diff options
author | pnv1 <pnv1@yandex-team.ru> | 2022-02-10 16:50:20 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:20 +0300 |
commit | 0689407cd3d1e44446526af0be1433d1c126788e (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 5f48504f9120f1bd635dfd2819efab47e7a203a5 (diff) | |
download | ydb-0689407cd3d1e44446526af0be1433d1c126788e.tar.gz |
Restoring authorship annotation for <pnv1@yandex-team.ru>. Commit 2 of 2.
272 files changed, 21045 insertions, 21045 deletions
diff --git a/build/rules/kikimr.policy b/build/rules/kikimr.policy index aa8bdc7171..5bb6426e30 100644 --- a/build/rules/kikimr.policy +++ b/build/rules/kikimr.policy @@ -67,7 +67,7 @@ ALLOW .* -> kikimr/persqueue ALLOW .* -> kikimr/public/api ALLOW .* -> kikimr/public/lib/json_value ALLOW .* -> kikimr/public/lib/yson_value -ALLOW .* -> kikimr/public/lib/ydb_cli +ALLOW .* -> kikimr/public/lib/ydb_cli ALLOW .* -> kikimr/yndx/api diff --git a/library/cpp/getopt/small/last_getopt_opts.cpp b/library/cpp/getopt/small/last_getopt_opts.cpp index f567a2745b..03c432849f 100644 --- a/library/cpp/getopt/small/last_getopt_opts.cpp +++ b/library/cpp/getopt/small/last_getopt_opts.cpp @@ -313,12 +313,12 @@ namespace NLastGetopt { } void TOpts::PrintCmdLine(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const { - os << colors.BoldColor() << "Usage" << colors.OldColor() << ": "; - if (CustomUsage) { - os << CustomUsage; - } else { - os << program << " "; - } + os << colors.BoldColor() << "Usage" << colors.OldColor() << ": "; + if (CustomUsage) { + os << CustomUsage; + } else { + os << program << " "; + } if (CustomCmdLineDescr) { os << CustomCmdLineDescr << Endl; return; diff --git a/library/cpp/getopt/small/last_getopt_opts.h b/library/cpp/getopt/small/last_getopt_opts.h index 4acce7ac5b..825b99c871 100644 --- a/library/cpp/getopt/small/last_getopt_opts.h +++ b/library/cpp/getopt/small/last_getopt_opts.h @@ -65,7 +65,7 @@ namespace NLastGetopt { TString Title; // title of the help string TString CustomCmdLineDescr; // user defined help string - TString CustomUsage; // user defined usage string + TString CustomUsage; // user defined usage string TVector<std::pair<TString, TString>> Sections; // additional help entries to print after usage @@ -420,11 +420,11 @@ namespace NLastGetopt { * * @param usage new usage string */ - void SetCustomUsage(const TString& usage) { - CustomUsage = usage; - } - - /** + void SetCustomUsage(const TString& usage) { + CustomUsage = usage; + } + + /** * Add a section to print after the main usage spec. */ void AddSection(TString title, TString text) { diff --git a/library/cpp/grpc/client/grpc_common.h b/library/cpp/grpc/client/grpc_common.h index 984fce2aa3..ffcdafe045 100644 --- a/library/cpp/grpc/client/grpc_common.h +++ b/library/cpp/grpc/client/grpc_common.h @@ -18,8 +18,8 @@ struct TGRpcClientConfig { ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests ui32 MaxInFlight = 0; - bool EnableSsl = false; - TString SslCaCert; //Implicitly enables Ssl if not empty + bool EnableSsl = false; + TString SslCaCert; //Implicitly enables Ssl if not empty grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; ui64 MemQuota = 0; std::unordered_map<TString, TString> StringChannelParams; @@ -35,12 +35,12 @@ struct TGRpcClientConfig { TGRpcClientConfig(const TString& locator, TDuration timeout = TDuration::Max(), ui64 maxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT, ui32 maxInFlight = 0, TString caCert = "", - grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) + grpc_compression_algorithm compressionAlgorithm = GRPC_COMPRESS_NONE, bool enableSsl = false) : Locator(locator) , Timeout(timeout) , MaxMessageSize(maxMessageSize) , MaxInFlight(maxInFlight) - , EnableSsl(enableSsl) + , EnableSsl(enableSsl) , SslCaCert(caCert) , CompressionAlgoritm(compressionAlgorithm) {} @@ -74,7 +74,7 @@ inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRp if (!config.SslTargetNameOverride.empty()) { args.SetSslTargetNameOverride(config.SslTargetNameOverride); } - if (config.EnableSsl || config.SslCaCert) { + if (config.EnableSsl || config.SslCaCert) { return grpc::CreateCustomChannel(config.Locator, grpc::SslCredentials(grpc::SslCredentialsOptions{config.SslCaCert, "", ""}), args); } else { return grpc::CreateCustomChannel(config.Locator, grpc::InsecureChannelCredentials(), args); diff --git a/ydb/apps/ya.make b/ydb/apps/ya.make index f061536de2..61c67bbce0 100644 --- a/ydb/apps/ya.make +++ b/ydb/apps/ya.make @@ -2,5 +2,5 @@ OWNER(g:kikimr) RECURSE( ydbd - ydb + ydb ) diff --git a/ydb/apps/ydb/README.md b/ydb/apps/ydb/README.md index b35a0d1458..66da458cc9 100644 --- a/ydb/apps/ydb/README.md +++ b/ydb/apps/ydb/README.md @@ -1,12 +1,12 @@ -# Yandex Database Command Line Interface - -This is the official YDB command line client that works via public API. - -The structure of commands in YDB CLI is similar to one in YDB API and consists of services (table, scheme, etc.). -There is also an additional "tools" section which provides useful utilities for common operations with the database. - -There is a couple of options that every call has to be provided with (or it can be once set with "ydb init" command and saved to config): -- endpoint - Endpoint to connect to; -- database - Database to work with. - -You can use --help option on root or any subcommand for more info.
\ No newline at end of file +# Yandex Database Command Line Interface + +This is the official YDB command line client that works via public API. + +The structure of commands in YDB CLI is similar to one in YDB API and consists of services (table, scheme, etc.). +There is also an additional "tools" section which provides useful utilities for common operations with the database. + +There is a couple of options that every call has to be provided with (or it can be once set with "ydb init" command and saved to config): +- endpoint - Endpoint to connect to; +- database - Database to work with. + +You can use --help option on root or any subcommand for more info.
\ No newline at end of file diff --git a/ydb/apps/ydb/commands/ya.make b/ydb/apps/ydb/commands/ya.make index 7d63dc1fea..58d40f75b1 100644 --- a/ydb/apps/ydb/commands/ya.make +++ b/ydb/apps/ydb/commands/ya.make @@ -1,13 +1,13 @@ -LIBRARY(commands) - -OWNER(g:kikimr) - -SRCS( - ydb_root.cpp -) - -PEERDIR( +LIBRARY(commands) + +OWNER(g:kikimr) + +SRCS( + ydb_root.cpp +) + +PEERDIR( ydb/public/lib/ydb_cli/commands -) - -END() +) + +END() diff --git a/ydb/apps/ydb/commands/ydb_root.cpp b/ydb/apps/ydb/commands/ydb_root.cpp index 97b7cb3331..fa54570f7c 100644 --- a/ydb/apps/ydb/commands/ydb_root.cpp +++ b/ydb/apps/ydb/commands/ydb_root.cpp @@ -1,24 +1,24 @@ -#include "ydb_root.h" - -namespace NYdb { -namespace NConsoleClient { - -int NewClient(int argc, char** argv) { - NYdb::NConsoleClient::TClientSettings settings; - settings.EnableSsl = true; - settings.UseOAuthToken = true; - settings.UseDefaultTokenFile = false; - settings.UseIamAuth = false; - settings.UseStaticCredentials = true; - settings.UseExportToYt = false; - settings.MentionUserAccount = false; - settings.YdbDir = "ydb"; - - THolder<TClientCommandRootCommon> commandsRoot = MakeHolder<TClientCommandRootCommon>(settings); - commandsRoot->Opts.SetTitle("YDB client"); - TClientCommand::TConfig config(argc, argv); - return commandsRoot->Process(config); -} - -} -} +#include "ydb_root.h" + +namespace NYdb { +namespace NConsoleClient { + +int NewClient(int argc, char** argv) { + NYdb::NConsoleClient::TClientSettings settings; + settings.EnableSsl = true; + settings.UseOAuthToken = true; + settings.UseDefaultTokenFile = false; + settings.UseIamAuth = false; + settings.UseStaticCredentials = true; + settings.UseExportToYt = false; + settings.MentionUserAccount = false; + settings.YdbDir = "ydb"; + + THolder<TClientCommandRootCommon> commandsRoot = MakeHolder<TClientCommandRootCommon>(settings); + commandsRoot->Opts.SetTitle("YDB client"); + TClientCommand::TConfig config(argc, argv); + return commandsRoot->Process(config); +} + +} +} diff --git a/ydb/apps/ydb/commands/ydb_root.h b/ydb/apps/ydb/commands/ydb_root.h index d2b619a3ab..bdde83eb56 100644 --- a/ydb/apps/ydb/commands/ydb_root.h +++ b/ydb/apps/ydb/commands/ydb_root.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/commands/ydb_root_common.h> - -namespace NYdb { -namespace NConsoleClient { - -int NewClient(int argc, char** argv); - -} -} + +namespace NYdb { +namespace NConsoleClient { + +int NewClient(int argc, char** argv); + +} +} diff --git a/ydb/apps/ydb/main.cpp b/ydb/apps/ydb/main.cpp index bc189fc1e0..5f1621744b 100644 --- a/ydb/apps/ydb/main.cpp +++ b/ydb/apps/ydb/main.cpp @@ -1,30 +1,30 @@ -#include "commands/ydb_root.h" +#include "commands/ydb_root.h" #include <ydb/public/lib/ydb_cli/commands/ydb_service_stream.h> - -TVector<NYdb::NPersQueue::ECodec> NYdb::NConsoleClient::InitAllowedCodecs() { - return TVector<NYdb::NPersQueue::ECodec>{ - NYdb::NPersQueue::ECodec::RAW, - NYdb::NPersQueue::ECodec::ZSTD, - NYdb::NPersQueue::ECodec::GZIP, - }; -} - - -int main(int argc, char **argv) { - try { - return NYdb::NConsoleClient::NewClient(argc, argv); - } - catch (const NYdb::NConsoleClient::TMissUseException& e) { - Cerr << e.what() << Endl; - Cerr << "Try \"--help\" option for more info." << Endl; - return EXIT_FAILURE; - } - catch (const NYdb::NConsoleClient::TYdbErrorException& e) { - Cerr << e; - return EXIT_FAILURE; - } - catch (const yexception& e) { - Cerr << e.what() << Endl; - return EXIT_FAILURE; - } -} + +TVector<NYdb::NPersQueue::ECodec> NYdb::NConsoleClient::InitAllowedCodecs() { + return TVector<NYdb::NPersQueue::ECodec>{ + NYdb::NPersQueue::ECodec::RAW, + NYdb::NPersQueue::ECodec::ZSTD, + NYdb::NPersQueue::ECodec::GZIP, + }; +} + + +int main(int argc, char **argv) { + try { + return NYdb::NConsoleClient::NewClient(argc, argv); + } + catch (const NYdb::NConsoleClient::TMissUseException& e) { + Cerr << e.what() << Endl; + Cerr << "Try \"--help\" option for more info." << Endl; + return EXIT_FAILURE; + } + catch (const NYdb::NConsoleClient::TYdbErrorException& e) { + Cerr << e; + return EXIT_FAILURE; + } + catch (const yexception& e) { + Cerr << e.what() << Endl; + return EXIT_FAILURE; + } +} diff --git a/ydb/apps/ydb/ya.make b/ydb/apps/ydb/ya.make index 292d312dda..5a39c8ad75 100644 --- a/ydb/apps/ydb/ya.make +++ b/ydb/apps/ydb/ya.make @@ -1,19 +1,19 @@ -PROGRAM(ydb) - -OWNER( - g:kikimr -) - -STRIP() - -SRCS( - main.cpp -) - -PEERDIR( +PROGRAM(ydb) + +OWNER( + g:kikimr +) + +STRIP() + +SRCS( + main.cpp +) + +PEERDIR( ydb/apps/ydb/commands -) - +) + # # DON'T ALLOW NEW DEPENDENCIES WITHOUT EXPLICIT APPROVE FROM kikimr-dev@ or fomichev@ # @@ -37,5 +37,5 @@ CHECK_DEPENDENT_DIRS( ydb/library/yql/public/issue ydb/library/yql/public/issue/protos ) -END() - +END() + diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index 360490b293..c666f7468c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -157,7 +157,7 @@ struct TAppData { TResourceProfilesPtr ResourceProfiles; TProgramShouldContinue * const KikimrShouldContinue; - bool EnableIntrospection = true; + bool EnableIntrospection = true; // Used to allow column families for testing bool AllowColumnFamiliesForTest = false; diff --git a/ydb/core/base/logoblob.h b/ydb/core/base/logoblob.h index d86c85e0e5..36eea54393 100644 --- a/ydb/core/base/logoblob.h +++ b/ydb/core/base/logoblob.h @@ -287,21 +287,21 @@ namespace NKikimr { ++begin; } } - - template<typename TContainer> - void LogoBlobIDRepatedFromLogoBlobIDUniversal( - ::google::protobuf::RepeatedPtrField<NKikimrProto::TLogoBlobID> *proto, - TContainer& container) - { - auto begin = container.begin(); - auto end = container.end(); - proto->Reserve(container.size()); - proto->Clear(); - while (begin != end) { - LogoBlobIDFromLogoBlobID(*begin, proto->Add()); - ++begin; - } - } + + template<typename TContainer> + void LogoBlobIDRepatedFromLogoBlobIDUniversal( + ::google::protobuf::RepeatedPtrField<NKikimrProto::TLogoBlobID> *proto, + TContainer& container) + { + auto begin = container.begin(); + auto end = container.end(); + proto->Reserve(container.size()); + proto->Clear(); + while (begin != end) { + LogoBlobIDFromLogoBlobID(*begin, proto->Add()); + ++begin; + } + } } template<> diff --git a/ydb/core/base/traceid.cpp b/ydb/core/base/traceid.cpp index 9c3fb80eb9..a1b2d3a977 100644 --- a/ydb/core/base/traceid.cpp +++ b/ydb/core/base/traceid.cpp @@ -1,70 +1,70 @@ -#include "traceid.h" -#include <util/random/random.h> -#include <util/stream/str.h> - -namespace NKikimr { -namespace NTracing { - -TTraceID::TTraceID() - : CreationTime(0) - , RandomID(0) -{} - -TTraceID::TTraceID(ui64 randomId, ui64 creationTime) - : CreationTime(creationTime) - , RandomID(randomId) -{} - -TTraceID TTraceID::GenerateNew() { - return TTraceID(RandomNumber<ui64>(), TInstant::Now().MicroSeconds()); -} - -TString TTraceID::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TTraceID::Out(IOutputStream &o) const { - char buf[240]; +#include "traceid.h" +#include <util/random/random.h> +#include <util/stream/str.h> + +namespace NKikimr { +namespace NTracing { + +TTraceID::TTraceID() + : CreationTime(0) + , RandomID(0) +{} + +TTraceID::TTraceID(ui64 randomId, ui64 creationTime) + : CreationTime(creationTime) + , RandomID(randomId) +{} + +TTraceID TTraceID::GenerateNew() { + return TTraceID(RandomNumber<ui64>(), TInstant::Now().MicroSeconds()); +} + +TString TTraceID::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TTraceID::Out(IOutputStream &o) const { + char buf[240]; sprintf(buf, "[ID:%" PRIu64 ", Created: %s]", RandomID, TInstant::MicroSeconds(CreationTime).ToRfc822StringLocal().data()); - o << buf; -} - -bool TTraceID::operator<(const TTraceID &x) const { - return CreationTime != x.CreationTime ? CreationTime < x.CreationTime : RandomID < x.RandomID; -} - -bool TTraceID::operator>(const TTraceID &x) const { - return (x < *this); -} - -bool TTraceID::operator<=(const TTraceID &x) const { - return CreationTime != x.CreationTime ? CreationTime < x.CreationTime : RandomID <= x.RandomID; -} - -bool TTraceID::operator>=(const TTraceID &x) const { - return (x <= *this); -} - -bool TTraceID::operator==(const TTraceID &x) const { - return CreationTime == x.CreationTime && RandomID == x.RandomID; -} - -bool TTraceID::operator!=(const TTraceID &x) const { - return CreationTime != x.CreationTime || RandomID != x.RandomID; -} - -void TraceIDFromTraceID(const TTraceID& src, NKikimrTracing::TTraceID* dest) { - Y_VERIFY_DEBUG(dest); - dest->SetCreationTime(src.CreationTime); - dest->SetRandomID(src.RandomID); -} - -TTraceID TraceIDFromTraceID(const NKikimrTracing::TTraceID &proto) { - return TTraceID(proto.GetRandomID(), proto.GetCreationTime()); -} - -} -} + o << buf; +} + +bool TTraceID::operator<(const TTraceID &x) const { + return CreationTime != x.CreationTime ? CreationTime < x.CreationTime : RandomID < x.RandomID; +} + +bool TTraceID::operator>(const TTraceID &x) const { + return (x < *this); +} + +bool TTraceID::operator<=(const TTraceID &x) const { + return CreationTime != x.CreationTime ? CreationTime < x.CreationTime : RandomID <= x.RandomID; +} + +bool TTraceID::operator>=(const TTraceID &x) const { + return (x <= *this); +} + +bool TTraceID::operator==(const TTraceID &x) const { + return CreationTime == x.CreationTime && RandomID == x.RandomID; +} + +bool TTraceID::operator!=(const TTraceID &x) const { + return CreationTime != x.CreationTime || RandomID != x.RandomID; +} + +void TraceIDFromTraceID(const TTraceID& src, NKikimrTracing::TTraceID* dest) { + Y_VERIFY_DEBUG(dest); + dest->SetCreationTime(src.CreationTime); + dest->SetRandomID(src.RandomID); +} + +TTraceID TraceIDFromTraceID(const NKikimrTracing::TTraceID &proto) { + return TTraceID(proto.GetRandomID(), proto.GetCreationTime()); +} + +} +} diff --git a/ydb/core/base/traceid.h b/ydb/core/base/traceid.h index 840daf88c7..5847f02c40 100644 --- a/ydb/core/base/traceid.h +++ b/ydb/core/base/traceid.h @@ -1,43 +1,43 @@ -#pragma once -#include "defs.h" +#pragma once +#include "defs.h" #include <ydb/core/protos/tracing.pb.h> - -namespace NKikimr { -namespace NTracing { - -struct TTraceID { - ui64 CreationTime; - ui64 RandomID; - - TTraceID(); - TTraceID(ui64 randomId, ui64 creationTime); - - static TTraceID GenerateNew(); - TString ToString() const; - void Out(IOutputStream &o) const; - - bool operator<(const TTraceID &x) const; - bool operator>(const TTraceID &x) const; - bool operator<=(const TTraceID &x) const; - bool operator>=(const TTraceID &x) const; - bool operator==(const TTraceID &x) const; - bool operator!=(const TTraceID &x) const; -}; - -void TraceIDFromTraceID(const TTraceID& src, NKikimrTracing::TTraceID* dest); -TTraceID TraceIDFromTraceID(const NKikimrTracing::TTraceID &proto); - -} -} - -template<> -struct THash<NKikimr::NTracing::TTraceID> { - inline ui64 operator()(const NKikimr::NTracing::TTraceID& x) const noexcept { - return x.RandomID; - } -}; - -template<> -inline void Out<NKikimr::NTracing::TTraceID>(IOutputStream& o, const NKikimr::NTracing::TTraceID &x) { - return x.Out(o); -} + +namespace NKikimr { +namespace NTracing { + +struct TTraceID { + ui64 CreationTime; + ui64 RandomID; + + TTraceID(); + TTraceID(ui64 randomId, ui64 creationTime); + + static TTraceID GenerateNew(); + TString ToString() const; + void Out(IOutputStream &o) const; + + bool operator<(const TTraceID &x) const; + bool operator>(const TTraceID &x) const; + bool operator<=(const TTraceID &x) const; + bool operator>=(const TTraceID &x) const; + bool operator==(const TTraceID &x) const; + bool operator!=(const TTraceID &x) const; +}; + +void TraceIDFromTraceID(const TTraceID& src, NKikimrTracing::TTraceID* dest); +TTraceID TraceIDFromTraceID(const NKikimrTracing::TTraceID &proto); + +} +} + +template<> +struct THash<NKikimr::NTracing::TTraceID> { + inline ui64 operator()(const NKikimr::NTracing::TTraceID& x) const noexcept { + return x.RandomID; + } +}; + +template<> +inline void Out<NKikimr::NTracing::TTraceID>(IOutputStream& o, const NKikimr::NTracing::TTraceID &x) { + return x.Out(o); +} diff --git a/ydb/core/base/tracing.h b/ydb/core/base/tracing.h index bfa81b1297..40420e4541 100644 --- a/ydb/core/base/tracing.h +++ b/ydb/core/base/tracing.h @@ -1,222 +1,222 @@ -#pragma once -#include "defs.h" -#include "traceid.h" +#pragma once +#include "defs.h" +#include "traceid.h" #include <library/cpp/monlib/dynamic_counters/counters.h> #include <library/cpp/threading/light_rw_lock/lightrwlock.h> - -namespace NKikimr { -namespace NTracing { - -struct TTimestampInfo { - enum EMode : ui32 { - ModeDefault, - ModeDisabled, - ModeAbsolute, - ModeFirst, - ModePrevious - } Mode; - - enum EPrecision : ui32 { - PrecisionDefault, - PrecisionSeconds, - PrecisionMilliseconds, - PrecisionMicroseconds - } Precision; - - TTimestampInfo() - : Mode(ModeDefault) - , Precision(PrecisionDefault) - {} - - TTimestampInfo(EMode mode, EPrecision precision) - : Mode(mode) - , Precision(precision) - {} - - static TString GetTextMode(EMode mode) { - switch (mode) { - case ModeDisabled: - return "Disabled"; - case ModeAbsolute: - case ModeDefault: - return "Absolute"; - case ModeFirst: - return "Relative to first"; - case ModePrevious: - return "Relative to previous"; - default: - return "Unknown mode"; - } - } - - static TString GetTextPrecision(EPrecision precision) { - switch (precision) { - case PrecisionSeconds: - return "Seconds"; - case PrecisionMilliseconds: - case PrecisionDefault: - return "Milliseconds"; - case PrecisionMicroseconds: - return "Microseconds"; - default: - return "Unknown precision"; - } - } -}; - -struct TTimestampData : TTimestampInfo { - TTimestampData(const TTimestampInfo& timestampInfo, TInstant baseTime) - : TTimestampInfo(timestampInfo) - , BaseTime(baseTime) - {} - - TInstant BaseTime; -}; - -struct TTraceInfo { - ui32 NodeId; - ui64 TabletId; - TTraceID TraceId; - TTimestampInfo TimestampInfo; -}; - -class ITraceSignal { -public: - typedef ui32 EType; - enum ETypeSpace { - ES_TABLET_BOOTSTRAP = 1 - // ES_NEXT_TYPE_GROUP = 1000 - }; - virtual ~ITraceSignal() = default; - - virtual bool SerializeToString(TString& str) const = 0; - virtual EType GetType() const = 0; - - virtual void OutHtmlHeader( - TStringStream& str, - TTimestampData& tsData, - std::function<TString()> getMyId - ) const = 0; - - virtual void OutHtmlBody( - TStringStream& str, - const TTimestampInfo& tsInfo, - std::function<TString()> getMyId, - TList<ui64>& signalAddress - ) const = 0; - - virtual void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix = TString()) const = 0; -}; - -class ITrace { -public: - enum EType { + +namespace NKikimr { +namespace NTracing { + +struct TTimestampInfo { + enum EMode : ui32 { + ModeDefault, + ModeDisabled, + ModeAbsolute, + ModeFirst, + ModePrevious + } Mode; + + enum EPrecision : ui32 { + PrecisionDefault, + PrecisionSeconds, + PrecisionMilliseconds, + PrecisionMicroseconds + } Precision; + + TTimestampInfo() + : Mode(ModeDefault) + , Precision(PrecisionDefault) + {} + + TTimestampInfo(EMode mode, EPrecision precision) + : Mode(mode) + , Precision(precision) + {} + + static TString GetTextMode(EMode mode) { + switch (mode) { + case ModeDisabled: + return "Disabled"; + case ModeAbsolute: + case ModeDefault: + return "Absolute"; + case ModeFirst: + return "Relative to first"; + case ModePrevious: + return "Relative to previous"; + default: + return "Unknown mode"; + } + } + + static TString GetTextPrecision(EPrecision precision) { + switch (precision) { + case PrecisionSeconds: + return "Seconds"; + case PrecisionMilliseconds: + case PrecisionDefault: + return "Milliseconds"; + case PrecisionMicroseconds: + return "Microseconds"; + default: + return "Unknown precision"; + } + } +}; + +struct TTimestampData : TTimestampInfo { + TTimestampData(const TTimestampInfo& timestampInfo, TInstant baseTime) + : TTimestampInfo(timestampInfo) + , BaseTime(baseTime) + {} + + TInstant BaseTime; +}; + +struct TTraceInfo { + ui32 NodeId; + ui64 TabletId; + TTraceID TraceId; + TTimestampInfo TimestampInfo; +}; + +class ITraceSignal { +public: + typedef ui32 EType; + enum ETypeSpace { + ES_TABLET_BOOTSTRAP = 1 + // ES_NEXT_TYPE_GROUP = 1000 + }; + virtual ~ITraceSignal() = default; + + virtual bool SerializeToString(TString& str) const = 0; + virtual EType GetType() const = 0; + + virtual void OutHtmlHeader( + TStringStream& str, + TTimestampData& tsData, + std::function<TString()> getMyId + ) const = 0; + + virtual void OutHtmlBody( + TStringStream& str, + const TTimestampInfo& tsInfo, + std::function<TString()> getMyId, + TList<ui64>& signalAddress + ) const = 0; + + virtual void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix = TString()) const = 0; +}; + +class ITrace { +public: + enum EType { TypeSysTabletBootstrap, - TypeReqRebuildHistoryGraph - }; - - struct TSerializedSignal { - ITraceSignal::EType Type; - TString Signal; - }; - - virtual ~ITrace() = default; - - virtual ITrace* CreateTrace(ITrace::EType type) = 0; - virtual bool Attach(THolder<ITraceSignal> signal) = 0; - virtual TTraceID GetSelfID() const = 0; - virtual TTraceID GetParentID() const = 0; - virtual TTraceID GetRootID() const = 0; - virtual ui64 GetSize() const = 0; - // Start of recursion - virtual void OutHtml(TStringStream& str, TTraceInfo& traceInfo) const = 0; - virtual void OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const = 0; - // Start of recursion - virtual void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) = 0; - virtual void OutSignalHtmlBody( - TStringStream& str, - const TTimestampInfo& tsInfo, - std::function<TString()> getMyId, - TList<ui64>& signalAddress - ) = 0; - virtual void OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix = TString()) const = 0; - virtual EType GetType() const = 0; - virtual bool SerializeToString(TString& str) const = 0; -}; - -ITrace* CreateTrace(ITrace::EType type); -ITrace* CreateTrace(const TString& serializedTrace); - -class ISignalCreator { -public: - virtual ITraceSignal* Create(const TString& serializedSignal) const = 0; - virtual ~ISignalCreator() = default; -}; - -template <typename TSignalType> -class TSignalCreator : public ISignalCreator { -public: - virtual ITraceSignal* Create(const TString& serializedSignal) const { - return new TSignalType(serializedSignal); - } -}; - -class TSignalFactory { -public: - static TSignalFactory& Instance() { - static TSignalFactory SingleFactory; - return SingleFactory; - } - - template <typename TSignalType> - void Add(ITraceSignal::EType signalType) { - TLightWriteGuard g{ Lock }; - auto it = Factorymap.find(signalType); - if (it == Factorymap.end()) { + TypeReqRebuildHistoryGraph + }; + + struct TSerializedSignal { + ITraceSignal::EType Type; + TString Signal; + }; + + virtual ~ITrace() = default; + + virtual ITrace* CreateTrace(ITrace::EType type) = 0; + virtual bool Attach(THolder<ITraceSignal> signal) = 0; + virtual TTraceID GetSelfID() const = 0; + virtual TTraceID GetParentID() const = 0; + virtual TTraceID GetRootID() const = 0; + virtual ui64 GetSize() const = 0; + // Start of recursion + virtual void OutHtml(TStringStream& str, TTraceInfo& traceInfo) const = 0; + virtual void OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const = 0; + // Start of recursion + virtual void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) = 0; + virtual void OutSignalHtmlBody( + TStringStream& str, + const TTimestampInfo& tsInfo, + std::function<TString()> getMyId, + TList<ui64>& signalAddress + ) = 0; + virtual void OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix = TString()) const = 0; + virtual EType GetType() const = 0; + virtual bool SerializeToString(TString& str) const = 0; +}; + +ITrace* CreateTrace(ITrace::EType type); +ITrace* CreateTrace(const TString& serializedTrace); + +class ISignalCreator { +public: + virtual ITraceSignal* Create(const TString& serializedSignal) const = 0; + virtual ~ISignalCreator() = default; +}; + +template <typename TSignalType> +class TSignalCreator : public ISignalCreator { +public: + virtual ITraceSignal* Create(const TString& serializedSignal) const { + return new TSignalType(serializedSignal); + } +}; + +class TSignalFactory { +public: + static TSignalFactory& Instance() { + static TSignalFactory SingleFactory; + return SingleFactory; + } + + template <typename TSignalType> + void Add(ITraceSignal::EType signalType) { + TLightWriteGuard g{ Lock }; + auto it = Factorymap.find(signalType); + if (it == Factorymap.end()) { Factorymap[signalType] = MakeHolder<TSignalCreator<TSignalType>>(); - } - } - - ITraceSignal* Create(const ITrace::TSerializedSignal& signal) { - TLightReadGuard g{ Lock }; - auto it = Factorymap.find(signal.Type); - if (it != Factorymap.end()) { - return it->second->Create(signal.Signal); - } - return nullptr; - } - -private: - using TFactoryMap = THashMap<ui32, THolder<ISignalCreator>>; - TFactoryMap Factorymap; - TLightRWLock Lock; -}; - -class ITraceCollection { -public: - virtual ~ITraceCollection() = default; - virtual void AddTrace(ui64 tabletID, ITrace* trace) = 0; - // Shows all tablet IDs for which it has data - virtual void GetTabletIDs(TVector<ui64>& tabletIDs) const = 0; - virtual bool HasTabletID(ui64 tabletID) const = 0; - // Returns a list of introspection trace ids and creation times for given tabletId - virtual bool GetTraces(ui64 tabletID, TVector<TTraceID>& result) = 0; - virtual ITrace* GetTrace(ui64 tabletID, TTraceID& traceID) = 0; -}; - -ITraceCollection* CreateTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters = nullptr); - -} -} - -template<> -inline void Out<NKikimr::NTracing::TTimestampInfo::EMode>(IOutputStream& o - , const NKikimr::NTracing::TTimestampInfo::EMode x) { - o << static_cast<ui32>(x); -} - -template<> -inline void Out<NKikimr::NTracing::TTimestampInfo::EPrecision>(IOutputStream& o - , const NKikimr::NTracing::TTimestampInfo::EPrecision x) { - o << static_cast<ui32>(x); -} + } + } + + ITraceSignal* Create(const ITrace::TSerializedSignal& signal) { + TLightReadGuard g{ Lock }; + auto it = Factorymap.find(signal.Type); + if (it != Factorymap.end()) { + return it->second->Create(signal.Signal); + } + return nullptr; + } + +private: + using TFactoryMap = THashMap<ui32, THolder<ISignalCreator>>; + TFactoryMap Factorymap; + TLightRWLock Lock; +}; + +class ITraceCollection { +public: + virtual ~ITraceCollection() = default; + virtual void AddTrace(ui64 tabletID, ITrace* trace) = 0; + // Shows all tablet IDs for which it has data + virtual void GetTabletIDs(TVector<ui64>& tabletIDs) const = 0; + virtual bool HasTabletID(ui64 tabletID) const = 0; + // Returns a list of introspection trace ids and creation times for given tabletId + virtual bool GetTraces(ui64 tabletID, TVector<TTraceID>& result) = 0; + virtual ITrace* GetTrace(ui64 tabletID, TTraceID& traceID) = 0; +}; + +ITraceCollection* CreateTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters = nullptr); + +} +} + +template<> +inline void Out<NKikimr::NTracing::TTimestampInfo::EMode>(IOutputStream& o + , const NKikimr::NTracing::TTimestampInfo::EMode x) { + o << static_cast<ui32>(x); +} + +template<> +inline void Out<NKikimr::NTracing::TTimestampInfo::EPrecision>(IOutputStream& o + , const NKikimr::NTracing::TTimestampInfo::EPrecision x) { + o << static_cast<ui32>(x); +} diff --git a/ydb/core/base/ya.make b/ydb/core/base/ya.make index d79d23de40..83db5825c3 100644 --- a/ydb/core/base/ya.make +++ b/ydb/core/base/ya.make @@ -79,9 +79,9 @@ SRCS( tablet_status_checker.cpp tabletid.h tablet_types.h - traceid.cpp - traceid.h - tracing.h + traceid.cpp + traceid.h + tracing.h tx_processing.h tx_processing.cpp user_registry.h diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index 78a1a14354..76f38bf35c 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -381,26 +381,26 @@ private: for (auto& tupleItem : selectTuple.Children()) { auto columnName = tupleItem->Content(); - const TTypeAnnotationNode *columnDataType; + const TTypeAnnotationNode *columnDataType; - auto systemColumnType = KikimrSystemColumns().find(columnName); - if (systemColumnType != KikimrSystemColumns().end()) { - columnDataType = ctx.MakeType<TDataExprType>(systemColumnType->second); + auto systemColumnType = KikimrSystemColumns().find(columnName); + if (systemColumnType != KikimrSystemColumns().end()) { + columnDataType = ctx.MakeType<TDataExprType>(systemColumnType->second); } else { - auto column = lookup->Columns.FindPtr(columnName); - YQL_ENSURE(column); - - // Decimal type is transformed into parametrized Decimal(22, 9). - if (column->Type == NYql::NProto::TypeIds::Decimal) { - columnDataType = ctx.MakeType<TDataExprParamsType>( - NUdf::GetDataSlot(column->Type), - ToString(NScheme::DECIMAL_PRECISION), - ToString(NScheme::DECIMAL_SCALE)); - } else { - columnDataType = GetMkqlDataTypeAnnotation( - TDataType::Create(column->Type, *MkqlCtx->TypeEnv), - ctx); - } + auto column = lookup->Columns.FindPtr(columnName); + YQL_ENSURE(column); + + // Decimal type is transformed into parametrized Decimal(22, 9). + if (column->Type == NYql::NProto::TypeIds::Decimal) { + columnDataType = ctx.MakeType<TDataExprParamsType>( + NUdf::GetDataSlot(column->Type), + ToString(NScheme::DECIMAL_PRECISION), + ToString(NScheme::DECIMAL_SCALE)); + } else { + columnDataType = GetMkqlDataTypeAnnotation( + TDataType::Create(column->Type, *MkqlCtx->TypeEnv), + ctx); + } } auto columnOptType = ctx.MakeType<TOptionalExprType>(columnDataType); @@ -503,13 +503,13 @@ private: ui32 selectIndex = 0; for (auto selectItem : selectTuple->Children()) { auto columnName = selectItem->Content(); - if (!NKikimr::IsSystemColumn(columnName)) { - auto column = lookup->Columns.FindPtr(columnName); - Y_ENSURE_EX(column, TNodeException(node) - << "Unknown column '" << columnName - << "' for table [" << lookup->Table.TableName - << "] at select position #" << selectIndex); - } + if (!NKikimr::IsSystemColumn(columnName)) { + auto column = lookup->Columns.FindPtr(columnName); + Y_ENSURE_EX(column, TNodeException(node) + << "Unknown column '" << columnName + << "' for table [" << lookup->Table.TableName + << "] at select position #" << selectIndex); + } ++selectIndex; } @@ -829,20 +829,20 @@ TRuntimeNode GetReadTargetNode(const TExprNode& callable, ui32 index, TMkqlBuild return mkqlContext->PgmBuilder->ReadTarget(readTarget); } -void FillColumnsToRead(IDbSchemeResolver::TTableResult* lookup, TExprNode* selectTuple, TVector<TSelectColumn>& columnsToRead) { - for (ui32 i = 0; i < selectTuple->ChildrenSize(); ++i) { - auto columnName = selectTuple->Child(i)->Content(); - const auto& systemColumn = GetSystemColumns().find(columnName); - if (systemColumn != GetSystemColumns().end()) { - columnsToRead.emplace_back(columnName, systemColumn->second.ColumnId, systemColumn->second.TypeId); - } else { - auto column = lookup->Columns.FindPtr(columnName); - YQL_ENSURE(column); - columnsToRead.emplace_back(columnName, column->Column, column->Type); - } - } -} - +void FillColumnsToRead(IDbSchemeResolver::TTableResult* lookup, TExprNode* selectTuple, TVector<TSelectColumn>& columnsToRead) { + for (ui32 i = 0; i < selectTuple->ChildrenSize(); ++i) { + auto columnName = selectTuple->Child(i)->Content(); + const auto& systemColumn = GetSystemColumns().find(columnName); + if (systemColumn != GetSystemColumns().end()) { + columnsToRead.emplace_back(columnName, systemColumn->second.ColumnId, systemColumn->second.TypeId); + } else { + auto column = lookup->Columns.FindPtr(columnName); + YQL_ENSURE(column); + columnsToRead.emplace_back(columnName, column->Column, column->Type); + } + } +} + void ValidateCompiledTable(const TExprNode& node, const TTableId& tableId) { auto currentVersion = ToString(tableId.SchemaVersion); auto programVersion = node.Child(0)->Child(1)->Content(); @@ -927,7 +927,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr } TVector<TSelectColumn> columnsToRead; - FillColumnsToRead(lookup, node.Child(2), columnsToRead); + FillColumnsToRead(lookup, node.Child(2), columnsToRead); auto readTargetNode = GetReadTargetNode(node, 3, ctx, mkqlContext); @@ -1032,7 +1032,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr options.ForbidNullArgsTo = forbidNullArgsTo; TVector<TSelectColumn> columnsToRead; - FillColumnsToRead(lookup, node.Child(2), columnsToRead); + FillColumnsToRead(lookup, node.Child(2), columnsToRead); auto optionsNode = node.Child(3); for (auto optionsItem : optionsNode->Children()) { diff --git a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp index ef336b6841..eb74fa36e9 100644 --- a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp +++ b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp @@ -63,14 +63,14 @@ class TTableProxyActor : public TActorBootstrapped<TTableProxyActor> { replyColumns.insert(reply.Table.ColumnNames.begin(), reply.Table.ColumnNames.end()); for (const auto &column : replyColumns) { - auto systemColumn = GetSystemColumns().find(column); - if (systemColumn != GetSystemColumns().end()) { - reply.Columns.insert({ - column, - {systemColumn->second.ColumnId, -1, systemColumn->second.TypeId, 0} - }); - continue; - } + auto systemColumn = GetSystemColumns().find(column); + if (systemColumn != GetSystemColumns().end()) { + reply.Columns.insert({ + column, + {systemColumn->second.ColumnId, -1, systemColumn->second.TypeId, 0} + }); + continue; + } const auto *x = backindex.FindPtr(column); if (x == nullptr) { reply.Status = TTableResult::Error; diff --git a/ydb/core/driver_lib/cli_base/cli.h b/ydb/core/driver_lib/cli_base/cli.h index f23d36087f..fdd4a61a58 100644 --- a/ydb/core/driver_lib/cli_base/cli.h +++ b/ydb/core/driver_lib/cli_base/cli.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/common/common.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> -#include <util/string/builder.h> - -namespace NKikimr { - -namespace NDriverClient { - int NewLiteClient(int argc, char** argv); -} -} +#include <util/string/builder.h> + +namespace NKikimr { + +namespace NDriverClient { + int NewLiteClient(int argc, char** argv); +} +} diff --git a/ydb/core/driver_lib/cli_base/cli_cmds.h b/ydb/core/driver_lib/cli_base/cli_cmds.h index a19f5c3008..4909636e43 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds.h +++ b/ydb/core/driver_lib/cli_base/cli_cmds.h @@ -1,85 +1,85 @@ -#pragma once - -#include "cli.h" -#include "cli_command.h" -#include "cli_grpc.h" -#include "cli_kicli.h" - +#pragma once + +#include "cli.h" +#include "cli_command.h" +#include "cli_grpc.h" +#include "cli_kicli.h" + #include <ydb/public/lib/ydb_cli/common/root.h> - + #include <ydb/public/api/protos/ydb_discovery.pb.h> #include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> - -namespace NKikimr { -namespace NDriverClient { - -class TClientCommandRootKikimrBase : public TClientCommandRootBase { -public: - TClientCommandRootKikimrBase(const TString& name); - void Config(TConfig& config) override; - void Parse(TConfig& config) override; - -protected: - bool GetProfileVariable(const TString& name, TString& value); - -private: - void ParseProfile(); - - THolder<TProfileConfig> ProfileConfig; - TString LocalProfileName; - bool DumpRequests = false; -}; - -class TClientCommandSchemaLite : public TClientCommandTree { -public: - TClientCommandSchemaLite(); -}; - -class TClientCommandDb : public TClientCommandTree { -public: - TClientCommandDb(); -}; - -class TClientCommandWhoAmI : public TClientCommand { -public: - TClientCommandWhoAmI(); - virtual void Config(TConfig& config) override; - virtual int Run(TConfig& config) override; - -protected: - bool WithGroups = false; - - int PrintResponse(const NMsgBusProxy::TBusResponse& response) const; -}; - -class TClientCommandDiscoveryBase - : public TClientGRpcCommand<Ydb::Discovery::V1::DiscoveryService, - Ydb::Discovery::ListEndpointsRequest, - Ydb::Discovery::ListEndpointsResponse, - decltype(&Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints), - &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints> { -public: - TClientCommandDiscoveryBase(const TString& name, const TString& description); - int Run(TConfig &config) override; - void PrintResponse(const Ydb::Operations::Operation &response) override; - -protected: - TString Database; -}; - -// kikimr -class TClientCommandDiscovery : public TClientCommandTree { -public: - TClientCommandDiscovery(); -}; - -// ydb -class TClientCommandDiscoveryLite : public TClientCommandDiscoveryBase { -public: - TClientCommandDiscoveryLite(); - void Config(TConfig& config) override; - void Parse(TConfig& config) override; -}; - -} -} + +namespace NKikimr { +namespace NDriverClient { + +class TClientCommandRootKikimrBase : public TClientCommandRootBase { +public: + TClientCommandRootKikimrBase(const TString& name); + void Config(TConfig& config) override; + void Parse(TConfig& config) override; + +protected: + bool GetProfileVariable(const TString& name, TString& value); + +private: + void ParseProfile(); + + THolder<TProfileConfig> ProfileConfig; + TString LocalProfileName; + bool DumpRequests = false; +}; + +class TClientCommandSchemaLite : public TClientCommandTree { +public: + TClientCommandSchemaLite(); +}; + +class TClientCommandDb : public TClientCommandTree { +public: + TClientCommandDb(); +}; + +class TClientCommandWhoAmI : public TClientCommand { +public: + TClientCommandWhoAmI(); + virtual void Config(TConfig& config) override; + virtual int Run(TConfig& config) override; + +protected: + bool WithGroups = false; + + int PrintResponse(const NMsgBusProxy::TBusResponse& response) const; +}; + +class TClientCommandDiscoveryBase + : public TClientGRpcCommand<Ydb::Discovery::V1::DiscoveryService, + Ydb::Discovery::ListEndpointsRequest, + Ydb::Discovery::ListEndpointsResponse, + decltype(&Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints), + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints> { +public: + TClientCommandDiscoveryBase(const TString& name, const TString& description); + int Run(TConfig &config) override; + void PrintResponse(const Ydb::Operations::Operation &response) override; + +protected: + TString Database; +}; + +// kikimr +class TClientCommandDiscovery : public TClientCommandTree { +public: + TClientCommandDiscovery(); +}; + +// ydb +class TClientCommandDiscoveryLite : public TClientCommandDiscoveryBase { +public: + TClientCommandDiscoveryLite(); + void Config(TConfig& config) override; + void Parse(TConfig& config) override; +}; + +} +} diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index d908b8b478..450d4135c6 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -21,10 +21,10 @@ namespace NKikimr { namespace NDriverClient { -void WarnProfilePathSet() { - Cout << "FYI: profile path is set. You can use short pathnames. Try --help for more info." << Endl; -} - +void WarnProfilePathSet() { + Cout << "FYI: profile path is set. You can use short pathnames. Try --help for more info." << Endl; +} + class TClientCommandSchemaMkdir : public TClientCommand { public: TClientCommandSchemaMkdir() @@ -35,9 +35,9 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<NAME>", "Full pathname of a directory (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<NAME>", "Full pathname of a directory (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); } TString Base; @@ -45,20 +45,20 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - TString pathname = config.ParseResult->GetFreeArgs()[0]; - size_t pos = pathname.rfind('/'); - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Base = config.Path; - Name = pathname; - return; - } else { - WarnProfilePathSet(); - } - } - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); + TString pathname = config.ParseResult->GetFreeArgs()[0]; + size_t pos = pathname.rfind('/'); + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Base = config.Path; + Name = pathname; + return; + } else { + WarnProfilePathSet(); + } + } + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); } virtual int Run(TConfig& config) override { @@ -80,9 +80,9 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<NAME>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<NAME>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); } TString Base; @@ -90,20 +90,20 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - TString pathname = config.ParseResult->GetFreeArgs()[0]; - size_t pos = pathname.rfind('/'); - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Base = config.Path; - Name = pathname; - return; - } else { - WarnProfilePathSet(); - } - } - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); + TString pathname = config.ParseResult->GetFreeArgs()[0]; + size_t pos = pathname.rfind('/'); + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Base = config.Path; + Name = pathname; + return; + } else { + WarnProfilePathSet(); + } + } + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); } virtual int Run(TConfig& config) override { @@ -115,10 +115,10 @@ public: } }; -class TClientCommandSchemaExec : public TClientCommandConfig { +class TClientCommandSchemaExec : public TClientCommandConfig { public: TClientCommandSchemaExec() - : TClientCommandConfig("execute", { "exec" }, "Execute schema protobuf") + : TClientCommandConfig("execute", { "exec" }, "Execute schema protobuf") {} bool ReturnTxId; @@ -129,8 +129,8 @@ public: TClientCommand::Config(config); ReturnTxId = false; config.Opts->AddLongOption('t', "txid", "Print TxId").NoArgument().SetFlag(&ReturnTxId); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<SCHEMA-PROTO>", "Schema protobuf or file with schema protobuf"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<SCHEMA-PROTO>", "Schema protobuf or file with schema protobuf"); } virtual void Parse(TConfig& config) override { @@ -200,8 +200,8 @@ public: Protobuf = false; PartitionStats = false; Boundaries = false; - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<PATH>", "Schema path or pathId (e.g. 72075186232623600/1225)"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<PATH>", "Schema path or pathId (e.g. 72075186232623600/1225)"); config.Opts->AddLongOption('t', "tree", "Show schema path tree").NoArgument().SetFlag(&Tree); config.Opts->AddLongOption('d', "details", "Show detailed information (like columns in a table)").NoArgument().SetFlag(&Details); config.Opts->AddLongOption('a', "acl", "Show owner and acl information").NoArgument().SetFlag(&AccessRights); @@ -438,8 +438,8 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<PATH>", "Schema path"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<PATH>", "Schema path"); } virtual void Parse(TConfig& config) override { @@ -523,8 +523,8 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); config.Opts->AddLongOption("pool", "default storage pool").RequiredArgument("NAME").StoreResult(&DefaultStoragePool); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<ROOT>", "Schema root name"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<ROOT>", "Schema root name"); } virtual void Parse(TConfig& config) override { @@ -565,10 +565,10 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<USER>", "User"); - SetFreeArgTitle(1, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<USER>", "User"); + SetFreeArgTitle(1, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); config.Opts->AddLongOption('R', "recursive", "Change owner on schema objects recursively").NoArgument().SetFlag(&Recursive); config.Opts->AddLongOption('v', "verbose", "Verbose output").NoArgument().SetFlag(&Verbose); } @@ -579,17 +579,17 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); Owner = config.ParseResult->GetFreeArgs()[0]; - TString pathname = config.ParseResult->GetFreeArgs()[1]; - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Path = config.Path + '/' + pathname; - return; - } else { - WarnProfilePathSet(); - } - } - Path = pathname; + TString pathname = config.ParseResult->GetFreeArgs()[1]; + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Path = config.Path + '/' + pathname; + return; + } else { + WarnProfilePathSet(); + } + } + Path = pathname; } int Chown(TConfig& config, const TString& path) { @@ -662,10 +662,10 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); - SetFreeArgTitle(1, "<ACCESS>", "ACCESS"); + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); + SetFreeArgTitle(1, "<ACCESS>", "ACCESS"); } TString Access; @@ -674,22 +674,22 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - TString pathname = config.ParseResult->GetFreeArgs()[0]; - size_t pos = pathname.rfind('/'); - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Base = config.Path; - Name = pathname; - } else { - WarnProfilePathSet(); - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); - } - } else { - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); - } + TString pathname = config.ParseResult->GetFreeArgs()[0]; + size_t pos = pathname.rfind('/'); + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Base = config.Path; + Name = pathname; + } else { + WarnProfilePathSet(); + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); + } + } else { + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); + } Access = config.ParseResult->GetFreeArgs()[1]; } @@ -733,10 +733,10 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" - " Or short pathname if profile path is set (e.g. test1/test2)."); - SetFreeArgTitle(1, "<ACCESS>", "ACCESS"); + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<PATH>", "Full pathname of an object (e.g. /ru/home/user/mydb/test1/test2).\n" + " Or short pathname if profile path is set (e.g. test1/test2)."); + SetFreeArgTitle(1, "<ACCESS>", "ACCESS"); } TString Access; @@ -745,22 +745,22 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - TString pathname = config.ParseResult->GetFreeArgs()[0]; - size_t pos = pathname.rfind('/'); - if (config.Path) { - // Profile path is set - if (!pathname.StartsWith('/')) { - Base = config.Path; - Name = pathname; - } else { - WarnProfilePathSet(); - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); - } - } else { - Base = pathname.substr(0, pos); - Name = pathname.substr(pos + 1); - } + TString pathname = config.ParseResult->GetFreeArgs()[0]; + size_t pos = pathname.rfind('/'); + if (config.Path) { + // Profile path is set + if (!pathname.StartsWith('/')) { + Base = config.Path; + Name = pathname; + } else { + WarnProfilePathSet(); + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); + } + } else { + Base = pathname.substr(0, pos); + Name = pathname.substr(pos + 1); + } Access = config.ParseResult->GetFreeArgs()[1]; } @@ -818,15 +818,15 @@ public: virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - if (CommandConfig.ClientConfig.Defined()) { + if (CommandConfig.ClientConfig.Defined()) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); if (p) { ClientConfig.Locator = p->Locator; ClientConfig.Timeout = p->Timeout; ClientConfig.MaxMessageSize = p->MaxMessageSize; ClientConfig.MaxInFlight = p->MaxInFlight; - ClientConfig.EnableSsl = p->EnableSsl; - ClientConfig.SslCaCert = p->SslCaCert; + ClientConfig.EnableSsl = p->EnableSsl; + ClientConfig.SslCaCert = p->SslCaCert; } } } @@ -936,15 +936,15 @@ public: TClientCommand::Config(config); config.SetFreeArgsNum(0); - config.Opts->AddLongOption("database", "Database name").AddLongName("db").Required().RequiredArgument("DB").StoreResult(&DatabaseName); - config.Opts->AddLongOption("source", "Source table path").AddLongName("src").RequiredArgument("PATH").AppendTo(&SrcValues); - config.Opts->AddLongOption("destination", "Destination table path").AddLongName("dst").RequiredArgument("PATH").AppendTo(&DstValues); + config.Opts->AddLongOption("database", "Database name").AddLongName("db").Required().RequiredArgument("DB").StoreResult(&DatabaseName); + config.Opts->AddLongOption("source", "Source table path").AddLongName("src").RequiredArgument("PATH").AppendTo(&SrcValues); + config.Opts->AddLongOption("destination", "Destination table path").AddLongName("dst").RequiredArgument("PATH").AppendTo(&DstValues); } virtual void Parse(TConfig& config) override { TClientCommand::Parse(config); - if (!CommandConfig.ClientConfig.Defined()) { + if (!CommandConfig.ClientConfig.Defined()) { return; } @@ -1256,9 +1256,9 @@ public: } }; -TClientCommandSchemaLite::TClientCommandSchemaLite() - : TClientCommandTree("schema", {}, "Schema operations") -{ +TClientCommandSchemaLite::TClientCommandSchemaLite() + : TClientCommandTree("schema", {}, "Schema operations") +{ AddCommand(std::make_unique<TClientCommandSchemaExec>()); AddCommand(std::make_unique<TClientCommandSchemaDescribe>()); AddCommand(std::make_unique<TClientCommandSchemaLs>()); @@ -1268,20 +1268,20 @@ TClientCommandSchemaLite::TClientCommandSchemaLite() AddCommand(std::make_unique<TClientCommandSchemaAccess>()); AddCommand(std::make_unique<TClientCommandSchemaTable>()); AddCommand(std::make_unique<TClientCommandSchemaUserAttribute>()); -} +} -class TClientCommandSchema : public TClientCommandSchemaLite { -public: - TClientCommandSchema() - { +class TClientCommandSchema : public TClientCommandSchemaLite { +public: + TClientCommandSchema() + { AddCommand(std::make_unique<TClientCommandSchemaInit>()); - } -}; - -class TClientCommandDbExec : public TClientCommandConfig { + } +}; + +class TClientCommandDbExec : public TClientCommandConfig { public: TClientCommandDbExec() - : TClientCommandConfig("minikql", { "execute", "exec", "mkql" }, "Execute Mini-KQL query") + : TClientCommandConfig("minikql", { "execute", "exec", "mkql" }, "Execute Mini-KQL query") {} TString MiniKQL; @@ -1290,9 +1290,9 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1, 2); - SetFreeArgTitle(0, "<MINIKQL>", "Text MiniKQL"); - SetFreeArgTitle(1, "<PARAMS>", "Text MiniKQL parameters"); + config.SetFreeArgsNum(1, 2); + SetFreeArgTitle(0, "<MINIKQL>", "Text MiniKQL"); + SetFreeArgTitle(1, "<PARAMS>", "Text MiniKQL parameters"); config.Opts->AddLongOption('p', "proto", "MiniKQL parameters are in protobuf format").NoArgument().SetFlag(&Proto); } @@ -1350,8 +1350,8 @@ public: .RequiredArgument("NUM").StoreResult(&MaxKeys).DefaultValue("1000"); config.Opts->AddLongOption("columns", "Comma separated list of columns to read") .RequiredArgument("NAMES").StoreResult(&Columns); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<PATH>", "path to table"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<PATH>", "path to table"); } virtual void Parse(TConfig& config) override { diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp index 1368790fcf..729ae432c3 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_discovery.cpp @@ -3,47 +3,47 @@ namespace NKikimr { namespace NDriverClient { -TClientCommandDiscoveryBase::TClientCommandDiscoveryBase(const TString& name, const TString& description) - : TClientGRpcCommand(name, {}, description) -{} +TClientCommandDiscoveryBase::TClientCommandDiscoveryBase(const TString& name, const TString& description) + : TClientGRpcCommand(name, {}, description) +{} -int TClientCommandDiscoveryBase::Run(TConfig &config) { - GRpcRequest.set_database(Database); - return TClientGRpcCommand::Run(config); +int TClientCommandDiscoveryBase::Run(TConfig &config) { + GRpcRequest.set_database(Database); + return TClientGRpcCommand::Run(config); } -//using TClientGRpcCommand::PrintResponse; -void TClientCommandDiscoveryBase::PrintResponse(const Ydb::Operations::Operation &response) { - if (response.status() != Ydb::StatusIds::SUCCESS) { - TClientGRpcCommand::PrintResponse(response); - } else { - Ydb::Discovery::ListEndpointsResult result; - Y_VERIFY(response.result().UnpackTo(&result)); - Cout << "OK [" << result.Getself_location() << "]" << Endl; - for (auto &endpoint : result.Getendpoints()) { - Cout << (endpoint.Getssl() ? "grpcs://" : "grpc://"); - Cout << endpoint.Getaddress() << ":" << endpoint.Getport(); - if (endpoint.Getlocation()) - Cout << " [" << endpoint.Getlocation() << "]"; - if (endpoint.serviceSize()) { - for (auto &x : endpoint.Getservice()) - Cout << " #" << x; +//using TClientGRpcCommand::PrintResponse; +void TClientCommandDiscoveryBase::PrintResponse(const Ydb::Operations::Operation &response) { + if (response.status() != Ydb::StatusIds::SUCCESS) { + TClientGRpcCommand::PrintResponse(response); + } else { + Ydb::Discovery::ListEndpointsResult result; + Y_VERIFY(response.result().UnpackTo(&result)); + Cout << "OK [" << result.Getself_location() << "]" << Endl; + for (auto &endpoint : result.Getendpoints()) { + Cout << (endpoint.Getssl() ? "grpcs://" : "grpc://"); + Cout << endpoint.Getaddress() << ":" << endpoint.Getport(); + if (endpoint.Getlocation()) + Cout << " [" << endpoint.Getlocation() << "]"; + if (endpoint.serviceSize()) { + for (auto &x : endpoint.Getservice()) + Cout << " #" << x; } - Cout << Endl; + Cout << Endl; } } -} +} -// Old kikimr behavior: -// ./kikimr discovery list -d <database> -struct TClientCommandDiscoveryListEndpoints - : TClientCommandDiscoveryBase -{ +// Old kikimr behavior: +// ./kikimr discovery list -d <database> +struct TClientCommandDiscoveryListEndpoints + : TClientCommandDiscoveryBase +{ TClientCommandDiscoveryListEndpoints() - : TClientCommandDiscoveryBase("list", "List existing tenants") + : TClientCommandDiscoveryBase("list", "List existing tenants") {} - void Config(TConfig &config) override { + void Config(TConfig &config) override { TClientCommand::Config(config); Database = ""; config.Opts->AddLongOption('d', "db", "Set target database") @@ -57,23 +57,23 @@ TClientCommandDiscovery::TClientCommandDiscovery() AddCommand(std::make_unique<TClientCommandDiscoveryListEndpoints>()); } -// New YDB behavior: -// ./ydb discover <database> - -TClientCommandDiscoveryLite::TClientCommandDiscoveryLite() - : TClientCommandDiscoveryBase("discover", "Endpoint discovery") -{} - -void TClientCommandDiscoveryLite::Config(TConfig& config) { - TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<database>", "Database to discover"); +// New YDB behavior: +// ./ydb discover <database> + +TClientCommandDiscoveryLite::TClientCommandDiscoveryLite() + : TClientCommandDiscoveryBase("discover", "Endpoint discovery") +{} + +void TClientCommandDiscoveryLite::Config(TConfig& config) { + TClientCommand::Config(config); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<database>", "Database to discover"); +} + +void TClientCommandDiscoveryLite::Parse(TConfig& config) { + TClientGRpcCommand::Parse(config); + Database = config.ParseResult->GetFreeArgs().at(0); +} + } - -void TClientCommandDiscoveryLite::Parse(TConfig& config) { - TClientGRpcCommand::Parse(config); - Database = config.ParseResult->GetFreeArgs().at(0); } - -} -} diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp index 79dcf9b7db..197eccfcfd 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp @@ -1,142 +1,142 @@ -#include "cli.h" -#include "cli_cmds.h" -#include <util/folder/path.h> -#include <util/folder/dirut.h> -#include <util/string/strip.h> -#include <util/system/env.h> - -namespace NKikimr { -namespace NDriverClient { - -const TString defaultProfileFile = "~/.ydb/default_profile"; - -TClientCommandRootKikimrBase::TClientCommandRootKikimrBase(const TString& name) - : TClientCommandRootBase(name) -{ -} - -void TClientCommandRootKikimrBase::Config(TConfig& config) { - TClientCommandRootBase::Config(config); - NLastGetopt::TOpts& opts = *config.Opts; - opts.AddLongOption('d', "dump", "Dump requests to error log").NoArgument().Hidden().SetFlag(&DumpRequests); - - TStringBuilder tokenHelp; - tokenHelp << "Security token file" << Endl - << " Token search order:" << Endl - << " 1. This option" << Endl - << " 2. \"YDB_TOKEN\" environment variable" << Endl - << " 3. Connection settings profile (\"~/.ydb/ProfileName/token\")" << Endl - << " 4. Default token file \"" << defaultTokenFile << "\" file"; - opts.AddLongOption('f', "token-file", tokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); - TStringBuilder profileHelp; - profileHelp << "Connection settings profile name" << Endl - << " Profile search order:" << Endl - << " 1. This option" << Endl - << " 2. \"YDB_PROFILE\" environment variable" << Endl - << " 3. Default profile file \"" << defaultProfileFile << "\" file"; - opts.AddLongOption("profile", profileHelp).RequiredArgument("NAME").StoreResult(&LocalProfileName); - - TStringStream stream; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - stream << " -s <[protocol://]host[:port]> [options] <subcommand>" << Endl << Endl - << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; - RenderCommandsDescription(stream, colors); - opts.SetCmdLineDescr(stream.Str()); -} - -void TClientCommandRootKikimrBase::Parse(TConfig& config) { - ParseProfile(); - GetProfileVariable("path", config.Path); - TClientCommandRootBase::Parse(config); - NClient::TKikimr::DUMP_REQUESTS = DumpRequests; -} - -void TClientCommandRootKikimrBase::ParseProfile() { - if (LocalProfileName.empty()) { - LocalProfileName = GetEnv("YDB_PROFILE"); - - if (LocalProfileName.empty()) { - TString profileFile = defaultProfileFile; - ReadFromFileIfExists(profileFile, "default profile", LocalProfileName); - } - } - - if (LocalProfileName) { - ProfileConfig = MakeHolder<TProfileConfig>(LocalProfileName); - } -} - -bool TClientCommandRootKikimrBase::GetProfileVariable(const TString& name, TString& value) { - if (!ProfileConfig) { - return false; - } - return ProfileConfig->GetVariable(name, value); -} - -class TClientCommandRootLite : public TClientCommandRootKikimrBase { -public: - TClientCommandRootLite() - : TClientCommandRootKikimrBase("ydb") - { +#include "cli.h" +#include "cli_cmds.h" +#include <util/folder/path.h> +#include <util/folder/dirut.h> +#include <util/string/strip.h> +#include <util/system/env.h> + +namespace NKikimr { +namespace NDriverClient { + +const TString defaultProfileFile = "~/.ydb/default_profile"; + +TClientCommandRootKikimrBase::TClientCommandRootKikimrBase(const TString& name) + : TClientCommandRootBase(name) +{ +} + +void TClientCommandRootKikimrBase::Config(TConfig& config) { + TClientCommandRootBase::Config(config); + NLastGetopt::TOpts& opts = *config.Opts; + opts.AddLongOption('d', "dump", "Dump requests to error log").NoArgument().Hidden().SetFlag(&DumpRequests); + + TStringBuilder tokenHelp; + tokenHelp << "Security token file" << Endl + << " Token search order:" << Endl + << " 1. This option" << Endl + << " 2. \"YDB_TOKEN\" environment variable" << Endl + << " 3. Connection settings profile (\"~/.ydb/ProfileName/token\")" << Endl + << " 4. Default token file \"" << defaultTokenFile << "\" file"; + opts.AddLongOption('f', "token-file", tokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); + TStringBuilder profileHelp; + profileHelp << "Connection settings profile name" << Endl + << " Profile search order:" << Endl + << " 1. This option" << Endl + << " 2. \"YDB_PROFILE\" environment variable" << Endl + << " 3. Default profile file \"" << defaultProfileFile << "\" file"; + opts.AddLongOption("profile", profileHelp).RequiredArgument("NAME").StoreResult(&LocalProfileName); + + TStringStream stream; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << " -s <[protocol://]host[:port]> [options] <subcommand>" << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + RenderCommandsDescription(stream, colors); + opts.SetCmdLineDescr(stream.Str()); +} + +void TClientCommandRootKikimrBase::Parse(TConfig& config) { + ParseProfile(); + GetProfileVariable("path", config.Path); + TClientCommandRootBase::Parse(config); + NClient::TKikimr::DUMP_REQUESTS = DumpRequests; +} + +void TClientCommandRootKikimrBase::ParseProfile() { + if (LocalProfileName.empty()) { + LocalProfileName = GetEnv("YDB_PROFILE"); + + if (LocalProfileName.empty()) { + TString profileFile = defaultProfileFile; + ReadFromFileIfExists(profileFile, "default profile", LocalProfileName); + } + } + + if (LocalProfileName) { + ProfileConfig = MakeHolder<TProfileConfig>(LocalProfileName); + } +} + +bool TClientCommandRootKikimrBase::GetProfileVariable(const TString& name, TString& value) { + if (!ProfileConfig) { + return false; + } + return ProfileConfig->GetVariable(name, value); +} + +class TClientCommandRootLite : public TClientCommandRootKikimrBase { +public: + TClientCommandRootLite() + : TClientCommandRootKikimrBase("ydb") + { AddCommand(std::make_unique<TClientCommandSchemaLite>()); AddCommand(std::make_unique<TClientCommandWhoAmI>()); AddCommand(std::make_unique<TClientCommandDiscoveryLite>()); - } - - void Config(TConfig& config) override { - config.Opts->AddLongOption('s', "server", "[Required] server address to connect") - .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); - TClientCommandRootKikimrBase::Config(config); - } - - void Parse(TConfig& config) override { - TClientCommandRootBase::Parse(config); - NClient::TKikimr::DUMP_REQUESTS = DumpRequests; - } - - void ParseAddress(TConfig& config) override { - TString hostname; - ui32 port = 2135; - if (Address.empty()) { - if (GetProfileVariable("host", hostname)) { - TString portStr; - if (GetProfileVariable("port", portStr)) { - port = FromString<ui32>(portStr); - } - } else { - if (config.IsHelpCommand()) { - return; - } else { - throw TMissUseException() - << "Missing required option 'server'. Also couldn't find 'host' variable in profile config."; - } - } - } - ParseProtocol(config); - ParseCaCerts(config); - config.Address = Address; - - if (!hostname) { - NMsgBusProxy::TMsgBusClientConfig::CrackAddress(Address, hostname, port); - } + } + + void Config(TConfig& config) override { + config.Opts->AddLongOption('s', "server", "[Required] server address to connect") + .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); + TClientCommandRootKikimrBase::Config(config); + } + + void Parse(TConfig& config) override { + TClientCommandRootBase::Parse(config); + NClient::TKikimr::DUMP_REQUESTS = DumpRequests; + } + + void ParseAddress(TConfig& config) override { + TString hostname; + ui32 port = 2135; + if (Address.empty()) { + if (GetProfileVariable("host", hostname)) { + TString portStr; + if (GetProfileVariable("port", portStr)) { + port = FromString<ui32>(portStr); + } + } else { + if (config.IsHelpCommand()) { + return; + } else { + throw TMissUseException() + << "Missing required option 'server'. Also couldn't find 'host' variable in profile config."; + } + } + } + ParseProtocol(config); + ParseCaCerts(config); + config.Address = Address; + + if (!hostname) { + NMsgBusProxy::TMsgBusClientConfig::CrackAddress(Address, hostname, port); + } CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(hostname + ':' + ToString(port)); - if (config.EnableSsl) { + if (config.EnableSsl) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); - p->EnableSsl = config.EnableSsl; - p->SslCaCert = config.CaCerts; - } - } - -private: - bool DumpRequests = false; -}; - -int NewLiteClient(int argc, char** argv) { + p->EnableSsl = config.EnableSsl; + p->SslCaCert = config.CaCerts; + } + } + +private: + bool DumpRequests = false; +}; + +int NewLiteClient(int argc, char** argv) { THolder<TClientCommandRootLite> commandsRoot = MakeHolder<TClientCommandRootLite>(); - commandsRoot->Opts.SetTitle("YDB client"); - TClientCommand::TConfig config(argc, argv); - return commandsRoot->Process(config); -} - -} -} + commandsRoot->Opts.SetTitle("YDB client"); + TClientCommand::TConfig config(argc, argv); + return commandsRoot->Process(config); +} + +} +} diff --git a/ydb/core/driver_lib/cli_base/cli_command.h b/ydb/core/driver_lib/cli_base/cli_command.h index 4873be0304..5e4ff6cd62 100644 --- a/ydb/core/driver_lib/cli_base/cli_command.h +++ b/ydb/core/driver_lib/cli_base/cli_command.h @@ -1,70 +1,70 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/common/command.h> #include <ydb/core/driver_lib/cli_config_base/config_base.h> -#include <util/string/builder.h> - -namespace NKikimr { - -namespace NDriverClient { - -using namespace NYdb::NConsoleClient; - -class TClientCommandConfig : public TClientCommand { -public: - TClientCommandConfig( - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ) - : TClientCommand(name, aliases, description) - {} - - static bool IsProtobuf(const TString& string) { - return string.empty() || string.find('{') != TString::npos; - } - - static bool IsMiniKQL(const TString& string) { - return string.empty() || string.find('(') != TString::npos; - } - - template <typename ProtobufType> - static TAutoPtr<ProtobufType> GetProtobuf(const TString& string) { - TAutoPtr<ProtobufType> pb(new ProtobufType); - if (IsProtobuf(string)) { - Y_VERIFY(::google::protobuf::TextFormat::ParseFromString(string, pb.Get())); - } else { - Y_VERIFY(ParsePBFromFile(string, pb.Get())); - } - return pb; - } - - template <typename ProtobufType> - static void ParseProtobuf(ProtobufType* pb, const TString& string) { - if (IsProtobuf(string)) { - Y_VERIFY(::google::protobuf::TextFormat::ParseFromString(string, pb)); - } else { - Y_VERIFY(ParsePBFromFile(string, pb)); - } - } - - static TString GetMiniKQL(const TString& string) { - if (IsMiniKQL(string)) { - return string; - } else { - return TUnbufferedFileInput(string).ReadAll(); - } - } - - template <typename ProtobufType> - static TString GetString(const ProtobufType& protobuf) { - TString string; - if (!::google::protobuf::TextFormat::PrintToString(protobuf, &string)) { - ythrow TWithBackTrace<yexception>() << "Error printing protobuf to string!"; - } - return string; - } -}; - -} -} +#include <util/string/builder.h> + +namespace NKikimr { + +namespace NDriverClient { + +using namespace NYdb::NConsoleClient; + +class TClientCommandConfig : public TClientCommand { +public: + TClientCommandConfig( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ) + : TClientCommand(name, aliases, description) + {} + + static bool IsProtobuf(const TString& string) { + return string.empty() || string.find('{') != TString::npos; + } + + static bool IsMiniKQL(const TString& string) { + return string.empty() || string.find('(') != TString::npos; + } + + template <typename ProtobufType> + static TAutoPtr<ProtobufType> GetProtobuf(const TString& string) { + TAutoPtr<ProtobufType> pb(new ProtobufType); + if (IsProtobuf(string)) { + Y_VERIFY(::google::protobuf::TextFormat::ParseFromString(string, pb.Get())); + } else { + Y_VERIFY(ParsePBFromFile(string, pb.Get())); + } + return pb; + } + + template <typename ProtobufType> + static void ParseProtobuf(ProtobufType* pb, const TString& string) { + if (IsProtobuf(string)) { + Y_VERIFY(::google::protobuf::TextFormat::ParseFromString(string, pb)); + } else { + Y_VERIFY(ParsePBFromFile(string, pb)); + } + } + + static TString GetMiniKQL(const TString& string) { + if (IsMiniKQL(string)) { + return string; + } else { + return TUnbufferedFileInput(string).ReadAll(); + } + } + + template <typename ProtobufType> + static TString GetString(const ProtobufType& protobuf) { + TString string; + if (!::google::protobuf::TextFormat::PrintToString(protobuf, &string)) { + ythrow TWithBackTrace<yexception>() << "Error printing protobuf to string!"; + } + return string; + } +}; + +} +} diff --git a/ydb/core/driver_lib/cli_base/cli_grpc.h b/ydb/core/driver_lib/cli_base/cli_grpc.h index 8e20a39441..61d09c9f70 100644 --- a/ydb/core/driver_lib/cli_base/cli_grpc.h +++ b/ydb/core/driver_lib/cli_base/cli_grpc.h @@ -1,132 +1,132 @@ -#pragma once - -#include "cli_command.h" - +#pragma once + +#include "cli_command.h" + #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> #include <ydb/public/lib/deprecated/client/grpc_client.h> #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> #include <util/string/split.h> -#include <util/string/type.h> - -namespace NKikimr { -namespace NDriverClient { - -template <typename TService, typename TRequest, typename TResponse, typename TFunction> -int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, - const TRequest &request, - Ydb::Operations::Operation &response, - TFunction function, - const TString& securityToken) -{ - int res = 0; - - if (!clientConfig.Locator) { - Cerr << "GRPC call error: GRPC server is not specified (MBus protocol is not supported for this command)." << Endl; - return -2; - } - +#include <util/string/type.h> + +namespace NKikimr { +namespace NDriverClient { + +template <typename TService, typename TRequest, typename TResponse, typename TFunction> +int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, + const TRequest &request, + Ydb::Operations::Operation &response, + TFunction function, + const TString& securityToken) +{ + int res = 0; + + if (!clientConfig.Locator) { + Cerr << "GRPC call error: GRPC server is not specified (MBus protocol is not supported for this command)." << Endl; + return -2; + } + NGrpc::TCallMeta meta; - if (securityToken) { - meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, securityToken}); - } - + if (securityToken) { + meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, securityToken}); + } + NGrpc::TResponseCallback<TResponse> responseCb = [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, TResponse &&resp) -> void { - res = (int)grpcStatus.GRpcStatusCode; - if (!res) { - response.CopyFrom(resp.operation()); - } else { - Cerr << "GRPC call error: " << grpcStatus.Msg << Endl; - } - }; - - { + res = (int)grpcStatus.GRpcStatusCode; + if (!res) { + response.CopyFrom(resp.operation()); + } else { + Cerr << "GRPC call error: " << grpcStatus.Msg << Endl; + } + }; + + { NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<TService>(clientConfig); + auto connection = clientLow.CreateGRpcServiceConnection<TService>(clientConfig); connection->DoRequest(request, responseCb, function, meta); - } - + } + NGrpc::TResponseCallback<Ydb::Operations::GetOperationResponse> operationCb = [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Operations::GetOperationResponse &&resp) -> void { - res = (int)grpcStatus.GRpcStatusCode; - if (!res) { - response.CopyFrom(resp.operation()); - } else { - Cerr << "GRPC call error: " << grpcStatus.Msg << Endl; - } - }; - - while (!res && !response.ready()) { - Sleep(TDuration::MilliSeconds(100)); + res = (int)grpcStatus.GRpcStatusCode; + if (!res) { + response.CopyFrom(resp.operation()); + } else { + Cerr << "GRPC call error: " << grpcStatus.Msg << Endl; + } + }; + + while (!res && !response.ready()) { + Sleep(TDuration::MilliSeconds(100)); NGrpc::TGRpcClientLow clientLow; - auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Operation::V1::OperationService>(clientConfig); - Ydb::Operations::GetOperationRequest request; - request.set_id(response.id()); + auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Operation::V1::OperationService>(clientConfig); + Ydb::Operations::GetOperationRequest request; + request.set_id(response.id()); connection->DoRequest(request, operationCb, &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, meta); - } - - return res; -} - -template <typename TService, typename TRequest, typename TResponse, - typename TFunction, TFunction function> -class TClientGRpcCommand : public TClientCommand { -public: - TRequest GRpcRequest; - NGRpcProxy::TGRpcClientConfig ClientConfig; - - TClientGRpcCommand(const TString &name, - const std::initializer_list<TString> &aliases, - const TString &description) - : TClientCommand(name, aliases, description) - {} - - void Parse(TConfig& config) override - { - TClientCommand::Parse(config); - - if (CommandConfig.ClientConfig.Defined()) { + } + + return res; +} + +template <typename TService, typename TRequest, typename TResponse, + typename TFunction, TFunction function> +class TClientGRpcCommand : public TClientCommand { +public: + TRequest GRpcRequest; + NGRpcProxy::TGRpcClientConfig ClientConfig; + + TClientGRpcCommand(const TString &name, + const std::initializer_list<TString> &aliases, + const TString &description) + : TClientCommand(name, aliases, description) + {} + + void Parse(TConfig& config) override + { + TClientCommand::Parse(config); + + if (CommandConfig.ClientConfig.Defined()) { auto *p = std::get_if<NGRpcProxy::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); - if (p) { - ClientConfig.Locator = p->Locator; - ClientConfig.Timeout = p->Timeout; - ClientConfig.MaxMessageSize = p->MaxMessageSize; - ClientConfig.MaxInFlight = p->MaxInFlight; - ClientConfig.EnableSsl = p->EnableSsl; - ClientConfig.SslCaCert = p->SslCaCert; - } - } - } - - int Run(TConfig& config) override - { - Ydb::Operations::Operation response; - int res; - - res = DoGRpcRequest<TService, TRequest, TResponse, TFunction> - (ClientConfig, GRpcRequest, response, function, config.SecurityToken); - - if (!res) { - PrintResponse(response); - } - - return res; - } - - virtual void PrintResponse(const Ydb::Operations::Operation &response) - { - if (response.status() == Ydb::StatusIds::SUCCESS) { - Cout << "OK" << Endl; - } else { - Cout << "ERROR: " << response.status() << Endl; - for (auto &issue : response.issues()) - Cout << issue.message() << Endl; - } - } -}; - -} -} - + if (p) { + ClientConfig.Locator = p->Locator; + ClientConfig.Timeout = p->Timeout; + ClientConfig.MaxMessageSize = p->MaxMessageSize; + ClientConfig.MaxInFlight = p->MaxInFlight; + ClientConfig.EnableSsl = p->EnableSsl; + ClientConfig.SslCaCert = p->SslCaCert; + } + } + } + + int Run(TConfig& config) override + { + Ydb::Operations::Operation response; + int res; + + res = DoGRpcRequest<TService, TRequest, TResponse, TFunction> + (ClientConfig, GRpcRequest, response, function, config.SecurityToken); + + if (!res) { + PrintResponse(response); + } + + return res; + } + + virtual void PrintResponse(const Ydb::Operations::Operation &response) + { + if (response.status() == Ydb::StatusIds::SUCCESS) { + Cout << "OK" << Endl; + } else { + Cout << "ERROR: " << response.status() << Endl; + for (auto &issue : response.issues()) + Cout << issue.message() << Endl; + } + } +}; + +} +} + diff --git a/ydb/core/driver_lib/cli_base/cli_kicli.cpp b/ydb/core/driver_lib/cli_base/cli_kicli.cpp index 3743cbe726..5ba51e77a6 100644 --- a/ydb/core/driver_lib/cli_base/cli_kicli.cpp +++ b/ydb/core/driver_lib/cli_base/cli_kicli.cpp @@ -1,88 +1,88 @@ -#include "cli_kicli.h" - -namespace NKikimr { -namespace NDriverClient { - -template <typename RequestType> -void SetToken(TClientCommand::TConfig& config, TAutoPtr<RequestType>& request) { - if (!config.SecurityToken.empty()) { - request->Record.SetSecurityToken(config.SecurityToken); - } -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusRequest>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeInitRoot>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeOperation>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeDescribe>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusWhoAmI>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusBSAdm>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusCmsRequest>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusConsoleRequest>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalMKQL>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalSchemeTx>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusFillNode>& request) { - SetToken(config, request); -} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusDrainNode>& request) { - SetToken(config, request); -} - -template <> +#include "cli_kicli.h" + +namespace NKikimr { +namespace NDriverClient { + +template <typename RequestType> +void SetToken(TClientCommand::TConfig& config, TAutoPtr<RequestType>& request) { + if (!config.SecurityToken.empty()) { + request->Record.SetSecurityToken(config.SecurityToken); + } +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusRequest>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeInitRoot>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeOperation>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeDescribe>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusWhoAmI>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusBSAdm>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusCmsRequest>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusConsoleRequest>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalMKQL>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalSchemeTx>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusFillNode>& request) { + SetToken(config, request); +} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusDrainNode>& request) { + SetToken(config, request); +} + +template <> int OnMessageBus(const TClientCommand::TConfig& config, const NMsgBusProxy::TBusResponse& response) { - const NKikimrClient::TResponse& resp(response.Record); - if (resp.HasExecutionEngineEvaluatedResponse()) { - NClient::TValue value = NClient::TValue::Create(resp.GetExecutionEngineEvaluatedResponse().GetValue(), resp.GetExecutionEngineEvaluatedResponse().GetType()); + const NKikimrClient::TResponse& resp(response.Record); + if (resp.HasExecutionEngineEvaluatedResponse()) { + NClient::TValue value = NClient::TValue::Create(resp.GetExecutionEngineEvaluatedResponse().GetValue(), resp.GetExecutionEngineEvaluatedResponse().GetType()); Cout << value.GetValueText<NClient::TFormatJSON>({config.JsonUi64AsText, config.JsonBinaryAsBase64}); - } - return 0; -} - -int InvokeThroughKikimr(TClientCommand::TConfig& config, std::function<int(NClient::TKikimr&)> handler) { - auto visitor = [&](const auto& conf) { - NClient::TKikimr kikimr(conf); - if (!config.SecurityToken.empty()) { - kikimr.SetSecurityToken(config.SecurityToken); - } - return handler(kikimr); - }; - if (const auto& conf = CommandConfig.ClientConfig) { + } + return 0; +} + +int InvokeThroughKikimr(TClientCommand::TConfig& config, std::function<int(NClient::TKikimr&)> handler) { + auto visitor = [&](const auto& conf) { + NClient::TKikimr kikimr(conf); + if (!config.SecurityToken.empty()) { + kikimr.SetSecurityToken(config.SecurityToken); + } + return handler(kikimr); + }; + if (const auto& conf = CommandConfig.ClientConfig) { return std::visit(std::move(visitor), *conf); - } else { - Cerr << "Client configuration is not provided" << Endl; - return 1; - } -} - -} -} + } else { + Cerr << "Client configuration is not provided" << Endl; + return 1; + } +} + +} +} diff --git a/ydb/core/driver_lib/cli_base/cli_kicli.h b/ydb/core/driver_lib/cli_base/cli_kicli.h index 0c4fdabba0..a93e2717fb 100644 --- a/ydb/core/driver_lib/cli_base/cli_kicli.h +++ b/ydb/core/driver_lib/cli_base/cli_kicli.h @@ -1,91 +1,91 @@ -#pragma once - -#include "cli_command.h" - +#pragma once + +#include "cli_command.h" + #include <ydb/public/lib/deprecated/kicli/kicli.h> - -namespace NKikimr { -namespace NDriverClient { - -template <typename ResultType> -int HandleResponse(const NThreading::TFuture<ResultType>& future, std::function<int(const ResultType&)> callback) { - static char animSprites[] = "\\|/-"; - size_t animSprite = 0; - NHPTimer::STime startTime; - if (TClientCommand::TIME_REQUESTS) { - NHPTimer::GetTime(&startTime); - } - while (!future.Wait(TInstant::MilliSeconds(200))) { - if (TClientCommand::PROGRESS_REQUESTS) { - Cerr << '\r' << animSprites[animSprite++]; - if (animSprite > 3) - animSprite = 0; - if (TClientCommand::TIME_REQUESTS) { - NHPTimer::STime tempTime = startTime; - TDuration timePassed = TDuration::Seconds(NHPTimer::GetTimePassed(&tempTime)); - Cerr << ' ' << timePassed; - } - } - } - if (TClientCommand::PROGRESS_REQUESTS) { - Cerr << '\r'; - } - if (TClientCommand::TIME_REQUESTS) { - TDuration timePassed = TDuration::Seconds(NHPTimer::GetTimePassed(&startTime)); - Cerr << timePassed << " " << Endl; - } - const ResultType& result(future.GetValue()); - NClient::TError error(result.GetError()); - if (!error.Success()) { - Cerr << error.GetCode() << ' ' << error.GetMessage() << Endl; - return 1; - } - return callback(result); -} - -int InvokeThroughKikimr(TClientCommand::TConfig& config, std::function<int(NClient::TKikimr&)> handler); - -template <typename RequestType> -void PrepareRequest(TClientCommand::TConfig&, TAutoPtr<RequestType>&) {} - -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusRequest>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeInitRoot>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeOperation>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeDescribe>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusWhoAmI>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusBSAdm>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusCmsRequest>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusConsoleRequest>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalMKQL>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalSchemeTx>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusFillNode>& request); -void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusDrainNode>& request); - -template <typename ResponseType> + +namespace NKikimr { +namespace NDriverClient { + +template <typename ResultType> +int HandleResponse(const NThreading::TFuture<ResultType>& future, std::function<int(const ResultType&)> callback) { + static char animSprites[] = "\\|/-"; + size_t animSprite = 0; + NHPTimer::STime startTime; + if (TClientCommand::TIME_REQUESTS) { + NHPTimer::GetTime(&startTime); + } + while (!future.Wait(TInstant::MilliSeconds(200))) { + if (TClientCommand::PROGRESS_REQUESTS) { + Cerr << '\r' << animSprites[animSprite++]; + if (animSprite > 3) + animSprite = 0; + if (TClientCommand::TIME_REQUESTS) { + NHPTimer::STime tempTime = startTime; + TDuration timePassed = TDuration::Seconds(NHPTimer::GetTimePassed(&tempTime)); + Cerr << ' ' << timePassed; + } + } + } + if (TClientCommand::PROGRESS_REQUESTS) { + Cerr << '\r'; + } + if (TClientCommand::TIME_REQUESTS) { + TDuration timePassed = TDuration::Seconds(NHPTimer::GetTimePassed(&startTime)); + Cerr << timePassed << " " << Endl; + } + const ResultType& result(future.GetValue()); + NClient::TError error(result.GetError()); + if (!error.Success()) { + Cerr << error.GetCode() << ' ' << error.GetMessage() << Endl; + return 1; + } + return callback(result); +} + +int InvokeThroughKikimr(TClientCommand::TConfig& config, std::function<int(NClient::TKikimr&)> handler); + +template <typename RequestType> +void PrepareRequest(TClientCommand::TConfig&, TAutoPtr<RequestType>&) {} + +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusRequest>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeInitRoot>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeOperation>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusSchemeDescribe>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusWhoAmI>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusBSAdm>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusCmsRequest>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusConsoleRequest>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalMKQL>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusTabletLocalSchemeTx>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusFillNode>& request); +void PrepareRequest(TClientCommand::TConfig& config, TAutoPtr<NMsgBusProxy::TBusDrainNode>& request); + +template <typename ResponseType> int OnMessageBus(const TClientCommand::TConfig& config, const ResponseType& response); - -template <typename RequestType, typename ResponseType> -int MessageBusCall(TClientCommand::TConfig& config, TAutoPtr<RequestType> request, std::function<int(const ResponseType&)> callback) { - PrepareRequest(config, request); - auto handler = [&](NClient::TKikimr& kikimr) { - NThreading::TFuture<NClient::TResult> future(kikimr.ExecuteRequest(request.Release())); - return HandleResponse<NClient::TResult>(future, [&](const NClient::TResult& result) -> int { - if (!result.HaveResponse<ResponseType>()) { - Cerr << "Unknown response received: " << result.GetType() << Endl; - return 1; - } - return callback(result.GetResponse<ResponseType>()); - }); - }; - return InvokeThroughKikimr(config, std::move(handler)); -} - -template <typename RequestType, typename ResponseType = NMsgBusProxy::TBusResponse> -int MessageBusCall(TClientCommand::TConfig& config, TAutoPtr<RequestType> request) { + +template <typename RequestType, typename ResponseType> +int MessageBusCall(TClientCommand::TConfig& config, TAutoPtr<RequestType> request, std::function<int(const ResponseType&)> callback) { + PrepareRequest(config, request); + auto handler = [&](NClient::TKikimr& kikimr) { + NThreading::TFuture<NClient::TResult> future(kikimr.ExecuteRequest(request.Release())); + return HandleResponse<NClient::TResult>(future, [&](const NClient::TResult& result) -> int { + if (!result.HaveResponse<ResponseType>()) { + Cerr << "Unknown response received: " << result.GetType() << Endl; + return 1; + } + return callback(result.GetResponse<ResponseType>()); + }); + }; + return InvokeThroughKikimr(config, std::move(handler)); +} + +template <typename RequestType, typename ResponseType = NMsgBusProxy::TBusResponse> +int MessageBusCall(TClientCommand::TConfig& config, TAutoPtr<RequestType> request) { return MessageBusCall<RequestType, ResponseType>(config, request, [&config](const ResponseType& response) -> int { return OnMessageBus(config, response); - }); -} - -} -} + }); +} + +} +} diff --git a/ydb/core/driver_lib/cli_base/ya.make b/ydb/core/driver_lib/cli_base/ya.make index ac124a9a1e..3945415de0 100644 --- a/ydb/core/driver_lib/cli_base/ya.make +++ b/ydb/core/driver_lib/cli_base/ya.make @@ -1,24 +1,24 @@ -LIBRARY(cli_base) - -OWNER(g:kikimr) - -SRCS( - cli_cmds_db.cpp - cli_cmds_discovery.cpp - cli_cmds_root.cpp - cli_cmds_whoami.cpp - cli_kicli.cpp -) - -PEERDIR( +LIBRARY(cli_base) + +OWNER(g:kikimr) + +SRCS( + cli_cmds_db.cpp + cli_cmds_discovery.cpp + cli_cmds_root.cpp + cli_cmds_whoami.cpp + cli_kicli.cpp +) + +PEERDIR( ydb/core/driver_lib/cli_config_base ydb/library/aclib ydb/public/lib/deprecated/kicli ydb/public/lib/ydb_cli/common ydb/public/sdk/cpp/client/resources ydb/public/sdk/cpp/client/ydb_table -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/driver_lib/cli_config_base/config_base.cpp b/ydb/core/driver_lib/cli_config_base/config_base.cpp index a6f899163c..3672ce3ec2 100644 --- a/ydb/core/driver_lib/cli_config_base/config_base.cpp +++ b/ydb/core/driver_lib/cli_config_base/config_base.cpp @@ -2,8 +2,8 @@ namespace NKikimr { -TCommandConfig CommandConfig; - +TCommandConfig CommandConfig; + TDuration ParseDuration(const TStringBuf& str) { TDuration result; if (TDuration::TryParse(str, result)) @@ -22,12 +22,12 @@ TCommandConfig::TServerEndpoint TCommandConfig::ParseServerAddress(const TString endpoint.ServerType = EServerType::GRpc; endpoint.Address = endpoint.Address.substr(7); port = 2135; - endpoint.EnableSsl = false; - } else if (address.StartsWith("grpcs://")) { - endpoint.ServerType = EServerType::GRpc; - endpoint.Address = endpoint.Address.substr(8); - port = 2135; - endpoint.EnableSsl = true; + endpoint.EnableSsl = false; + } else if (address.StartsWith("grpcs://")) { + endpoint.ServerType = EServerType::GRpc; + endpoint.Address = endpoint.Address.substr(8); + port = 2135; + endpoint.EnableSsl = true; } else if (address.StartsWith("mbus://")) { endpoint.ServerType = EServerType::MessageBus; endpoint.Address = endpoint.Address.substr(7); diff --git a/ydb/core/driver_lib/cli_config_base/config_base.h b/ydb/core/driver_lib/cli_config_base/config_base.h index 7dc1e533eb..6db66dc7d1 100644 --- a/ydb/core/driver_lib/cli_config_base/config_base.h +++ b/ydb/core/driver_lib/cli_config_base/config_base.h @@ -44,14 +44,14 @@ struct TCommandConfig { struct TServerEndpoint { EServerType ServerType; TString Address; - TMaybe<bool> EnableSsl = Nothing(); + TMaybe<bool> EnableSsl = Nothing(); }; static TServerEndpoint ParseServerAddress(const TString& address); }; -extern TCommandConfig CommandConfig; - +extern TCommandConfig CommandConfig; + } // namespace NKikimr @@ -87,10 +87,10 @@ template<> struct TCliCommands<MODES_ENUM> {\ \ static TString CommandsDescription(const TString& toolName) {\ TStringStream helpStream;\ - helpStream << "[global options...] <command> [command options...] [mbus] [messagebus options...]";\ + helpStream << "[global options...] <command> [command options...] [mbus] [messagebus options...]";\ helpStream << "\n\nAvailable commands are:\n";\ MODES_MAP(MODE_GEN_HELP);\ - helpStream << "\nSee '" << toolName << " <command> --help' for mode options\n";\ + helpStream << "\nSee '" << toolName << " <command> --help' for mode options\n";\ return helpStream.Str();\ } \ \ diff --git a/ydb/core/driver_lib/cli_utils/cli.cpp b/ydb/core/driver_lib/cli_utils/cli.cpp index 69e08c2aa4..f84f6784df 100644 --- a/ydb/core/driver_lib/cli_utils/cli.cpp +++ b/ydb/core/driver_lib/cli_utils/cli.cpp @@ -13,19 +13,19 @@ void DumpProxyErrorCodes(IOutputStream &o, const NKikimrClient::TResponse &respo } } -void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix) { - for (auto opt : opts.Opts_) { - if (opt.Get()->GetName().StartsWith(prefix)) { - opt.Get()->Hidden_ = true; - } - } +void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix) { + for (auto opt : opts.Opts_) { + if (opt.Get()->GetName().StartsWith(prefix)) { + opt.Get()->Hidden_ = true; + } + } +} + +void HideOptions(NLastGetopt::TOpts& opts) { + for (auto opt : opts.Opts_) { + opt.Get()->Hidden_ = true; + } +} + } - -void HideOptions(NLastGetopt::TOpts& opts) { - for (auto opt : opts.Opts_) { - opt.Get()->Hidden_ = true; - } } - -} -} diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h index adefaa019a..099664792e 100644 --- a/ydb/core/driver_lib/cli_utils/cli.h +++ b/ydb/core/driver_lib/cli_utils/cli.h @@ -4,7 +4,7 @@ #include <ydb/core/driver_lib/cli_base/cli.h> #include <ydb/core/driver_lib/run/factories.h> - + #include <library/cpp/actors/interconnect/poller_tcp.h> #include <ydb/public/lib/deprecated/client/msgbus_client.h> @@ -37,8 +37,8 @@ namespace NDriverClient { int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv); int LoadRequest(TCommandConfig &cmdConf, int argc, char **argv); int ActorsysPerfTest(TCommandConfig &cmdConf, int argc, char **argv); - void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix); - void HideOptions(NLastGetopt::TOpts& opts); + void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix); + void HideOptions(NLastGetopt::TOpts& opts); int NewClient(int argc, char** argv, std::shared_ptr<TModuleFactories> factories); TString NewClientCommandsDescription(std::shared_ptr<TModuleFactories> factories); } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp index 664b55e919..fcc9305996 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmd_config.cpp @@ -28,10 +28,10 @@ namespace NDriverClient { switch (endpoint.ServerType) { case TCommandConfig::EServerType::GRpc: ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); - if (endpoint.EnableSsl.Defined()) { + if (endpoint.EnableSsl.Defined()) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&ClientConfig.GetRef()); - p->EnableSsl = endpoint.EnableSsl.GetRef(); - } + p->EnableSsl = endpoint.EnableSsl.GetRef(); + } break; case TCommandConfig::EServerType::MessageBus: if (!endpoint.Address.empty()) { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp index 3f15e6b22e..8d8e54af65 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_cms.cpp @@ -10,7 +10,7 @@ namespace NKikimr { namespace NDriverClient { -class TCmsClientCommand : public TClientCommandConfig { +class TCmsClientCommand : public TClientCommandConfig { public: TString Domain; NKikimrClient::TCmsRequest Request; @@ -18,7 +18,7 @@ public: TCmsClientCommand(const TString &name, const std::initializer_list<TString> &aliases, const TString &description) - : TClientCommandConfig(name, aliases, description) + : TClientCommandConfig(name, aliases, description) { } @@ -99,7 +99,7 @@ public: config.Opts->AddLongOption("host", "Get state for specified host(s)") .RequiredArgument("NAME").AppendTo(&Hosts); - config.SetFreeArgsNum(0); + config.SetFreeArgsNum(0); } void Parse(TConfig& config) override @@ -200,10 +200,10 @@ public: config.Opts->AddLongOption("user", "User name").Required() .RequiredArgument("NAME").StoreResult(&User); if (HasId) { - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<ID>", "Request ID"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<ID>", "Request ID"); } else { - config.SetFreeArgsNum(0); + config.SetFreeArgsNum(0); } config.Opts->AddLongOption("dry", "Dry run").NoArgument().SetFlag(&DryRun); } @@ -274,8 +274,8 @@ public: config.Opts->AddLongOption("user", "User name").Required() .RequiredArgument("NAME").StoreResult(&User); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<ID>", "Request ID"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<ID>", "Request ID"); config.Opts->AddLongOption("dry", "Dry run").NoArgument().SetFlag(&DryRun); } @@ -331,7 +331,7 @@ public: if (HasDuration) config.Opts->AddLongOption("duration", "Action duration in minutes") .Required().RequiredArgument("NUM").StoreResult(&Duration); - config.SetFreeArgsMin(1); + config.SetFreeArgsMin(1); config.Opts->SetFreeArgDefaultTitle("<NAME>", FreeArgDescr(FreeArgField)); } @@ -697,10 +697,10 @@ public: config.Opts->AddLongOption("user", "User name").Required() .RequiredArgument("NAME").StoreResult(&User); if (HasId) { - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<ID>", "Notification ID"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<ID>", "Notification ID"); } else { - config.SetFreeArgsNum(0); + config.SetFreeArgsNum(0); } config.Opts->AddLongOption("dry", "Dry run").NoArgument().SetFlag(&DryRun); } @@ -819,10 +819,10 @@ public: config.Opts->AddLongOption("user", "User name").Required() .RequiredArgument("NAME").StoreResult(&User); if (HasId) { - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<ID>", "Permission ID"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<ID>", "Permission ID"); } else { - config.SetFreeArgsNum(0); + config.SetFreeArgsNum(0); } if (HasDeadline) { config.Opts->AddLongOption("hours", "New permission duration") @@ -945,8 +945,8 @@ public: virtual void Config(TConfig& config) override { TCmsClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<CONFIG>", "Config protobuf or file with config protobuf"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<CONFIG>", "Config protobuf or file with config protobuf"); } void Parse(TConfig& config) override @@ -997,8 +997,8 @@ public: allMarkers.push_back(static_cast<NKikimrCms::EMarker>(i)); TString desc = Sprintf("Marker name (%s)", JoinSeq(", ", allMarkers).data()); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<MARKER>", desc); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<MARKER>", desc); config.Opts->AddLongOption("host", "Host(s) to mark") .RequiredArgument("NAME").AppendTo(&Hosts); config.Opts->AddLongOption("node", "Node(s) to mark") diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp index 1dbd2f5a21..0497d4ea8f 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp @@ -8,7 +8,7 @@ namespace NKikimr { namespace NDriverClient { -class TConsoleClientCommand : public TClientCommandConfig { +class TConsoleClientCommand : public TClientCommandConfig { public: TString Domain; ui32 Retries; @@ -17,7 +17,7 @@ public: TConsoleClientCommand(const TString &name, const std::initializer_list<TString> &aliases, const TString &description) - : TClientCommandConfig(name, aliases, description) + : TClientCommandConfig(name, aliases, description) , Retries(0) { } @@ -329,8 +329,8 @@ public: virtual void Config(TConfig& config) override { TConsoleClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<SCHEMA-PROTO>", "Console request protobuf or file with protobuf"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<SCHEMA-PROTO>", "Console request protobuf or file with protobuf"); } virtual void Parse(TConfig& config) override { @@ -383,8 +383,8 @@ public: config.Opts->AddLongOption("merge-overwrite-repeated", "Merge provided config with the current one" " but overwrite those repeated field which are not empty in provided config") .NoArgument().SetFlag(&MergeOverwriteRepeated); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<CONFIG-PROTO>", "Console config protobuf or file with protobuf"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<CONFIG-PROTO>", "Console config protobuf or file with protobuf"); } virtual void Parse(TConfig& config) override { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp index eb471f51a9..a9be370d37 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_disk.cpp @@ -22,7 +22,7 @@ public: IsVerbose = false; LockDevice = false; MainKey = 0; - config.SetFreeArgsNum(1); + config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<PATH>", "Disk path"); config.Opts->AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM") .Optional().StoreResult(&MainKey); // TODO: make required @@ -120,7 +120,7 @@ public: SectorSize = 4 << 10; Guid = 0; IsErasureEncode = false; - config.SetFreeArgsNum(1); + config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<PATH>", "Disk path"); config.Opts->AddLongOption('d', "disk-size", "disk size to set (supports K/M/G/T suffixes, 0 to autodetect, default = 0)\n" "kikimr needs disk of at least 16 GiB, disk must be large enough to contain at least 100 chunks") @@ -144,7 +144,7 @@ public: config.Opts->AddLongOption('e', "erasure-encode", "erasure-encode data to recover from single-sector failures") .Optional().NoArgument().SetFlag(&IsErasureEncode); - config.Opts->SetCmdLineDescr("\n\n" + config.Opts->SetCmdLineDescr("\n\n" "Kikimr was designed to work with large block-devices, like 4 TiB HDDs and 1 TiB SSDs\n" "It will work with files, but don't expect good performance.\n" "If you still want to run kikimr with a tiny file, specify --disk-size 17179869184 --chunk-size 33554432"); diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp index 726907f54a..af2801de36 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp @@ -12,7 +12,7 @@ namespace NDriverClient { using NBsController::TCandidate; using NBsController::GroupFromCandidates; -class TClientCommandGenConfigStatic : public TClientCommandConfig { +class TClientCommandGenConfigStatic : public TClientCommandConfig { TString ErasureList; TString ErasureStr; @@ -109,7 +109,7 @@ class TClientCommandGenConfigStatic : public TClientCommandConfig { public: TClientCommandGenConfigStatic() - : TClientCommandConfig("static", {}, "Generate configuration for static group") + : TClientCommandConfig("static", {}, "Generate configuration for static group") { TStringStream erasureList("{"); for (ui32 species = 0; species < TBlobStorageGroupType::ErasureSpeciesCount; ++species) { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp index efba2cb4df..11e89d1960 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_node.cpp @@ -43,7 +43,7 @@ public: virtual void Config(TConfig& config) override { TClientCommandTree::Config(config); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); } NKikimrClient::TResponse BusResponse; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp index 223d687e10..ec049fab11 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_root.cpp @@ -9,14 +9,14 @@ namespace NKikimr { namespace NDriverClient { -using namespace NYdb::NConsoleClient; - +using namespace NYdb::NConsoleClient; + extern void AddClientCommandServer(TClientCommandTree& parent, std::shared_ptr<TModuleFactories> factories); -class TClientCommandRoot : public TClientCommandRootKikimrBase { +class TClientCommandRoot : public TClientCommandRootKikimrBase { public: TClientCommandRoot(std::shared_ptr<TModuleFactories> factories) - : TClientCommandRootKikimrBase("kikimr") + : TClientCommandRootKikimrBase("kikimr") { AddCommand(std::make_unique<TClientCommandAdmin>()); AddCommand(std::make_unique<TClientCommandDb>()); @@ -25,63 +25,63 @@ public: AddCommand(std::make_unique<TClientCommandDiscovery>()); AddClientCommandServer(*this, std::move(factories)); } - - void Config(TConfig& config) override { - NLastGetopt::TOpts& opts = *config.Opts; - MsgBusClientConfig.ConfigureLastGetopt(opts, "mb-"); - HideOptions(*config.Opts); - opts.AddLongOption('k', "token", "security token").RequiredArgument("TOKEN").StoreResult(&Token); - opts.AddLongOption('s', "server", "server address to connect") - .RequiredArgument("HOST[:PORT]").StoreResult(&Address); - TClientCommandRootKikimrBase::Config(config); - } - - void ParseAddress(TConfig& config) override { - if (Address.empty()) { - TString ydbServer = GetEnv("YDB_SERVER"); - if (ydbServer == nullptr) { - ydbServer = GetEnv("KIKIMR_SERVER"); - } - if (ydbServer != nullptr) { - Address = ydbServer; - } - } - config.Address = Address; - - TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(Address); - if (endpoint.EnableSsl.Defined()) { - config.EnableSsl = endpoint.EnableSsl.GetRef(); - } - ParseCaCerts(config); - - switch (endpoint.ServerType) { - case TCommandConfig::EServerType::GRpc: + + void Config(TConfig& config) override { + NLastGetopt::TOpts& opts = *config.Opts; + MsgBusClientConfig.ConfigureLastGetopt(opts, "mb-"); + HideOptions(*config.Opts); + opts.AddLongOption('k', "token", "security token").RequiredArgument("TOKEN").StoreResult(&Token); + opts.AddLongOption('s', "server", "server address to connect") + .RequiredArgument("HOST[:PORT]").StoreResult(&Address); + TClientCommandRootKikimrBase::Config(config); + } + + void ParseAddress(TConfig& config) override { + if (Address.empty()) { + TString ydbServer = GetEnv("YDB_SERVER"); + if (ydbServer == nullptr) { + ydbServer = GetEnv("KIKIMR_SERVER"); + } + if (ydbServer != nullptr) { + Address = ydbServer; + } + } + config.Address = Address; + + TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(Address); + if (endpoint.EnableSsl.Defined()) { + config.EnableSsl = endpoint.EnableSsl.GetRef(); + } + ParseCaCerts(config); + + switch (endpoint.ServerType) { + case TCommandConfig::EServerType::GRpc: CommandConfig.ClientConfig = NGrpc::TGRpcClientConfig(endpoint.Address); - if (config.EnableSsl) { + if (config.EnableSsl) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); - p->EnableSsl = config.EnableSsl; - p->SslCaCert = config.CaCerts; - } - break; - case TCommandConfig::EServerType::MessageBus: - if (!endpoint.Address.empty()) { - NMsgBusProxy::TMsgBusClientConfig::CrackAddress( - endpoint.Address, - MsgBusClientConfig.Ip, - MsgBusClientConfig.Port); - } - CommandConfig.ClientConfig = std::move(MsgBusClientConfig); - break; - } - } - -private: - NMsgBusProxy::TMsgBusClientConfig MsgBusClientConfig; + p->EnableSsl = config.EnableSsl; + p->SslCaCert = config.CaCerts; + } + break; + case TCommandConfig::EServerType::MessageBus: + if (!endpoint.Address.empty()) { + NMsgBusProxy::TMsgBusClientConfig::CrackAddress( + endpoint.Address, + MsgBusClientConfig.Ip, + MsgBusClientConfig.Port); + } + CommandConfig.ClientConfig = std::move(MsgBusClientConfig); + break; + } + } + +private: + NMsgBusProxy::TMsgBusClientConfig MsgBusClientConfig; }; int NewClient(int argc, char** argv, std::shared_ptr<TModuleFactories> factories) { THolder<TClientCommandRoot> commandsRoot = MakeHolder<TClientCommandRoot>(std::move(factories)); - TClientCommand::TConfig config(argc, argv); + TClientCommand::TConfig config(argc, argv); // TODO: process flags from environment KIKIMR_FLAGS before command line processing return commandsRoot->Process(config); } @@ -89,10 +89,10 @@ int NewClient(int argc, char** argv, std::shared_ptr<TModuleFactories> factories TString NewClientCommandsDescription(std::shared_ptr<TModuleFactories> factories) { THolder<TClientCommandRoot> commandsRoot = MakeHolder<TClientCommandRoot>(std::move(factories)); TStringStream stream; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - stream << " [options] <subcommand>" << Endl << Endl - << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; - commandsRoot->RenderCommandsDescription(stream, colors); + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << " [options] <subcommand>" << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + commandsRoot->RenderCommandsDescription(stream, colors); return stream.Str(); } diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 7cbe821ed0..6766dd3171 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -58,7 +58,7 @@ protected: ui64 TenantNetwork; TVector<TString> NodeBrokerAddresses; ui32 NodeBrokerPort; - bool NodeBrokerUseTls; + bool NodeBrokerUseTls; bool FixedNodeID; bool IgnoreCmsConfigs; bool HierarchicalCfg; @@ -111,7 +111,7 @@ protected: TenantMemory = 0; TenantNetwork = 0; NodeBrokerPort = 0; - NodeBrokerUseTls = false; + NodeBrokerUseTls = false; FixedNodeID = false; InterconnectPort = 0; SqsHttpPort = 0; @@ -150,8 +150,8 @@ protected: .RequiredArgument("ADDR").AppendTo(&NodeBrokerAddresses); config.Opts->AddLongOption("node-broker-port", "node broker port (hosts from naming file are used)") .RequiredArgument("PORT").StoreResult(&NodeBrokerPort); - config.Opts->AddLongOption("node-broker-use-tls", "use tls for node broker (hosts from naming file are used)") - .RequiredArgument("PORT").StoreResult(&NodeBrokerUseTls); + config.Opts->AddLongOption("node-broker-use-tls", "use tls for node broker (hosts from naming file are used)") + .RequiredArgument("PORT").StoreResult(&NodeBrokerUseTls); config.Opts->AddLongOption("node-address", "address for dynamic node") .RequiredArgument("ADDR").StoreResult(&NodeAddress); config.Opts->AddLongOption("node-host", "hostname for dynamic node") @@ -259,7 +259,7 @@ protected: config.Opts->AddLongOption("hierarchic-cfg", "Use hierarchical approach for configuration parts overriding") .NoArgument().SetFlag(&HierarchicalCfg); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); config.Opts->SetFreeArgDefaultTitle("PATH", "path to protobuf file; files are merged in order in which they are enlisted"); } @@ -681,7 +681,7 @@ protected: if (attempts > 0) Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000))); - NClient::TKikimr kikimr(GetKikimr(addr)); + NClient::TKikimr kikimr(GetKikimr(addr)); auto configurator = kikimr.GetNodeConfigurator(); Cout << "Trying to get configs from " << addr << Endl; @@ -841,7 +841,7 @@ protected: void MaybeRegisterAndLoadConfigs() { - // static node + // static node if (NodeBrokerAddresses.empty() && !NodeBrokerPort) { if (!NodeId) ythrow yexception() << "Either --node [NUM|'static'] or --node-broker[-port] should be specified"; @@ -862,8 +862,8 @@ protected: const TString &nodeHost, const TString &nodeAddress, const TString &nodeResolveHost, - const TMaybe<TString>& path) { - NClient::TKikimr kikimr(GetKikimr(addr)); + const TMaybe<TString>& path) { + NClient::TKikimr kikimr(GetKikimr(addr)); auto registrant = kikimr.GetNodeRegistrant(); NActorsInterconnect::TNodeLocation location; @@ -900,7 +900,7 @@ protected: } else { Y_VERIFY(NodeBrokerPort); for (auto &node : RunConfig.AppConfig.MutableNameserviceConfig()->GetNode()) { - addrs.emplace_back(TStringBuilder() << (NodeBrokerUseTls ? "grpcs://" : "") << node.GetHost() << ':' << NodeBrokerPort); + addrs.emplace_back(TStringBuilder() << (NodeBrokerUseTls ? "grpcs://" : "") << node.GetHost() << ':' << NodeBrokerPort); } } ShuffleRange(addrs); @@ -1017,7 +1017,7 @@ protected: } bool TryToLoadConfigForDynamicNodeFromCMS(const TString &addr, TString &error) { - NClient::TKikimr kikimr(GetKikimr(addr)); + NClient::TKikimr kikimr(GetKikimr(addr)); auto configurator = kikimr.GetNodeConfigurator(); Cout << "Trying to get configs from " << addr << Endl; @@ -1094,20 +1094,20 @@ protected: } } } - -private: - NClient::TKikimr GetKikimr(const TString& addr) { - TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(addr); + +private: + NClient::TKikimr GetKikimr(const TString& addr) { + TCommandConfig::TServerEndpoint endpoint = TCommandConfig::ParseServerAddress(addr); NGrpc::TGRpcClientConfig grpcConfig(endpoint.Address, TDuration::Seconds(5)); grpcConfig.LoadBalancingPolicy = "round_robin"; - if (endpoint.EnableSsl.Defined()) { - grpcConfig.EnableSsl = endpoint.EnableSsl.GetRef(); - if (PathToCA) { - grpcConfig.SslCaCert = ReadFromFile(PathToCA, "CA certificates"); - } - } - return NClient::TKikimr(grpcConfig); - } + if (endpoint.EnableSsl.Defined()) { + grpcConfig.EnableSsl = endpoint.EnableSsl.GetRef(); + if (PathToCA) { + grpcConfig.SslCaCert = ReadFromFile(PathToCA, "CA certificates"); + } + } + return NClient::TKikimr(grpcConfig); + } }; class TClientCommandServerConfig : public TClientCommandServerBase { diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp index b6e05b24e6..361e757ca1 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tablet.cpp @@ -4,10 +4,10 @@ namespace NKikimr { namespace NDriverClient { -class TClientCommandKeyValueRequest : public TClientCommandConfig { +class TClientCommandKeyValueRequest : public TClientCommandConfig { public: TClientCommandKeyValueRequest() - : TClientCommandConfig("request", { "req" }, "Request to KV tablet") + : TClientCommandConfig("request", { "req" }, "Request to KV tablet") {} TString ProtoBuf; @@ -47,8 +47,8 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<REQUEST>", "Request protobuf or file with request protobuf"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<REQUEST>", "Request protobuf or file with request protobuf"); config.Opts->AddLongOption("of", "output file path, protobuf must contain single read command!").Optional() .RequiredArgument("PATH").StoreResult(&OutputFile); config.Opts->AddLongOption("if", "input file path, protobuf must contain single write command!").Optional() @@ -81,10 +81,10 @@ public: } }; -class TClientCommandTabletExec : public TClientCommandConfig { +class TClientCommandTabletExec : public TClientCommandConfig { public: TClientCommandTabletExec() - : TClientCommandConfig("execute", { "exec" }) + : TClientCommandConfig("execute", { "exec" }) { } @@ -97,9 +97,9 @@ public: config.Opts->AddLongOption("follower", "connect to follower").NoArgument(); config.Opts->AddLongOption("json-ui64-as-string", "json output ui64 as string").NoArgument(); config.Opts->AddLongOption("json-binary-as-base64", "json output binary data in base64").NoArgument(); - config.SetFreeArgsNum(1, 2); - SetFreeArgTitle(0, "<PROGRAM>", "Program to execute"); - SetFreeArgTitle(1, "<PARAMS>", "Parameters of the program"); + config.SetFreeArgsNum(1, 2); + SetFreeArgTitle(0, "<PROGRAM>", "Program to execute"); + SetFreeArgTitle(1, "<PARAMS>", "Parameters of the program"); } virtual void Parse(TConfig& config) override { @@ -147,7 +147,7 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(0); + config.SetFreeArgsNum(0); } virtual void Parse(TConfig& config) override { @@ -175,8 +175,8 @@ public: TClientCommand::Config(config); config.Opts->AddLongOption("follower", "connect to follower"); config.Opts->AddLongOption("dry-run", "test changes without applying"); - config.SetFreeArgsNum(1, 1); - SetFreeArgTitle(0, "<SCHEME CHANGES>", "Scheme changes to apply"); + config.SetFreeArgsNum(1, 1); + SetFreeArgTitle(0, "<SCHEME CHANGES>", "Scheme changes to apply"); } virtual void Parse(TConfig& config) override { @@ -209,8 +209,8 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<NODE ID>", "Node ID to drain tablets from"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<NODE ID>", "Node ID to drain tablets from"); } ui32 NodeId; @@ -235,8 +235,8 @@ public: virtual void Config(TConfig& config) override { TClientCommand::Config(config); - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<NODE ID>", "Node ID to fill tablets to"); + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<NODE ID>", "Node ID to fill tablets to"); } ui32 NodeId; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp index 0704b15a8b..db44a3a516 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_tenant.cpp @@ -134,15 +134,15 @@ public: { TTenantClientCommand::Parse(config); - if (CommandConfig.ClientConfig.Defined()) { + if (CommandConfig.ClientConfig.Defined()) { auto *p = std::get_if<NGrpc::TGRpcClientConfig>(&CommandConfig.ClientConfig.GetRef()); if (p) { ClientConfig.Locator = p->Locator; ClientConfig.Timeout = p->Timeout; ClientConfig.MaxMessageSize = p->MaxMessageSize; ClientConfig.MaxInFlight = p->MaxInFlight; - ClientConfig.EnableSsl = p->EnableSsl; - ClientConfig.SslCaCert = p->SslCaCert; + ClientConfig.EnableSsl = p->EnableSsl; + ClientConfig.SslCaCert = p->SslCaCert; } } } @@ -331,7 +331,7 @@ public: .RequiredArgument("NAME=VALUE").AppendTo(&Attributes); config.Opts->AddLongOption("serverless", "Create a serverless database (free arg must specify shared database for resources)") .NoArgument().StoreTrue(&Serverless); - config.SetFreeArgsMin(1); + config.SetFreeArgsMin(1); config.Opts->SetFreeArgDefaultTitle("<pool type>:<pool size>", "Pairs describing storage pool type and size (number of groups)."); } @@ -425,7 +425,7 @@ public: void Config(TConfig& config) override { TTenantClientGRpcCommand::Config(config); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); config.Opts->SetFreeArgDefaultTitle("[[<availability zone>:]<unit kind>:]<units count>", "Triples describing units."); } @@ -460,7 +460,7 @@ public: void Config(TConfig& config) override { TTenantClientGRpcCommand::Config(config); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); config.Opts->SetFreeArgDefaultTitle("[[<availability zone>:]<unit kind>:]<units count>", "Triples describing units."); } @@ -495,7 +495,7 @@ public: void Config(TConfig& config) override { TTenantClientGRpcCommand::Config(config); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); config.Opts->SetFreeArgDefaultTitle("<host>:<port>:<kind>", "Triples describing registered units."); } @@ -536,7 +536,7 @@ public: void Config(TConfig& config) override { TTenantClientGRpcCommand::Config(config); - config.SetFreeArgsMin(0); + config.SetFreeArgsMin(0); config.Opts->SetFreeArgDefaultTitle("<host>:<port>", "Pairs describing deregistered units."); } @@ -575,7 +575,7 @@ public: void Config(TConfig& config) override { TTenantClientGRpcCommand::Config(config); - config.SetFreeArgsMin(1); + config.SetFreeArgsMin(1); config.Opts->SetFreeArgDefaultTitle("<pool kind>:<pool size>", "Pairs describing storage pool type and size (number of groups)."); } diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index ddf9dd8b37..faf1797413 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -51,7 +51,7 @@ union TBasicKikimrServicesMask { bool EnableSqs:1; bool EnableConfigsDispatcher:1; bool EnableSecurityServices:1; - bool EnableTabletInfo:1; + bool EnableTabletInfo:1; bool EnableQuoterService:1; bool EnablePersQueueClusterDiscovery:1; bool EnableNetClassifier:1; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index d3600e19eb..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2141,24 +2141,24 @@ void TConfigsCacheInitializer::InitializeServices(NActors::TActorSystemSetup* se } } -// TTabletInfoInitializer - -TTabletInfoInitializer::TTabletInfoInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TTabletInfoInitializer::InitializeServices( - NActors::TActorSystemSetup* setup, - const NKikimr::TAppData* appData) { - TActorSetupCmd tabletInfoSetup(NTabletInfo::CreateTabletInfo(), TMailboxType::ReadAsFilled, appData->UserPoolId); +// TTabletInfoInitializer + +TTabletInfoInitializer::TTabletInfoInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TTabletInfoInitializer::InitializeServices( + NActors::TActorSystemSetup* setup, + const NKikimr::TAppData* appData) { + TActorSetupCmd tabletInfoSetup(NTabletInfo::CreateTabletInfo(), TMailboxType::ReadAsFilled, appData->UserPoolId); setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(NTabletInfo::MakeTabletInfoID(), tabletInfoSetup)); -} - +} + TConfigValidatorsInitializer::TConfigValidatorsInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } - + void TConfigValidatorsInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { Y_UNUSED(setup); Y_UNUSED(appData); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 272b7442d7..407ce1bb7b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -437,17 +437,17 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TTabletInfoInitializer : public IKikimrServicesInitializer { -public: - TTabletInfoInitializer(const TKikimrRunConfig& runConfig); +class TTabletInfoInitializer : public IKikimrServicesInitializer { +public: + TTabletInfoInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - class TConfigValidatorsInitializer : public IKikimrServicesInitializer { public: TConfigValidatorsInitializer(const TKikimrRunConfig& runConfig); - + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index 3fe09138a0..f0a9315685 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -70,17 +70,17 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> configParser.SetupGlobalOpts(opts); NMsgBusProxy::TMsgBusClientConfig mbusConfig; mbusConfig.ConfigureLastGetopt(opts, "mb-"); - NDriverClient::HideOptions(opts); - opts.AddLongOption('s', "server", "Server address to connect (default $KIKIMR_SERVER)").RequiredArgument("ADDR[:NUM]"); - opts.AddLongOption('k', "token", "Security token").RequiredArgument("TOKEN"); - opts.AddLongOption('f', "token-file", "Security token file").RequiredArgument("PATH"); - opts.AddLongOption('d', "dump", "Dump requests to error log").NoArgument(); - opts.AddLongOption('t', "time", "Show request execution time").NoArgument(); - opts.AddLongOption('o', "progress", "Show progress of long requests").NoArgument(); - opts.AddLongOption(0, "allocator-info", "Print the name of allocator linked to the binary and exit") + NDriverClient::HideOptions(opts); + opts.AddLongOption('s', "server", "Server address to connect (default $KIKIMR_SERVER)").RequiredArgument("ADDR[:NUM]"); + opts.AddLongOption('k', "token", "Security token").RequiredArgument("TOKEN"); + opts.AddLongOption('f', "token-file", "Security token file").RequiredArgument("PATH"); + opts.AddLongOption('d', "dump", "Dump requests to error log").NoArgument(); + opts.AddLongOption('t', "time", "Show request execution time").NoArgument(); + opts.AddLongOption('o', "progress", "Show progress of long requests").NoArgument(); + opts.AddLongOption(0, "allocator-info", "Print the name of allocator linked to the binary and exit") .NoArgument().Handler(&PrintAllocatorInfoAndExit); - opts.SetFreeArgsMin(1); - opts.SetFreeArgTitle(0, "<command>", TDriverModeParser::CommandsCsv()); + opts.SetFreeArgsMin(1); + opts.SetFreeArgTitle(0, "<command>", TDriverModeParser::CommandsCsv()); opts.SetCmdLineDescr(NDriverClient::NewClientCommandsDescription(factories)); opts.AddHelpOption('h'); @@ -174,13 +174,13 @@ void SetupTerminateHandler() { int ParameterizedMain(int argc, char **argv, std::shared_ptr<NKikimr::TModuleFactories> factories) { try { return NKikimr::Main(argc, argv, std::move(factories)); - } - catch (const NYdb::NConsoleClient::TMissUseException& e) { - Cerr << e.what() << Endl; - Cerr << "Try \"--help\" option for more info." << Endl; - return 1; - } - catch (const yexception& e) { + } + catch (const NYdb::NConsoleClient::TMissUseException& e) { + Cerr << e.what() << Endl; + Cerr << "Try \"--help\" option for more info." << Endl; + return 1; + } + catch (const yexception& e) { Cerr << "Caught exception: " << e.what() << Endl; return 1; } diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 618e75ffc4..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -124,7 +124,7 @@ #include <util/system/hostname.h> #include <ydb/core/tracing/tablet_info.h> - + namespace NKikimr { class TDomainsInitializer : public IAppDataInitializer { @@ -919,9 +919,9 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize()) AppData->ResourceProfiles->LoadProfiles(runConfig.AppConfig.GetBootstrapConfig().GetResourceProfiles()); - if (runConfig.AppConfig.GetBootstrapConfig().HasEnableIntrospection()) - AppData->EnableIntrospection = runConfig.AppConfig.GetBootstrapConfig().GetEnableIntrospection(); - + if (runConfig.AppConfig.GetBootstrapConfig().HasEnableIntrospection()) + AppData->EnableIntrospection = runConfig.AppConfig.GetBootstrapConfig().GetEnableIntrospection(); + TAppDataInitializersList appDataInitializers; // setup domain info appDataInitializers.AddAppDataInitializer(new TDomainsInitializer(runConfig)); @@ -1288,10 +1288,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TConfigsCacheInitializer(runConfig)); } - if (serviceMask.EnableTabletInfo) { - sil->AddServiceInitializer(new TTabletInfoInitializer(runConfig)); - } - + if (serviceMask.EnableTabletInfo) { + sil->AddServiceInitializer(new TTabletInfoInitializer(runConfig)); + } + sil->AddServiceInitializer(new TLeaseHolderInitializer(runConfig)); sil->AddServiceInitializer(new TConfigValidatorsInitializer(runConfig)); diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index 41e9523584..36c35f32cc 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -88,9 +88,9 @@ bool TEngineHost::IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTi for (size_t i = 0; i < key.Columns.size(); i++) { const TKeyDesc::TColumnOp& cop = key.Columns[i]; - if (IsSystemColumn(cop.Column)) { - continue; - } + if (IsSystemColumn(cop.Column)) { + continue; + } auto* cinfo = Scheme.GetColumnInfo(tableInfo, cop.Column); EH_VALIDATE(cinfo, TypeCheckFailed); // Unknown column NScheme::TTypeId vtype = cinfo->PType; @@ -279,8 +279,8 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray ConvertKeys(tableInfo, row, key); TSmallVec<NTable::TTag> tags; - TSmallVec<NTable::TTag> systemColumnTags; - AnalyzeRowType(columnIds, tags, systemColumnTags); + TSmallVec<NTable::TTag> systemColumnTags; + AnalyzeRowType(columnIds, tags, systemColumnTags); TSmallVec<NScheme::TTypeId> cellTypes; cellTypes.reserve(tags.size()); @@ -296,7 +296,7 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray Settings.KeyAccessSampler->AddSample(tableId, row); NTable::TSelectStats stats; - ui64 flags = Settings.DisableByKeyFilter ? (ui64)NTable::NoByKey : 0; + ui64 flags = Settings.DisableByKeyFilter ? (ui64)NTable::NoByKey : 0; const auto ready = Db.Select(localTid, key, tags, dbRow, stats, flags, GetReadVersion(tableId)); Counters.InvisibleRowSkips += stats.Invisible; @@ -320,15 +320,15 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray rowItems[i] = GetCellValue(dbRow.Get(i), cellTypes[i]); rowBytes += dbRow.Get(i).IsNull() ? 1 : dbRow.Get(i).Size(); } - for (ui32 i = 0; i < systemColumnTags.size(); ++i) { - switch (systemColumnTags[i]) { - case TKeyDesc::EColumnIdDataShard: - rowItems[tags.size() + i] = NUdf::TUnboxedValuePod(GetShardId()); - break; - default: - ythrow yexception() << "Unknown system column tag: " << systemColumnTags[i]; - } - } + for (ui32 i = 0; i < systemColumnTags.size(); ++i) { + switch (systemColumnTags[i]) { + case TKeyDesc::EColumnIdDataShard: + rowItems[tags.size() + i] = NUdf::TUnboxedValuePod(GetShardId()); + break; + default: + ythrow yexception() << "Unknown system column tag: " << systemColumnTags[i]; + } + } rowBytes = std::max(rowBytes, (ui64)8); Counters.SelectRowBytes += rowBytes; @@ -361,16 +361,16 @@ public: void operator delete[](void *mem, std::size_t sz) = delete; - static NUdf::TUnboxedValue Create(const TDbTupleRef& dbData, const THolderFactory& holderFactory, - const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) { - ui32 size = dbData.ColumnCount + systemColumnTags.size(); + static NUdf::TUnboxedValue Create(const TDbTupleRef& dbData, const THolderFactory& holderFactory, + const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) { + ui32 size = dbData.ColumnCount + systemColumnTags.size(); ui32 maskSize = size > 0 ? (size - 1) / 64 + 1 : 0; void* buffer = MKQLAllocWithSize(sizeof(TSelectRangeLazyRow) + size * sizeof(NUdf::TUnboxedValue) + maskSize * sizeof(ui64)); - return NUdf::TUnboxedValuePod(::new(buffer) TSelectRangeLazyRow(dbData, holderFactory, maskSize, systemColumnTags, shardId)); + return NUdf::TUnboxedValuePod(::new(buffer) TSelectRangeLazyRow(dbData, holderFactory, maskSize, systemColumnTags, shardId)); } void InvalidateDb() { @@ -390,14 +390,14 @@ public: } private: - TSelectRangeLazyRow(const TDbTupleRef& dbData, const THolderFactory& holderFactory, ui32 maskSize, - const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) + TSelectRangeLazyRow(const TDbTupleRef& dbData, const THolderFactory& holderFactory, ui32 maskSize, + const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) : TComputationValue<TSelectRangeLazyRow<TTableIt>>(&holderFactory.GetMemInfo()) , Iter() , DbData(dbData) , MaskSize(maskSize) - , SystemColumnTags(systemColumnTags) - , ShardId(shardId) + , SystemColumnTags(systemColumnTags) + , ShardId(shardId) { ClearMask(); for (ui32 i = 0; i < Size(); ++i) { @@ -415,7 +415,7 @@ private: } inline ui32 Size() const { - return DbData.ColumnCount + SystemColumnTags.size(); + return DbData.ColumnCount + SystemColumnTags.size(); } inline ui32 GetMaskSize() const { @@ -426,17 +426,17 @@ private: Y_VERIFY_DEBUG(MaskSize > 0); if (!TestMask(index)) { - if (index < DbData.ColumnCount) { - GetPtr()[index] = GetCellValue(DbData.Columns[index], DbData.Types[index]); - } else { - switch (SystemColumnTags[index - DbData.ColumnCount]) { - case TKeyDesc::EColumnIdDataShard: - GetPtr()[index] = NUdf::TUnboxedValuePod(ShardId); - break; - default: - throw TSchemeErrorTabletException(); - } - } + if (index < DbData.ColumnCount) { + GetPtr()[index] = GetCellValue(DbData.Columns[index], DbData.Types[index]); + } else { + switch (SystemColumnTags[index - DbData.ColumnCount]) { + case TKeyDesc::EColumnIdDataShard: + GetPtr()[index] = NUdf::TUnboxedValuePod(ShardId); + break; + default: + throw TSchemeErrorTabletException(); + } + } SetMask(index); } Y_VERIFY_DEBUG(TestMask(index)); @@ -460,8 +460,8 @@ private: THolder<TTableIt> Iter; TDbTupleRef DbData; ui32 MaskSize; - TSmallVec<NTable::TTag> SystemColumnTags; - ui64 ShardId; + TSmallVec<NTable::TTag> SystemColumnTags; + ui64 ShardId; }; class TSelectRangeLazyRowsList : public TCustomListValue { @@ -477,16 +477,16 @@ public: public: TIterator(TMemoryUsageInfo* memInfo, const TSelectRangeLazyRowsList& list, TAutoPtr<TTableIt>&& iter, - const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) + const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId) : TBase(memInfo) , List(list) , Iter(iter.Release()) , HasCurrent(false) , Iterations(0) , Items(0) - , Bytes(0) - , SystemColumnTags(systemColumnTags) - , ShardId(shardId) {} + , Bytes(0) + , SystemColumnTags(systemColumnTags) + , ShardId(shardId) {} bool Next(NUdf::TUnboxedValue& value) override { bool truncated = false; @@ -560,7 +560,7 @@ public: if (HasCurrent && CurrentRowValue.UniqueBoxed()) { CurrentRow()->Reuse(rowValues); } else { - CurrentRowValue = TSelectRangeLazyRow<TTableIt>::Create(rowValues, List.HolderFactory, SystemColumnTags, ShardId); + CurrentRowValue = TSelectRangeLazyRow<TTableIt>::Create(rowValues, List.HolderFactory, SystemColumnTags, ShardId); } value = CurrentRowValue; @@ -619,8 +619,8 @@ public: ui64 Items; ui64 Bytes; NUdf::TUnboxedValue CurrentRowValue; - TSmallVec<NTable::TTag> SystemColumnTags; - ui64 ShardId; + TSmallVec<NTable::TTag> SystemColumnTags; + ui64 ShardId; }; public: @@ -635,8 +635,8 @@ public: , TableId(tableId) , LocalTid(localTid) , Tags(tags) - , SystemColumnTags(systemColumnTags) - , ShardId(shardId) + , SystemColumnTags(systemColumnTags) + , ShardId(shardId) , SkipNullKeys(skipNullKeys) , ItemsLimit(itemsLimit) , BytesLimit(bytesLimit) @@ -663,15 +663,15 @@ public: if (Reverse) { auto read = Db.IterateRangeReverse(LocalTid, keyRange, Tags, EngineHost.GetReadVersion(TableId)); - return NUdf::TUnboxedValuePod( - new TIterator<NTable::TTableReverseIt>(GetMemInfo(), *this, std::move(read), SystemColumnTags, ShardId) - ); + return NUdf::TUnboxedValuePod( + new TIterator<NTable::TTableReverseIt>(GetMemInfo(), *this, std::move(read), SystemColumnTags, ShardId) + ); } else { auto read = Db.IterateRange(LocalTid, keyRange, Tags, EngineHost.GetReadVersion(TableId)); - return NUdf::TUnboxedValuePod( - new TIterator<NTable::TTableIt>(GetMemInfo(), *this, std::move(read), SystemColumnTags, ShardId) - ); + return NUdf::TUnboxedValuePod( + new TIterator<NTable::TTableIt>(GetMemInfo(), *this, std::move(read), SystemColumnTags, ShardId) + ); } } @@ -711,8 +711,8 @@ private: TTableId TableId; ui64 LocalTid; TSmallVec<NTable::TTag> Tags; - TSmallVec<NTable::TTag> SystemColumnTags; - ui64 ShardId; + TSmallVec<NTable::TTag> SystemColumnTags; + ui64 ShardId; TSmallVec<bool> SkipNullKeys; ui64 ItemsLimit; ui64 BytesLimit; @@ -817,8 +817,8 @@ NUdf::TUnboxedValue TEngineHost::SelectRange(const TTableId& tableId, const TTab "Unexpected type structure of returnType in TEngineHost::SelectRange()"); TSmallVec<NTable::TTag> tags; - TSmallVec<NTable::TTag> systemColumnTags; - AnalyzeRowType(columnIds, tags, systemColumnTags); + TSmallVec<NTable::TTag> systemColumnTags; + AnalyzeRowType(columnIds, tags, systemColumnTags); TSmallVec<bool> skipNullKeysFlags = CreateBoolVec(skipNullKeys); TSmallVec<bool> forbidNullArgsFrom = CreateBoolVec(forbidNullArgs.first); @@ -946,16 +946,16 @@ void TEngineHost::SetPeriodicCallback(TPeriodicCallback&& callback) { PeriodicCallback = std::move(callback); } -void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags) { +void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags) { // Find out tags that should be read in Select*() functions tags.reserve(columnIds->GetValuesCount()); for (ui32 i = 0; i < columnIds->GetValuesCount(); i++) { - NTable::TTag columnId = AS_VALUE(TDataLiteral, columnIds->GetValue(i))->AsValue().Get<ui32>(); - if (IsSystemColumn(columnId)) { - systemColumnTags.push_back(columnId); - } else { - tags.push_back(columnId); - } + NTable::TTag columnId = AS_VALUE(TDataLiteral, columnIds->GetValue(i))->AsValue().Get<ui32>(); + if (IsSystemColumn(columnId)) { + systemColumnTags.push_back(columnId); + } else { + tags.push_back(columnId); + } } } diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h index c9a3e5ca5d..012ee6891b 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.h +++ b/ydb/core/engine/minikql/minikql_engine_host.h @@ -163,12 +163,12 @@ public: } }; -void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags); +void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags); NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeId type); NUdf::TUnboxedValue CreateSelectRangeLazyRowsList(NTable::TDatabase& db, const NTable::TScheme& scheme, const THolderFactory& holderFactory, const TTableId& tableId, ui64 localTid, const TSmallVec<NTable::TTag>& tags, const TSmallVec<bool>& skipNullKeys, const TTableRange& range, ui64 itemsLimit, ui64 bytesLimit, - bool reverse, TEngineHostCounters& counters, const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId); + bool reverse, TEngineHostCounters& counters, const TSmallVec<NTable::TTag>& systemColumnTags, ui64 shardId); void ConvertTableKeys(const NTable::TScheme& scheme, const NTable::TScheme::TTableInfo* tableInfo, const TArrayRef<const TCell>& row, TSmallVec<TRawTypeValue>& key, ui64* keyDataBytes); diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index cb3fce3ad8..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -106,7 +106,7 @@ struct TRpcServices { EvExportToYt, EvDiscoverPQClusters, EvBulkUpsert, - EvWhoAmI, + EvWhoAmI, EvKikhouseDescribeTable, EvCreateRateLimiterResource, EvAlterRateLimiterResource, @@ -167,7 +167,7 @@ struct TRpcServices { EvDataStreamsRemoveTagsFromStream, EvDataStreamsSplitShard, EvDataStreamsStartStreamEncryption, - EvDataStreamsStopStreamEncryption, + EvDataStreamsStopStreamEncryption, EvStreamExecuteYqlScript, EvYandexQueryCreateQuery, EvYandexQueryListQueries, diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index d847bf8968..97315f6e9f 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -74,7 +74,7 @@ protected: void Handle(TEvBeginTransactionRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCommitTransactionRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvRollbackTransactionRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvListEndpointsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvListEndpointsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDescribeTenantOptionsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDescribeTableOptionsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCreateCoordinationNode::TPtr& ev, const TActorContext& ctx); @@ -102,7 +102,7 @@ protected: void Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvBulkUpsertRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvCreateRateLimiterResource::TPtr& ev, const TActorContext& ctx); void Handle(TEvAlterRateLimiterResource::TPtr& ev, const TActorContext& ctx); void Handle(TEvDropRateLimiterResource::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/grpc_services/rpc_alter_table.cpp b/ydb/core/grpc_services/rpc_alter_table.cpp index 392af448c1..d1f2755b66 100644 --- a/ydb/core/grpc_services/rpc_alter_table.cpp +++ b/ydb/core/grpc_services/rpc_alter_table.cpp @@ -3,7 +3,7 @@ #include "rpc_scheme_base.h" #include "rpc_common.h" #include "operation_helpers.h" -#include "table_settings.h" +#include "table_settings.h" #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/cms/console/configs_dispatcher.h> @@ -25,7 +25,7 @@ namespace NKikimr { namespace NGRpcService { using namespace NActors; -using namespace NConsole; +using namespace NConsole; using namespace Ydb; static bool CheckAccess(const NACLib::TUserToken& userToken, const NSchemeCache::TSchemeCacheNavigate* navigate) { @@ -106,10 +106,10 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab if (req->add_columns_size() || req->drop_columns_size() || req->alter_columns_size() || req->ttl_action_case() != Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET || req->has_alter_storage_settings() - || req->add_column_families_size() || req->alter_column_families_size() - || req->set_compaction_policy() || req->has_alter_partitioning_settings() - || req->set_key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED - || req->has_set_read_replicas_settings()) { + || req->add_column_families_size() || req->alter_column_families_size() + || req->set_compaction_policy() || req->has_alter_partitioning_settings() + || req->set_key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED + || req->has_set_read_replicas_settings()) { ops.emplace(EOp::Common); } @@ -152,11 +152,11 @@ public: switch (*ops.begin()) { case EOp::Common: - // Altering table settings will need table profiles - SendConfigRequest(ctx); - ctx.Schedule(TDuration::Seconds(15), new TEvents::TEvWakeup(WakeupTagGetConfig)); - Become(&TAlterTableRPC::AlterStateGetConfig); - return; + // Altering table settings will need table profiles + SendConfigRequest(ctx); + ctx.Schedule(TDuration::Seconds(15), new TEvents::TEvWakeup(WakeupTagGetConfig)); + Become(&TAlterTableRPC::AlterStateGetConfig); + return; case EOp::AddIndex: if (req->add_indexes_size() == 1) { @@ -200,52 +200,52 @@ private: } } - void AlterStateGetConfig(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvConfigsDispatcher::TEvGetConfigResponse, Handle); - HFunc(TEvents::TEvUndelivered, Handle); - HFunc(TEvents::TEvWakeup, HandleWakeup); - default: TBase::StateFuncBase(ev, ctx); - } - } - - void Handle(TEvents::TEvUndelivered::TPtr &/*ev*/, const TActorContext &ctx) - { - LOG_CRIT_S(ctx, NKikimrServices::GRPC_PROXY, - "TAlterTableRPC: cannot deliver config request to Configs Dispatcher" - " (empty default profile is available only)"); - AlterTable(ctx); - Become(&TAlterTableRPC::AlterStateWork); - } - - void Handle(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr &ev, const TActorContext &ctx) { - auto &config = ev->Get()->Config->GetTableProfilesConfig(); - Profiles.Load(config); - - AlterTable(ctx); - Become(&TAlterTableRPC::AlterStateWork); - } - - void HandleWakeup(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { - switch (ev->Get()->Tag) { - case WakeupTagGetConfig: { - LOG_CRIT_S(ctx, NKikimrServices::GRPC_PROXY, "TAlterTableRPC: cannot get table profiles (timeout)"); - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue("Tables profiles config not available.")); - return Reply(StatusIds::UNAVAILABLE, issues, ctx); - } - default: - TBase::HandleWakeup(ev, ctx); - } - } - - void SendConfigRequest(const TActorContext &ctx) { - ui32 configKind = (ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem; - ctx.Send(MakeConfigsDispatcherID(ctx.SelfID.NodeId()), - new TEvConfigsDispatcher::TEvGetConfigRequest(configKind), - IEventHandle::FlagTrackDelivery); - } - + void AlterStateGetConfig(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvConfigsDispatcher::TEvGetConfigResponse, Handle); + HFunc(TEvents::TEvUndelivered, Handle); + HFunc(TEvents::TEvWakeup, HandleWakeup); + default: TBase::StateFuncBase(ev, ctx); + } + } + + void Handle(TEvents::TEvUndelivered::TPtr &/*ev*/, const TActorContext &ctx) + { + LOG_CRIT_S(ctx, NKikimrServices::GRPC_PROXY, + "TAlterTableRPC: cannot deliver config request to Configs Dispatcher" + " (empty default profile is available only)"); + AlterTable(ctx); + Become(&TAlterTableRPC::AlterStateWork); + } + + void Handle(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr &ev, const TActorContext &ctx) { + auto &config = ev->Get()->Config->GetTableProfilesConfig(); + Profiles.Load(config); + + AlterTable(ctx); + Become(&TAlterTableRPC::AlterStateWork); + } + + void HandleWakeup(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { + switch (ev->Get()->Tag) { + case WakeupTagGetConfig: { + LOG_CRIT_S(ctx, NKikimrServices::GRPC_PROXY, "TAlterTableRPC: cannot get table profiles (timeout)"); + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue("Tables profiles config not available.")); + return Reply(StatusIds::UNAVAILABLE, issues, ctx); + } + default: + TBase::HandleWakeup(ev, ctx); + } + } + + void SendConfigRequest(const TActorContext &ctx) { + ui32 configKind = (ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem; + ctx.Send(MakeConfigsDispatcherID(ctx.SelfID.NodeId()), + new TEvConfigsDispatcher::TEvGetConfigRequest(configKind), + IEventHandle::FlagTrackDelivery); + } + void PrepareAlterTableAddIndex() { using namespace NTxProxy; LogPrefix = TStringBuilder() << "[AlterTableAddIndexOp " << SelfId() << "] "; @@ -479,11 +479,11 @@ private: } if (!FillAlterTableSettingsDesc(*desc, *req, Profiles, code, error, AppData())) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); - } - + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } + ctx.Send(MakeTxProxyID(), proposeRequest.release()); } @@ -534,7 +534,7 @@ private: TString LogPrefix; TActorId SSPipeClient; THolder<const NACLib::TUserToken> UserToken; - TTableProfiles Profiles; + TTableProfiles Profiles; }; void TGRpcRequestProxy::Handle(TEvAlterTableRequest::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/grpc_services/rpc_create_table.cpp b/ydb/core/grpc_services/rpc_create_table.cpp index 2808f90cb6..302dfde57a 100644 --- a/ydb/core/grpc_services/rpc_create_table.cpp +++ b/ydb/core/grpc_services/rpc_create_table.cpp @@ -4,7 +4,7 @@ #include "rpc_scheme_base.h" #include "rpc_common.h" #include "table_profiles.h" -#include "table_settings.h" +#include "table_settings.h" #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/engine/mkql_proto.h> @@ -89,15 +89,15 @@ private: IEventHandle::FlagTrackDelivery); } - // Mutually exclusive settings - void MEWarning(const TString& settingName) { - Request_->RaiseIssue( - NYql::TIssue(TStringBuilder() << "Table profile and " << settingName - << " are set. They are mutually exclusive. Use either one of them.") - .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) - ); - } - + // Mutually exclusive settings + void MEWarning(const TString& settingName) { + Request_->RaiseIssue( + NYql::TIssue(TStringBuilder() << "Table profile and " << settingName + << " are set. They are mutually exclusive. Use either one of them.") + .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) + ); + } + void SendProposeRequest(const TActorContext &ctx) { const auto req = GetProtoRequest(); std::pair<TString, TString> pathPair; @@ -154,13 +154,13 @@ private: return Reply(code, issues, ctx); } - bool tableProfileSet = false; - if (req->has_profile()) { - const auto& profile = req->profile(); - tableProfileSet = profile.preset_name() || profile.has_compaction_policy() || profile.has_execution_policy() - || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() - || profile.has_caching_policy(); - } + bool tableProfileSet = false; + if (req->has_profile()) { + const auto& profile = req->profile(); + tableProfileSet = profile.preset_name() || profile.has_compaction_policy() || profile.has_execution_policy() + || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() + || profile.has_caching_policy(); + } if (!Profiles.ApplyTableProfile(req->profile(), *tableDesc, code, error)) { NYql::TIssues issues; @@ -172,9 +172,9 @@ private: // Apply storage settings to the default column family if (req->has_storage_settings()) { - if (tableProfileSet) { - MEWarning("StorageSettings"); - } + if (tableProfileSet) { + MEWarning("StorageSettings"); + } if (!families.ApplyStorageSettings(req->storage_settings(), &code, &error)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); @@ -182,9 +182,9 @@ private: } } - if (tableProfileSet && req->column_familiesSize()) { - MEWarning("ColumnFamilies"); - } + if (tableProfileSet && req->column_familiesSize()) { + MEWarning("ColumnFamilies"); + } for (const auto& familySettings : req->column_families()) { if (!families.ApplyFamilySettings(familySettings, &code, &error)) { NYql::TIssues issues; @@ -206,19 +206,19 @@ private: attr.SetValue(value); } - TList<TString> warnings; + TList<TString> warnings; if (!FillCreateTableSettingsDesc(*tableDesc, *req, Profiles, code, error, warnings)) { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue(error)); - return Reply(code, issues, ctx); - } - for (const auto& warning : warnings) { - Request_->RaiseIssue( - NYql::TIssue(warning) - .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) - ); - } - + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } + for (const auto& warning : warnings) { + Request_->RaiseIssue( + NYql::TIssue(warning) + .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) + ); + } + ctx.Send(MakeTxProxyID(), proposeRequest.release()); } diff --git a/ydb/core/grpc_services/rpc_deferrable.h b/ydb/core/grpc_services/rpc_deferrable.h index b55f7aca69..644e7b84c4 100644 --- a/ydb/core/grpc_services/rpc_deferrable.h +++ b/ydb/core/grpc_services/rpc_deferrable.h @@ -19,11 +19,11 @@ namespace NKikimr { namespace NGRpcService { -template <typename TDerived, typename TRequest, bool IsOperation> -class TRpcRequestWithOperationParamsActor : public TActorBootstrapped<TDerived> { +template <typename TDerived, typename TRequest, bool IsOperation> +class TRpcRequestWithOperationParamsActor : public TActorBootstrapped<TDerived> { private: typedef TActorBootstrapped<TDerived> TBase; - typedef typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type TRequestBase; + typedef typename std::conditional<IsOperation, IRequestOpCtx, IRequestNoOpCtx>::type TRequestBase; public: enum EWakeupTag { @@ -34,7 +34,7 @@ public: }; public: - TRpcRequestWithOperationParamsActor(TRequestBase* request) + TRpcRequestWithOperationParamsActor(TRequestBase* request) : Request_(request) { auto& operationParams = GetProtoRequest()->operation_params(); @@ -43,14 +43,14 @@ public: ReportCostInfo_ = operationParams.report_cost_info() == Ydb::FeatureFlag::ENABLED; } - const typename TRequest::TRequest* GetProtoRequest() const { - return TRequest::GetProtoRequest(Request_); + const typename TRequest::TRequest* GetProtoRequest() const { + return TRequest::GetProtoRequest(Request_); + } + + Ydb::Operations::OperationParams::OperationMode GetOperationMode() const { + return GetProtoRequest()->operation_params().operation_mode(); } - Ydb::Operations::OperationParams::OperationMode GetOperationMode() const { - return GetProtoRequest()->operation_params().operation_mode(); - } - void Bootstrap(const TActorContext &ctx) { HasCancel_ = static_cast<TDerived*>(this)->HasCancelOperation(); @@ -75,68 +75,68 @@ public: Request_->SetClientLostAction(std::move(clientLostCb)); } - bool HasCancelOperation() { - return false; - } - - TRequestBase& Request() const { + bool HasCancelOperation() { + return false; + } + + TRequestBase& Request() const { return *Request_; } -protected: - TDuration GetOperationTimeout() { - return OperationTimeout_; +protected: + TDuration GetOperationTimeout() { + return OperationTimeout_; } - TDuration GetCancelAfter() { - return CancelAfter_; - } - - void DestroyTimers() { - auto& ctx = TlsActivationContext->AsActorContext(); - if (OperationTimeoutTimer) { - ctx.Send(OperationTimeoutTimer, new TEvents::TEvPoisonPill); - } - if (CancelAfterTimer) { - ctx.Send(CancelAfterTimer, new TEvents::TEvPoisonPill); - } - } - - void PassAway() override { - DestroyTimers(); - TBase::PassAway(); - } - - TRequest* RequestPtr() { - return static_cast<TRequest*>(Request_.get()); - } - -protected: - std::unique_ptr<TRequestBase> Request_; + TDuration GetCancelAfter() { + return CancelAfter_; + } + + void DestroyTimers() { + auto& ctx = TlsActivationContext->AsActorContext(); + if (OperationTimeoutTimer) { + ctx.Send(OperationTimeoutTimer, new TEvents::TEvPoisonPill); + } + if (CancelAfterTimer) { + ctx.Send(CancelAfterTimer, new TEvents::TEvPoisonPill); + } + } + + void PassAway() override { + DestroyTimers(); + TBase::PassAway(); + } + + TRequest* RequestPtr() { + return static_cast<TRequest*>(Request_.get()); + } + +protected: + std::unique_ptr<TRequestBase> Request_; TActorId OperationTimeoutTimer; TActorId CancelAfterTimer; - TDuration OperationTimeout_; - TDuration CancelAfter_; - bool HasCancel_ = false; + TDuration OperationTimeout_; + TDuration CancelAfter_; + bool HasCancel_ = false; bool ReportCostInfo_ = false; -}; - -template <typename TDerived, typename TRequest> -class TRpcOperationRequestActor : public TRpcRequestWithOperationParamsActor<TDerived, TRequest, true> { -private: - typedef TRpcRequestWithOperationParamsActor<TDerived, TRequest, true> TBase; - -public: - - TRpcOperationRequestActor(IRequestOpCtx* request) - : TBase(request) - {} - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::DEFERRABLE_RPC; - } - +}; + +template <typename TDerived, typename TRequest> +class TRpcOperationRequestActor : public TRpcRequestWithOperationParamsActor<TDerived, TRequest, true> { +private: + typedef TRpcRequestWithOperationParamsActor<TDerived, TRequest, true> TBase; + +public: + + TRpcOperationRequestActor(IRequestOpCtx* request) + : TBase(request) + {} + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::DEFERRABLE_RPC; + } + void OnCancelOperation(const TActorContext& ctx) { Y_UNUSED(ctx); } @@ -173,7 +173,7 @@ protected: } protected: - using TBase::Request_; + using TBase::Request_; void Reply(Ydb::StatusIds::StatusCode status, const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx) @@ -240,10 +240,10 @@ protected: protected: void HandleWakeup(TEvents::TEvWakeup::TPtr &ev, const TActorContext &ctx) { switch (ev->Get()->Tag) { - case TBase::WakeupTagTimeout: + case TBase::WakeupTagTimeout: static_cast<TDerived*>(this)->OnOperationTimeout(ctx); break; - case TBase::WakeupTagCancel: + case TBase::WakeupTagCancel: static_cast<TDerived*>(this)->OnCancelOperation(ctx); break; default: diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp index da8129b695..261f10d467 100644 --- a/ydb/core/grpc_services/rpc_describe_table.cpp +++ b/ydb/core/grpc_services/rpc_describe_table.cpp @@ -83,9 +83,9 @@ private: FillStorageSettings(describeTableResult, tableDescription); FillColumnFamilies(describeTableResult, tableDescription); FillAttributes(describeTableResult, pathDescription); - FillPartitioningSettings(describeTableResult, tableDescription); - FillKeyBloomFilter(describeTableResult, tableDescription); - FillReadReplicasSettings(describeTableResult, tableDescription); + FillPartitioningSettings(describeTableResult, tableDescription); + FillKeyBloomFilter(describeTableResult, tableDescription); + FillReadReplicasSettings(describeTableResult, tableDescription); return ReplyWithResult(Ydb::StatusIds::SUCCESS, describeTableResult, ctx); } diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp index 004e41c45a..317a3488ce 100644 --- a/ydb/core/grpc_services/rpc_execute_data_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp @@ -216,7 +216,7 @@ public: const auto& issueMessage = kqpResponse.GetQueryIssues(); auto queryResult = TEvExecuteDataQueryRequest::AllocateResult<Ydb::Table::ExecuteQueryResult>(Request_); - ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); + ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); ConvertQueryStats(kqpResponse, queryResult); if (kqpResponse.HasTxMeta()) { queryResult->mutable_tx_meta()->CopyFrom(kqpResponse.GetTxMeta()); diff --git a/ydb/core/grpc_services/rpc_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_execute_yql_script.cpp index ae3750dacd..2650d32df5 100644 --- a/ydb/core/grpc_services/rpc_execute_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_execute_yql_script.cpp @@ -97,7 +97,7 @@ public: const auto& issueMessage = kqpResponse.GetQueryIssues(); auto queryResult = TEvExecuteYqlScriptRequest::AllocateResult<TResult>(Request_); - ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); + ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); if (kqpResponse.HasQueryStats()) { FillQueryStats(*queryResult->mutable_query_stats(), kqpResponse); diff --git a/ydb/core/grpc_services/rpc_explain_yql_script.cpp b/ydb/core/grpc_services/rpc_explain_yql_script.cpp index c88edc6d37..1d4667feec 100644 --- a/ydb/core/grpc_services/rpc_explain_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_explain_yql_script.cpp @@ -57,10 +57,10 @@ public: } switch (req->mode()) { - // KIKIMR-10990 - //case Ydb::Scripting::ExplainYqlRequest_Mode_PARSE: - // ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_PARSE); - // break; + // KIKIMR-10990 + //case Ydb::Scripting::ExplainYqlRequest_Mode_PARSE: + // ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_PARSE); + // break; case Ydb::Scripting::ExplainYqlRequest_Mode_VALIDATE: ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_VALIDATE); break; @@ -72,7 +72,7 @@ public: << "Unknown explain mode")); return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); } - + ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT); ev->Record.MutableRequest()->SetQuery(script); ev->Record.MutableRequest()->SetKeepSession(false); @@ -87,15 +87,15 @@ public: if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { const auto& kqpResponse = record.GetResponse(); const auto& issueMessage = kqpResponse.GetQueryIssues(); - const auto& queryParameters = kqpResponse.GetQueryParameters(); + const auto& queryParameters = kqpResponse.GetQueryParameters(); Ydb::Scripting::ExplainYqlResult queryResult; queryResult.set_plan(kqpResponse.GetQueryPlan()); - for (const auto& queryParameter : queryParameters) { - Ydb::Type parameterType; - ConvertMiniKQLTypeToYdbType(queryParameter.GetType(), parameterType); - queryResult.mutable_parameters_types()->insert({ queryParameter.GetName(), parameterType }); - } + for (const auto& queryParameter : queryParameters) { + Ydb::Type parameterType; + ConvertMiniKQLTypeToYdbType(queryParameter.GetType(), parameterType); + queryResult.mutable_parameters_types()->insert({ queryParameter.GetName(), parameterType }); + } ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, queryResult, ctx); } else { diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h index 0fb4fa48ce..2811a49e78 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.h +++ b/ydb/core/grpc_services/rpc_kqp_base.h @@ -72,44 +72,44 @@ inline bool CheckQuery(const TString& query, NYql::TIssues& issues) { void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse); -inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, Ydb::ResultSet* to) { - const auto& type = from.GetType(); - TStackVec<NKikimrMiniKQL::TType> columnTypes; - Y_ENSURE(type.GetKind() == NKikimrMiniKQL::ETypeKind::Struct); - for (const auto& member : type.GetStruct().GetMember()) { - if (member.GetType().GetKind() == NKikimrMiniKQL::ETypeKind::List) { - for (const auto& column : member.GetType().GetList().GetItem().GetStruct().GetMember()) { - auto columnMeta = to->add_columns(); - columnMeta->set_name(column.GetName()); - columnTypes.push_back(column.GetType()); - ConvertMiniKQLTypeToYdbType(column.GetType(), *columnMeta->mutable_type()); +inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, Ydb::ResultSet* to) { + const auto& type = from.GetType(); + TStackVec<NKikimrMiniKQL::TType> columnTypes; + Y_ENSURE(type.GetKind() == NKikimrMiniKQL::ETypeKind::Struct); + for (const auto& member : type.GetStruct().GetMember()) { + if (member.GetType().GetKind() == NKikimrMiniKQL::ETypeKind::List) { + for (const auto& column : member.GetType().GetList().GetItem().GetStruct().GetMember()) { + auto columnMeta = to->add_columns(); + columnMeta->set_name(column.GetName()); + columnTypes.push_back(column.GetType()); + ConvertMiniKQLTypeToYdbType(column.GetType(), *columnMeta->mutable_type()); } } - } - for (const auto& responseStruct : from.GetValue().GetStruct()) { - for (const auto& row : responseStruct.GetList()) { - auto newRow = to->add_rows(); - ui32 columnCount = static_cast<ui32>(row.StructSize()); - Y_ENSURE(columnCount == columnTypes.size()); - for (ui32 i = 0; i < columnCount; i++) { - const auto& column = row.GetStruct(i); - ConvertMiniKQLValueToYdbValue(columnTypes[i], column, *newRow->add_items()); + } + for (const auto& responseStruct : from.GetValue().GetStruct()) { + for (const auto& row : responseStruct.GetList()) { + auto newRow = to->add_rows(); + ui32 columnCount = static_cast<ui32>(row.StructSize()); + Y_ENSURE(columnCount == columnTypes.size()); + for (ui32 i = 0; i < columnCount; i++) { + const auto& column = row.GetStruct(i); + ConvertMiniKQLValueToYdbValue(columnTypes[i], column, *newRow->add_items()); } } - if (responseStruct.Getvalue_valueCase() == NKikimrMiniKQL::TValue::kBool) { - to->set_truncated(responseStruct.GetBool()); - } + if (responseStruct.Getvalue_valueCase() == NKikimrMiniKQL::TValue::kBool) { + to->set_truncated(responseStruct.GetBool()); + } + } +} + +template<typename TFrom, typename TTo> +inline void ConvertKqpQueryResultsToDbResult(const TFrom& from, TTo* to) { + const auto& results = from.GetResults(); + for (const auto& result : results) { + ConvertKqpQueryResultToDbResult(result, to->add_result_sets()); } } -template<typename TFrom, typename TTo> -inline void ConvertKqpQueryResultsToDbResult(const TFrom& from, TTo* to) { - const auto& results = from.GetResults(); - for (const auto& result : results) { - ConvertKqpQueryResultToDbResult(result, to->add_result_sets()); - } -} - template <typename TDerived, typename TRequest> class TRpcKqpRequestActor : public TRpcOperationRequestActor<TDerived, TRequest> { using TBase = TRpcOperationRequestActor<TDerived, TRequest>; diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp index 09d321a3c6..5f95f263c6 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp @@ -515,7 +515,7 @@ private: LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Client lost, send abort event to executer " << ExecuterActorId_); if (ExecuterActorId_) { - auto abortEv = TEvKqp::TEvAbortExecution::Aborted("Client lost"); // any status code can be here + auto abortEv = TEvKqp::TEvAbortExecution::Aborted("Client lost"); // any status code can be here ctx.Send(ExecuterActorId_, abortEv.Release()); } diff --git a/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp index e0da1aa816..aa936e7b6e 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_yql_script.cpp @@ -1,6 +1,6 @@ #include "service_yql_scripting.h" -#include "rpc_kqp_base.h" - +#include "rpc_kqp_base.h" + #include <ydb/public/api/protos/ydb_scripting.pb.h> #include <ydb/core/actorlib_impl/long_timer.h> @@ -8,507 +8,507 @@ #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/kqp/executer/kqp_executer.h> #include <ydb/core/kqp/prepare/kqp_query_plan.h> - + #include <ydb/core/protos/services.pb.h> #include <ydb/core/ydb_convert/ydb_convert.h> - - -namespace NKikimr { -namespace NGRpcService { - + + +namespace NKikimr { +namespace NGRpcService { + using TEvStreamExecuteYqlScriptRequest = TGrpcRequestNoOperationCall<Ydb::Scripting::ExecuteYqlRequest, Ydb::Scripting::ExecuteYqlPartialResponse>; -namespace { - -using namespace NActors; -using namespace Ydb; - -namespace { - struct TParseRequestError { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - - TParseRequestError() - : Status(Ydb::StatusIds::INTERNAL_ERROR) - , Issues({ MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, - "Unexpected error while parsing request.") }) {} - - TParseRequestError(const Ydb::StatusIds::StatusCode& status, const NYql::TIssues& issues) - : Status(status) - , Issues(issues) {} - }; - - // Stores ResultSets from data query response until they are all sent to client one by one - struct TDataQueryStreamContext { - TDataQueryStreamContext(NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr& handle) - : Handle(handle.Release()) - , ResultIterator(Handle->Get()->Record.GetResults().begin()) - {} - - NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr Handle; - google::protobuf::RepeatedPtrField<NKikimrMiniKQL::TResult>::const_iterator ResultIterator; - }; - - enum EStreamRpcWakeupTag : ui64 { - ClientLostTag = 1, - ClientTimeoutTag = 2 - }; - - bool FillKqpParameters(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, - NKikimrMiniKQL::TParams& output, TParseRequestError& error) - { - if (input.size() != 0) { - try { - ConvertYdbParamsToMiniKQLParams(input, output); - } - catch (const std::exception& ex) { - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Failed to parse query parameters."); - issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(NYql::ExceptionToIssue(ex))); - - error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, { issue }); - return false; - } - } - - return true; - } - - bool FillKqpRequest(const Ydb::Scripting::ExecuteYqlRequest& req, NKikimrKqp::TEvQueryRequest& kqpRequest, - TParseRequestError& error) - { - if (!FillKqpParameters(req.parameters(), *kqpRequest.MutableRequest()->MutableParameters(), error)) { - return false; - } - - auto& script = req.script(); - NYql::TIssues issues; - if (!CheckQuery(script, issues)) { - error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues); - return false; - } - - kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING); - kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats())); - kqpRequest.MutableRequest()->SetKeepSession(false); - kqpRequest.MutableRequest()->SetQuery(script); - - return true; - } -} - -class TStreamExecuteYqlScriptRPC - : public TRpcRequestWithOperationParamsActor<TStreamExecuteYqlScriptRPC, TEvStreamExecuteYqlScriptRequest, false> { - -private: - typedef TRpcRequestWithOperationParamsActor<TStreamExecuteYqlScriptRPC, TEvStreamExecuteYqlScriptRequest, false> TBase; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::GRPC_STREAM_REQ; - } - +namespace { + +using namespace NActors; +using namespace Ydb; + +namespace { + struct TParseRequestError { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + + TParseRequestError() + : Status(Ydb::StatusIds::INTERNAL_ERROR) + , Issues({ MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, + "Unexpected error while parsing request.") }) {} + + TParseRequestError(const Ydb::StatusIds::StatusCode& status, const NYql::TIssues& issues) + : Status(status) + , Issues(issues) {} + }; + + // Stores ResultSets from data query response until they are all sent to client one by one + struct TDataQueryStreamContext { + TDataQueryStreamContext(NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr& handle) + : Handle(handle.Release()) + , ResultIterator(Handle->Get()->Record.GetResults().begin()) + {} + + NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr Handle; + google::protobuf::RepeatedPtrField<NKikimrMiniKQL::TResult>::const_iterator ResultIterator; + }; + + enum EStreamRpcWakeupTag : ui64 { + ClientLostTag = 1, + ClientTimeoutTag = 2 + }; + + bool FillKqpParameters(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input, + NKikimrMiniKQL::TParams& output, TParseRequestError& error) + { + if (input.size() != 0) { + try { + ConvertYdbParamsToMiniKQLParams(input, output); + } + catch (const std::exception& ex) { + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Failed to parse query parameters."); + issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(NYql::ExceptionToIssue(ex))); + + error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, { issue }); + return false; + } + } + + return true; + } + + bool FillKqpRequest(const Ydb::Scripting::ExecuteYqlRequest& req, NKikimrKqp::TEvQueryRequest& kqpRequest, + TParseRequestError& error) + { + if (!FillKqpParameters(req.parameters(), *kqpRequest.MutableRequest()->MutableParameters(), error)) { + return false; + } + + auto& script = req.script(); + NYql::TIssues issues; + if (!CheckQuery(script, issues)) { + error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues); + return false; + } + + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING); + kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats())); + kqpRequest.MutableRequest()->SetKeepSession(false); + kqpRequest.MutableRequest()->SetQuery(script); + + return true; + } +} + +class TStreamExecuteYqlScriptRPC + : public TRpcRequestWithOperationParamsActor<TStreamExecuteYqlScriptRPC, TEvStreamExecuteYqlScriptRequest, false> { + +private: + typedef TRpcRequestWithOperationParamsActor<TStreamExecuteYqlScriptRPC, TEvStreamExecuteYqlScriptRequest, false> TBase; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::GRPC_STREAM_REQ; + } + TStreamExecuteYqlScriptRPC(IRequestNoOpCtx* request, ui64 rpcBufferSize) - : TBase(request) - , RpcBufferSize_(rpcBufferSize) - {} - - using TBase::Request_; - - void Bootstrap(const TActorContext &ctx) { - this->Become(&TStreamExecuteYqlScriptRPC::StateWork); - TBase::Bootstrap(ctx); - - const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); - - InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout()); - if (InactiveClientTimeout_) { - SetClientTimeoutTimer(InactiveClientTimeout_, ctx); - } - - LastDataStreamTimestamp_ = TAppData::TimeProvider->Now(); - - auto selfId = this->SelfId(); - auto as = TActivationContext::ActorSystem(); - - RequestPtr()->SetClientLostAction([selfId, as]() { - as->Send(selfId, new TEvents::TEvWakeup(EStreamRpcWakeupTag::ClientLostTag)); - }); - - RequestPtr()->SetStreamingNotify([selfId, as](size_t left) { - as->Send(selfId, new TRpcServices::TEvGrpcNextReply(left)); - }); - - Proceed(ctx); - } - -private: - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvWakeup, Handle); - HFunc(NKqp::TEvKqp::TEvDataQueryStreamPart, Handle); - HFunc(TRpcServices::TEvGrpcNextReply, Handle); - HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); - HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle); - HFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle); - default: { - return ReplyFinishStream(TStringBuilder() - << "Unexpected event received in TStreamExecuteYqlScriptRPC::StateWork: " << ev->GetTypeRewrite(), ctx); - } - } - } - - void Proceed(const TActorContext &ctx) { - const auto& featureFlags = AppData(ctx)->FeatureFlags; - if (!featureFlags.GetAllowStreamExecuteYqlScript()) { - return ReplyFinishStream("StreamExecuteYqlScript request is not supported", ctx); - } - - const auto req = GetProtoRequest(); - const auto traceId = Request_->GetTraceId(); - - auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - - if (traceId) { - ev->Record.SetTraceId(traceId.GetRef()); - } - - ActorIdToProto(this->SelfId(), ev->Record.MutableRequestActorId()); - - TParseRequestError parseError; - if (!FillKqpRequest(*req, ev->Record, parseError)) { - return ReplyFinishStream(parseError.Status, parseError.Issues, ctx); - } - if (!ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release())) { - return ReplyFinishStream("Couldn't send request to KqpProxy", ctx); - } - } - - void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) { - switch ((EStreamRpcWakeupTag) ev->Get()->Tag) { - case EStreamRpcWakeupTag::ClientLostTag: - return HandleClientLost(ctx); - case EStreamRpcWakeupTag::ClientTimeoutTag: - return HandleClientTimeout(ctx); - default: - break; - } - switch ((TBase::EWakeupTag)ev->Get()->Tag) { - case TBase::WakeupTagTimeout: - return HandleOperationTimeout(ctx); - default: - break; - } - } - - void SendDataQueryResultPart() { - ++ResultsReceived_; - const auto& kqpResult = *DataQueryStreamContext->ResultIterator; - - Ydb::Scripting::ExecuteYqlPartialResponse response; - response.set_status(StatusIds::SUCCESS); - auto result = response.mutable_result(); - - ConvertKqpQueryResultToDbResult(kqpResult, result->mutable_result_set()); - result->set_result_set_index(ResultsReceived_ - 1); - - TString out; + : TBase(request) + , RpcBufferSize_(rpcBufferSize) + {} + + using TBase::Request_; + + void Bootstrap(const TActorContext &ctx) { + this->Become(&TStreamExecuteYqlScriptRPC::StateWork); + TBase::Bootstrap(ctx); + + const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig(); + + InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout()); + if (InactiveClientTimeout_) { + SetClientTimeoutTimer(InactiveClientTimeout_, ctx); + } + + LastDataStreamTimestamp_ = TAppData::TimeProvider->Now(); + + auto selfId = this->SelfId(); + auto as = TActivationContext::ActorSystem(); + + RequestPtr()->SetClientLostAction([selfId, as]() { + as->Send(selfId, new TEvents::TEvWakeup(EStreamRpcWakeupTag::ClientLostTag)); + }); + + RequestPtr()->SetStreamingNotify([selfId, as](size_t left) { + as->Send(selfId, new TRpcServices::TEvGrpcNextReply(left)); + }); + + Proceed(ctx); + } + +private: + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvWakeup, Handle); + HFunc(NKqp::TEvKqp::TEvDataQueryStreamPart, Handle); + HFunc(TRpcServices::TEvGrpcNextReply, Handle); + HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); + HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle); + HFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle); + default: { + return ReplyFinishStream(TStringBuilder() + << "Unexpected event received in TStreamExecuteYqlScriptRPC::StateWork: " << ev->GetTypeRewrite(), ctx); + } + } + } + + void Proceed(const TActorContext &ctx) { + const auto& featureFlags = AppData(ctx)->FeatureFlags; + if (!featureFlags.GetAllowStreamExecuteYqlScript()) { + return ReplyFinishStream("StreamExecuteYqlScript request is not supported", ctx); + } + + const auto req = GetProtoRequest(); + const auto traceId = Request_->GetTraceId(); + + auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + + if (traceId) { + ev->Record.SetTraceId(traceId.GetRef()); + } + + ActorIdToProto(this->SelfId(), ev->Record.MutableRequestActorId()); + + TParseRequestError parseError; + if (!FillKqpRequest(*req, ev->Record, parseError)) { + return ReplyFinishStream(parseError.Status, parseError.Issues, ctx); + } + if (!ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release())) { + return ReplyFinishStream("Couldn't send request to KqpProxy", ctx); + } + } + + void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) { + switch ((EStreamRpcWakeupTag) ev->Get()->Tag) { + case EStreamRpcWakeupTag::ClientLostTag: + return HandleClientLost(ctx); + case EStreamRpcWakeupTag::ClientTimeoutTag: + return HandleClientTimeout(ctx); + default: + break; + } + switch ((TBase::EWakeupTag)ev->Get()->Tag) { + case TBase::WakeupTagTimeout: + return HandleOperationTimeout(ctx); + default: + break; + } + } + + void SendDataQueryResultPart() { + ++ResultsReceived_; + const auto& kqpResult = *DataQueryStreamContext->ResultIterator; + + Ydb::Scripting::ExecuteYqlPartialResponse response; + response.set_status(StatusIds::SUCCESS); + auto result = response.mutable_result(); + + ConvertKqpQueryResultToDbResult(kqpResult, result->mutable_result_set()); + result->set_result_set_index(ResultsReceived_ - 1); + + TString out; Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); - - GRpcResponsesSizeQueue_.push(out.size()); - GRpcResponsesSize_ += out.size(); - - RequestPtr()->SendSerializedResult(std::move(out), StatusIds::SUCCESS); - } - - // From TKqpStreamRequestHandler - void Handle(NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr& ev, const TActorContext& ctx) { - GatewayRequestHandlerActorId_ = ActorIdFromProto(ev->Get()->Record.GetGatewayActorId()); - - if (!ev->Get()->Record.GetResults().size()) { - return ReplyFinishStream("Received TEvDataQueryStreamPart with no results", - ctx); - } - if (DataQueryStreamContext) { - return ReplyFinishStream("Received TEvDataQueryStreamPart event while previous data query is in progress", - ctx); - } - - DataQueryStreamContext = MakeHolder<TDataQueryStreamContext>(ev); - - SendDataQueryResultPart(); - } - - // From TKqpScanQueryStreamRequestHandler - void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) { - GatewayRequestHandlerActorId_ = ActorIdFromProto(ev->Get()->Record.GetExecuterActorId()); - ProcessingScanQuery_ = false; - LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " GatewayRequestHandlerActorId_: " << GatewayRequestHandlerActorId_); - } - - // From TKqpScanQueryStreamRequestHandler - void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) { - if (!GatewayRequestHandlerActorId_) { - return ReplyFinishStream("Received StreamData event from unknown executer", ctx); - } - if (!ProcessingScanQuery_) { - // First data part from this scan query - ++ResultsReceived_; - } - ProcessingScanQuery_ = true; - - Ydb::Scripting::ExecuteYqlPartialResponse response; - response.set_status(StatusIds::SUCCESS); - auto result = response.mutable_result(); - result->mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet()); - result->set_result_set_index(ResultsReceived_ - 1); - - TString out; + + GRpcResponsesSizeQueue_.push(out.size()); + GRpcResponsesSize_ += out.size(); + + RequestPtr()->SendSerializedResult(std::move(out), StatusIds::SUCCESS); + } + + // From TKqpStreamRequestHandler + void Handle(NKqp::TEvKqp::TEvDataQueryStreamPart::TPtr& ev, const TActorContext& ctx) { + GatewayRequestHandlerActorId_ = ActorIdFromProto(ev->Get()->Record.GetGatewayActorId()); + + if (!ev->Get()->Record.GetResults().size()) { + return ReplyFinishStream("Received TEvDataQueryStreamPart with no results", + ctx); + } + if (DataQueryStreamContext) { + return ReplyFinishStream("Received TEvDataQueryStreamPart event while previous data query is in progress", + ctx); + } + + DataQueryStreamContext = MakeHolder<TDataQueryStreamContext>(ev); + + SendDataQueryResultPart(); + } + + // From TKqpScanQueryStreamRequestHandler + void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) { + GatewayRequestHandlerActorId_ = ActorIdFromProto(ev->Get()->Record.GetExecuterActorId()); + ProcessingScanQuery_ = false; + LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " GatewayRequestHandlerActorId_: " << GatewayRequestHandlerActorId_); + } + + // From TKqpScanQueryStreamRequestHandler + void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) { + if (!GatewayRequestHandlerActorId_) { + return ReplyFinishStream("Received StreamData event from unknown executer", ctx); + } + if (!ProcessingScanQuery_) { + // First data part from this scan query + ++ResultsReceived_; + } + ProcessingScanQuery_ = true; + + Ydb::Scripting::ExecuteYqlPartialResponse response; + response.set_status(StatusIds::SUCCESS); + auto result = response.mutable_result(); + result->mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet()); + result->set_result_set_index(ResultsReceived_ - 1); + + TString out; Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); - - GRpcResponsesSizeQueue_.push(out.size()); - GRpcResponsesSize_ += out.size(); - - RequestPtr()->SendSerializedResult(std::move(out), StatusIds::SUCCESS); - - ui64 freeSpace = GRpcResponsesSize_ < RpcBufferSize_ - ? RpcBufferSize_ - GRpcResponsesSize_ - : 0; - - if (freeSpace == 0) { - WaitOnSeqNo_ = ev->Get()->Record.GetSeqNo(); - } - - LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack" - << ", seqNo: " << ev->Get()->Record.GetSeqNo() - << ", freeSpace: " << freeSpace - << ", to: " << ev->Sender - << ", queue: " << GRpcResponsesSizeQueue_.size()); - - auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); - resp->Record.SetFreeSpace(freeSpace); - - ctx.Send(ev->Sender, resp.Release()); - } - - void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " NextReply" - << ", left: " << ev->Get()->LeftInQueue - << ", queue: " << GRpcResponsesSizeQueue_.size() - << ", used memory: " << GRpcResponsesSize_ - << ", buffer size: " << RpcBufferSize_); - LastDataStreamTimestamp_ = TAppData::TimeProvider->Now(); - - if (DataQueryStreamContext) { - //DataQuery in progress - if (++DataQueryStreamContext->ResultIterator != DataQueryStreamContext->Handle->Get()->Record.GetResults().end()) { - // Send next ResultSet to client - return SendDataQueryResultPart(); - } else { - // Send ack to gateway request handler actor - auto resp = MakeHolder<NKqp::TEvKqp::TEvDataQueryStreamPartAck>(); - ctx.Send(GatewayRequestHandlerActorId_, resp.Release()); - DataQueryStreamContext.Reset(); - return; - } - - } else { - //ScanQuery in progress - while (GRpcResponsesSizeQueue_.size() > ev->Get()->LeftInQueue) { - GRpcResponsesSize_ -= GRpcResponsesSizeQueue_.front(); - GRpcResponsesSizeQueue_.pop(); - } - Y_VERIFY_DEBUG(GRpcResponsesSizeQueue_.empty() == (GRpcResponsesSize_ == 0)); - - if (WaitOnSeqNo_ && RpcBufferSize_ > GRpcResponsesSize_) { - ui64 freeSpace = RpcBufferSize_ - GRpcResponsesSize_; - - LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack" - << ", seqNo: " << *WaitOnSeqNo_ - << ", freeSpace: " << freeSpace - << ", to: " << GatewayRequestHandlerActorId_); - - auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetSeqNo(*WaitOnSeqNo_); - resp->Record.SetFreeSpace(freeSpace); - - ctx.Send(GatewayRequestHandlerActorId_, resp.Release()); - - WaitOnSeqNo_.Clear(); - } - } - } - - // Final response - void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record.GetRef(); - - NYql::TIssues issues; - const auto& issueMessage = record.GetResponse().GetQueryIssues(); - NYql::IssuesFromMessage(issueMessage, issues); - - if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { - Ydb::Scripting::ExecuteYqlPartialResponse response; - TString out; - auto& kqpResponse = record.GetResponse(); - response.set_status(Ydb::StatusIds::SUCCESS); - - if (kqpResponse.HasQueryStats()) { - FillQueryStats(*response.mutable_result()->mutable_query_stats(), kqpResponse); - } else if (kqpResponse.HasQueryPlan()) { - response.mutable_result()->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan()); - } - + + GRpcResponsesSizeQueue_.push(out.size()); + GRpcResponsesSize_ += out.size(); + + RequestPtr()->SendSerializedResult(std::move(out), StatusIds::SUCCESS); + + ui64 freeSpace = GRpcResponsesSize_ < RpcBufferSize_ + ? RpcBufferSize_ - GRpcResponsesSize_ + : 0; + + if (freeSpace == 0) { + WaitOnSeqNo_ = ev->Get()->Record.GetSeqNo(); + } + + LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack" + << ", seqNo: " << ev->Get()->Record.GetSeqNo() + << ", freeSpace: " << freeSpace + << ", to: " << ev->Sender + << ", queue: " << GRpcResponsesSizeQueue_.size()); + + auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); + resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); + resp->Record.SetFreeSpace(freeSpace); + + ctx.Send(ev->Sender, resp.Release()); + } + + void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " NextReply" + << ", left: " << ev->Get()->LeftInQueue + << ", queue: " << GRpcResponsesSizeQueue_.size() + << ", used memory: " << GRpcResponsesSize_ + << ", buffer size: " << RpcBufferSize_); + LastDataStreamTimestamp_ = TAppData::TimeProvider->Now(); + + if (DataQueryStreamContext) { + //DataQuery in progress + if (++DataQueryStreamContext->ResultIterator != DataQueryStreamContext->Handle->Get()->Record.GetResults().end()) { + // Send next ResultSet to client + return SendDataQueryResultPart(); + } else { + // Send ack to gateway request handler actor + auto resp = MakeHolder<NKqp::TEvKqp::TEvDataQueryStreamPartAck>(); + ctx.Send(GatewayRequestHandlerActorId_, resp.Release()); + DataQueryStreamContext.Reset(); + return; + } + + } else { + //ScanQuery in progress + while (GRpcResponsesSizeQueue_.size() > ev->Get()->LeftInQueue) { + GRpcResponsesSize_ -= GRpcResponsesSizeQueue_.front(); + GRpcResponsesSizeQueue_.pop(); + } + Y_VERIFY_DEBUG(GRpcResponsesSizeQueue_.empty() == (GRpcResponsesSize_ == 0)); + + if (WaitOnSeqNo_ && RpcBufferSize_ > GRpcResponsesSize_) { + ui64 freeSpace = RpcBufferSize_ - GRpcResponsesSize_; + + LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack" + << ", seqNo: " << *WaitOnSeqNo_ + << ", freeSpace: " << freeSpace + << ", to: " << GatewayRequestHandlerActorId_); + + auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); + resp->Record.SetSeqNo(*WaitOnSeqNo_); + resp->Record.SetFreeSpace(freeSpace); + + ctx.Send(GatewayRequestHandlerActorId_, resp.Release()); + + WaitOnSeqNo_.Clear(); + } + } + } + + // Final response + void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record.GetRef(); + + NYql::TIssues issues; + const auto& issueMessage = record.GetResponse().GetQueryIssues(); + NYql::IssuesFromMessage(issueMessage, issues); + + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { + Ydb::Scripting::ExecuteYqlPartialResponse response; + TString out; + auto& kqpResponse = record.GetResponse(); + response.set_status(Ydb::StatusIds::SUCCESS); + + if (kqpResponse.HasQueryStats()) { + FillQueryStats(*response.mutable_result()->mutable_query_stats(), kqpResponse); + } else if (kqpResponse.HasQueryPlan()) { + response.mutable_result()->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan()); + } + Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); - RequestPtr()->SendSerializedResult(std::move(out), record.GetYdbStatus()); - } - - ReplyFinishStream(record.GetYdbStatus(), issues, ctx); - } - -private: - void SetClientTimeoutTimer(TDuration timeout, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Set stream timeout timer for " << timeout); - - auto *ev = new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(EStreamRpcWakeupTag::ClientTimeoutTag)); - ClientTimeoutTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way()); - CreateLongTimer(ctx, timeout, ev, 0, ClientTimeoutTimerCookieHolder_.Get()); - } - - void HandleClientLost(const TActorContext& ctx) { - LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Client lost, send abort event to executer " << GatewayRequestHandlerActorId_); - - if (GatewayRequestHandlerActorId_) { - auto abortEv = NKqp::TEvKqp::TEvAbortExecution::Aborted("Client lost"); - - ctx.Send(GatewayRequestHandlerActorId_, abortEv.Release()); - } - - // We must try to finish stream otherwise grpc will not free allocated memory - // If stream already scheduled to be finished (ReplyFinishStream already called) - // this call do nothing but Die will be called after reply to grpc - ReplyFinishStream("Client should not see this message, if so... may the force be with you", ctx); - } - - void HandleClientTimeout(const TActorContext& ctx) { - TInstant now = TAppData::TimeProvider->Now(); - TDuration timeout; - - if (InactiveClientTimeout_ && GRpcResponsesSizeQueue_.size() > 0) { - TDuration processTime = now - LastDataStreamTimestamp_; - if (processTime >= InactiveClientTimeout_) { - auto message = TStringBuilder() << this->SelfId() << " Client cannot process data in " << processTime - << " which exceeds client timeout " << InactiveClientTimeout_; - LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, message); - - if (GatewayRequestHandlerActorId_) { - auto timeoutEv = MakeHolder<NKqp::TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Client timeout"); - ctx.Send(GatewayRequestHandlerActorId_, timeoutEv.Release()); - } - - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message); - return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx); - } - TDuration remain = InactiveClientTimeout_ - processTime; - timeout = timeout ? Min(timeout, remain) : remain; - } - - if (timeout) { - SetClientTimeoutTimer(timeout, ctx); - } - } - - void HandleOperationTimeout(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, TStringBuilder() << this->SelfId() << " Operation timeout."); - - if (GatewayRequestHandlerActorId_) { - auto timeoutEv = MakeHolder<NKqp::TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Operation timeout"); - ctx.Send(GatewayRequestHandlerActorId_, timeoutEv.Release()); - } - - auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Operation timeout"); - return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx); - } - - void ReplyFinishStream(const TString& message, const TActorContext& ctx) { - NYql::TIssues issues; - issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message)); - ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issues, ctx); - } - - void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue, const TActorContext& ctx) { - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage; - NYql::IssueToMessage(issue, issuesMessage.Add()); - - ReplyFinishStream(status, issuesMessage, ctx); - } - - void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { - google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage; - for (auto& issue : issues) { - auto item = issuesMessage.Add(); - NYql::IssueToMessage(issue, item); - } - - ReplyFinishStream(status, issuesMessage, ctx); - } - - void ReplyFinishStream(Ydb::StatusIds::StatusCode status, - const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx) - { - LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, "Finish grpc stream, status: " - << Ydb::StatusIds::StatusCode_Name(status)); - - // Skip sending empty result in case of success status - simplify client logic - if (status != Ydb::StatusIds::SUCCESS) { - TString out = NullSerializeResponse(message, status); - RequestPtr()->SendSerializedResult(std::move(out), status); - } - - RequestPtr()->FinishStream(); - this->PassAway(); - } - - static TString NullSerializeResponse(const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, - Ydb::StatusIds::StatusCode status) - { - TString out; - Ydb::Scripting::ExecuteYqlPartialResponse response; - response.set_status(status); - response.mutable_issues()->CopyFrom(message); + RequestPtr()->SendSerializedResult(std::move(out), record.GetYdbStatus()); + } + + ReplyFinishStream(record.GetYdbStatus(), issues, ctx); + } + +private: + void SetClientTimeoutTimer(TDuration timeout, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Set stream timeout timer for " << timeout); + + auto *ev = new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(EStreamRpcWakeupTag::ClientTimeoutTag)); + ClientTimeoutTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way()); + CreateLongTimer(ctx, timeout, ev, 0, ClientTimeoutTimerCookieHolder_.Get()); + } + + void HandleClientLost(const TActorContext& ctx) { + LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Client lost, send abort event to executer " << GatewayRequestHandlerActorId_); + + if (GatewayRequestHandlerActorId_) { + auto abortEv = NKqp::TEvKqp::TEvAbortExecution::Aborted("Client lost"); + + ctx.Send(GatewayRequestHandlerActorId_, abortEv.Release()); + } + + // We must try to finish stream otherwise grpc will not free allocated memory + // If stream already scheduled to be finished (ReplyFinishStream already called) + // this call do nothing but Die will be called after reply to grpc + ReplyFinishStream("Client should not see this message, if so... may the force be with you", ctx); + } + + void HandleClientTimeout(const TActorContext& ctx) { + TInstant now = TAppData::TimeProvider->Now(); + TDuration timeout; + + if (InactiveClientTimeout_ && GRpcResponsesSizeQueue_.size() > 0) { + TDuration processTime = now - LastDataStreamTimestamp_; + if (processTime >= InactiveClientTimeout_) { + auto message = TStringBuilder() << this->SelfId() << " Client cannot process data in " << processTime + << " which exceeds client timeout " << InactiveClientTimeout_; + LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, message); + + if (GatewayRequestHandlerActorId_) { + auto timeoutEv = MakeHolder<NKqp::TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Client timeout"); + ctx.Send(GatewayRequestHandlerActorId_, timeoutEv.Release()); + } + + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message); + return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx); + } + TDuration remain = InactiveClientTimeout_ - processTime; + timeout = timeout ? Min(timeout, remain) : remain; + } + + if (timeout) { + SetClientTimeoutTimer(timeout, ctx); + } + } + + void HandleOperationTimeout(const TActorContext& ctx) { + LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, TStringBuilder() << this->SelfId() << " Operation timeout."); + + if (GatewayRequestHandlerActorId_) { + auto timeoutEv = MakeHolder<NKqp::TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Operation timeout"); + ctx.Send(GatewayRequestHandlerActorId_, timeoutEv.Release()); + } + + auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Operation timeout"); + return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx); + } + + void ReplyFinishStream(const TString& message, const TActorContext& ctx) { + NYql::TIssues issues; + issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message)); + ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issues, ctx); + } + + void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue, const TActorContext& ctx) { + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage; + NYql::IssueToMessage(issue, issuesMessage.Add()); + + ReplyFinishStream(status, issuesMessage, ctx); + } + + void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) { + google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage; + for (auto& issue : issues) { + auto item = issuesMessage.Add(); + NYql::IssueToMessage(issue, item); + } + + ReplyFinishStream(status, issuesMessage, ctx); + } + + void ReplyFinishStream(Ydb::StatusIds::StatusCode status, + const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx) + { + LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, "Finish grpc stream, status: " + << Ydb::StatusIds::StatusCode_Name(status)); + + // Skip sending empty result in case of success status - simplify client logic + if (status != Ydb::StatusIds::SUCCESS) { + TString out = NullSerializeResponse(message, status); + RequestPtr()->SendSerializedResult(std::move(out), status); + } + + RequestPtr()->FinishStream(); + this->PassAway(); + } + + static TString NullSerializeResponse(const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, + Ydb::StatusIds::StatusCode status) + { + TString out; + Ydb::Scripting::ExecuteYqlPartialResponse response; + response.set_status(status); + response.mutable_issues()->CopyFrom(message); Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); - return out; - } - -private: - const ui64 RpcBufferSize_; - - TDuration InactiveClientTimeout_; - TQueue<ui64> GRpcResponsesSizeQueue_; - ui64 GRpcResponsesSize_ = 0; - TInstant LastDataStreamTimestamp_; - TMaybe<ui64> WaitOnSeqNo_; - - TSchedulerCookieHolder ClientTimeoutTimerCookieHolder_; - - TActorId GatewayRequestHandlerActorId_; - ui64 ResultsReceived_ = 0; - bool ProcessingScanQuery_ = false; - - // DataQuery - THolder<TDataQueryStreamContext> DataQueryStreamContext; -}; - -} // namespace - + return out; + } + +private: + const ui64 RpcBufferSize_; + + TDuration InactiveClientTimeout_; + TQueue<ui64> GRpcResponsesSizeQueue_; + ui64 GRpcResponsesSize_ = 0; + TInstant LastDataStreamTimestamp_; + TMaybe<ui64> WaitOnSeqNo_; + + TSchedulerCookieHolder ClientTimeoutTimerCookieHolder_; + + TActorId GatewayRequestHandlerActorId_; + ui64 ResultsReceived_ = 0; + bool ProcessingScanQuery_ = false; + + // DataQuery + THolder<TDataQueryStreamContext> DataQueryStreamContext; +}; + +} // namespace + void DoStreamExecuteYqlScript(std::unique_ptr<IRequestNoOpCtx> p, const IFacilityProvider& facility) { ui64 rpcBufferSize = facility.GetAppConfig().GetTableServiceConfig().GetResourceManager().GetChannelBufferSize(); TActivationContext::AsActorContext().Register(new TStreamExecuteYqlScriptRPC(p.release(), rpcBufferSize)); -} - -} // namespace NGRpcService -} // namespace NKikimr +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_whoami.cpp b/ydb/core/grpc_services/rpc_whoami.cpp index 6d0f1ea1a1..b8b2dadfcf 100644 --- a/ydb/core/grpc_services/rpc_whoami.cpp +++ b/ydb/core/grpc_services/rpc_whoami.cpp @@ -1,86 +1,86 @@ -#include "grpc_request_proxy.h" -#include "rpc_calls.h" - +#include "grpc_request_proxy.h" +#include "rpc_calls.h" + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <ydb/core/security/ticket_parser.h> - - -namespace NKikimr { -namespace NGRpcService { - -class TWhoAmIRPC : public TActorBootstrapped<TWhoAmIRPC> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::GRPC_REQ; - } - - TWhoAmIRPC(TEvWhoAmIRequest::TPtr& request) - : Request(request->Release().Release()) - {} - - void Bootstrap(const TActorContext& ctx) { - TMaybe<TString> authToken = Request->GetYdbToken(); - if (authToken) { + + +namespace NKikimr { +namespace NGRpcService { + +class TWhoAmIRPC : public TActorBootstrapped<TWhoAmIRPC> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::GRPC_REQ; + } + + TWhoAmIRPC(TEvWhoAmIRequest::TPtr& request) + : Request(request->Release().Release()) + {} + + void Bootstrap(const TActorContext& ctx) { + TMaybe<TString> authToken = Request->GetYdbToken(); + if (authToken) { TMaybe<TString> database = Request->GetDatabaseName(); ctx.Send(MakeTicketParserID(), new TEvTicketParser::TEvAuthorizeTicket({ .Database = database ? database.GetRef() : TString(), .Ticket = authToken.GetRef(), .PeerName = Request->GetPeerName() })); - Become(&TThis::StateWaitForTicket); - } else { - ReplyError("No token provided"); - PassAway(); - } - } - - STFUNC(StateWaitForTicket) { - Y_UNUSED(ctx); - switch (ev->GetTypeRewrite()) { + Become(&TThis::StateWaitForTicket); + } else { + ReplyError("No token provided"); + PassAway(); + } + } + + STFUNC(StateWaitForTicket) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { hFunc(TEvTicketParser::TEvAuthorizeTicketResult, Handle); - hFunc(TEvents::TEvUndelivered, Handle); - } - } - -private: + hFunc(TEvents::TEvUndelivered, Handle); + } + } + +private: void Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev) { const TEvTicketParser::TEvAuthorizeTicketResult& result(*ev->Get()); auto *response = TEvWhoAmIRequest::AllocateResult<Ydb::Discovery::WhoAmIResult>(Request); - if (!result.Error) { - if (result.Token != nullptr) { - response->set_user(result.Token->GetUserSID()); - if (Request->GetProtoRequest()->include_groups()) { - for (const auto& group : result.Token->GetGroupSIDs()) { - response->add_groups(group); - } - } - Request->SendResult(*response, Ydb::StatusIds::SUCCESS); - } else { - ReplyError("Empty token in Staff response"); - } - } else { - ReplyError(result.Error.Message); - } - PassAway(); - } - - void Handle(TEvents::TEvUndelivered::TPtr&) { - ReplyError("Error parsing token"); - PassAway(); - } - - void ReplyError(const TString& error) { - auto issue = NYql::TIssue(error); - Request->RaiseIssue(issue); + if (!result.Error) { + if (result.Token != nullptr) { + response->set_user(result.Token->GetUserSID()); + if (Request->GetProtoRequest()->include_groups()) { + for (const auto& group : result.Token->GetGroupSIDs()) { + response->add_groups(group); + } + } + Request->SendResult(*response, Ydb::StatusIds::SUCCESS); + } else { + ReplyError("Empty token in Staff response"); + } + } else { + ReplyError(result.Error.Message); + } + PassAway(); + } + + void Handle(TEvents::TEvUndelivered::TPtr&) { + ReplyError("Error parsing token"); + PassAway(); + } + + void ReplyError(const TString& error) { + auto issue = NYql::TIssue(error); + Request->RaiseIssue(issue); Request->ReplyWithYdbStatus(Ydb::StatusIds::GENERIC_ERROR); - } - - THolder<TEvWhoAmIRequest> Request; -}; - -void TGRpcRequestProxy::Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TWhoAmIRPC(ev)); -} - -} // namespace NGRpcService -} // namespace NKikimr + } + + THolder<TEvWhoAmIRequest> Request; +}; + +void TGRpcRequestProxy::Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx) { + ctx.Register(new TWhoAmIRPC(ev)); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/table_profiles.cpp b/ydb/core/grpc_services/table_profiles.cpp index 213ccd65bf..367de4cc89 100644 --- a/ydb/core/grpc_services/table_profiles.cpp +++ b/ydb/core/grpc_services/table_profiles.cpp @@ -427,7 +427,7 @@ bool TTableProfiles::ApplyTableProfile(const Ydb::Table::TableProfile &profile, followerGroup.SetRequireAllDataCenters(false); if (replicationPolicy.HasAllowFollowerPromotion()) { followerGroup.SetAllowLeaderPromotion(replicationPolicy.GetAllowFollowerPromotion()); - } + } } if (cachingPolicy.HasExecutorCacheSize()) @@ -436,28 +436,28 @@ bool TTableProfiles::ApplyTableProfile(const Ydb::Table::TableProfile &profile, return true; } -bool TTableProfiles::ApplyCompactionPolicy(const TString &name, +bool TTableProfiles::ApplyCompactionPolicy(const TString &name, NKikimrSchemeOp::TPartitionConfig &partitionConfig, Ydb::StatusIds::StatusCode &code, - TString &error, const TAppData* appData) const -{ - NKikimrConfig::TCompactionPolicy compactionPolicy; - - if (!CompactionPolicies.contains(name)) { - code = Ydb::StatusIds::BAD_REQUEST; - error = Sprintf("unknown compaction policy preset '%s'", name.c_str()); - return false; - } - compactionPolicy = CompactionPolicies.at(name); - // Apply compaction policy to table description. - if (compactionPolicy.HasCompactionPolicy()) { - partitionConfig.MutableCompactionPolicy()->CopyFrom(compactionPolicy.GetCompactionPolicy()); - } else if (appData) { - TIntrusiveConstPtr<NLocalDb::TCompactionPolicy> defaultPolicy = appData->DomainsInfo->GetDefaultUserTablePolicy(); - defaultPolicy->Serialize(*partitionConfig.MutableCompactionPolicy()); - } - return true; -} - - + TString &error, const TAppData* appData) const +{ + NKikimrConfig::TCompactionPolicy compactionPolicy; + + if (!CompactionPolicies.contains(name)) { + code = Ydb::StatusIds::BAD_REQUEST; + error = Sprintf("unknown compaction policy preset '%s'", name.c_str()); + return false; + } + compactionPolicy = CompactionPolicies.at(name); + // Apply compaction policy to table description. + if (compactionPolicy.HasCompactionPolicy()) { + partitionConfig.MutableCompactionPolicy()->CopyFrom(compactionPolicy.GetCompactionPolicy()); + } else if (appData) { + TIntrusiveConstPtr<NLocalDb::TCompactionPolicy> defaultPolicy = appData->DomainsInfo->GetDefaultUserTablePolicy(); + defaultPolicy->Serialize(*partitionConfig.MutableCompactionPolicy()); + } + return true; +} + + } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/table_profiles.h b/ydb/core/grpc_services/table_profiles.h index ab8dfdd347..2acec6f115 100644 --- a/ydb/core/grpc_services/table_profiles.h +++ b/ydb/core/grpc_services/table_profiles.h @@ -23,10 +23,10 @@ public: NKikimrSchemeOp::TTableDescription &tableDesc, Ydb::StatusIds::StatusCode &code, TString &error) const; - + bool ApplyCompactionPolicy(const TString &name, NKikimrSchemeOp::TPartitionConfig &partitionConfig, - Ydb::StatusIds::StatusCode &code, TString &error, const TAppData* appData = nullptr) const; - + Ydb::StatusIds::StatusCode &code, TString &error, const TAppData* appData = nullptr) const; + private: NKikimrSchemeOp::TFamilyDescription *GetNamedFamilyDescription(NKikimrConfig::TStoragePolicy &policy, const TString& name) const; NKikimrSchemeOp::TFamilyDescription *GetDefaultFamilyDescription(NKikimrConfig::TStoragePolicy &policy) const; diff --git a/ydb/core/grpc_services/table_settings.cpp b/ydb/core/grpc_services/table_settings.cpp index 82b033c6f0..97dc63a53c 100644 --- a/ydb/core/grpc_services/table_settings.cpp +++ b/ydb/core/grpc_services/table_settings.cpp @@ -1,50 +1,50 @@ -#include "table_settings.h" - -namespace NKikimr { -namespace NGRpcService { - +#include "table_settings.h" + +namespace NKikimr { +namespace NGRpcService { + bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings) { - - bool tableProfileSet = false; + + bool tableProfileSet = false; if (in.has_profile()) { const auto& profile = in.profile(); - tableProfileSet = profile.preset_name() || profile.has_compaction_policy() || profile.has_execution_policy() - || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() - || profile.has_caching_policy(); - } + tableProfileSet = profile.preset_name() || profile.has_compaction_policy() || profile.has_execution_policy() + || profile.has_partitioning_policy() || profile.has_storage_policy() || profile.has_replication_policy() + || profile.has_caching_policy(); + } auto &partitionConfig = *out.MutablePartitionConfig(); if (!in.compaction_policy().empty()) { - if (tableProfileSet) { - MEWarning("CompactionPolicy", warnings); - } + if (tableProfileSet) { + MEWarning("CompactionPolicy", warnings); + } if (!profiles.ApplyCompactionPolicy(in.compaction_policy(), partitionConfig, code, error)) { - return false; - } - } - + return false; + } + } + return NKikimr::FillCreateTableSettingsDesc(out, in, code, error, warnings, tableProfileSet); } - + bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData) { - - bool changed = false; + + bool changed = false; auto &partitionConfig = *out.MutablePartitionConfig(); - + if (in.set_compaction_policy()) { if (!profiles.ApplyCompactionPolicy(in.set_compaction_policy(), partitionConfig, code, error, appData)) { - return false; - } - - changed = true; - } - + return false; + } + + changed = true; + } + return NKikimr::FillAlterTableSettingsDesc(out, in, code, error, changed); -} - -} // namespace NGRpcService -} // namespace NKikimr +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/table_settings.h b/ydb/core/grpc_services/table_settings.h index 78108b3b97..c8c6d26281 100644 --- a/ydb/core/grpc_services/table_settings.h +++ b/ydb/core/grpc_services/table_settings.h @@ -1,19 +1,19 @@ -#pragma once - -#include "table_profiles.h" - +#pragma once + +#include "table_profiles.h" + #include <ydb/core/ydb_convert/table_settings.h> - -namespace NKikimr { -namespace NGRpcService { - + +namespace NKikimr { +namespace NGRpcService { + bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, const Ydb::Table::CreateTableRequest& in, const NGRpcService::TTableProfiles& profiles, - Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings); - + Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings); + bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, const Ydb::Table::AlterTableRequest& in, const NGRpcService::TTableProfiles& profiles, - Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData); - -} // namespace NGRpcService -} // namespace NKikimr + Ydb::StatusIds::StatusCode& code, TString& error, const TAppData* appData); + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index fbc6d33bd9..05b156bf04 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -67,11 +67,11 @@ SRCS( rpc_s3_listing.cpp rpc_scheme_base.cpp rpc_stream_execute_scan_query.cpp - rpc_stream_execute_yql_script.cpp - rpc_whoami.cpp + rpc_stream_execute_yql_script.cpp + rpc_whoami.cpp rpc_yq.cpp table_profiles.cpp - table_settings.cpp + table_settings.cpp rpc_analytics_internal.cpp ) diff --git a/ydb/core/kqp/common/kqp_common.h b/ydb/core/kqp/common/kqp_common.h index 67d096ec4c..f9379ad129 100644 --- a/ydb/core/kqp/common/kqp_common.h +++ b/ydb/core/kqp/common/kqp_common.h @@ -27,8 +27,8 @@ struct TKqpEvents { EvInitiateShutdownRequest, EvInitiateSessionShutdown, EvContinueShutdown, - EvDataQueryStreamPart, - EvDataQueryStreamPartAck, + EvDataQueryStreamPart, + EvDataQueryStreamPartAck, EvRecompileRequest, }; diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h index 2c5805d6eb..1e06beca39 100644 --- a/ydb/core/kqp/common/kqp_gateway.h +++ b/ydb/core/kqp/common/kqp_gateway.h @@ -173,13 +173,13 @@ public: virtual NThreading::TFuture<TQueryResult> ExecScanQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, ui64 rowsLimit) = 0; - virtual NThreading::TFuture<TQueryResult> StreamExecDataQueryAst(const TString& cluster, const TString& query, + virtual NThreading::TFuture<TQueryResult> StreamExecDataQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, - const Ydb::Table::TransactionSettings& txSettings, const NActors::TActorId& target) = 0; - - virtual NThreading::TFuture<TQueryResult> StreamExecScanQueryAst(const TString& cluster, const TString& query, + const Ydb::Table::TransactionSettings& txSettings, const NActors::TActorId& target) = 0; + + virtual NThreading::TFuture<TQueryResult> StreamExecScanQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, const NActors::TActorId& target) = 0; - + public: virtual TInstant GetCurrentTime() const = 0; }; diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp index 59c5acddd0..26a28a0249 100644 --- a/ydb/core/kqp/counters/kqp_counters.cpp +++ b/ydb/core/kqp/counters/kqp_counters.cpp @@ -96,8 +96,8 @@ void TKqpCountersBase::Init() { KqpGroup->GetCounter("Request/QueryTypeAstDml", true); QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT] = KqpGroup->GetCounter("Request/QueryTypeSqlScript", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING] = - KqpGroup->GetCounter("Request/QueryTypeSqlScriptStreaming", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING] = + KqpGroup->GetCounter("Request/QueryTypeSqlScriptStreaming", true); QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN] = KqpGroup->GetCounter("Request/QueryTypeSqlScan", true); QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN] = diff --git a/ydb/core/kqp/counters/kqp_db_counters.h b/ydb/core/kqp/counters/kqp_db_counters.h index e5a54b1473..814c382d4c 100644 --- a/ydb/core/kqp/counters/kqp_db_counters.h +++ b/ydb/core/kqp/counters/kqp_db_counters.h @@ -35,7 +35,7 @@ namespace NKqp { XX(DB_KQP_QUERY_TYPE_PREPARED_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML]) \ XX(DB_KQP_QUERY_TYPE_AST_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML]) \ XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT]) \ - XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT_STREAMING, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING]) \ + XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT_STREAMING, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING]) \ XX(DB_KQP_QUERY_TYPE_SQL_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN]) \ XX(DB_KQP_QUERY_TYPE_AST_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN]) \ XX(DB_KQP_RSP_STATUS_CODE_UNSPECIFIED, YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \ diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 14300142e8..44af6d1168 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -119,32 +119,32 @@ void FillAstAndPlan(IKqpHost::TQueryResult& queryResult, const NKikimrKqp::TPrep } /* - * Validate YqlScript. - */ -class TAsyncValidateYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { -public: - using TResult = IKqpHost::TQueryResult; - - TAsyncValidateYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx, + * Validate YqlScript. + */ +class TAsyncValidateYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { +public: + using TResult = IKqpHost::TQueryResult; + + TAsyncValidateYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx, TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer, TMaybe<TSqlVersion> sqlVersion) - : TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get()) - , SessionCtx(sessionCtx) - , Transformer(transformer) - , SqlVersion(sqlVersion) {} - - void FillResult(TResult& validateResult) const override { - YQL_ENSURE(SessionCtx->Query().PrepareOnly); + : TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get()) + , SessionCtx(sessionCtx) + , Transformer(transformer) + , SqlVersion(sqlVersion) {} + + void FillResult(TResult& validateResult) const override { + YQL_ENSURE(SessionCtx->Query().PrepareOnly); validateResult.PreparedQuery.reset(SessionCtx->Query().PreparingQuery.release()); - validateResult.SqlVersion = SqlVersion; - } - -private: - TIntrusivePtr<TKikimrSessionContext> SessionCtx; - TAutoPtr<IGraphTransformer> Transformer; - TMaybe<TSqlVersion> SqlVersion; -}; - -/* + validateResult.SqlVersion = SqlVersion; + } + +private: + TIntrusivePtr<TKikimrSessionContext> SessionCtx; + TAutoPtr<IGraphTransformer> Transformer; + TMaybe<TSqlVersion> SqlVersion; +}; + +/* * Explain Yql/YqlScript. */ class TAsyncExplainYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { @@ -565,7 +565,7 @@ public: if (queryCtx->Type == EKikimrQueryType::YqlScript || queryCtx->Type == EKikimrQueryType::YqlScriptStreaming) - { + { return ret; } @@ -819,7 +819,7 @@ public: case EKikimrQueryType::Scan: return KqpRunner->PrepareScanQuery(cluster, query, ctx, settings); case EKikimrQueryType::YqlScript: - case EKikimrQueryType::YqlScriptStreaming: + case EKikimrQueryType::YqlScriptStreaming: break; default: YQL_ENSURE(false, "Unexpected query type for prepare action: " << queryType); @@ -828,9 +828,9 @@ public: } switch (queryType) { - case EKikimrQueryType::YqlScript: - case EKikimrQueryType::YqlScriptStreaming: - { + case EKikimrQueryType::YqlScript: + case EKikimrQueryType::YqlScriptStreaming: + { YQL_ENSURE(TMaybeNode<TKiDataQuery>(query)); TKiDataQuery dataQuery(query); @@ -843,46 +843,46 @@ public: TFuture<TQueryResult> future; switch (queryType) { - case EKikimrQueryType::YqlScript: - if (useScanQuery) { - ui64 rowsLimit = 0; - if (!dataQuery.Results().Empty()) { - rowsLimit = FromString<ui64>(dataQuery.Results().Item(0).RowsLimit()); - } - - if (SessionCtx->Query().PrepareOnly) { - future = Gateway->ExplainScanQueryAst(Cluster, queryAstStr); - } else { - future = Gateway->ExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), - querySettings, rowsLimit); - } + case EKikimrQueryType::YqlScript: + if (useScanQuery) { + ui64 rowsLimit = 0; + if (!dataQuery.Results().Empty()) { + rowsLimit = FromString<ui64>(dataQuery.Results().Item(0).RowsLimit()); + } + + if (SessionCtx->Query().PrepareOnly) { + future = Gateway->ExplainScanQueryAst(Cluster, queryAstStr); + } else { + future = Gateway->ExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), + querySettings, rowsLimit); + } } else { - Ydb::Table::TransactionSettings txSettings; - txSettings.mutable_serializable_read_write(); - if (SessionCtx->Query().PrepareOnly) { - future = Gateway->ExplainDataQueryAst(Cluster, queryAstStr); - } else { - future = Gateway->ExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), - querySettings, txSettings); - } + Ydb::Table::TransactionSettings txSettings; + txSettings.mutable_serializable_read_write(); + if (SessionCtx->Query().PrepareOnly) { + future = Gateway->ExplainDataQueryAst(Cluster, queryAstStr); + } else { + future = Gateway->ExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), + querySettings, txSettings); + } } - break; - case EKikimrQueryType::YqlScriptStreaming: - if (useScanQuery) { - future = Gateway->StreamExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), - querySettings, SessionCtx->Query().ReplyTarget); + break; + case EKikimrQueryType::YqlScriptStreaming: + if (useScanQuery) { + future = Gateway->StreamExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), + querySettings, SessionCtx->Query().ReplyTarget); } else { - Ydb::Table::TransactionSettings txSettings; - txSettings.mutable_serializable_read_write(); - - future = Gateway->StreamExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), - querySettings, txSettings, SessionCtx->Query().ReplyTarget); + Ydb::Table::TransactionSettings txSettings; + txSettings.mutable_serializable_read_write(); + + future = Gateway->StreamExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), + querySettings, txSettings, SessionCtx->Query().ReplyTarget); } - break; - - default: + break; + + default: YQL_ENSURE(false, "Unexpected query type for execute action: " << queryType); - return nullptr; + return nullptr; } return MakeIntrusive<TKikimrFutureResult<TQueryResult>>(future, ctx); @@ -1332,29 +1332,29 @@ public: }); } - IAsyncQueryResultPtr StreamExecuteYqlScript(const TString& script, NKikimrMiniKQL::TParams&& parameters, - const NActors::TActorId& target, const TExecScriptSettings& settings) override - { - return CheckedProcessQuery(*ExprCtx, - [this, &script, parameters = std::move(parameters), target, settings](TExprContext& ctx) mutable { - return StreamExecuteYqlScriptInternal(script, std::move(parameters), target, settings, ctx); - }); - } - - IAsyncQueryResultPtr ValidateYqlScript(const TString& script) override { - return CheckedProcessQuery(*ExprCtx, - [this, &script](TExprContext& ctx) mutable { - return ValidateYqlScriptInternal(script, ctx); - }); - } - - TQueryResult SyncValidateYqlScript(const TString& script) override { - return CheckedSyncProcessQuery( - [this, &script]() mutable { - return ValidateYqlScript(script); - }); - } - + IAsyncQueryResultPtr StreamExecuteYqlScript(const TString& script, NKikimrMiniKQL::TParams&& parameters, + const NActors::TActorId& target, const TExecScriptSettings& settings) override + { + return CheckedProcessQuery(*ExprCtx, + [this, &script, parameters = std::move(parameters), target, settings](TExprContext& ctx) mutable { + return StreamExecuteYqlScriptInternal(script, std::move(parameters), target, settings, ctx); + }); + } + + IAsyncQueryResultPtr ValidateYqlScript(const TString& script) override { + return CheckedProcessQuery(*ExprCtx, + [this, &script](TExprContext& ctx) mutable { + return ValidateYqlScriptInternal(script, ctx); + }); + } + + TQueryResult SyncValidateYqlScript(const TString& script) override { + return CheckedSyncProcessQuery( + [this, &script]() mutable { + return ValidateYqlScript(script); + }); + } + IAsyncQueryResultPtr ExplainYqlScript(const TString& script) override { return CheckedProcessQuery(*ExprCtx, [this, &script] (TExprContext& ctx) mutable { @@ -1992,58 +1992,58 @@ private: *ResultProviderConfig, *PlanBuilder, sqlVersion); } - IAsyncQueryResultPtr StreamExecuteYqlScriptInternal(const TString& script, NKikimrMiniKQL::TParams&& parameters, - const NActors::TActorId& target,const TExecScriptSettings& settings, TExprContext& ctx) - { - SetupYqlTransformer(nullptr); - - SessionCtx->Query().Type = EKikimrQueryType::YqlScriptStreaming; - SessionCtx->Query().Deadlines = settings.Deadlines; - SessionCtx->Query().StatsMode = settings.StatsMode; - SessionCtx->Query().ReplyTarget = target; + IAsyncQueryResultPtr StreamExecuteYqlScriptInternal(const TString& script, NKikimrMiniKQL::TParams&& parameters, + const NActors::TActorId& target,const TExecScriptSettings& settings, TExprContext& ctx) + { + SetupYqlTransformer(nullptr); + + SessionCtx->Query().Type = EKikimrQueryType::YqlScriptStreaming; + SessionCtx->Query().Deadlines = settings.Deadlines; + SessionCtx->Query().StatsMode = settings.StatsMode; + SessionCtx->Query().ReplyTarget = target; SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>(); SessionCtx->Query().PreparedQuery.reset(); - - TMaybe<TSqlVersion> sqlVersion; - auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); - if (!scriptExpr) { - return nullptr; - } - - if (!ParseParameters(std::move(parameters), SessionCtx->Query().Parameters, ctx)) { - return nullptr; - } - - return MakeIntrusive<TAsyncExecuteYqlResult>(scriptExpr.Get(), ctx, *YqlTransformer, Cluster, SessionCtx, - *ResultProviderConfig, *PlanBuilder, sqlVersion); - } - - IAsyncQueryResultPtr ValidateYqlScriptInternal(const TString& script, TExprContext& ctx) { - SetupSession(nullptr); - - SessionCtx->Query().Type = EKikimrQueryType::YqlScript; - SessionCtx->Query().PrepareOnly = true; + + TMaybe<TSqlVersion> sqlVersion; + auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); + if (!scriptExpr) { + return nullptr; + } + + if (!ParseParameters(std::move(parameters), SessionCtx->Query().Parameters, ctx)) { + return nullptr; + } + + return MakeIntrusive<TAsyncExecuteYqlResult>(scriptExpr.Get(), ctx, *YqlTransformer, Cluster, SessionCtx, + *ResultProviderConfig, *PlanBuilder, sqlVersion); + } + + IAsyncQueryResultPtr ValidateYqlScriptInternal(const TString& script, TExprContext& ctx) { + SetupSession(nullptr); + + SessionCtx->Query().Type = EKikimrQueryType::YqlScript; + SessionCtx->Query().PrepareOnly = true; SessionCtx->Query().SuppressDdlChecks = true; SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>(); SessionCtx->Query().PreparedQuery.reset(); - - TMaybe<TSqlVersion> sqlVersion; - auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); - if (!scriptExpr) { - return nullptr; - } - - auto transformer = TTransformationPipeline(TypesCtx) - .AddServiceTransformers() - .AddPreTypeAnnotation() - .AddIOAnnotation() - .AddTypeAnnotation() + + TMaybe<TSqlVersion> sqlVersion; + auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); + if (!scriptExpr) { + return nullptr; + } + + auto transformer = TTransformationPipeline(TypesCtx) + .AddServiceTransformers() + .AddPreTypeAnnotation() + .AddIOAnnotation() + .AddTypeAnnotation() .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters") - .Build(false); - - return MakeIntrusive<TAsyncValidateYqlResult>(scriptExpr.Get(), SessionCtx, ctx, transformer, sqlVersion); - } - + .Build(false); + + return MakeIntrusive<TAsyncValidateYqlResult>(scriptExpr.Get(), SessionCtx, ctx, transformer, sqlVersion); + } + IAsyncQueryResultPtr ExplainYqlScriptInternal(const TString& script, TExprContext& ctx) { SetupYqlTransformer(nullptr); diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h index 9e550be97a..11282d0ebe 100644 --- a/ydb/core/kqp/host/kqp_host.h +++ b/ydb/core/kqp/host/kqp_host.h @@ -101,9 +101,9 @@ public: virtual IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) = 0; /* Scripting */ - virtual IAsyncQueryResultPtr ValidateYqlScript(const TString& script) = 0; - virtual TQueryResult SyncValidateYqlScript(const TString& script) = 0; - + virtual IAsyncQueryResultPtr ValidateYqlScript(const TString& script) = 0; + virtual TQueryResult SyncValidateYqlScript(const TString& script) = 0; + virtual IAsyncQueryResultPtr ExplainYqlScript(const TString& script) = 0; virtual TQueryResult SyncExplainYqlScript(const TString& script) = 0; @@ -111,9 +111,9 @@ public: const TExecScriptSettings& settings) = 0; virtual TQueryResult SyncExecuteYqlScript(const TString& script, NKikimrMiniKQL::TParams&& parameters, const TExecScriptSettings& settings) = 0; - - virtual IAsyncQueryResultPtr StreamExecuteYqlScript(const TString& script, NKikimrMiniKQL::TParams&& parameters, - const NActors::TActorId& target, const TExecScriptSettings& settings) = 0; + + virtual IAsyncQueryResultPtr StreamExecuteYqlScript(const TString& script, NKikimrMiniKQL::TParams&& parameters, + const NActors::TActorId& target, const TExecScriptSettings& settings) = 0; }; TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway, diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index f9cd5ecd99..42b4eca7c6 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -196,7 +196,7 @@ public: .AddServiceTransformers() .Add(TLogExprTransformer::Sync("PhysicalOptimizeTransformer", logComp, logLevel), "LogPhysicalOptimize") .AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), - *typesCtx, Config)) + *typesCtx, Config)) .Add(CreateKqpCheckQueryTransformer(), "CheckKqlQuery") .AddPostTypeAnnotation() .AddCommonOptimization() @@ -383,7 +383,7 @@ private: // scan query } - bool sysColumnsEnabled = TransformCtx->Config->SystemColumnsEnabled(); + bool sysColumnsEnabled = TransformCtx->Config->SystemColumnsEnabled(); std::optional<TKqpTransactionInfo::EEngine> engine; if (settings.UseNewEngine.Defined()) { @@ -419,7 +419,7 @@ private: } } - auto program = BuildKiProgram(dataQuery, *TransformCtx->Tables, ctx, sysColumnsEnabled); + auto program = BuildKiProgram(dataQuery, *TransformCtx->Tables, ctx, sysColumnsEnabled); KqlOptimizeTransformer->Rewind(); diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h index 17df3f63f5..09f794591d 100644 --- a/ydb/core/kqp/kqp.h +++ b/ydb/core/kqp/kqp.h @@ -246,11 +246,11 @@ struct TEvKqp { } }; - struct TEvDataQueryStreamPart : public TEventPB<TEvDataQueryStreamPart, - NKikimrKqp::TEvDataQueryStreamPart, TKqpEvents::EvDataQueryStreamPart> {}; - - struct TEvDataQueryStreamPartAck : public TEventLocal<TEvDataQueryStreamPartAck, TKqpEvents::EvDataQueryStreamPartAck> {}; - + struct TEvDataQueryStreamPart : public TEventPB<TEvDataQueryStreamPart, + NKikimrKqp::TEvDataQueryStreamPart, TKqpEvents::EvDataQueryStreamPart> {}; + + struct TEvDataQueryStreamPartAck : public TEventLocal<TEvDataQueryStreamPartAck, TKqpEvents::EvDataQueryStreamPartAck> {}; + // Wrapper to use Arena allocated protobuf with ActorSystem (for serialization path). // Arena deserialization is not supported. // TODO: Add arena support to actor system TEventPB? diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 1b2117f131..47489a9bd5 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -31,9 +31,9 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> -#include <util/string/split.h> +#include <util/string/split.h> #include <util/string/vector.h> - + namespace NKikimr { namespace NKqp { @@ -277,208 +277,208 @@ private: TVector<NYql::NDqProto::TDqExecutionStats> Executions; }; -// Handles data query request for StreamExecuteYqlScript -template<typename TRequest, typename TResponse, typename TResult> -class TKqpStreamRequestHandler : public TRequestHandlerBase< - TKqpStreamRequestHandler<TRequest, TResponse, TResult>, - TRequest, - TResponse, - TResult> -{ -public: - using TBase = typename TKqpStreamRequestHandler::TBase; - using TCallbackFunc = typename TBase::TCallbackFunc; - - TKqpStreamRequestHandler(TRequest* request, const TActorId& target, TPromise<TResult> promise, - TCallbackFunc callback) - : TBase(request, promise, callback) - , TargetActorId(target) {} - - void Bootstrap(const TActorContext& ctx) { - TActorId kqpProxy = MakeKqpProxyID(ctx.SelfID.NodeId()); - ctx.Send(kqpProxy, this->Request.Release()); - - this->Become(&TKqpStreamRequestHandler::AwaitState); - } - - void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { - const auto& kqpResponse = ev->Get()->Record; - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, ctx.SelfID +// Handles data query request for StreamExecuteYqlScript +template<typename TRequest, typename TResponse, typename TResult> +class TKqpStreamRequestHandler : public TRequestHandlerBase< + TKqpStreamRequestHandler<TRequest, TResponse, TResult>, + TRequest, + TResponse, + TResult> +{ +public: + using TBase = typename TKqpStreamRequestHandler::TBase; + using TCallbackFunc = typename TBase::TCallbackFunc; + + TKqpStreamRequestHandler(TRequest* request, const TActorId& target, TPromise<TResult> promise, + TCallbackFunc callback) + : TBase(request, promise, callback) + , TargetActorId(target) {} + + void Bootstrap(const TActorContext& ctx) { + TActorId kqpProxy = MakeKqpProxyID(ctx.SelfID.NodeId()); + ctx.Send(kqpProxy, this->Request.Release()); + + this->Become(&TKqpStreamRequestHandler::AwaitState); + } + + void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { + const auto& kqpResponse = ev->Get()->Record; + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, ctx.SelfID << "Received process error for kqp data query: " << kqpResponse.GetError()); - + TBase::HandleError(kqpResponse.GetError(), ctx); - } - - using TBase::Promise; - using TBase::Callback; - - virtual void HandleResponse(typename TResponse::TPtr &ev, const TActorContext &ctx) { - auto& record = ev->Get()->Record.GetRef(); - if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { - if (record.MutableResponse()->GetResults().size()) { - // Send result sets to RPC actor TStreamExecuteYqlScriptRPC - auto evStreamPart = MakeHolder<NKqp::TEvKqp::TEvDataQueryStreamPart>(); - ActorIdToProto(this->SelfId(), evStreamPart->Record.MutableGatewayActorId()); - - for (int i = 0; i < record.MutableResponse()->MutableResults()->size(); ++i) { - // Workaround to avoid errors on Pull execution stage which would expect some results - Ydb::ResultSet resultSet; - NKikimr::ConvertYdbResultToKqpResult(resultSet, *evStreamPart->Record.AddResults()); - } - - evStreamPart->Record.MutableResults()->Swap(record.MutableResponse()->MutableResults()); - this->Send(TargetActorId, evStreamPart.Release()); - - // Save response without data to send it later - ResponseHandle = ev.Release(); - } else { - // Response has no result sets. Forward to main pipeline - Callback(Promise, std::move(*ev->Get())); - this->Die(ctx); - } - } else { - // Forward error to main pipeline - Callback(Promise, std::move(*ev->Get())); - this->Die(ctx); - } - } - - void Handle(NKqp::TEvKqp::TEvDataQueryStreamPartAck::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Callback(Promise, std::move(*ResponseHandle->Get())); - this->Die(ctx); - } - - void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, this->SelfId() - << "Received abort execution event for data query: " << record.GetMessage()); - - TBase::HandleError(record.GetMessage(), ctx); - } - - using TBase::Handle; - using TBase::HandleResponse; - - STFUNC(AwaitState) { - switch (ev->GetTypeRewrite()) { - HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); - HFunc(TResponse, HandleResponse); - HFunc(NKqp::TEvKqp::TEvDataQueryStreamPartAck, Handle); - HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle); - - default: - TBase::HandleUnexpectedEvent("TKqpStreamRequestHandler", ev->GetTypeRewrite(), ctx); - } - } - -private: - TActorId TargetActorId; - typename TResponse::TPtr ResponseHandle; -}; - -// Handles scan query request for StreamExecuteYqlScript -class TKqpScanQueryStreamRequestHandler : public TRequestHandlerBase< - TKqpScanQueryStreamRequestHandler, - NKqp::TEvKqp::TEvQueryRequest, - NKqp::TEvKqp::TEvQueryResponse, - IKqpGateway::TQueryResult> -{ -public: - using TRequest = NKqp::TEvKqp::TEvQueryRequest; - using TResponse = NKqp::TEvKqp::TEvQueryResponse; - using TResult = IKqpGateway::TQueryResult; - - using TBase = TKqpScanQueryStreamRequestHandler::TBase; - - TKqpScanQueryStreamRequestHandler(TRequest* request, const TActorId& target, TPromise<TResult> promise, - TCallbackFunc callback) - : TBase(request, promise, callback) - , TargetActorId(target) {} - - void Bootstrap(const TActorContext& ctx) { - ActorIdToProto(SelfId(), this->Request->Record.MutableRequestActorId()); - - TActorId kqpProxy = MakeKqpProxyID(ctx.SelfID.NodeId()); - ctx.Send(kqpProxy, this->Request.Release()); - - this->Become(&TKqpScanQueryStreamRequestHandler::AwaitState); - } - - void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ctx); - TlsActivationContext->Send(ev->Forward(TargetActorId)); - } - - void Handle(NKqp::TEvKqpExecuter::TEvStreamDataAck::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ctx); - TlsActivationContext->Send(ev->Forward(ExecuterActorId)); - } - - void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) { - ExecuterActorId = ActorIdFromProto(ev->Get()->Record.GetExecuterActorId()); - ActorIdToProto(SelfId(), ev->Get()->Record.MutableExecuterActorId()); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() - << "Received executer progress for scan query, id: " << ExecuterActorId); - TlsActivationContext->Send(ev->Forward(TargetActorId)); - } - - void Handle(NKqp::TEvKqpExecuter::TEvStreamProfile::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ctx); - Executions.push_back(std::move(*ev->Get()->Record.MutableProfile())); - } - - void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { - const auto& kqpResponse = ev->Get()->Record; - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() + } + + using TBase::Promise; + using TBase::Callback; + + virtual void HandleResponse(typename TResponse::TPtr &ev, const TActorContext &ctx) { + auto& record = ev->Get()->Record.GetRef(); + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { + if (record.MutableResponse()->GetResults().size()) { + // Send result sets to RPC actor TStreamExecuteYqlScriptRPC + auto evStreamPart = MakeHolder<NKqp::TEvKqp::TEvDataQueryStreamPart>(); + ActorIdToProto(this->SelfId(), evStreamPart->Record.MutableGatewayActorId()); + + for (int i = 0; i < record.MutableResponse()->MutableResults()->size(); ++i) { + // Workaround to avoid errors on Pull execution stage which would expect some results + Ydb::ResultSet resultSet; + NKikimr::ConvertYdbResultToKqpResult(resultSet, *evStreamPart->Record.AddResults()); + } + + evStreamPart->Record.MutableResults()->Swap(record.MutableResponse()->MutableResults()); + this->Send(TargetActorId, evStreamPart.Release()); + + // Save response without data to send it later + ResponseHandle = ev.Release(); + } else { + // Response has no result sets. Forward to main pipeline + Callback(Promise, std::move(*ev->Get())); + this->Die(ctx); + } + } else { + // Forward error to main pipeline + Callback(Promise, std::move(*ev->Get())); + this->Die(ctx); + } + } + + void Handle(NKqp::TEvKqp::TEvDataQueryStreamPartAck::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + Callback(Promise, std::move(*ResponseHandle->Get())); + this->Die(ctx); + } + + void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, this->SelfId() + << "Received abort execution event for data query: " << record.GetMessage()); + + TBase::HandleError(record.GetMessage(), ctx); + } + + using TBase::Handle; + using TBase::HandleResponse; + + STFUNC(AwaitState) { + switch (ev->GetTypeRewrite()) { + HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); + HFunc(TResponse, HandleResponse); + HFunc(NKqp::TEvKqp::TEvDataQueryStreamPartAck, Handle); + HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle); + + default: + TBase::HandleUnexpectedEvent("TKqpStreamRequestHandler", ev->GetTypeRewrite(), ctx); + } + } + +private: + TActorId TargetActorId; + typename TResponse::TPtr ResponseHandle; +}; + +// Handles scan query request for StreamExecuteYqlScript +class TKqpScanQueryStreamRequestHandler : public TRequestHandlerBase< + TKqpScanQueryStreamRequestHandler, + NKqp::TEvKqp::TEvQueryRequest, + NKqp::TEvKqp::TEvQueryResponse, + IKqpGateway::TQueryResult> +{ +public: + using TRequest = NKqp::TEvKqp::TEvQueryRequest; + using TResponse = NKqp::TEvKqp::TEvQueryResponse; + using TResult = IKqpGateway::TQueryResult; + + using TBase = TKqpScanQueryStreamRequestHandler::TBase; + + TKqpScanQueryStreamRequestHandler(TRequest* request, const TActorId& target, TPromise<TResult> promise, + TCallbackFunc callback) + : TBase(request, promise, callback) + , TargetActorId(target) {} + + void Bootstrap(const TActorContext& ctx) { + ActorIdToProto(SelfId(), this->Request->Record.MutableRequestActorId()); + + TActorId kqpProxy = MakeKqpProxyID(ctx.SelfID.NodeId()); + ctx.Send(kqpProxy, this->Request.Release()); + + this->Become(&TKqpScanQueryStreamRequestHandler::AwaitState); + } + + void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + TlsActivationContext->Send(ev->Forward(TargetActorId)); + } + + void Handle(NKqp::TEvKqpExecuter::TEvStreamDataAck::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + TlsActivationContext->Send(ev->Forward(ExecuterActorId)); + } + + void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) { + ExecuterActorId = ActorIdFromProto(ev->Get()->Record.GetExecuterActorId()); + ActorIdToProto(SelfId(), ev->Get()->Record.MutableExecuterActorId()); + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() + << "Received executer progress for scan query, id: " << ExecuterActorId); + TlsActivationContext->Send(ev->Forward(TargetActorId)); + } + + void Handle(NKqp::TEvKqpExecuter::TEvStreamProfile::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + Executions.push_back(std::move(*ev->Get()->Record.MutableProfile())); + } + + void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { + const auto& kqpResponse = ev->Get()->Record; + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() << "Received process error for scan query: " << kqpResponse.GetError()); - + TBase::HandleError(kqpResponse.GetError(), ctx); - } - - void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() - << "Received abort execution event for scan query: " << record.GetMessage()); - - TBase::HandleError(record.GetMessage(), ctx); - } - - using TBase::HandleResponse; - - void HandleResponse(typename TResponse::TPtr &ev, const TActorContext &ctx) { - auto& response = *ev->Get()->Record.GetRef().MutableResponse(); - - Ydb::ResultSet resultSet; - NKikimr::ConvertYdbResultToKqpResult(resultSet, *response.AddResults()); - for (auto& execStats : Executions) { - response.MutableQueryStats()->AddExecutions()->Swap(&execStats); - } - Executions.clear(); - - TBase::HandleResponse(ev, ctx); - } - - STFUNC(AwaitState) { - switch (ev->GetTypeRewrite()) { - HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); - HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle); - HFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle); - HFunc(NKqp::TEvKqpExecuter::TEvStreamProfile, Handle); - HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle); - HFunc(TResponse, HandleResponse); - - default: - TBase::HandleUnexpectedEvent("TKqpScanQueryStreamRequestHandler", ev->GetTypeRewrite(), ctx); - } - } - -private: - TActorId ExecuterActorId; - TActorId TargetActorId; + } + + void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() + << "Received abort execution event for scan query: " << record.GetMessage()); + + TBase::HandleError(record.GetMessage(), ctx); + } + + using TBase::HandleResponse; + + void HandleResponse(typename TResponse::TPtr &ev, const TActorContext &ctx) { + auto& response = *ev->Get()->Record.GetRef().MutableResponse(); + + Ydb::ResultSet resultSet; + NKikimr::ConvertYdbResultToKqpResult(resultSet, *response.AddResults()); + for (auto& execStats : Executions) { + response.MutableQueryStats()->AddExecutions()->Swap(&execStats); + } + Executions.clear(); + + TBase::HandleResponse(ev, ctx); + } + + STFUNC(AwaitState) { + switch (ev->GetTypeRewrite()) { + HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle); + HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle); + HFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle); + HFunc(NKqp::TEvKqpExecuter::TEvStreamProfile, Handle); + HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle); + HFunc(TResponse, HandleResponse); + + default: + TBase::HandleUnexpectedEvent("TKqpScanQueryStreamRequestHandler", ev->GetTypeRewrite(), ctx); + } + } + +private: + TActorId ExecuterActorId; + TActorId TargetActorId; TVector<NYql::NDqProto::TDqExecutionStats> Executions; -}; - +}; + class TMkqlRequestHandler : public TRequestHandlerBase< TMkqlRequestHandler, TEvTxUserProxy::TEvProposeTransaction, @@ -963,13 +963,13 @@ void KqpResponseToQueryResult(const NKikimrKqp::TEvQueryResponse& response, IKqp queryResult.QueryStats = queryResponse.GetQueryStats(); } -namespace { - struct TSendRoleWrapper : public TThrRefBase { - using TMethod = std::function<void(TString&&, NYql::TAlterGroupSettings::EAction, std::vector<TString>&&)>; - TMethod SendNextRole; - }; -} - +namespace { + struct TSendRoleWrapper : public TThrRefBase { + using TMethod = std::function<void(TString&&, NYql::TAlterGroupSettings::EAction, std::vector<TString>&&)>; + TMethod SendNextRole; + }; +} + class TKikimrIcGateway : public IKqpGateway { private: struct TUserTokenData { @@ -1171,39 +1171,39 @@ public: indexDesc->AddDataColumnNames(col); } } - FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); + FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); } else { schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateTable); tableDesc = schemeTx.MutableCreateTable(); - FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); + FillCreateTableColumnDesc(*tableDesc, pathPair.second, metadata); } Ydb::StatusIds::StatusCode code; TString error; - TList<TString> warnings; - - if (!FillCreateTableDesc(metadata, *tableDesc, profiles, code, error, warnings)) { + TList<TString> warnings; + + if (!FillCreateTableDesc(metadata, *tableDesc, profiles, code, error, warnings)) { IKqpGateway::TGenericResult errResult; errResult.AddIssue(NYql::TIssue(error)); - errResult.SetStatus(NYql::YqlStatusFromYdbStatus(code)); + errResult.SetStatus(NYql::YqlStatusFromYdbStatus(code)); tablePromise.SetValue(errResult); return; } SendSchemeRequest(ev.Release()).Apply( - [tablePromise, warnings{std::move(warnings)}](const TFuture<TGenericResult>& future) mutable { - if (warnings.size()) { - auto result = future.GetValue(); - for (const auto& warning : warnings) { - result.AddIssue( - NYql::TIssue(warning) - .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) - ); - tablePromise.SetValue(result); - } - } else { - tablePromise.SetValue(future.GetValue()); - } + [tablePromise, warnings{std::move(warnings)}](const TFuture<TGenericResult>& future) mutable { + if (warnings.size()) { + auto result = future.GetValue(); + for (const auto& warning : warnings) { + result.AddIssue( + NYql::TIssue(warning) + .SetCode(NKikimrIssues::TIssuesIds::WARNING, NYql::TSeverityIds::S_WARNING) + ); + tablePromise.SetValue(result); + } + } else { + tablePromise.SetValue(future.GetValue()); + } }); }); @@ -1285,315 +1285,315 @@ public: } } - TFuture<TGenericResult> CreateUser(const TString& cluster, const NYql::TCreateUserSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - auto createUserPromise = NewPromise<TGenericResult>(); - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - auto& createUser = *schemeTx.MutableAlterLogin()->MutableCreateUser(); - - createUser.SetUser(settings.UserName); - if (settings.Password) { - createUser.SetPassword(settings.Password); - } - - SendSchemeRequest(ev.Release()).Apply( - [createUserPromise](const TFuture<TGenericResult>& future) mutable { - createUserPromise.SetValue(future.GetValue()); - } - ); - - return createUserPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> AlterUser(const TString& cluster, const NYql::TAlterUserSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - auto alterUserPromise = NewPromise<TGenericResult>(); - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - auto& alterUser = *schemeTx.MutableAlterLogin()->MutableModifyUser(); - - alterUser.SetUser(settings.UserName); - if (settings.Password) { - alterUser.SetPassword(settings.Password); - } - - SendSchemeRequest(ev.Release()).Apply( - [alterUserPromise](const TFuture<TGenericResult>& future) mutable { - alterUserPromise.SetValue(future.GetValue()); - } - ); - - return alterUserPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> DropUser(const TString& cluster, const NYql::TDropUserSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - auto dropUserPromise = NewPromise<TGenericResult>(); - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - auto& dropUser = *schemeTx.MutableAlterLogin()->MutableRemoveUser(); - - dropUser.SetUser(settings.UserName); - - SendSchemeRequest(ev.Release()).Apply( - [dropUserPromise, &settings](const TFuture<TGenericResult>& future) mutable { - const auto& realResult = future.GetValue(); - if (!realResult.Success() && realResult.Status() == TIssuesIds::DEFAULT_ERROR && settings.Force) { - IKqpGateway::TGenericResult fakeResult; - fakeResult.SetSuccess(); - dropUserPromise.SetValue(std::move(fakeResult)); - } else { - dropUserPromise.SetValue(realResult); - } - } - ); - - return dropUserPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> CreateGroup(const TString& cluster, const NYql::TCreateGroupSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - auto createGroupPromise = NewPromise<TGenericResult>(); - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - auto& createGroup = *schemeTx.MutableAlterLogin()->MutableCreateGroup(); - - createGroup.SetGroup(settings.GroupName); - - SendSchemeRequest(ev.Release()).Apply( - [createGroupPromise](const TFuture<TGenericResult>& future) mutable { - createGroupPromise.SetValue(future.GetValue()); - } - ); - - return createGroupPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> AlterGroup(const TString& cluster, NYql::TAlterGroupSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - if (!settings.Roles.size()) { - return MakeFuture(ResultFromError<TGenericResult>("No roles given for AlterGroup request")); - } - - TPromise<TGenericResult> alterGroupPromise = NewPromise<TGenericResult>(); - - auto sendRoleWrapper = MakeIntrusive<TSendRoleWrapper>(); - - sendRoleWrapper->SendNextRole = [alterGroupPromise, sendRoleWrapper, this, database = std::move(database)] - (TString&& groupName, NYql::TAlterGroupSettings::EAction action, std::vector<TString>&& rolesToSend) - mutable - { - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - switch (action) { - case NYql::TAlterGroupSettings::EAction::AddRoles: - { - auto& alterGroup = *schemeTx.MutableAlterLogin()->MutableAddGroupMembership(); - alterGroup.SetGroup(groupName); - alterGroup.SetMember(*rolesToSend.begin()); - break; - } - case NYql::TAlterGroupSettings::EAction::RemoveRoles: - { - auto& alterGroup = *schemeTx.MutableAlterLogin()->MutableRemoveGroupMembership(); - alterGroup.SetGroup(groupName); - alterGroup.SetMember(*rolesToSend.begin()); - break; - } - default: - break; - } - - std::vector<TString> restOfRoles( - std::make_move_iterator(rolesToSend.begin() + 1), - std::make_move_iterator(rolesToSend.end()) - ); - - SendSchemeRequest(ev.Release()).Apply( - [alterGroupPromise, sendRoleWrapper, groupName = std::move(groupName), action, restOfRoles = std::move(restOfRoles)] - (const TFuture<TGenericResult>& future) mutable - { - auto result = future.GetValue(); - if (!result.Success()) { - alterGroupPromise.SetValue(result); - return; - } - if (restOfRoles.size()) { - try { - sendRoleWrapper->SendNextRole(std::move(groupName), action, std::move(restOfRoles)); - } - catch (yexception& e) { - return alterGroupPromise.SetValue(ResultFromException<TGenericResult>(e)); - } - } else { - alterGroupPromise.SetValue(result); - } - } - ); - }; - - sendRoleWrapper->SendNextRole(std::move(settings.GroupName), settings.Action, std::move(settings.Roles)); - - return alterGroupPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - - TFuture<TGenericResult> DropGroup(const TString& cluster, const NYql::TDropGroupSettings& settings) override { - using TRequest = TEvTxUserProxy::TEvProposeTransaction; - - try { - if (!CheckCluster(cluster)) { - return InvalidCluster<TGenericResult>(cluster); - } - - TString database; - if (!GetDatabaseForLoginOperation(database)) { - return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); - } - - auto dropGroupPromise = NewPromise<TGenericResult>(); - - auto ev = MakeHolder<TRequest>(); - ev->Record.SetDatabaseName(database); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); - schemeTx.SetWorkingDir(database); - schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); - auto& dropGroup = *schemeTx.MutableAlterLogin()->MutableRemoveGroup(); - - dropGroup.SetGroup(settings.GroupName); - - SendSchemeRequest(ev.Release()).Apply( - [dropGroupPromise, &settings](const TFuture<TGenericResult>& future) mutable { - const auto& realResult = future.GetValue(); - if (!realResult.Success() && realResult.Status() == TIssuesIds::DEFAULT_ERROR && settings.Force) { - IKqpGateway::TGenericResult fakeResult; - fakeResult.SetSuccess(); - dropGroupPromise.SetValue(std::move(fakeResult)); - } else { - dropGroupPromise.SetValue(realResult); - } - } - ); - - return dropGroupPromise.GetFuture(); - } - catch (yexception& e) { - return MakeFuture(ResultFromException<TGenericResult>(e)); - } - } - + TFuture<TGenericResult> CreateUser(const TString& cluster, const NYql::TCreateUserSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + auto createUserPromise = NewPromise<TGenericResult>(); + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + auto& createUser = *schemeTx.MutableAlterLogin()->MutableCreateUser(); + + createUser.SetUser(settings.UserName); + if (settings.Password) { + createUser.SetPassword(settings.Password); + } + + SendSchemeRequest(ev.Release()).Apply( + [createUserPromise](const TFuture<TGenericResult>& future) mutable { + createUserPromise.SetValue(future.GetValue()); + } + ); + + return createUserPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> AlterUser(const TString& cluster, const NYql::TAlterUserSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + auto alterUserPromise = NewPromise<TGenericResult>(); + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + auto& alterUser = *schemeTx.MutableAlterLogin()->MutableModifyUser(); + + alterUser.SetUser(settings.UserName); + if (settings.Password) { + alterUser.SetPassword(settings.Password); + } + + SendSchemeRequest(ev.Release()).Apply( + [alterUserPromise](const TFuture<TGenericResult>& future) mutable { + alterUserPromise.SetValue(future.GetValue()); + } + ); + + return alterUserPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> DropUser(const TString& cluster, const NYql::TDropUserSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + auto dropUserPromise = NewPromise<TGenericResult>(); + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + auto& dropUser = *schemeTx.MutableAlterLogin()->MutableRemoveUser(); + + dropUser.SetUser(settings.UserName); + + SendSchemeRequest(ev.Release()).Apply( + [dropUserPromise, &settings](const TFuture<TGenericResult>& future) mutable { + const auto& realResult = future.GetValue(); + if (!realResult.Success() && realResult.Status() == TIssuesIds::DEFAULT_ERROR && settings.Force) { + IKqpGateway::TGenericResult fakeResult; + fakeResult.SetSuccess(); + dropUserPromise.SetValue(std::move(fakeResult)); + } else { + dropUserPromise.SetValue(realResult); + } + } + ); + + return dropUserPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> CreateGroup(const TString& cluster, const NYql::TCreateGroupSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + auto createGroupPromise = NewPromise<TGenericResult>(); + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + auto& createGroup = *schemeTx.MutableAlterLogin()->MutableCreateGroup(); + + createGroup.SetGroup(settings.GroupName); + + SendSchemeRequest(ev.Release()).Apply( + [createGroupPromise](const TFuture<TGenericResult>& future) mutable { + createGroupPromise.SetValue(future.GetValue()); + } + ); + + return createGroupPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> AlterGroup(const TString& cluster, NYql::TAlterGroupSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + if (!settings.Roles.size()) { + return MakeFuture(ResultFromError<TGenericResult>("No roles given for AlterGroup request")); + } + + TPromise<TGenericResult> alterGroupPromise = NewPromise<TGenericResult>(); + + auto sendRoleWrapper = MakeIntrusive<TSendRoleWrapper>(); + + sendRoleWrapper->SendNextRole = [alterGroupPromise, sendRoleWrapper, this, database = std::move(database)] + (TString&& groupName, NYql::TAlterGroupSettings::EAction action, std::vector<TString>&& rolesToSend) + mutable + { + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + switch (action) { + case NYql::TAlterGroupSettings::EAction::AddRoles: + { + auto& alterGroup = *schemeTx.MutableAlterLogin()->MutableAddGroupMembership(); + alterGroup.SetGroup(groupName); + alterGroup.SetMember(*rolesToSend.begin()); + break; + } + case NYql::TAlterGroupSettings::EAction::RemoveRoles: + { + auto& alterGroup = *schemeTx.MutableAlterLogin()->MutableRemoveGroupMembership(); + alterGroup.SetGroup(groupName); + alterGroup.SetMember(*rolesToSend.begin()); + break; + } + default: + break; + } + + std::vector<TString> restOfRoles( + std::make_move_iterator(rolesToSend.begin() + 1), + std::make_move_iterator(rolesToSend.end()) + ); + + SendSchemeRequest(ev.Release()).Apply( + [alterGroupPromise, sendRoleWrapper, groupName = std::move(groupName), action, restOfRoles = std::move(restOfRoles)] + (const TFuture<TGenericResult>& future) mutable + { + auto result = future.GetValue(); + if (!result.Success()) { + alterGroupPromise.SetValue(result); + return; + } + if (restOfRoles.size()) { + try { + sendRoleWrapper->SendNextRole(std::move(groupName), action, std::move(restOfRoles)); + } + catch (yexception& e) { + return alterGroupPromise.SetValue(ResultFromException<TGenericResult>(e)); + } + } else { + alterGroupPromise.SetValue(result); + } + } + ); + }; + + sendRoleWrapper->SendNextRole(std::move(settings.GroupName), settings.Action, std::move(settings.Roles)); + + return alterGroupPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> DropGroup(const TString& cluster, const NYql::TDropGroupSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + TString database; + if (!GetDatabaseForLoginOperation(database)) { + return MakeFuture(ResultFromError<TGenericResult>("Couldn't get domain name")); + } + + auto dropGroupPromise = NewPromise<TGenericResult>(); + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(database); + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterLogin); + auto& dropGroup = *schemeTx.MutableAlterLogin()->MutableRemoveGroup(); + + dropGroup.SetGroup(settings.GroupName); + + SendSchemeRequest(ev.Release()).Apply( + [dropGroupPromise, &settings](const TFuture<TGenericResult>& future) mutable { + const auto& realResult = future.GetValue(); + if (!realResult.Success() && realResult.Status() == TIssuesIds::DEFAULT_ERROR && settings.Force) { + IKqpGateway::TGenericResult fakeResult; + fakeResult.SetSuccess(); + dropGroupPromise.SetValue(std::move(fakeResult)); + } else { + dropGroupPromise.SetValue(realResult); + } + } + ); + + return dropGroupPromise.GetFuture(); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + TFuture<TMkqlResult> ExecuteMkql(const TString& cluster, const TString& program, TKqpParamsMap&& params, const TMkqlSettings& settings, const TKqpSnapshot& snapshot) override { @@ -1670,78 +1670,78 @@ public: }); } - TFuture<TQueryResult> StreamExecDataQueryAst(const TString& cluster, const TString& query, + TFuture<TQueryResult> StreamExecDataQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, - const Ydb::Table::TransactionSettings& txSettings, const NActors::TActorId& target) override - { - using TRequest = NKqp::TEvKqp::TEvQueryRequest; - using TResponse = NKqp::TEvKqp::TEvQueryResponse; - - auto ev = MakeHolder<TRequest>(); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - - ev->Record.MutableRequest()->SetDatabase(Database); - ev->Record.MutableRequest()->SetCluster(cluster); - ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_DML); - ev->Record.MutableRequest()->SetQuery(query); - ev->Record.MutableRequest()->SetKeepSession(false); - ev->Record.MutableRequest()->SetStatsMode(settings.StatsMode); - + const Ydb::Table::TransactionSettings& txSettings, const NActors::TActorId& target) override + { + using TRequest = NKqp::TEvKqp::TEvQueryRequest; + using TResponse = NKqp::TEvKqp::TEvQueryResponse; + + auto ev = MakeHolder<TRequest>(); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + + ev->Record.MutableRequest()->SetDatabase(Database); + ev->Record.MutableRequest()->SetCluster(cluster); + ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_DML); + ev->Record.MutableRequest()->SetQuery(query); + ev->Record.MutableRequest()->SetKeepSession(false); + ev->Record.MutableRequest()->SetStatsMode(settings.StatsMode); + if (!params.Values.empty()) { - FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters()); - } - + FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters()); + } + //auto& querySettings = *ev->Record.MutableRequest()->MutableQuerySettings(); //querySettings.set_use_new_engine(NYql::GetFlagValue(settings.UseNewEngine)); - - auto& txControl = *ev->Record.MutableRequest()->MutableTxControl(); - txControl.mutable_begin_tx()->CopyFrom(txSettings); - txControl.set_commit_tx(true); - - return SendKqpStreamRequest<TRequest, TResponse, TQueryResult>(ev.Release(), target, - [](TPromise<TQueryResult> promise, TResponse&& responseEv) { - TQueryResult queryResult; - queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); - KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); + + auto& txControl = *ev->Record.MutableRequest()->MutableTxControl(); + txControl.mutable_begin_tx()->CopyFrom(txSettings); + txControl.set_commit_tx(true); + + return SendKqpStreamRequest<TRequest, TResponse, TQueryResult>(ev.Release(), target, + [](TPromise<TQueryResult> promise, TResponse&& responseEv) { + TQueryResult queryResult; + queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); + KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); promise.SetValue(std::move(queryResult)); - }); - } - - TFuture<TQueryResult> StreamExecScanQueryAst(const TString& cluster, const TString& query, + }); + } + + TFuture<TQueryResult> StreamExecScanQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, const NActors::TActorId& target) override - { - using TRequest = NKqp::TEvKqp::TEvQueryRequest; - using TResponse = NKqp::TEvKqp::TEvQueryResponse; - - auto ev = MakeHolder<TRequest>(); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - - ev->Record.MutableRequest()->SetDatabase(Database); - ev->Record.MutableRequest()->SetCluster(cluster); - ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN); - ev->Record.MutableRequest()->SetQuery(query); - ev->Record.MutableRequest()->SetKeepSession(false); - ev->Record.MutableRequest()->SetStatsMode(settings.StatsMode); - + { + using TRequest = NKqp::TEvKqp::TEvQueryRequest; + using TResponse = NKqp::TEvKqp::TEvQueryResponse; + + auto ev = MakeHolder<TRequest>(); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + + ev->Record.MutableRequest()->SetDatabase(Database); + ev->Record.MutableRequest()->SetCluster(cluster); + ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN); + ev->Record.MutableRequest()->SetQuery(query); + ev->Record.MutableRequest()->SetKeepSession(false); + ev->Record.MutableRequest()->SetStatsMode(settings.StatsMode); + if (!params.Values.empty()) { - FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters()); - } - - return SendKqpScanQueryStreamRequest(ev.Release(), target, - [](TPromise<TQueryResult> promise, TResponse&& responseEv) { - TQueryResult queryResult; - queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); - KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); + FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters()); + } + + return SendKqpScanQueryStreamRequest(ev.Release(), target, + [](TPromise<TQueryResult> promise, TResponse&& responseEv) { + TQueryResult queryResult; + queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); + KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); promise.SetValue(std::move(queryResult)); - }); - } - + }); + } + TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) override { using TRequest = NKqp::TEvKqp::TEvQueryRequest; @@ -1954,28 +1954,28 @@ private: } template<typename TRequest, typename TResponse, typename TResult> - TFuture<TResult> SendKqpStreamRequest(TRequest* request, const NActors::TActorId& target, - typename TKqpStreamRequestHandler<TRequest, TResponse, TResult>::TCallbackFunc callback) - { - auto promise = NewPromise<TResult>(); - IActor* requestHandler = new TKqpStreamRequestHandler<TRequest, TResponse, TResult>(request, - target, promise, callback); - RegisterActor(requestHandler); - - return promise.GetFuture(); - } - - TFuture<TQueryResult> SendKqpScanQueryStreamRequest(NKqp::TEvKqp::TEvQueryRequest* request, - const NActors::TActorId& target, TKqpScanQueryStreamRequestHandler::TCallbackFunc callback) - { - auto promise = NewPromise<TQueryResult>(); - IActor* requestHandler = new TKqpScanQueryStreamRequestHandler(request, target, promise, callback); - RegisterActor(requestHandler); - - return promise.GetFuture(); - } - - template<typename TRequest, typename TResponse, typename TResult> + TFuture<TResult> SendKqpStreamRequest(TRequest* request, const NActors::TActorId& target, + typename TKqpStreamRequestHandler<TRequest, TResponse, TResult>::TCallbackFunc callback) + { + auto promise = NewPromise<TResult>(); + IActor* requestHandler = new TKqpStreamRequestHandler<TRequest, TResponse, TResult>(request, + target, promise, callback); + RegisterActor(requestHandler); + + return promise.GetFuture(); + } + + TFuture<TQueryResult> SendKqpScanQueryStreamRequest(NKqp::TEvKqp::TEvQueryRequest* request, + const NActors::TActorId& target, TKqpScanQueryStreamRequestHandler::TCallbackFunc callback) + { + auto promise = NewPromise<TQueryResult>(); + IActor* requestHandler = new TKqpScanQueryStreamRequestHandler(request, target, promise, callback); + RegisterActor(requestHandler); + + return promise.GetFuture(); + } + + template<typename TRequest, typename TResponse, typename TResult> TFuture<TResult> SendActorRequest(const TActorId& actorId, TRequest* request, typename TActorRequestHandler<TRequest, TResponse, TResult>::TCallbackFunc callback) { @@ -2138,20 +2138,20 @@ private: return promise.GetFuture(); } - bool GetDatabaseForLoginOperation(TString& database) { - TAppData* appData = AppData(ActorSystem); - if (appData && appData->AuthConfig.GetDomainLoginOnly()) { - if (appData->DomainsInfo && !appData->DomainsInfo->Domains.empty()) { - database = "/" + appData->DomainsInfo->Domains.begin()->second->Name; - return true; - } - } else { - database = Database; - return true; - } - return false; - } - + bool GetDatabaseForLoginOperation(TString& database) { + TAppData* appData = AppData(ActorSystem); + if (appData && appData->AuthConfig.GetDomainLoginOnly()) { + if (appData->DomainsInfo && !appData->DomainsInfo->Domains.empty()) { + database = "/" + appData->DomainsInfo->Domains.begin()->second->Name; + return true; + } + } else { + database = Database; + return true; + } + return false; + } + private: static TRunResponse GetRunResponse(NKikimrTxUserProxy::TEvProposeTransactionStatus&& ev) { IKqpGateway::TRunResponse response; @@ -2206,7 +2206,7 @@ private: } static void FillCreateTableColumnDesc(NKikimrSchemeOp::TTableDescription& tableDesc, - const TString& name, NYql::TKikimrTableMetadataPtr metadata) + const TString& name, NYql::TKikimrTableMetadataPtr metadata) { tableDesc.SetName(name); @@ -2221,7 +2221,7 @@ private: columnDesc.SetNotNull(columnIt->second.NotNull); if (columnIt->second.Families) { columnDesc.SetFamilyName(*columnIt->second.Families.begin()); - } + } } for (TString& keyColumn : metadata->KeyColumnNames) { @@ -2372,154 +2372,154 @@ private: } } - static bool ConvertDataSlotToYdbTypedValue(NYql::EDataSlot fromType, const TString& fromValue, Ydb::Type* toType, - Ydb::Value* toValue) - { - switch (fromType) { - case NYql::EDataSlot::Bool: - toType->set_type_id(Ydb::Type::BOOL); - toValue->set_bool_value(FromString<bool>(fromValue)); - break; - case NYql::EDataSlot::Int8: - toType->set_type_id(Ydb::Type::INT8); - toValue->set_int32_value(FromString<i32>(fromValue)); - break; - case NYql::EDataSlot::Uint8: - toType->set_type_id(Ydb::Type::UINT8); - toValue->set_uint32_value(FromString<ui32>(fromValue)); - break; - case NYql::EDataSlot::Int16: - toType->set_type_id(Ydb::Type::INT16); - toValue->set_int32_value(FromString<i32>(fromValue)); - break; - case NYql::EDataSlot::Uint16: - toType->set_type_id(Ydb::Type::UINT16); - toValue->set_uint32_value(FromString<ui32>(fromValue)); - break; - case NYql::EDataSlot::Int32: - toType->set_type_id(Ydb::Type::INT32); - toValue->set_int32_value(FromString<i32>(fromValue)); - break; - case NYql::EDataSlot::Uint32: - toType->set_type_id(Ydb::Type::UINT32); - toValue->set_uint32_value(FromString<ui32>(fromValue)); - break; - case NYql::EDataSlot::Uint64: - toType->set_type_id(Ydb::Type::UINT64); - toValue->set_uint64_value(FromString<ui64>(fromValue)); - break; - case NYql::EDataSlot::Float: - toType->set_type_id(Ydb::Type::FLOAT); - toValue->set_float_value(FromString<float>(fromValue)); - break; - case NYql::EDataSlot::Double: - toType->set_type_id(Ydb::Type::DOUBLE); - toValue->set_double_value(FromString<double>(fromValue)); - break; - case NYql::EDataSlot::Json: - toType->set_type_id(Ydb::Type::JSON); - toValue->set_text_value(fromValue); - break; - case NYql::EDataSlot::String: - toType->set_type_id(Ydb::Type::STRING); - toValue->set_bytes_value(fromValue); - break; - case NYql::EDataSlot::Utf8: - toType->set_type_id(Ydb::Type::UTF8); - toValue->set_text_value(fromValue); - break; - case NYql::EDataSlot::Date: - toType->set_type_id(Ydb::Type::DATE); - toValue->set_uint32_value(FromString<ui32>(fromValue)); - break; - case NYql::EDataSlot::Datetime: - toType->set_type_id(Ydb::Type::DATETIME); - toValue->set_uint32_value(FromString<ui32>(fromValue)); - break; - case NYql::EDataSlot::Timestamp: - toType->set_type_id(Ydb::Type::TIMESTAMP); - toValue->set_uint64_value(FromString<ui64>(fromValue)); - break; - case NYql::EDataSlot::Interval: - toType->set_type_id(Ydb::Type::INTERVAL); - toValue->set_int64_value(FromString<i64>(fromValue)); - break; - default: - return false; - } - return true; - } - - - // Convert TKikimrTableMetadata struct to public API proto - static bool ConvertCreateTableSettingsToProto(NYql::TKikimrTableMetadataPtr metadata, - Ydb::Table::CreateTableRequest& proto, Ydb::StatusIds::StatusCode& code, TString& error) - { - for (const auto& family : metadata->ColumnFamilies) { - auto* familyProto = proto.add_column_families(); - familyProto->set_name(family.Name); - if (family.Data) { - familyProto->mutable_data()->set_media(family.Data.GetRef()); - } - if (family.Compression) { - if (to_lower(family.Compression.GetRef()) == "off") { - familyProto->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); - } else if (to_lower(family.Compression.GetRef()) == "lz4") { - familyProto->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_LZ4); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown compression '" << family.Compression << "' for a column family"; - return false; - } - } - } - - if (metadata->TableSettings.CompactionPolicy) { - proto.set_compaction_policy(metadata->TableSettings.CompactionPolicy.GetRef()); - } - - if (metadata->TableSettings.PartitionBy) { - if (metadata->TableSettings.PartitionBy.size() > metadata->KeyColumnNames.size()) { - code = Ydb::StatusIds::BAD_REQUEST; - error = "\"Partition by\" contains more columns than primary key does"; - return false; - } else if (metadata->TableSettings.PartitionBy.size() == metadata->KeyColumnNames.size()) { - for (size_t i = 0; i < metadata->TableSettings.PartitionBy.size(); ++i) { - if (metadata->TableSettings.PartitionBy[i] != metadata->KeyColumnNames[i]) { - code = Ydb::StatusIds::BAD_REQUEST; - error = "\"Partition by\" doesn't match primary key"; - return false; - } - } - } else { - code = Ydb::StatusIds::UNSUPPORTED; - error = "\"Partition by\" is not supported yet"; - return false; - } - } - - if (metadata->TableSettings.AutoPartitioningBySize) { - auto& partitioningSettings = *proto.mutable_partitioning_settings(); - TString value = to_lower(metadata->TableSettings.AutoPartitioningBySize.GetRef()); - if (value == "enabled") { - partitioningSettings.set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); - } else if (value == "disabled") { - partitioningSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown feature flag '" - << metadata->TableSettings.AutoPartitioningBySize.GetRef() - << "' for auto partitioning by size"; - return false; - } - } - - if (metadata->TableSettings.PartitionSizeMb) { - auto& partitioningSettings = *proto.mutable_partitioning_settings(); - partitioningSettings.set_partition_size_mb(metadata->TableSettings.PartitionSizeMb.GetRef()); - } - - if (metadata->TableSettings.AutoPartitioningByLoad) { + static bool ConvertDataSlotToYdbTypedValue(NYql::EDataSlot fromType, const TString& fromValue, Ydb::Type* toType, + Ydb::Value* toValue) + { + switch (fromType) { + case NYql::EDataSlot::Bool: + toType->set_type_id(Ydb::Type::BOOL); + toValue->set_bool_value(FromString<bool>(fromValue)); + break; + case NYql::EDataSlot::Int8: + toType->set_type_id(Ydb::Type::INT8); + toValue->set_int32_value(FromString<i32>(fromValue)); + break; + case NYql::EDataSlot::Uint8: + toType->set_type_id(Ydb::Type::UINT8); + toValue->set_uint32_value(FromString<ui32>(fromValue)); + break; + case NYql::EDataSlot::Int16: + toType->set_type_id(Ydb::Type::INT16); + toValue->set_int32_value(FromString<i32>(fromValue)); + break; + case NYql::EDataSlot::Uint16: + toType->set_type_id(Ydb::Type::UINT16); + toValue->set_uint32_value(FromString<ui32>(fromValue)); + break; + case NYql::EDataSlot::Int32: + toType->set_type_id(Ydb::Type::INT32); + toValue->set_int32_value(FromString<i32>(fromValue)); + break; + case NYql::EDataSlot::Uint32: + toType->set_type_id(Ydb::Type::UINT32); + toValue->set_uint32_value(FromString<ui32>(fromValue)); + break; + case NYql::EDataSlot::Uint64: + toType->set_type_id(Ydb::Type::UINT64); + toValue->set_uint64_value(FromString<ui64>(fromValue)); + break; + case NYql::EDataSlot::Float: + toType->set_type_id(Ydb::Type::FLOAT); + toValue->set_float_value(FromString<float>(fromValue)); + break; + case NYql::EDataSlot::Double: + toType->set_type_id(Ydb::Type::DOUBLE); + toValue->set_double_value(FromString<double>(fromValue)); + break; + case NYql::EDataSlot::Json: + toType->set_type_id(Ydb::Type::JSON); + toValue->set_text_value(fromValue); + break; + case NYql::EDataSlot::String: + toType->set_type_id(Ydb::Type::STRING); + toValue->set_bytes_value(fromValue); + break; + case NYql::EDataSlot::Utf8: + toType->set_type_id(Ydb::Type::UTF8); + toValue->set_text_value(fromValue); + break; + case NYql::EDataSlot::Date: + toType->set_type_id(Ydb::Type::DATE); + toValue->set_uint32_value(FromString<ui32>(fromValue)); + break; + case NYql::EDataSlot::Datetime: + toType->set_type_id(Ydb::Type::DATETIME); + toValue->set_uint32_value(FromString<ui32>(fromValue)); + break; + case NYql::EDataSlot::Timestamp: + toType->set_type_id(Ydb::Type::TIMESTAMP); + toValue->set_uint64_value(FromString<ui64>(fromValue)); + break; + case NYql::EDataSlot::Interval: + toType->set_type_id(Ydb::Type::INTERVAL); + toValue->set_int64_value(FromString<i64>(fromValue)); + break; + default: + return false; + } + return true; + } + + + // Convert TKikimrTableMetadata struct to public API proto + static bool ConvertCreateTableSettingsToProto(NYql::TKikimrTableMetadataPtr metadata, + Ydb::Table::CreateTableRequest& proto, Ydb::StatusIds::StatusCode& code, TString& error) + { + for (const auto& family : metadata->ColumnFamilies) { + auto* familyProto = proto.add_column_families(); + familyProto->set_name(family.Name); + if (family.Data) { + familyProto->mutable_data()->set_media(family.Data.GetRef()); + } + if (family.Compression) { + if (to_lower(family.Compression.GetRef()) == "off") { + familyProto->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); + } else if (to_lower(family.Compression.GetRef()) == "lz4") { + familyProto->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_LZ4); + } else { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown compression '" << family.Compression << "' for a column family"; + return false; + } + } + } + + if (metadata->TableSettings.CompactionPolicy) { + proto.set_compaction_policy(metadata->TableSettings.CompactionPolicy.GetRef()); + } + + if (metadata->TableSettings.PartitionBy) { + if (metadata->TableSettings.PartitionBy.size() > metadata->KeyColumnNames.size()) { + code = Ydb::StatusIds::BAD_REQUEST; + error = "\"Partition by\" contains more columns than primary key does"; + return false; + } else if (metadata->TableSettings.PartitionBy.size() == metadata->KeyColumnNames.size()) { + for (size_t i = 0; i < metadata->TableSettings.PartitionBy.size(); ++i) { + if (metadata->TableSettings.PartitionBy[i] != metadata->KeyColumnNames[i]) { + code = Ydb::StatusIds::BAD_REQUEST; + error = "\"Partition by\" doesn't match primary key"; + return false; + } + } + } else { + code = Ydb::StatusIds::UNSUPPORTED; + error = "\"Partition by\" is not supported yet"; + return false; + } + } + + if (metadata->TableSettings.AutoPartitioningBySize) { + auto& partitioningSettings = *proto.mutable_partitioning_settings(); + TString value = to_lower(metadata->TableSettings.AutoPartitioningBySize.GetRef()); + if (value == "enabled") { + partitioningSettings.set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); + } else if (value == "disabled") { + partitioningSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); + } else { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown feature flag '" + << metadata->TableSettings.AutoPartitioningBySize.GetRef() + << "' for auto partitioning by size"; + return false; + } + } + + if (metadata->TableSettings.PartitionSizeMb) { + auto& partitioningSettings = *proto.mutable_partitioning_settings(); + partitioningSettings.set_partition_size_mb(metadata->TableSettings.PartitionSizeMb.GetRef()); + } + + if (metadata->TableSettings.AutoPartitioningByLoad) { auto& partitioningSettings = *proto.mutable_partitioning_settings(); TString value = to_lower(metadata->TableSettings.AutoPartitioningByLoad.GetRef()); if (value == "enabled") { @@ -2533,68 +2533,68 @@ private: << "' for auto partitioning by load"; return false; } - } - - if (metadata->TableSettings.MinPartitions) { - auto& partitioningSettings = *proto.mutable_partitioning_settings(); - partitioningSettings.set_min_partitions_count(metadata->TableSettings.MinPartitions.GetRef()); - } - - if (metadata->TableSettings.MaxPartitions) { - auto& partitioningSettings = *proto.mutable_partitioning_settings(); - partitioningSettings.set_max_partitions_count(metadata->TableSettings.MaxPartitions.GetRef()); - } - - if (metadata->TableSettings.UniformPartitions) { - if (metadata->TableSettings.PartitionAtKeys) { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Uniform partitions and partitions at keys settings are mutually exclusive." - << " Use either one of them."; - return false; - } - proto.set_uniform_partitions(metadata->TableSettings.UniformPartitions.GetRef()); - } - - if (metadata->TableSettings.PartitionAtKeys) { - auto* borders = proto.mutable_partition_at_keys(); - for (const auto& splitPoint : metadata->TableSettings.PartitionAtKeys) { - auto* border = borders->Addsplit_points(); - auto &keyType = *border->mutable_type()->mutable_tuple_type(); - for (const auto& key : splitPoint) { - auto* type = keyType.add_elements()->mutable_optional_type()->mutable_item(); - auto* value = border->mutable_value()->add_items(); - if (!ConvertDataSlotToYdbTypedValue(key.first, key.second, type, value)) { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unsupported type for PartitionAtKeys: '" - << key.first << "'"; - return false; - } - } - } - } - - if (metadata->TableSettings.KeyBloomFilter) { - TString value = to_lower(metadata->TableSettings.KeyBloomFilter.GetRef()); - if (value == "enabled") { - proto.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); - } else if (value == "disabled") { - proto.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown feature flag '" - << metadata->TableSettings.KeyBloomFilter.GetRef() - << "' for key bloom filter"; - return false; - } - } - - if (metadata->TableSettings.ReadReplicasSettings) { + } + + if (metadata->TableSettings.MinPartitions) { + auto& partitioningSettings = *proto.mutable_partitioning_settings(); + partitioningSettings.set_min_partitions_count(metadata->TableSettings.MinPartitions.GetRef()); + } + + if (metadata->TableSettings.MaxPartitions) { + auto& partitioningSettings = *proto.mutable_partitioning_settings(); + partitioningSettings.set_max_partitions_count(metadata->TableSettings.MaxPartitions.GetRef()); + } + + if (metadata->TableSettings.UniformPartitions) { + if (metadata->TableSettings.PartitionAtKeys) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Uniform partitions and partitions at keys settings are mutually exclusive." + << " Use either one of them."; + return false; + } + proto.set_uniform_partitions(metadata->TableSettings.UniformPartitions.GetRef()); + } + + if (metadata->TableSettings.PartitionAtKeys) { + auto* borders = proto.mutable_partition_at_keys(); + for (const auto& splitPoint : metadata->TableSettings.PartitionAtKeys) { + auto* border = borders->Addsplit_points(); + auto &keyType = *border->mutable_type()->mutable_tuple_type(); + for (const auto& key : splitPoint) { + auto* type = keyType.add_elements()->mutable_optional_type()->mutable_item(); + auto* value = border->mutable_value()->add_items(); + if (!ConvertDataSlotToYdbTypedValue(key.first, key.second, type, value)) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unsupported type for PartitionAtKeys: '" + << key.first << "'"; + return false; + } + } + } + } + + if (metadata->TableSettings.KeyBloomFilter) { + TString value = to_lower(metadata->TableSettings.KeyBloomFilter.GetRef()); + if (value == "enabled") { + proto.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); + } else if (value == "disabled") { + proto.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); + } else { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown feature flag '" + << metadata->TableSettings.KeyBloomFilter.GetRef() + << "' for key bloom filter"; + return false; + } + } + + if (metadata->TableSettings.ReadReplicasSettings) { if (!NYql::ConvertReadReplicasSettingsToProto(metadata->TableSettings.ReadReplicasSettings.GetRef(), - *proto.mutable_read_replicas_settings(), code, error)) { - return false; - } - } - + *proto.mutable_read_replicas_settings(), code, error)) { + return false; + } + } + if (const auto& ttl = metadata->TableSettings.TtlSettings) { if (ttl.IsSet()) { ConvertTtlSettingsToProto(ttl.GetValueSet(), *proto.mutable_ttl_settings()); @@ -2605,37 +2605,37 @@ private: } } - return true; - } - - static bool FillCreateTableDesc(NYql::TKikimrTableMetadataPtr metadata, + return true; + } + + static bool FillCreateTableDesc(NYql::TKikimrTableMetadataPtr metadata, NKikimrSchemeOp::TTableDescription& tableDesc, const NGRpcService::TTableProfiles& profiles, - Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings) - { - Ydb::Table::CreateTableRequest createTableProto; - if (!profiles.ApplyTableProfile(*createTableProto.mutable_profile(), tableDesc, code, error) - || !ConvertCreateTableSettingsToProto(metadata, createTableProto, code, error)) { - return false; - } - + Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings) + { + Ydb::Table::CreateTableRequest createTableProto; + if (!profiles.ApplyTableProfile(*createTableProto.mutable_profile(), tableDesc, code, error) + || !ConvertCreateTableSettingsToProto(metadata, createTableProto, code, error)) { + return false; + } + TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); - - for (const auto& familySettings : createTableProto.column_families()) { - if (!families.ApplyFamilySettings(familySettings, &code, &error)) { - return false; - } - } - - if (families.Modified && !families.ValidateColumnFamilies(&code, &error)) { - return false; - } - + + for (const auto& familySettings : createTableProto.column_families()) { + if (!families.ApplyFamilySettings(familySettings, &code, &error)) { + return false; + } + } + + if (families.Modified && !families.ValidateColumnFamilies(&code, &error)) { + return false; + } + if (!NGRpcService::FillCreateTableSettingsDesc(tableDesc, createTableProto, profiles, code, error, warnings)) { - return false; - } - return true; - } - + return false; + } + return true; + } + private: TString Cluster; TString Database; diff --git a/ydb/core/kqp/kqp_response.cpp b/ydb/core/kqp/kqp_response.cpp index da18bd0b60..d3859cd39c 100644 --- a/ydb/core/kqp/kqp_response.cpp +++ b/ydb/core/kqp/kqp_response.cpp @@ -38,7 +38,7 @@ TMaybe<Ydb::StatusIds::StatusCode> GetYdbStatus(const TIssue& issue) { return Ydb::StatusIds::NOT_FOUND; case TIssuesIds::KIKIMR_BAD_REQUEST: - case TIssuesIds::KIKIMR_BAD_COLUMN_TYPE: + case TIssuesIds::KIKIMR_BAD_COLUMN_TYPE: case TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE: return Ydb::StatusIds::BAD_REQUEST; diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index d091714af5..cccb8a51e8 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -989,11 +989,11 @@ private: break; } - case NKikimrKqp::QUERY_ACTION_PARSE: { - onBadRequest("Parse mode is not supported yet"); - return; - } - + case NKikimrKqp::QUERY_ACTION_PARSE: { + onBadRequest("Parse mode is not supported yet"); + return; + } + case NKikimrKqp::QUERY_ACTION_VALIDATE: { if (!ValidateQuery(ctx, queryRequest.GetQuery(), queryType)) { onBadRequest(QueryState->Error); @@ -1309,15 +1309,15 @@ private: break; } - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { - IKqpHost::TExecScriptSettings execSettings; - execSettings.Deadlines = QueryState->QueryDeadlines; - execSettings.StatsMode = statsMode; - QueryState->AsyncQueryResult = KqpHost->StreamExecuteYqlScript(query, std::move(*parameters), - requestActorId, execSettings); - break; - } - + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { + IKqpHost::TExecScriptSettings execSettings; + execSettings.Deadlines = QueryState->QueryDeadlines; + execSettings.StatsMode = statsMode; + QueryState->AsyncQueryResult = KqpHost->StreamExecuteYqlScript(query, std::move(*parameters), + requestActorId, execSettings); + break; + } + case NKikimrKqp::QUERY_TYPE_SQL_SCAN: case NKikimrKqp::QUERY_TYPE_AST_SCAN: { bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN); @@ -1359,8 +1359,8 @@ private: break; } - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { QueryState->AsyncQueryResult = KqpHost->ExplainYqlScript(query); break; } @@ -1374,16 +1374,16 @@ private: } bool ValidateQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type) { - switch (type) { - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { - QueryState->AsyncQueryResult = KqpHost->ValidateYqlScript(query); - break; - } - - default: - QueryState->Error = "Unexpected query type."; - return false; + switch (type) { + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { + QueryState->AsyncQueryResult = KqpHost->ValidateYqlScript(query); + break; + } + + default: + QueryState->Error = "Unexpected query type."; + return false; } return true; @@ -1805,8 +1805,8 @@ private: case NKikimrKqp::QUERY_TYPE_SQL_DML: case NKikimrKqp::QUERY_TYPE_PREPARED_DML: case NKikimrKqp::QUERY_TYPE_SQL_SCAN: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { auto userSID = NACLib::TUserToken(QueryState->UserToken).GetUserSID(); NSysView::CollectQueryStats(ctx, stats, queryDuration, ExtractQueryText(), userSID, QueryState->ParametersSize, database, type, requestUnits); @@ -1965,43 +1965,43 @@ private: } } - /* - * TODO: - * For Scan/NewEngine plan will be set later on rpc_* level from stats and execution profiles, so - * QUERY_REPLY_FLAG_PLAN doesn't matter much. However it's may be a good idea to move FillQueryStats here, - * but for that we need to set QueryStats for scan query earlier in pipeline (now ExecutionProfiles are - * handled in rpc_stream_execute_scan_query). Other option is to remove REPLY_FLAGs at all. - */ - if (replyAst && !queryResult.QueryAst.empty()) { - ev.MutableResponse()->SetQueryAst(queryResult.QueryAst); - } - if (replyPlan && !queryResult.QueryPlan.empty()) { - ev.MutableResponse()->SetQueryPlan(queryResult.QueryPlan); - } - + /* + * TODO: + * For Scan/NewEngine plan will be set later on rpc_* level from stats and execution profiles, so + * QUERY_REPLY_FLAG_PLAN doesn't matter much. However it's may be a good idea to move FillQueryStats here, + * but for that we need to set QueryStats for scan query earlier in pipeline (now ExecutionProfiles are + * handled in rpc_stream_execute_scan_query). Other option is to remove REPLY_FLAGs at all. + */ + if (replyAst && !queryResult.QueryAst.empty()) { + ev.MutableResponse()->SetQueryAst(queryResult.QueryAst); + } + if (replyPlan && !queryResult.QueryPlan.empty()) { + ev.MutableResponse()->SetQueryPlan(queryResult.QueryPlan); + } + if (ydbStatus != Ydb::StatusIds::SUCCESS) { - return; - } - + return; + } + bool replyQueryId = false; - bool replyQueryParameters = false; + bool replyQueryParameters = false; switch (queryRequest.GetAction()) { - case NKikimrKqp::QUERY_ACTION_PREPARE: - replyQueryId = true; - replyQueryParameters = true; - break; + case NKikimrKqp::QUERY_ACTION_PREPARE: + replyQueryId = true; + replyQueryParameters = true; + break; + + case NKikimrKqp::QUERY_ACTION_EXECUTE: + replyQueryParameters = replyQueryId = queryRequest.GetQueryCachePolicy().keep_in_cache(); + break; - case NKikimrKqp::QUERY_ACTION_EXECUTE: - replyQueryParameters = replyQueryId = queryRequest.GetQueryCachePolicy().keep_in_cache(); - break; + case NKikimrKqp::QUERY_ACTION_PARSE: + case NKikimrKqp::QUERY_ACTION_VALIDATE: + replyQueryParameters = true; + break; - case NKikimrKqp::QUERY_ACTION_PARSE: - case NKikimrKqp::QUERY_ACTION_VALIDATE: - replyQueryParameters = true; - break; - - default: - break; + default: + break; } if (replyQueryParameters) { @@ -2015,7 +2015,7 @@ private: } } - if (replyQueryId) { + if (replyQueryId) { TString queryId; if (QueryState->QueryCompileResult) { queryId = QueryState->QueryCompileResult->Uid; diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index 2f76fdc6db..1eeb95daee 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -566,18 +566,18 @@ TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputC { auto op = GetTableOp(write); switch (op) { - case TYdbOperation::Upsert: + case TYdbOperation::Upsert: return BuildUpsertTable(write, inputColumns, tableData, ctx); - case TYdbOperation::Replace: + case TYdbOperation::Replace: return BuildReplaceTable(write, inputColumns, tableData, ctx); - case TYdbOperation::InsertAbort: - case TYdbOperation::InsertRevert: - return BuildInsertTable(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx); - case TYdbOperation::UpdateOn: + case TYdbOperation::InsertAbort: + case TYdbOperation::InsertRevert: + return BuildInsertTable(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx); + case TYdbOperation::UpdateOn: return BuildUpdateOnTable(write, inputColumns, tableData, ctx); - case TYdbOperation::Delete: + case TYdbOperation::Delete: return BuildDeleteTable(write, tableData, ctx); - case TYdbOperation::DeleteOn: + case TYdbOperation::DeleteOn: return BuildDeleteTable(write, tableData, ctx); default: YQL_ENSURE(false, "Unsupported table operation: " << op << ", table: " << tableData.Metadata->Name); @@ -589,16 +589,16 @@ TExprBase WriteTableWithIndexUpdate(const TKiWriteTable& write, const TCoAtomLis { auto op = GetTableOp(write); switch (op) { - case TYdbOperation::Upsert: + case TYdbOperation::Upsert: return BuildUpsertTableWithIndex(write, inputColumns, tableData, ctx); - case TYdbOperation::Replace: + case TYdbOperation::Replace: return BuildReplaceTableWithIndex(write, inputColumns, tableData, ctx); - case TYdbOperation::InsertAbort: - case TYdbOperation::InsertRevert: - return BuildInsertTableWithIndex(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx); - case TYdbOperation::UpdateOn: + case TYdbOperation::InsertAbort: + case TYdbOperation::InsertRevert: + return BuildInsertTableWithIndex(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx); + case TYdbOperation::UpdateOn: return BuildUpdateOnTableWithIndex(write, inputColumns, tableData, ctx); - case TYdbOperation::DeleteOn: + case TYdbOperation::DeleteOn: return BuildDeleteTableWithIndex(write, tableData, ctx); default: YQL_ENSURE(false, "Unsupported table operation: " << (ui32)op << ", table: " << tableData.Metadata->Name); diff --git a/ydb/core/kqp/prepare/kqp_prepare.h b/ydb/core/kqp/prepare/kqp_prepare.h index f8d1ab7dda..d114a44945 100644 --- a/ydb/core/kqp/prepare/kqp_prepare.h +++ b/ydb/core/kqp/prepare/kqp_prepare.h @@ -184,8 +184,8 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpFinalizeTransformer(TIntrusivePtr<IKq TIntrusivePtr<TKqlTransformContext> transformCtx); TAutoPtr<NYql::IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cluster, - TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TTypeAnnotationContext& typesCtx, - NYql::TKikimrConfiguration::TPtr config); + TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TTypeAnnotationContext& typesCtx, + NYql::TKikimrConfiguration::TPtr config); TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckKiProgramTransformer(); TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckQueryTransformer(); diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index b35de2b8de..ee51c7349c 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -167,7 +167,7 @@ bool CalcKeyColumnsCount(TExprContext& ctx, const TPositionHandle pos, const TSt } TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster, - const TKikimrTablesData& tablesData, bool withSystemColumns) + const TKikimrTablesData& tablesData, bool withSystemColumns) { const bool readIndex = TKqlReadTableIndex::Match(node.Get()); if (readIndex && !EnsureArgsCount(*node, 5, ctx)) { @@ -1027,12 +1027,12 @@ TStatus AnnotateKqpEnsure(const TExprNode::TPtr& node, TExprContext& ctx) { } // namespace TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cluster, - TIntrusivePtr<TKikimrTablesData> tablesData, TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config) + TIntrusivePtr<TKikimrTablesData> tablesData, TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config) { TAutoPtr<IGraphTransformer> dqTransformer = CreateDqTypeAnnotationTransformer(typesCtx); return CreateFunctorTransformer( - [cluster, tablesData, dqTransformer, config](const TExprNode::TPtr& input, TExprNode::TPtr& output, + [cluster, tablesData, dqTransformer, config](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus { output = input; diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index da3b2860cf..035d6b7701 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -84,42 +84,42 @@ private: return TStatus::Ok; } - TStatus HandleCreateUser(TKiCreateUser node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "CreateUser is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - - TStatus HandleAlterUser(TKiAlterUser node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "AlterUser is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - - TStatus HandleDropUser(TKiDropUser node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "DropUser is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - - TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "CreateGroup is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - - TStatus HandleAlterGroup(TKiAlterGroup node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "AlterGroup is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - - TStatus HandleDropGroup(TKiDropGroup node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "DropGroup is not yet implemented for intent determination transformer")); - return TStatus::Error; - } - + TStatus HandleCreateUser(TKiCreateUser node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "CreateUser is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + + TStatus HandleAlterUser(TKiAlterUser node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "AlterUser is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + + TStatus HandleDropUser(TKiDropUser node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "DropUser is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + + TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "CreateGroup is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + + TStatus HandleAlterGroup(TKiAlterGroup node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "AlterGroup is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + + TStatus HandleDropGroup(TKiDropGroup node, TExprContext& ctx) override { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + << "DropGroup is not yet implemented for intent determination transformer")); + return TStatus::Error; + } + TStatus HandleWrite(TExprBase node, TExprContext& ctx) override { auto cluster = node.Ref().Child(1)->Child(1)->Content(); TKikimrKey key(ctx); @@ -129,8 +129,8 @@ private: switch (key.GetKeyType()) { case TKikimrKey::Type::Table: { - NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( - TExprList(node.Ref().ChildPtr(4)), ctx); + NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( + TExprList(node.Ref().ChildPtr(4)), ctx); if (!settings.Mode) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Mode option is required for Kikimr table writes.")); @@ -181,8 +181,8 @@ private: } case TKikimrKey::Type::TableScheme: { - NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( - TExprList(node.Ref().ChildPtr(4)), ctx); + NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( + TExprList(node.Ref().ChildPtr(4)), ctx); if (!settings.Mode) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Mode option is required for Kikimr scheme writes.")); @@ -200,9 +200,9 @@ private: SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "alter") { - if (!settings.AlterActions) { + if (!settings.AlterActions) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() - << "No actions provided for alter mode.")); + << "No actions provided for alter mode.")); return TStatus::Error; } @@ -217,9 +217,9 @@ private: case TKikimrKey::Type::TableList: break; - - case TKikimrKey::Type::Role: - return TStatus::Ok; + + case TKikimrKey::Type::Role: + return TStatus::Ok; } ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid table key type.")); @@ -428,15 +428,15 @@ public: return true; } - if (node.IsCallable(TKiCreateUser::CallableName()) - || node.IsCallable(TKiAlterUser::CallableName()) - || node.IsCallable(TKiDropUser::CallableName()) - || node.IsCallable(TKiCreateGroup::CallableName()) - || node.IsCallable(TKiAlterGroup::CallableName()) - || node.IsCallable(TKiDropGroup::CallableName())) { - return true; - } - + if (node.IsCallable(TKiCreateUser::CallableName()) + || node.IsCallable(TKiAlterUser::CallableName()) + || node.IsCallable(TKiDropUser::CallableName()) + || node.IsCallable(TKiCreateGroup::CallableName()) + || node.IsCallable(TKiAlterGroup::CallableName()) + || node.IsCallable(TKiDropGroup::CallableName())) { + return true; + } + if (auto maybeRight = TMaybeNode<TCoNth>(&node).Tuple().Maybe<TCoRight>()) { if (maybeRight.Input().Maybe<TKiExecDataQuery>()) { return true; @@ -480,11 +480,11 @@ public: YQL_ENSURE(node->IsCallable(WriteName), "Expected Write!, got: " << node->Content()); TKikimrKey key(ctx); - YQL_ENSURE(key.Extract(*node->Child(2)), "Failed to extract ydb key."); + YQL_ENSURE(key.Extract(*node->Child(2)), "Failed to extract ydb key."); switch (key.GetKeyType()) { case TKikimrKey::Type::Table: { - NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); + NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); YQL_ENSURE(settings.Mode); auto mode = settings.Mode.Cast(); @@ -531,22 +531,22 @@ public: } case TKikimrKey::Type::TableScheme: { - NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); + NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); YQL_ENSURE(settings.Mode); auto mode = settings.Mode.Cast(); if (mode == "create") { - YQL_ENSURE(settings.Columns); - YQL_ENSURE(!settings.Columns.Cast().Empty()); + YQL_ENSURE(settings.Columns); + YQL_ENSURE(!settings.Columns.Cast().Empty()); if (!settings.PrimaryKey) { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Primary key is required for ydb tables.")); return nullptr; } - if (!settings.PartitionBy.IsValid()) { - settings.PartitionBy = Build<TCoAtomList>(ctx, node->Pos()).Done(); - } - + if (!settings.PartitionBy.IsValid()) { + settings.PartitionBy = Build<TCoAtomList>(ctx, node->Pos()).Done(); + } + return Build<TKiCreateTable>(ctx, node->Pos()) .World(node->Child(0)) .DataSink(node->Child(1)) @@ -556,95 +556,95 @@ public: .Settings(settings.Other) .Indexes(settings.Indexes.Cast()) .Changefeeds(settings.Changefeeds.Cast()) - .PartitionBy(settings.PartitionBy.Cast()) - .ColumnFamilies(settings.ColumnFamilies.Cast()) - .TableSettings(settings.TableSettings.Cast()) + .PartitionBy(settings.PartitionBy.Cast()) + .ColumnFamilies(settings.ColumnFamilies.Cast()) + .TableSettings(settings.TableSettings.Cast()) .Done() .Ptr(); } else if (mode == "alter") { for (auto setting : settings.Other) { if (setting.Name().Value() == "intent") { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Old AST format for AlterTable")); - return nullptr; + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Old AST format for AlterTable")); + return nullptr; } } - YQL_ENSURE(settings.AlterActions); - YQL_ENSURE(!settings.AlterActions.Cast().Empty()); - + YQL_ENSURE(settings.AlterActions); + YQL_ENSURE(!settings.AlterActions.Cast().Empty()); + return Build<TKiAlterTable>(ctx, node->Pos()) .World(node->Child(0)) .DataSink(node->Child(1)) .Table().Build(key.GetTablePath()) - .Actions(settings.AlterActions.Cast()) + .Actions(settings.AlterActions.Cast()) .Done() .Ptr(); } else { - YQL_ENSURE(false, "unknown TableScheme mode \"" << TString(mode) << "\""); + YQL_ENSURE(false, "unknown TableScheme mode \"" << TString(mode) << "\""); } } case TKikimrKey::Type::TableList: break; - - case TKikimrKey::Type::Role: { - NCommon::TWriteRoleSettings settings = NCommon::ParseWriteRoleSettings(TExprList(node->Child(4)), ctx); - YQL_ENSURE(settings.Mode); - auto mode = settings.Mode.Cast(); - - if (mode == "createUser") { - return Build<TKiCreateUser>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .UserName().Build(key.GetRoleName()) - .Settings(settings.Other) - .Done() - .Ptr(); - } else if (mode == "alterUser") { - return Build<TKiAlterUser>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .UserName().Build(key.GetRoleName()) - .Settings(settings.Other) - .Done() - .Ptr(); - } else if (mode == "dropUser") { - return Build<TKiDropUser>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .UserName().Build(key.GetRoleName()) - .Settings(settings.Other) - .Done() - .Ptr(); - } else if (mode == "createGroup") { - return Build<TKiCreateGroup>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .GroupName().Build(key.GetRoleName()) - .Done() - .Ptr(); - } else if (mode == "addUsersToGroup" || mode == "dropUsersFromGroup") { - return Build<TKiAlterGroup>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .GroupName().Build(key.GetRoleName()) - .Action().Build(mode) - .Roles(settings.Roles.Cast()) - .Done() - .Ptr(); - } else if (mode == "dropGroup") { - return Build<TKiDropGroup>(ctx, node->Pos()) - .World(node->Child(0)) - .DataSink(node->Child(1)) - .GroupName().Build(key.GetRoleName()) - .Settings(settings.Other) - .Done() - .Ptr(); - } else { - YQL_ENSURE(false, "unknown Role mode \"" << TString(mode) << "\""); - } - break; - } + + case TKikimrKey::Type::Role: { + NCommon::TWriteRoleSettings settings = NCommon::ParseWriteRoleSettings(TExprList(node->Child(4)), ctx); + YQL_ENSURE(settings.Mode); + auto mode = settings.Mode.Cast(); + + if (mode == "createUser") { + return Build<TKiCreateUser>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .UserName().Build(key.GetRoleName()) + .Settings(settings.Other) + .Done() + .Ptr(); + } else if (mode == "alterUser") { + return Build<TKiAlterUser>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .UserName().Build(key.GetRoleName()) + .Settings(settings.Other) + .Done() + .Ptr(); + } else if (mode == "dropUser") { + return Build<TKiDropUser>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .UserName().Build(key.GetRoleName()) + .Settings(settings.Other) + .Done() + .Ptr(); + } else if (mode == "createGroup") { + return Build<TKiCreateGroup>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .GroupName().Build(key.GetRoleName()) + .Done() + .Ptr(); + } else if (mode == "addUsersToGroup" || mode == "dropUsersFromGroup") { + return Build<TKiAlterGroup>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .GroupName().Build(key.GetRoleName()) + .Action().Build(mode) + .Roles(settings.Roles.Cast()) + .Done() + .Ptr(); + } else if (mode == "dropGroup") { + return Build<TKiDropGroup>(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .GroupName().Build(key.GetRoleName()) + .Settings(settings.Other) + .Done() + .Ptr(); + } else { + YQL_ENSURE(false, "unknown Role mode \"" << TString(mode) << "\""); + } + break; + } } ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Failed to rewrite IO.")); @@ -726,30 +726,30 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt return HandleDropTable(node.Cast(), ctx); } - if (auto node = TMaybeNode<TKiCreateUser>(input)) { - return HandleCreateUser(node.Cast(), ctx); - } - - if (auto node = TMaybeNode<TKiAlterUser>(input)) { - return HandleAlterUser(node.Cast(), ctx); - } - - if (auto node = TMaybeNode<TKiDropUser>(input)) { - return HandleDropUser(node.Cast(), ctx); - } - - if (auto node = TMaybeNode<TKiCreateGroup>(input)) { - return HandleCreateGroup(node.Cast(), ctx); - } - - if (auto node = TMaybeNode<TKiAlterGroup>(input)) { - return HandleAlterGroup(node.Cast(), ctx); - } - - if (auto node = TMaybeNode<TKiDropGroup>(input)) { - return HandleDropGroup(node.Cast(), ctx); - } - + if (auto node = TMaybeNode<TKiCreateUser>(input)) { + return HandleCreateUser(node.Cast(), ctx); + } + + if (auto node = TMaybeNode<TKiAlterUser>(input)) { + return HandleAlterUser(node.Cast(), ctx); + } + + if (auto node = TMaybeNode<TKiDropUser>(input)) { + return HandleDropUser(node.Cast(), ctx); + } + + if (auto node = TMaybeNode<TKiCreateGroup>(input)) { + return HandleCreateGroup(node.Cast(), ctx); + } + + if (auto node = TMaybeNode<TKiAlterGroup>(input)) { + return HandleAlterGroup(node.Cast(), ctx); + } + + if (auto node = TMaybeNode<TKiDropGroup>(input)) { + return HandleDropGroup(node.Cast(), ctx); + } + if (input->IsCallable(WriteName)) { return HandleWrite(TExprBase(input), ctx); } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index e93f65c6ea..65ef7eb932 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -25,7 +25,7 @@ private: return TStatus::Error; } - return HandleKey(cluster, key); + return HandleKey(cluster, key); } TStatus HandleRead(TExprBase node, TExprContext& ctx) override { @@ -35,7 +35,7 @@ private: return TStatus::Error; } - return HandleKey(cluster, key); + return HandleKey(cluster, key); } TStatus HandleLength(TExprBase node, TExprContext& ctx) override { @@ -51,7 +51,7 @@ private: } private: - TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key) { + TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key) { switch (key.GetKeyType()) { case TKikimrKey::Type::Table: case TKikimrKey::Type::TableScheme: { @@ -66,9 +66,9 @@ private: case TKikimrKey::Type::TableList: return TStatus::Ok; - - case TKikimrKey::Type::Role: - return TStatus::Ok; + + case TKikimrKey::Type::Role: + return TStatus::Ok; } return TStatus::Error; @@ -167,16 +167,16 @@ public: YQL_ENSURE(res.Metadata); tableDesc.Metadata = res.Metadata; - bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); + bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); YQL_ENSURE(res.Metadata->Indexes.size() == res.Metadata->SecondaryGlobalIndexMetadata.size()); for (const auto& indexMeta : res.Metadata->SecondaryGlobalIndexMetadata) { YQL_ENSURE(indexMeta); auto& desc = SessionCtx->Tables().GetOrAddTable(indexMeta->Cluster, SessionCtx->GetDatabase(), indexMeta->Name); desc.Metadata = indexMeta; - desc.Load(ctx, sysColumnsEnabled); + desc.Load(ctx, sysColumnsEnabled); } - if (!tableDesc.Load(ctx, sysColumnsEnabled)) { + if (!tableDesc.Load(ctx, sysColumnsEnabled)) { LoadResults.clear(); return TStatus::Error; } @@ -233,7 +233,7 @@ protected: break; case EKikimrQueryType::YqlScript: - case EKikimrQueryType::YqlScriptStreaming: + case EKikimrQueryType::YqlScriptStreaming: pragmaAllowed = AllowedScriptingPragmas.contains(name); break; diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index eda5ad7db7..ad360fcaeb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -16,111 +16,111 @@ using namespace NNodes; using namespace NCommon; using namespace NThreading; -namespace { - bool EnsureNotPrepare(const TString featureName, TPositionHandle pos, const TKikimrQueryContext& queryCtx, - TExprContext& ctx) - { - if (queryCtx.PrepareOnly && !queryCtx.SuppressDdlChecks) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() - << "'" << featureName << "' not supported in query prepare mode.")); - return false; - } - - return true; +namespace { + bool EnsureNotPrepare(const TString featureName, TPositionHandle pos, const TKikimrQueryContext& queryCtx, + TExprContext& ctx) + { + if (queryCtx.PrepareOnly && !queryCtx.SuppressDdlChecks) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() + << "'" << featureName << "' not supported in query prepare mode.")); + return false; + } + + return true; + } + + void FillExecDataQueryAst(TKiExecDataQuery exec, const TString& ast, TExprContext& ctx) { + auto astNode = Build<TCoAtom>(ctx, exec.Pos()) + .Value(ast) + .Done(); + + astNode.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); + + exec.Ptr()->ChildRef(TKiExecDataQuery::idx_Ast) = astNode.Ptr(); + } + + TCreateUserSettings ParseCreateUserSettings(TKiCreateUser createUser) { + TCreateUserSettings createUserSettings; + createUserSettings.UserName = TString(createUser.UserName()); + + for (auto setting : createUser.Settings()) { + auto name = setting.Name().Value(); + if (name == "password") { + createUserSettings.Password = setting.Value().Cast<TCoAtom>().StringValue(); + } else if (name == "nullPassword") { + // Default value + } else if (name == "passwordEncrypted") { + createUserSettings.PasswordEncrypted = true; + } + } + return createUserSettings; + } + + TAlterUserSettings ParseAlterUserSettings(TKiAlterUser alterUser) { + TAlterUserSettings alterUserSettings; + alterUserSettings.UserName = TString(alterUser.UserName()); + + for (auto setting : alterUser.Settings()) { + auto name = setting.Name().Value(); + if (name == "password") { + alterUserSettings.Password = setting.Value().Cast<TCoAtom>().StringValue(); + } else if (name == "nullPassword") { + // Default value + } else if (name == "passwordEncrypted") { + alterUserSettings.PasswordEncrypted = true; + } + } + return alterUserSettings; } - void FillExecDataQueryAst(TKiExecDataQuery exec, const TString& ast, TExprContext& ctx) { - auto astNode = Build<TCoAtom>(ctx, exec.Pos()) - .Value(ast) - .Done(); - - astNode.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - - exec.Ptr()->ChildRef(TKiExecDataQuery::idx_Ast) = astNode.Ptr(); - } - - TCreateUserSettings ParseCreateUserSettings(TKiCreateUser createUser) { - TCreateUserSettings createUserSettings; - createUserSettings.UserName = TString(createUser.UserName()); - - for (auto setting : createUser.Settings()) { - auto name = setting.Name().Value(); - if (name == "password") { - createUserSettings.Password = setting.Value().Cast<TCoAtom>().StringValue(); - } else if (name == "nullPassword") { - // Default value - } else if (name == "passwordEncrypted") { - createUserSettings.PasswordEncrypted = true; - } - } - return createUserSettings; - } - - TAlterUserSettings ParseAlterUserSettings(TKiAlterUser alterUser) { - TAlterUserSettings alterUserSettings; - alterUserSettings.UserName = TString(alterUser.UserName()); - - for (auto setting : alterUser.Settings()) { - auto name = setting.Name().Value(); - if (name == "password") { - alterUserSettings.Password = setting.Value().Cast<TCoAtom>().StringValue(); - } else if (name == "nullPassword") { - // Default value - } else if (name == "passwordEncrypted") { - alterUserSettings.PasswordEncrypted = true; - } - } - return alterUserSettings; - } - - TDropUserSettings ParseDropUserSettings(TKiDropUser dropUser) { - TDropUserSettings dropUserSettings; - dropUserSettings.UserName = TString(dropUser.UserName()); - - for (auto setting : dropUser.Settings()) { - auto name = setting.Name().Value(); - if (name == "force") { - dropUserSettings.Force = true; - } - } - return dropUserSettings; - } - - TCreateGroupSettings ParseCreateGroupSettings(TKiCreateGroup createGroup) { - TCreateGroupSettings createGroupSettings; - createGroupSettings.GroupName = TString(createGroup.GroupName()); - return createGroupSettings; - } - - TAlterGroupSettings ParseAlterGroupSettings(TKiAlterGroup alterGroup) { - TAlterGroupSettings alterGroupSettings; - alterGroupSettings.GroupName = TString(alterGroup.GroupName()); - - TString action = TString(alterGroup.Action()); - if (action == "addUsersToGroup") { - alterGroupSettings.Action = TAlterGroupSettings::EAction::AddRoles; - } else if (action == "dropUsersFromGroup") { - alterGroupSettings.Action = TAlterGroupSettings::EAction::RemoveRoles; - } - - for (auto role : alterGroup.Roles()) { - alterGroupSettings.Roles.push_back(role.Cast<TCoAtom>().StringValue()); - } - return alterGroupSettings; - } - - TDropGroupSettings ParseDropGroupSettings(TKiDropGroup dropGroup) { - TDropGroupSettings dropGroupSettings; - dropGroupSettings.GroupName = TString(dropGroup.GroupName()); - - for (auto setting : dropGroup.Settings()) { - auto name = setting.Name().Value(); - if (name == "force") { - dropGroupSettings.Force = true; - } - } - return dropGroupSettings; - } + TDropUserSettings ParseDropUserSettings(TKiDropUser dropUser) { + TDropUserSettings dropUserSettings; + dropUserSettings.UserName = TString(dropUser.UserName()); + + for (auto setting : dropUser.Settings()) { + auto name = setting.Name().Value(); + if (name == "force") { + dropUserSettings.Force = true; + } + } + return dropUserSettings; + } + + TCreateGroupSettings ParseCreateGroupSettings(TKiCreateGroup createGroup) { + TCreateGroupSettings createGroupSettings; + createGroupSettings.GroupName = TString(createGroup.GroupName()); + return createGroupSettings; + } + + TAlterGroupSettings ParseAlterGroupSettings(TKiAlterGroup alterGroup) { + TAlterGroupSettings alterGroupSettings; + alterGroupSettings.GroupName = TString(alterGroup.GroupName()); + + TString action = TString(alterGroup.Action()); + if (action == "addUsersToGroup") { + alterGroupSettings.Action = TAlterGroupSettings::EAction::AddRoles; + } else if (action == "dropUsersFromGroup") { + alterGroupSettings.Action = TAlterGroupSettings::EAction::RemoveRoles; + } + + for (auto role : alterGroup.Roles()) { + alterGroupSettings.Roles.push_back(role.Cast<TCoAtom>().StringValue()); + } + return alterGroupSettings; + } + + TDropGroupSettings ParseDropGroupSettings(TKiDropGroup dropGroup) { + TDropGroupSettings dropGroupSettings; + dropGroupSettings.GroupName = TString(dropGroup.GroupName()); + + for (auto setting : dropGroup.Settings()) { + auto name = setting.Name().Value(); + if (name == "force") { + dropGroupSettings.Force = true; + } + } + return dropGroupSettings; + } } class TKiSinkPlanInfoTransformer : public TGraphTransformerBase { @@ -220,11 +220,11 @@ public: YQL_ENSURE(key.Extract(maybeTableScheme.Cast().TableKey().Ref())); auto& tableDesc = SessionCtx->Tables().ExistingTable(TString(cluster), key.GetTablePath()); - TKikimrTableDescription rawTableDesc; - rawTableDesc.Metadata = tableDesc.Metadata; - rawTableDesc.Load(ctx); + TKikimrTableDescription rawTableDesc; + rawTableDesc.Metadata = tableDesc.Metadata; + rawTableDesc.Load(ctx); - auto result = GetTableMetadataResult(rawTableDesc, fillSettings, ctx); + auto result = GetTableMetadataResult(rawTableDesc, fillSettings, ctx); YQL_ENSURE(result); auto resultNode = ctx.NewAtom(input->Pos(), *result); @@ -463,7 +463,7 @@ public: auto cluster = TString(maybeCreate.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeCreate.Cast().Table())); - if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { + if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { return SyncError(); } @@ -490,7 +490,7 @@ public: auto cluster = TString(maybeDrop.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeDrop.Cast().Table())); - if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::DropTable, ctx)) { + if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::DropTable, ctx)) { return SyncError(); } @@ -521,15 +521,15 @@ public: auto cluster = TString(maybeAlter.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeAlter.Cast().Table())); - if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::AlterTable, ctx)) { + if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::AlterTable, ctx)) { return SyncError(); } Ydb::Table::AlterTableRequest alterTableRequest; alterTableRequest.set_path(table.Metadata->Name); - for (auto action : maybeAlter.Cast().Actions()) { - auto name = action.Name().Value(); + for (auto action : maybeAlter.Cast().Actions()) { + auto name = action.Name().Value(); if (name == "renameTo") { auto destination = action.Value().Cast<TCoAtom>().StringValue(); auto future = Gateway->RenameTable(table.Metadata->Name, destination, cluster); @@ -540,80 +540,80 @@ public: return resultNode; }); } else if (name == "addColumns") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { auto add_column = alterTableRequest.add_add_columns(); - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); add_column->set_name(TString(columnName)); - auto typeNode = columnTuple.Item(1); - auto columnType = typeNode.Ref().GetTypeAnn(); - auto type = columnType->Cast<TTypeExprType>()->GetType(); + auto typeNode = columnTuple.Item(1); + auto columnType = typeNode.Ref().GetTypeAnn(); + auto type = columnType->Cast<TTypeExprType>()->GetType(); auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); - auto dataType = actualType->Cast<TDataExprType>(); + auto dataType = actualType->Cast<TDataExprType>(); SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull); - if (columnTuple.Size() > 2) { - auto families = columnTuple.Item(2).Cast<TCoAtomList>(); + if (columnTuple.Size() > 2) { + auto families = columnTuple.Item(2).Cast<TCoAtomList>(); if (families.Size() > 1) { ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), "Unsupported number of families")); return SyncError(); } - for (auto family : families) { + for (auto family : families) { add_column->set_family(TString(family.Value())); - } - } - } - } else if (name == "dropColumns") { - auto listNode = action.Value().Cast<TCoAtomList>(); - for (auto dropColumn : listNode) { + } + } + } + } else if (name == "dropColumns") { + auto listNode = action.Value().Cast<TCoAtomList>(); + for (auto dropColumn : listNode) { alterTableRequest.add_drop_columns(TString(dropColumn.Value())); - } - } else if (name == "alterColumns") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { + } + } else if (name == "alterColumns") { + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { auto alter_columns = alterTableRequest.add_alter_columns(); - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto columnName = columnTuple.Item(0).Cast<TCoAtom>(); alter_columns->set_name(TString(columnName)); - auto families = columnTuple.Item(1).Cast<TCoAtomList>(); + auto families = columnTuple.Item(1).Cast<TCoAtomList>(); if (families.Size() > 1) { ctx.AddError(TIssue(ctx.GetPosition(families.Pos()), "Unsupported number of families")); return SyncError(); } - for (auto family : families) { + for (auto family : families) { alter_columns->set_family(TString(family.Value())); - } - } - } else if (name == "addColumnFamilies" || name == "alterColumnFamilies") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - if (auto maybeTupleList = item.Maybe<TCoNameValueTupleList>()) { + } + } + } else if (name == "addColumnFamilies" || name == "alterColumnFamilies") { + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + if (auto maybeTupleList = item.Maybe<TCoNameValueTupleList>()) { auto f = (name == "addColumnFamilies") ? alterTableRequest.add_add_column_families() : alterTableRequest.add_alter_column_families(); - for (auto familySetting : maybeTupleList.Cast()) { - auto name = familySetting.Name().Value(); - if (name == "name") { + for (auto familySetting : maybeTupleList.Cast()) { + auto name = familySetting.Name().Value(); + if (name == "name") { f->set_name(TString(familySetting.Value().Cast<TCoAtom>().Value())); - } else if (name == "data") { + } else if (name == "data") { auto data = TString( familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() .Value()); f->mutable_data()->set_media(data); - } else if (name == "compression") { + } else if (name == "compression") { auto comp = TString( - familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() - .Value() - ); + familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>() + .Value() + ); if (to_lower(comp) == "off") { f->set_compression(Ydb::Table::ColumnFamily::COMPRESSION_NONE); } else if (to_lower(comp) == "lz4") { @@ -626,24 +626,24 @@ public: return SyncError(); } - } else { - ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), - TStringBuilder() << "Unknown column family setting name: " << name)); - return SyncError(); - } - } - } - } - } else if (name == "setTableSettings") { - auto listNode = action.Value().Cast<TCoNameValueTupleList>(); - - for (const auto& setting : listNode) { - auto name = setting.Name().Value(); - if (name == "compactionPolicy") { + } else { + ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), + TStringBuilder() << "Unknown column family setting name: " << name)); + return SyncError(); + } + } + } + } + } else if (name == "setTableSettings") { + auto listNode = action.Value().Cast<TCoNameValueTupleList>(); + + for (const auto& setting : listNode) { + auto name = setting.Name().Value(); + if (name == "compactionPolicy") { alterTableRequest.set_set_compaction_policy(TString( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() )); - } else if (name == "autoPartitioningBySize") { + } else if (name == "autoPartitioningBySize") { auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); if (val == "enabled") { @@ -658,20 +658,20 @@ public: errText)); return SyncError(); } - } else if (name == "partitionSizeMb") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { + } else if (name == "partitionSizeMb") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); partitioningSettings->set_partition_size_mb(value); - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set preferred partition size to 0. " - "To disable auto partitioning by size use 'SET AUTO_PARTITIONING_BY_SIZE DISABLED'")); - return SyncError(); - } - } else if (name == "autoPartitioningByLoad") { + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set preferred partition size to 0. " + "To disable auto partitioning by size use 'SET AUTO_PARTITIONING_BY_SIZE DISABLED'")); + return SyncError(); + } + } else if (name == "autoPartitioningByLoad") { auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); TString val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); if (val == "enabled") { @@ -686,31 +686,31 @@ public: errText)); return SyncError(); } - } else if (name == "minPartitions") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { + } else if (name == "minPartitions") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); partitioningSettings->set_min_partitions_count(value); - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set min partition count to 0")); - return SyncError(); - } - } else if (name == "maxPartitions") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set min partition count to 0")); + return SyncError(); + } + } else if (name == "maxPartitions") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { auto partitioningSettings = alterTableRequest.mutable_alter_partitioning_settings(); partitioningSettings->set_max_partitions_count(value); - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set max partition count to 0")); - return SyncError(); - } - } else if (name == "keyBloomFilter") { + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set max partition count to 0")); + return SyncError(); + } + } else if (name == "keyBloomFilter") { auto val = to_lower(TString(setting.Value().Cast<TCoAtom>().Value())); if (val == "enabled") { alterTableRequest.set_set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); @@ -725,10 +725,10 @@ public: return SyncError(); } - } else if (name == "readReplicasSettings") { + } else if (name == "readReplicasSettings") { const auto replicasSettings = TString( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); Ydb::StatusIds::StatusCode code; TString errText; if (!ConvertReadReplicasSettingsToProto(replicasSettings, @@ -753,12 +753,12 @@ public: ConvertTtlSettingsToProto(ttlSettings, *alterTableRequest.mutable_set_ttl_settings()); } else if (name == "resetTtlSettings") { alterTableRequest.mutable_drop_ttl_settings(); - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown table profile setting: " << name)); - return SyncError(); - } - } + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown table profile setting: " << name)); + return SyncError(); + } + } } else if (name == "addIndex") { auto listNode = action.Value().Cast<TExprList>(); auto add_index = alterTableRequest.add_add_indexes(); @@ -811,10 +811,10 @@ public: } else if (name == "dropIndex") { auto nameNode = action.Value().Cast<TCoAtom>(); alterTableRequest.add_drop_indexes(TString(nameNode.Value())); - } else { - ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), - TStringBuilder() << "Unknown alter table action: " << name)); - return SyncError(); + } else { + ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), + TStringBuilder() << "Unknown alter table action: " << name)); + return SyncError(); } } @@ -829,144 +829,144 @@ public: } - if (auto maybeCreateUser = TMaybeNode<TKiCreateUser>(input)) { - if (!EnsureNotPrepare("CREATE USER", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeCreateUser.Cast().DataSink().Cluster()); - TCreateUserSettings createUserSettings = ParseCreateUserSettings(maybeCreateUser.Cast()); - - auto future = Gateway->CreateUser(cluster, createUserSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing CREATE USER"); - } - - if (auto maybeAlterUser = TMaybeNode<TKiAlterUser>(input)) { - if (!EnsureNotPrepare("ALTER USER", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeAlterUser.Cast().DataSink().Cluster()); - TAlterUserSettings alterUserSettings = ParseAlterUserSettings(maybeAlterUser.Cast()); - - auto future = Gateway->AlterUser(cluster, alterUserSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing ALTER USER"); - } - - if (auto maybeDropUser = TMaybeNode<TKiDropUser>(input)) { - if (!EnsureNotPrepare("DROP USER", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeDropUser.Cast().DataSink().Cluster()); - TDropUserSettings dropUserSettings = ParseDropUserSettings(maybeDropUser.Cast()); - - auto future = Gateway->DropUser(cluster, dropUserSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing DROP USER"); - } - - if (auto maybeCreateGroup = TMaybeNode<TKiCreateGroup>(input)) { - if (!EnsureNotPrepare("CREATE GROUP", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeCreateGroup.Cast().DataSink().Cluster()); - TCreateGroupSettings createGroupSettings = ParseCreateGroupSettings(maybeCreateGroup.Cast()); - - auto future = Gateway->CreateGroup(cluster, createGroupSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing CREATE GROUP"); - } - - if (auto maybeAlterGroup = TMaybeNode<TKiAlterGroup>(input)) { - if (!EnsureNotPrepare("ALTER GROUP", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeAlterGroup.Cast().DataSink().Cluster()); - TAlterGroupSettings alterGroupSettings = ParseAlterGroupSettings(maybeAlterGroup.Cast()); - - auto future = Gateway->AlterGroup(cluster, alterGroupSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing ALTER GROUP"); - } - - if (auto maybeDropGroup = TMaybeNode<TKiDropGroup>(input)) { - if (!EnsureNotPrepare("DROP GROUP", input->Pos(), SessionCtx->Query(), ctx)) { - return SyncError(); - } - - auto requireStatus = RequireChild(*input, 0); - if (requireStatus.Level != TStatus::Ok) { - return SyncStatus(requireStatus); - } - - auto cluster = TString(maybeDropGroup.Cast().DataSink().Cluster()); - TDropGroupSettings dropGroupSettings = ParseDropGroupSettings(maybeDropGroup.Cast()); - - auto future = Gateway->DropGroup(cluster, dropGroupSettings); - - return WrapFuture(future, - [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { - Y_UNUSED(res); - auto resultNode = ctx.NewWorld(input->Pos()); - return resultNode; - }, "Executing DROP GROUP"); - } - + if (auto maybeCreateUser = TMaybeNode<TKiCreateUser>(input)) { + if (!EnsureNotPrepare("CREATE USER", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeCreateUser.Cast().DataSink().Cluster()); + TCreateUserSettings createUserSettings = ParseCreateUserSettings(maybeCreateUser.Cast()); + + auto future = Gateway->CreateUser(cluster, createUserSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing CREATE USER"); + } + + if (auto maybeAlterUser = TMaybeNode<TKiAlterUser>(input)) { + if (!EnsureNotPrepare("ALTER USER", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeAlterUser.Cast().DataSink().Cluster()); + TAlterUserSettings alterUserSettings = ParseAlterUserSettings(maybeAlterUser.Cast()); + + auto future = Gateway->AlterUser(cluster, alterUserSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing ALTER USER"); + } + + if (auto maybeDropUser = TMaybeNode<TKiDropUser>(input)) { + if (!EnsureNotPrepare("DROP USER", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeDropUser.Cast().DataSink().Cluster()); + TDropUserSettings dropUserSettings = ParseDropUserSettings(maybeDropUser.Cast()); + + auto future = Gateway->DropUser(cluster, dropUserSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing DROP USER"); + } + + if (auto maybeCreateGroup = TMaybeNode<TKiCreateGroup>(input)) { + if (!EnsureNotPrepare("CREATE GROUP", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeCreateGroup.Cast().DataSink().Cluster()); + TCreateGroupSettings createGroupSettings = ParseCreateGroupSettings(maybeCreateGroup.Cast()); + + auto future = Gateway->CreateGroup(cluster, createGroupSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing CREATE GROUP"); + } + + if (auto maybeAlterGroup = TMaybeNode<TKiAlterGroup>(input)) { + if (!EnsureNotPrepare("ALTER GROUP", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeAlterGroup.Cast().DataSink().Cluster()); + TAlterGroupSettings alterGroupSettings = ParseAlterGroupSettings(maybeAlterGroup.Cast()); + + auto future = Gateway->AlterGroup(cluster, alterGroupSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing ALTER GROUP"); + } + + if (auto maybeDropGroup = TMaybeNode<TKiDropGroup>(input)) { + if (!EnsureNotPrepare("DROP GROUP", input->Pos(), SessionCtx->Query(), ctx)) { + return SyncError(); + } + + auto requireStatus = RequireChild(*input, 0); + if (requireStatus.Level != TStatus::Ok) { + return SyncStatus(requireStatus); + } + + auto cluster = TString(maybeDropGroup.Cast().DataSink().Cluster()); + TDropGroupSettings dropGroupSettings = ParseDropGroupSettings(maybeDropGroup.Cast()); + + auto future = Gateway->DropGroup(cluster, dropGroupSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing DROP GROUP"); + } + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Failed to execute node: " << input->Content())); return SyncError(); @@ -1116,7 +1116,7 @@ private: } bool ApplyDdlOperation(const TString& cluster, TPositionHandle pos, const TString& table, - TYdbOperation op, TExprContext& ctx) + TYdbOperation op, TExprContext& ctx) { YQL_ENSURE(op & KikimrSchemeOps()); diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index 923c820763..c54d82a2ca 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -113,9 +113,9 @@ {"Index": 3, "Name": "Columns", "Type": "TExprList"}, {"Index": 4, "Name": "PrimaryKey", "Type": "TCoAtomList"}, {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 6, "Name": "Indexes", "Type": "TCoIndexList"}, - {"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"}, - {"Index": 8, "Name": "ColumnFamilies", "Type": "TExprList"}, + {"Index": 6, "Name": "Indexes", "Type": "TCoIndexList"}, + {"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"}, + {"Index": 8, "Name": "ColumnFamilies", "Type": "TExprList"}, {"Index": 9, "Name": "TableSettings", "Type": "TCoNameValueTupleList"}, {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"} ] @@ -128,7 +128,7 @@ {"Index": 0, "Name": "World", "Type": "TExprBase"}, {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"} + {"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"} ] }, { @@ -143,72 +143,72 @@ ] }, { - "Name": "TKiCreateUser", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiCreateUser!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} - ] - }, - { - "Name": "TKiAlterUser", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiAlterUser!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} - ] - }, - { - "Name": "TKiDropUser", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiDropUser!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} - ] - }, - { - "Name": "TKiCreateGroup", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiCreateGroup!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"} - ] - }, - { - "Name": "TKiAlterGroup", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiAlterGroup!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Action", "Type": "TCoAtom"}, - {"Index": 4, "Name": "Roles", "Type": "TCoAtomList"} - ] - }, - { - "Name": "TKiDropGroup", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KiDropGroup!"}, - "Children": [ - {"Index": 0, "Name": "World", "Type": "TExprBase"}, - {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, - {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} - ] - }, - { + "Name": "TKiCreateUser", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiCreateUser!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TKiAlterUser", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiAlterUser!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TKiDropUser", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiDropUser!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "UserName", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TKiCreateGroup", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiCreateGroup!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"} + ] + }, + { + "Name": "TKiAlterGroup", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiAlterGroup!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Action", "Type": "TCoAtom"}, + {"Index": 4, "Name": "Roles", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TKiDropGroup", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiDropGroup!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "GroupName", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { "Name": "TKiOperation", "Base": "TExprBase", "Match": {"Type": "Tuple"}, diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index dd5218eece..e49f84b5e1 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -232,12 +232,12 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti return true; } -bool TTableSettings::IsSet() const { - return CompactionPolicy || PartitionBy || AutoPartitioningBySize || UniformPartitions || PartitionAtKeys - || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || KeyBloomFilter +bool TTableSettings::IsSet() const { + return CompactionPolicy || PartitionBy || AutoPartitioningBySize || UniformPartitions || PartitionAtKeys + || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || KeyBloomFilter || ReadReplicasSettings || TtlSettings; -} - +} + EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { switch (ydbStatus) { case Ydb::StatusIds::SUCCESS: @@ -258,16 +258,16 @@ EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { return TIssuesIds::KIKIMR_SCHEME_ERROR; case Ydb::StatusIds::GENERIC_ERROR: return TIssuesIds::DEFAULT_ERROR; - case Ydb::StatusIds::TIMEOUT: - return TIssuesIds::KIKIMR_TIMEOUT; + case Ydb::StatusIds::TIMEOUT: + return TIssuesIds::KIKIMR_TIMEOUT; case Ydb::StatusIds::BAD_SESSION: return TIssuesIds::KIKIMR_TOO_MANY_TRANSACTIONS; case Ydb::StatusIds::PRECONDITION_FAILED: return TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION; - case Ydb::StatusIds::CANCELLED: - return TIssuesIds::KIKIMR_OPERATION_CANCELLED; - case Ydb::StatusIds::UNSUPPORTED: - return TIssuesIds::KIKIMR_UNSUPPORTED; + case Ydb::StatusIds::CANCELLED: + return TIssuesIds::KIKIMR_OPERATION_CANCELLED; + case Ydb::StatusIds::UNSUPPORTED: + return TIssuesIds::KIKIMR_UNSUPPORTED; default: return TIssuesIds::DEFAULT_ERROR; } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 90475d7cee..7598494a2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -28,8 +28,8 @@ namespace NYql { -using NUdf::EDataSlot; - +using NUdf::EDataSlot; + class TKikimrGatewayConfig; class TKikimrClusterConfig; @@ -147,12 +147,12 @@ struct TIndexDescription { } }; -struct TColumnFamily { - TString Name; - TMaybe<TString> Data; - TMaybe<TString> Compression; -}; - +struct TColumnFamily { + TString Name; + TMaybe<TString> Data; + TMaybe<TString> Compression; +}; + struct TTtlSettings { TString ColumnName; TDuration ExpireAfter; @@ -160,23 +160,23 @@ struct TTtlSettings { static bool TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error); }; -struct TTableSettings { - TMaybe<TString> CompactionPolicy; - TVector<TString> PartitionBy; - TMaybe<TString> AutoPartitioningBySize; - TMaybe<ui64> PartitionSizeMb; - TMaybe<TString> AutoPartitioningByLoad; - TMaybe<ui64> MinPartitions; - TMaybe<ui64> MaxPartitions; - TMaybe<ui64> UniformPartitions; - TVector<TVector<std::pair<EDataSlot, TString>>> PartitionAtKeys; - TMaybe<TString> KeyBloomFilter; - TMaybe<TString> ReadReplicasSettings; +struct TTableSettings { + TMaybe<TString> CompactionPolicy; + TVector<TString> PartitionBy; + TMaybe<TString> AutoPartitioningBySize; + TMaybe<ui64> PartitionSizeMb; + TMaybe<TString> AutoPartitioningByLoad; + TMaybe<ui64> MinPartitions; + TMaybe<ui64> MaxPartitions; + TMaybe<ui64> UniformPartitions; + TVector<TVector<std::pair<EDataSlot, TString>>> PartitionAtKeys; + TMaybe<TString> KeyBloomFilter; + TMaybe<TString> ReadReplicasSettings; TResetableSetting<TTtlSettings, void> TtlSettings; - - bool IsSet() const; -}; - + + bool IsSet() const; +}; + struct TKikimrColumnMetadata { TString Name; ui32 Id = 0; @@ -296,8 +296,8 @@ struct TKikimrTableMetadata : public TThrRefBase { TVector<TIndexDescription> Indexes; TVector<TIntrusivePtr<TKikimrTableMetadata>> SecondaryGlobalIndexMetadata; - TVector<TColumnFamily> ColumnFamilies; - TTableSettings TableSettings; + TVector<TColumnFamily> ColumnFamilies; + TTableSettings TableSettings; TKikimrTableMetadata(const TString& cluster, const TString& table) : Cluster(cluster) @@ -421,43 +421,43 @@ struct TKikimrTableMetadata : public TThrRefBase { } }; -struct TCreateUserSettings { - TString UserName; - TString Password; - bool PasswordEncrypted = false; -}; - -struct TAlterUserSettings { - TString UserName; - TString Password; - bool PasswordEncrypted = false; -}; - -struct TDropUserSettings { - TString UserName; - bool Force = false; -}; - -struct TCreateGroupSettings { - TString GroupName; -}; - -struct TAlterGroupSettings { - enum class EAction : ui32 { - AddRoles = 0, - RemoveRoles = 1, - }; - - TString GroupName; - EAction Action; - std::vector<TString> Roles; -}; - -struct TDropGroupSettings { - TString GroupName; - bool Force = false; -}; - +struct TCreateUserSettings { + TString UserName; + TString Password; + bool PasswordEncrypted = false; +}; + +struct TAlterUserSettings { + TString UserName; + TString Password; + bool PasswordEncrypted = false; +}; + +struct TDropUserSettings { + TString UserName; + bool Force = false; +}; + +struct TCreateGroupSettings { + TString GroupName; +}; + +struct TAlterGroupSettings { + enum class EAction : ui32 { + AddRoles = 0, + RemoveRoles = 1, + }; + + TString GroupName; + EAction Action; + std::vector<TString> Roles; +}; + +struct TDropGroupSettings { + TString GroupName; + bool Force = false; +}; + struct TKikimrListPathItem { TKikimrListPathItem(TString name, bool isDirectory) { Name = name; @@ -603,18 +603,18 @@ public: virtual NThreading::TFuture<TGenericResult> DropTable(const TString& cluster, const TString& table) = 0; - virtual NThreading::TFuture<TGenericResult> CreateUser(const TString& cluster, const TCreateUserSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> AlterUser(const TString& cluster, const TAlterUserSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> DropUser(const TString& cluster, const TDropUserSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> CreateGroup(const TString& cluster, const TCreateGroupSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> AlterGroup(const TString& cluster, TAlterGroupSettings& settings) = 0; - - virtual NThreading::TFuture<TGenericResult> DropGroup(const TString& cluster, const TDropGroupSettings& settings) = 0; - + virtual NThreading::TFuture<TGenericResult> CreateUser(const TString& cluster, const TCreateUserSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> AlterUser(const TString& cluster, const TAlterUserSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> DropUser(const TString& cluster, const TDropUserSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> CreateGroup(const TString& cluster, const TCreateGroupSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> AlterGroup(const TString& cluster, TAlterGroupSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> DropGroup(const TString& cluster, const TDropGroupSettings& settings) = 0; + virtual TVector<TString> GetCollectedSchemeData() = 0; public: diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index 8b263bbcc6..791a617c08 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -161,33 +161,33 @@ void TestRunSimpleCommon(TIntrusivePtr<IKqpGateway> gateway) { } void CheckPolicies(Tests::TClient& client, const TString& tableName) { - auto describeResult = client.Ls(tableName); - UNIT_ASSERT(describeResult->Record.GetPathDescription().HasTableStats()); - const auto& desc = describeResult->Record.GetPathDescription(); - UNIT_ASSERT_VALUES_EQUAL(desc.GetTableStats().GetPartCount(), 4); - for (const auto column : desc.GetTable().GetColumns()) { - if (column.GetName() == "Column2") { - UNIT_ASSERT_VALUES_EQUAL(column.GetFamilyName(), "Family2"); - } - } - for (const auto family : desc.GetTable().GetPartitionConfig().GetColumnFamilies()) { - if (family.HasId() && family.GetId() == 0) { - UNIT_ASSERT_VALUES_EQUAL(static_cast<size_t>(family.GetColumnCodec()), + auto describeResult = client.Ls(tableName); + UNIT_ASSERT(describeResult->Record.GetPathDescription().HasTableStats()); + const auto& desc = describeResult->Record.GetPathDescription(); + UNIT_ASSERT_VALUES_EQUAL(desc.GetTableStats().GetPartCount(), 4); + for (const auto column : desc.GetTable().GetColumns()) { + if (column.GetName() == "Column2") { + UNIT_ASSERT_VALUES_EQUAL(column.GetFamilyName(), "Family2"); + } + } + for (const auto family : desc.GetTable().GetPartitionConfig().GetColumnFamilies()) { + if (family.HasId() && family.GetId() == 0) { + UNIT_ASSERT_VALUES_EQUAL(static_cast<size_t>(family.GetColumnCodec()), static_cast<size_t>(NKikimrSchemeOp::ColumnCodecPlain)); - } else if (family.HasName() && family.GetName() == "Family2") { - UNIT_ASSERT_VALUES_EQUAL(static_cast<size_t>(family.GetColumnCodec()), + } else if (family.HasName() && family.GetName() == "Family2") { + UNIT_ASSERT_VALUES_EQUAL(static_cast<size_t>(family.GetColumnCodec()), static_cast<size_t>(NKikimrSchemeOp::ColumnCodecLZ4)); - } - } -} - + } + } +} + struct TTestIndexSettings { const bool WithDataColumns; }; void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient& client, bool createFolders = true, const TMaybe<TTestIndexSettings> withIndex = Nothing(), bool withExtendedDdl = false, - const TMaybe<bool>& shouldCreate = Nothing()) { + const TMaybe<bool>& shouldCreate = Nothing()) { auto metadata = MakeIntrusive<TKikimrTableMetadata>(); metadata->Cluster = TestCluster; @@ -200,11 +200,11 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient metadata->Columns.insert(std::make_pair("Column2", TKikimrColumnMetadata{"Column2", 0, "String", false})); metadata->ColumnOrder.push_back("Column2"); - - if (withExtendedDdl) { - metadata->Columns["Column2"].Families.push_back("Family2"); - } - + + if (withExtendedDdl) { + metadata->Columns["Column2"].Families.push_back("Family2"); + } + metadata->KeyColumnNames.push_back("Column1"); if (withIndex) { @@ -227,19 +227,19 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient metadata->Indexes.push_back(indexDesc); } - if (withExtendedDdl) { - metadata->TableSettings.AutoPartitioningBySize = "disabled"; - metadata->TableSettings.PartitionAtKeys = { - {std::make_pair(EDataSlot::Uint32, "10")}, - {std::make_pair(EDataSlot::Uint32, "100")}, - {std::make_pair(EDataSlot::Uint32, "1000")} - }; - metadata->ColumnFamilies = { - {"default", "test", "off"}, - {"Family2", "test", "lz4"} - }; - } - + if (withExtendedDdl) { + metadata->TableSettings.AutoPartitioningBySize = "disabled"; + metadata->TableSettings.PartitionAtKeys = { + {std::make_pair(EDataSlot::Uint32, "10")}, + {std::make_pair(EDataSlot::Uint32, "100")}, + {std::make_pair(EDataSlot::Uint32, "1000")} + }; + metadata->ColumnFamilies = { + {"default", "test", "off"}, + {"Family2", "test", "lz4"} + }; + } + auto responseFuture = gateway->CreateTable(metadata, createFolders); responseFuture.Wait(); auto response = responseFuture.GetValue(); @@ -282,10 +282,10 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient UNIT_ASSERT_VALUES_EQUAL(indexDescResult.second.DataColumns[i], expectedDesc->second.DataColumns[i]); } } - - if (withExtendedDdl) { + + if (withExtendedDdl) { CheckPolicies(client, metadata->Name); - } + } } } @@ -371,12 +371,12 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { CreateSampleTables(kikimr); TestDropTableCommon(GetIcGateway(kikimr.GetTestServer())); } - - Y_UNIT_TEST(TestCreateTableWithExtendedDdl) { + + Y_UNIT_TEST(TestCreateTableWithExtendedDdl) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); CreateSampleTables(kikimr); TestCreateTableCommon(GetIcGateway(kikimr.GetTestServer()), kikimr.GetTestClient(), true, Nothing(), true); - } + } } } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_kql.cpp b/ydb/core/kqp/provider/yql_kikimr_kql.cpp index bf42b79cdf..95f5915a57 100644 --- a/ydb/core/kqp/provider/yql_kikimr_kql.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_kql.cpp @@ -274,7 +274,7 @@ TExprNode::TPtr KiUpsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, valueTuples.push_back(tuple); } else if (replace) { auto type = table.GetColumnType(name); - YQL_ENSURE(type, "No such column: " << name); + YQL_ENSURE(type, "No such column: " << name); auto tuple = Build<TCoNameValueTuple>(ctx, node.Pos()) .Name().Build(name) .Value<TCoNothing>() @@ -325,7 +325,7 @@ TExprNode::TPtr KiUpsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, } TExprNode::TPtr KiInsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, const TKikimrTableDescription& table, - const TYdbOperation& op, TExprNode::TPtr& effect) + const TYdbOperation& op, TExprNode::TPtr& effect) { auto fetchItemArg = Build<TCoArgument>(ctx, node.Pos()) .Name("fetchItem") @@ -394,14 +394,14 @@ TExprNode::TPtr KiInsertTableToKql(const TKiWriteTable& node, TExprContext& ctx, TExprNode::TPtr insertEffect; auto insertKql = KiUpsertTableToKql(node, ctx, table, false, true, insertEffect); - if (op == TYdbOperation::InsertAbort) { + if (op == TYdbOperation::InsertAbort) { effect = Build<TKiAbortIf>(ctx, node.Pos()) .Predicate(predicate) .Effect(insertEffect) .Constraint().Build("insert_pk") .Done() .Ptr(); - } else if (op == TYdbOperation::InsertRevert) { + } else if (op == TYdbOperation::InsertRevert) { effect = Build<TKiRevertIf>(ctx, node.Pos()) .Predicate(predicate) .Effect(insertEffect) @@ -515,7 +515,7 @@ TExprNode::TPtr KiDeleteOnTableToKql(const TKiWriteTable& node, TExprContext& ct .Ptr(); } -TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns) { +TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns) { const auto& read = right.Input().Cast<TKiReadTable>(); bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); @@ -560,7 +560,7 @@ TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikim .Cluster(cluster) .Table(versionedTable) .Range(range.ToRangeExpr(read, ctx)) - .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) + .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) .Settings() .Build() .IndexName() @@ -580,7 +580,7 @@ TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikim .Cluster(cluster) .Table(versionedTable) .Range(range.ToRangeExpr(read, ctx)) - .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) + .Select(read.GetSelectColumns(ctx, tablesData, withSystemColumns)) .Settings() .Build() .Done(); @@ -790,15 +790,15 @@ TExprNode::TPtr KiWriteTableToKql(TKiWriteTable write, TExprContext& ctx, auto& tableDesc = tablesData.ExistingTable(TString(cluster), TString(table)); switch (op) { - case TYdbOperation::Upsert: - case TYdbOperation::Replace: - return KiUpsertTableToKql(write, ctx, tableDesc, op == TYdbOperation::Replace, false, effect); - case TYdbOperation::InsertRevert: - case TYdbOperation::InsertAbort: + case TYdbOperation::Upsert: + case TYdbOperation::Replace: + return KiUpsertTableToKql(write, ctx, tableDesc, op == TYdbOperation::Replace, false, effect); + case TYdbOperation::InsertRevert: + case TYdbOperation::InsertAbort: return KiInsertTableToKql(write, ctx, tableDesc, op, effect); - case TYdbOperation::DeleteOn: + case TYdbOperation::DeleteOn: return KiDeleteOnTableToKql(write, ctx, tableDesc, effect); - case TYdbOperation::UpdateOn: + case TYdbOperation::UpdateOn: return KiUpdateOnTableToKql(write, ctx, tableDesc, effect); default: return nullptr; @@ -806,7 +806,7 @@ TExprNode::TPtr KiWriteTableToKql(TKiWriteTable write, TExprContext& ctx, } TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, - const TKikimrTablesData& tablesData, TExprNode::TPtr& effect, bool withSystemColumns) + const TKikimrTablesData& tablesData, TExprNode::TPtr& effect, bool withSystemColumns) { YQL_ENSURE(update.Update().Ref().GetTypeAnn()); @@ -987,7 +987,7 @@ TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, } TExprNode::TPtr KiDeleteTableToKql(TKiDeleteTable del, TExprContext& ctx, - const TKikimrTablesData& tablesData, TExprNode::TPtr& effect, bool withSystemColumns) + const TKikimrTablesData& tablesData, TExprNode::TPtr& effect, bool withSystemColumns) { const auto& cluster = del.DataSink().Cluster(); const auto& table = del.Table(); @@ -1094,23 +1094,23 @@ TExprNode::TPtr KiDeleteTableToKql(TKiDeleteTable del, TExprContext& ctx, } // namespace TKiProgram BuildKiProgram(TKiDataQuery query, const TKikimrTablesData& tablesData, - TExprContext& ctx, bool withSystemColumns) + TExprContext& ctx, bool withSystemColumns) { TExprNode::TPtr optResult; TOptimizeExprSettings optSettings(nullptr); optSettings.VisitChanges = true; IGraphTransformer::TStatus status(IGraphTransformer::TStatus::Ok); status = OptimizeExpr(query.Effects().Ptr(), optResult, - [&tablesData, withSystemColumns](const TExprNode::TPtr& input, TExprContext& ctx) { + [&tablesData, withSystemColumns](const TExprNode::TPtr& input, TExprContext& ctx) { auto node = TExprBase(input); auto ret = input; if (auto maybeWrite = node.Maybe<TKiWriteTable>()) { KiWriteTableToKql(maybeWrite.Cast(), ctx, tablesData, ret); } else if (auto maybeUpdate = node.Maybe<TKiUpdateTable>()) { - KiUpdateTableToKql(maybeUpdate.Cast(), ctx, tablesData, ret, withSystemColumns); + KiUpdateTableToKql(maybeUpdate.Cast(), ctx, tablesData, ret, withSystemColumns); } else if (auto maybeDelete = node.Maybe<TKiDeleteTable>()) { - KiDeleteTableToKql(maybeDelete.Cast(), ctx, tablesData, ret, withSystemColumns); + KiDeleteTableToKql(maybeDelete.Cast(), ctx, tablesData, ret, withSystemColumns); } return ret; @@ -1149,11 +1149,11 @@ TKiProgram BuildKiProgram(TKiDataQuery query, const TKikimrTablesData& tablesDat TExprNode::TPtr newProgram; status = OptimizeExpr(program.Ptr(), newProgram, - [&tablesData, withSystemColumns](const TExprNode::TPtr& input, TExprContext& ctx) { + [&tablesData, withSystemColumns](const TExprNode::TPtr& input, TExprContext& ctx) { auto node = TExprBase(input); if (node.Maybe<TCoRight>().Input().Maybe<TKiReadTable>()) { - return KiReadTableToKql(node.Cast<TCoRight>(), ctx, tablesData, withSystemColumns); + return KiReadTableToKql(node.Cast<TCoRight>(), ctx, tablesData, withSystemColumns); } return input; @@ -1175,7 +1175,7 @@ TExprBase UnwrapKiReadTableValues(TExprBase input, const TKikimrTableDescription TVector<TExprBase> structItems; for (auto atom : columns) { auto columnType = tableDesc.GetColumnType(TString(atom.Value())); - YQL_ENSURE(columnType); + YQL_ENSURE(columnType); auto item = Build<TCoNameValueTuple>(ctx, input.Pos()) .Name(atom) diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index 5d6f80a193..0ba0b34952 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -10,7 +10,7 @@ namespace { using namespace NNodes; using namespace NCommon; -TKiOperation BuildTableOpNode(const TCoAtom& cluster, const TStringBuf& table, TYdbOperation op, TPositionHandle pos, +TKiOperation BuildTableOpNode(const TCoAtom& cluster, const TStringBuf& table, TYdbOperation op, TPositionHandle pos, TExprContext& ctx) { return Build<TKiOperation>(ctx, pos) @@ -20,14 +20,14 @@ TKiOperation BuildTableOpNode(const TCoAtom& cluster, const TStringBuf& table, T .Done(); } -TKiOperation BuildYdbOpNode(const TCoAtom& cluster, TYdbOperation op, TPositionHandle pos, TExprContext& ctx) { - return Build<TKiOperation>(ctx, pos) - .Cluster().Build(cluster) - .Table().Build("") - .Operation<TCoAtom>().Build(ToString(op)) - .Done(); -} - +TKiOperation BuildYdbOpNode(const TCoAtom& cluster, TYdbOperation op, TPositionHandle pos, TExprContext& ctx) { + return Build<TKiOperation>(ctx, pos) + .Cluster().Build(cluster) + .Table().Build("") + .Operation<TCoAtom>().Build(ToString(op)) + .Done(); +} + TCoAtomList GetResultColumns(const TResWriteBase& resWrite, TExprContext& ctx) { TExprNode::TListType columns; auto columnsSetting = GetSetting(resWrite.Settings().Ref(), "columns"); @@ -83,7 +83,7 @@ struct TKiExploreTxResults { hasScheme = false; hasData = false; for (auto& node : TableOperations) { - auto op = FromString<TYdbOperation>(TString(node.Operation())); + auto op = FromString<TYdbOperation>(TString(node.Operation())); hasScheme = hasScheme || (op & KikimrSchemeOps()); hasData = hasData || (op & KikimrDataOps()); } @@ -130,7 +130,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = key.GetTablePath(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(maybeRead.Cast().World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Select, read.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Select, read.Pos(), ctx)); return result; } @@ -158,7 +158,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = update.Table().Value(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(update.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Update, update.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Update, update.Pos(), ctx)); txRes.Effects.push_back(node); return result; } @@ -172,7 +172,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = del.Table().Value(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(del.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Delete, del.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::Delete, del.Pos(), ctx)); txRes.Effects.push_back(node); return result; } @@ -186,7 +186,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = create.Table().Value(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(create.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::CreateTable, create.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::CreateTable, create.Pos(), ctx)); return result; } @@ -199,7 +199,7 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = drop.Table().Value(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(drop.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::DropTable, drop.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::DropTable, drop.Pos(), ctx)); return result; } @@ -212,82 +212,82 @@ bool ExploreTx(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, T auto table = alter.Table().Value(); txRes.Ops.insert(node.Raw()); auto result = ExploreTx(alter.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::AlterTable, alter.Pos(), ctx)); + txRes.TableOperations.push_back(BuildTableOpNode(cluster, table, TYdbOperation::AlterTable, alter.Pos(), ctx)); + return result; + } + + if (auto maybeCreateUser = node.Maybe<TKiCreateUser>()) { + auto createUser = maybeCreateUser.Cast(); + if (!checkDataSink(createUser.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(createUser.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::CreateUser, createUser.Pos(), ctx)); + return result; + } + + if (auto maybeAlterUser = node.Maybe<TKiAlterUser>()) { + auto alterUser = maybeAlterUser.Cast(); + if (!checkDataSink(alterUser.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(alterUser.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::AlterUser, alterUser.Pos(), ctx)); + return result; + } + + if (auto maybeDropUser = node.Maybe<TKiDropUser>()) { + auto dropUser = maybeDropUser.Cast(); + if (!checkDataSink(dropUser.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(dropUser.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::DropUser, dropUser.Pos(), ctx)); + return result; + } + + if (auto maybeCreateGroup = node.Maybe<TKiCreateGroup>()) { + auto createGroup = maybeCreateGroup.Cast(); + if (!checkDataSink(createGroup.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(createGroup.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::CreateGroup, createGroup.Pos(), ctx)); + return result; + } + + if (auto maybeAlterGroup = node.Maybe<TKiAlterGroup>()) { + auto alterGroup = maybeAlterGroup.Cast(); + if (!checkDataSink(alterGroup.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(alterGroup.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::AlterGroup, alterGroup.Pos(), ctx)); + return result; + } + + if (auto maybeDropGroup = node.Maybe<TKiDropGroup>()) { + auto dropGroup = maybeDropGroup.Cast(); + if (!checkDataSink(dropGroup.DataSink())) { + return false; + } + + txRes.Ops.insert(node.Raw()); + auto result = ExploreTx(dropGroup.World(), ctx, dataSink, txRes); + txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::DropGroup, dropGroup.Pos(), ctx)); return result; } - if (auto maybeCreateUser = node.Maybe<TKiCreateUser>()) { - auto createUser = maybeCreateUser.Cast(); - if (!checkDataSink(createUser.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(createUser.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::CreateUser, createUser.Pos(), ctx)); - return result; - } - - if (auto maybeAlterUser = node.Maybe<TKiAlterUser>()) { - auto alterUser = maybeAlterUser.Cast(); - if (!checkDataSink(alterUser.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(alterUser.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::AlterUser, alterUser.Pos(), ctx)); - return result; - } - - if (auto maybeDropUser = node.Maybe<TKiDropUser>()) { - auto dropUser = maybeDropUser.Cast(); - if (!checkDataSink(dropUser.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(dropUser.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::DropUser, dropUser.Pos(), ctx)); - return result; - } - - if (auto maybeCreateGroup = node.Maybe<TKiCreateGroup>()) { - auto createGroup = maybeCreateGroup.Cast(); - if (!checkDataSink(createGroup.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(createGroup.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::CreateGroup, createGroup.Pos(), ctx)); - return result; - } - - if (auto maybeAlterGroup = node.Maybe<TKiAlterGroup>()) { - auto alterGroup = maybeAlterGroup.Cast(); - if (!checkDataSink(alterGroup.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(alterGroup.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::AlterGroup, alterGroup.Pos(), ctx)); - return result; - } - - if (auto maybeDropGroup = node.Maybe<TKiDropGroup>()) { - auto dropGroup = maybeDropGroup.Cast(); - if (!checkDataSink(dropGroup.DataSink())) { - return false; - } - - txRes.Ops.insert(node.Raw()); - auto result = ExploreTx(dropGroup.World(), ctx, dataSink, txRes); - txRes.TableOperations.push_back(BuildYdbOpNode(cluster, TYdbOperation::DropGroup, dropGroup.Pos(), ctx)); - return result; - } - if (auto maybeExecQuery = node.Maybe<TKiExecDataQuery>()) { auto execQuery = maybeExecQuery.Cast(); if (!checkDataSink(execQuery.DataSink())) { @@ -625,20 +625,20 @@ TExprNode::TPtr KiBuildResult(TExprBase node, const TString& cluster, TExprCont return ctx.ChangeChild(*ctx.ChangeChild(resFill.Ref(), 0, world.Ptr()), 3, data.Ptr()); } -TYdbOperation GetTableOp(const TKiWriteTable& write) { +TYdbOperation GetTableOp(const TKiWriteTable& write) { auto mode = write.Mode().Value(); if (mode == "upsert") { - return TYdbOperation::Upsert; + return TYdbOperation::Upsert; } else if (mode == "replace") { - return TYdbOperation::Replace; + return TYdbOperation::Replace; } else if (mode == "insert_revert") { - return TYdbOperation::InsertRevert; + return TYdbOperation::InsertRevert; } else if (mode == "insert_abort") { - return TYdbOperation::InsertAbort; + return TYdbOperation::InsertAbort; } else if (mode == "delete_on") { - return TYdbOperation::DeleteOn; + return TYdbOperation::DeleteOn; } else if (mode == "update_on") { - return TYdbOperation::UpdateOn; + return TYdbOperation::UpdateOn; } YQL_ENSURE(false, "Unexpected TKiWriteTable mode: " << mode); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 2ede51c810..635d164827 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -25,14 +25,14 @@ struct TKikimrData { THashSet<TStringBuf> CommitModes; THashSet<TStringBuf> SupportedEffects; - TYdbOperations SchemeOps; - TYdbOperations DataOps; - TYdbOperations ModifyOps; - TYdbOperations ReadOps; - TYdbOperations RequireUnmodifiedOps; - - TMap<TString, NKikimr::NUdf::EDataSlot> SystemColumns; - + TYdbOperations SchemeOps; + TYdbOperations DataOps; + TYdbOperations ModifyOps; + TYdbOperations ReadOps; + TYdbOperations RequireUnmodifiedOps; + + TMap<TString, NKikimr::NUdf::EDataSlot> SystemColumns; + TKikimrData() { DataSourceNames.insert(TKiReadTable::CallableName()); DataSourceNames.insert(TKiReadTableScheme::CallableName()); @@ -45,12 +45,12 @@ struct TKikimrData { DataSinkNames.insert(TKiCreateTable::CallableName()); DataSinkNames.insert(TKiAlterTable::CallableName()); DataSinkNames.insert(TKiDropTable::CallableName()); - DataSinkNames.insert(TKiCreateUser::CallableName()); - DataSinkNames.insert(TKiAlterUser::CallableName()); - DataSinkNames.insert(TKiDropUser::CallableName()); - DataSinkNames.insert(TKiCreateGroup::CallableName()); - DataSinkNames.insert(TKiAlterGroup::CallableName()); - DataSinkNames.insert(TKiDropGroup::CallableName()); + DataSinkNames.insert(TKiCreateUser::CallableName()); + DataSinkNames.insert(TKiAlterUser::CallableName()); + DataSinkNames.insert(TKiDropUser::CallableName()); + DataSinkNames.insert(TKiCreateGroup::CallableName()); + DataSinkNames.insert(TKiAlterGroup::CallableName()); + DataSinkNames.insert(TKiDropGroup::CallableName()); DataSinkNames.insert(TKiDataQuery::CallableName()); DataSinkNames.insert(TKiExecDataQuery::CallableName()); DataSinkNames.insert(TKiEffects::CallableName()); @@ -77,41 +77,41 @@ struct TKikimrData { SupportedEffects.insert(TKiDeleteTable::CallableName()); ModifyOps = - TYdbOperation::Upsert | - TYdbOperation::Replace | - TYdbOperation::Update | - TYdbOperation::UpdateOn | - TYdbOperation::Delete | - TYdbOperation::DeleteOn | - TYdbOperation::InsertRevert | - TYdbOperation::InsertAbort; + TYdbOperation::Upsert | + TYdbOperation::Replace | + TYdbOperation::Update | + TYdbOperation::UpdateOn | + TYdbOperation::Delete | + TYdbOperation::DeleteOn | + TYdbOperation::InsertRevert | + TYdbOperation::InsertAbort; ReadOps = - TYdbOperation::Select | - TYdbOperation::Update | - TYdbOperation::Delete; + TYdbOperation::Select | + TYdbOperation::Update | + TYdbOperation::Delete; DataOps = ModifyOps | ReadOps; SchemeOps = - TYdbOperation::CreateTable | - TYdbOperation::DropTable | - TYdbOperation::AlterTable | - TYdbOperation::CreateUser | - TYdbOperation::AlterUser | - TYdbOperation::DropUser | - TYdbOperation::CreateGroup | - TYdbOperation::AlterGroup | - TYdbOperation::DropGroup; + TYdbOperation::CreateTable | + TYdbOperation::DropTable | + TYdbOperation::AlterTable | + TYdbOperation::CreateUser | + TYdbOperation::AlterUser | + TYdbOperation::DropUser | + TYdbOperation::CreateGroup | + TYdbOperation::AlterGroup | + TYdbOperation::DropGroup; RequireUnmodifiedOps = - TYdbOperation::InsertRevert | - TYdbOperation::InsertAbort | - TYdbOperation::UpdateOn; // TODO: KIKIMR-3206 - - SystemColumns = { - {"_yql_partition_id", NKikimr::NUdf::EDataSlot::Uint64} - }; + TYdbOperation::InsertRevert | + TYdbOperation::InsertAbort | + TYdbOperation::UpdateOn; // TODO: KIKIMR-3206 + + SystemColumns = { + {"_yql_partition_id", NKikimr::NUdf::EDataSlot::Uint64} + }; } }; @@ -165,7 +165,7 @@ const TKikimrTableDescription& TKikimrTablesData::ExistingTable(const TStringBuf return *desc; } -bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { +bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { ColumnTypes.clear(); TVector<const TItemExprType*> items; @@ -194,16 +194,16 @@ bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { YQL_ENSURE(insertResult.second); } - if (withSystemColumns) { - for (const auto& [name, type] : KikimrSystemColumns()) { - const TOptionalExprType* optType = ctx.MakeType<TOptionalExprType>(ctx.MakeType<TDataExprType>(type)); - items.push_back(ctx.MakeType<TItemExprType>(name, optType)); - - auto insertResult = ColumnTypes.insert(std::make_pair(name, optType)); - YQL_ENSURE(insertResult.second); - } - } - + if (withSystemColumns) { + for (const auto& [name, type] : KikimrSystemColumns()) { + const TOptionalExprType* optType = ctx.MakeType<TOptionalExprType>(ctx.MakeType<TDataExprType>(type)); + items.push_back(ctx.MakeType<TItemExprType>(name, optType)); + + auto insertResult = ColumnTypes.insert(std::make_pair(name, optType)); + YQL_ENSURE(insertResult.second); + } + } + SchemeNode = ctx.MakeType<TStructExprType>(items); return true; } @@ -340,18 +340,18 @@ bool TKikimrKey::Extract(const TExprNode& key) { return false; } - Target = nameNode->Child(0)->Content(); + Target = nameNode->Child(0)->Content(); } else if (tagName == "tablescheme") { KeyType = Type::TableScheme; - Target = key.Child(0)->Child(1)->Child(0)->Content(); + Target = key.Child(0)->Child(1)->Child(0)->Content(); } else if (tagName == "tablelist") { KeyType = Type::TableList; - Target = key.Child(0)->Child(1)->Child(0)->Content(); - } else if (tagName == "role") { - KeyType = Type::Role; - Target = key.Child(0)->Child(1)->Child(0)->Content(); + Target = key.Child(0)->Child(1)->Child(0)->Content(); + } else if (tagName == "role") { + KeyType = Type::Role; + Target = key.Child(0)->Child(1)->Child(0)->Content(); } else { - Ctx.AddError(TIssue(Ctx.GetPosition(key.Child(0)->Pos()), TString("Unexpected tag for kikimr key: ") + tagName)); + Ctx.AddError(TIssue(Ctx.GetPosition(key.Child(0)->Pos()), TString("Unexpected tag for kikimr key: ") + tagName)); return false; } @@ -376,7 +376,7 @@ bool TKikimrKey::Extract(const TExprNode& key) { View = viewNode->Child(0)->Content(); } else { - Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), TStringBuilder() << "Unexpected tag for kikimr key child: " << tag->Content())); + Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), TStringBuilder() << "Unexpected tag for kikimr key child: " << tag->Content())); return false; } } @@ -393,30 +393,30 @@ NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metada } NNodes::TCoAtomList BuildColumnsList( - const TKikimrTableDescription& table, - TPositionHandle pos, - TExprContext& ctx, - bool withSystemColumns -) { + const TKikimrTableDescription& table, + TPositionHandle pos, + TExprContext& ctx, + bool withSystemColumns +) { TVector<TExprBase> columnsToSelect; - for (const auto& pair : table.Metadata->Columns) { + for (const auto& pair : table.Metadata->Columns) { auto atom = Build<TCoAtom>(ctx, pos) .Value(pair.second.Name) .Done(); - columnsToSelect.emplace_back(std::move(atom)); + columnsToSelect.emplace_back(std::move(atom)); + } + + if (withSystemColumns) { + for (const auto& pair : KikimrSystemColumns()) { + auto atom = Build<TCoAtom>(ctx, pos) + .Value(pair.first) + .Done(); + + columnsToSelect.emplace_back(std::move(atom)); + } } - if (withSystemColumns) { - for (const auto& pair : KikimrSystemColumns()) { - auto atom = Build<TCoAtom>(ctx, pos) - .Value(pair.first) - .Done(); - - columnsToSelect.emplace_back(std::move(atom)); - } - } - return Build<TCoAtomList>(ctx, pos) .Add(columnsToSelect) .Done(); @@ -488,7 +488,7 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TCoNameValueTupleLi TVector<NKqpProto::TKqpTableOp> protoOps; for (const auto& op : operations) { auto table = TString(op.Name()); - auto tableOp = FromString<TYdbOperation>(TString(op.Value().Cast<TCoAtom>())); + auto tableOp = FromString<TYdbOperation>(TString(op.Value().Cast<TCoAtom>())); auto pos = ctx.GetPosition(op.Pos()); NKqpProto::TKqpTableOp protoOp; @@ -507,7 +507,7 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TKiOperationList& o TVector<NKqpProto::TKqpTableOp> protoOps; for (const auto& op : operations) { auto table = TString(op.Table()); - auto tableOp = FromString<TYdbOperation>(TString(op.Operation())); + auto tableOp = FromString<TYdbOperation>(TString(op.Operation())); auto pos = ctx.GetPosition(op.Pos()); NKqpProto::TKqpTableOp protoOp; @@ -565,7 +565,7 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto for (const auto& op : operations) { const auto& table = op.GetTable(); - auto newOp = TYdbOperation(op.GetOperation()); + auto newOp = TYdbOperation(op.GetOperation()); TPosition pos(op.GetPosition().GetColumn(), op.GetPosition().GetRow()); const auto info = tableInfoMap.FindPtr(table); @@ -644,8 +644,8 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto } // TODO: KIKIMR-3206 - bool currentDelete = currentOps & (TYdbOperation::Delete | TYdbOperation::DeleteOn); - bool newUpdate = newOp == TYdbOperation::Update; + bool currentDelete = currentOps & (TYdbOperation::Delete | TYdbOperation::DeleteOn); + bool newUpdate = newOp == TYdbOperation::Update; if (currentDelete && newUpdate) { TString message = TStringBuilder() << "Operation '" << newOp << "' may lead to unexpected results when applied to table with deleted rows: " << table; @@ -692,34 +692,34 @@ const TStringBuf& KikimrCommitModeScheme() { return CommitModeScheme; } -const TYdbOperations& KikimrSchemeOps() { +const TYdbOperations& KikimrSchemeOps() { return Singleton<TKikimrData>()->SchemeOps; } -const TYdbOperations& KikimrDataOps() { +const TYdbOperations& KikimrDataOps() { return Singleton<TKikimrData>()->DataOps; } -const TYdbOperations& KikimrModifyOps() { +const TYdbOperations& KikimrModifyOps() { return Singleton<TKikimrData>()->ModifyOps; } -const TYdbOperations& KikimrReadOps() { +const TYdbOperations& KikimrReadOps() { return Singleton<TKikimrData>()->ReadOps; } -const TYdbOperations& KikimrRequireUnmodifiedOps() { +const TYdbOperations& KikimrRequireUnmodifiedOps() { return Singleton<TKikimrData>()->RequireUnmodifiedOps; } -const TMap<TString, NKikimr::NUdf::EDataSlot>& KikimrSystemColumns() { - return Singleton<TKikimrData>()->SystemColumns; -} - -bool IsKikimrSystemColumn(const TStringBuf columnName) { - return KikimrSystemColumns().FindPtr(columnName); -} - +const TMap<TString, NKikimr::NUdf::EDataSlot>& KikimrSystemColumns() { + return Singleton<TKikimrData>()->SystemColumns; +} + +bool IsKikimrSystemColumn(const TStringBuf columnName) { + return KikimrSystemColumns().FindPtr(columnName); +} + bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos) { if (metadata->Indexes.empty()) { ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index cb1df6c021..6903e6f193 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -8,7 +8,7 @@ #include <ydb/library/yql/core/yql_type_annotation.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> -#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/actor.h> #include <library/cpp/cache/cache.h> #include <util/generic/strbuf.h> @@ -71,8 +71,8 @@ enum class EKikimrQueryType { Ddl, YqlScript, YqlInternal, - Scan, - YqlScriptStreaming, + Scan, + YqlScriptStreaming, }; struct TKikimrQueryContext : TThrRefBase { @@ -108,9 +108,9 @@ struct TKikimrQueryContext : TThrRefBase { THashMap<ui64, TIntrusivePtr<IKikimrQueryExecutor::TAsyncQueryResult>> InProgress; TVector<ui64> ExecutionOrder; - NActors::TActorId ReplyTarget; + NActors::TActorId ReplyTarget; TMaybe<NKikimrKqp::TRlPath> RlPath; - + TIntrusivePtr<ITimeProvider> TimeProvider; TIntrusivePtr<IRandomProvider> RandomProvider; @@ -187,7 +187,7 @@ public: const TStructExprType* SchemeNode = nullptr; TMaybe<TString> RelativePath; - bool Load(TExprContext& ctx, bool withVirtualColumns = false); + bool Load(TExprContext& ctx, bool withVirtualColumns = false); void ToYson(NYson::TYsonWriter& writer) const; TMaybe<ui32> GetKeyColumnIndex(const TString& name) const; @@ -231,10 +231,10 @@ private: THashMap<std::pair<TString, TString>, TKikimrTableDescription> Tables; }; -enum class TYdbOperation : ui32 { - CreateTable = 1 << 0, - DropTable = 1 << 1, - AlterTable = 1 << 2, +enum class TYdbOperation : ui32 { + CreateTable = 1 << 0, + DropTable = 1 << 1, + AlterTable = 1 << 2, Select = 1 << 3, Upsert = 1 << 4, Replace = 1 << 5, @@ -245,16 +245,16 @@ enum class TYdbOperation : ui32 { ReservedInsertIgnore = 1 << 10, UpdateOn = 1 << 11, DeleteOn = 1 << 12, - CreateUser = 1 << 13, - AlterUser = 1 << 14, - DropUser = 1 << 15, - CreateGroup = 1 << 16, - AlterGroup = 1 << 17, - DropGroup = 1 << 18 + CreateUser = 1 << 13, + AlterUser = 1 << 14, + DropUser = 1 << 15, + CreateGroup = 1 << 16, + AlterGroup = 1 << 17, + DropGroup = 1 << 18 }; -Y_DECLARE_FLAGS(TYdbOperations, TYdbOperation) -Y_DECLARE_OPERATORS_FOR_FLAGS(TYdbOperations) +Y_DECLARE_FLAGS(TYdbOperations, TYdbOperation) +Y_DECLARE_OPERATORS_FOR_FLAGS(TYdbOperations) class IKikimrTransactionContext : public TThrRefBase { public: @@ -273,7 +273,7 @@ public: class TKikimrTransactionContextBase : public IKikimrTransactionContext { public: - THashMap<TString, TYdbOperations> TableOperations; + THashMap<TString, TYdbOperations> TableOperations; THashMap<TKikimrPathId, TString> TableByIdMap; TMaybe<NKikimrKqp::EIsolationLevel> EffectiveIsolationLevel; bool Readonly = false; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index fb18cc35cb..e1aa3cda2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -34,12 +34,12 @@ private: virtual TStatus HandleCreateTable(NNodes::TKiCreateTable node, TExprContext& ctx) = 0; virtual TStatus HandleAlterTable(NNodes::TKiAlterTable node, TExprContext& ctx) = 0; virtual TStatus HandleDropTable(NNodes::TKiDropTable node, TExprContext& ctx) = 0; - virtual TStatus HandleCreateUser(NNodes::TKiCreateUser node, TExprContext& ctx) = 0; - virtual TStatus HandleAlterUser(NNodes::TKiAlterUser node, TExprContext& ctx) = 0; - virtual TStatus HandleDropUser(NNodes::TKiDropUser node, TExprContext& ctx) = 0; - virtual TStatus HandleCreateGroup(NNodes::TKiCreateGroup node, TExprContext& ctx) = 0; - virtual TStatus HandleAlterGroup(NNodes::TKiAlterGroup node, TExprContext& ctx) = 0; - virtual TStatus HandleDropGroup(NNodes::TKiDropGroup node, TExprContext& ctx) = 0; + virtual TStatus HandleCreateUser(NNodes::TKiCreateUser node, TExprContext& ctx) = 0; + virtual TStatus HandleAlterUser(NNodes::TKiAlterUser node, TExprContext& ctx) = 0; + virtual TStatus HandleDropUser(NNodes::TKiDropUser node, TExprContext& ctx) = 0; + virtual TStatus HandleCreateGroup(NNodes::TKiCreateGroup node, TExprContext& ctx) = 0; + virtual TStatus HandleAlterGroup(NNodes::TKiAlterGroup node, TExprContext& ctx) = 0; + virtual TStatus HandleDropGroup(NNodes::TKiDropGroup node, TExprContext& ctx) = 0; virtual TStatus HandleWrite(NNodes::TExprBase node, TExprContext& ctx) = 0; virtual TStatus HandleCommit(NNodes::TCoCommit node, TExprContext& ctx) = 0; virtual TStatus HandleKql(NNodes::TCallable node, TExprContext& ctx) = 0; @@ -53,8 +53,8 @@ public: enum class Type { Table, TableList, - TableScheme, - Role + TableScheme, + Role }; public: @@ -69,21 +69,21 @@ public: TString GetTablePath() const { Y_VERIFY_DEBUG(KeyType.Defined()); Y_VERIFY_DEBUG(KeyType == Type::Table || KeyType == Type::TableScheme); - return Target; + return Target; } TString GetFolderPath() const { Y_VERIFY_DEBUG(KeyType.Defined()); Y_VERIFY_DEBUG(KeyType == Type::TableList); - return Target; + return Target; + } + + TString GetRoleName() const { + Y_VERIFY_DEBUG(KeyType.Defined()); + Y_VERIFY_DEBUG(KeyType == Type::Role); + return Target; } - TString GetRoleName() const { - Y_VERIFY_DEBUG(KeyType.Defined()); - Y_VERIFY_DEBUG(KeyType == Type::Role); - return Target; - } - const TMaybe<TString>& GetView() const { return View; } @@ -93,7 +93,7 @@ public: private: TExprContext& Ctx; TMaybe<Type> KeyType; - TString Target; + TString Target; TMaybe<TString> View; }; @@ -206,11 +206,11 @@ NNodes::TExprBase UnwrapKiReadTableValues(NNodes::TExprBase input, const TKikimr NNodes::TKiVersionedTable BuildVersionedTable(const TKikimrTableMetadata& metadata, TPositionHandle pos, TExprContext& ctx); NNodes::TCoAtomList BuildColumnsList( - const TKikimrTableDescription& table, - TPositionHandle pos, - TExprContext& ctx, - bool withSystemColumns = false -); + const TKikimrTableDescription& table, + TPositionHandle pos, + TExprContext& ctx, + bool withSystemColumns = false +); NNodes::TCoAtomList BuildKeyColumnsList( const TKikimrTableDescription& table, TPositionHandle pos, @@ -227,12 +227,12 @@ NNodes::TCoNameValueTupleList ExtractNamedKeyTuples(NNodes::TCoArgument arg, const TKikimrTableDescription& desc, TExprContext& ctx, const TString& tablePrefix = TString()); const TTypeAnnotationNode* GetReadTableRowType(TExprContext& ctx, const TKikimrTablesData& tablesData, - const TString& cluster, const TString& table, NNodes::TCoAtomList select, bool withSystemColumns = false); + const TString& cluster, const TString& table, NNodes::TCoAtomList select, bool withSystemColumns = false); NKikimrKqp::EIsolationLevel GetIsolationLevel(const TMaybe<TString>& isolationLevel); TMaybe<TString> GetIsolationLevel(const NKikimrKqp::EIsolationLevel& isolationLevel); -TYdbOperation GetTableOp(const NNodes::TKiWriteTable& write); +TYdbOperation GetTableOp(const NNodes::TKiWriteTable& write); TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const NNodes::TCoNameValueTupleList& operations, TExprContext& ctx); TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const NNodes::TKiOperationList& operations, TExprContext& ctx); void TableDescriptionToTableInfo(const TKikimrTableDescription& desc, NKqpProto::TKqpTableInfo* info); @@ -257,8 +257,8 @@ NNodes::TMaybeNode<NNodes::TExprBase> KiTableLookupGetValue(NNodes::TExprBase no TExprContext& ctx); NCommon::TTableLookup::TCompareResult KiTableLookupCompare(NNodes::TExprBase left, NNodes::TExprBase right); -NNodes::TKiProgram BuildKiProgram(NNodes::TKiDataQuery query, const TKikimrTablesData& tablesData, TExprContext& ctx, - bool withSystemColumns); +NNodes::TKiProgram BuildKiProgram(NNodes::TKiDataQuery query, const TKikimrTablesData& tablesData, TExprContext& ctx, + bool withSystemColumns); const THashSet<TStringBuf>& KikimrDataSourceFunctions(); const THashSet<TStringBuf>& KikimrDataSinkFunctions(); @@ -270,15 +270,15 @@ const TStringBuf& KikimrCommitModeFlush(); const TStringBuf& KikimrCommitModeRollback(); const TStringBuf& KikimrCommitModeScheme(); -const TYdbOperations& KikimrSchemeOps(); -const TYdbOperations& KikimrDataOps(); -const TYdbOperations& KikimrModifyOps(); -const TYdbOperations& KikimrReadOps(); -const TYdbOperations& KikimrRequireUnmodifiedOps(); +const TYdbOperations& KikimrSchemeOps(); +const TYdbOperations& KikimrDataOps(); +const TYdbOperations& KikimrModifyOps(); +const TYdbOperations& KikimrReadOps(); +const TYdbOperations& KikimrRequireUnmodifiedOps(); + +const TMap<TString, NKikimr::NUdf::EDataSlot>& KikimrSystemColumns(); +bool IsKikimrSystemColumn(const TStringBuf columnName); -const TMap<TString, NKikimr::NUdf::EDataSlot>& KikimrSystemColumns(); -bool IsKikimrSystemColumn(const TStringBuf columnName); - bool ValidateTableHasIndex(TKikimrTableMetadataPtr metadata, TExprContext& ctx, const TPositionHandle& pos); } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp index 979ce9a922..ebd392e16c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp @@ -58,7 +58,7 @@ TKikimrConfiguration::TKikimrConfiguration() { REGISTER_SETTING(*this, UseNewEngine); REGISTER_SETTING(*this, UnwrapReadTableValues); REGISTER_SETTING(*this, AllowNullCompareInIndex); - REGISTER_SETTING(*this, EnableSystemColumns); + REGISTER_SETTING(*this, EnableSystemColumns); REGISTER_SETTING(*this, EnableLlvm); REGISTER_SETTING(*this, OptDisableJoinRewrite); @@ -111,10 +111,10 @@ bool TKikimrSettings::HasDefaultCluster() const { return _DefaultCluster.Get() && !_DefaultCluster.Get().GetRef().empty(); } -bool TKikimrSettings::SystemColumnsEnabled() const { - return GetFlagValue(EnableSystemColumns.Get()); -} - +bool TKikimrSettings::SystemColumnsEnabled() const { + return GetFlagValue(EnableSystemColumns.Get()); +} + bool TKikimrSettings::SpillingEnabled() const { return GetFlagValue(_KqpEnableSpilling.Get()); } diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index e5ce52abd8..530cfb2855 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -50,7 +50,7 @@ struct TKikimrSettings { NCommon::TConfSetting<bool, false> UseNewEngine; NCommon::TConfSetting<bool, false> UnwrapReadTableValues; NCommon::TConfSetting<bool, false> AllowNullCompareInIndex; - NCommon::TConfSetting<bool, false> EnableSystemColumns; + NCommon::TConfSetting<bool, false> EnableSystemColumns; NCommon::TConfSetting<bool, false> EnableLlvm; /* Disable optimizer rules */ @@ -80,7 +80,7 @@ struct TKikimrSettings { bool AllowReverseRange() const; bool HasDefaultCluster() const; bool HasAllowKqpUnsafeCommit() const; - bool SystemColumnsEnabled() const; + bool SystemColumnsEnabled() const; bool SpillingEnabled() const; bool DisableLlvmForUdfStages() const; bool PushOlapProcess() const; diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index a617f77836..692165204a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -132,11 +132,11 @@ private: return TStatus::Error; } } - bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); - auto selectType = GetReadTableRowType( - ctx, SessionCtx->Tables(), TString(readTable.DataSource().Cluster()), key.GetTablePath(), - readTable.GetSelectColumns(ctx, SessionCtx->Tables(), sysColumnsEnabled), sysColumnsEnabled - ); + bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); + auto selectType = GetReadTableRowType( + ctx, SessionCtx->Tables(), TString(readTable.DataSource().Cluster()), key.GetTablePath(), + readTable.GetSelectColumns(ctx, SessionCtx->Tables(), sysColumnsEnabled), sysColumnsEnabled + ); if (!selectType) { return TStatus::Error; @@ -190,11 +190,11 @@ private: node.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(children)); return TStatus::Ok; } - - case TKikimrKey::Type::Role: - { - return TStatus::Ok; - } + + case TKikimrKey::Type::Role: + { + return TStatus::Ok; + } } return TStatus::Error; @@ -225,39 +225,39 @@ private: TTypeAnnotationContext& Types; }; -namespace { - std::function<void(TPositionHandle pos, const TString& column, const TString& message)> GetColumnTypeErrorFn(TExprContext& ctx) { - auto columnTypeError = [&ctx](TPositionHandle pos, const TString& column, const TString& message) { - ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, - TStringBuilder() << "Invalid type for column: " << column << ". " << message)); - }; - return columnTypeError; - } - - bool ValidateColumnDataType(const TDataExprType* type, const TExprBase& typeNode, const TString& columnName, - TExprContext& ctx) { - auto columnTypeError = GetColumnTypeErrorFn(ctx); - switch (type->GetSlot()) { - case EDataSlot::Decimal: - if (const auto dataExprParamsType = dynamic_cast<const TDataExprParamsType*>(type)) { - if (dataExprParamsType->GetParamOne() != "22") { - columnTypeError(typeNode.Pos(), columnName, TStringBuilder() << "Bad decimal precision \"" - << dataExprParamsType->GetParamOne() << "\". Only Decimal(22,9) is supported for table columns"); - return false; - } - if (dataExprParamsType->GetParamTwo() != "9") { - columnTypeError(typeNode.Pos(), columnName, TStringBuilder() << "Bad decimal scale \"" - << dataExprParamsType->GetParamTwo() << "\". Only Decimal(22,9) is supported for table columns"); - return false; - } - } +namespace { + std::function<void(TPositionHandle pos, const TString& column, const TString& message)> GetColumnTypeErrorFn(TExprContext& ctx) { + auto columnTypeError = [&ctx](TPositionHandle pos, const TString& column, const TString& message) { + ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, + TStringBuilder() << "Invalid type for column: " << column << ". " << message)); + }; + return columnTypeError; + } + + bool ValidateColumnDataType(const TDataExprType* type, const TExprBase& typeNode, const TString& columnName, + TExprContext& ctx) { + auto columnTypeError = GetColumnTypeErrorFn(ctx); + switch (type->GetSlot()) { + case EDataSlot::Decimal: + if (const auto dataExprParamsType = dynamic_cast<const TDataExprParamsType*>(type)) { + if (dataExprParamsType->GetParamOne() != "22") { + columnTypeError(typeNode.Pos(), columnName, TStringBuilder() << "Bad decimal precision \"" + << dataExprParamsType->GetParamOne() << "\". Only Decimal(22,9) is supported for table columns"); + return false; + } + if (dataExprParamsType->GetParamTwo() != "9") { + columnTypeError(typeNode.Pos(), columnName, TStringBuilder() << "Bad decimal scale \"" + << dataExprParamsType->GetParamTwo() << "\". Only Decimal(22,9) is supported for table columns"); + return false; + } + } break; - - default: - break; - } - return true; - } + + default: + break; + } + return true; + } } class TKiSinkTypeAnnotationTransformer : public TKiSinkVisitorTransformer @@ -371,8 +371,8 @@ private: } auto op = GetTableOp(node); - if (op == TYdbOperation::InsertAbort || op == TYdbOperation::InsertRevert || - op == TYdbOperation::Upsert || op == TYdbOperation::Replace) { + if (op == TYdbOperation::InsertAbort || op == TYdbOperation::InsertRevert || + op == TYdbOperation::Upsert || op == TYdbOperation::Replace) { for (const auto& [name, meta] : table->Metadata->Columns) { if (meta.NotNull && !rowType->FindItem(name)) { ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() @@ -388,7 +388,7 @@ private: return TStatus::Error; } } - } else if (op == TYdbOperation::UpdateOn) { + } else if (op == TYdbOperation::UpdateOn) { for (const auto& item : rowType->GetItems()) { auto column = table->Metadata->Columns.FindPtr(TString(item->GetName())); YQL_ENSURE(column); @@ -561,13 +561,13 @@ private: meta->ColumnOrder.reserve(create.Columns().Size()); for (auto atom : create.PrimaryKey()) { - meta->KeyColumnNames.emplace_back(atom.Value()); + meta->KeyColumnNames.emplace_back(atom.Value()); + } + + for (const auto& column : create.PartitionBy()) { + meta->TableSettings.PartitionBy.emplace_back(column.Value()); } - for (const auto& column : create.PartitionBy()) { - meta->TableSettings.PartitionBy.emplace_back(column.Value()); - } - for (auto item : create.Columns()) { auto columnTuple = item.Cast<TExprList>(); auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); @@ -587,22 +587,22 @@ private: auto dataType = actualType->Cast<TDataExprType>(); - if (!ValidateColumnDataType(dataType, typeNode, columnName, ctx)) { - return IGraphTransformer::TStatus::Error; - } - + if (!ValidateColumnDataType(dataType, typeNode, columnName, ctx)) { + return IGraphTransformer::TStatus::Error; + } + TKikimrColumnMetadata columnMeta; columnMeta.Name = columnName; columnMeta.Type = dataType->GetName(); columnMeta.NotNull = notNull; - if (columnTuple.Size() > 2) { - auto families = columnTuple.Item(2).Cast<TCoAtomList>(); - for (auto family : families) { - columnMeta.Families.push_back(TString(family.Value())); - } - } - + if (columnTuple.Size() > 2) { + auto families = columnTuple.Item(2).Cast<TCoAtomList>(); + for (auto family : families) { + columnMeta.Families.push_back(TString(family.Value())); + } + } + meta->ColumnOrder.push_back(columnName); auto insertRes = meta->Columns.insert(std::make_pair(columnName, columnMeta)); if (!insertRes.second) { @@ -660,139 +660,139 @@ private: meta->Indexes.push_back(indexDesc); } - for (auto columnFamily : create.ColumnFamilies()) { - if (auto maybeTupleList = columnFamily.Maybe<TCoNameValueTupleList>()) { - TColumnFamily family; - for (auto familySetting : maybeTupleList.Cast()) { - auto name = familySetting.Name().Value(); - if (name == "name") { - family.Name = TString(familySetting.Value().Cast<TCoAtom>().Value()); - } else if (name == "data") { - family.Data = TString( - familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - } else if (name == "compression") { - family.Compression = TString( - familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - } else { - ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), - TStringBuilder() << "Unknown column family setting name: " << name)); - return TStatus::Error; - } - } - meta->ColumnFamilies.push_back(family); - } - } - - for (const auto& setting : create.TableSettings()) { - auto name = setting.Name().Value(); - if (name == "compactionPolicy") { - meta->TableSettings.CompactionPolicy = TString( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - } else if (name == "autoPartitioningBySize") { - meta->TableSettings.AutoPartitioningBySize = TString(setting.Value().Cast<TCoAtom>().Value()); - } else if (name == "partitionSizeMb") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { - meta->TableSettings.PartitionSizeMb = value; - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set preferred partition size to 0. " - "To disable auto partitioning by size use 'SET AUTO_PARTITIONING_BY_SIZE DISABLED'")); - return TStatus::Error; - } - } else if (name == "autoPartitioningByLoad") { - meta->TableSettings.AutoPartitioningByLoad = TString(setting.Value().Cast<TCoAtom>().Value()); - } else if (name == "minPartitions") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { - meta->TableSettings.MinPartitions = value; - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set min partition count to 0")); - return TStatus::Error; - } - } else if (name == "maxPartitions") { - ui64 value = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - if (value) { - meta->TableSettings.MaxPartitions = value; - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't set max partition count to 0")); - return TStatus::Error; - } - } else if (name == "uniformPartitions") { - meta->TableSettings.UniformPartitions = FromString<ui64>( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); - } else if (name == "partitionAtKeys") { - TVector<const TDataExprType*> keyTypes; - keyTypes.reserve(meta->KeyColumnNames.size() + 1); - - // Getting key column types - for (const auto& key : meta->KeyColumnNames) { - for (auto item : create.Columns()) { - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); - auto columnName = TString(nameNode.Value()); - if (columnName == key) { - auto typeNode = columnTuple.Item(1); - auto keyType = typeNode.Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (keyType->HasOptional()) { - keyType = keyType->Cast<TOptionalExprType>()->GetItemType(); - } - keyTypes.push_back(keyType->Cast<TDataExprType>()); - } - } - } - if (keyTypes.size() != create.PrimaryKey().Size()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Pos()), "Can't get all key column types")); - return IGraphTransformer::TStatus::Error; - } - auto listNode = setting.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto partitionNode = listNode.Item(i); - TVector<std::pair<EDataSlot, TString>> keys; - auto boundaries = partitionNode.Cast<TExprList>(); - if (boundaries.Size() > keyTypes.size()) { - ctx.AddError(TIssue(ctx.GetPosition(partitionNode.Pos()), TStringBuilder() - << "Partition at keys has " << boundaries.Size() << " key values while there are only " - << keyTypes.size() << " key columns")); - return IGraphTransformer::TStatus::Error; - } - for (size_t j = 0; j < boundaries.Size(); ++j) { - TExprNode::TPtr keyNode = boundaries.Item(j).Ptr(); - TString content(keyNode->Child(0)->Content()); - if (keyNode->GetTypeAnn()->Cast<TDataExprType>()->GetSlot() != keyTypes[j]->GetSlot()) { - if (TryConvertTo(keyNode, *keyTypes[j], ctx) == TStatus::Error) { - ctx.AddError(TIssue(ctx.GetPosition(keyNode->Pos()), TStringBuilder() - << "Failed to convert value \"" << content - << "\" to a corresponding key column type")); - return TStatus::Error; - } - auto newTypeAnn = ctx.MakeType<TDataExprType>(keyTypes[j]->GetSlot()); - keyNode->SetTypeAnn(newTypeAnn); - } - - keys.emplace_back(keyTypes[j]->GetSlot(), content); - } - - meta->TableSettings.PartitionAtKeys.push_back(keys); - } - } else if (name == "keyBloomFilter") { - meta->TableSettings.KeyBloomFilter = TString(setting.Value().Cast<TCoAtom>().Value()); - } else if (name == "readReplicasSettings") { - meta->TableSettings.ReadReplicasSettings = TString( - setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() - ); + for (auto columnFamily : create.ColumnFamilies()) { + if (auto maybeTupleList = columnFamily.Maybe<TCoNameValueTupleList>()) { + TColumnFamily family; + for (auto familySetting : maybeTupleList.Cast()) { + auto name = familySetting.Name().Value(); + if (name == "name") { + family.Name = TString(familySetting.Value().Cast<TCoAtom>().Value()); + } else if (name == "data") { + family.Data = TString( + familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else if (name == "compression") { + family.Compression = TString( + familySetting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else { + ctx.AddError(TIssue(ctx.GetPosition(familySetting.Name().Pos()), + TStringBuilder() << "Unknown column family setting name: " << name)); + return TStatus::Error; + } + } + meta->ColumnFamilies.push_back(family); + } + } + + for (const auto& setting : create.TableSettings()) { + auto name = setting.Name().Value(); + if (name == "compactionPolicy") { + meta->TableSettings.CompactionPolicy = TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else if (name == "autoPartitioningBySize") { + meta->TableSettings.AutoPartitioningBySize = TString(setting.Value().Cast<TCoAtom>().Value()); + } else if (name == "partitionSizeMb") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { + meta->TableSettings.PartitionSizeMb = value; + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set preferred partition size to 0. " + "To disable auto partitioning by size use 'SET AUTO_PARTITIONING_BY_SIZE DISABLED'")); + return TStatus::Error; + } + } else if (name == "autoPartitioningByLoad") { + meta->TableSettings.AutoPartitioningByLoad = TString(setting.Value().Cast<TCoAtom>().Value()); + } else if (name == "minPartitions") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { + meta->TableSettings.MinPartitions = value; + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set min partition count to 0")); + return TStatus::Error; + } + } else if (name == "maxPartitions") { + ui64 value = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + if (value) { + meta->TableSettings.MaxPartitions = value; + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + "Can't set max partition count to 0")); + return TStatus::Error; + } + } else if (name == "uniformPartitions") { + meta->TableSettings.UniformPartitions = FromString<ui64>( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else if (name == "partitionAtKeys") { + TVector<const TDataExprType*> keyTypes; + keyTypes.reserve(meta->KeyColumnNames.size() + 1); + + // Getting key column types + for (const auto& key : meta->KeyColumnNames) { + for (auto item : create.Columns()) { + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto columnName = TString(nameNode.Value()); + if (columnName == key) { + auto typeNode = columnTuple.Item(1); + auto keyType = typeNode.Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (keyType->HasOptional()) { + keyType = keyType->Cast<TOptionalExprType>()->GetItemType(); + } + keyTypes.push_back(keyType->Cast<TDataExprType>()); + } + } + } + if (keyTypes.size() != create.PrimaryKey().Size()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Pos()), "Can't get all key column types")); + return IGraphTransformer::TStatus::Error; + } + auto listNode = setting.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto partitionNode = listNode.Item(i); + TVector<std::pair<EDataSlot, TString>> keys; + auto boundaries = partitionNode.Cast<TExprList>(); + if (boundaries.Size() > keyTypes.size()) { + ctx.AddError(TIssue(ctx.GetPosition(partitionNode.Pos()), TStringBuilder() + << "Partition at keys has " << boundaries.Size() << " key values while there are only " + << keyTypes.size() << " key columns")); + return IGraphTransformer::TStatus::Error; + } + for (size_t j = 0; j < boundaries.Size(); ++j) { + TExprNode::TPtr keyNode = boundaries.Item(j).Ptr(); + TString content(keyNode->Child(0)->Content()); + if (keyNode->GetTypeAnn()->Cast<TDataExprType>()->GetSlot() != keyTypes[j]->GetSlot()) { + if (TryConvertTo(keyNode, *keyTypes[j], ctx) == TStatus::Error) { + ctx.AddError(TIssue(ctx.GetPosition(keyNode->Pos()), TStringBuilder() + << "Failed to convert value \"" << content + << "\" to a corresponding key column type")); + return TStatus::Error; + } + auto newTypeAnn = ctx.MakeType<TDataExprType>(keyTypes[j]->GetSlot()); + keyNode->SetTypeAnn(newTypeAnn); + } + + keys.emplace_back(keyTypes[j]->GetSlot(), content); + } + + meta->TableSettings.PartitionAtKeys.push_back(keys); + } + } else if (name == "keyBloomFilter") { + meta->TableSettings.KeyBloomFilter = TString(setting.Value().Cast<TCoAtom>().Value()); + } else if (name == "readReplicasSettings") { + meta->TableSettings.ReadReplicasSettings = TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); } else if (name == "setTtlSettings") { TTtlSettings ttlSettings; TString error; @@ -809,13 +809,13 @@ private: ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), "Can't reset TTL settings")); return TStatus::Error; - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown table profile setting: " << name)); - return TStatus::Error; - } - } - + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown table profile setting: " << name)); + return TStatus::Error; + } + } + if (!EnsureModifyPermissions(cluster, table, create.Pos(), ctx)) { return TStatus::Error; } @@ -829,8 +829,8 @@ private: } tableDesc.Metadata = meta; - bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); - YQL_ENSURE(tableDesc.Load(ctx, sysColumnsEnabled)); + bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); + YQL_ENSURE(tableDesc.Load(ctx, sysColumnsEnabled)); create.Ptr()->SetTypeAnn(create.World().Ref().GetTypeAnn()); return TStatus::Ok; @@ -872,106 +872,106 @@ private: return TStatus::Error; } - YQL_ENSURE(!node.Actions().Empty()); + YQL_ENSURE(!node.Actions().Empty()); - for (const auto& action : node.Actions()) { - auto name = action.Name().Value(); + for (const auto& action : node.Actions()) { + auto name = action.Name().Value(); if (name == "renameTo") { YQL_ENSURE(action.Value().Cast<TCoAtom>().Value()); } else if (name == "addColumns") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); - auto name = TString(nameNode.Value()); - if (table->Metadata->Columns.FindPtr(name)) { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << name << "\" already exists")); - return TStatus::Error; - } - } - auto columnTypeError = GetColumnTypeErrorFn(ctx); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); - auto name = TString(nameNode.Value()); - columnTuple.Item(0).Cast<TCoAtom>(); - auto typeNode = columnTuple.Item(1); - auto columnType = typeNode.Ref().GetTypeAnn(); - YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); - auto type = columnType->Cast<TTypeExprType>()->GetType(); + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto name = TString(nameNode.Value()); + if (table->Metadata->Columns.FindPtr(name)) { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << name << "\" already exists")); + return TStatus::Error; + } + } + auto columnTypeError = GetColumnTypeErrorFn(ctx); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto name = TString(nameNode.Value()); + columnTuple.Item(0).Cast<TCoAtom>(); + auto typeNode = columnTuple.Item(1); + auto columnType = typeNode.Ref().GetTypeAnn(); + YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); + auto type = columnType->Cast<TTypeExprType>()->GetType(); auto actualType = (type->GetKind() == ETypeAnnotationKind::Optional) ? type->Cast<TOptionalExprType>()->GetItemType() : type; - if (actualType->GetKind() != ETypeAnnotationKind::Data) { - columnTypeError(typeNode.Pos(), name, "Only core YQL data types are currently supported"); - return TStatus::Error; - } - - auto dataType = actualType->Cast<TDataExprType>(); - - if (!ValidateColumnDataType(dataType, typeNode, name, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (columnTuple.Size() > 2) { - auto families = columnTuple.Item(2); - if (families.Cast<TCoAtomList>().Size() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << name - << "\". Several column families for a single column are not yet supported")); - return TStatus::Error; - } - } + if (actualType->GetKind() != ETypeAnnotationKind::Data) { + columnTypeError(typeNode.Pos(), name, "Only core YQL data types are currently supported"); + return TStatus::Error; + } + + auto dataType = actualType->Cast<TDataExprType>(); + + if (!ValidateColumnDataType(dataType, typeNode, name, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (columnTuple.Size() > 2) { + auto families = columnTuple.Item(2); + if (families.Cast<TCoAtomList>().Size() > 1) { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << name + << "\". Several column families for a single column are not yet supported")); + return TStatus::Error; + } + } } - } else if (name == "dropColumns") { - auto listNode = action.Value().Cast<TCoAtomList>(); - THashSet<TString> keyColumns; - for (const auto& keyColumnName : table->Metadata->KeyColumnNames) { - keyColumns.insert(keyColumnName); - } - for (auto dropColumn : listNode) { - TString name(dropColumn.Value()); - - if (!table->Metadata->Columns.FindPtr(name)) { - ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column \"" << name << "\" does not exist")); - return TStatus::Error; - } - - if (keyColumns.find(name) != keyColumns.end()) { - ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << name << "\" is a key column. Key column drop is not supported")); - return TStatus::Error; - } + } else if (name == "dropColumns") { + auto listNode = action.Value().Cast<TCoAtomList>(); + THashSet<TString> keyColumns; + for (const auto& keyColumnName : table->Metadata->KeyColumnNames) { + keyColumns.insert(keyColumnName); } - } else if (name == "alterColumns") { - auto listNode = action.Value().Cast<TExprList>(); - for (size_t i = 0; i < listNode.Size(); ++i) { - auto item = listNode.Item(i); - auto columnTuple = item.Cast<TExprList>(); - auto nameNode = columnTuple.Item(0).Cast<TCoAtom>();; - auto name = TString(nameNode.Value()); - if (!table->Metadata->Columns.FindPtr(name)) { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << name << "\" does not exist")); - return TStatus::Error; - } - auto families = columnTuple.Item(1); - if (families.Cast<TCoAtomList>().Size() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() - << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) - << " Column: \"" << name - << "\". Several column families for a single column are not yet supported")); - return TStatus::Error; - } + for (auto dropColumn : listNode) { + TString name(dropColumn.Value()); + + if (!table->Metadata->Columns.FindPtr(name)) { + ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column \"" << name << "\" does not exist")); + return TStatus::Error; + } + + if (keyColumns.find(name) != keyColumns.end()) { + ctx.AddError(TIssue(ctx.GetPosition(dropColumn.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << name << "\" is a key column. Key column drop is not supported")); + return TStatus::Error; + } + } + } else if (name == "alterColumns") { + auto listNode = action.Value().Cast<TExprList>(); + for (size_t i = 0; i < listNode.Size(); ++i) { + auto item = listNode.Item(i); + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>();; + auto name = TString(nameNode.Value()); + if (!table->Metadata->Columns.FindPtr(name)) { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << name << "\" does not exist")); + return TStatus::Error; + } + auto families = columnTuple.Item(1); + if (families.Cast<TCoAtomList>().Size() > 1) { + ctx.AddError(TIssue(ctx.GetPosition(nameNode.Pos()), TStringBuilder() + << "AlterTable : " << NCommon::FullTableName(table->Metadata->Cluster, table->Metadata->Name) + << " Column: \"" << name + << "\". Several column families for a single column are not yet supported")); + return TStatus::Error; + } } } else if (name == "addIndex") { auto listNode = action.Value().Cast<TExprList>(); @@ -1009,12 +1009,118 @@ private: << " Index: \"" << name << "\" does not exist")); return TStatus::Error; } - } else if (name != "addColumnFamilies" && name != "alterColumnFamilies" - && name != "setTableSettings") - { - ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), - TStringBuilder() << "Unknown alter table action: " << name)); - return TStatus::Error; + } else if (name != "addColumnFamilies" && name != "alterColumnFamilies" + && name != "setTableSettings") + { + ctx.AddError(TIssue(ctx.GetPosition(action.Name().Pos()), + TStringBuilder() << "Unknown alter table action: " << name)); + return TStatus::Error; + } + } + + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleCreateUser(TKiCreateUser node, TExprContext& ctx) override { + for (const auto& setting : node.Settings()) { + auto name = setting.Name().Value(); + if (name == "password") { + if (!EnsureAtom(setting.Value().Ref(), ctx)) { + return TStatus::Error; + } + } else if (name == "passwordEncrypted") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "passwordEncrypted node shouldn't have value" << name)); + } + } else if (name == "nullPassword") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "nullPassword node shouldn't have value" << name)); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown create user setting: " << name)); + return TStatus::Error; + } + } + + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleAlterUser(TKiAlterUser node, TExprContext& ctx) override { + for (const auto& setting : node.Settings()) { + auto name = setting.Name().Value(); + if (name == "password") { + if (!EnsureAtom(setting.Value().Ref(), ctx)) { + return TStatus::Error; + } + } else if (name == "passwordEncrypted") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "passwordEncrypted node shouldn't have value" << name)); + } + } else if (name == "nullPassword") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "nullPassword node shouldn't have value" << name)); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown alter user setting: " << name)); + return TStatus::Error; + } + } + + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleDropUser(TKiDropUser node, TExprContext& ctx) override { + for (const auto& setting : node.Settings()) { + auto name = setting.Name().Value(); + if (name == "force") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "force node shouldn't have value" << name)); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown drop user setting: " << name)); + return TStatus::Error; + } + } + + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override { + Y_UNUSED(ctx); + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleAlterGroup(TKiAlterGroup node, TExprContext& ctx) override { + Y_UNUSED(ctx); + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + virtual TStatus HandleDropGroup(TKiDropGroup node, TExprContext& ctx) override { + for (const auto& setting : node.Settings()) { + auto name = setting.Name().Value(); + if (name == "force") { + if (setting.Value()) { + ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), + TStringBuilder() << "force node shouldn't have value" << name)); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), + TStringBuilder() << "Unknown drop group setting: " << name)); + return TStatus::Error; } } @@ -1022,112 +1128,6 @@ private: return TStatus::Ok; } - virtual TStatus HandleCreateUser(TKiCreateUser node, TExprContext& ctx) override { - for (const auto& setting : node.Settings()) { - auto name = setting.Name().Value(); - if (name == "password") { - if (!EnsureAtom(setting.Value().Ref(), ctx)) { - return TStatus::Error; - } - } else if (name == "passwordEncrypted") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "passwordEncrypted node shouldn't have value" << name)); - } - } else if (name == "nullPassword") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "nullPassword node shouldn't have value" << name)); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown create user setting: " << name)); - return TStatus::Error; - } - } - - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - virtual TStatus HandleAlterUser(TKiAlterUser node, TExprContext& ctx) override { - for (const auto& setting : node.Settings()) { - auto name = setting.Name().Value(); - if (name == "password") { - if (!EnsureAtom(setting.Value().Ref(), ctx)) { - return TStatus::Error; - } - } else if (name == "passwordEncrypted") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "passwordEncrypted node shouldn't have value" << name)); - } - } else if (name == "nullPassword") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "nullPassword node shouldn't have value" << name)); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown alter user setting: " << name)); - return TStatus::Error; - } - } - - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - virtual TStatus HandleDropUser(TKiDropUser node, TExprContext& ctx) override { - for (const auto& setting : node.Settings()) { - auto name = setting.Name().Value(); - if (name == "force") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "force node shouldn't have value" << name)); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown drop user setting: " << name)); - return TStatus::Error; - } - } - - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - virtual TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override { - Y_UNUSED(ctx); - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - virtual TStatus HandleAlterGroup(TKiAlterGroup node, TExprContext& ctx) override { - Y_UNUSED(ctx); - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - - virtual TStatus HandleDropGroup(TKiDropGroup node, TExprContext& ctx) override { - for (const auto& setting : node.Settings()) { - auto name = setting.Name().Value(); - if (name == "force") { - if (setting.Value()) { - ctx.AddError(TIssue(ctx.GetPosition(setting.Value().Ref().Pos()), - TStringBuilder() << "force node shouldn't have value" << name)); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - TStringBuilder() << "Unknown drop group setting: " << name)); - return TStatus::Error; - } - } - - node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); - return TStatus::Ok; - } - virtual TStatus HandleWrite(TExprBase node, TExprContext& ctx) override { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Failed to annotate Write!, IO rewrite should handle this")); return TStatus::Error; @@ -1158,7 +1158,7 @@ private: switch (SessionCtx->Query().Type) { case EKikimrQueryType::YqlScript: - case EKikimrQueryType::YqlScriptStreaming: + case EKikimrQueryType::YqlScriptStreaming: case EKikimrQueryType::YqlInternal: break; @@ -1244,7 +1244,7 @@ private: } virtual TStatus HandleKql(TCallable node, TExprContext& ctx) override { - bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); + bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); if (auto call = node.Maybe<TKiSelectRow>()) { auto selectRow = call.Cast(); @@ -1448,7 +1448,7 @@ TAutoPtr<IGraphTransformer> CreateKiSinkTypeAnnotationTransformer(TIntrusivePtr< } const TTypeAnnotationNode* GetReadTableRowType(TExprContext& ctx, const TKikimrTablesData& tablesData, - const TString& cluster, const TString& table, TCoAtomList select, bool withSystemColumns) + const TString& cluster, const TString& table, TCoAtomList select, bool withSystemColumns) { auto tableDesc = tablesData.EnsureTableExists(cluster, table, select.Pos(), ctx); if (!tableDesc) { @@ -1458,23 +1458,23 @@ const TTypeAnnotationNode* GetReadTableRowType(TExprContext& ctx, const TKikimrT TVector<const TItemExprType*> resultItems; for (auto item : select) { auto column = tableDesc->Metadata->Columns.FindPtr(item.Value()); - TString columnName; - if (column) { - columnName = column->Name; - } else { - if (withSystemColumns && IsKikimrSystemColumn(item.Value())) { - columnName = TString(item.Value()); - } else { - ctx.AddError(TIssue(ctx.GetPosition(select.Pos()), TStringBuilder() - << "Column not found: " << item.Value())); - return nullptr; - } - } - - auto type = tableDesc->GetColumnType(columnName); - YQL_ENSURE(type, "No such column: " << columnName); - - auto itemType = ctx.MakeType<TItemExprType>(columnName, type); + TString columnName; + if (column) { + columnName = column->Name; + } else { + if (withSystemColumns && IsKikimrSystemColumn(item.Value())) { + columnName = TString(item.Value()); + } else { + ctx.AddError(TIssue(ctx.GetPosition(select.Pos()), TStringBuilder() + << "Column not found: " << item.Value())); + return nullptr; + } + } + + auto type = tableDesc->GetColumnType(columnName); + YQL_ENSURE(type, "No such column: " << columnName); + + auto itemType = ctx.MakeType<TItemExprType>(columnName, type); if (!itemType->Validate(select.Pos(), ctx)) { return nullptr; } diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp index 13b453afe2..decd2cb301 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp @@ -92,7 +92,7 @@ bool IsSqlQuery(const NKikimrKqp::EQueryType& queryType) { case NKikimrKqp::QUERY_TYPE_SQL_DML: case NKikimrKqp::QUERY_TYPE_SQL_DDL: case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: case NKikimrKqp::QUERY_TYPE_SQL_SCAN: return true; diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h index 09b96f4387..99c9cc3f05 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.h +++ b/ydb/core/kqp/runtime/kqp_read_table.h @@ -18,7 +18,7 @@ struct TParseReadTableResultBase { TTableId TableId; TSmallVec<TKqpComputeContextBase::TColumn> Columns; - TSmallVec<TKqpComputeContextBase::TColumn> SystemColumns; + TSmallVec<TKqpComputeContextBase::TColumn> SystemColumns; TSmallVec<bool> SkipNullKeys; TNode* ItemsLimit = nullptr; bool Reverse = false; diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp index 8d05520af8..c4d8e24196 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp @@ -254,7 +254,7 @@ TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTab , Range(range) , SkipNullKeys(skipNullKeys) , Columns(columns) - , SystemColumns(systemColumns) + , SystemColumns(systemColumns) {} TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, @@ -446,8 +446,8 @@ TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const { class TKqpTableReader : public IKqpTableReader { public: TKqpTableReader(TKqpScanComputeContext::TScanData& scanData) - : ScanData(scanData) - {} + : ScanData(scanData) + {} NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) override { if (ScanData.IsEmpty()) { diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 5ffc03a08e..9bc7889f92 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -114,7 +114,7 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { Server.Reset(MakeHolder<Tests::TServer>(*ServerSettings)); Server->EnableGRpc(grpcPort); - Server->SetupDefaultProfiles(); + Server->SetupDefaultProfiles(); Client.Reset(MakeHolder<Tests::TClient>(*ServerSettings)); @@ -348,29 +348,29 @@ void TKikimrRunner::CreateSampleTables() { ("kikimr-db", 3, "kikimr-db-21", "Stream Read Data"), ("kikimr-db", 4, "kikimr-db-53", "Discover"), ("ydb", 0, "ydb-1000", "some very very very very long string"); - + REPLACE INTO `Join1` (Key, Fk21, Fk22, Value) VALUES - (1, 101, "One", "Value1"), - (2, 102, "Two", "Value1"), - (3, 103, "One", "Value2"), - (4, 104, "Two", "Value2"), - (5, 105, "One", "Value3"), - (6, 106, "Two", "Value3"), - (7, 107, "One", "Value4"), + (1, 101, "One", "Value1"), + (2, 102, "Two", "Value1"), + (3, 103, "One", "Value2"), + (4, 104, "Two", "Value2"), + (5, 105, "One", "Value3"), + (6, 106, "Two", "Value3"), + (7, 107, "One", "Value4"), (8, 108, "One", "Value5"), (9, 101, "Two", "Value1"); - + REPLACE INTO `Join2` (Key1, Key2, Name, Value2) VALUES - (101, "One", "Name1", "Value21"), - (101, "Two", "Name1", "Value22"), - (101, "Three", "Name3", "Value23"), - (102, "One", "Name2", "Value24"), - (103, "One", "Name1", "Value25"), - (104, "One", "Name3", "Value26"), - (105, "One", "Name2", "Value27"), - (105, "Two", "Name4", "Value28"), - (106, "One", "Name3", "Value29"), - (108, "One", NULL, "Value31"); + (101, "One", "Name1", "Value21"), + (101, "Two", "Name1", "Value22"), + (101, "Three", "Name3", "Value23"), + (102, "One", "Name2", "Value24"), + (103, "One", "Name1", "Value25"), + (104, "One", "Name3", "Value26"), + (105, "One", "Name2", "Value27"), + (105, "Two", "Name4", "Value28"), + (106, "One", "Name3", "Value29"), + (108, "One", NULL, "Value31"); )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync()); } @@ -550,20 +550,20 @@ void FillProfile(NYdb::NExperimental::TStreamPart& streamPart, NYson::TYsonWrite } void PrintResultSet(const NYdb::TResultSet& resultSet, NYson::TYsonWriter& writer) { - auto columns = resultSet.GetColumnsMeta(); - - NYdb::TResultSetParser parser(resultSet); - while (parser.TryNextRow()) { - writer.OnListItem(); - writer.OnBeginList(); - for (ui32 i = 0; i < columns.size(); ++i) { - writer.OnListItem(); - FormatValueYson(parser.GetValue(i), writer); - } - writer.OnEndList(); - } -} - + auto columns = resultSet.GetColumnsMeta(); + + NYdb::TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + writer.OnListItem(); + writer.OnBeginList(); + for (ui32 i = 0; i < columns.size(); ++i) { + writer.OnListItem(); + FormatValueYson(parser.GetValue(i), writer); + } + writer.OnEndList(); + } +} + template<typename TIterator> TString StreamResultToYsonImpl(TIterator& it, TVector<TString>* profiles) { TStringStream out; @@ -580,8 +580,8 @@ TString StreamResultToYsonImpl(TIterator& it, TVector<TString>* profiles) { } if (streamPart.HasResultSet()) { - auto resultSet = streamPart.ExtractResultSet(); - PrintResultSet(resultSet, writer); + auto resultSet = streamPart.ExtractResultSet(); + PrintResultSet(resultSet, writer); } FillProfile(streamPart, writer, profiles, profileIndex); @@ -601,43 +601,43 @@ TString StreamResultToYson(NYdb::NTable::TScanQueryPartIterator& it) { return StreamResultToYsonImpl(it, nullptr); } -TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it) { - TStringStream out; +TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it) { + TStringStream out; NYson::TYsonWriter writer(&out, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true); - writer.OnBeginList(); - - ui32 currentIndex = 0; - writer.OnListItem(); - writer.OnBeginList(); - - for (;;) { - auto streamPart = it.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); - break; - } - - if (streamPart.HasPartialResult()) { - const auto& partialResult = streamPart.GetPartialResult(); - - ui32 resultSetIndex = partialResult.GetResultSetIndex(); - if (currentIndex != resultSetIndex) { - currentIndex = resultSetIndex; - writer.OnEndList(); - writer.OnListItem(); - writer.OnBeginList(); - } - - PrintResultSet(partialResult.GetResultSet(), writer); - } - } - - writer.OnEndList(); - writer.OnEndList(); - - return out.Str(); -} - + writer.OnBeginList(); + + ui32 currentIndex = 0; + writer.OnListItem(); + writer.OnBeginList(); + + for (;;) { + auto streamPart = it.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); + break; + } + + if (streamPart.HasPartialResult()) { + const auto& partialResult = streamPart.GetPartialResult(); + + ui32 resultSetIndex = partialResult.GetResultSetIndex(); + if (currentIndex != resultSetIndex) { + currentIndex = resultSetIndex; + writer.OnEndList(); + writer.OnListItem(); + writer.OnBeginList(); + } + + PrintResultSet(partialResult.GetResultSet(), writer); + } + } + + writer.OnEndList(); + writer.OnEndList(); + + return out.Str(); +} + template<typename TIterator> TCollectedStreamResult CollectStreamResultImpl(TIterator& it) { TCollectedStreamResult res; @@ -659,8 +659,8 @@ TCollectedStreamResult CollectStreamResultImpl(TIterator& it) { } if (streamPart.HasResultSet()) { - auto resultSet = streamPart.ExtractResultSet(); - PrintResultSet(resultSet, resultSetWriter); + auto resultSet = streamPart.ExtractResultSet(); + PrintResultSet(resultSet, resultSetWriter); } if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) { diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index 440514002a..c1d64b65c3 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -220,7 +220,7 @@ inline NYdb::NTable::TDataQueryResult ExecQueryAndTestResult(NYdb::NTable::TSess TString StreamResultToYson(NYdb::NExperimental::TStreamPartIterator& it, TVector<TString>* profiles = nullptr); TString StreamResultToYson(NYdb::NTable::TScanQueryPartIterator& it); -TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it); +TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it); ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp index 7136b99873..ca7ffa42a4 100644 --- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp @@ -457,32 +457,32 @@ Y_UNIT_TEST_SUITE(KqpScheme) { Y_UNIT_TEST(InvalidationAfterDropCreateTable2MultiStageTxNoEffects) { CheckInvalidationAfterDropCreateTable2(true, true); } - - Y_UNIT_TEST(CreateTableWithDefaultSettings) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitioningBySize"; - auto query = TStringBuilder() << R"( - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - { - TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - } - } - + + Y_UNIT_TEST(CreateTableWithDefaultSettings) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitioningBySize"; + auto query = TStringBuilder() << R"( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + { + TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + } + } + void AlterTableSetttings( NYdb::NTable::TSession& session, const TString& tableName, const THashMap<TString, TString>& settings, bool compat, @@ -523,37 +523,37 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateTableWithCompactionPolicy(bool compat) { - TKikimrRunner kikimr = TKikimrRunner(); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithCompactionPolicy"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - COMPACTION_POLICY = "compaction2" - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 2); - } + TKikimrRunner kikimr = TKikimrRunner(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithCompactionPolicy"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + COMPACTION_POLICY = "compaction2" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 2); + } AlterTableSetttings(session, tableName, {{"COMPACTION_POLICY", "\"default\""}}, compat); - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 3); - } - } - + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 3); + } + } + Y_UNIT_TEST(CreateTableWithCompactionPolicyUncompat) { CreateTableWithCompactionPolicy(false); } @@ -562,123 +562,123 @@ Y_UNIT_TEST_SUITE(KqpScheme) { CreateTableWithCompactionPolicy(true); } - Y_UNIT_TEST(CreateAndAlterTableWithPartitionBy) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitionBy"; - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 Uint64, - Key3 Uint64, - Value String, - PRIMARY KEY (Key1, Key2, Key3), - PARTITION BY (Key1, Key2, Key3, Value) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), - "\"Partition by\" contains more columns than primary key does", - "Unexpected error message"); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 Uint64, - Key3 Uint64, - Value String, - PRIMARY KEY (Key1, Key2, Key3), - PARTITION BY (Key1, Key3, Key2) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), - "\"Partition by\" doesn't match primary key", - "Unexpected error message"); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 Uint64, - Key3 Uint64, - Value String, - PRIMARY KEY (Key1, Key2, Key3), - PARTITION BY (Key1, Key2) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), - "\"Partition by\" is not supported yet", - "Unexpected error message"); - } - - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 Uint64, - Key3 Uint64, - Value String, - PRIMARY KEY (Key1, Key2, Key3), - PARTITION BY (Key1, Key2, Key3) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - + Y_UNIT_TEST(CreateAndAlterTableWithPartitionBy) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionBy"; + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 Uint64, + Key3 Uint64, + Value String, + PRIMARY KEY (Key1, Key2, Key3), + PARTITION BY (Key1, Key2, Key3, Value) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), + "\"Partition by\" contains more columns than primary key does", + "Unexpected error message"); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 Uint64, + Key3 Uint64, + Value String, + PRIMARY KEY (Key1, Key2, Key3), + PARTITION BY (Key1, Key3, Key2) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), + "\"Partition by\" doesn't match primary key", + "Unexpected error message"); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 Uint64, + Key3 Uint64, + Value String, + PRIMARY KEY (Key1, Key2, Key3), + PARTITION BY (Key1, Key2) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), + "\"Partition by\" is not supported yet", + "Unexpected error message"); + } + + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 Uint64, + Key3 Uint64, + Value String, + PRIMARY KEY (Key1, Key2, Key3), + PARTITION BY (Key1, Key2, Key3) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + void CreateAndAlterTableWithPartitioningBySize(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitioningBySize"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_SIZE = ENABLED - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - { - TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitioningBySize"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_BY_SIZE = ENABLED + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + { + TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); + } + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); - { - TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - } - } - + { + TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describe.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + } + } + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningBySizeUncompat) { CreateAndAlterTableWithPartitioningBySize(false); } @@ -688,65 +688,65 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateAndAlterTableWithPartitionSize(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitionSize"; - auto queryCreate1 = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_PARTITION_SIZE_MB = 0 - );)"; - auto resultCreate1 = session.ExecuteSchemeQuery(queryCreate1).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate1.GetStatus(), EStatus::GENERIC_ERROR, resultCreate1.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(resultCreate1.GetIssues().ToString(), "Can't set preferred partition size to 0", - "Unexpected error message"); - - auto queryCreate2 = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000 - );)"; - auto resultCreate2 = session.ExecuteSchemeQuery(queryCreate2).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate2.GetStatus(), EStatus::SUCCESS, resultCreate2.GetIssues().ToString()); - - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); - UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 1000 * 1024 * 1024); - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionSize"; + auto queryCreate1 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_PARTITION_SIZE_MB = 0 + );)"; + auto resultCreate1 = session.ExecuteSchemeQuery(queryCreate1).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate1.GetStatus(), EStatus::GENERIC_ERROR, resultCreate1.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(resultCreate1.GetIssues().ToString(), "Can't set preferred partition size to 0", + "Unexpected error message"); + + auto queryCreate2 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000 + );)"; + auto resultCreate2 = session.ExecuteSchemeQuery(queryCreate2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate2.GetStatus(), EStatus::SUCCESS, resultCreate2.GetIssues().ToString()); + + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); + UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 1000 * 1024 * 1024); + } + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "0"}}, compat, EStatus::GENERIC_ERROR, "Can't set preferred partition size to 0"); - + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_BY_SIZE", "DISABLED"}}, compat); - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); - UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 0); - } - + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); + UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 0); + } + AlterTableSetttings(session, tableName, {{"AUTO_PARTITIONING_PARTITION_SIZE_MB", "500"}}, compat); - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); - UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 500 * 1024 * 1024); - } - } - + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + size_t sizeToSplit = describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetPartitioningPolicy().GetSizeToSplit(); + UNIT_ASSERT_VALUES_EQUAL(sizeToSplit, 500 * 1024 * 1024); + } + } + Y_UNIT_TEST(CreateAndAlterTableWithPartitionSizeUncompat) { CreateAndAlterTableWithPartitionSize(false); } @@ -867,26 +867,26 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateAndAlterTableWithPartitioningByLoad(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitioningByLoad"; - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_LOAD = ENABLED - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitioningByLoad"; + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_BY_LOAD = ENABLED + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { + } + + { auto describeResult = kikimr.GetTestClient().Ls(tableName); bool enabled = describeResult->Record.GetPathDescription().GetTable() .GetPartitionConfig().GetPartitioningPolicy().GetSplitByLoadSettings().GetEnabled(); @@ -908,8 +908,8 @@ Y_UNIT_TEST_SUITE(KqpScheme) { .GetPartitionConfig().GetPartitioningPolicy().GetSplitByLoadSettings().GetEnabled(); UNIT_ASSERT_VALUES_EQUAL(enabled, true); } - } - + } + Y_UNIT_TEST(CreateAndAlterTableWithPartitioningByLoadUncompat) { CreateAndAlterTableWithPartitioningByLoad(false); } @@ -919,67 +919,67 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateAndAlterTableWithMinMaxPartitions(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithMinMaxPartitions"; - auto queryCreate1 = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 0, - AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100 - );)"; - auto resultCreate1 = session.ExecuteSchemeQuery(queryCreate1).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate1.GetStatus(), EStatus::GENERIC_ERROR, resultCreate1.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(resultCreate1.GetIssues().ToString(), "Can't set min partition count to 0", - "Unexpected error message"); - - auto queryCreate2 = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10, - AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100 - );)"; - auto resultCreate2 = session.ExecuteSchemeQuery(queryCreate2).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate2.GetStatus(), EStatus::SUCCESS, resultCreate2.GetIssues().ToString()); - - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetPartitioningPolicy().GetMinPartitionsCount(), 10); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetPartitioningPolicy().GetMaxPartitionsCount(), 100); - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithMinMaxPartitions"; + auto queryCreate1 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 0, + AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100 + );)"; + auto resultCreate1 = session.ExecuteSchemeQuery(queryCreate1).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate1.GetStatus(), EStatus::GENERIC_ERROR, resultCreate1.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(resultCreate1.GetIssues().ToString(), "Can't set min partition count to 0", + "Unexpected error message"); + + auto queryCreate2 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10, + AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100 + );)"; + auto resultCreate2 = session.ExecuteSchemeQuery(queryCreate2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate2.GetStatus(), EStatus::SUCCESS, resultCreate2.GetIssues().ToString()); + + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetPartitioningPolicy().GetMinPartitionsCount(), 10); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetPartitioningPolicy().GetMaxPartitionsCount(), 100); + } + AlterTableSetttings(session, tableName, { {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "0"} }, compat, EStatus::GENERIC_ERROR, "Can't set max partition count to 0"); - + AlterTableSetttings(session, tableName, { {"AUTO_PARTITIONING_MIN_PARTITIONS_COUNT", "20"}, {"AUTO_PARTITIONING_MAX_PARTITIONS_COUNT", "50"} }, compat); - - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetPartitioningPolicy().GetMinPartitionsCount(), 20); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetPartitioningPolicy().GetMaxPartitionsCount(), 50); - } - } - + + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetPartitioningPolicy().GetMinPartitionsCount(), 20); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetPartitioningPolicy().GetMaxPartitionsCount(), 50); + } + } + Y_UNIT_TEST(CreateAndAlterTableWithMinMaxPartitionsUncompat) { CreateAndAlterTableWithMinMaxPartitions(false); } @@ -989,44 +989,44 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateAndAlterTableWithBloomFilter(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithBloomFilter"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - KEY_BLOOM_FILTER = ENABLED - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetEnableFilterByKey(), true); - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithBloomFilter"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + KEY_BLOOM_FILTER = ENABLED + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetEnableFilterByKey(), true); + } + AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "DISABLED"}}, compat); - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetEnableFilterByKey(), false); - } - + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetEnableFilterByKey(), false); + } + AlterTableSetttings(session, tableName, {{"KEY_BLOOM_FILTER", "ENABLED"}}, compat); - { - auto describeResult = kikimr.GetTestClient().Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() - .GetEnableFilterByKey(), true); - } - } - + { + auto describeResult = kikimr.GetTestClient().Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable().GetPartitionConfig() + .GetEnableFilterByKey(), true); + } + } + Y_UNIT_TEST(CreateAndAlterTableWithBloomFilterUncompat) { CreateAndAlterTableWithBloomFilter(false); } @@ -1036,77 +1036,77 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateTableWithReadReplicas(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithReadReplicas"; - { - auto queryCreate = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - READ_REPLICAS_SETTINGS = "SOME_AZ:2" - );)"; - auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::GENERIC_ERROR, resultCreate.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(resultCreate.GetIssues().ToString(), - "Specifying read replicas count for each AZ in cluster is not supported yet", - "Unexpected error message"); - } - { - auto queryCreate = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - READ_REPLICAS_SETTINGS = "VLA: 2, MAN:1 " - );)"; - auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::GENERIC_ERROR, resultCreate.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(resultCreate.GetIssues().ToString(), - "Specifying read replicas count for each AZ in cluster is not supported yet", - "Unexpected error message"); - } - { - auto queryCreate = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - READ_REPLICAS_SETTINGS = "any_az:1" - );)"; - auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::SUCCESS, resultCreate.GetIssues().ToString()); - } - - { - const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); - const auto readReplicasSettings = tableDesc.GetReadReplicasSettings(); - UNIT_ASSERT(readReplicasSettings); - UNIT_ASSERT(readReplicasSettings->GetMode() == NYdb::NTable::TReadReplicasSettings::EMode::AnyAz); - UNIT_ASSERT_VALUES_EQUAL(readReplicasSettings->GetReadReplicasCount(), 1); - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithReadReplicas"; + { + auto queryCreate = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + READ_REPLICAS_SETTINGS = "SOME_AZ:2" + );)"; + auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::GENERIC_ERROR, resultCreate.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(resultCreate.GetIssues().ToString(), + "Specifying read replicas count for each AZ in cluster is not supported yet", + "Unexpected error message"); + } + { + auto queryCreate = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + READ_REPLICAS_SETTINGS = "VLA: 2, MAN:1 " + );)"; + auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::GENERIC_ERROR, resultCreate.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(resultCreate.GetIssues().ToString(), + "Specifying read replicas count for each AZ in cluster is not supported yet", + "Unexpected error message"); + } + { + auto queryCreate = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + READ_REPLICAS_SETTINGS = "any_az:1" + );)"; + auto resultCreate = session.ExecuteSchemeQuery(queryCreate).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), EStatus::SUCCESS, resultCreate.GetIssues().ToString()); + } + + { + const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); + const auto readReplicasSettings = tableDesc.GetReadReplicasSettings(); + UNIT_ASSERT(readReplicasSettings); + UNIT_ASSERT(readReplicasSettings->GetMode() == NYdb::NTable::TReadReplicasSettings::EMode::AnyAz); + UNIT_ASSERT_VALUES_EQUAL(readReplicasSettings->GetReadReplicasCount(), 1); + } + AlterTableSetttings(session, tableName, {{"READ_REPLICAS_SETTINGS", "\"PER_AZ:2\""}}, compat); - { - const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); - const auto readReplicasSettings = tableDesc.GetReadReplicasSettings(); - UNIT_ASSERT(readReplicasSettings); - UNIT_ASSERT(readReplicasSettings->GetMode() == NYdb::NTable::TReadReplicasSettings::EMode::PerAz); - UNIT_ASSERT_VALUES_EQUAL(readReplicasSettings->GetReadReplicasCount(), 2); - } - } - + { + const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); + const auto readReplicasSettings = tableDesc.GetReadReplicasSettings(); + UNIT_ASSERT(readReplicasSettings); + UNIT_ASSERT(readReplicasSettings->GetMode() == NYdb::NTable::TReadReplicasSettings::EMode::PerAz); + UNIT_ASSERT_VALUES_EQUAL(readReplicasSettings->GetReadReplicasCount(), 2); + } + } + Y_UNIT_TEST(CreateTableWithReadReplicasUncompat) { CreateTableWithReadReplicas(false); } @@ -1254,32 +1254,32 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateTableWithUniformPartitions(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithUniformPartitions"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - UNIFORM_PARTITIONS = 4 - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto describeResult = session.DescribeTable(tableName, - NYdb::NTable::TDescribeTableSettings().WithTableStatistics(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithUniformPartitions"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + UNIFORM_PARTITIONS = 4 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto describeResult = session.DescribeTable(tableName, + NYdb::NTable::TDescribeTableSettings().WithTableStatistics(true)).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); + AlterTableSetttings(session, tableName, {{"UNIFORM_PARTITIONS", "8"}}, compat, EStatus::GENERIC_ERROR, "UNIFORM_PARTITIONS alter is not supported"); - } - + } + Y_UNIT_TEST(CreateTableWithUniformPartitionsUncompat) { CreateTableWithUniformPartitions(false); } @@ -1289,63 +1289,63 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } void CreateTableWithPartitionAtKeysSimple(bool compat) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitionAtKeysSimple"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 String, - Value String, - PRIMARY KEY (Key1, Key2) - ) - WITH ( - PARTITION_AT_KEYS = (10, 100, 1000, 10000) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto describeResult = session.DescribeTable(tableName, - TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 5); - - auto extractValue = [](const TValue& val) { - auto parser = TValueParser(val); - parser.OpenTuple(); - UNIT_ASSERT(parser.TryNextElement()); - return parser.GetOptionalUint64().GetRef(); - }; - - const TVector<TKeyRange>& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); - - size_t n = 0; - const TVector<ui64> expectedRanges = { 10ul, 100ul, 1000ul, 10000ul }; - - for (const auto& range : keyRanges) { - if (n == 0) { - UNIT_ASSERT(!range.From()); - } else { - UNIT_ASSERT(range.From()->IsInclusive()); - auto left = extractValue(range.From()->GetValue()); - UNIT_ASSERT_VALUES_EQUAL(left, expectedRanges[n - 1]); - } - if (n == expectedRanges.size()) { - UNIT_ASSERT(!range.To()); - } else { - UNIT_ASSERT(!range.To()->IsInclusive()); - auto right = extractValue(range.To()->GetValue()); - UNIT_ASSERT_VALUES_EQUAL(right, expectedRanges[n]); - } - ++n; - } - + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionAtKeysSimple"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 String, + Value String, + PRIMARY KEY (Key1, Key2) + ) + WITH ( + PARTITION_AT_KEYS = (10, 100, 1000, 10000) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto describeResult = session.DescribeTable(tableName, + TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 5); + + auto extractValue = [](const TValue& val) { + auto parser = TValueParser(val); + parser.OpenTuple(); + UNIT_ASSERT(parser.TryNextElement()); + return parser.GetOptionalUint64().GetRef(); + }; + + const TVector<TKeyRange>& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); + + size_t n = 0; + const TVector<ui64> expectedRanges = { 10ul, 100ul, 1000ul, 10000ul }; + + for (const auto& range : keyRanges) { + if (n == 0) { + UNIT_ASSERT(!range.From()); + } else { + UNIT_ASSERT(range.From()->IsInclusive()); + auto left = extractValue(range.From()->GetValue()); + UNIT_ASSERT_VALUES_EQUAL(left, expectedRanges[n - 1]); + } + if (n == expectedRanges.size()) { + UNIT_ASSERT(!range.To()); + } else { + UNIT_ASSERT(!range.To()->IsInclusive()); + auto right = extractValue(range.To()->GetValue()); + UNIT_ASSERT_VALUES_EQUAL(right, expectedRanges[n]); + } + ++n; + } + AlterTableSetttings(session, tableName, {{"PARTITION_AT_KEYS", "(100, 500, 1000, 10000)"}}, compat, EStatus::GENERIC_ERROR, "PARTITION_AT_KEYS alter is not supported"); - } - + } + Y_UNIT_TEST(CreateTableWithPartitionAtKeysSimpleUncompat) { CreateTableWithPartitionAtKeysSimple(false); } @@ -1354,254 +1354,254 @@ Y_UNIT_TEST_SUITE(KqpScheme) { CreateTableWithPartitionAtKeysSimple(true); } - Y_UNIT_TEST(CreateTableWithPartitionAtKeysComplex) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithPartitionAtKeysComplex"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key1 Uint64, - Key2 String, - Value String, - PRIMARY KEY (Key1, Key2) - ) - WITH ( - PARTITION_AT_KEYS = ((10), (100, "123"), (1000, "cde")) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto describeResult = session.DescribeTable(tableName, - TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); - - auto extractValue = [](const TValue& val) { - auto parser = TValueParser(val); - parser.OpenTuple(); - UNIT_ASSERT(parser.TryNextElement()); - ui64 pk1 = parser.GetOptionalUint64().GetRef(); - UNIT_ASSERT(parser.TryNextElement()); - auto pk2 = parser.GetOptionalString(); - return std::pair<ui64, TMaybe<TString>>(pk1, pk2); - }; - - const TVector<TKeyRange>& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); - - size_t n = 0; - const TVector<std::pair<ui64, TString>> expectedRanges = { - { 10ul, "" }, - { 100ul, "123" }, - { 1000ul, "cde" } - }; - - for (const auto& range : keyRanges) { - if (n == 0) { - UNIT_ASSERT(!range.From()); - } else { - UNIT_ASSERT(range.From()->IsInclusive()); - const auto& [pk1, pk2] = extractValue(range.From()->GetValue()); - const auto& [expectedPk1, expectedPk2] = expectedRanges[n - 1]; - - UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); - if (pk2.Defined()) { - UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); - } else { - UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); - } - } - if (n == expectedRanges.size()) { - UNIT_ASSERT(!range.To()); - } else { - UNIT_ASSERT(!range.To()->IsInclusive()); - const auto&[pk1, pk2] = extractValue(range.To()->GetValue()); - const auto&[expectedPk1, expectedPk2] = expectedRanges[n]; - - UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); - if (pk2.Defined()) { - UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); - } else { - UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); - } - } - ++n; - } - } - - Y_UNIT_TEST(CreateTableWithWrongPartitionAtKeys) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `/Root/TableWithWrongPartitionAtKeysComplex` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ) - WITH ( - PARTITION_AT_KEYS = ((10), (100, "123"), (1000, "cde")) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), - "Partition at keys has 2 key values while there are only 1 key columns", "Unexpected error message"); - } - - Y_UNIT_TEST(CreateTableWithFamiliesRegular) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithFamiliesRegular"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value1 String FAMILY Family1, - Value2 Uint32 FAMILY Family2, - PRIMARY KEY (Key), - FAMILY Family1 ( - DATA = "test", - COMPRESSION = "off" - ), - FAMILY Family2 - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); - UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); - for (const auto& family : columnFamilies) { - if (family.GetName() == "Family1") { - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); - } else { - UNIT_ASSERT(family.GetName() == "default" || family.GetName() == "Family2"); - } - } - } - - Y_UNIT_TEST(CreateTableWithDefaultFamily) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithDefaultFamily"; - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value1 String FAMILY Family1, - Value2 String, - PRIMARY KEY (Key), - FAMILY default ( - DATA = "test", - COMPRESSION = "lz4" - ), - FAMILY Family1 ( - DATA = "test", - COMPRESSION = "off" - ) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - { - auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); - UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 2); - for (const auto& family : columnFamilies) { - if (family.GetName() == "Family1") { - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); - } else { - UNIT_ASSERT(family.GetName() == "default"); - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); - } - } - } - - auto queryAlter1 = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` - ADD FAMILY Family2 ( - DATA = "test", - COMPRESSION = "off" - ), - ADD COLUMN Value3 Uint32 FAMILY Family1, - ADD COLUMN Value4 Uint32 FAMILY Family2, - DROP COLUMN Value2, - ALTER COLUMN Value1 SET FAMILY Family2, - ALTER FAMILY Family1 SET COMPRESSION "LZ4";)"; - auto resultAlter1 = session.ExecuteSchemeQuery(queryAlter1).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultAlter1.GetStatus(), EStatus::SUCCESS, resultAlter1.GetIssues().ToString()); - - { - auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); - const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); - UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); - for (const auto& family : columnFamilies) { - if (family.GetName() == "Family1") { - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); - } else if (family.GetName() == "Family2") { - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); - } else { - UNIT_ASSERT(family.GetName() == "default"); - UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); - UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); - } - } - const auto& columns = describeResult.GetTableDescription().GetColumns(); - UNIT_ASSERT_VALUES_EQUAL(columns.size(), 4); - for (const auto& column : columns) { - UNIT_ASSERT_C(column.Name == "Key" || column.Name == "Value1" - || column.Name == "Value3" || column.Name == "Value4", column.Name); - } - } - } - - Y_UNIT_TEST(CreateAndAlterTableComplex) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE TABLE `/Root/TableToAlter` ( - Key Uint64, - Value1 String FAMILY Family1, - PRIMARY KEY (Key), - FAMILY Family1 ( - DATA = "test", - COMPRESSION = "off" - ), - FAMILY Family2 ( - DATA = "test", - COMPRESSION = "lz4" - ) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto queryAddColumn = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `/Root/TableToAlter` ADD COLUMN Value2 Uint32 FAMILY Family2;)"; - auto resultAddColumn = session.ExecuteSchemeQuery(queryAddColumn).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultAddColumn.GetStatus(), EStatus::SUCCESS, resultAddColumn.GetIssues().ToString()); - - auto queryDropColumn = TStringBuilder() << R"( - --!syntax_v1 - ALTER TABLE `/Root/TableToAlter` DROP COLUMN Value2;)"; - auto resultDropColumn = session.ExecuteSchemeQuery(queryDropColumn).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resultDropColumn.GetStatus(), EStatus::SUCCESS, resultDropColumn.GetIssues().ToString()); - } + Y_UNIT_TEST(CreateTableWithPartitionAtKeysComplex) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionAtKeysComplex"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key1 Uint64, + Key2 String, + Value String, + PRIMARY KEY (Key1, Key2) + ) + WITH ( + PARTITION_AT_KEYS = ((10), (100, "123"), (1000, "cde")) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto describeResult = session.DescribeTable(tableName, + TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); + + auto extractValue = [](const TValue& val) { + auto parser = TValueParser(val); + parser.OpenTuple(); + UNIT_ASSERT(parser.TryNextElement()); + ui64 pk1 = parser.GetOptionalUint64().GetRef(); + UNIT_ASSERT(parser.TryNextElement()); + auto pk2 = parser.GetOptionalString(); + return std::pair<ui64, TMaybe<TString>>(pk1, pk2); + }; + + const TVector<TKeyRange>& keyRanges = describeResult.GetTableDescription().GetKeyRanges(); + + size_t n = 0; + const TVector<std::pair<ui64, TString>> expectedRanges = { + { 10ul, "" }, + { 100ul, "123" }, + { 1000ul, "cde" } + }; + + for (const auto& range : keyRanges) { + if (n == 0) { + UNIT_ASSERT(!range.From()); + } else { + UNIT_ASSERT(range.From()->IsInclusive()); + const auto& [pk1, pk2] = extractValue(range.From()->GetValue()); + const auto& [expectedPk1, expectedPk2] = expectedRanges[n - 1]; + + UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); + if (pk2.Defined()) { + UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); + } else { + UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); + } + } + if (n == expectedRanges.size()) { + UNIT_ASSERT(!range.To()); + } else { + UNIT_ASSERT(!range.To()->IsInclusive()); + const auto&[pk1, pk2] = extractValue(range.To()->GetValue()); + const auto&[expectedPk1, expectedPk2] = expectedRanges[n]; + + UNIT_ASSERT_VALUES_EQUAL(pk1, expectedPk1); + if (pk2.Defined()) { + UNIT_ASSERT_VALUES_EQUAL(pk2.GetRef(), expectedPk2); + } else { + UNIT_ASSERT_VALUES_EQUAL("", expectedPk2); + } + } + ++n; + } + } + + Y_UNIT_TEST(CreateTableWithWrongPartitionAtKeys) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `/Root/TableWithWrongPartitionAtKeysComplex` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ) + WITH ( + PARTITION_AT_KEYS = ((10), (100, "123"), (1000, "cde")) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), + "Partition at keys has 2 key values while there are only 1 key columns", "Unexpected error message"); + } + + Y_UNIT_TEST(CreateTableWithFamiliesRegular) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithFamiliesRegular"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String FAMILY Family1, + Value2 Uint32 FAMILY Family2, + PRIMARY KEY (Key), + FAMILY Family1 ( + DATA = "test", + COMPRESSION = "off" + ), + FAMILY Family2 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); + UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); + for (const auto& family : columnFamilies) { + if (family.GetName() == "Family1") { + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + } else { + UNIT_ASSERT(family.GetName() == "default" || family.GetName() == "Family2"); + } + } + } + + Y_UNIT_TEST(CreateTableWithDefaultFamily) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithDefaultFamily"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String FAMILY Family1, + Value2 String, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4" + ), + FAMILY Family1 ( + DATA = "test", + COMPRESSION = "off" + ) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + { + auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); + UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 2); + for (const auto& family : columnFamilies) { + if (family.GetName() == "Family1") { + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + } else { + UNIT_ASSERT(family.GetName() == "default"); + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + } + } + } + + auto queryAlter1 = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(` + ADD FAMILY Family2 ( + DATA = "test", + COMPRESSION = "off" + ), + ADD COLUMN Value3 Uint32 FAMILY Family1, + ADD COLUMN Value4 Uint32 FAMILY Family2, + DROP COLUMN Value2, + ALTER COLUMN Value1 SET FAMILY Family2, + ALTER FAMILY Family1 SET COMPRESSION "LZ4";)"; + auto resultAlter1 = session.ExecuteSchemeQuery(queryAlter1).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultAlter1.GetStatus(), EStatus::SUCCESS, resultAlter1.GetIssues().ToString()); + + { + auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); + UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); + for (const auto& family : columnFamilies) { + if (family.GetName() == "Family1") { + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + } else if (family.GetName() == "Family2") { + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::None); + } else { + UNIT_ASSERT(family.GetName() == "default"); + UNIT_ASSERT_VALUES_EQUAL(family.GetData(), "test"); + UNIT_ASSERT_VALUES_EQUAL(family.GetCompression(), EColumnFamilyCompression::LZ4); + } + } + const auto& columns = describeResult.GetTableDescription().GetColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 4); + for (const auto& column : columns) { + UNIT_ASSERT_C(column.Name == "Key" || column.Name == "Value1" + || column.Name == "Value3" || column.Name == "Value4", column.Name); + } + } + } + + Y_UNIT_TEST(CreateAndAlterTableComplex) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `/Root/TableToAlter` ( + Key Uint64, + Value1 String FAMILY Family1, + PRIMARY KEY (Key), + FAMILY Family1 ( + DATA = "test", + COMPRESSION = "off" + ), + FAMILY Family2 ( + DATA = "test", + COMPRESSION = "lz4" + ) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto queryAddColumn = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `/Root/TableToAlter` ADD COLUMN Value2 Uint32 FAMILY Family2;)"; + auto resultAddColumn = session.ExecuteSchemeQuery(queryAddColumn).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultAddColumn.GetStatus(), EStatus::SUCCESS, resultAddColumn.GetIssues().ToString()); + + auto queryDropColumn = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `/Root/TableToAlter` DROP COLUMN Value2;)"; + auto resultDropColumn = session.ExecuteSchemeQuery(queryDropColumn).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultDropColumn.GetStatus(), EStatus::SUCCESS, resultDropColumn.GetIssues().ToString()); + } Y_UNIT_TEST(AddDropColumn) { TKikimrRunner kikimr; @@ -1682,64 +1682,64 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::CORE_EXEC)); UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR)); } - - Y_UNIT_TEST(CreateTableWithDecimalColumn) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithDecimalColumn"; - { - auto query = TStringBuilder() << R"( - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value Decimal(35,9), - PRIMARY KEY (Key) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); - } - { - auto query = TStringBuilder() << R"( - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value Decimal(22,20), - PRIMARY KEY (Key) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); - } - { - auto query = TStringBuilder() << R"( - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value Decimal(22,9), - PRIMARY KEY (Key) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - auto tableDesc = describe.GetTableDescription(); - TVector<TTableColumn> columns = tableDesc.GetTableColumns(); - UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2); - TType valueType = columns[1].Type; - TTypeParser parser(valueType); - auto optionalKind = parser.GetKind(); - UNIT_ASSERT_EQUAL(optionalKind, TTypeParser::ETypeKind::Optional); - parser.OpenOptional(); - auto kind = parser.GetKind(); - UNIT_ASSERT_EQUAL(kind, TTypeParser::ETypeKind::Decimal); - TDecimalType decimalType = parser.GetDecimal(); - UNIT_ASSERT_EQUAL(decimalType.Precision, 22); - UNIT_ASSERT_EQUAL(decimalType.Scale, 9); - } - } - + + Y_UNIT_TEST(CreateTableWithDecimalColumn) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithDecimalColumn"; + { + auto query = TStringBuilder() << R"( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value Decimal(35,9), + PRIMARY KEY (Key) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); + } + { + auto query = TStringBuilder() << R"( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value Decimal(22,20), + PRIMARY KEY (Key) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); + } + { + auto query = TStringBuilder() << R"( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value Decimal(22,9), + PRIMARY KEY (Key) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + auto tableDesc = describe.GetTableDescription(); + TVector<TTableColumn> columns = tableDesc.GetTableColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2); + TType valueType = columns[1].Type; + TTypeParser parser(valueType); + auto optionalKind = parser.GetKind(); + UNIT_ASSERT_EQUAL(optionalKind, TTypeParser::ETypeKind::Optional); + parser.OpenOptional(); + auto kind = parser.GetKind(); + UNIT_ASSERT_EQUAL(kind, TTypeParser::ETypeKind::Decimal); + TDecimalType decimalType = parser.GetDecimal(); + UNIT_ASSERT_EQUAL(decimalType.Precision, 22); + UNIT_ASSERT_EQUAL(decimalType.Scale, 9); + } + } + void AlterTableAddIndex(EIndexTypeSql type, bool enableAsyncIndexes = false) { TKikimrRunner kikimr(TKikimrSettings().SetEnableAsyncIndexes(enableAsyncIndexes)); auto db = kikimr.GetTableClient(); @@ -1827,331 +1827,331 @@ Y_UNIT_TEST_SUITE(KqpScheme) { AlterTableAddIndex(EIndexTypeSql::GlobalAsync, true); } - Y_UNIT_TEST(AlterTableWithDecimalColumn) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableName = "/Root/TableWithDecimalColumn"; - { - auto query = TStringBuilder() << R"( - CREATE TABLE `)" << tableName << R"(` ( - Key Uint64, - Value1 String, - PRIMARY KEY (Key) - );)"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - ALTER TABLE `)" << tableName << R"(` - ADD COLUMN Value2 Decimal(35,9); - )"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); - } - { - auto query = TStringBuilder() << R"( - ALTER TABLE `)" << tableName << R"(` - ADD COLUMN Value2 Decimal(22,20); - )"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); - } - { - auto query = TStringBuilder() << R"( - ALTER TABLE `)" << tableName << R"(` - ADD COLUMN Value2 Decimal(22,9); - )"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); - auto tableDesc = describe.GetTableDescription(); - TVector<TTableColumn> columns = tableDesc.GetTableColumns(); - UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); - TType valueType = columns[2].Type; - TTypeParser parser(valueType); - auto optionalKind = parser.GetKind(); - UNIT_ASSERT_EQUAL(optionalKind, TTypeParser::ETypeKind::Optional); - parser.OpenOptional(); - auto kind = parser.GetKind(); - UNIT_ASSERT_EQUAL(kind, TTypeParser::ETypeKind::Decimal); - TDecimalType decimalType = parser.GetDecimal(); - UNIT_ASSERT_EQUAL(decimalType.Precision, 22); - UNIT_ASSERT_EQUAL(decimalType.Scale, 9); - } - } - - Y_UNIT_TEST(CreateUserWithPassword) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 ENCRYPTED PASSWORD 'password1'; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 PASSWORD NULL; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(CreateUserWithoutPassword) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1; - )"; - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(CreateAndDropUser) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - { - // Drop non-existing user force - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP USER IF EXISTS user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 PASSWORD 'password1'; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop existing user - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP USER user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 PASSWORD NULL; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop existing user force - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP USER IF EXISTS user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop non-existing user - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP USER user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(AlterUser) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 PASSWORD 'password1'; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER USER user1 WITH PASSWORD 'password2'; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER USER user1 WITH ENCRYPTED PASSWORD 'password3'; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER USER user1 WITH PASSWORD NULL; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(CreateAndDropGroup) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - { - // Drop non-existing group force - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP GROUP IF EXISTS group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE GROUP group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop existing group - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP GROUP group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE GROUP group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop existing group force - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP GROUP IF EXISTS group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - // Drop non-existing group - auto query = TStringBuilder() << R"( - --!syntax_v1 - DROP GROUP group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(AlterGroup) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE USER user1 PASSWORD 'password1'; - CREATE USER user2 PASSWORD 'password2'; - CREATE USER user3; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - CREATE GROUP group1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER GROUP group1 ADD USER user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER GROUP group1 DROP USER user1; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER GROUP group1 ADD USER user1, user2; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - auto query = TStringBuilder() << R"( - --!syntax_v1 - ALTER GROUP group1 DROP USER user1, user2; - )"; - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } + Y_UNIT_TEST(AlterTableWithDecimalColumn) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithDecimalColumn"; + { + auto query = TStringBuilder() << R"( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + ALTER TABLE `)" << tableName << R"(` + ADD COLUMN Value2 Decimal(35,9); + )"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); + } + { + auto query = TStringBuilder() << R"( + ALTER TABLE `)" << tableName << R"(` + ADD COLUMN Value2 Decimal(22,20); + )"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE)); + } + { + auto query = TStringBuilder() << R"( + ALTER TABLE `)" << tableName << R"(` + ADD COLUMN Value2 Decimal(22,9); + )"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS); + auto tableDesc = describe.GetTableDescription(); + TVector<TTableColumn> columns = tableDesc.GetTableColumns(); + UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); + TType valueType = columns[2].Type; + TTypeParser parser(valueType); + auto optionalKind = parser.GetKind(); + UNIT_ASSERT_EQUAL(optionalKind, TTypeParser::ETypeKind::Optional); + parser.OpenOptional(); + auto kind = parser.GetKind(); + UNIT_ASSERT_EQUAL(kind, TTypeParser::ETypeKind::Decimal); + TDecimalType decimalType = parser.GetDecimal(); + UNIT_ASSERT_EQUAL(decimalType.Precision, 22); + UNIT_ASSERT_EQUAL(decimalType.Scale, 9); + } + } + + Y_UNIT_TEST(CreateUserWithPassword) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 ENCRYPTED PASSWORD 'password1'; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 PASSWORD NULL; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(CreateUserWithoutPassword) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1; + )"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(CreateAndDropUser) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + { + // Drop non-existing user force + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP USER IF EXISTS user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 PASSWORD 'password1'; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop existing user + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP USER user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 PASSWORD NULL; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop existing user force + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP USER IF EXISTS user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop non-existing user + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP USER user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(AlterUser) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 PASSWORD 'password1'; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER USER user1 WITH PASSWORD 'password2'; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER USER user1 WITH ENCRYPTED PASSWORD 'password3'; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER USER user1 WITH PASSWORD NULL; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(CreateAndDropGroup) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + { + // Drop non-existing group force + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP GROUP IF EXISTS group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE GROUP group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop existing group + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP GROUP group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE GROUP group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop existing group force + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP GROUP IF EXISTS group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + // Drop non-existing group + auto query = TStringBuilder() << R"( + --!syntax_v1 + DROP GROUP group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(AlterGroup) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE USER user1 PASSWORD 'password1'; + CREATE USER user2 PASSWORD 'password2'; + CREATE USER user3; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE GROUP group1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER GROUP group1 ADD USER user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER GROUP group1 DROP USER user1; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER GROUP group1 ADD USER user1, user2; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + auto query = TStringBuilder() << R"( + --!syntax_v1 + ALTER GROUP group1 DROP USER user1, user2; + )"; + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } } -} // namespace NKqp +} // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/ut/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/kqp_scripting_ut.cpp index 193dadfec7..65e75ac303 100644 --- a/ydb/core/kqp/ut/kqp_scripting_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scripting_ut.cpp @@ -340,27 +340,27 @@ Y_UNIT_TEST_SUITE(KqpScripting) { settings.Mode(ExplainYqlRequestMode::Validate); auto result = client.ExplainYqlScript(R"( - DECLARE $value1 as Utf8; - DECLARE $value2 as UInt32; - SELECT $value1 as value1, $value2 as value2; + DECLARE $value1 as Utf8; + DECLARE $value2 as UInt32; + SELECT $value1 as value1, $value2 as value2; )", settings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - auto paramTypes = result.GetParameterTypes(); - UNIT_ASSERT_VALUES_EQUAL(paramTypes.size(), 2); - - UNIT_ASSERT(paramTypes.find("$value1") != paramTypes.end()); - TType type1 = paramTypes.at("$value1"); - TTypeParser parser1(type1); - UNIT_ASSERT_EQUAL(parser1.GetKind(), TTypeParser::ETypeKind::Primitive); - UNIT_ASSERT_EQUAL(parser1.GetPrimitive(), EPrimitiveType::Utf8); - - UNIT_ASSERT(paramTypes.find("$value2") != paramTypes.end()); - TType type2 = paramTypes.at("$value2"); - TTypeParser parser2(type2); - UNIT_ASSERT_EQUAL(parser2.GetKind(), TTypeParser::ETypeKind::Primitive); - UNIT_ASSERT_EQUAL(parser2.GetPrimitive(), EPrimitiveType::Uint32); - + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + auto paramTypes = result.GetParameterTypes(); + UNIT_ASSERT_VALUES_EQUAL(paramTypes.size(), 2); + + UNIT_ASSERT(paramTypes.find("$value1") != paramTypes.end()); + TType type1 = paramTypes.at("$value1"); + TTypeParser parser1(type1); + UNIT_ASSERT_EQUAL(parser1.GetKind(), TTypeParser::ETypeKind::Primitive); + UNIT_ASSERT_EQUAL(parser1.GetPrimitive(), EPrimitiveType::Utf8); + + UNIT_ASSERT(paramTypes.find("$value2") != paramTypes.end()); + TType type2 = paramTypes.at("$value2"); + TTypeParser parser2(type2); + UNIT_ASSERT_EQUAL(parser2.GetKind(), TTypeParser::ETypeKind::Primitive); + UNIT_ASSERT_EQUAL(parser2.GetPrimitive(), EPrimitiveType::Uint32); + UNIT_ASSERT_EQUAL(result.GetPlan(), ""); } @@ -431,19 +431,19 @@ Y_UNIT_TEST_SUITE(KqpScripting) { auto node = FindPlanNodeByKv(plan.GetMap().at("queries").GetArray()[2], "Node Type", "Aggregate-TableFullScan"); UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); } - - Y_UNIT_TEST(StreamExecuteYqlScriptScan) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - PRAGMA kikimr.ScanQuery = "true"; - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[8u]]])", StreamResultToYson(it)); - } - + + Y_UNIT_TEST(StreamExecuteYqlScriptScan) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + PRAGMA kikimr.ScanQuery = "true"; + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[8u]]])", StreamResultToYson(it)); + } + Y_UNIT_TEST(StreamExecuteYqlScriptScanScalar) { TKikimrRunner kikimr; TScriptingClient client(kikimr.GetDriver()); @@ -456,181 +456,181 @@ Y_UNIT_TEST_SUITE(KqpScripting) { SELECT Data FROM [/Root/EightShard] WHERE Key = $key1 OR Key = $key2 LIMIT COALESCE($limit, 1u); )").GetValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[[1]];[[3]]]])", StreamResultToYson(it)); + CompareYson(R"([[[[1]];[[3]]]])", StreamResultToYson(it)); } - Y_UNIT_TEST(StreamExecuteYqlScriptData) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[8u]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamExecuteYqlScriptMixed) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( + Y_UNIT_TEST(StreamExecuteYqlScriptData) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[8u]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamExecuteYqlScriptMixed) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( SELECT Key FROM [/Root/EightShard] WHERE Text = "Value1" ORDER BY Key; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value2"; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value3"; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value2"; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value3"; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); CompareYson(R"([ [[[101u]];[[201u]];[[301u]];[[401u]];[[501u]];[[601u]];[[701u]];[[801u]]]; [[8u]]; [[8u]]; [[8u]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamExecuteYqlScriptLeadingEmptyScan) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - PRAGMA kikimr.ScanQuery = "true"; - SELECT Key FROM [/Root/EightShard] WHERE Text = "Value123"; - COMMIT; - - SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[];[[8u]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamExecuteYqlScriptSeveralQueries) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - --!syntax_v1 - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[[101u]]];[[[102u]]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamExecuteYqlScriptSeveralQueriesComplex) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - --!syntax_v1 - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; - COMMIT; - - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 3; - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 4; - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 5; - - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - TString res = StreamResultToYson(it); - Cerr << "Result: " << res << Endl; - CompareYson(R"([[[[101u]]];[[[102u]]];[[[103u]]];[[[104u]]];[[[105u]]]])", res); - } - - Y_UNIT_TEST(SyncExecuteYqlScriptSeveralQueries) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - - auto result = client.ExecuteYqlScript(R"( - --!syntax_v1 - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; - SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; - )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets()[0].RowsCount(), 1); - TResultSetParser rs0(result.GetResultSets()[0]); - UNIT_ASSERT(rs0.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(*rs0.ColumnParser(0).GetOptionalUint32().Get(), 101u); - TResultSetParser rs1(result.GetResultSets()[1]); - UNIT_ASSERT(rs1.TryNextRow()); - UNIT_ASSERT_VALUES_EQUAL(*rs1.ColumnParser(0).GetOptionalUint32().Get(), 102u); - } - - Y_UNIT_TEST(StreamExecuteYqlScriptEmptyResults) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - auto it = client.StreamExecuteYqlScript(R"( - SELECT Key FROM [/Root/EightShard] WHERE Text = "Value"; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - SELECT Key FROM [/Root/EightShard] WHERE Text = "Value5"; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[];[]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamDdlAndDml) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - { - auto it = client.StreamExecuteYqlScript(R"( - CREATE TABLE [/Root/StreamScriptingTest] ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - COMMIT; - - REPLACE INTO [/Root/StreamScriptingTest] (Key, Value) VALUES - (1, "One"), - (2, "Two"); - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - { - auto streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_C(streamPart.IsSuccess(), streamPart.GetIssues().ToString()); - UNIT_ASSERT_C(!streamPart.HasPartialResult(), streamPart.GetIssues().ToString()); - } - { - auto streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_C(!streamPart.IsSuccess(), streamPart.GetIssues().ToString()); - UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); - } - } - - auto it = client.StreamExecuteYqlScript(R"( - SELECT COUNT(*) FROM [/Root/StreamScriptingTest]; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[2u]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamOperationTimeout) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - - TExecuteYqlRequestSettings settings; - settings.OperationTimeout(TDuration::MilliSeconds(1)); - auto it = client.StreamExecuteYqlScript(R"( - SELECT Key FROM [/Root/EightShard] WHERE Text = "Value1"; - )", settings).GetValueSync(); - - auto streamPart = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(streamPart.GetStatus(), EStatus::TIMEOUT, it.GetIssues().ToString()); - } + } + + Y_UNIT_TEST(StreamExecuteYqlScriptLeadingEmptyScan) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + PRAGMA kikimr.ScanQuery = "true"; + SELECT Key FROM [/Root/EightShard] WHERE Text = "Value123"; + COMMIT; + + SELECT COUNT(*) FROM [/Root/EightShard] WHERE Text = "Value1"; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[];[[8u]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamExecuteYqlScriptSeveralQueries) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + --!syntax_v1 + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[[101u]]];[[[102u]]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamExecuteYqlScriptSeveralQueriesComplex) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + --!syntax_v1 + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; + COMMIT; + + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 3; + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 4; + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 5; + + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + TString res = StreamResultToYson(it); + Cerr << "Result: " << res << Endl; + CompareYson(R"([[[[101u]]];[[[102u]]];[[[103u]]];[[[104u]]];[[[105u]]]])", res); + } + + Y_UNIT_TEST(SyncExecuteYqlScriptSeveralQueries) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + + auto result = client.ExecuteYqlScript(R"( + --!syntax_v1 + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 1; + SELECT Fk21 FROM `/Root/Join1` WHERE Key = 2; + )").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets()[0].RowsCount(), 1); + TResultSetParser rs0(result.GetResultSets()[0]); + UNIT_ASSERT(rs0.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(*rs0.ColumnParser(0).GetOptionalUint32().Get(), 101u); + TResultSetParser rs1(result.GetResultSets()[1]); + UNIT_ASSERT(rs1.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(*rs1.ColumnParser(0).GetOptionalUint32().Get(), 102u); + } + + Y_UNIT_TEST(StreamExecuteYqlScriptEmptyResults) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + auto it = client.StreamExecuteYqlScript(R"( + SELECT Key FROM [/Root/EightShard] WHERE Text = "Value"; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + SELECT Key FROM [/Root/EightShard] WHERE Text = "Value5"; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[];[]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamDdlAndDml) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + { + auto it = client.StreamExecuteYqlScript(R"( + CREATE TABLE [/Root/StreamScriptingTest] ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + COMMIT; + + REPLACE INTO [/Root/StreamScriptingTest] (Key, Value) VALUES + (1, "One"), + (2, "Two"); + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + { + auto streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_C(streamPart.IsSuccess(), streamPart.GetIssues().ToString()); + UNIT_ASSERT_C(!streamPart.HasPartialResult(), streamPart.GetIssues().ToString()); + } + { + auto streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_C(!streamPart.IsSuccess(), streamPart.GetIssues().ToString()); + UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); + } + } + + auto it = client.StreamExecuteYqlScript(R"( + SELECT COUNT(*) FROM [/Root/StreamScriptingTest]; + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[2u]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamOperationTimeout) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + + TExecuteYqlRequestSettings settings; + settings.OperationTimeout(TDuration::MilliSeconds(1)); + auto it = client.StreamExecuteYqlScript(R"( + SELECT Key FROM [/Root/EightShard] WHERE Text = "Value1"; + )", settings).GetValueSync(); + + auto streamPart = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(streamPart.GetStatus(), EStatus::TIMEOUT, it.GetIssues().ToString()); + } Y_UNIT_TEST(SecondaryIndexes) { TKikimrRunner kikimr; @@ -688,5 +688,5 @@ Y_UNIT_TEST_SUITE(KqpScripting) { } } -} // namespace NKqp +} // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/ut/kqp_sys_col_ut.cpp b/ydb/core/kqp/ut/kqp_sys_col_ut.cpp index d06bd9ea1c..13fb8fa8ec 100644 --- a/ydb/core/kqp/ut/kqp_sys_col_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sys_col_ut.cpp @@ -1,412 +1,412 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NExperimental; - -TDataQueryResult ExecuteDataQuery(TKikimrRunner& kikimr, const TString& query) { - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - auto txControl = TTxControl::BeginTx().CommitTx(); - auto result = session.ExecuteDataQuery( - query, - txControl, - TExecDataQuerySettings() + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NExperimental; + +TDataQueryResult ExecuteDataQuery(TKikimrRunner& kikimr, const TString& query) { + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto txControl = TTxControl::BeginTx().CommitTx(); + auto result = session.ExecuteDataQuery( + query, + txControl, + TExecDataQuerySettings() .OperationTimeout(TDuration::Seconds(10)) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - return result; -} - + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + return result; +} + TDataQueryResult ExecuteDataQuery(const TString& query) { TKikimrRunner kikimr; return ExecuteDataQuery(kikimr, query); -} - -TStreamPartIterator ExecuteStreamQuery(TKikimrRunner& kikimr, const TString& query) { - NExperimental::TStreamQueryClient db{ kikimr.GetDriver() }; - auto it = db.ExecuteStreamQuery(query).GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - return it; -} - +} + +TStreamPartIterator ExecuteStreamQuery(TKikimrRunner& kikimr, const TString& query) { + NExperimental::TStreamQueryClient db{ kikimr.GetDriver() }; + auto it = db.ExecuteStreamQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + return it; +} + void SelectRowAsteriskCommon(bool useNewEngine) { - TStringBuilder query; - query << R"( + TStringBuilder query; + query << R"( PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"("; - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE Key = 1; - )"; + )"; auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[[1u];["One"];[-1]]])", - FormatResultSetYson(result.GetResultSet(0))); -} - + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[[1u];["One"];[-1]]])", + FormatResultSetYson(result.GetResultSet(0))); +} + void SelectRowByIdCommon(bool useNewEngine) { - TStringBuilder query; - query << R"( + TStringBuilder query; + query << R"( PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"("; - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul; - )"; + )"; auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", - FormatResultSetYson(result.GetResultSet(0))); -} - + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", + FormatResultSetYson(result.GetResultSet(0))); +} + void SelectRangeCommon(bool useNewEngine) { - TStringBuilder query; - query << R"( + TStringBuilder query; + query << R"( PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"("; - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3; - )"; + )"; auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[72075186224037888u]];[[72075186224037888u]]])", - FormatResultSetYson(result.GetResultSet(0))); -} - -Y_UNIT_TEST_SUITE(KqpSysColV0) { - Y_UNIT_TEST(SelectRowAsterisk) { + FormatResultSetYson(result.GetResultSet(0))); +} + +Y_UNIT_TEST_SUITE(KqpSysColV0) { + Y_UNIT_TEST(SelectRowAsterisk) { SelectRowAsteriskCommon(false); - } - - Y_UNIT_TEST(SelectRowAsteriskNewEngine) { + } + + Y_UNIT_TEST(SelectRowAsteriskNewEngine) { SelectRowAsteriskCommon(true); - } - - Y_UNIT_TEST(SelectRowById) { + } + + Y_UNIT_TEST(SelectRowById) { SelectRowByIdCommon(false); - } - - Y_UNIT_TEST(SelectRowByIdNewEngine) { + } + + Y_UNIT_TEST(SelectRowByIdNewEngine) { SelectRowByIdCommon(true); - } - - Y_UNIT_TEST(SelectRange) { + } + + Y_UNIT_TEST(SelectRange) { SelectRangeCommon(false); - } - - Y_UNIT_TEST(SelectRangeNewEngine) { + } + + Y_UNIT_TEST(SelectRangeNewEngine) { SelectRangeCommon(true); - } - - Y_UNIT_TEST(UpdateAndDelete) { - TKikimrRunner kikimr; - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + } + + Y_UNIT_TEST(UpdateAndDelete) { + TKikimrRunner kikimr; + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; REPLACE INTO [/Root/TwoShard] (Key, Value1, Value2) - VALUES (4u, "Four", -4); - )"; - auto result = ExecuteDataQuery(kikimr, query); - } - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + VALUES (4u, "Four", -4); + )"; + auto result = ExecuteDataQuery(kikimr, query); + } + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM [/Root/TwoShard] WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4; - )"; - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[1u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[1u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; UPDATE [/Root/TwoShard] SET Value2 = -44 WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4; - )"; - ExecuteDataQuery(kikimr, query); - } - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + ExecuteDataQuery(kikimr, query); + } + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM [/Root/TwoShard] WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; - )"; - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[1u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[1u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; DELETE FROM [/Root/TwoShard] WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; - )"; - auto result = ExecuteDataQuery(kikimr, query); - } - { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto result = ExecuteDataQuery(kikimr, query); + } + { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM [/Root/TwoShard] WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; - )"; - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[0u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - } - - Y_UNIT_TEST(InnerJoinTables) { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[0u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + } + + Y_UNIT_TEST(InnerJoinTables) { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM [/Root/Join1] AS t1 INNER JOIN [/Root/Join2] AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - - Y_UNIT_TEST(InnerJoinSelect) { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + FormatResultSetYson(result.GetResultSet(0))); + } + + Y_UNIT_TEST(InnerJoinSelect) { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM [/Root/Join1] AS t1 INNER JOIN (SELECT Key1, Key2, Value2 FROM [/Root/Join2]) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - - Y_UNIT_TEST(InnerJoinSelectAsterisk) { - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + FormatResultSetYson(result.GetResultSet(0))); + } + + Y_UNIT_TEST(InnerJoinSelectAsterisk) { + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM [/Root/Join1] AS t1 INNER JOIN (SELECT * FROM [/Root/Join2]) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } -} - -Y_UNIT_TEST_SUITE(KqpSysColV1) { + FormatResultSetYson(result.GetResultSet(0))); + } +} + +Y_UNIT_TEST_SUITE(KqpSysColV1) { Y_UNIT_TEST_NEW_ENGINE(SelectRowAsterisk) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE Key = 1; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[[1u];["One"];[-1]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[[1u];["One"];[-1]]])", + FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST_NEW_ENGINE(SelectRowById) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", + FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST_NEW_ENGINE(SelectRange) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[72075186224037888u]];[[72075186224037888u]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - + FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST_NEW_ENGINE(UpdateAndDelete) { - TKikimrRunner kikimr; - { + TKikimrRunner kikimr; + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; REPLACE INTO `/Root/TwoShard` (Key, Value1, Value2) - VALUES (4u, "Four", -4); + VALUES (4u, "Four", -4); )"); - auto result = ExecuteDataQuery(kikimr, query); - } - { + auto result = ExecuteDataQuery(kikimr, query); + } + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4; )"); - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[1u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - { + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[1u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; UPDATE `/Root/TwoShard` SET Value2 = -44 WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4; )"); - ExecuteDataQuery(kikimr, query); - } - { + ExecuteDataQuery(kikimr, query); + } + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; )"); - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[1u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - { + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[1u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; DELETE FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; )"); - auto result = ExecuteDataQuery(kikimr, query); - } - { + auto result = ExecuteDataQuery(kikimr, query); + } + { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT COUNT(*) FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44; )"); - auto result = ExecuteDataQuery(kikimr, query); - UNIT_ASSERT(result.GetResultSets().size()); - CompareYson(R"([[0u]])", - FormatResultSetYson(result.GetResultSet(0))); - } - } - + auto result = ExecuteDataQuery(kikimr, query); + UNIT_ASSERT(result.GetResultSets().size()); + CompareYson(R"([[0u]])", + FormatResultSetYson(result.GetResultSet(0))); + } + } + Y_UNIT_TEST_NEW_ENGINE(InnerJoinTables) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN `/Root/Join2` AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - + FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelect) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN (SELECT Key1, Key2, Value2 FROM `/Root/Join2`) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - + FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelectAsterisk) { auto query = Q_(R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN (SELECT * FROM `/Root/Join2`) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; )"); - auto result = ExecuteDataQuery(query); - UNIT_ASSERT(result.GetResultSets().size()); + auto result = ExecuteDataQuery(query); + UNIT_ASSERT(result.GetResultSets().size()); CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])", - FormatResultSetYson(result.GetResultSet(0))); - } - - Y_UNIT_TEST(StreamSelectRowAsterisk) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + FormatResultSetYson(result.GetResultSet(0))); + } + + Y_UNIT_TEST(StreamSelectRowAsterisk) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE Key = 1; - )"; - auto it = ExecuteStreamQuery(kikimr, query); - CompareYson(R"([[[1u];["One"];[-1]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamSelectRowById) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto it = ExecuteStreamQuery(kikimr, query); + CompareYson(R"([[[1u];["One"];[-1]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamSelectRowById) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul; - )"; - auto it = ExecuteStreamQuery(kikimr, query); - CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamSelectRange) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + )"; + auto it = ExecuteStreamQuery(kikimr, query); + CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])", StreamResultToYson(it)); + } + + Y_UNIT_TEST(StreamSelectRange) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3; - )"; - auto it = ExecuteStreamQuery(kikimr, query); + )"; + auto it = ExecuteStreamQuery(kikimr, query); CompareYson(R"([[[72075186224037888u]];[[72075186224037888u]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamInnerJoinTables) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + } + + Y_UNIT_TEST(StreamInnerJoinTables) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN `/Root/Join2` AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto it = ExecuteStreamQuery(kikimr, query); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto it = ExecuteStreamQuery(kikimr, query); CompareYson(R"([[[108u];["One"];[8];[108u];["One"];#;["Value5"];["Value31"]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamInnerJoinSelect) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + } + + Y_UNIT_TEST(StreamInnerJoinSelect) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN (SELECT Key1, Key2, Value2 FROM `/Root/Join2`) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto it = ExecuteStreamQuery(kikimr, query); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto it = ExecuteStreamQuery(kikimr, query); CompareYson(R"([[[108u];["One"];[8];[108u];["One"];["Value5"];["Value31"]]])", StreamResultToYson(it)); - } - - Y_UNIT_TEST(StreamInnerJoinSelectAsterisk) { - TKikimrRunner kikimr; - auto query = R"( - PRAGMA kikimr.EnableSystemColumns = "true"; + } + + Y_UNIT_TEST(StreamInnerJoinSelectAsterisk) { + TKikimrRunner kikimr; + auto query = R"( + PRAGMA kikimr.EnableSystemColumns = "true"; SELECT * FROM `/Root/Join1` AS t1 INNER JOIN (SELECT * FROM `/Root/Join2`) AS t2 - ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 - WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; - )"; - auto it = ExecuteStreamQuery(kikimr, query); + ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 + WHERE t1.Value == "Value5" AND t2.Value2 == "Value31"; + )"; + auto it = ExecuteStreamQuery(kikimr, query); CompareYson(R"([[[108u];["One"];[8];[108u];["One"];#;["Value5"];["Value31"]]])", StreamResultToYson(it)); - } -} - + } +} + } // namespace NKqp -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make index 93f73d1cbf..609ec3237f 100644 --- a/ydb/core/kqp/ut/ya.make +++ b/ydb/core/kqp/ut/ya.make @@ -50,7 +50,7 @@ SRCS( kqp_stats_ut.cpp kqp_sqlin_ut.cpp kqp_sys_view_ut.cpp - kqp_sys_col_ut.cpp + kqp_sys_col_ut.cpp kqp_table_predicate_ut.cpp kqp_tx_ut.cpp kqp_types_arrow_ut.cpp diff --git a/ydb/core/node_whiteboard/node_whiteboard.h b/ydb/core/node_whiteboard/node_whiteboard.h index a5c02d1185..cdf7601807 100644 --- a/ydb/core/node_whiteboard/node_whiteboard.h +++ b/ydb/core/node_whiteboard/node_whiteboard.h @@ -46,15 +46,15 @@ struct TEvWhiteboard{ EvSystemStateSetTenant, EvSystemStateRemoveTenant, EvBSGroupStateDelete, - EvIntrospectionData, - EvTabletLookupRequest, - EvTabletLookupResponse, - EvTraceLookupRequest, - EvTraceLookupResponse, - EvTraceRequest, - EvTraceResponse, - EvSignalBodyRequest, - EvSignalBodyResponse, + EvIntrospectionData, + EvTabletLookupRequest, + EvTabletLookupResponse, + EvTraceLookupRequest, + EvTraceLookupResponse, + EvTraceRequest, + EvTraceResponse, + EvSignalBodyRequest, + EvSignalBodyResponse, EvPDiskStateDelete, EvVDiskStateGenerationChange, EvEnd @@ -375,29 +375,29 @@ struct TEvWhiteboard{ struct TEvNodeStateResponse : TEventPB<TEvNodeStateResponse, NKikimrWhiteboard::TEvNodeStateResponse, EvNodeStateResponse> {}; - - struct TEvIntrospectionData : TEventLocal<TEvIntrospectionData, EvIntrospectionData> { - - THolder<NTracing::ITrace> Trace; - TTabletId TabletId; - - TEvIntrospectionData(TTabletId tabletId, NTracing::ITrace* trace) - : Trace(trace) - , TabletId(tabletId) - {} - }; - - struct TEvTabletLookupRequest : TEventPB<TEvTabletLookupRequest, NKikimrWhiteboard::TEvTabletLookupRequest, EvTabletLookupRequest> {}; - struct TEvTabletLookupResponse : TEventPB<TEvTabletLookupResponse, NKikimrWhiteboard::TEvTabletLookupResponse, EvTabletLookupResponse> {}; - - struct TEvTraceLookupRequest : TEventPB<TEvTraceLookupRequest, NKikimrWhiteboard::TEvTraceLookupRequest, EvTraceLookupRequest> {}; - struct TEvTraceLookupResponse : TEventPB<TEvTraceLookupResponse, NKikimrWhiteboard::TEvTraceLookupResponse, EvTraceLookupResponse> {}; - - struct TEvTraceRequest : TEventPB<TEvTraceRequest, NKikimrWhiteboard::TEvTraceRequest, EvTraceRequest> {}; - struct TEvTraceResponse : TEventPB<TEvTraceResponse, NKikimrWhiteboard::TEvTraceResponse, EvTraceResponse> {}; - - struct TEvSignalBodyRequest : TEventPB<TEvSignalBodyRequest, NKikimrWhiteboard::TEvSignalBodyRequest, EvSignalBodyRequest> {}; - struct TEvSignalBodyResponse : TEventPB<TEvSignalBodyResponse, NKikimrWhiteboard::TEvSignalBodyResponse, EvSignalBodyResponse> {}; + + struct TEvIntrospectionData : TEventLocal<TEvIntrospectionData, EvIntrospectionData> { + + THolder<NTracing::ITrace> Trace; + TTabletId TabletId; + + TEvIntrospectionData(TTabletId tabletId, NTracing::ITrace* trace) + : Trace(trace) + , TabletId(tabletId) + {} + }; + + struct TEvTabletLookupRequest : TEventPB<TEvTabletLookupRequest, NKikimrWhiteboard::TEvTabletLookupRequest, EvTabletLookupRequest> {}; + struct TEvTabletLookupResponse : TEventPB<TEvTabletLookupResponse, NKikimrWhiteboard::TEvTabletLookupResponse, EvTabletLookupResponse> {}; + + struct TEvTraceLookupRequest : TEventPB<TEvTraceLookupRequest, NKikimrWhiteboard::TEvTraceLookupRequest, EvTraceLookupRequest> {}; + struct TEvTraceLookupResponse : TEventPB<TEvTraceLookupResponse, NKikimrWhiteboard::TEvTraceLookupResponse, EvTraceLookupResponse> {}; + + struct TEvTraceRequest : TEventPB<TEvTraceRequest, NKikimrWhiteboard::TEvTraceRequest, EvTraceRequest> {}; + struct TEvTraceResponse : TEventPB<TEvTraceResponse, NKikimrWhiteboard::TEvTraceResponse, EvTraceResponse> {}; + + struct TEvSignalBodyRequest : TEventPB<TEvSignalBodyRequest, NKikimrWhiteboard::TEvSignalBodyRequest, EvSignalBodyRequest> {}; + struct TEvSignalBodyResponse : TEventPB<TEvSignalBodyResponse, NKikimrWhiteboard::TEvSignalBodyResponse, EvSignalBodyResponse> {}; }; inline TActorId MakeNodeWhiteboardServiceId(ui32 node) { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d3b0992fea..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -348,7 +348,7 @@ message TBootstrap { optional NKikimrSharedCache.TSharedCacheConfig SharedCacheConfig = 7; repeated NKikimrSchemeOp.TResourceProfile ResourceProfiles = 8; optional TCompileServiceConfig CompileServiceConfig = 9; //may be need special file with resource limits? - optional bool EnableIntrospection = 10; + optional bool EnableIntrospection = 10; } message TInterconnectConfig { @@ -654,7 +654,7 @@ message TFeatureFlags { optional bool EnableDbCounters = 39 [default = false]; optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false]; optional bool EnableAsyncIndexes = 41 [default = true]; - optional bool AllowStreamExecuteYqlScript = 42 [default = true]; + optional bool AllowStreamExecuteYqlScript = 42 [default = true]; optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true optional bool EnableOlapSchemaOperations = 44 [default = false]; optional bool EnableVPatch = 45 [default = false]; diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index cf81316553..9b46b720e1 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -34,7 +34,7 @@ enum EQueryType { QUERY_TYPE_SQL_SCRIPT = 8; QUERY_TYPE_SQL_SCAN = 9; QUERY_TYPE_AST_SCAN = 10; - QUERY_TYPE_SQL_SCRIPT_STREAMING = 11; + QUERY_TYPE_SQL_SCRIPT_STREAMING = 11; }; enum EQueryAction { @@ -343,11 +343,11 @@ message TEvPingSessionResponse { optional bool WorkerIsClosing = 4 [default = false]; } -message TEvDataQueryStreamPart { - optional NActorsProto.TActorId GatewayActorId = 1; - repeated NKikimrMiniKQL.TResult Results = 2; -}; - +message TEvDataQueryStreamPart { + optional NActorsProto.TActorId GatewayActorId = 1; + repeated NKikimrMiniKQL.TResult Results = 2; +}; + // Executer message TExecuterTxRequest { diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index 898cbe9f8d..b96713ca29 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -301,40 +301,40 @@ message TEvSystemStateResponse { optional uint32 ResponseDuration = 3; // filled during collect } -message TEvTabletLookupRequest { -} - -message TEvTabletLookupResponse { - repeated uint64 TabletIDs = 1; -} - -message TEvTraceLookupRequest { - optional uint64 TabletID = 1; -} - -message TEvTraceLookupResponse { - repeated NKikimrTracing.TTraceID TraceIDs = 1; -} - -message TEvTraceRequest { - optional uint64 TabletID = 1; - optional NKikimrTracing.TTraceID TraceID = 2; - optional uint32 Mode = 3; - optional uint32 Precision = 4; -} - -message TEvTraceResponse { - optional string Trace = 1; -} - -message TEvSignalBodyRequest { - optional uint64 TabletID = 1; - optional NKikimrTracing.TTraceID TraceID = 2; - optional uint32 Mode = 3; - optional uint32 Precision = 4; - optional string SignalID = 5; -} - -message TEvSignalBodyResponse { - optional string SignalBody = 1; -} +message TEvTabletLookupRequest { +} + +message TEvTabletLookupResponse { + repeated uint64 TabletIDs = 1; +} + +message TEvTraceLookupRequest { + optional uint64 TabletID = 1; +} + +message TEvTraceLookupResponse { + repeated NKikimrTracing.TTraceID TraceIDs = 1; +} + +message TEvTraceRequest { + optional uint64 TabletID = 1; + optional NKikimrTracing.TTraceID TraceID = 2; + optional uint32 Mode = 3; + optional uint32 Precision = 4; +} + +message TEvTraceResponse { + optional string Trace = 1; +} + +message TEvSignalBodyRequest { + optional uint64 TabletID = 1; + optional NKikimrTracing.TTraceID TraceID = 2; + optional uint32 Mode = 3; + optional uint32 Precision = 4; + optional string SignalID = 5; +} + +message TEvSignalBodyResponse { + optional string SignalBody = 1; +} diff --git a/ydb/core/protos/tablet_tracing_signals.proto b/ydb/core/protos/tablet_tracing_signals.proto index 0a976b8f59..51c1c7dd4b 100644 --- a/ydb/core/protos/tablet_tracing_signals.proto +++ b/ydb/core/protos/tablet_tracing_signals.proto @@ -2,171 +2,171 @@ import "library/cpp/actors/protos/actors.proto"; import "ydb/core/protos/tablet.proto"; import "ydb/core/protos/base.proto"; import "ydb/core/protos/tracing.proto"; - -package NKikimrTracing; -option java_package = "ru.yandex.kikimr.proto"; - -message TOnCancelTablet { - optional fixed64 TimeStamp = 1; - optional uint64 TabletID = 2; - optional NKikimrTabletBase.TTabletTypes.EType TabletType = 3; - optional uint32 TabletDeadReason = 4; - optional uint32 SuggestedGeneration = 5; - optional uint32 KnownGeneration = 6; -} - -message TOnTabletBootstrap { - optional fixed64 TimeStamp = 1; + +package NKikimrTracing; +option java_package = "ru.yandex.kikimr.proto"; + +message TOnCancelTablet { + optional fixed64 TimeStamp = 1; + optional uint64 TabletID = 2; + optional NKikimrTabletBase.TTabletTypes.EType TabletType = 3; + optional uint32 TabletDeadReason = 4; + optional uint32 SuggestedGeneration = 5; + optional uint32 KnownGeneration = 6; +} + +message TOnTabletBootstrap { + optional fixed64 TimeStamp = 1; optional bool Leader = 2; - optional uint32 SuggestedGeneration = 3; + optional uint32 SuggestedGeneration = 3; optional NActorsProto.TActorId StateStorageProxyID = 4; -} - -message TOnHandleStateStorageInfoResolve { - optional fixed64 TimeStamp = 1; - optional uint32 KnownGeneration = 2; - optional uint32 KnownStep = 3; - optional uint32 SignatureSz = 4; -} - -message TOnLockedInitializationPath { - optional fixed64 TimeStamp = 1; - optional uint32 KnownGeneration = 2; - optional uint32 KnownStep = 3; - optional uint32 SignatureSz = 4; -} - -message TOnHandleStateStorageInfoLock { - optional fixed64 TimeStamp = 1; - optional uint32 KnownGeneration = 2; - optional uint32 KnownStep = 3; - optional uint32 SignatureSz = 4; -} - -message TOnPromoteToCandidate { - optional fixed64 TimeStamp = 1; - optional uint32 KnownGeneration = 2; -} - -message TOnTabletBlockBlobStorage { - optional fixed64 TimeStamp = 1; +} + +message TOnHandleStateStorageInfoResolve { + optional fixed64 TimeStamp = 1; + optional uint32 KnownGeneration = 2; + optional uint32 KnownStep = 3; + optional uint32 SignatureSz = 4; +} + +message TOnLockedInitializationPath { + optional fixed64 TimeStamp = 1; + optional uint32 KnownGeneration = 2; + optional uint32 KnownStep = 3; + optional uint32 SignatureSz = 4; +} + +message TOnHandleStateStorageInfoLock { + optional fixed64 TimeStamp = 1; + optional uint32 KnownGeneration = 2; + optional uint32 KnownStep = 3; + optional uint32 SignatureSz = 4; +} + +message TOnPromoteToCandidate { + optional fixed64 TimeStamp = 1; + optional uint32 KnownGeneration = 2; +} + +message TOnTabletBlockBlobStorage { + optional fixed64 TimeStamp = 1; optional NActorsProto.TActorId ReqBlockBlobStorageID = 2; - optional uint32 KnownGeneration = 3; -} - -message TOnTabletRebuildGraph { - optional fixed64 TimeStamp = 1; + optional uint32 KnownGeneration = 3; +} + +message TOnTabletRebuildGraph { + optional fixed64 TimeStamp = 1; optional NActorsProto.TActorId TabletReqRebuildGraphID = 2; - optional TTraceID RebuildGraphTraceID = 3; -} - -message TOnWriteZeroEntry { - optional fixed64 TimeStamp = 1; - optional uint64 Snapshot = 2; - optional uint64 ZeroConfirmed = 3; - optional uint32 LastInGeneration = 4; -} - + optional TTraceID RebuildGraphTraceID = 3; +} + +message TOnWriteZeroEntry { + optional fixed64 TimeStamp = 1; + optional uint64 Snapshot = 2; + optional uint64 ZeroConfirmed = 3; + optional uint32 LastInGeneration = 4; +} + message TOnFollowerPromoteToLeader { - optional fixed64 TimeStamp = 1; - optional uint32 SuggestedGeneration = 2; + optional fixed64 TimeStamp = 1; + optional uint32 SuggestedGeneration = 2; optional NActorsProto.TActorId KnownLeaderID = 3; optional NActorsProto.TActorId FollowerStStGuardian = 4; -} - -message TRebuildGraphBootstrap { - optional fixed64 TimeStamp = 1; - optional uint32 BlockedGen = 2; -} - -message TErrorEntryBeyondBlocked { - optional fixed64 TimeStamp = 1; - optional NKikimrProto.TLogoBlobID Latest= 2; - optional uint32 BlockedGen = 3; -} - -message TErrorUnknownStatus { - optional fixed64 TimeStamp = 1; - optional NKikimrProto.EReplyStatus Status = 2; - optional string Reason = 3; -} - -message TErrorParsingFromString { - optional fixed64 TimeStamp = 1; - optional NKikimrProto.TLogoBlobID BlobID = 2; -} - -message TErrorSendRefsCheck { - optional fixed64 TimeStamp = 1; -} - -message TErrorRebuildGraph { - optional fixed64 TimeStamp = 1; - optional uint32 Generation = 2; - optional uint32 Step = 3; -} - -message TOnProcessKeyEntry { - optional fixed64 TimeStamp = 1; - optional NKikimrProto.TLogoBlobID LatestEntry = 2; -} - -message TOnProcessZeroEntry { - optional fixed64 TimeStamp = 1; - optional uint32 Generation = 2; - optional uint64 Snapshot = 3; - optional uint64 Confirmed = 4; -} - -message TOnProcessLogEntry { - optional fixed64 TimeStamp = 1; - optional NKikimrProto.TLogoBlobID BlobID = 2; - optional uint64 Snapshot = 3; - optional uint64 Confirmed = 4; - repeated NKikimrProto.TLogoBlobID References = 5; - repeated NKikimrProto.TLogoBlobID GcDiscovered = 6; - repeated NKikimrProto.TLogoBlobID GcLeft = 7; -} - -message TOnDiscoverRangeRequest { - optional fixed64 TimeStamp = 1; - optional uint32 Group = 2; - optional NKikimrProto.TLogoBlobID From = 3; - optional NKikimrProto.TLogoBlobID To = 4; -} - -message TOnApplyDiscoveryRange { - optional fixed64 TimeStamp = 1; - optional uint32 Group = 2; - optional NKikimrProto.TLogoBlobID From = 3; - optional NKikimrProto.TLogoBlobID To = 4; -} - -message TOnMakeHistory { - optional fixed64 TimeStamp = 1; - repeated NKikimrProto.TLogoBlobID RefsToCheck = 2; -} - -message TOnCheckRefsGetResult { - optional fixed64 TimeStamp = 1; - optional uint32 ResponseSz = 2; -} - -message TOnBuildHistoryGraph { - message TEntry { - optional uint32 Generation = 1; - optional uint32 Step = 2; - optional bool IsSnapshot = 3; - repeated NKikimrProto.TLogoBlobID References = 4; - repeated NKikimrProto.TLogoBlobID GcDiscovered = 5; - repeated NKikimrProto.TLogoBlobID GcLeft = 6; - } - - optional fixed64 TimeStamp = 1; - repeated TEntry Entries = 2; -} - -message TOnRebuildGraphResult { - optional fixed64 TimeStamp = 1; - optional string SerializedTrace = 2; -} +} + +message TRebuildGraphBootstrap { + optional fixed64 TimeStamp = 1; + optional uint32 BlockedGen = 2; +} + +message TErrorEntryBeyondBlocked { + optional fixed64 TimeStamp = 1; + optional NKikimrProto.TLogoBlobID Latest= 2; + optional uint32 BlockedGen = 3; +} + +message TErrorUnknownStatus { + optional fixed64 TimeStamp = 1; + optional NKikimrProto.EReplyStatus Status = 2; + optional string Reason = 3; +} + +message TErrorParsingFromString { + optional fixed64 TimeStamp = 1; + optional NKikimrProto.TLogoBlobID BlobID = 2; +} + +message TErrorSendRefsCheck { + optional fixed64 TimeStamp = 1; +} + +message TErrorRebuildGraph { + optional fixed64 TimeStamp = 1; + optional uint32 Generation = 2; + optional uint32 Step = 3; +} + +message TOnProcessKeyEntry { + optional fixed64 TimeStamp = 1; + optional NKikimrProto.TLogoBlobID LatestEntry = 2; +} + +message TOnProcessZeroEntry { + optional fixed64 TimeStamp = 1; + optional uint32 Generation = 2; + optional uint64 Snapshot = 3; + optional uint64 Confirmed = 4; +} + +message TOnProcessLogEntry { + optional fixed64 TimeStamp = 1; + optional NKikimrProto.TLogoBlobID BlobID = 2; + optional uint64 Snapshot = 3; + optional uint64 Confirmed = 4; + repeated NKikimrProto.TLogoBlobID References = 5; + repeated NKikimrProto.TLogoBlobID GcDiscovered = 6; + repeated NKikimrProto.TLogoBlobID GcLeft = 7; +} + +message TOnDiscoverRangeRequest { + optional fixed64 TimeStamp = 1; + optional uint32 Group = 2; + optional NKikimrProto.TLogoBlobID From = 3; + optional NKikimrProto.TLogoBlobID To = 4; +} + +message TOnApplyDiscoveryRange { + optional fixed64 TimeStamp = 1; + optional uint32 Group = 2; + optional NKikimrProto.TLogoBlobID From = 3; + optional NKikimrProto.TLogoBlobID To = 4; +} + +message TOnMakeHistory { + optional fixed64 TimeStamp = 1; + repeated NKikimrProto.TLogoBlobID RefsToCheck = 2; +} + +message TOnCheckRefsGetResult { + optional fixed64 TimeStamp = 1; + optional uint32 ResponseSz = 2; +} + +message TOnBuildHistoryGraph { + message TEntry { + optional uint32 Generation = 1; + optional uint32 Step = 2; + optional bool IsSnapshot = 3; + repeated NKikimrProto.TLogoBlobID References = 4; + repeated NKikimrProto.TLogoBlobID GcDiscovered = 5; + repeated NKikimrProto.TLogoBlobID GcLeft = 6; + } + + optional fixed64 TimeStamp = 1; + repeated TEntry Entries = 2; +} + +message TOnRebuildGraphResult { + optional fixed64 TimeStamp = 1; + optional string SerializedTrace = 2; +} diff --git a/ydb/core/protos/tracing.proto b/ydb/core/protos/tracing.proto index ba6eeb6bc6..83c9b72b94 100644 --- a/ydb/core/protos/tracing.proto +++ b/ydb/core/protos/tracing.proto @@ -1,20 +1,20 @@ -package NKikimrTracing; -option java_package = "ru.yandex.kikimr.proto"; - -message TTraceID { - optional fixed64 CreationTime = 1; - optional fixed64 RandomID = 2; -} - -message TSerializedSignal { - optional uint32 Type = 1; - optional string Signal = 2; -} - -message TTraceSignal { - optional uint32 Type = 1; - optional TTraceID SelfID = 2; - optional TTraceID ParentID = 3; - optional TTraceID RootID = 4; - repeated TSerializedSignal Signals = 5; -} +package NKikimrTracing; +option java_package = "ru.yandex.kikimr.proto"; + +message TTraceID { + optional fixed64 CreationTime = 1; + optional fixed64 RandomID = 2; +} + +message TSerializedSignal { + optional uint32 Type = 1; + optional string Signal = 2; +} + +message TTraceSignal { + optional uint32 Type = 1; + optional TTraceID SelfID = 2; + optional TTraceID ParentID = 3; + optional TTraceID RootID = 4; + repeated TSerializedSignal Signals = 5; +} diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 3d69579887..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -107,8 +107,8 @@ SRCS( tenant_pool.proto tenant_slot_broker.proto test_shard.proto - tracing.proto - tablet_tracing_signals.proto + tracing.proto + tablet_tracing_signals.proto node_whiteboard.proto tx.proto tx_columnshard.proto diff --git a/ydb/core/scheme/scheme_tabledefs.cpp b/ydb/core/scheme/scheme_tabledefs.cpp index 7ebdbf9bb2..8c91e03d0e 100644 --- a/ydb/core/scheme/scheme_tabledefs.cpp +++ b/ydb/core/scheme/scheme_tabledefs.cpp @@ -30,29 +30,29 @@ void TKeyDesc::Out(IOutputStream& o, TKeyDesc::EStatus x) { } } -struct TSystemColumnsData { - const TString PartitionColumnName = "_yql_partition_id"; - - const TMap<TString, TSystemColumnInfo> SystemColumns = { - {PartitionColumnName, {TKeyDesc::EColumnIdDataShard, NScheme::NTypeIds::Uint64}} - }; -}; - -bool IsSystemColumn(ui32 columnId) { - switch (columnId) { - case TKeyDesc::EColumnIdDataShard: - return true; - default: - return false; - } +struct TSystemColumnsData { + const TString PartitionColumnName = "_yql_partition_id"; + + const TMap<TString, TSystemColumnInfo> SystemColumns = { + {PartitionColumnName, {TKeyDesc::EColumnIdDataShard, NScheme::NTypeIds::Uint64}} + }; +}; + +bool IsSystemColumn(ui32 columnId) { + switch (columnId) { + case TKeyDesc::EColumnIdDataShard: + return true; + default: + return false; + } +} + +bool IsSystemColumn(const TStringBuf columnName) { + return GetSystemColumns().FindPtr(columnName); +} + +const TMap<TString, TSystemColumnInfo>& GetSystemColumns() { + return Singleton<TSystemColumnsData>()->SystemColumns; +} + } - -bool IsSystemColumn(const TStringBuf columnName) { - return GetSystemColumns().FindPtr(columnName); -} - -const TMap<TString, TSystemColumnInfo>& GetSystemColumns() { - return Singleton<TSystemColumnsData>()->SystemColumns; -} - -} diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h index 8e1ad8064b..190233fdee 100644 --- a/ydb/core/scheme/scheme_tabledefs.h +++ b/ydb/core/scheme/scheme_tabledefs.h @@ -11,7 +11,7 @@ #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> #include <util/generic/maybe.h> -#include <util/generic/map.h> +#include <util/generic/map.h> namespace NKikimr { @@ -598,12 +598,12 @@ public: InplaceUpdate = 3, }; - enum ESystemColumnIds : ui32 { - EColumnIdInvalid = Max<ui32>(), - EColumnIdSystemStart = EColumnIdInvalid - 1000, - EColumnIdDataShard = EColumnIdSystemStart + 1 - }; - + enum ESystemColumnIds : ui32 { + EColumnIdInvalid = Max<ui32>(), + EColumnIdSystemStart = EColumnIdInvalid - 1000, + EColumnIdDataShard = EColumnIdSystemStart + 1 + }; + #define SCHEME_KEY_DESCRIPTION_STATUS_MAP(XX) \ XX(Unknown, 0) \ XX(Ok, 1) \ @@ -695,16 +695,16 @@ public: {} }; -struct TSystemColumnInfo { - TKeyDesc::ESystemColumnIds ColumnId; - NKikimr::NScheme::TTypeId TypeId; -}; - -const TMap<TString, TSystemColumnInfo>& GetSystemColumns(); - -bool IsSystemColumn(ui32 columnId); -bool IsSystemColumn(const TStringBuf columnName); - +struct TSystemColumnInfo { + TKeyDesc::ESystemColumnIds ColumnId; + NKikimr::NScheme::TTypeId TypeId; +}; + +const TMap<TString, TSystemColumnInfo>& GetSystemColumns(); + +bool IsSystemColumn(ui32 columnId); +bool IsSystemColumn(const TStringBuf columnName); + inline int ComparePointKeys(const TKeyDesc& point1, const TKeyDesc& point2) { Y_VERIFY(point1.Range.Point); Y_VERIFY(point2.Range.Point); diff --git a/ydb/core/sys_view/service/sysview_service.cpp b/ydb/core/sys_view/service/sysview_service.cpp index 6e51cc79f5..1f56906599 100644 --- a/ydb/core/sys_view/service/sysview_service.cpp +++ b/ydb/core/sys_view/service/sysview_service.cpp @@ -67,7 +67,7 @@ void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery strType = "scan"; break; case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: - case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: strType = "script"; break; default: diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp index 3d362c61e1..783758e99d 100644 --- a/ydb/core/tablet/node_whiteboard.cpp +++ b/ydb/core/tablet/node_whiteboard.cpp @@ -36,10 +36,10 @@ public: } void Bootstrap(const TActorContext &ctx) { - TIntrusivePtr<NMonitoring::TDynamicCounters> tabletsGroup = GetServiceCounters(AppData(ctx)->Counters, "tablets"); - TIntrusivePtr<NMonitoring::TDynamicCounters> introspectionGroup = tabletsGroup->GetSubgroup("type", "introspection"); + TIntrusivePtr<NMonitoring::TDynamicCounters> tabletsGroup = GetServiceCounters(AppData(ctx)->Counters, "tablets"); + TIntrusivePtr<NMonitoring::TDynamicCounters> introspectionGroup = tabletsGroup->GetSubgroup("type", "introspection"); TabletIntrospectionData.Reset(NTracing::CreateTraceCollection(introspectionGroup)); - + SystemStateInfo.SetNumberOfCpus(NSystemInfo::NumberOfCpus()); TString branch = GetTag(); if (branch.empty()) { @@ -76,7 +76,7 @@ protected: std::unordered_map<TVDiskID, NKikimrWhiteboard::TVDiskStateInfo, THash<TVDiskID>> VDiskStateInfo; std::unordered_map<ui32, NKikimrWhiteboard::TBSGroupStateInfo> BSGroupStateInfo; NKikimrWhiteboard::TSystemStateInfo SystemStateInfo; - THolder<NTracing::ITraceCollection> TabletIntrospectionData; + THolder<NTracing::ITraceCollection> TabletIntrospectionData; TProcStat ProcessStats; template <typename PropertyType> @@ -363,11 +363,11 @@ protected: HFunc(TEvWhiteboard::TEvSystemStateAddRole, Handle); HFunc(TEvWhiteboard::TEvSystemStateSetTenant, Handle); HFunc(TEvWhiteboard::TEvSystemStateRequest, Handle); - hFunc(TEvWhiteboard::TEvIntrospectionData, Handle); - HFunc(TEvWhiteboard::TEvTabletLookupRequest, Handle); - HFunc(TEvWhiteboard::TEvTraceLookupRequest, Handle); - HFunc(TEvWhiteboard::TEvTraceRequest, Handle); - HFunc(TEvWhiteboard::TEvSignalBodyRequest, Handle); + hFunc(TEvWhiteboard::TEvIntrospectionData, Handle); + HFunc(TEvWhiteboard::TEvTabletLookupRequest, Handle); + HFunc(TEvWhiteboard::TEvTraceLookupRequest, Handle); + HFunc(TEvWhiteboard::TEvTraceRequest, Handle); + HFunc(TEvWhiteboard::TEvSignalBodyRequest, Handle); HFunc(TEvPrivate::TEvUpdateRuntimeStats, Handle); HFunc(TEvPrivate::TEvCleanupDeadTablets, Handle); } @@ -699,86 +699,86 @@ protected: ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); } - void Handle(TEvWhiteboard::TEvIntrospectionData::TPtr &ev) { - TEvWhiteboard::TEvIntrospectionData *msg = ev->Get(); - TabletIntrospectionData->AddTrace(msg->TabletId, msg->Trace.Release()); - } - - void Handle(TEvWhiteboard::TEvTabletLookupRequest::TPtr &ev, const TActorContext &ctx) { + void Handle(TEvWhiteboard::TEvIntrospectionData::TPtr &ev) { + TEvWhiteboard::TEvIntrospectionData *msg = ev->Get(); + TabletIntrospectionData->AddTrace(msg->TabletId, msg->Trace.Release()); + } + + void Handle(TEvWhiteboard::TEvTabletLookupRequest::TPtr &ev, const TActorContext &ctx) { THolder<TEvWhiteboard::TEvTabletLookupResponse> response = MakeHolder<TEvWhiteboard::TEvTabletLookupResponse>(); - auto& record = response->Record; - TVector<ui64> tabletIDs; - TabletIntrospectionData->GetTabletIDs(tabletIDs); - for (auto id : tabletIDs) { - record.AddTabletIDs(id); - } - ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); - } - - void Handle(TEvWhiteboard::TEvTraceLookupRequest::TPtr &ev, const TActorContext &ctx) { - ui64 tabletID = ev->Get()->Record.GetTabletID(); + auto& record = response->Record; + TVector<ui64> tabletIDs; + TabletIntrospectionData->GetTabletIDs(tabletIDs); + for (auto id : tabletIDs) { + record.AddTabletIDs(id); + } + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } + + void Handle(TEvWhiteboard::TEvTraceLookupRequest::TPtr &ev, const TActorContext &ctx) { + ui64 tabletID = ev->Get()->Record.GetTabletID(); THolder<TEvWhiteboard::TEvTraceLookupResponse> response = MakeHolder<TEvWhiteboard::TEvTraceLookupResponse>(); - auto& record = response->Record; - TVector<NTracing::TTraceID> tabletTraces; - TabletIntrospectionData->GetTraces(tabletID, tabletTraces); - for (auto& tabletTrace : tabletTraces) { - TraceIDFromTraceID(tabletTrace, record.AddTraceIDs()); - } - ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); - } - - void Handle(TEvWhiteboard::TEvTraceRequest::TPtr &ev, const TActorContext &ctx) { - auto& requestRecord = ev->Get()->Record; - ui64 tabletID = requestRecord.GetTabletID(); - NTracing::TTraceID traceID = NTracing::TraceIDFromTraceID(requestRecord.GetTraceID()); - + auto& record = response->Record; + TVector<NTracing::TTraceID> tabletTraces; + TabletIntrospectionData->GetTraces(tabletID, tabletTraces); + for (auto& tabletTrace : tabletTraces) { + TraceIDFromTraceID(tabletTrace, record.AddTraceIDs()); + } + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } + + void Handle(TEvWhiteboard::TEvTraceRequest::TPtr &ev, const TActorContext &ctx) { + auto& requestRecord = ev->Get()->Record; + ui64 tabletID = requestRecord.GetTabletID(); + NTracing::TTraceID traceID = NTracing::TraceIDFromTraceID(requestRecord.GetTraceID()); + THolder<TEvWhiteboard::TEvTraceResponse> response = MakeHolder<TEvWhiteboard::TEvTraceResponse>(); - auto& responseRecord = response->Record; - auto trace = TabletIntrospectionData->GetTrace(tabletID, traceID); - NTracing::TTraceInfo traceInfo = { - ctx.SelfID.NodeId(), - tabletID, - traceID, - NTracing::TTimestampInfo( - static_cast<NTracing::TTimestampInfo::EMode>(requestRecord.GetMode()), - static_cast<NTracing::TTimestampInfo::EPrecision>(requestRecord.GetPrecision()) - ) - }; - TStringStream str; - if (trace) { - trace->OutHtml(str, traceInfo); - } else { - str << "Trace not found."; - } - responseRecord.SetTrace(str.Str()); - ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); - } - - void Handle(TEvWhiteboard::TEvSignalBodyRequest::TPtr &ev, const TActorContext &ctx) { - auto& requestRecord = ev->Get()->Record; - ui64 tabletID = requestRecord.GetTabletID(); - NTracing::TTraceID traceID = NTracing::TraceIDFromTraceID(requestRecord.GetTraceID()); - TString signalID = requestRecord.GetSignalID(); - + auto& responseRecord = response->Record; + auto trace = TabletIntrospectionData->GetTrace(tabletID, traceID); + NTracing::TTraceInfo traceInfo = { + ctx.SelfID.NodeId(), + tabletID, + traceID, + NTracing::TTimestampInfo( + static_cast<NTracing::TTimestampInfo::EMode>(requestRecord.GetMode()), + static_cast<NTracing::TTimestampInfo::EPrecision>(requestRecord.GetPrecision()) + ) + }; + TStringStream str; + if (trace) { + trace->OutHtml(str, traceInfo); + } else { + str << "Trace not found."; + } + responseRecord.SetTrace(str.Str()); + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } + + void Handle(TEvWhiteboard::TEvSignalBodyRequest::TPtr &ev, const TActorContext &ctx) { + auto& requestRecord = ev->Get()->Record; + ui64 tabletID = requestRecord.GetTabletID(); + NTracing::TTraceID traceID = NTracing::TraceIDFromTraceID(requestRecord.GetTraceID()); + TString signalID = requestRecord.GetSignalID(); + THolder<TEvWhiteboard::TEvSignalBodyResponse> response = MakeHolder<TEvWhiteboard::TEvSignalBodyResponse>(); - auto& responseRecord = response->Record; - auto trace = TabletIntrospectionData->GetTrace(tabletID, traceID); - TStringStream str; - if (trace) { - trace->OutSignalHtmlBody( - str, - NTracing::TTimestampInfo( - static_cast<NTracing::TTimestampInfo::EMode>(requestRecord.GetMode()), - static_cast<NTracing::TTimestampInfo::EPrecision>(requestRecord.GetPrecision()) - ), - signalID); - } else { - str << "Trace not found."; - } - responseRecord.SetSignalBody(str.Str()); - ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); - } - + auto& responseRecord = response->Record; + auto trace = TabletIntrospectionData->GetTrace(tabletID, traceID); + TStringStream str; + if (trace) { + trace->OutSignalHtmlBody( + str, + NTracing::TTimestampInfo( + static_cast<NTracing::TTimestampInfo::EMode>(requestRecord.GetMode()), + static_cast<NTracing::TTimestampInfo::EPrecision>(requestRecord.GetPrecision()) + ), + signalID); + } else { + str << "Trace not found."; + } + responseRecord.SetSignalBody(str.Str()); + ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); + } + static TVector<double> GetLoadAverage() { TVector<double> loadAvg(3); loadAvg.resize(NSystemInfo::LoadAverage(loadAvg.data(), loadAvg.size())); diff --git a/ydb/core/tablet/tablet_impl.h b/ydb/core/tablet/tablet_impl.h index 641b5a9ae1..b47b1f1097 100644 --- a/ydb/core/tablet/tablet_impl.h +++ b/ydb/core/tablet/tablet_impl.h @@ -55,29 +55,29 @@ struct TEvTabletBase { TIntrusivePtr<TEvTablet::TDependencyGraph> DependencyGraph; NMetrics::TTabletThroughputRawValue GroupReadBytes; NMetrics::TTabletIopsRawValue GroupReadOps; - THolder<NTracing::ITrace> Trace; + THolder<NTracing::ITrace> Trace; - TEvRebuildGraphResult( - NKikimrProto::EReplyStatus status, - NTracing::ITrace *trace, - const TString &reason = TString() - ) + TEvRebuildGraphResult( + NKikimrProto::EReplyStatus status, + NTracing::ITrace *trace, + const TString &reason = TString() + ) : Status(status) , ErrorReason(reason) - , Trace(trace) + , Trace(trace) {} - TEvRebuildGraphResult( - const TIntrusivePtr<TEvTablet::TDependencyGraph> &graph, - NMetrics::TTabletThroughputRawValue &&read, + TEvRebuildGraphResult( + const TIntrusivePtr<TEvTablet::TDependencyGraph> &graph, + NMetrics::TTabletThroughputRawValue &&read, NMetrics::TTabletIopsRawValue &&readOps, - NTracing::ITrace *trace - ) + NTracing::ITrace *trace + ) : Status(NKikimrProto::OK) , DependencyGraph(graph) , GroupReadBytes(std::move(read)) , GroupReadOps(std::move(readOps)) - , Trace(trace) + , Trace(trace) {} }; diff --git a/ydb/core/tablet/tablet_req_rebuildhistory.cpp b/ydb/core/tablet/tablet_req_rebuildhistory.cpp index 1eaffe9de7..c0b4c0c2bc 100644 --- a/ydb/core/tablet/tablet_req_rebuildhistory.cpp +++ b/ydb/core/tablet/tablet_req_rebuildhistory.cpp @@ -10,8 +10,8 @@ #include <ydb/core/protos/services.pb.h> #include <google/protobuf/text_format.h> -#include "tablet_tracing_signals.h" - +#include "tablet_tracing_signals.h" + #if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR #error log macro definition clash #endif @@ -202,9 +202,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil NMetrics::TTabletThroughputRawValue GroupReadBytes; NMetrics::TTabletIopsRawValue GroupReadOps; - THolder<NTracing::ITrace> IntrospectionTrace; + THolder<NTracing::ITrace> IntrospectionTrace; const ui64 FollowerCookie; - + TGenerationEntry& GenerationInfo(ui32 gen) { TGenerationEntry& x = LogInfo[gen]; if (gen == Snapshot.first) @@ -220,9 +220,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil void ProcessZeroEntry(ui32 gen, NKikimrTabletBase::TTabletLogEntry &logEntry) { BLOG_D("TTabletReqRebuildHistoryGraph::ProcessZeroEntry - generation " << gen); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnProcessZeroEntry>(gen, Snapshot, Confirmed)); - } + } Y_VERIFY(logEntry.HasZeroConfirmed() && logEntry.HasZeroTailSz()); @@ -326,9 +326,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil } void ProcessLogEntry(const TLogoBlobID &id, NKikimrTabletBase::TTabletLogEntry &logEntry) { - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnProcessLogEntry>(id, Snapshot, Confirmed, logEntry)); - } + } Y_VERIFY(logEntry.HasSnapshot() && logEntry.HasConfirmed()); LOG_DEBUG(*TlsActivationContext, NKikimrServices::TABLET_MAIN, [&](){ @@ -380,9 +380,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil SendToBSProxy(SelfId(), group, new TEvBlobStorage::TEvRange(tabletId, fromId, toId, mustRestoreFirst, TInstant::Max(), false, BlockedGen)); RangesToDiscover.insert(toId); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnDiscoverRangeRequest>(group, fromId, toId)); - } + } if (lastGen) return; @@ -395,9 +395,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil NKikimrTabletBase::TTabletLogEntry logEntry; if (!logEntry.ParseFromString(logBody)) { BLOG_ERROR("TTabletReqRebuildHistoryGraph::ProcessKeyEntry logBody ParseFromString error, id# " << id); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorParsingFromString>(id)); - } + } return ReplyAndDie(NKikimrProto::ERROR, "Log entry parse failed"); } @@ -446,9 +446,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil } void ApplyDiscoveryRange(TEvBlobStorage::TEvRangeResult *msg) { - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnApplyDiscoveryRange>(msg->GroupId, msg->From, msg->To)); - } + } Y_VERIFY(RangesToDiscover.erase(msg->To)); for (TVector<TEvBlobStorage::TEvRangeResult::TResponse>::iterator it = msg->Responses.begin(), end = msg->Responses.end(); it != end; ++it) { const TLogoBlobID &id = it->Id; @@ -477,18 +477,18 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil } ScanRefsToCheck(); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnMakeHistory>(RefsToCheck)); - } + } if (RefsToCheckByGroup.empty()) return BuildHistory(); for (auto &xpair : RefsToCheckByGroup) { if (!SendRefsCheck(xpair.second, xpair.first)) { BLOG_ERROR("TTabletReqRebuildHistoryGraph::MakeHistory SendRefsCheck A error"); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorSendRefsCheck>()); - } + } return ReplyAndDie(NKikimrProto::ERROR, "SendRefsCheck failed"); } } @@ -567,9 +567,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil } void CheckReferences(TEvBlobStorage::TEvGetResult *msg) { - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnCheckRefsGetResult>(msg->ResponseSz)); - } + } Y_VERIFY_DEBUG(msg->Status == NKikimrProto::OK); for (ui32 i = 0, e = msg->ResponseSz; i != e; ++i) { @@ -586,9 +586,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil default: BLOG_ERROR("TTabletReqRebuildHistoryGraph::CheckReferences - blob " << response.Id << " Status# " << NKikimrProto::EReplyStatus_Name(response.Status)); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorUnknownStatus>(response.Status, msg->ErrorReason)); - } + } return ReplyAndDie(NKikimrProto::ERROR, msg->ErrorReason); } } @@ -761,15 +761,15 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil sb << Info->TabletID << ":" << invalidLogEntry.first << ":" << invalidLogEntry.second; return (TString)sb; }()); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorRebuildGraph>(invalidLogEntry.first, invalidLogEntry.second)); - } + } return ReplyAndDie(NKikimrProto::ERROR, "Graph has missing log entries"); } - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnBuildHistoryGraph>(graph.Get())); - } + } Send(Owner, new TEvTabletBase::TEvRebuildGraphResult( graph.Release(), @@ -793,15 +793,15 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil if (FollowerCookie == 0 && msg->Latest.Generation() > BlockedGen) { BLOG_ERROR("TTabletReqRebuildHistoryGraph - Found entry beyond blocked generation" << " LastBlobID: " << msg->Latest.ToString() << ". Blocked: " << BlockedGen); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorEntryBeyondBlocked>(msg->Latest, BlockedGen)); - } + } return ReplyAndDie(NKikimrProto::ERROR, "Found entry beyond blocked generation"); } - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnProcessKeyEntry>(msg->Latest)); - } + } return ProcessKeyEntry(msg->Latest, msg->Buffer); case NKikimrProto::RACE: @@ -810,9 +810,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil default: BLOG_ERROR("TTabletReqRebuildHistoryGraph::Handle TEvFindLatestLogEntryResult" << " Status# " << NKikimrProto::EReplyStatus_Name(msg->Status)); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorUnknownStatus>(msg->Status, msg->ErrorReason)); - } + } return ReplyAndDie(NKikimrProto::ERROR, msg->ErrorReason); } } @@ -830,9 +830,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil BLOG_ERROR("TTabletReqRebuildHistoryGraph::HandleDiscover TEvRangeResult" << " Status# " << NKikimrProto::EReplyStatus_Name(msg->Status) << " Result# " << msg->Print(false)); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorUnknownStatus>(msg->Status, msg->ErrorReason)); - } + } return ReplyAndDie(NKikimrProto::ERROR, msg->ErrorReason); } @@ -854,9 +854,9 @@ class TTabletReqRebuildHistoryGraph : public TActorBootstrapped<TTabletReqRebuil BLOG_ERROR("TTabletReqRebuildHistoryGraph::Handle TEvGetResult" << " Status# " << NKikimrProto::EReplyStatus_Name(msg->Status) << " Result# " << msg->Print(false)); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TErrorUnknownStatus>(msg->Status, msg->ErrorReason)); - } + } return ReplyAndDie(NKikimrProto::ERROR, msg->ErrorReason); } } @@ -871,15 +871,15 @@ public: , Info(info) , BlockedGen(blockedGen) , RequestsLeft(0) - , IntrospectionTrace(trace) + , IntrospectionTrace(trace) , FollowerCookie(followerCookie) {} void Bootstrap() { - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TRebuildGraphBootstrap>(BlockedGen)); - } - + } + if (FollowerCookie == 0) Register(CreateTabletFindLastEntry(SelfId(), true, Info.Get(), BlockedGen)); else diff --git a/ydb/core/tablet/tablet_resolver.cpp b/ydb/core/tablet/tablet_resolver.cpp index b73d2b34ef..c0a39a2216 100644 --- a/ydb/core/tablet/tablet_resolver.cpp +++ b/ydb/core/tablet/tablet_resolver.cpp @@ -268,11 +268,11 @@ class TTabletResolver : public TActorBootstrapped<TTabletResolver> { LOG_DEBUG(ctx, NKikimrServices::TABLET_RESOLVER, "SelectForward node %" PRIu32 " selfDC %s leaderDC %s %s" - " local %" PRIu32 " localDc %" PRIu32 " other %" PRIu32 " disallowed %" PRIu32 + " local %" PRIu32 " localDc %" PRIu32 " other %" PRIu32 " disallowed %" PRIu32 " tabletId: %" PRIu64 " followers: %" PRIu64 " countLeader %" PRIu32 " allowFollowers %" PRIu32 " winner: %s", selfNode, dcName(selfDc), dcName(leaderDc), info.ResFlags.ToString().data(), - info.NumLocal, info.NumLocalDc, info.NumOtherDc, disallowed, + info.NumLocal, info.NumLocalDc, info.NumOtherDc, disallowed, tabletId, entry.KnownFollowers.size(), countLeader, info.ResFlags.AllowFollower(), winner.KnownLeader.ToString().c_str()); diff --git a/ydb/core/tablet/tablet_sys.cpp b/ydb/core/tablet/tablet_sys.cpp index 84f327e80f..44bfac7b2d 100644 --- a/ydb/core/tablet/tablet_sys.cpp +++ b/ydb/core/tablet/tablet_sys.cpp @@ -76,9 +76,9 @@ void TTablet::PromoteToCandidate(ui32 gen) { Y_VERIFY_DEBUG(SetupInfo); if (!UserTablet) UserTablet = SetupInfo->Apply(Info.Get(), SelfId()); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnPromoteToCandidate>(StateStorageInfo.KnownGeneration)); - } + } // todo: handle 'proxy not found' case Send(StateStorageInfo.ProxyID, new TEvStateStorage::TEvUpdate(TabletID(), 0, SelfId(), UserTablet, StateStorageInfo.KnownGeneration, 0, StateStorageInfo.Signature.Get(), StateStorageInfo.SignatureSz, TEvStateStorage::TProxyOptions::SigAsync)); @@ -93,30 +93,30 @@ void TTablet::TabletBlockBlobStorage() { IActor * const x = CreateTabletReqBlockBlobStorage(SelfId(), Info.Get(), StateStorageInfo.KnownGeneration - 1, false); TActorId newActorId = Register(x); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnTabletBlockBlobStorage>(newActorId, StateStorageInfo.KnownGeneration)); - } - + } + Become(&TThis::StateBlockBlobStorage); ReportTabletStateChange(TTabletStateInfo::BlockBlobStorage); } void TTablet::TabletRebuildGraph() { - THolder<NTracing::ITrace> newTrace; - NTracing::TTraceID rebuildGraphTraceID; - if (IntrospectionTrace) { + THolder<NTracing::ITrace> newTrace; + NTracing::TTraceID rebuildGraphTraceID; + if (IntrospectionTrace) { newTrace = THolder<NTracing::ITrace>(IntrospectionTrace->CreateTrace(NTracing::ITrace::TypeReqRebuildHistoryGraph)); - rebuildGraphTraceID = newTrace->GetSelfID(); - } + rebuildGraphTraceID = newTrace->GetSelfID(); + } RebuildGraphRequest = Register(CreateTabletReqRebuildHistoryGraph( SelfId(), Info.Get(), StateStorageInfo.KnownGeneration - 1, IntrospectionTrace ? newTrace.Release() : nullptr, 0) ); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnTabletRebuildGraph>(RebuildGraphRequest, rebuildGraphTraceID)); - } - + } + Become(&TThis::StateRebuildGraph); ReportTabletStateChange(TTabletStateInfo::RebuildGraph); } @@ -168,10 +168,10 @@ void TTablet::WriteZeroEntry(TEvTablet::TDependencyGraph *graph) { } } - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnWriteZeroEntry>(snapshot, lastGeneration, confirmedStep, lastInGeneration)); - } - + } + // fill tail bitmask (beyond continuous confirmed range) (if any present) TAutoPtr<NKikimrTabletBase::TTabletLogEntry> entry(new NKikimrTabletBase::TTabletLogEntry()); @@ -514,14 +514,14 @@ void TTablet::HandleByFollower(TEvTablet::TEvFollowerUpdate::TPtr &ev) { void TTablet::HandleByFollower(TEvTablet::TEvPromoteToLeader::TPtr &ev) { TEvTablet::TEvPromoteToLeader *msg = ev->Get(); BLOG_TRACE("Follower got TEvPromoteToLeader Sender# " << ev->Sender << " Generation# " << msg->SuggestedGeneration); - - if (IntrospectionTrace) { + + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnFollowerPromoteToLeader>( - msg->SuggestedGeneration + msg->SuggestedGeneration , FollowerInfo.KnownLeaderID , FollowerStStGuardian)); - } - + } + Info = msg->TabletStorageInfo; // detach from leader @@ -778,14 +778,14 @@ void TTablet::HandleStateStorageInfoResolve(TEvStateStorage::TEvInfo::TPtr &ev) if (SuggestedGeneration && StateStorageInfo.KnownGeneration >= SuggestedGeneration) return CancelTablet(TEvTablet::TEvTabletDead::ReasonBootSuggestOutdated); - if (IntrospectionTrace) { - IntrospectionTrace->Attach( + if (IntrospectionTrace) { + IntrospectionTrace->Attach( MakeHolder<NTracing::TOnHandleStateStorageInfoResolve>( - StateStorageInfo.KnownGeneration - , StateStorageInfo.KnownStep - , StateStorageInfo.SignatureSz)); - } - + StateStorageInfo.KnownGeneration + , StateStorageInfo.KnownStep + , StateStorageInfo.SignatureSz)); + } + switch (msg->Status) { case NKikimrProto::OK: { // enough replicas replied and we have state info now @@ -831,14 +831,14 @@ void TTablet::HandleStateStorageInfoLock(TEvStateStorage::TEvInfo::TPtr &ev) { case NKikimrProto::OK: { // ok, we had successfully locked state storage for synthetic generation, now find actual one StateStorageInfo.Update(msg); - - if (IntrospectionTrace) { + + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnHandleStateStorageInfoLock>( - StateStorageInfo.KnownGeneration - , StateStorageInfo.KnownStep - , StateStorageInfo.SignatureSz)); - } - + StateStorageInfo.KnownGeneration + , StateStorageInfo.KnownStep + , StateStorageInfo.SignatureSz)); + } + Register(CreateTabletFindLastEntry(SelfId(), false, Info.Get(), 0)); Become(&TThis::StateDiscover); ReportTabletStateChange(TTabletStateInfo::Discover); @@ -942,9 +942,9 @@ void TTablet::HandleRebuildGraphResult(TEvTabletBase::TEvRebuildGraphResult::TPt RebuildGraphRequest = TActorId(); // check consistency?? TEvTabletBase::TEvRebuildGraphResult *msg = ev->Get(); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnRebuildGraphResult>(msg->Trace.Get())); - } + } TIntrusivePtr<TEvTablet::TDependencyGraph> graph; switch (msg->Status) { case NKikimrProto::OK: @@ -1734,16 +1734,16 @@ void TTablet::CancelTablet(TEvTablet::TEvTabletDead::EReason reason, const TStri PipeConnectAcceptor->Detach(SelfId()); const ui32 reportedGeneration = SuggestedGeneration ? SuggestedGeneration : StateStorageInfo.KnownGeneration; - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnCancelTablet>( - this->TabletID() - , Info->TabletType - , reason - , SuggestedGeneration - , StateStorageInfo.KnownGeneration)); + this->TabletID() + , Info->TabletType + , reason + , SuggestedGeneration + , StateStorageInfo.KnownGeneration)); SendIntrospectionData(); - } - + } + Send(Launcher, new TEvTablet::TEvTabletDead(TabletID(), reason, reportedGeneration)); if (UserTablet) @@ -1801,12 +1801,12 @@ void TTablet::LockedInitializationPath() { StateStorageInfo.KnownGeneration = latestChangeGeneration; StateStorageInfo.KnownStep = 0; } - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnLockedInitializationPath>( - StateStorageInfo.KnownGeneration - , StateStorageInfo.KnownStep - , StateStorageInfo.SignatureSz)); - } + StateStorageInfo.KnownGeneration + , StateStorageInfo.KnownStep + , StateStorageInfo.SignatureSz)); + } // lock => find latest => update => normal path Send(StateStorageInfo.ProxyID, new TEvStateStorage::TEvLock(TabletID(), 0, SelfId(), StateStorageInfo.KnownGeneration + 1, StateStorageInfo.Signature.Get(), StateStorageInfo.SignatureSz, TEvStateStorage::TProxyOptions::SigAsync)); @@ -1870,15 +1870,15 @@ void TTablet::BootstrapFollower() { Leader = false; BoostrapTime = AppData()->TimeProvider->Now(); bool enInt = AppData()->EnableIntrospection; - if (enInt) { + if (enInt) { IntrospectionTrace.Reset(NTracing::CreateTrace(NTracing::ITrace::TypeSysTabletBootstrap)); - } + } StateStorageInfo.ProxyID = MakeStateStorageProxyID(StateStorageGroup()); Send(StateStorageInfo.ProxyID, new TEvStateStorage::TEvLookup(TabletID(), 0, TEvStateStorage::TProxyOptions(TEvStateStorage::TProxyOptions::SigAsync))); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnTabletBootstrap>(SuggestedGeneration, false, StateStorageInfo.ProxyID)); - } + } Become(&TThis::StateResolveLeader); ReportTabletStateChange(TTabletStateInfo::ResolveLeader); @@ -1889,15 +1889,15 @@ void TTablet::Bootstrap() { Leader = true; BoostrapTime = AppData()->TimeProvider->Now(); bool enInt = AppData()->EnableIntrospection; - if (enInt) { + if (enInt) { IntrospectionTrace.Reset(NTracing::CreateTrace(NTracing::ITrace::TypeSysTabletBootstrap)); - } + } ReportTabletStateChange(TTabletStateInfo::Created); // useless? StateStorageInfo.ProxyID = MakeStateStorageProxyID(StateStorageGroup()); Send(StateStorageInfo.ProxyID, new TEvStateStorage::TEvLookup(TabletID(), 0, TEvStateStorage::TProxyOptions(TEvStateStorage::TProxyOptions::SigAsync))); - if (IntrospectionTrace) { + if (IntrospectionTrace) { IntrospectionTrace->Attach(MakeHolder<NTracing::TOnTabletBootstrap>(SuggestedGeneration, true, StateStorageInfo.ProxyID)); - } + } // todo: handle "proxy unknown" case (normal timeouts are handled by proxy) PipeConnectAcceptor->Detach(SelfId()); Become(&TThis::StateResolveStateStorage); @@ -1949,5 +1949,5 @@ void TTablet::SendIntrospectionData() { Send(tabletStateServiceId, new NNodeWhiteboard::TEvWhiteboard::TEvIntrospectionData(TabletID(), IntrospectionTrace.Release())); IntrospectionTrace.Reset(NTracing::CreateTrace(NTracing::ITrace::TypeSysTabletBootstrap)); } - -} + +} diff --git a/ydb/core/tablet/tablet_sys.h b/ydb/core/tablet/tablet_sys.h index 6067e22822..661d75a4e0 100644 --- a/ydb/core/tablet/tablet_sys.h +++ b/ydb/core/tablet/tablet_sys.h @@ -224,7 +224,7 @@ class TTablet : public TActor<TTablet> { ui32 GcNextStep; TResourceProfilesPtr ResourceProfiles; TSharedQuotaPtr TxCacheQuota; - THolder<NTracing::ITrace> IntrospectionTrace; + THolder<NTracing::ITrace> IntrospectionTrace; TActorId RebuildGraphRequest; // Delayed cancellation reason @@ -359,7 +359,7 @@ class TTablet : public TActor<TTablet> { void RetryFollowerBootstrapOrWait(); void SendIntrospectionData(); - + STATEFN(StateResolveStateStorage) { switch (ev->GetTypeRewrite()) { hFunc(TEvStateStorage::TEvInfo, HandleStateStorageInfoResolve); diff --git a/ydb/core/tablet/tablet_tracing_signals.cpp b/ydb/core/tablet/tablet_tracing_signals.cpp index 13f10d849c..9d1923f822 100644 --- a/ydb/core/tablet/tablet_tracing_signals.cpp +++ b/ydb/core/tablet/tablet_tracing_signals.cpp @@ -1,840 +1,840 @@ -#include "tablet_tracing_signals.h" +#include "tablet_tracing_signals.h" #include <ydb/core/tracing/http.h> #include <ydb/core/base/tablet.h> - -namespace NKikimr { -namespace NTracing { - -template <typename TStreamImpl> -void PrintGenStepPair(TStreamImpl& str, ui64 genStepInt) { - auto genStepPair = ExpandGenStepPair(genStepInt); - str << genStepPair.first << ":" << genStepPair.second; -} - -template <typename TStreamImpl> -void PrintLogoBlob(TStreamImpl& str, const NKikimrProto::TLogoBlobID& protoLogoBlobId) { - TLogoBlobID logoBlobId = LogoBlobIDFromLogoBlobID(protoLogoBlobId); - str << logoBlobId.ToString(); -} - -TSignalFactory& GetSignalFactoryInstance() { - static TSignalFactory SingleFactory; - return SingleFactory; -} - -struct TSignalTypeRegistrator { - TSignalTypeRegistrator() { - Initialize(); - } - - void Initialize() { - Register<TOnTabletBootstrap>(NSignalTypes::TypeOnTabletBootstrap); - Register<TOnHandleStateStorageInfoResolve>(NSignalTypes::TypeOnHandleStateStorageInfoResolve); - Register<TOnCancelTablet>(NSignalTypes::TypeOnCancelTablet); - Register<TOnLockedInitializationPath>(NSignalTypes::TypeOnLockedInitializationPath); - Register<TOnHandleStateStorageInfoLock>(NSignalTypes::TypeOnHandleStateStorageInfoLock); - Register<TOnPromoteToCandidate>(NSignalTypes::TypeOnPromoteToCandidate); - Register<TOnTabletBlockBlobStorage>(NSignalTypes::TypeOnTabletBlockBlobStorage); - Register<TOnTabletRebuildGraph>(NSignalTypes::TypeOnTabletRebuildGraph); - Register<TOnWriteZeroEntry>(NSignalTypes::TypeOnWriteZeroEntry); + +namespace NKikimr { +namespace NTracing { + +template <typename TStreamImpl> +void PrintGenStepPair(TStreamImpl& str, ui64 genStepInt) { + auto genStepPair = ExpandGenStepPair(genStepInt); + str << genStepPair.first << ":" << genStepPair.second; +} + +template <typename TStreamImpl> +void PrintLogoBlob(TStreamImpl& str, const NKikimrProto::TLogoBlobID& protoLogoBlobId) { + TLogoBlobID logoBlobId = LogoBlobIDFromLogoBlobID(protoLogoBlobId); + str << logoBlobId.ToString(); +} + +TSignalFactory& GetSignalFactoryInstance() { + static TSignalFactory SingleFactory; + return SingleFactory; +} + +struct TSignalTypeRegistrator { + TSignalTypeRegistrator() { + Initialize(); + } + + void Initialize() { + Register<TOnTabletBootstrap>(NSignalTypes::TypeOnTabletBootstrap); + Register<TOnHandleStateStorageInfoResolve>(NSignalTypes::TypeOnHandleStateStorageInfoResolve); + Register<TOnCancelTablet>(NSignalTypes::TypeOnCancelTablet); + Register<TOnLockedInitializationPath>(NSignalTypes::TypeOnLockedInitializationPath); + Register<TOnHandleStateStorageInfoLock>(NSignalTypes::TypeOnHandleStateStorageInfoLock); + Register<TOnPromoteToCandidate>(NSignalTypes::TypeOnPromoteToCandidate); + Register<TOnTabletBlockBlobStorage>(NSignalTypes::TypeOnTabletBlockBlobStorage); + Register<TOnTabletRebuildGraph>(NSignalTypes::TypeOnTabletRebuildGraph); + Register<TOnWriteZeroEntry>(NSignalTypes::TypeOnWriteZeroEntry); Register<TOnFollowerPromoteToLeader>(NSignalTypes::TypeOnFollowerPromoteToLeader); - Register<TRebuildGraphBootstrap>(NSignalTypes::TypeRebuildGraphBootstrap); - Register<TErrorEntryBeyondBlocked>(NSignalTypes::TypeErrorEntryBeyondBlocked); - Register<TErrorUnknownStatus>(NSignalTypes::TypeErrorUnknownStatus); - Register<TErrorParsingFromString>(NSignalTypes::TypeErrorParsingFromString); - Register<TErrorSendRefsCheck>(NSignalTypes::TypeErrorSendRefsCheck); - Register<TErrorRebuildGraph>(NSignalTypes::TypeErrorRebuildGraph); - Register<TOnProcessKeyEntry>(NSignalTypes::TypeOnProcessKeyEntry); - Register<TOnProcessZeroEntry>(NSignalTypes::TypeOnProcessZeroEntry); - Register<TOnProcessLogEntry>(NSignalTypes::TypeOnProcessLogEntry); - Register<TOnDiscoverRangeRequest>(NSignalTypes::TypeOnDiscoverRangeRequest); - Register<TOnApplyDiscoveryRange>(NSignalTypes::TypeOnApplyDiscoveryRange); - Register<TOnMakeHistory>(NSignalTypes::TypeOnMakeHistory); - Register<TOnCheckRefsGetResult>(NSignalTypes::TypeOnCheckRefsGetResult); - Register<TOnBuildHistoryGraph>(NSignalTypes::TypeOnBuildHistoryGraph); - Register<TOnRebuildGraphResult>(NSignalTypes::TypeOnRebuildGraphResult); - } - -private: - template <typename TDerived> - void Register(ITraceSignal::EType signalType) { - TSignalFactory::Instance().Add<TDerived>(signalType); - } -}; - -// Singleton -static TSignalTypeRegistrator SignalTypeRegistrator; - + Register<TRebuildGraphBootstrap>(NSignalTypes::TypeRebuildGraphBootstrap); + Register<TErrorEntryBeyondBlocked>(NSignalTypes::TypeErrorEntryBeyondBlocked); + Register<TErrorUnknownStatus>(NSignalTypes::TypeErrorUnknownStatus); + Register<TErrorParsingFromString>(NSignalTypes::TypeErrorParsingFromString); + Register<TErrorSendRefsCheck>(NSignalTypes::TypeErrorSendRefsCheck); + Register<TErrorRebuildGraph>(NSignalTypes::TypeErrorRebuildGraph); + Register<TOnProcessKeyEntry>(NSignalTypes::TypeOnProcessKeyEntry); + Register<TOnProcessZeroEntry>(NSignalTypes::TypeOnProcessZeroEntry); + Register<TOnProcessLogEntry>(NSignalTypes::TypeOnProcessLogEntry); + Register<TOnDiscoverRangeRequest>(NSignalTypes::TypeOnDiscoverRangeRequest); + Register<TOnApplyDiscoveryRange>(NSignalTypes::TypeOnApplyDiscoveryRange); + Register<TOnMakeHistory>(NSignalTypes::TypeOnMakeHistory); + Register<TOnCheckRefsGetResult>(NSignalTypes::TypeOnCheckRefsGetResult); + Register<TOnBuildHistoryGraph>(NSignalTypes::TypeOnBuildHistoryGraph); + Register<TOnRebuildGraphResult>(NSignalTypes::TypeOnRebuildGraphResult); + } + +private: + template <typename TDerived> + void Register(ITraceSignal::EType signalType) { + TSignalFactory::Instance().Add<TDerived>(signalType); + } +}; + +// Singleton +static TSignalTypeRegistrator SignalTypeRegistrator; + TOnTabletBootstrap::TOnTabletBootstrap(ui32 suggestedGeneration, bool leader, const TActorId& proxyID) { - PbSignal.SetSuggestedGeneration(suggestedGeneration); + PbSignal.SetSuggestedGeneration(suggestedGeneration); PbSignal.SetLeader(leader); NActors::ActorIdToProto(proxyID, PbSignal.MutableStateStorageProxyID()); -} - -TOnTabletBootstrap::TOnTabletBootstrap(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnTabletBootstrap::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { +} + +TOnTabletBootstrap::TOnTabletBootstrap(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnTabletBootstrap::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { TActorId ID = NActors::ActorIdFromProto(PbSignal.GetStateStorageProxyID()); - str << prefix << TimeStamp(tsData) << " Bootstrapped. Sent TEvLookup to SS proxy " << ID.ToString() << Endl; -} - -TOnHandleStateStorageInfoResolve::TOnHandleStateStorageInfoResolve( - ui32 knownGeneration - , ui32 knownStep - , ui32 signatureSz -) { - PbSignal.SetKnownGeneration(knownGeneration); - PbSignal.SetKnownStep(knownStep); - PbSignal.SetSignatureSz(signatureSz); -} - -TOnHandleStateStorageInfoResolve::TOnHandleStateStorageInfoResolve(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnHandleStateStorageInfoResolve::OutText( - TStringStream& str - , TTimestampData& tsData - , const TString& prefix -) const { - str << prefix << TimeStamp(tsData) - << " Received TEvStateStorage::TEvInfo (KnownGeneration=" << PbSignal.GetKnownGeneration() - << ", KnownStep=" << PbSignal.GetKnownStep() - << ", SignatureSz=" << PbSignal.GetSignatureSz() - << ")" << Endl; -} - -TOnCancelTablet::TOnCancelTablet(ui64 tabletID - , TTabletTypes::EType tabletType - , ui32 tabletDeadReason - , ui32 suggestedGeneration - , ui32 knownGeneration -) { - PbSignal.SetTabletID(tabletID); - PbSignal.SetTabletType(tabletType); - PbSignal.SetTabletDeadReason(tabletDeadReason); - PbSignal.SetSuggestedGeneration(suggestedGeneration); - PbSignal.SetKnownGeneration(knownGeneration); -} - -TOnCancelTablet::TOnCancelTablet(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnCancelTablet::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " CancelTablet called (TabletID=" << PbSignal.GetTabletID() - << ", TabletType=" << TTabletTypes::TypeToStr((TTabletTypes::EType)PbSignal.GetTabletType()) - << ", DeadReason=" - << TEvTablet::TEvTabletDead::Str((TEvTablet::TEvTabletDead::EReason)PbSignal.GetTabletDeadReason()) - << ", SuggestedGeneration=" << PbSignal.GetSuggestedGeneration() - << ")" << Endl; -} - -TOnLockedInitializationPath::TOnLockedInitializationPath(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz) { - PbSignal.SetKnownGeneration(knownGeneration); - PbSignal.SetKnownStep(knownStep); - PbSignal.SetSignatureSz(signatureSz); -} - -TOnLockedInitializationPath::TOnLockedInitializationPath(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnLockedInitializationPath::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " LockedInitializationPath (KnownGeneration=" << PbSignal.GetKnownGeneration() - << ", KnownStep=" << PbSignal.GetKnownStep() - << ", SignatureSz=" << PbSignal.GetSignatureSz() - << ")" << Endl; -} - -TOnHandleStateStorageInfoLock::TOnHandleStateStorageInfoLock( - ui32 knownGeneration - , ui32 knownStep - , ui32 signatureSz -) { - PbSignal.SetKnownGeneration(knownGeneration); - PbSignal.SetKnownStep(knownStep); - PbSignal.SetSignatureSz(signatureSz); -} - -TOnHandleStateStorageInfoLock::TOnHandleStateStorageInfoLock(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnHandleStateStorageInfoLock::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) - << " Successfully locked the storage (KnownGeneration=" << PbSignal.GetKnownGeneration() - << ", KnownStep=" << PbSignal.GetKnownStep() - << ", SignatureSz=" << PbSignal.GetSignatureSz() - << ")" << Endl; -} - -TOnPromoteToCandidate::TOnPromoteToCandidate(ui32 knownGeneration) { - PbSignal.SetKnownGeneration(knownGeneration); -} - -TOnPromoteToCandidate::TOnPromoteToCandidate(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnPromoteToCandidate::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) - << " Promoted to candidate (KnownGeneration=" << PbSignal.GetKnownGeneration() << ")" << Endl; -} - + str << prefix << TimeStamp(tsData) << " Bootstrapped. Sent TEvLookup to SS proxy " << ID.ToString() << Endl; +} + +TOnHandleStateStorageInfoResolve::TOnHandleStateStorageInfoResolve( + ui32 knownGeneration + , ui32 knownStep + , ui32 signatureSz +) { + PbSignal.SetKnownGeneration(knownGeneration); + PbSignal.SetKnownStep(knownStep); + PbSignal.SetSignatureSz(signatureSz); +} + +TOnHandleStateStorageInfoResolve::TOnHandleStateStorageInfoResolve(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnHandleStateStorageInfoResolve::OutText( + TStringStream& str + , TTimestampData& tsData + , const TString& prefix +) const { + str << prefix << TimeStamp(tsData) + << " Received TEvStateStorage::TEvInfo (KnownGeneration=" << PbSignal.GetKnownGeneration() + << ", KnownStep=" << PbSignal.GetKnownStep() + << ", SignatureSz=" << PbSignal.GetSignatureSz() + << ")" << Endl; +} + +TOnCancelTablet::TOnCancelTablet(ui64 tabletID + , TTabletTypes::EType tabletType + , ui32 tabletDeadReason + , ui32 suggestedGeneration + , ui32 knownGeneration +) { + PbSignal.SetTabletID(tabletID); + PbSignal.SetTabletType(tabletType); + PbSignal.SetTabletDeadReason(tabletDeadReason); + PbSignal.SetSuggestedGeneration(suggestedGeneration); + PbSignal.SetKnownGeneration(knownGeneration); +} + +TOnCancelTablet::TOnCancelTablet(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnCancelTablet::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " CancelTablet called (TabletID=" << PbSignal.GetTabletID() + << ", TabletType=" << TTabletTypes::TypeToStr((TTabletTypes::EType)PbSignal.GetTabletType()) + << ", DeadReason=" + << TEvTablet::TEvTabletDead::Str((TEvTablet::TEvTabletDead::EReason)PbSignal.GetTabletDeadReason()) + << ", SuggestedGeneration=" << PbSignal.GetSuggestedGeneration() + << ")" << Endl; +} + +TOnLockedInitializationPath::TOnLockedInitializationPath(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz) { + PbSignal.SetKnownGeneration(knownGeneration); + PbSignal.SetKnownStep(knownStep); + PbSignal.SetSignatureSz(signatureSz); +} + +TOnLockedInitializationPath::TOnLockedInitializationPath(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnLockedInitializationPath::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " LockedInitializationPath (KnownGeneration=" << PbSignal.GetKnownGeneration() + << ", KnownStep=" << PbSignal.GetKnownStep() + << ", SignatureSz=" << PbSignal.GetSignatureSz() + << ")" << Endl; +} + +TOnHandleStateStorageInfoLock::TOnHandleStateStorageInfoLock( + ui32 knownGeneration + , ui32 knownStep + , ui32 signatureSz +) { + PbSignal.SetKnownGeneration(knownGeneration); + PbSignal.SetKnownStep(knownStep); + PbSignal.SetSignatureSz(signatureSz); +} + +TOnHandleStateStorageInfoLock::TOnHandleStateStorageInfoLock(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnHandleStateStorageInfoLock::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) + << " Successfully locked the storage (KnownGeneration=" << PbSignal.GetKnownGeneration() + << ", KnownStep=" << PbSignal.GetKnownStep() + << ", SignatureSz=" << PbSignal.GetSignatureSz() + << ")" << Endl; +} + +TOnPromoteToCandidate::TOnPromoteToCandidate(ui32 knownGeneration) { + PbSignal.SetKnownGeneration(knownGeneration); +} + +TOnPromoteToCandidate::TOnPromoteToCandidate(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnPromoteToCandidate::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) + << " Promoted to candidate (KnownGeneration=" << PbSignal.GetKnownGeneration() << ")" << Endl; +} + TOnTabletBlockBlobStorage::TOnTabletBlockBlobStorage(const TActorId& reqBlockBlobStorageID, ui32 knownGeneration) { NActors::ActorIdToProto(reqBlockBlobStorageID, PbSignal.MutableReqBlockBlobStorageID()); - PbSignal.SetKnownGeneration(knownGeneration); -} - -TOnTabletBlockBlobStorage::TOnTabletBlockBlobStorage(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnTabletBlockBlobStorage::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + PbSignal.SetKnownGeneration(knownGeneration); +} + +TOnTabletBlockBlobStorage::TOnTabletBlockBlobStorage(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnTabletBlockBlobStorage::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { TActorId ID = NActors::ActorIdFromProto(PbSignal.GetReqBlockBlobStorageID()); - str << prefix << TimeStamp(tsData) << " Created ReqBlockBlobStorage Actor (" << ID.ToString() - << ") with KnownGeneration=" << PbSignal.GetKnownGeneration() << Endl; -} - -TOnTabletRebuildGraph::TOnTabletRebuildGraph( + str << prefix << TimeStamp(tsData) << " Created ReqBlockBlobStorage Actor (" << ID.ToString() + << ") with KnownGeneration=" << PbSignal.GetKnownGeneration() << Endl; +} + +TOnTabletRebuildGraph::TOnTabletRebuildGraph( const TActorId& tabletReqRebuildGraphID - , const TTraceID& rebuildGraphTraceID -) { + , const TTraceID& rebuildGraphTraceID +) { NActors::ActorIdToProto(tabletReqRebuildGraphID, PbSignal.MutableTabletReqRebuildGraphID()); - TraceIDFromTraceID(rebuildGraphTraceID, PbSignal.MutableRebuildGraphTraceID()); -} - -TOnTabletRebuildGraph::TOnTabletRebuildGraph(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnTabletRebuildGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + TraceIDFromTraceID(rebuildGraphTraceID, PbSignal.MutableRebuildGraphTraceID()); +} + +TOnTabletRebuildGraph::TOnTabletRebuildGraph(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnTabletRebuildGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { TActorId ID = NActors::ActorIdFromProto(PbSignal.GetTabletReqRebuildGraphID()); - str << prefix << TimeStamp(tsData) << " Created ReqRebuildHistoryGraph Actor (" << ID.ToString() << ")" << Endl; -} - -TOnWriteZeroEntry::TOnWriteZeroEntry( - const std::pair<ui32, ui32>& snapshot - , ui32 lastGeneration - , ui32 confirmedStep - , ui32 lastInGeneration -) { - PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); - PbSignal.SetZeroConfirmed(MakeGenStepPair(lastGeneration, confirmedStep)); - PbSignal.SetLastInGeneration(lastInGeneration); -} - -TOnWriteZeroEntry::TOnWriteZeroEntry(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnWriteZeroEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " WriteZeroEntry (Snapshot="; - PrintGenStepPair(str, PbSignal.GetSnapshot()); - str << ", ZeroConfirmed="; - PrintGenStepPair(str, PbSignal.GetZeroConfirmed()); - str << ", LastInGeneration=" << PbSignal.GetLastInGeneration() << ")" << Endl; -} - + str << prefix << TimeStamp(tsData) << " Created ReqRebuildHistoryGraph Actor (" << ID.ToString() << ")" << Endl; +} + +TOnWriteZeroEntry::TOnWriteZeroEntry( + const std::pair<ui32, ui32>& snapshot + , ui32 lastGeneration + , ui32 confirmedStep + , ui32 lastInGeneration +) { + PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); + PbSignal.SetZeroConfirmed(MakeGenStepPair(lastGeneration, confirmedStep)); + PbSignal.SetLastInGeneration(lastInGeneration); +} + +TOnWriteZeroEntry::TOnWriteZeroEntry(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnWriteZeroEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " WriteZeroEntry (Snapshot="; + PrintGenStepPair(str, PbSignal.GetSnapshot()); + str << ", ZeroConfirmed="; + PrintGenStepPair(str, PbSignal.GetZeroConfirmed()); + str << ", LastInGeneration=" << PbSignal.GetLastInGeneration() << ")" << Endl; +} + TOnFollowerPromoteToLeader::TOnFollowerPromoteToLeader( - ui32 suggestedGeneration + ui32 suggestedGeneration , const TActorId& knownLeaderID , const TActorId& followerStStGuardian -) { - PbSignal.SetSuggestedGeneration(suggestedGeneration); +) { + PbSignal.SetSuggestedGeneration(suggestedGeneration); NActors::ActorIdToProto(knownLeaderID, PbSignal.MutableKnownLeaderID()); NActors::ActorIdToProto(followerStStGuardian, PbSignal.MutableFollowerStStGuardian()); -} - +} + TOnFollowerPromoteToLeader::TOnFollowerPromoteToLeader(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - + : TMyBase(serializedSignal) +{} + void TOnFollowerPromoteToLeader::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { TActorId knownLeaderId = NActors::ActorIdFromProto(PbSignal.GetKnownLeaderID()); TActorId followerStStGuardianId = NActors::ActorIdFromProto(PbSignal.GetFollowerStStGuardian()); - str << prefix << TimeStamp(tsData) + str << prefix << TimeStamp(tsData) << " Follower promoted to leader (SuggestedGeneration=" << PbSignal.GetSuggestedGeneration() << ", KnownLeaderID=" << knownLeaderId.ToString() << ", FollowerStStGuardian=" << followerStStGuardianId.ToString() << ")" << Endl; -} - -TRebuildGraphBootstrap::TRebuildGraphBootstrap(ui32 blockedGen) { - PbSignal.SetBlockedGen(blockedGen); -} - -TRebuildGraphBootstrap::TRebuildGraphBootstrap(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TRebuildGraphBootstrap::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) - << " ReqRebuildHistoryGraph actor bootstrapped. BlockedGen=" << PbSignal.GetBlockedGen() << Endl; -} - -TErrorEntryBeyondBlocked::TErrorEntryBeyondBlocked(const TLogoBlobID& latest, ui32 blockedGen) { - LogoBlobIDFromLogoBlobID(latest, PbSignal.MutableLatest()); - PbSignal.SetBlockedGen(blockedGen); -} - -TErrorEntryBeyondBlocked::TErrorEntryBeyondBlocked(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TErrorEntryBeyondBlocked::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " Found entry beyond blocked generation. LastBlobID="; - PrintLogoBlob(str, PbSignal.GetLatest()); - str << ", BlockedGen=" << PbSignal.GetBlockedGen() << Endl; -} - -TErrorUnknownStatus::TErrorUnknownStatus(NKikimrProto::EReplyStatus status, const TString& reason) { - PbSignal.SetStatus(status); - PbSignal.SetReason(reason); -} - -TErrorUnknownStatus::TErrorUnknownStatus(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TErrorUnknownStatus::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) - << " Handle TEvFindLatestLogEntryResult: Status=" << NKikimrProto::EReplyStatus_Name(PbSignal.GetStatus()) - << ", Reason=\"" << PbSignal.GetReason() << "\"" << Endl; -} - -TErrorParsingFromString::TErrorParsingFromString(const TLogoBlobID& blobID) { - LogoBlobIDFromLogoBlobID(blobID, PbSignal.MutableBlobID()); -} - -TErrorParsingFromString::TErrorParsingFromString(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TErrorParsingFromString::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " logBody ParseFromString error for BlobID="; - PrintLogoBlob(str, PbSignal.GetBlobID()); - str << Endl; -} - -TErrorSendRefsCheck::TErrorSendRefsCheck() { -} - -TErrorSendRefsCheck::TErrorSendRefsCheck(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TErrorSendRefsCheck::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " SendRefsCheck error" << Endl; -} - -TErrorRebuildGraph::TErrorRebuildGraph(ui32 generation, ui32 step) { - PbSignal.SetGeneration(generation); - PbSignal.SetStep(step); -} - -TErrorRebuildGraph::TErrorRebuildGraph(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TErrorRebuildGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " Graph rebuild error - no Log entry for " << PbSignal.GetGeneration() - << ":" << PbSignal.GetStep() << Endl; -} - -TOnProcessKeyEntry::TOnProcessKeyEntry(const TLogoBlobID& latestEntry) { - LogoBlobIDFromLogoBlobID(latestEntry, PbSignal.MutableLatestEntry()); -} - -TOnProcessKeyEntry::TOnProcessKeyEntry(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnProcessKeyEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " Last entry found: "; - PrintLogoBlob(str, PbSignal.GetLatestEntry()); - str << Endl; -} - -TOnProcessZeroEntry::TOnProcessZeroEntry( - ui32 generation - , const std::pair<ui32, ui32>& snapshot - , const std::pair<ui32, ui32>& confirmed -) { - PbSignal.SetGeneration(generation); - PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); - PbSignal.SetConfirmed(MakeGenStepPair(confirmed.first, confirmed.second)); -} - -TOnProcessZeroEntry::TOnProcessZeroEntry(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnProcessZeroEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) - << " ProcessZeroEntry (Generation=" << PbSignal.GetGeneration() << ", Snapshot="; - PrintGenStepPair(str, PbSignal.GetSnapshot()); - str << ", Confirmed="; - PrintGenStepPair(str, PbSignal.GetConfirmed()); - str << ")" << Endl; -} - -TOnProcessLogEntry::TOnProcessLogEntry( - const TLogoBlobID& blobID - , const std::pair<ui32, ui32>& snapshot - , const std::pair<ui32, ui32>& confirmed - , NKikimrTabletBase::TTabletLogEntry& logEntry -) { - LogoBlobIDFromLogoBlobID(blobID, PbSignal.MutableBlobID()); - PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); - PbSignal.SetConfirmed(MakeGenStepPair(confirmed.first, confirmed.second)); - *PbSignal.MutableReferences() = logEntry.GetReferences(); - *PbSignal.MutableGcDiscovered() = logEntry.GetGcDiscovered(); - *PbSignal.MutableGcLeft() = logEntry.GetGcLeft(); -} - -TOnProcessLogEntry::TOnProcessLogEntry(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnProcessLogEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " ProcessLogEntry (ID="; - PrintLogoBlob(str, PbSignal.GetBlobID()); - str << ", Snapshot="; - PrintGenStepPair(str, PbSignal.GetSnapshot()); - str << ", Confirmed="; - PrintGenStepPair(str, PbSignal.GetConfirmed()); - str << ")" << Endl; - - size_t refSize = PbSignal.ReferencesSize(); - str << prefix << " " << refSize << " References"; - if (refSize) { - str << ":" << Endl; - for (size_t i = 0; i < refSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, PbSignal.references(i)); - str << Endl; - } - } else { - str << Endl; - } - - size_t gcDiscoveredSize = PbSignal.GcDiscoveredSize(); - str << prefix << " " << gcDiscoveredSize << " GcDiscovered"; - if (gcDiscoveredSize) { - str << ":" << Endl; - for (size_t i = 0; i < gcDiscoveredSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, PbSignal.gcdiscovered(i)); - str << Endl; - } - } else { - str << Endl; - } - - size_t gcLeftSize = PbSignal.GcLeftSize(); - str << prefix << " " << gcLeftSize << " GcLeft"; - if (gcLeftSize) { - str << ":" << Endl; - for (size_t i = 0; i < gcLeftSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, PbSignal.gcleft(i)); - str << Endl; - } - } else { - str << Endl; - } -} - -void TOnProcessLogEntry::OutHtmlHeader( - TStringStream& str - , TTimestampData& tsData - , std::function<TString()> getMyId -) const { - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << TimeStamp(tsData) << " "; - TString myId = getMyId(); - TStringBuilder textHeader; - textHeader << "ProcessLogEntry (ID="; - PrintLogoBlob(textHeader, PbSignal.GetBlobID()); - textHeader << ", Snapshot="; - PrintGenStepPair(textHeader, PbSignal.GetSnapshot()); - textHeader << ", Confirmed="; - PrintGenStepPair(textHeader, PbSignal.GetConfirmed()); - textHeader << ")"; - COLLAPSED_REF_CONTENT(myId, textHeader) { - DIV_CLASS("tab-left") { - size_t refSize = PbSignal.ReferencesSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << refSize << " References:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < refSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, PbSignal.references(i)); - } - } - } - } - - size_t gcDiscoveredSize = PbSignal.GcDiscoveredSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << gcDiscoveredSize << " GcDiscovered:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < gcDiscoveredSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, PbSignal.gcdiscovered(i)); - } - } - } - } - - size_t gcLeftSize = PbSignal.GcLeftSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << gcLeftSize << " GcLeft:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < gcLeftSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, PbSignal.gcleft(i)); - } - } - } - } - } - } - } - } - } -} - -TOnDiscoverRangeRequest::TOnDiscoverRangeRequest(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to) { - PbSignal.SetGroup(group); - LogoBlobIDFromLogoBlobID(from, PbSignal.MutableFrom()); - LogoBlobIDFromLogoBlobID(to, PbSignal.MutableTo()); -} - -TOnDiscoverRangeRequest::TOnDiscoverRangeRequest(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnDiscoverRangeRequest::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " DiscoverRangeRequest sent (Group=" << PbSignal.GetGroup() << ", From="; - PrintLogoBlob(str, PbSignal.GetFrom()); - str << ", To="; - PrintLogoBlob(str, PbSignal.GetTo()); - str << ")" << Endl; -} - -TOnApplyDiscoveryRange::TOnApplyDiscoveryRange(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to) { - PbSignal.SetGroup(group); - LogoBlobIDFromLogoBlobID(from, PbSignal.MutableFrom()); - LogoBlobIDFromLogoBlobID(to, PbSignal.MutableTo()); -} - -TOnApplyDiscoveryRange::TOnApplyDiscoveryRange(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnApplyDiscoveryRange::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " ApplyDiscoveryRange (Group=" << PbSignal.GetGroup() << ", From="; - PrintLogoBlob(str, PbSignal.GetFrom()); - str << ", To="; - PrintLogoBlob(str, PbSignal.GetTo()); - str << ")" << Endl; -} - -TOnMakeHistory::TOnMakeHistory(TSet<TLogoBlobID>& refsToCheck) { - LogoBlobIDRepatedFromLogoBlobIDUniversal(PbSignal.MutableRefsToCheck(), refsToCheck); -} - -TOnMakeHistory::TOnMakeHistory(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnMakeHistory::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " OnMakeHistory(" << PbSignal.RefsToCheckSize() << " entries)" << Endl; - for (size_t i = 0; i < PbSignal.RefsToCheckSize(); ++i) { - str << prefix << " "; - PrintLogoBlob(str, PbSignal.refstocheck(i)); - str << Endl; - } -} - -void TOnMakeHistory::OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const { - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << TimeStamp(tsData) << " "; - TString myId = getMyId(); - TString textHeader = " OnMakeHistory(" + ToString(PbSignal.RefsToCheckSize()) + " entries)"; - COLLAPSED_REF_CONTENT(myId, textHeader) { - DIV_CLASS("tab-left") { - for (size_t i = 0; i < PbSignal.RefsToCheckSize(); ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, PbSignal.refstocheck(i)); - } - } - } - } - } - } - } - } -} - -TOnCheckRefsGetResult::TOnCheckRefsGetResult(ui32 responseSz) { - PbSignal.SetResponseSz(responseSz); -} - -TOnCheckRefsGetResult::TOnCheckRefsGetResult(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnCheckRefsGetResult::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " CheckReferences(" << PbSignal.GetResponseSz() << " refs)" << Endl; -} - -TOnBuildHistoryGraph::TOnBuildHistoryGraph(TEvTablet::TDependencyGraph* graph) { - if (!graph) { - return; - } - for (const auto& graphEntry : graph->Entries) { - auto& pbEntry = *PbSignal.AddEntries(); - pbEntry.SetGeneration(graphEntry.Id.first); - pbEntry.SetStep(graphEntry.Id.second); - pbEntry.SetIsSnapshot(graphEntry.IsSnapshot); - LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableReferences() - , graphEntry.References.begin() - , graphEntry.References.end()); - LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableGcDiscovered() - , graphEntry.GcDiscovered.begin() - , graphEntry.GcDiscovered.end()); - LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableGcLeft() - , graphEntry.GcLeft.begin() - , graphEntry.GcLeft.end()); - } -} - -TOnBuildHistoryGraph::TOnBuildHistoryGraph(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void PrintBuildHistoryGraphEntry(TStringStream& str - , const NKikimrTracing::TOnBuildHistoryGraph::TEntry& entry - , const TString& prefix) { - size_t refSize = entry.ReferencesSize(); - str << prefix << "{" << Endl - << prefix << " " << entry.GetGeneration() << ":" << entry.GetStep() - << ", IsSnapshot=" << entry.GetIsSnapshot() << Endl - << prefix << " " << refSize << " References"; - if (refSize) { - str << ":" << Endl; - for (size_t i = 0; i < refSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, entry.references(i)); - str << Endl; - } - } else { - str << Endl; - } - size_t gcDiscoveredSize = entry.GcDiscoveredSize(); - str << prefix << " " << gcDiscoveredSize << " GcDiscovered"; - if (gcDiscoveredSize) { - str << ":" << Endl; - for (size_t i = 0; i < gcDiscoveredSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, entry.gcdiscovered(i)); - str << Endl; - } - } else { - str << Endl; - } - size_t gcLeftSize = entry.GcLeftSize(); - str << prefix << " " << gcLeftSize << " GcLeft"; - if (gcLeftSize) { - str << ":" << Endl; - for (size_t i = 0; i < gcLeftSize; ++i) { - str << prefix << " "; - PrintLogoBlob(str, entry.gcleft(i)); - str << Endl; - } - } else { - str << Endl; - } - str << prefix << "}" << Endl; -} - -void PrintBuildHistoryGraphEntryHtml(TStringStream& str - , const NKikimrTracing::TOnBuildHistoryGraph::TEntry& entry - , std::function<TString()> getMyId -) { - HTML(str) { - TString myId = getMyId(); - TStringStream textHeader; - textHeader << entry.GetGeneration() << ":" << entry.GetStep() - << " (IsSnapshot = " << entry.GetIsSnapshot() << ")"; - COLLAPSED_REF_CONTENT(myId, textHeader.Str()) { - DIV_CLASS("tab-left") { - size_t refSize = entry.ReferencesSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << refSize << " References:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < refSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, entry.references(i)); - } - } - } - } - size_t gcDiscoveredSize = entry.GcDiscoveredSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << gcDiscoveredSize << " GcDiscovered:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < gcDiscoveredSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, entry.gcdiscovered(i)); - } - } - } - } - size_t gcLeftSize = entry.GcLeftSize(); - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << gcLeftSize << " GcLeft:"; - } - } - DIV_CLASS("tab-left") { - for (size_t i = 0; i < gcLeftSize; ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintLogoBlob(str, entry.gcleft(i)); - } - } - } - } - } - } - } -} - -void TOnBuildHistoryGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " History graph built(" << PbSignal.EntriesSize() << " entries)" << Endl; - for (size_t i = 0; i < PbSignal.EntriesSize(); ++i) { - PrintBuildHistoryGraphEntry(str, PbSignal.entries(i), prefix); - } -} - -void TOnBuildHistoryGraph::OutHtmlHeader( - TStringStream& str - , TTimestampData& tsData - , std::function<TString()> getMyId -) const { - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << TimeStamp(tsData) << " "; - TString myId = getMyId(); - TString textHeader = " History graph built(" + ToString(PbSignal.EntriesSize()) + " entries)"; - COLLAPSED_REF_CONTENT(myId, textHeader) { - DIV_CLASS("tab-left") { - for (size_t i = 0; i < PbSignal.EntriesSize(); ++i) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - PrintBuildHistoryGraphEntryHtml( - str, - PbSignal.entries(i), - [i, getMyId](){ return getMyId() + ToString(i); } - ); - } - } - } - } - } - } - } - } -} - -TOnRebuildGraphResult::TOnRebuildGraphResult(ITrace* trace) { - TString str; - if (trace->SerializeToString(str)) { - PbSignal.SetSerializedTrace(str); - } -} - -TOnRebuildGraphResult::TOnRebuildGraphResult(const TString& serializedSignal) - : TMyBase(serializedSignal) -{} - -void TOnRebuildGraphResult::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { - str << prefix << TimeStamp(tsData) << " RebuildGraph result received. Here's the trace:" << Endl; - str << prefix << "{" << Endl; +} + +TRebuildGraphBootstrap::TRebuildGraphBootstrap(ui32 blockedGen) { + PbSignal.SetBlockedGen(blockedGen); +} + +TRebuildGraphBootstrap::TRebuildGraphBootstrap(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TRebuildGraphBootstrap::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) + << " ReqRebuildHistoryGraph actor bootstrapped. BlockedGen=" << PbSignal.GetBlockedGen() << Endl; +} + +TErrorEntryBeyondBlocked::TErrorEntryBeyondBlocked(const TLogoBlobID& latest, ui32 blockedGen) { + LogoBlobIDFromLogoBlobID(latest, PbSignal.MutableLatest()); + PbSignal.SetBlockedGen(blockedGen); +} + +TErrorEntryBeyondBlocked::TErrorEntryBeyondBlocked(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TErrorEntryBeyondBlocked::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " Found entry beyond blocked generation. LastBlobID="; + PrintLogoBlob(str, PbSignal.GetLatest()); + str << ", BlockedGen=" << PbSignal.GetBlockedGen() << Endl; +} + +TErrorUnknownStatus::TErrorUnknownStatus(NKikimrProto::EReplyStatus status, const TString& reason) { + PbSignal.SetStatus(status); + PbSignal.SetReason(reason); +} + +TErrorUnknownStatus::TErrorUnknownStatus(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TErrorUnknownStatus::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) + << " Handle TEvFindLatestLogEntryResult: Status=" << NKikimrProto::EReplyStatus_Name(PbSignal.GetStatus()) + << ", Reason=\"" << PbSignal.GetReason() << "\"" << Endl; +} + +TErrorParsingFromString::TErrorParsingFromString(const TLogoBlobID& blobID) { + LogoBlobIDFromLogoBlobID(blobID, PbSignal.MutableBlobID()); +} + +TErrorParsingFromString::TErrorParsingFromString(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TErrorParsingFromString::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " logBody ParseFromString error for BlobID="; + PrintLogoBlob(str, PbSignal.GetBlobID()); + str << Endl; +} + +TErrorSendRefsCheck::TErrorSendRefsCheck() { +} + +TErrorSendRefsCheck::TErrorSendRefsCheck(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TErrorSendRefsCheck::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " SendRefsCheck error" << Endl; +} + +TErrorRebuildGraph::TErrorRebuildGraph(ui32 generation, ui32 step) { + PbSignal.SetGeneration(generation); + PbSignal.SetStep(step); +} + +TErrorRebuildGraph::TErrorRebuildGraph(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TErrorRebuildGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " Graph rebuild error - no Log entry for " << PbSignal.GetGeneration() + << ":" << PbSignal.GetStep() << Endl; +} + +TOnProcessKeyEntry::TOnProcessKeyEntry(const TLogoBlobID& latestEntry) { + LogoBlobIDFromLogoBlobID(latestEntry, PbSignal.MutableLatestEntry()); +} + +TOnProcessKeyEntry::TOnProcessKeyEntry(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnProcessKeyEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " Last entry found: "; + PrintLogoBlob(str, PbSignal.GetLatestEntry()); + str << Endl; +} + +TOnProcessZeroEntry::TOnProcessZeroEntry( + ui32 generation + , const std::pair<ui32, ui32>& snapshot + , const std::pair<ui32, ui32>& confirmed +) { + PbSignal.SetGeneration(generation); + PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); + PbSignal.SetConfirmed(MakeGenStepPair(confirmed.first, confirmed.second)); +} + +TOnProcessZeroEntry::TOnProcessZeroEntry(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnProcessZeroEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) + << " ProcessZeroEntry (Generation=" << PbSignal.GetGeneration() << ", Snapshot="; + PrintGenStepPair(str, PbSignal.GetSnapshot()); + str << ", Confirmed="; + PrintGenStepPair(str, PbSignal.GetConfirmed()); + str << ")" << Endl; +} + +TOnProcessLogEntry::TOnProcessLogEntry( + const TLogoBlobID& blobID + , const std::pair<ui32, ui32>& snapshot + , const std::pair<ui32, ui32>& confirmed + , NKikimrTabletBase::TTabletLogEntry& logEntry +) { + LogoBlobIDFromLogoBlobID(blobID, PbSignal.MutableBlobID()); + PbSignal.SetSnapshot(MakeGenStepPair(snapshot.first, snapshot.second)); + PbSignal.SetConfirmed(MakeGenStepPair(confirmed.first, confirmed.second)); + *PbSignal.MutableReferences() = logEntry.GetReferences(); + *PbSignal.MutableGcDiscovered() = logEntry.GetGcDiscovered(); + *PbSignal.MutableGcLeft() = logEntry.GetGcLeft(); +} + +TOnProcessLogEntry::TOnProcessLogEntry(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnProcessLogEntry::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " ProcessLogEntry (ID="; + PrintLogoBlob(str, PbSignal.GetBlobID()); + str << ", Snapshot="; + PrintGenStepPair(str, PbSignal.GetSnapshot()); + str << ", Confirmed="; + PrintGenStepPair(str, PbSignal.GetConfirmed()); + str << ")" << Endl; + + size_t refSize = PbSignal.ReferencesSize(); + str << prefix << " " << refSize << " References"; + if (refSize) { + str << ":" << Endl; + for (size_t i = 0; i < refSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, PbSignal.references(i)); + str << Endl; + } + } else { + str << Endl; + } + + size_t gcDiscoveredSize = PbSignal.GcDiscoveredSize(); + str << prefix << " " << gcDiscoveredSize << " GcDiscovered"; + if (gcDiscoveredSize) { + str << ":" << Endl; + for (size_t i = 0; i < gcDiscoveredSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, PbSignal.gcdiscovered(i)); + str << Endl; + } + } else { + str << Endl; + } + + size_t gcLeftSize = PbSignal.GcLeftSize(); + str << prefix << " " << gcLeftSize << " GcLeft"; + if (gcLeftSize) { + str << ":" << Endl; + for (size_t i = 0; i < gcLeftSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, PbSignal.gcleft(i)); + str << Endl; + } + } else { + str << Endl; + } +} + +void TOnProcessLogEntry::OutHtmlHeader( + TStringStream& str + , TTimestampData& tsData + , std::function<TString()> getMyId +) const { + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << TimeStamp(tsData) << " "; + TString myId = getMyId(); + TStringBuilder textHeader; + textHeader << "ProcessLogEntry (ID="; + PrintLogoBlob(textHeader, PbSignal.GetBlobID()); + textHeader << ", Snapshot="; + PrintGenStepPair(textHeader, PbSignal.GetSnapshot()); + textHeader << ", Confirmed="; + PrintGenStepPair(textHeader, PbSignal.GetConfirmed()); + textHeader << ")"; + COLLAPSED_REF_CONTENT(myId, textHeader) { + DIV_CLASS("tab-left") { + size_t refSize = PbSignal.ReferencesSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << refSize << " References:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < refSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, PbSignal.references(i)); + } + } + } + } + + size_t gcDiscoveredSize = PbSignal.GcDiscoveredSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << gcDiscoveredSize << " GcDiscovered:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < gcDiscoveredSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, PbSignal.gcdiscovered(i)); + } + } + } + } + + size_t gcLeftSize = PbSignal.GcLeftSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << gcLeftSize << " GcLeft:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < gcLeftSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, PbSignal.gcleft(i)); + } + } + } + } + } + } + } + } + } +} + +TOnDiscoverRangeRequest::TOnDiscoverRangeRequest(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to) { + PbSignal.SetGroup(group); + LogoBlobIDFromLogoBlobID(from, PbSignal.MutableFrom()); + LogoBlobIDFromLogoBlobID(to, PbSignal.MutableTo()); +} + +TOnDiscoverRangeRequest::TOnDiscoverRangeRequest(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnDiscoverRangeRequest::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " DiscoverRangeRequest sent (Group=" << PbSignal.GetGroup() << ", From="; + PrintLogoBlob(str, PbSignal.GetFrom()); + str << ", To="; + PrintLogoBlob(str, PbSignal.GetTo()); + str << ")" << Endl; +} + +TOnApplyDiscoveryRange::TOnApplyDiscoveryRange(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to) { + PbSignal.SetGroup(group); + LogoBlobIDFromLogoBlobID(from, PbSignal.MutableFrom()); + LogoBlobIDFromLogoBlobID(to, PbSignal.MutableTo()); +} + +TOnApplyDiscoveryRange::TOnApplyDiscoveryRange(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnApplyDiscoveryRange::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " ApplyDiscoveryRange (Group=" << PbSignal.GetGroup() << ", From="; + PrintLogoBlob(str, PbSignal.GetFrom()); + str << ", To="; + PrintLogoBlob(str, PbSignal.GetTo()); + str << ")" << Endl; +} + +TOnMakeHistory::TOnMakeHistory(TSet<TLogoBlobID>& refsToCheck) { + LogoBlobIDRepatedFromLogoBlobIDUniversal(PbSignal.MutableRefsToCheck(), refsToCheck); +} + +TOnMakeHistory::TOnMakeHistory(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnMakeHistory::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " OnMakeHistory(" << PbSignal.RefsToCheckSize() << " entries)" << Endl; + for (size_t i = 0; i < PbSignal.RefsToCheckSize(); ++i) { + str << prefix << " "; + PrintLogoBlob(str, PbSignal.refstocheck(i)); + str << Endl; + } +} + +void TOnMakeHistory::OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const { + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << TimeStamp(tsData) << " "; + TString myId = getMyId(); + TString textHeader = " OnMakeHistory(" + ToString(PbSignal.RefsToCheckSize()) + " entries)"; + COLLAPSED_REF_CONTENT(myId, textHeader) { + DIV_CLASS("tab-left") { + for (size_t i = 0; i < PbSignal.RefsToCheckSize(); ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, PbSignal.refstocheck(i)); + } + } + } + } + } + } + } + } +} + +TOnCheckRefsGetResult::TOnCheckRefsGetResult(ui32 responseSz) { + PbSignal.SetResponseSz(responseSz); +} + +TOnCheckRefsGetResult::TOnCheckRefsGetResult(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnCheckRefsGetResult::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " CheckReferences(" << PbSignal.GetResponseSz() << " refs)" << Endl; +} + +TOnBuildHistoryGraph::TOnBuildHistoryGraph(TEvTablet::TDependencyGraph* graph) { + if (!graph) { + return; + } + for (const auto& graphEntry : graph->Entries) { + auto& pbEntry = *PbSignal.AddEntries(); + pbEntry.SetGeneration(graphEntry.Id.first); + pbEntry.SetStep(graphEntry.Id.second); + pbEntry.SetIsSnapshot(graphEntry.IsSnapshot); + LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableReferences() + , graphEntry.References.begin() + , graphEntry.References.end()); + LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableGcDiscovered() + , graphEntry.GcDiscovered.begin() + , graphEntry.GcDiscovered.end()); + LogoBlobIDRepatedFromLogoBlobIDVector(pbEntry.MutableGcLeft() + , graphEntry.GcLeft.begin() + , graphEntry.GcLeft.end()); + } +} + +TOnBuildHistoryGraph::TOnBuildHistoryGraph(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void PrintBuildHistoryGraphEntry(TStringStream& str + , const NKikimrTracing::TOnBuildHistoryGraph::TEntry& entry + , const TString& prefix) { + size_t refSize = entry.ReferencesSize(); + str << prefix << "{" << Endl + << prefix << " " << entry.GetGeneration() << ":" << entry.GetStep() + << ", IsSnapshot=" << entry.GetIsSnapshot() << Endl + << prefix << " " << refSize << " References"; + if (refSize) { + str << ":" << Endl; + for (size_t i = 0; i < refSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, entry.references(i)); + str << Endl; + } + } else { + str << Endl; + } + size_t gcDiscoveredSize = entry.GcDiscoveredSize(); + str << prefix << " " << gcDiscoveredSize << " GcDiscovered"; + if (gcDiscoveredSize) { + str << ":" << Endl; + for (size_t i = 0; i < gcDiscoveredSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, entry.gcdiscovered(i)); + str << Endl; + } + } else { + str << Endl; + } + size_t gcLeftSize = entry.GcLeftSize(); + str << prefix << " " << gcLeftSize << " GcLeft"; + if (gcLeftSize) { + str << ":" << Endl; + for (size_t i = 0; i < gcLeftSize; ++i) { + str << prefix << " "; + PrintLogoBlob(str, entry.gcleft(i)); + str << Endl; + } + } else { + str << Endl; + } + str << prefix << "}" << Endl; +} + +void PrintBuildHistoryGraphEntryHtml(TStringStream& str + , const NKikimrTracing::TOnBuildHistoryGraph::TEntry& entry + , std::function<TString()> getMyId +) { + HTML(str) { + TString myId = getMyId(); + TStringStream textHeader; + textHeader << entry.GetGeneration() << ":" << entry.GetStep() + << " (IsSnapshot = " << entry.GetIsSnapshot() << ")"; + COLLAPSED_REF_CONTENT(myId, textHeader.Str()) { + DIV_CLASS("tab-left") { + size_t refSize = entry.ReferencesSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << refSize << " References:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < refSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, entry.references(i)); + } + } + } + } + size_t gcDiscoveredSize = entry.GcDiscoveredSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << gcDiscoveredSize << " GcDiscovered:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < gcDiscoveredSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, entry.gcdiscovered(i)); + } + } + } + } + size_t gcLeftSize = entry.GcLeftSize(); + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << gcLeftSize << " GcLeft:"; + } + } + DIV_CLASS("tab-left") { + for (size_t i = 0; i < gcLeftSize; ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintLogoBlob(str, entry.gcleft(i)); + } + } + } + } + } + } + } +} + +void TOnBuildHistoryGraph::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " History graph built(" << PbSignal.EntriesSize() << " entries)" << Endl; + for (size_t i = 0; i < PbSignal.EntriesSize(); ++i) { + PrintBuildHistoryGraphEntry(str, PbSignal.entries(i), prefix); + } +} + +void TOnBuildHistoryGraph::OutHtmlHeader( + TStringStream& str + , TTimestampData& tsData + , std::function<TString()> getMyId +) const { + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << TimeStamp(tsData) << " "; + TString myId = getMyId(); + TString textHeader = " History graph built(" + ToString(PbSignal.EntriesSize()) + " entries)"; + COLLAPSED_REF_CONTENT(myId, textHeader) { + DIV_CLASS("tab-left") { + for (size_t i = 0; i < PbSignal.EntriesSize(); ++i) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + PrintBuildHistoryGraphEntryHtml( + str, + PbSignal.entries(i), + [i, getMyId](){ return getMyId() + ToString(i); } + ); + } + } + } + } + } + } + } + } +} + +TOnRebuildGraphResult::TOnRebuildGraphResult(ITrace* trace) { + TString str; + if (trace->SerializeToString(str)) { + PbSignal.SetSerializedTrace(str); + } +} + +TOnRebuildGraphResult::TOnRebuildGraphResult(const TString& serializedSignal) + : TMyBase(serializedSignal) +{} + +void TOnRebuildGraphResult::OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const { + str << prefix << TimeStamp(tsData) << " RebuildGraph result received. Here's the trace:" << Endl; + str << prefix << "{" << Endl; THolder<ITrace> trace(CreateTrace(PbSignal.GetSerializedTrace())); - trace->OutText(str, tsData, " "); - str << prefix << "}" << Endl; -} - -void TOnRebuildGraphResult::OutHtmlHeader( - TStringStream& str - , TTimestampData& tsData - , std::function<TString()> getMyId -) const { - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << TimeStamp(tsData) << " "; - TString myId = getMyId(); - COLLAPSED_REF_CONTENT_AJAX(myId, tsData, "RebuildGraph result received") { - DIV_CLASS("tab-left") { - str << "Loading data..."; - } - } - } - } - } -} - -void TOnRebuildGraphResult::OutHtmlBody( - TStringStream& str, - const TTimestampInfo& tsInfo, - std::function<TString()> getMyId, - TList<ui64>& signalAddress -) const { - Y_UNUSED(getMyId); - Y_UNUSED(signalAddress); + trace->OutText(str, tsData, " "); + str << prefix << "}" << Endl; +} + +void TOnRebuildGraphResult::OutHtmlHeader( + TStringStream& str + , TTimestampData& tsData + , std::function<TString()> getMyId +) const { + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << TimeStamp(tsData) << " "; + TString myId = getMyId(); + COLLAPSED_REF_CONTENT_AJAX(myId, tsData, "RebuildGraph result received") { + DIV_CLASS("tab-left") { + str << "Loading data..."; + } + } + } + } + } +} + +void TOnRebuildGraphResult::OutHtmlBody( + TStringStream& str, + const TTimestampInfo& tsInfo, + std::function<TString()> getMyId, + TList<ui64>& signalAddress +) const { + Y_UNUSED(getMyId); + Y_UNUSED(signalAddress); THolder<ITrace> trace(CreateTrace(PbSignal.GetSerializedTrace())); - if (signalAddress.size()) { - // cuttent signal is not the target - trace->OutSignalHtmlBody(str, tsInfo, getMyId, signalAddress); - } else { - // current signal is the target - trace->OutHtml(str, tsInfo, getMyId); - } -} - -} -} + if (signalAddress.size()) { + // cuttent signal is not the target + trace->OutSignalHtmlBody(str, tsInfo, getMyId, signalAddress); + } else { + // current signal is the target + trace->OutHtml(str, tsInfo, getMyId); + } +} + +} +} diff --git a/ydb/core/tablet/tablet_tracing_signals.h b/ydb/core/tablet/tablet_tracing_signals.h index 8c376f99dd..a8e73087e1 100644 --- a/ydb/core/tablet/tablet_tracing_signals.h +++ b/ydb/core/tablet/tablet_tracing_signals.h @@ -1,299 +1,299 @@ -#pragma once - -#include "defs.h" +#pragma once + +#include "defs.h" #include <ydb/core/protos/tablet_tracing_signals.pb.h> #include <ydb/core/tracing/signal.h> #include <ydb/core/base/tracing.h> #include <ydb/core/base/tablet.h> #include <ydb/core/base/tablet_types.h> -#include <util/generic/set.h> +#include <util/generic/set.h> #include <ydb/core/protos/tablet.pb.h> - -namespace NKikimr { -namespace NTracing { - - namespace NSignalTypes { - enum ESignalType : ITraceSignal::EType { - TypeOnTabletBootstrap = ITraceSignal::ES_TABLET_BOOTSTRAP, - TypeOnHandleStateStorageInfoResolve, - TypeOnCancelTablet, - TypeOnLockedInitializationPath, - TypeOnHandleStateStorageInfoLock, - TypeOnPromoteToCandidate, - TypeOnTabletBlockBlobStorage, - TypeOnTabletRebuildGraph, - TypeOnWriteZeroEntry, + +namespace NKikimr { +namespace NTracing { + + namespace NSignalTypes { + enum ESignalType : ITraceSignal::EType { + TypeOnTabletBootstrap = ITraceSignal::ES_TABLET_BOOTSTRAP, + TypeOnHandleStateStorageInfoResolve, + TypeOnCancelTablet, + TypeOnLockedInitializationPath, + TypeOnHandleStateStorageInfoLock, + TypeOnPromoteToCandidate, + TypeOnTabletBlockBlobStorage, + TypeOnTabletRebuildGraph, + TypeOnWriteZeroEntry, TypeOnFollowerPromoteToLeader, - TypeRebuildGraphBootstrap, - TypeErrorEntryBeyondBlocked, - TypeErrorUnknownStatus, - TypeErrorParsingFromString, - TypeErrorSendRefsCheck, - TypeErrorRebuildGraph, - TypeOnProcessKeyEntry, - TypeOnProcessZeroEntry, - TypeOnProcessLogEntry, - TypeOnDiscoverRangeRequest, - TypeOnApplyDiscoveryRange, - TypeOnMakeHistory, - TypeOnCheckRefsGetResult, - TypeOnBuildHistoryGraph, - TypeOnRebuildGraphResult - }; - } - - class TOnTabletBootstrap : public TTraceSignal<TOnTabletBootstrap, - NKikimrTracing::TOnTabletBootstrap, - NSignalTypes::TypeOnTabletBootstrap> { - public: + TypeRebuildGraphBootstrap, + TypeErrorEntryBeyondBlocked, + TypeErrorUnknownStatus, + TypeErrorParsingFromString, + TypeErrorSendRefsCheck, + TypeErrorRebuildGraph, + TypeOnProcessKeyEntry, + TypeOnProcessZeroEntry, + TypeOnProcessLogEntry, + TypeOnDiscoverRangeRequest, + TypeOnApplyDiscoveryRange, + TypeOnMakeHistory, + TypeOnCheckRefsGetResult, + TypeOnBuildHistoryGraph, + TypeOnRebuildGraphResult + }; + } + + class TOnTabletBootstrap : public TTraceSignal<TOnTabletBootstrap, + NKikimrTracing::TOnTabletBootstrap, + NSignalTypes::TypeOnTabletBootstrap> { + public: TOnTabletBootstrap(ui32 suggestedGeneration, bool leader, const TActorId& proxyID); - TOnTabletBootstrap(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnHandleStateStorageInfoResolve : - public TTraceSignal<TOnHandleStateStorageInfoResolve, - NKikimrTracing::TOnHandleStateStorageInfoResolve, - NSignalTypes::TypeOnHandleStateStorageInfoResolve> { - public: - TOnHandleStateStorageInfoResolve(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); - TOnHandleStateStorageInfoResolve(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnCancelTablet : public TTraceSignal<TOnCancelTablet, - NKikimrTracing::TOnCancelTablet, - NSignalTypes::TypeOnCancelTablet> { - public: - TOnCancelTablet(ui64 tabletID - , TTabletTypes::EType tabletType - , ui32 tabletDeadReason - , ui32 suggestedGeneration - , ui32 knownGeneration - ); - TOnCancelTablet(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnLockedInitializationPath : public TTraceSignal<TOnLockedInitializationPath, - NKikimrTracing::TOnLockedInitializationPath, - NSignalTypes::TypeOnLockedInitializationPath> { - public: - TOnLockedInitializationPath(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); - TOnLockedInitializationPath(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnHandleStateStorageInfoLock : public TTraceSignal<TOnHandleStateStorageInfoLock, - NKikimrTracing::TOnHandleStateStorageInfoLock, - NSignalTypes::TypeOnHandleStateStorageInfoLock> { - public: - TOnHandleStateStorageInfoLock(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); - TOnHandleStateStorageInfoLock(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnPromoteToCandidate : public TTraceSignal<TOnPromoteToCandidate, - NKikimrTracing::TOnPromoteToCandidate, - NSignalTypes::TypeOnPromoteToCandidate> { - public: - TOnPromoteToCandidate(ui32 knownGeneration); - TOnPromoteToCandidate(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnTabletBlockBlobStorage : public TTraceSignal<TOnTabletBlockBlobStorage, - NKikimrTracing::TOnTabletBlockBlobStorage, - NSignalTypes::TypeOnTabletBlockBlobStorage> { - public: + TOnTabletBootstrap(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnHandleStateStorageInfoResolve : + public TTraceSignal<TOnHandleStateStorageInfoResolve, + NKikimrTracing::TOnHandleStateStorageInfoResolve, + NSignalTypes::TypeOnHandleStateStorageInfoResolve> { + public: + TOnHandleStateStorageInfoResolve(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); + TOnHandleStateStorageInfoResolve(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnCancelTablet : public TTraceSignal<TOnCancelTablet, + NKikimrTracing::TOnCancelTablet, + NSignalTypes::TypeOnCancelTablet> { + public: + TOnCancelTablet(ui64 tabletID + , TTabletTypes::EType tabletType + , ui32 tabletDeadReason + , ui32 suggestedGeneration + , ui32 knownGeneration + ); + TOnCancelTablet(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnLockedInitializationPath : public TTraceSignal<TOnLockedInitializationPath, + NKikimrTracing::TOnLockedInitializationPath, + NSignalTypes::TypeOnLockedInitializationPath> { + public: + TOnLockedInitializationPath(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); + TOnLockedInitializationPath(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnHandleStateStorageInfoLock : public TTraceSignal<TOnHandleStateStorageInfoLock, + NKikimrTracing::TOnHandleStateStorageInfoLock, + NSignalTypes::TypeOnHandleStateStorageInfoLock> { + public: + TOnHandleStateStorageInfoLock(ui32 knownGeneration, ui32 knownStep, ui32 signatureSz); + TOnHandleStateStorageInfoLock(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnPromoteToCandidate : public TTraceSignal<TOnPromoteToCandidate, + NKikimrTracing::TOnPromoteToCandidate, + NSignalTypes::TypeOnPromoteToCandidate> { + public: + TOnPromoteToCandidate(ui32 knownGeneration); + TOnPromoteToCandidate(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnTabletBlockBlobStorage : public TTraceSignal<TOnTabletBlockBlobStorage, + NKikimrTracing::TOnTabletBlockBlobStorage, + NSignalTypes::TypeOnTabletBlockBlobStorage> { + public: TOnTabletBlockBlobStorage(const TActorId& reqBlockBlobStorageID, ui32 knownGeneration); - TOnTabletBlockBlobStorage(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnTabletRebuildGraph : public TTraceSignal<TOnTabletRebuildGraph, - NKikimrTracing::TOnTabletRebuildGraph, - NSignalTypes::TypeOnTabletRebuildGraph> { - public: + TOnTabletBlockBlobStorage(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnTabletRebuildGraph : public TTraceSignal<TOnTabletRebuildGraph, + NKikimrTracing::TOnTabletRebuildGraph, + NSignalTypes::TypeOnTabletRebuildGraph> { + public: TOnTabletRebuildGraph(const TActorId& tabletReqRebuildGraphID, const TTraceID& rebuildGraphTraceID); - TOnTabletRebuildGraph(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnWriteZeroEntry : public TTraceSignal<TOnWriteZeroEntry, - NKikimrTracing::TOnWriteZeroEntry, - NSignalTypes::TypeOnWriteZeroEntry> { - public: - TOnWriteZeroEntry( - const std::pair<ui32, ui32>& snapshot - , ui32 lastGeneration - , ui32 confirmedStep - , ui32 lastInGeneration - ); - TOnWriteZeroEntry(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - + TOnTabletRebuildGraph(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnWriteZeroEntry : public TTraceSignal<TOnWriteZeroEntry, + NKikimrTracing::TOnWriteZeroEntry, + NSignalTypes::TypeOnWriteZeroEntry> { + public: + TOnWriteZeroEntry( + const std::pair<ui32, ui32>& snapshot + , ui32 lastGeneration + , ui32 confirmedStep + , ui32 lastInGeneration + ); + TOnWriteZeroEntry(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + class TOnFollowerPromoteToLeader : public TTraceSignal<TOnFollowerPromoteToLeader, NKikimrTracing::TOnFollowerPromoteToLeader, NSignalTypes::TypeOnFollowerPromoteToLeader> { - public: + public: TOnFollowerPromoteToLeader( - ui32 suggestedGeneration + ui32 suggestedGeneration , const TActorId& knownLeaderID , const TActorId& followerStStGuardian - ); + ); TOnFollowerPromoteToLeader(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TRebuildGraphBootstrap : public TTraceSignal<TRebuildGraphBootstrap, - NKikimrTracing::TRebuildGraphBootstrap, - NSignalTypes::TypeRebuildGraphBootstrap> { - public: - TRebuildGraphBootstrap(ui32 blockedGen); - TRebuildGraphBootstrap(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TErrorEntryBeyondBlocked : public TTraceSignal<TErrorEntryBeyondBlocked, - NKikimrTracing::TErrorEntryBeyondBlocked, - NSignalTypes::TypeErrorEntryBeyondBlocked> { - public: - TErrorEntryBeyondBlocked(const TLogoBlobID& latest, ui32 blockedGen); - TErrorEntryBeyondBlocked(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TErrorUnknownStatus : public TTraceSignal<TErrorUnknownStatus, - NKikimrTracing::TErrorUnknownStatus, - NSignalTypes::TypeErrorUnknownStatus> { - public: - TErrorUnknownStatus(NKikimrProto::EReplyStatus status, const TString& reason); - TErrorUnknownStatus(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TErrorParsingFromString : public TTraceSignal<TErrorParsingFromString, - NKikimrTracing::TErrorParsingFromString, - NSignalTypes::TypeErrorParsingFromString> { - public: - TErrorParsingFromString(const TLogoBlobID& blobID); - TErrorParsingFromString(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TErrorSendRefsCheck : public TTraceSignal<TErrorSendRefsCheck, - NKikimrTracing::TErrorSendRefsCheck, - NSignalTypes::TypeErrorSendRefsCheck> { - public: - TErrorSendRefsCheck(); - TErrorSendRefsCheck(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TErrorRebuildGraph : public TTraceSignal<TErrorRebuildGraph, - NKikimrTracing::TErrorRebuildGraph, - NSignalTypes::TypeErrorRebuildGraph> { - public: - TErrorRebuildGraph(ui32 generation, ui32 step); - TErrorRebuildGraph(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnProcessKeyEntry : public TTraceSignal<TOnProcessKeyEntry, - NKikimrTracing::TOnProcessKeyEntry, - NSignalTypes::TypeOnProcessKeyEntry> { - public: - TOnProcessKeyEntry(const TLogoBlobID& latestEntry); - TOnProcessKeyEntry(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnProcessZeroEntry : public TTraceSignal<TOnProcessZeroEntry, - NKikimrTracing::TOnProcessZeroEntry, - NSignalTypes::TypeOnProcessZeroEntry> { - public: - TOnProcessZeroEntry(ui32 generation, const std::pair<ui32, ui32>& snapshot, const std::pair<ui32, ui32>& confirmed); - TOnProcessZeroEntry(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnProcessLogEntry : public TTraceSignal<TOnProcessLogEntry, - NKikimrTracing::TOnProcessLogEntry, - NSignalTypes::TypeOnProcessLogEntry> { - public: - TOnProcessLogEntry(const TLogoBlobID& blobID - , const std::pair<ui32, ui32>& snapshot - , const std::pair<ui32, ui32>& confirmed - , NKikimrTabletBase::TTabletLogEntry& logEntry - ); - TOnProcessLogEntry(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; - }; - - class TOnDiscoverRangeRequest : public TTraceSignal<TOnDiscoverRangeRequest, - NKikimrTracing::TOnDiscoverRangeRequest, - NSignalTypes::TypeOnDiscoverRangeRequest> { - public: - TOnDiscoverRangeRequest(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to); - TOnDiscoverRangeRequest(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnApplyDiscoveryRange : public TTraceSignal<TOnApplyDiscoveryRange, - NKikimrTracing::TOnApplyDiscoveryRange, - NSignalTypes::TypeOnApplyDiscoveryRange> { - public: - TOnApplyDiscoveryRange(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to); - TOnApplyDiscoveryRange(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnMakeHistory : public TTraceSignal<TOnMakeHistory, - NKikimrTracing::TOnMakeHistory, - NSignalTypes::TypeOnMakeHistory> { - public: - TOnMakeHistory(TSet<TLogoBlobID>& refsToCheck); - TOnMakeHistory(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; - }; - - class TOnCheckRefsGetResult : public TTraceSignal<TOnCheckRefsGetResult, - NKikimrTracing::TOnCheckRefsGetResult, - NSignalTypes::TypeOnCheckRefsGetResult> { - public: - TOnCheckRefsGetResult(ui32 responseSz); - TOnCheckRefsGetResult(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - }; - - class TOnBuildHistoryGraph : public TTraceSignal<TOnBuildHistoryGraph, - NKikimrTracing::TOnBuildHistoryGraph, - NSignalTypes::TypeOnBuildHistoryGraph> { - public: - TOnBuildHistoryGraph(TEvTablet::TDependencyGraph* graph); - TOnBuildHistoryGraph(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; - }; - - class TOnRebuildGraphResult : public TTraceSignal<TOnRebuildGraphResult, - NKikimrTracing::TOnRebuildGraphResult, - NSignalTypes::TypeOnRebuildGraphResult> { - public: - TOnRebuildGraphResult(NTracing::ITrace *trace); - TOnRebuildGraphResult(const TString& serializedSignal); - void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; - void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; - void OutHtmlBody(TStringStream& str - , const TTimestampInfo& tsInfo - , std::function<TString()> getMyId - , TList<ui64>& signalAddress - ) const override; - }; - -} -} + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TRebuildGraphBootstrap : public TTraceSignal<TRebuildGraphBootstrap, + NKikimrTracing::TRebuildGraphBootstrap, + NSignalTypes::TypeRebuildGraphBootstrap> { + public: + TRebuildGraphBootstrap(ui32 blockedGen); + TRebuildGraphBootstrap(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TErrorEntryBeyondBlocked : public TTraceSignal<TErrorEntryBeyondBlocked, + NKikimrTracing::TErrorEntryBeyondBlocked, + NSignalTypes::TypeErrorEntryBeyondBlocked> { + public: + TErrorEntryBeyondBlocked(const TLogoBlobID& latest, ui32 blockedGen); + TErrorEntryBeyondBlocked(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TErrorUnknownStatus : public TTraceSignal<TErrorUnknownStatus, + NKikimrTracing::TErrorUnknownStatus, + NSignalTypes::TypeErrorUnknownStatus> { + public: + TErrorUnknownStatus(NKikimrProto::EReplyStatus status, const TString& reason); + TErrorUnknownStatus(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TErrorParsingFromString : public TTraceSignal<TErrorParsingFromString, + NKikimrTracing::TErrorParsingFromString, + NSignalTypes::TypeErrorParsingFromString> { + public: + TErrorParsingFromString(const TLogoBlobID& blobID); + TErrorParsingFromString(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TErrorSendRefsCheck : public TTraceSignal<TErrorSendRefsCheck, + NKikimrTracing::TErrorSendRefsCheck, + NSignalTypes::TypeErrorSendRefsCheck> { + public: + TErrorSendRefsCheck(); + TErrorSendRefsCheck(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TErrorRebuildGraph : public TTraceSignal<TErrorRebuildGraph, + NKikimrTracing::TErrorRebuildGraph, + NSignalTypes::TypeErrorRebuildGraph> { + public: + TErrorRebuildGraph(ui32 generation, ui32 step); + TErrorRebuildGraph(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnProcessKeyEntry : public TTraceSignal<TOnProcessKeyEntry, + NKikimrTracing::TOnProcessKeyEntry, + NSignalTypes::TypeOnProcessKeyEntry> { + public: + TOnProcessKeyEntry(const TLogoBlobID& latestEntry); + TOnProcessKeyEntry(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnProcessZeroEntry : public TTraceSignal<TOnProcessZeroEntry, + NKikimrTracing::TOnProcessZeroEntry, + NSignalTypes::TypeOnProcessZeroEntry> { + public: + TOnProcessZeroEntry(ui32 generation, const std::pair<ui32, ui32>& snapshot, const std::pair<ui32, ui32>& confirmed); + TOnProcessZeroEntry(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnProcessLogEntry : public TTraceSignal<TOnProcessLogEntry, + NKikimrTracing::TOnProcessLogEntry, + NSignalTypes::TypeOnProcessLogEntry> { + public: + TOnProcessLogEntry(const TLogoBlobID& blobID + , const std::pair<ui32, ui32>& snapshot + , const std::pair<ui32, ui32>& confirmed + , NKikimrTabletBase::TTabletLogEntry& logEntry + ); + TOnProcessLogEntry(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; + }; + + class TOnDiscoverRangeRequest : public TTraceSignal<TOnDiscoverRangeRequest, + NKikimrTracing::TOnDiscoverRangeRequest, + NSignalTypes::TypeOnDiscoverRangeRequest> { + public: + TOnDiscoverRangeRequest(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to); + TOnDiscoverRangeRequest(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnApplyDiscoveryRange : public TTraceSignal<TOnApplyDiscoveryRange, + NKikimrTracing::TOnApplyDiscoveryRange, + NSignalTypes::TypeOnApplyDiscoveryRange> { + public: + TOnApplyDiscoveryRange(ui32 group, const TLogoBlobID& from, const TLogoBlobID& to); + TOnApplyDiscoveryRange(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnMakeHistory : public TTraceSignal<TOnMakeHistory, + NKikimrTracing::TOnMakeHistory, + NSignalTypes::TypeOnMakeHistory> { + public: + TOnMakeHistory(TSet<TLogoBlobID>& refsToCheck); + TOnMakeHistory(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; + }; + + class TOnCheckRefsGetResult : public TTraceSignal<TOnCheckRefsGetResult, + NKikimrTracing::TOnCheckRefsGetResult, + NSignalTypes::TypeOnCheckRefsGetResult> { + public: + TOnCheckRefsGetResult(ui32 responseSz); + TOnCheckRefsGetResult(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + }; + + class TOnBuildHistoryGraph : public TTraceSignal<TOnBuildHistoryGraph, + NKikimrTracing::TOnBuildHistoryGraph, + NSignalTypes::TypeOnBuildHistoryGraph> { + public: + TOnBuildHistoryGraph(TEvTablet::TDependencyGraph* graph); + TOnBuildHistoryGraph(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; + }; + + class TOnRebuildGraphResult : public TTraceSignal<TOnRebuildGraphResult, + NKikimrTracing::TOnRebuildGraphResult, + NSignalTypes::TypeOnRebuildGraphResult> { + public: + TOnRebuildGraphResult(NTracing::ITrace *trace); + TOnRebuildGraphResult(const TString& serializedSignal); + void OutText(TStringStream& str, TTimestampData& tsData, const TString& prefix) const override; + void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override; + void OutHtmlBody(TStringStream& str + , const TTimestampInfo& tsInfo + , std::function<TString()> getMyId + , TList<ui64>& signalAddress + ) const override; + }; + +} +} diff --git a/ydb/core/tablet/ya.make b/ydb/core/tablet/ya.make index f086090e17..25e611b408 100644 --- a/ydb/core/tablet/ya.make +++ b/ydb/core/tablet/ya.make @@ -25,8 +25,8 @@ SRCS( tablet_counters_protobuf.h tablet_exception.h tablet_impl.h - tablet_tracing_signals.cpp - tablet_tracing_signals.h + tablet_tracing_signals.cpp + tablet_tracing_signals.h tablet_list_renderer.cpp tablet_list_renderer.h tablet_metrics.h diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 28fb173169..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -423,60 +423,60 @@ namespace Tests { UNIT_ASSERT(configureResponse->Record.GetResponse().GetSuccess()); } - void TServer::SetupDefaultProfiles() { - NKikimr::Tests::TClient client(*Settings); - TAutoPtr<NMsgBusProxy::TBusConsoleRequest> request(new NMsgBusProxy::TBusConsoleRequest()); - auto &item = *request->Record.MutableConfigureRequest()->AddActions()->MutableAddConfigItem()->MutableConfigItem(); - item.SetKind((ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem); - auto &profiles = *item.MutableConfig()->MutableTableProfilesConfig(); - { - // Storage policy: - auto& policy = *profiles.AddStoragePolicies(); - policy.SetName("default"); - auto& family = *policy.AddColumnFamilies(); - family.SetId(0); - family.MutableStorageConfig()->MutableSysLog()->SetPreferredPoolKind("test"); - family.MutableStorageConfig()->MutableLog()->SetPreferredPoolKind("test"); - family.MutableStorageConfig()->MutableData()->SetPreferredPoolKind("test"); - } - { - // Compaction policy: - NLocalDb::TCompactionPolicyPtr defaultPolicy = NLocalDb::CreateDefaultUserTablePolicy(); + void TServer::SetupDefaultProfiles() { + NKikimr::Tests::TClient client(*Settings); + TAutoPtr<NMsgBusProxy::TBusConsoleRequest> request(new NMsgBusProxy::TBusConsoleRequest()); + auto &item = *request->Record.MutableConfigureRequest()->AddActions()->MutableAddConfigItem()->MutableConfigItem(); + item.SetKind((ui32)NKikimrConsole::TConfigItem::TableProfilesConfigItem); + auto &profiles = *item.MutableConfig()->MutableTableProfilesConfig(); + { + // Storage policy: + auto& policy = *profiles.AddStoragePolicies(); + policy.SetName("default"); + auto& family = *policy.AddColumnFamilies(); + family.SetId(0); + family.MutableStorageConfig()->MutableSysLog()->SetPreferredPoolKind("test"); + family.MutableStorageConfig()->MutableLog()->SetPreferredPoolKind("test"); + family.MutableStorageConfig()->MutableData()->SetPreferredPoolKind("test"); + } + { + // Compaction policy: + NLocalDb::TCompactionPolicyPtr defaultPolicy = NLocalDb::CreateDefaultUserTablePolicy(); NKikimrSchemeOp::TCompactionPolicy defaultflatSchemePolicy; - defaultPolicy->Serialize(defaultflatSchemePolicy); - auto &defaultCompactionPolicy = *profiles.AddCompactionPolicies(); - defaultCompactionPolicy.SetName("default"); - defaultCompactionPolicy.MutableCompactionPolicy()->CopyFrom(defaultflatSchemePolicy); - - NLocalDb::TCompactionPolicy policy1; - policy1.Generations.push_back({ 0, 8, 8, 128 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(1), true }); + defaultPolicy->Serialize(defaultflatSchemePolicy); + auto &defaultCompactionPolicy = *profiles.AddCompactionPolicies(); + defaultCompactionPolicy.SetName("default"); + defaultCompactionPolicy.MutableCompactionPolicy()->CopyFrom(defaultflatSchemePolicy); + + NLocalDb::TCompactionPolicy policy1; + policy1.Generations.push_back({ 0, 8, 8, 128 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(1), true }); NKikimrSchemeOp::TCompactionPolicy flatSchemePolicy1; - policy1.Serialize(flatSchemePolicy1); - auto &compactionPolicy1 = *profiles.AddCompactionPolicies(); - compactionPolicy1.SetName("compaction1"); - compactionPolicy1.MutableCompactionPolicy()->CopyFrom(flatSchemePolicy1); - - NLocalDb::TCompactionPolicy policy2; - policy2.Generations.push_back({ 0, 8, 8, 128 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(1), true }); - policy2.Generations.push_back({ 40 * 1024 * 1024, 5, 16, 512 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(2), false }); + policy1.Serialize(flatSchemePolicy1); + auto &compactionPolicy1 = *profiles.AddCompactionPolicies(); + compactionPolicy1.SetName("compaction1"); + compactionPolicy1.MutableCompactionPolicy()->CopyFrom(flatSchemePolicy1); + + NLocalDb::TCompactionPolicy policy2; + policy2.Generations.push_back({ 0, 8, 8, 128 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(1), true }); + policy2.Generations.push_back({ 40 * 1024 * 1024, 5, 16, 512 * 1024 * 1024, NLocalDb::LegacyQueueIdToTaskName(2), false }); NKikimrSchemeOp::TCompactionPolicy flatSchemePolicy2; - policy2.Serialize(flatSchemePolicy2); - auto &compactionPolicy2 = *profiles.AddCompactionPolicies(); - compactionPolicy2.SetName("compaction2"); - compactionPolicy2.MutableCompactionPolicy()->CopyFrom(flatSchemePolicy2); - } - { - auto& profile = *profiles.AddTableProfiles(); - profile.SetName("default"); - profile.SetStoragePolicy("default"); - } - TAutoPtr<NBus::TBusMessage> reply; - NBus::EMessageStatus msgStatus = client.SyncCall(request, reply); - UNIT_ASSERT_VALUES_EQUAL(msgStatus, NBus::MESSAGE_OK); - auto resp = dynamic_cast<NMsgBusProxy::TBusConsoleResponse*>(reply.Get())->Record; - UNIT_ASSERT_VALUES_EQUAL(resp.GetStatus().GetCode(), Ydb::StatusIds::SUCCESS); - } - + policy2.Serialize(flatSchemePolicy2); + auto &compactionPolicy2 = *profiles.AddCompactionPolicies(); + compactionPolicy2.SetName("compaction2"); + compactionPolicy2.MutableCompactionPolicy()->CopyFrom(flatSchemePolicy2); + } + { + auto& profile = *profiles.AddTableProfiles(); + profile.SetName("default"); + profile.SetStoragePolicy("default"); + } + TAutoPtr<NBus::TBusMessage> reply; + NBus::EMessageStatus msgStatus = client.SyncCall(request, reply); + UNIT_ASSERT_VALUES_EQUAL(msgStatus, NBus::MESSAGE_OK); + auto resp = dynamic_cast<NMsgBusProxy::TBusConsoleResponse*>(reply.Get())->Record; + UNIT_ASSERT_VALUES_EQUAL(resp.GetStatus().GetCode(), Ydb::StatusIds::SUCCESS); + } + void TServer::SetupDomainLocalService(ui32 nodeIdx) { SetupLocalService(nodeIdx, Settings->DomainName); } diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 05c1265192..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -220,7 +220,7 @@ namespace Tests { void EnableGRpc(const NGrpc::TServerOptions& options); void EnableGRpc(ui16 port); - void SetupDefaultProfiles(); + void SetupDefaultProfiles(); TIntrusivePtr<NMonitoring::TDynamicCounters> GetGRpcServerRootCounters() const { return GRpcServerRootCounters; diff --git a/ydb/core/tracing/http.cpp b/ydb/core/tracing/http.cpp index bc8bd01a5c..1fe4b33706 100644 --- a/ydb/core/tracing/http.cpp +++ b/ydb/core/tracing/http.cpp @@ -1,175 +1,175 @@ -#include "http.h" -#include <util/string/builder.h> - -namespace NKikimr { -namespace NTracing { -namespace NHttp { - -TCollapsedRef::TCollapsedRef(IOutputStream& str, const TString& target, const TString& text) - : Str(str) { - Str << "<a class='collapse-ref' data-toggle='collapse' data-target='#" << target << "'>" - << text << "</a>"; - Str << "<div id='" << target << "' class='collapse'>"; -} - -TCollapsedRef::~TCollapsedRef() { - try { - Str << "</div>"; - } - catch (...) { - } -} - -TCollapsedRefAjax::TCollapsedRefAjax( - IOutputStream& str, - const TString& target, - const TTimestampInfo& tsInfo, - const TString& text -) - : Str(str) { - Str << "<a class='collapse-ref' data-toggle='collapse' data-target='#" << target - << "' onclick=\"getCollapsedData(this, '" << target << "', " - << tsInfo.Mode << ", " << tsInfo.Precision - << ")\" data-filled=false>" << text << "</a>" << Endl - << "<div id='" << target << "' class='collapse'>"; -} - -TCollapsedRefAjax::~TCollapsedRefAjax() { - try { - Str << "</div>"; - } - catch (...) { - } -} - -void OutputStyles(IOutputStream& os) { - os << R"(<style> - .collapse-ref { - cursor: pointer; - } - .tab-left { - margin-left: 20px; - } - .first-dropdown-block { - float: left; - } - .dropdown-block { - margin-left: 20px; - float: left; - } - .panel { - display: inline-block; - } - </style> -)"; -} - -void OutputScripts(IOutputStream& str) { - str << R"( -<script> -function getCollapsedData(elem, signalId, tmode, tprecision) { - if (elem.getAttribute('data-filled') === 'false') { - $.ajax({ - url: document.URL + '&SignalID=' + signalId + '&TMode=' + tmode + '&TPrecision=' + tprecision, - success: function(result) { - var targetElem = document.getElementById(signalId); - if (targetElem === undefined) { - alert('cant find element with id: ' + signalId); - } else { - targetElem.innerHTML = result; - } - }, - error: function() { document.getElementById(signalId).innerHTML = 'Error retrieving data'; } - }); - } -} -</script> -)"; -} - -void OutputStaticPart(IOutputStream& str) { - OutputStyles(str); - OutputScripts(str); -} - -class TInfoPanel { -public: - TInfoPanel(IOutputStream& str, const TString& heading) - : Str(str) { - Str << "<div class = \"panel panel-default\">" << Endl - << "<div class = \"panel-heading\">" << heading << "</div>" << Endl - << "<div class = \"panel-body\">" << Endl; - } - - ~TInfoPanel() { - Str << "</div>" << Endl << "</div>"; - } - -private: - IOutputStream& Str; -}; - -void OutputTimeDropdown(IOutputStream& str, const TTraceInfo& traceInfo) { - TInfoPanel panel(str, "Timestamp settings"); - TStringBuilder hrefBase; - hrefBase << " <li role = \"presentation\"> <a role = \"menuitem\" tabindex = \"-1\" href = \"tablet?" - << "NodeID=" << traceInfo.NodeId - << "&TabletID=" << traceInfo.TabletId - << "&RandomID=" << traceInfo.TraceId.RandomID - << "&CreationTime=" << traceInfo.TraceId.CreationTime; - - TStringBuilder hrefPrecision; - if (traceInfo.TimestampInfo.Precision != TTimestampInfo::PrecisionDefault) { - hrefPrecision << "&TPrecision=" << traceInfo.TimestampInfo.Precision; - } - - str << R"( -<div class="first-dropdown-block"> -<div class="dropdown"> -<button class="btn btn-primary dropdown-toggle" id="timemode" type="button" data-toggle="dropdown">)"; - str << TTimestampInfo::GetTextMode(traceInfo.TimestampInfo.Mode); - str << R"(<span class="caret"></span></button> -<ul class="dropdown-menu" role="menu" aria-labelledby="timemode"> -)"; - str << hrefBase << "&TMode=" - << TTimestampInfo::ModeDisabled << hrefPrecision << "\">disabled</a></li>"; - str << " <li role=\"presentation\" class=\"divider\"></li>" << Endl; - str << hrefBase << "&TMode=" - << TTimestampInfo::ModeAbsolute << hrefPrecision << "\">Absolute</a></li>" << Endl; - str << hrefBase << "&TMode=" - << TTimestampInfo::ModeFirst << hrefPrecision << "\">Relative to the first</a></li>" << Endl; - str << hrefBase << "&TMode=" - << TTimestampInfo::ModePrevious << hrefPrecision << "\">Relative to the previous</a></li>" << Endl; - str << R"(</ul> -</div> -</div> -)"; - - if (traceInfo.TimestampInfo.Mode == TTimestampInfo::ModeDisabled) { - return; - } - - if (traceInfo.TimestampInfo.Mode != TTimestampInfo::ModeDefault) { - hrefBase << "&TMode=" << traceInfo.TimestampInfo.Mode; - } - - str << R"( -<div class="dropdown-block"> -<div class="dropdown"> -<button class="btn btn-primary dropdown-toggle" id="timeprecision" type="button" data-toggle="dropdown">)"; - str << TTimestampInfo::GetTextPrecision(traceInfo.TimestampInfo.Precision); - str << R"(<span class="caret"></span></button> -<ul class="dropdown-menu" role="menu" aria-labelledby="timeprecision"> -)"; - str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionSeconds << "\">Seconds</a></li>" << Endl; - str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionMilliseconds << "\">Milliseconds</a></li>" << Endl; - str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionMicroseconds << "\">Microseconds</a></li>" << Endl; - str << R"(</ul> -</div> -</div> -)"; -} - -} -} -} +#include "http.h" +#include <util/string/builder.h> + +namespace NKikimr { +namespace NTracing { +namespace NHttp { + +TCollapsedRef::TCollapsedRef(IOutputStream& str, const TString& target, const TString& text) + : Str(str) { + Str << "<a class='collapse-ref' data-toggle='collapse' data-target='#" << target << "'>" + << text << "</a>"; + Str << "<div id='" << target << "' class='collapse'>"; +} + +TCollapsedRef::~TCollapsedRef() { + try { + Str << "</div>"; + } + catch (...) { + } +} + +TCollapsedRefAjax::TCollapsedRefAjax( + IOutputStream& str, + const TString& target, + const TTimestampInfo& tsInfo, + const TString& text +) + : Str(str) { + Str << "<a class='collapse-ref' data-toggle='collapse' data-target='#" << target + << "' onclick=\"getCollapsedData(this, '" << target << "', " + << tsInfo.Mode << ", " << tsInfo.Precision + << ")\" data-filled=false>" << text << "</a>" << Endl + << "<div id='" << target << "' class='collapse'>"; +} + +TCollapsedRefAjax::~TCollapsedRefAjax() { + try { + Str << "</div>"; + } + catch (...) { + } +} + +void OutputStyles(IOutputStream& os) { + os << R"(<style> + .collapse-ref { + cursor: pointer; + } + .tab-left { + margin-left: 20px; + } + .first-dropdown-block { + float: left; + } + .dropdown-block { + margin-left: 20px; + float: left; + } + .panel { + display: inline-block; + } + </style> +)"; +} + +void OutputScripts(IOutputStream& str) { + str << R"( +<script> +function getCollapsedData(elem, signalId, tmode, tprecision) { + if (elem.getAttribute('data-filled') === 'false') { + $.ajax({ + url: document.URL + '&SignalID=' + signalId + '&TMode=' + tmode + '&TPrecision=' + tprecision, + success: function(result) { + var targetElem = document.getElementById(signalId); + if (targetElem === undefined) { + alert('cant find element with id: ' + signalId); + } else { + targetElem.innerHTML = result; + } + }, + error: function() { document.getElementById(signalId).innerHTML = 'Error retrieving data'; } + }); + } +} +</script> +)"; +} + +void OutputStaticPart(IOutputStream& str) { + OutputStyles(str); + OutputScripts(str); +} + +class TInfoPanel { +public: + TInfoPanel(IOutputStream& str, const TString& heading) + : Str(str) { + Str << "<div class = \"panel panel-default\">" << Endl + << "<div class = \"panel-heading\">" << heading << "</div>" << Endl + << "<div class = \"panel-body\">" << Endl; + } + + ~TInfoPanel() { + Str << "</div>" << Endl << "</div>"; + } + +private: + IOutputStream& Str; +}; + +void OutputTimeDropdown(IOutputStream& str, const TTraceInfo& traceInfo) { + TInfoPanel panel(str, "Timestamp settings"); + TStringBuilder hrefBase; + hrefBase << " <li role = \"presentation\"> <a role = \"menuitem\" tabindex = \"-1\" href = \"tablet?" + << "NodeID=" << traceInfo.NodeId + << "&TabletID=" << traceInfo.TabletId + << "&RandomID=" << traceInfo.TraceId.RandomID + << "&CreationTime=" << traceInfo.TraceId.CreationTime; + + TStringBuilder hrefPrecision; + if (traceInfo.TimestampInfo.Precision != TTimestampInfo::PrecisionDefault) { + hrefPrecision << "&TPrecision=" << traceInfo.TimestampInfo.Precision; + } + + str << R"( +<div class="first-dropdown-block"> +<div class="dropdown"> +<button class="btn btn-primary dropdown-toggle" id="timemode" type="button" data-toggle="dropdown">)"; + str << TTimestampInfo::GetTextMode(traceInfo.TimestampInfo.Mode); + str << R"(<span class="caret"></span></button> +<ul class="dropdown-menu" role="menu" aria-labelledby="timemode"> +)"; + str << hrefBase << "&TMode=" + << TTimestampInfo::ModeDisabled << hrefPrecision << "\">disabled</a></li>"; + str << " <li role=\"presentation\" class=\"divider\"></li>" << Endl; + str << hrefBase << "&TMode=" + << TTimestampInfo::ModeAbsolute << hrefPrecision << "\">Absolute</a></li>" << Endl; + str << hrefBase << "&TMode=" + << TTimestampInfo::ModeFirst << hrefPrecision << "\">Relative to the first</a></li>" << Endl; + str << hrefBase << "&TMode=" + << TTimestampInfo::ModePrevious << hrefPrecision << "\">Relative to the previous</a></li>" << Endl; + str << R"(</ul> +</div> +</div> +)"; + + if (traceInfo.TimestampInfo.Mode == TTimestampInfo::ModeDisabled) { + return; + } + + if (traceInfo.TimestampInfo.Mode != TTimestampInfo::ModeDefault) { + hrefBase << "&TMode=" << traceInfo.TimestampInfo.Mode; + } + + str << R"( +<div class="dropdown-block"> +<div class="dropdown"> +<button class="btn btn-primary dropdown-toggle" id="timeprecision" type="button" data-toggle="dropdown">)"; + str << TTimestampInfo::GetTextPrecision(traceInfo.TimestampInfo.Precision); + str << R"(<span class="caret"></span></button> +<ul class="dropdown-menu" role="menu" aria-labelledby="timeprecision"> +)"; + str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionSeconds << "\">Seconds</a></li>" << Endl; + str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionMilliseconds << "\">Milliseconds</a></li>" << Endl; + str << hrefBase << "&TPrecision=" << TTimestampInfo::PrecisionMicroseconds << "\">Microseconds</a></li>" << Endl; + str << R"(</ul> +</div> +</div> +)"; +} + +} +} +} diff --git a/ydb/core/tracing/http.h b/ydb/core/tracing/http.h index 6688df89d4..25d04e1088 100644 --- a/ydb/core/tracing/http.h +++ b/ydb/core/tracing/http.h @@ -1,44 +1,44 @@ -#pragma once +#pragma once #include <library/cpp/monlib/service/pages/templates.h> #include <ydb/core/base/tracing.h> - -#define COLLAPSED_REF_CONTENT(target, text) \ - WITH_SCOPED(tmp, NKikimr::NTracing::NHttp::TCollapsedRef(__stream, target, text)) - -#define COLLAPSED_REF_CONTENT_AJAX(target, tsInfo, text) \ - WITH_SCOPED(tmp, NKikimr::NTracing::NHttp::TCollapsedRefAjax(__stream, target, tsInfo, text)) - -namespace NKikimr { -namespace NTracing { -namespace NHttp { - -struct TCollapsedRef { - TCollapsedRef(IOutputStream& str, const TString& target, const TString& text); - ~TCollapsedRef(); - - explicit inline operator bool() const noexcept { - return true; // just to work with WITH_SCOPED - } - - IOutputStream& Str; -}; - -struct TCollapsedRefAjax { - TCollapsedRefAjax(IOutputStream& str, const TString& target, const TTimestampInfo& tsInfo, const TString& text); - ~TCollapsedRefAjax(); - - explicit inline operator bool() const noexcept { - return true; // just to work with WITH_SCOPED - } - - IOutputStream& Str; -}; - -void OutputStyles(IOutputStream& os); -void OutputScripts(IOutputStream& str); -void OutputStaticPart(IOutputStream& str); -void OutputTimeDropdown(IOutputStream& str, const TTraceInfo& traceInfo); - -} -} -} + +#define COLLAPSED_REF_CONTENT(target, text) \ + WITH_SCOPED(tmp, NKikimr::NTracing::NHttp::TCollapsedRef(__stream, target, text)) + +#define COLLAPSED_REF_CONTENT_AJAX(target, tsInfo, text) \ + WITH_SCOPED(tmp, NKikimr::NTracing::NHttp::TCollapsedRefAjax(__stream, target, tsInfo, text)) + +namespace NKikimr { +namespace NTracing { +namespace NHttp { + +struct TCollapsedRef { + TCollapsedRef(IOutputStream& str, const TString& target, const TString& text); + ~TCollapsedRef(); + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; +}; + +struct TCollapsedRefAjax { + TCollapsedRefAjax(IOutputStream& str, const TString& target, const TTimestampInfo& tsInfo, const TString& text); + ~TCollapsedRefAjax(); + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; +}; + +void OutputStyles(IOutputStream& os); +void OutputScripts(IOutputStream& str); +void OutputStaticPart(IOutputStream& str); +void OutputTimeDropdown(IOutputStream& str, const TTraceInfo& traceInfo); + +} +} +} diff --git a/ydb/core/tracing/signal.h b/ydb/core/tracing/signal.h index 1e8757a253..7f317436e9 100644 --- a/ydb/core/tracing/signal.h +++ b/ydb/core/tracing/signal.h @@ -1,169 +1,169 @@ -#pragma once +#pragma once #include <ydb/core/base/defs.h> #include <ydb/core/base/tracing.h> -#include <util/string/builder.h> -#include "http.h" - -namespace NKikimr { -namespace NTracing { - -template <size_t N> -struct TPad { - int I; -}; - -template <size_t N> -inline TPad<N> Pad(int i) { - return {i}; -} - -inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<2>& p) { - if (p.I < 10) { - if (p.I >= 0) { - o << '0'; - } - } - - return o << p.I; -} - -inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<3>& p) { - if (p.I < 100) { - if (p.I >= 0) { - if (p.I < 10) { - o << '0' << '0'; - } else if (p.I < 10) { - o << '0'; - } - } - } - - return o << p.I; -} - -inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<6>& p) { - if (p.I < 100000) { - if (p.I >= 0) { - if (p.I < 10) { - o << '0' << '0' << '0' << '0' << '0'; - } else if (p.I < 100) { - o << '0' << '0' << '0' << '0'; - } else if (p.I < 1000) { - o << '0' << '0' << '0'; - } else if (p.I < 10000) { - o << '0' << '0'; - } else { - o << '0'; - } - } - } - - return o << p.I; -} - -inline void WriteMicroSecondsToStream(TStringBuilder& os, ui32 microSeconds) { - os << '.' << Pad<6>(microSeconds); -} - -inline void WriteMilliSecondsToStream(TStringBuilder& os, ui32 milliSeconds) { - os << '.' << Pad<3>(milliSeconds); -} - -template <typename TDerived, typename TPbType, ITraceSignal::EType TSignalType> -class TTraceSignal : public ITraceSignal { -public: - TTraceSignal() { - PbSignal.SetTimeStamp(TInstant::Now().MicroSeconds()); - } - - TTraceSignal(const TString& serializedSignal) { +#include <util/string/builder.h> +#include "http.h" + +namespace NKikimr { +namespace NTracing { + +template <size_t N> +struct TPad { + int I; +}; + +template <size_t N> +inline TPad<N> Pad(int i) { + return {i}; +} + +inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<2>& p) { + if (p.I < 10) { + if (p.I >= 0) { + o << '0'; + } + } + + return o << p.I; +} + +inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<3>& p) { + if (p.I < 100) { + if (p.I >= 0) { + if (p.I < 10) { + o << '0' << '0'; + } else if (p.I < 10) { + o << '0'; + } + } + } + + return o << p.I; +} + +inline TStringBuilder& operator<<(TStringBuilder& o, const TPad<6>& p) { + if (p.I < 100000) { + if (p.I >= 0) { + if (p.I < 10) { + o << '0' << '0' << '0' << '0' << '0'; + } else if (p.I < 100) { + o << '0' << '0' << '0' << '0'; + } else if (p.I < 1000) { + o << '0' << '0' << '0'; + } else if (p.I < 10000) { + o << '0' << '0'; + } else { + o << '0'; + } + } + } + + return o << p.I; +} + +inline void WriteMicroSecondsToStream(TStringBuilder& os, ui32 microSeconds) { + os << '.' << Pad<6>(microSeconds); +} + +inline void WriteMilliSecondsToStream(TStringBuilder& os, ui32 milliSeconds) { + os << '.' << Pad<3>(milliSeconds); +} + +template <typename TDerived, typename TPbType, ITraceSignal::EType TSignalType> +class TTraceSignal : public ITraceSignal { +public: + TTraceSignal() { + PbSignal.SetTimeStamp(TInstant::Now().MicroSeconds()); + } + + TTraceSignal(const TString& serializedSignal) { Y_PROTOBUF_SUPPRESS_NODISCARD PbSignal.ParseFromString(serializedSignal); - } - - bool SerializeToString(TString& str) const override { - return PbSignal.SerializeToString(&str); - } - + } + + bool SerializeToString(TString& str) const override { + return PbSignal.SerializeToString(&str); + } + EType GetType() const override { - return TSignalType; - } - - // Default realizations: - - virtual void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override { - Y_UNUSED(getMyId); - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - OutText(str, tsData); - } - } - } - } - - virtual void OutHtmlBody( - TStringStream& str, - const TTimestampInfo& tsInfo, - std::function<TString()> getMyId, - TList<ui64>& signalAddress - ) const override { - Y_UNUSED(tsInfo); - Y_UNUSED(getMyId); - Y_UNUSED(signalAddress); - str << "ITraceSignal::OutHtmlBody is undefined for type " << TSignalType; - } - -protected: - TString TimeStamp(TTimestampData& tsData) const { - TStringBuilder str; - switch (tsData.Mode) { - case TTimestampInfo::ModeDisabled: { - return TString(); - } - case TTimestampInfo::ModeAbsolute: - case TTimestampInfo::ModeDefault: { - TInstant timestamp = TInstant::MicroSeconds(PbSignal.GetTimeStamp()); - str << timestamp.FormatLocalTime("%Y-%m-%d %H:%M:%S"); - WriteWithPrecision(str, timestamp, tsData.Precision); - break; - } - case TTimestampInfo::ModeFirst: - case TTimestampInfo::ModePrevious: { - TInstant timestamp = TInstant::MicroSeconds(PbSignal.GetTimeStamp()); - TDuration diffTime = timestamp - tsData.BaseTime; - str << diffTime.Hours() << "h " << Pad<2>(diffTime.Minutes() % 60) << "m " - << Pad<2>(diffTime.Seconds() % 60); - WriteWithPrecision(str, diffTime, tsData.Precision); - str << "s"; - if (tsData.Mode == TTimestampInfo::ModePrevious) { - tsData.BaseTime = timestamp; - } - break; - } - } - return str; - } - - template <typename TimeType> - void WriteWithPrecision(TStringBuilder& str, const TimeType& time, TTimestampInfo::EPrecision precision) const { - switch (precision) { - case TTimestampInfo::PrecisionSeconds: - break; - case TTimestampInfo::PrecisionMilliseconds: - case TTimestampInfo::PrecisionDefault: - WriteMilliSecondsToStream(str, time.MilliSecondsOfSecond()); - break; - case TTimestampInfo::PrecisionMicroseconds: - WriteMicroSecondsToStream(str, time.MicroSecondsOfSecond()); - break; - } - } - - using MyPbType = TPbType; - using TMyBase = TTraceSignal<TDerived, TPbType, TSignalType>; - MyPbType PbSignal; -}; - - -} -} + return TSignalType; + } + + // Default realizations: + + virtual void OutHtmlHeader(TStringStream& str, TTimestampData& tsData, std::function<TString()> getMyId) const override { + Y_UNUSED(getMyId); + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + OutText(str, tsData); + } + } + } + } + + virtual void OutHtmlBody( + TStringStream& str, + const TTimestampInfo& tsInfo, + std::function<TString()> getMyId, + TList<ui64>& signalAddress + ) const override { + Y_UNUSED(tsInfo); + Y_UNUSED(getMyId); + Y_UNUSED(signalAddress); + str << "ITraceSignal::OutHtmlBody is undefined for type " << TSignalType; + } + +protected: + TString TimeStamp(TTimestampData& tsData) const { + TStringBuilder str; + switch (tsData.Mode) { + case TTimestampInfo::ModeDisabled: { + return TString(); + } + case TTimestampInfo::ModeAbsolute: + case TTimestampInfo::ModeDefault: { + TInstant timestamp = TInstant::MicroSeconds(PbSignal.GetTimeStamp()); + str << timestamp.FormatLocalTime("%Y-%m-%d %H:%M:%S"); + WriteWithPrecision(str, timestamp, tsData.Precision); + break; + } + case TTimestampInfo::ModeFirst: + case TTimestampInfo::ModePrevious: { + TInstant timestamp = TInstant::MicroSeconds(PbSignal.GetTimeStamp()); + TDuration diffTime = timestamp - tsData.BaseTime; + str << diffTime.Hours() << "h " << Pad<2>(diffTime.Minutes() % 60) << "m " + << Pad<2>(diffTime.Seconds() % 60); + WriteWithPrecision(str, diffTime, tsData.Precision); + str << "s"; + if (tsData.Mode == TTimestampInfo::ModePrevious) { + tsData.BaseTime = timestamp; + } + break; + } + } + return str; + } + + template <typename TimeType> + void WriteWithPrecision(TStringBuilder& str, const TimeType& time, TTimestampInfo::EPrecision precision) const { + switch (precision) { + case TTimestampInfo::PrecisionSeconds: + break; + case TTimestampInfo::PrecisionMilliseconds: + case TTimestampInfo::PrecisionDefault: + WriteMilliSecondsToStream(str, time.MilliSecondsOfSecond()); + break; + case TTimestampInfo::PrecisionMicroseconds: + WriteMicroSecondsToStream(str, time.MicroSecondsOfSecond()); + break; + } + } + + using MyPbType = TPbType; + using TMyBase = TTraceSignal<TDerived, TPbType, TSignalType>; + MyPbType PbSignal; +}; + + +} +} diff --git a/ydb/core/tracing/tablet_info.cpp b/ydb/core/tracing/tablet_info.cpp index a0c3edc3e1..6eb99ffc60 100644 --- a/ydb/core/tracing/tablet_info.cpp +++ b/ydb/core/tracing/tablet_info.cpp @@ -1,5 +1,5 @@ -#include "tablet_info.h" - +#include "tablet_info.h" + #include <library/cpp/actors/core/log.h> #include <ydb/core/mon/mon.h> #include <library/cpp/actors/core/mon.h> @@ -9,832 +9,832 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/interconnect/interconnect.h> -#include "http.h" - -namespace NKikimr { -namespace NTabletInfo { - -using namespace NActors; -using namespace NNodeWhiteboard; - -namespace { - -template<typename TStatus> -struct TWBReplyStatus { - enum EStatus { - OK = 0, - UNDELIVERED, - DISCONNECTED, - TIMEOUT - }; - - static TString StatusDescription(EStatus status) { - switch (status) { - case OK: - return "Reply received"; - break; - case UNDELIVERED: - return "Request undelivered"; - break; - case DISCONNECTED: - return "Node disconnected"; - break; - case TIMEOUT: - return "Request timeout"; - default: - return "Unknown status"; - break; - } - } - - EStatus Status; - THolder<TStatus> Response; -}; - -void AddTabletLookupForm(TStringStream& str) { - str << "<br><form method=\"GET\" id=\"tabletMonFrm\" name=\"tabletMonFrm\">" << Endl; - str << "<h3>Tablet lookup</h3>" << Endl; - str << "TabletID: <input type=\"text\" id=\"iTabletID\" name=\"iTabletID\"/>" << Endl; - str << "<input class=\"btn btn-primary\" type=\"submit\" value=\"Watch\"/>" << Endl; - str << "</form>" << Endl; -} - -void AddTraceLookupForm(TStringStream& str) { - str << "<br><form method=\"GET\" id=\"tblIntMonFrm\" name=\"tblIntMonFrm\">" << Endl; - str << "<h3>Trace lookup</h3>" << Endl; - str << "<a class='collapse-ref' data-toggle='collapse' data-target='#LookupInfo'>Help</a>"; - str << "<div id='LookupInfo' class='collapse'>"; - str << "<div class=\"tab-left\">" - << "NodeID - Get all tablets with traces(NodeID=0 - all nodes)</div>" << Endl; - str << "<div class=\"tab-left\">" - << "TabletID - Get all traces for given Tablet on all Nodes </div>" << Endl; - str << "<div class=\"tab-left\">" - << "NodeID && TabletID - Get all traces for given Tablet on given Node </div>" << Endl; - str << "<div class=\"tab-left\">" - << "NodeID && TabletID && RandomID && CreationTime - show exact trace </div>" << Endl; - str << "</div><br>" << Endl; - str << "NodeID: <input type=\"text\" id=\"NodeID\" name=\"NodeID\"/>" << Endl; - str << "TabletID: <input type=\"text\" id=\"TabletID\" name=\"TabletID\"/>" << Endl; - str << "RandomID: <input type=\"text\" id=\"RandomID\" name=\"RandomID\"/>" << Endl; - str << "CreationTime: <input type=\"text\" id=\"CreationTime\" name=\"CreationTime\"/>" << Endl; - str << "<input class=\"btn btn-primary\" type=\"submit\" value=\"Watch\"/>" << Endl; - str << "</form>" << Endl; -} - -class TTabletLookupActor : public TActorBootstrapped<TTabletLookupActor> { -public: - using TReplyStatus = TWBReplyStatus<TEvWhiteboard::TEvTabletLookupResponse>; - +#include "http.h" + +namespace NKikimr { +namespace NTabletInfo { + +using namespace NActors; +using namespace NNodeWhiteboard; + +namespace { + +template<typename TStatus> +struct TWBReplyStatus { + enum EStatus { + OK = 0, + UNDELIVERED, + DISCONNECTED, + TIMEOUT + }; + + static TString StatusDescription(EStatus status) { + switch (status) { + case OK: + return "Reply received"; + break; + case UNDELIVERED: + return "Request undelivered"; + break; + case DISCONNECTED: + return "Node disconnected"; + break; + case TIMEOUT: + return "Request timeout"; + default: + return "Unknown status"; + break; + } + } + + EStatus Status; + THolder<TStatus> Response; +}; + +void AddTabletLookupForm(TStringStream& str) { + str << "<br><form method=\"GET\" id=\"tabletMonFrm\" name=\"tabletMonFrm\">" << Endl; + str << "<h3>Tablet lookup</h3>" << Endl; + str << "TabletID: <input type=\"text\" id=\"iTabletID\" name=\"iTabletID\"/>" << Endl; + str << "<input class=\"btn btn-primary\" type=\"submit\" value=\"Watch\"/>" << Endl; + str << "</form>" << Endl; +} + +void AddTraceLookupForm(TStringStream& str) { + str << "<br><form method=\"GET\" id=\"tblIntMonFrm\" name=\"tblIntMonFrm\">" << Endl; + str << "<h3>Trace lookup</h3>" << Endl; + str << "<a class='collapse-ref' data-toggle='collapse' data-target='#LookupInfo'>Help</a>"; + str << "<div id='LookupInfo' class='collapse'>"; + str << "<div class=\"tab-left\">" + << "NodeID - Get all tablets with traces(NodeID=0 - all nodes)</div>" << Endl; + str << "<div class=\"tab-left\">" + << "TabletID - Get all traces for given Tablet on all Nodes </div>" << Endl; + str << "<div class=\"tab-left\">" + << "NodeID && TabletID - Get all traces for given Tablet on given Node </div>" << Endl; + str << "<div class=\"tab-left\">" + << "NodeID && TabletID && RandomID && CreationTime - show exact trace </div>" << Endl; + str << "</div><br>" << Endl; + str << "NodeID: <input type=\"text\" id=\"NodeID\" name=\"NodeID\"/>" << Endl; + str << "TabletID: <input type=\"text\" id=\"TabletID\" name=\"TabletID\"/>" << Endl; + str << "RandomID: <input type=\"text\" id=\"RandomID\" name=\"RandomID\"/>" << Endl; + str << "CreationTime: <input type=\"text\" id=\"CreationTime\" name=\"CreationTime\"/>" << Endl; + str << "<input class=\"btn btn-primary\" type=\"submit\" value=\"Watch\"/>" << Endl; + str << "</form>" << Endl; +} + +class TTabletLookupActor : public TActorBootstrapped<TTabletLookupActor> { +public: + using TReplyStatus = TWBReplyStatus<TEvWhiteboard::TEvTabletLookupResponse>; + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TABLET_LOOKUP_ACTOR; - } - + } + TTabletLookupActor(ui32 nodeId, const TActorId& sender, ui32 timeout) - : NodeId(nodeId) - , Sender(sender) - , Timeout(timeout) - {} - - void Bootstrap(const TActorContext& ctx) { - if (NodeId) { - // Direct request - SendRequest(NodeId, ctx); - Become(&TThis::StateTabletLookupRequested); - } else { - // Broadcast request + : NodeId(nodeId) + , Sender(sender) + , Timeout(timeout) + {} + + void Bootstrap(const TActorContext& ctx) { + if (NodeId) { + // Direct request + SendRequest(NodeId, ctx); + Become(&TThis::StateTabletLookupRequested); + } else { + // Broadcast request const TActorId nameserviceId = NActors::GetNameserviceActorId(); - ctx.Send(nameserviceId, new TEvInterconnect::TEvListNodes()); - Become(&TThis::StateNodeListRequested); - } - ctx.Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); - } - - STFUNC(StateNodeListRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvInterconnect::TEvNodesInfo, HandleNodeList); - CFunc(TEvents::TSystem::Wakeup, HandleNodeListTimeout); - } - } - - void HandleNodeList(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) { - for (const auto& ni : ev->Get()->Nodes) { - SendRequest(ni.NodeId, ctx); - } - if (Requested > 0) { - Become(&TThis::StateTabletLookupRequested); - } else { - ReplyAndDie(ctx); - } - } - - void HandleNodeListTimeout(const TActorContext &ctx) { - Notify(ctx, "Timeout getting node list"); - Die(ctx); - } - - void SendRequest(ui32 nodeId, const TActorContext& ctx) { + ctx.Send(nameserviceId, new TEvInterconnect::TEvListNodes()); + Become(&TThis::StateNodeListRequested); + } + ctx.Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); + } + + STFUNC(StateNodeListRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvInterconnect::TEvNodesInfo, HandleNodeList); + CFunc(TEvents::TSystem::Wakeup, HandleNodeListTimeout); + } + } + + void HandleNodeList(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) { + for (const auto& ni : ev->Get()->Nodes) { + SendRequest(ni.NodeId, ctx); + } + if (Requested > 0) { + Become(&TThis::StateTabletLookupRequested); + } else { + ReplyAndDie(ctx); + } + } + + void HandleNodeListTimeout(const TActorContext &ctx) { + Notify(ctx, "Timeout getting node list"); + Die(ctx); + } + + void SendRequest(ui32 nodeId, const TActorContext& ctx) { TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); - ctx.Send(whiteboardServiceId - , new TEvWhiteboard::TEvTabletLookupRequest() - , IEventHandle::FlagTrackDelivery, nodeId); - ++Requested; - TracedTablets[nodeId] = nullptr; - } - - STFUNC(StateTabletLookupRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvWhiteboard::TEvTabletLookupResponse, Handle); - HFunc(TEvents::TEvUndelivered, Undelivered); - HFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); - CFunc(TEvents::TSystem::Wakeup, HandleTimeout); - } - } - - void Handle(TEvWhiteboard::TEvTabletLookupResponse::TPtr& ev, const TActorContext& ctx) { - ui64 nodeId = ev.Get()->Cookie; + ctx.Send(whiteboardServiceId + , new TEvWhiteboard::TEvTabletLookupRequest() + , IEventHandle::FlagTrackDelivery, nodeId); + ++Requested; + TracedTablets[nodeId] = nullptr; + } + + STFUNC(StateTabletLookupRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvWhiteboard::TEvTabletLookupResponse, Handle); + HFunc(TEvents::TEvUndelivered, Undelivered); + HFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + } + } + + void Handle(TEvWhiteboard::TEvTabletLookupResponse::TPtr& ev, const TActorContext& ctx) { + ui64 nodeId = ev.Get()->Cookie; TracedTablets[nodeId] = THolder<TReplyStatus>(new TReplyStatus({ TReplyStatus::OK, THolder<TEvWhiteboard::TEvTabletLookupResponse>(ev->Release()) })); - NodeTabletInfoReceived(ctx); - } - - void NodeTabletInfoReceived(const TActorContext& ctx) { - ++Received; - if (Received >= Requested) { - ReplyAndDie(ctx); - } - } - - void BadResponseReceived(ui32 nodeId, const TActorContext& ctx, TReplyStatus::EStatus status) { - if (TracedTablets[nodeId] == nullptr) { + NodeTabletInfoReceived(ctx); + } + + void NodeTabletInfoReceived(const TActorContext& ctx) { + ++Received; + if (Received >= Requested) { + ReplyAndDie(ctx); + } + } + + void BadResponseReceived(ui32 nodeId, const TActorContext& ctx, TReplyStatus::EStatus status) { + if (TracedTablets[nodeId] == nullptr) { TracedTablets[nodeId] = THolder<TReplyStatus>(new TReplyStatus({ status, nullptr })); - NodeTabletInfoReceived(ctx); - } - } - - void Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { - BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::UNDELIVERED); - } - - void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { - BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::DISCONNECTED); - } - - void HandleTimeout(const TActorContext &ctx) { - for (auto& nodeTabletTracesPair : TracedTablets) { - if (!nodeTabletTracesPair.second) { + NodeTabletInfoReceived(ctx); + } + } + + void Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { + BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::UNDELIVERED); + } + + void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { + BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::DISCONNECTED); + } + + void HandleTimeout(const TActorContext &ctx) { + for (auto& nodeTabletTracesPair : TracedTablets) { + if (!nodeTabletTracesPair.second) { nodeTabletTracesPair.second = THolder<TReplyStatus>(new TReplyStatus({ TReplyStatus::TIMEOUT, nullptr })); - } - } - ReplyAndDie(ctx); - } - - void ReplyAndDie(const TActorContext& ctx) { - TStringStream str; - - HTML(str) { - NTracing::NHttp::OutputStaticPart(str); - AddTabletLookupForm(str); - str << "<br>"; - - H3() { - str << "All tablets that have traces"; - if (NodeId) { - str << " on Node " << NodeId << ":"; - } else { - str << ":"; - } - } - for (auto& nodeTabletsPair : TracedTablets) { - ui32 nodeId = nodeTabletsPair.first; - if (nodeTabletsPair.second->Status == TReplyStatus::OK) { - auto& nodeTablets = nodeTabletsPair.second->Response.Get()->Record; - size_t tabletIDsSize = nodeTablets.TabletIDsSize(); - if (!NodeId) { - H4() { - str << "Node " << nodeId; - if (!tabletIDsSize) { - str << " - no tablets with traces found"; - } - } - } - if (tabletIDsSize) { - for (ui32 i = 0; i < tabletIDsSize; ++i) { - ui64 tabletId = nodeTablets.GetTabletIDs(i); - DIV_CLASS("tab-left") { - str << "<a href=\"tablet?NodeID=" << nodeId << "&TabletID=" << tabletId << "\">" - << tabletId << "</a>"; - } - } - } else { - if (NodeId) { - DIV_CLASS("tab-left") { - str << "No tablets with traces found"; - } - } - } - } else { - // Bad response - if (NodeId) { - DIV_CLASS("tab-left") { - str << "<font color=\"red\">" - << TReplyStatus::StatusDescription(nodeTabletsPair.second->Status) - << "</font>"; - } - } else { - H4() { - str << "<font color=\"red\">Node " << nodeId << ": " - << TReplyStatus::StatusDescription(nodeTabletsPair.second->Status) - << "</font>"; - } - } - } - } - AddTraceLookupForm(str); - } - Notify(ctx, str.Str()); - Die(ctx); - } - - void Notify(const TActorContext &ctx, const TString& html) { - ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); - } - -private: - const ui32 NodeId; + } + } + ReplyAndDie(ctx); + } + + void ReplyAndDie(const TActorContext& ctx) { + TStringStream str; + + HTML(str) { + NTracing::NHttp::OutputStaticPart(str); + AddTabletLookupForm(str); + str << "<br>"; + + H3() { + str << "All tablets that have traces"; + if (NodeId) { + str << " on Node " << NodeId << ":"; + } else { + str << ":"; + } + } + for (auto& nodeTabletsPair : TracedTablets) { + ui32 nodeId = nodeTabletsPair.first; + if (nodeTabletsPair.second->Status == TReplyStatus::OK) { + auto& nodeTablets = nodeTabletsPair.second->Response.Get()->Record; + size_t tabletIDsSize = nodeTablets.TabletIDsSize(); + if (!NodeId) { + H4() { + str << "Node " << nodeId; + if (!tabletIDsSize) { + str << " - no tablets with traces found"; + } + } + } + if (tabletIDsSize) { + for (ui32 i = 0; i < tabletIDsSize; ++i) { + ui64 tabletId = nodeTablets.GetTabletIDs(i); + DIV_CLASS("tab-left") { + str << "<a href=\"tablet?NodeID=" << nodeId << "&TabletID=" << tabletId << "\">" + << tabletId << "</a>"; + } + } + } else { + if (NodeId) { + DIV_CLASS("tab-left") { + str << "No tablets with traces found"; + } + } + } + } else { + // Bad response + if (NodeId) { + DIV_CLASS("tab-left") { + str << "<font color=\"red\">" + << TReplyStatus::StatusDescription(nodeTabletsPair.second->Status) + << "</font>"; + } + } else { + H4() { + str << "<font color=\"red\">Node " << nodeId << ": " + << TReplyStatus::StatusDescription(nodeTabletsPair.second->Status) + << "</font>"; + } + } + } + } + AddTraceLookupForm(str); + } + Notify(ctx, str.Str()); + Die(ctx); + } + + void Notify(const TActorContext &ctx, const TString& html) { + ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); + } + +private: + const ui32 NodeId; const TActorId Sender; - ui32 Timeout; - ui32 Requested = 0; - ui32 Received = 0; - TMap<ui32, THolder<TReplyStatus>> TracedTablets; -}; - -class TTraceLookupActor : public TActorBootstrapped<TTraceLookupActor> { -public: - using TReplyStatus = TWBReplyStatus<TEvWhiteboard::TEvTraceLookupResponse>; - + ui32 Timeout; + ui32 Requested = 0; + ui32 Received = 0; + TMap<ui32, THolder<TReplyStatus>> TracedTablets; +}; + +class TTraceLookupActor : public TActorBootstrapped<TTraceLookupActor> { +public: + using TReplyStatus = TWBReplyStatus<TEvWhiteboard::TEvTraceLookupResponse>; + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::TRACE_LOOKUP_ACTOR; - } - + return NKikimrServices::TActivity::TRACE_LOOKUP_ACTOR; + } + TTraceLookupActor(ui32 nodeId, ui64 tabletId, const TActorId& sender, ui32 timeout) - : NodeId(nodeId) - , TabletId(tabletId) - , Sender(sender) - , Timeout(timeout) - {} - - void Bootstrap(const TActorContext& ctx) { - if (NodeId) { - // Direct request - SendRequest(NodeId, ctx); - Become(&TThis::StateTraceLookupRequested); - } else { - // Broadcast request + : NodeId(nodeId) + , TabletId(tabletId) + , Sender(sender) + , Timeout(timeout) + {} + + void Bootstrap(const TActorContext& ctx) { + if (NodeId) { + // Direct request + SendRequest(NodeId, ctx); + Become(&TThis::StateTraceLookupRequested); + } else { + // Broadcast request const TActorId nameserviceId = NActors::GetNameserviceActorId(); - ctx.Send(nameserviceId, new TEvInterconnect::TEvListNodes()); - Become(&TThis::StateNodeListRequested); - ctx.Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); - } - } - - STFUNC(StateNodeListRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvInterconnect::TEvNodesInfo, HandleNodeList); - CFunc(TEvents::TSystem::Wakeup, HandleTimeout); - } - } - - void HandleNodeList(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) { - for (const auto& ni : ev->Get()->Nodes) { - SendRequest(ni.NodeId, ctx); - } - if (Requested > 0) { - Become(&TThis::StateTraceLookupRequested); - } else { - ReplyAndDie(ctx); - } - } - - void SendRequest(ui32 nodeId, const TActorContext& ctx) { + ctx.Send(nameserviceId, new TEvInterconnect::TEvListNodes()); + Become(&TThis::StateNodeListRequested); + ctx.Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); + } + } + + STFUNC(StateNodeListRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvInterconnect::TEvNodesInfo, HandleNodeList); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + } + } + + void HandleNodeList(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx) { + for (const auto& ni : ev->Get()->Nodes) { + SendRequest(ni.NodeId, ctx); + } + if (Requested > 0) { + Become(&TThis::StateTraceLookupRequested); + } else { + ReplyAndDie(ctx); + } + } + + void SendRequest(ui32 nodeId, const TActorContext& ctx) { TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); THolder<TEvWhiteboard::TEvTraceLookupRequest> request = MakeHolder<TEvWhiteboard::TEvTraceLookupRequest>(); - auto& record = request->Record; - record.SetTabletID(TabletId); - ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, nodeId); - ++Requested; - TabletTraces[nodeId] = nullptr; - } - - STFUNC(StateTraceLookupRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvWhiteboard::TEvTraceLookupResponse, Handle); - HFunc(TEvents::TEvUndelivered, Undelivered); - HFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); - CFunc(TEvents::TSystem::Wakeup, HandleTimeout); - } - } - - void Handle(TEvWhiteboard::TEvTraceLookupResponse::TPtr& ev, const TActorContext& ctx) { - ui64 nodeId = ev.Get()->Cookie; + auto& record = request->Record; + record.SetTabletID(TabletId); + ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, nodeId); + ++Requested; + TabletTraces[nodeId] = nullptr; + } + + STFUNC(StateTraceLookupRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvWhiteboard::TEvTraceLookupResponse, Handle); + HFunc(TEvents::TEvUndelivered, Undelivered); + HFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + } + } + + void Handle(TEvWhiteboard::TEvTraceLookupResponse::TPtr& ev, const TActorContext& ctx) { + ui64 nodeId = ev.Get()->Cookie; TabletTraces[nodeId] = THolder<TReplyStatus>(new TReplyStatus({ TReplyStatus::OK, THolder<TEvWhiteboard::TEvTraceLookupResponse>(ev->Release()) })); - NodeTabletInfoReceived(ctx); - } - - void NodeTabletInfoReceived(const TActorContext& ctx) { - ++Received; - if (Received >= Requested) { - ReplyAndDie(ctx); - } - } - - void BadResponseReceived(ui32 nodeId, const TActorContext& ctx, TReplyStatus::EStatus status) { - if (TabletTraces[nodeId] == nullptr) { + NodeTabletInfoReceived(ctx); + } + + void NodeTabletInfoReceived(const TActorContext& ctx) { + ++Received; + if (Received >= Requested) { + ReplyAndDie(ctx); + } + } + + void BadResponseReceived(ui32 nodeId, const TActorContext& ctx, TReplyStatus::EStatus status) { + if (TabletTraces[nodeId] == nullptr) { TabletTraces[nodeId] = THolder<TReplyStatus>(new TReplyStatus({ status, nullptr })); - NodeTabletInfoReceived(ctx); - } - } - - void Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { - BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::UNDELIVERED); - } - - void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { - BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::DISCONNECTED); - } - - void ReplyAndDie(const TActorContext& ctx) { - TStringStream str; - TList<ui32> nodesWithNoTraces; - THashMap<TReplyStatus::EStatus, TList<ui32>> failNodes; - - HTML(str) { - NTracing::NHttp::OutputStaticPart(str); - str << "<br>"; - H3() { - str << "Traces for Tablet " << "<a href=\"tablet?iTabletID=" << TabletId << "\">" - << TabletId << "</a>" << ":"; - } - for (auto& nodeTabletTracesPair : TabletTraces) { - ui32 nodeId = nodeTabletTracesPair.first; - if (nodeTabletTracesPair.second->Status == TReplyStatus::OK) { - auto& nodeTabletTraces = nodeTabletTracesPair.second->Response.Get()->Record; - size_t traceIDsSize = nodeTabletTraces.TraceIDsSize(); - if (traceIDsSize) { - H4() { - str << "Node " << nodeId << ":"; - } - // Sorting traces by Id - TVector<NTracing::TTraceID> traceIDs; - traceIDs.reserve(traceIDsSize); - for (ui32 i = 0; i < traceIDsSize; ++i) { - traceIDs.push_back(NTracing::TraceIDFromTraceID(nodeTabletTraces.GetTraceIDs(i))); - } - Sort(traceIDs.begin(), traceIDs.end()); - for (auto& traceID : traceIDs) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << "<a href=\"tablet?NodeID=" << nodeId - << "&TabletID=" << TabletId - << "&RandomID=" << traceID.RandomID - << "&CreationTime=" << traceID.CreationTime - << "\">" << traceID.RandomID << ":" << traceID.CreationTime << "</a>" - << " - created at " - << TInstant::MicroSeconds(traceID.CreationTime).ToRfc822StringLocal(); - } - } - } - } else { - nodesWithNoTraces.push_back(nodeId); - } - } else { - failNodes[nodeTabletTracesPair.second->Status].push_back(nodeId); - } - } - size_t emptyNodes = nodesWithNoTraces.size(); - if (emptyNodes) { - str << "<br>"; - H4() { - str << "No traces found on " << emptyNodes << " node" << (emptyNodes == 1 ? ":" : "s:"); - } - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - for (ui32 nodeId : nodesWithNoTraces) { - if (nodeId != *nodesWithNoTraces.begin()) { - str << ", "; - } - str << nodeId; - } - } - } - } - - for (auto& failNodesPair : failNodes) { - size_t failedNodes = failNodesPair.second.size(); - if (failedNodes) { - str << "<br>"; - H4() { - str << "<font color=\"red\">" - << TReplyStatus::StatusDescription(failNodesPair.first) - << "</font> on " << failedNodes - << " node" << (failedNodes == 1 ? ":" : "s:"); - } - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - for (ui32 nodeId : failNodesPair.second) { - if (nodeId != *failNodesPair.second.begin()) { - str << ", "; - } - str << nodeId; - } - } - } - } - } - AddTraceLookupForm(str); - } - Notify(ctx, str.Str()); - Die(ctx); - } - - void Notify(const TActorContext &ctx, const TString& html) { - ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); - } - - void HandleTimeout(const TActorContext &ctx) { - for (auto& nodeTabletTracesPair : TabletTraces) { - if (!nodeTabletTracesPair.second) { + NodeTabletInfoReceived(ctx); + } + } + + void Undelivered(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { + BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::UNDELIVERED); + } + + void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &ev, const TActorContext &ctx) { + BadResponseReceived(ev.Get()->Cookie, ctx, TReplyStatus::DISCONNECTED); + } + + void ReplyAndDie(const TActorContext& ctx) { + TStringStream str; + TList<ui32> nodesWithNoTraces; + THashMap<TReplyStatus::EStatus, TList<ui32>> failNodes; + + HTML(str) { + NTracing::NHttp::OutputStaticPart(str); + str << "<br>"; + H3() { + str << "Traces for Tablet " << "<a href=\"tablet?iTabletID=" << TabletId << "\">" + << TabletId << "</a>" << ":"; + } + for (auto& nodeTabletTracesPair : TabletTraces) { + ui32 nodeId = nodeTabletTracesPair.first; + if (nodeTabletTracesPair.second->Status == TReplyStatus::OK) { + auto& nodeTabletTraces = nodeTabletTracesPair.second->Response.Get()->Record; + size_t traceIDsSize = nodeTabletTraces.TraceIDsSize(); + if (traceIDsSize) { + H4() { + str << "Node " << nodeId << ":"; + } + // Sorting traces by Id + TVector<NTracing::TTraceID> traceIDs; + traceIDs.reserve(traceIDsSize); + for (ui32 i = 0; i < traceIDsSize; ++i) { + traceIDs.push_back(NTracing::TraceIDFromTraceID(nodeTabletTraces.GetTraceIDs(i))); + } + Sort(traceIDs.begin(), traceIDs.end()); + for (auto& traceID : traceIDs) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << "<a href=\"tablet?NodeID=" << nodeId + << "&TabletID=" << TabletId + << "&RandomID=" << traceID.RandomID + << "&CreationTime=" << traceID.CreationTime + << "\">" << traceID.RandomID << ":" << traceID.CreationTime << "</a>" + << " - created at " + << TInstant::MicroSeconds(traceID.CreationTime).ToRfc822StringLocal(); + } + } + } + } else { + nodesWithNoTraces.push_back(nodeId); + } + } else { + failNodes[nodeTabletTracesPair.second->Status].push_back(nodeId); + } + } + size_t emptyNodes = nodesWithNoTraces.size(); + if (emptyNodes) { + str << "<br>"; + H4() { + str << "No traces found on " << emptyNodes << " node" << (emptyNodes == 1 ? ":" : "s:"); + } + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + for (ui32 nodeId : nodesWithNoTraces) { + if (nodeId != *nodesWithNoTraces.begin()) { + str << ", "; + } + str << nodeId; + } + } + } + } + + for (auto& failNodesPair : failNodes) { + size_t failedNodes = failNodesPair.second.size(); + if (failedNodes) { + str << "<br>"; + H4() { + str << "<font color=\"red\">" + << TReplyStatus::StatusDescription(failNodesPair.first) + << "</font> on " << failedNodes + << " node" << (failedNodes == 1 ? ":" : "s:"); + } + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + for (ui32 nodeId : failNodesPair.second) { + if (nodeId != *failNodesPair.second.begin()) { + str << ", "; + } + str << nodeId; + } + } + } + } + } + AddTraceLookupForm(str); + } + Notify(ctx, str.Str()); + Die(ctx); + } + + void Notify(const TActorContext &ctx, const TString& html) { + ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); + } + + void HandleTimeout(const TActorContext &ctx) { + for (auto& nodeTabletTracesPair : TabletTraces) { + if (!nodeTabletTracesPair.second) { nodeTabletTracesPair.second = THolder<TReplyStatus>(new TReplyStatus({ TReplyStatus::TIMEOUT, nullptr })); - } - } - ReplyAndDie(ctx); - } - -private: - const ui32 NodeId; - const ui64 TabletId; + } + } + ReplyAndDie(ctx); + } + +private: + const ui32 NodeId; + const ui64 TabletId; const TActorId Sender; - ui32 Timeout; - ui32 Requested = 0; - ui32 Received = 0; - THashMap<ui32, THolder<TReplyStatus>> TabletTraces; -}; - -class TTraceRequestActor : public TActorBootstrapped<TTraceRequestActor> { -public: + ui32 Timeout; + ui32 Requested = 0; + ui32 Received = 0; + THashMap<ui32, THolder<TReplyStatus>> TabletTraces; +}; + +class TTraceRequestActor : public TActorBootstrapped<TTraceRequestActor> { +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TRACE_REQUEST_ACTOR; - } - + } + TTraceRequestActor(const NTracing::TTraceInfo& traceInfo, const TActorId& sender) - : TraceInfo(traceInfo) - , Sender(sender) - {} - - void Bootstrap(const TActorContext& ctx) { + : TraceInfo(traceInfo) + , Sender(sender) + {} + + void Bootstrap(const TActorContext& ctx) { TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(TraceInfo.NodeId); THolder<TEvWhiteboard::TEvTraceRequest> request = MakeHolder<TEvWhiteboard::TEvTraceRequest>(); - auto& record = request->Record; - record.SetTabletID(TraceInfo.TabletId); - NTracing::TraceIDFromTraceID(TraceInfo.TraceId, record.MutableTraceID()); - record.SetMode(TraceInfo.TimestampInfo.Mode); - record.SetPrecision(TraceInfo.TimestampInfo.Precision); - record.SetTabletID(TraceInfo.TabletId); - ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, TraceInfo.NodeId); - Become(&TThis::StateTraceRequested); - } - - STFUNC(StateTraceRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvWhiteboard::TEvTraceResponse, Handle); - CFunc(TEvents::TEvUndelivered::EventType, Undelivered); - CFunc(TEvInterconnect::TEvNodeDisconnected::EventType, Disconnected); - } - } - - void Handle(TEvWhiteboard::TEvTraceResponse::TPtr& ev, const TActorContext& ctx) { - TStringStream str; - try { - HTML(str) { - NTracing::NHttp::OutputStaticPart(str); - H4() { - str << "Trace " << TraceInfo.TraceId << "<br>at Tablet " - << "<a href=\"tablet?iTabletID=" << TraceInfo.TabletId << "\">" << TraceInfo.TabletId << "</a>" - << " on Node " << TraceInfo.NodeId << ":"; - } - auto& record = ev->Get()->Record; - auto& tabletTrace = record.GetTrace(); - str << tabletTrace; - AddTraceLookupForm(str); - } - } - catch (yexception&) { - Notify(ctx, "Error rendering trace"); - Die(ctx); - return; - } - Notify(ctx, str.Str()); - Die(ctx); - } - - void Undelivered(const TActorContext &ctx) { - Notify(ctx, "Request is undelivered"); - Die(ctx); - } - - void Disconnected(const TActorContext &ctx) { - Notify(ctx, "Interconnect: Node Disconnected"); - Die(ctx); - } - - void Notify(const TActorContext &ctx, const TString& html) { - ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); - } - -private: - const NTracing::TTraceInfo TraceInfo; + auto& record = request->Record; + record.SetTabletID(TraceInfo.TabletId); + NTracing::TraceIDFromTraceID(TraceInfo.TraceId, record.MutableTraceID()); + record.SetMode(TraceInfo.TimestampInfo.Mode); + record.SetPrecision(TraceInfo.TimestampInfo.Precision); + record.SetTabletID(TraceInfo.TabletId); + ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, TraceInfo.NodeId); + Become(&TThis::StateTraceRequested); + } + + STFUNC(StateTraceRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvWhiteboard::TEvTraceResponse, Handle); + CFunc(TEvents::TEvUndelivered::EventType, Undelivered); + CFunc(TEvInterconnect::TEvNodeDisconnected::EventType, Disconnected); + } + } + + void Handle(TEvWhiteboard::TEvTraceResponse::TPtr& ev, const TActorContext& ctx) { + TStringStream str; + try { + HTML(str) { + NTracing::NHttp::OutputStaticPart(str); + H4() { + str << "Trace " << TraceInfo.TraceId << "<br>at Tablet " + << "<a href=\"tablet?iTabletID=" << TraceInfo.TabletId << "\">" << TraceInfo.TabletId << "</a>" + << " on Node " << TraceInfo.NodeId << ":"; + } + auto& record = ev->Get()->Record; + auto& tabletTrace = record.GetTrace(); + str << tabletTrace; + AddTraceLookupForm(str); + } + } + catch (yexception&) { + Notify(ctx, "Error rendering trace"); + Die(ctx); + return; + } + Notify(ctx, str.Str()); + Die(ctx); + } + + void Undelivered(const TActorContext &ctx) { + Notify(ctx, "Request is undelivered"); + Die(ctx); + } + + void Disconnected(const TActorContext &ctx) { + Notify(ctx, "Interconnect: Node Disconnected"); + Die(ctx); + } + + void Notify(const TActorContext &ctx, const TString& html) { + ctx.Send(Sender, new NMon::TEvHttpInfoRes(html)); + } + +private: + const NTracing::TTraceInfo TraceInfo; const TActorId Sender; -}; - -class TSignalBodyRequestActor : public TActorBootstrapped<TSignalBodyRequestActor> { -public: +}; + +class TSignalBodyRequestActor : public TActorBootstrapped<TSignalBodyRequestActor> { +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::SIGNAL_BODY_REQUEST_ACTOR; - } - + } + TSignalBodyRequestActor(const NTracing::TTraceInfo& traceInfo, const TString& signalId, const TActorId& sender) - : TraceInfo(traceInfo) - , SignalId(signalId) - , Sender(sender) - {} - - void Bootstrap(const TActorContext& ctx) { + : TraceInfo(traceInfo) + , SignalId(signalId) + , Sender(sender) + {} + + void Bootstrap(const TActorContext& ctx) { TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(TraceInfo.NodeId); THolder<TEvWhiteboard::TEvSignalBodyRequest> request = MakeHolder<TEvWhiteboard::TEvSignalBodyRequest>(); - auto& record = request->Record; - record.SetTabletID(TraceInfo.TabletId); - NTracing::TraceIDFromTraceID(TraceInfo.TraceId, record.MutableTraceID()); - record.SetMode(TraceInfo.TimestampInfo.Mode); - record.SetPrecision(TraceInfo.TimestampInfo.Precision); - record.SetSignalID(SignalId); - ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, TraceInfo.NodeId); - Become(&TThis::StateTraceRequested); - } - - STFUNC(StateTraceRequested) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvWhiteboard::TEvSignalBodyResponse, Handle); - CFunc(TEvents::TEvUndelivered::EventType, Undelivered); - CFunc(TEvInterconnect::TEvNodeDisconnected::EventType, Disconnected); - } - } - - void Handle(TEvWhiteboard::TEvSignalBodyResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - auto& signalBody = record.GetSignalBody(); - TStringStream str; - str << NMonitoring::HTTPOKTEXT; - HTML(str) { - DIV_CLASS("tab-left") { - str << signalBody; - } - } - Notify(ctx, str.Str()); - Die(ctx); - } - - void Undelivered(const TActorContext &ctx) { - Notify(ctx, "Request is undelivered"); - Die(ctx); - } - - void Disconnected(const TActorContext &ctx) { - Notify(ctx, "Interconnect: Node Disconnected"); - Die(ctx); - } - - void Notify(const TActorContext &ctx, const TString& html) { - ctx.Send(Sender, new NMon::TEvHttpInfoRes(html, 0, NMon::IEvHttpInfoRes::Custom)); - } - -private: - const NTracing::TTraceInfo TraceInfo; - const TString SignalId; + auto& record = request->Record; + record.SetTabletID(TraceInfo.TabletId); + NTracing::TraceIDFromTraceID(TraceInfo.TraceId, record.MutableTraceID()); + record.SetMode(TraceInfo.TimestampInfo.Mode); + record.SetPrecision(TraceInfo.TimestampInfo.Precision); + record.SetSignalID(SignalId); + ctx.Send(whiteboardServiceId, request.Release(), IEventHandle::FlagTrackDelivery, TraceInfo.NodeId); + Become(&TThis::StateTraceRequested); + } + + STFUNC(StateTraceRequested) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvWhiteboard::TEvSignalBodyResponse, Handle); + CFunc(TEvents::TEvUndelivered::EventType, Undelivered); + CFunc(TEvInterconnect::TEvNodeDisconnected::EventType, Disconnected); + } + } + + void Handle(TEvWhiteboard::TEvSignalBodyResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + auto& signalBody = record.GetSignalBody(); + TStringStream str; + str << NMonitoring::HTTPOKTEXT; + HTML(str) { + DIV_CLASS("tab-left") { + str << signalBody; + } + } + Notify(ctx, str.Str()); + Die(ctx); + } + + void Undelivered(const TActorContext &ctx) { + Notify(ctx, "Request is undelivered"); + Die(ctx); + } + + void Disconnected(const TActorContext &ctx) { + Notify(ctx, "Interconnect: Node Disconnected"); + Die(ctx); + } + + void Notify(const TActorContext &ctx, const TString& html) { + ctx.Send(Sender, new NMon::TEvHttpInfoRes(html, 0, NMon::IEvHttpInfoRes::Custom)); + } + +private: + const NTracing::TTraceInfo TraceInfo; + const TString SignalId; const TActorId Sender; -}; - -ui64 TryParseTabletId(TStringBuf tabletIdParam) { - if (tabletIdParam.StartsWith("0x")) { - return IntFromString<ui64, 16>(tabletIdParam.substr(2)); - } else { - return FromString<ui64>(tabletIdParam); - } -} - -void RenderTabletPage(NMon::TEvHttpInfo::TPtr ev, const TActorContext &ctx, ui64 tabletId) { - TStringStream str; - HTML(str) { - NTracing::NHttp::OutputStaticPart(str); - H3() { - str << "TabletID: " << tabletId; - } - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { str << "<a href=\"tablets?TabletID=" << tabletId << "\">Tablet details</a>"; } - } - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { str << "<a href=\"tablets?SsId=" << tabletId << "\">State Storage</a>"; } - } - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << "<a href=\"tablet?TabletID=" << tabletId << "\">Introspection traces</a>"; - } - } - AddTraceLookupForm(str); - } - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); -} - -} - -class TTabletInfoActor : public TActorBootstrapped<TTabletInfoActor> { -public: +}; + +ui64 TryParseTabletId(TStringBuf tabletIdParam) { + if (tabletIdParam.StartsWith("0x")) { + return IntFromString<ui64, 16>(tabletIdParam.substr(2)); + } else { + return FromString<ui64>(tabletIdParam); + } +} + +void RenderTabletPage(NMon::TEvHttpInfo::TPtr ev, const TActorContext &ctx, ui64 tabletId) { + TStringStream str; + HTML(str) { + NTracing::NHttp::OutputStaticPart(str); + H3() { + str << "TabletID: " << tabletId; + } + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { str << "<a href=\"tablets?TabletID=" << tabletId << "\">Tablet details</a>"; } + } + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { str << "<a href=\"tablets?SsId=" << tabletId << "\">State Storage</a>"; } + } + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << "<a href=\"tablet?TabletID=" << tabletId << "\">Introspection traces</a>"; + } + } + AddTraceLookupForm(str); + } + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); +} + +} + +class TTabletInfoActor : public TActorBootstrapped<TTabletInfoActor> { +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TABLET_INFO; - } - void Bootstrap(const TActorContext &ctx); - STFUNC(StateWork); - -private: - void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx); -}; - -void TTabletInfoActor::Bootstrap(const TActorContext &ctx) { - Become(&TThis::StateWork); - - TMon* mon = AppData(ctx)->Mon; - - if (mon) { - mon->RegisterActorPage(nullptr, "tablet", "Tablet boot tracing", false, ctx.ExecutorThread.ActorSystem, ctx.SelfID); - } -} - -bool HasProperty(const TCgiParameters& cgi, const TString& name) { - return cgi.Has(name) && cgi.Get(name).length(); -} - -void TTabletInfoActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { - if (!ev) { - return; - } - NMon::TEvHttpInfo* msg = ev->Get(); + } + void Bootstrap(const TActorContext &ctx); + STFUNC(StateWork); + +private: + void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx); +}; + +void TTabletInfoActor::Bootstrap(const TActorContext &ctx) { + Become(&TThis::StateWork); + + TMon* mon = AppData(ctx)->Mon; + + if (mon) { + mon->RegisterActorPage(nullptr, "tablet", "Tablet boot tracing", false, ctx.ExecutorThread.ActorSystem, ctx.SelfID); + } +} + +bool HasProperty(const TCgiParameters& cgi, const TString& name) { + return cgi.Has(name) && cgi.Get(name).length(); +} + +void TTabletInfoActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { + if (!ev) { + return; + } + NMon::TEvHttpInfo* msg = ev->Get(); const TCgiParameters& cgi = msg->Request.GetParams(); - ui32 timeout = FromStringWithDefault<ui32>(cgi.Get("timeout"), 10000); - - bool hasNodeIdParam = HasProperty(cgi, "NodeID"); - bool hasTabletIdParam = HasProperty(cgi, "TabletID"); - bool hasTraceIdParam = HasProperty(cgi, "RandomID") && HasProperty(cgi, "CreationTime"); - - if (cgi.Has("SignalID")) { - // SignalID => Signal body direct request - const TString& signalID = cgi.Get("SignalID"); - if (!hasNodeIdParam || !hasTabletIdParam || !hasTraceIdParam || !signalID.length()) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes( - "Need NodeID, TabletID, TraceID and signal address to search for the signal body." - )); - return; - } - const TString& nodeIdParam = cgi.Get("NodeID"); - const TString& tabletIdParam = cgi.Get("TabletID"); - const TString& randomIdParam = cgi.Get("RandomID"); - const TString& creationTimeParam = cgi.Get("CreationTime"); - try { - NTracing::TTraceInfo traceInfo = { - FromString<ui32>(nodeIdParam), - TryParseTabletId(tabletIdParam), - { - FromString<ui64>(randomIdParam), - FromString<ui64>(creationTimeParam) - }, - NTracing::TTimestampInfo( - HasProperty(cgi, "TMode") ? - static_cast<NTracing::TTimestampInfo::EMode>(FromString<ui32>(cgi.Get("TMode"))) - : NTracing::TTimestampInfo::ModeDefault, - HasProperty(cgi, "TPrecision") ? - static_cast<NTracing::TTimestampInfo::EPrecision>(FromString<ui32>(cgi.Get("TPrecision"))) - : NTracing::TTimestampInfo::PrecisionDefault - ) - }; - ctx.ExecutorThread.RegisterActor( - new TSignalBodyRequestActor(traceInfo, signalID, ev->Sender) - ); - return; - } - catch (yexception&) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); - return; - } - } - - if (hasTraceIdParam) { - if (!hasNodeIdParam || !hasTabletIdParam) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Need NodeID and TabletID to search for the trace.")); - return; - } - // NodeID && TabletID && TraceID => Direct trace request - const TString& nodeIdParam = cgi.Get("NodeID"); - const TString& tabletIdParam = cgi.Get("TabletID"); - const TString& randomIdParam = cgi.Get("RandomID"); - const TString& creationTimeParam = cgi.Get("CreationTime"); - try { - NTracing::TTraceInfo traceInfo = { - FromString<ui32>(nodeIdParam), - TryParseTabletId(tabletIdParam), - { - FromString<ui64>(randomIdParam), - FromString<ui64>(creationTimeParam) - }, - NTracing::TTimestampInfo( - HasProperty(cgi, "TMode") ? - static_cast<NTracing::TTimestampInfo::EMode>(FromString<ui32>(cgi.Get("TMode"))) - : NTracing::TTimestampInfo::ModeDefault, - HasProperty(cgi, "TPrecision") ? - static_cast<NTracing::TTimestampInfo::EPrecision>(FromString<ui32>(cgi.Get("TPrecision"))) - : NTracing::TTimestampInfo::PrecisionDefault - ) - }; - ctx.ExecutorThread.RegisterActor( - new TTraceRequestActor(traceInfo, ev->Sender) - ); - return; - } - catch (yexception&) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); - return; - } - } - - if (hasTabletIdParam) { - // TabletID && !TraceID => TraceLookup - const TString& tabletIdParam = cgi.Get("TabletID"); - try { - ui32 nodeId = hasNodeIdParam ? FromString<ui32>(cgi.Get("NodeID")) : 0; - ui64 tabletId = TryParseTabletId(tabletIdParam); - ctx.ExecutorThread.RegisterActor( - new TTraceLookupActor(nodeId, tabletId, ev->Sender, timeout) - ); - return; - } - catch (yexception&) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); - return; - } - } - - if (HasProperty(cgi, "iTabletID")) { - const TString& tabletIdParam = cgi.Get("iTabletID"); - ui64 tabletId; - try { - tabletId = TryParseTabletId(tabletIdParam); - } - catch (yexception&) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); - return; - } - RenderTabletPage(ev, ctx, tabletId); - return; - } - - // NodeID && !TabletID && !TraceID => TabletLookup - if (hasNodeIdParam) { - try { - ui32 nodeId = FromString<ui32>(cgi.Get("NodeID")); - ctx.ExecutorThread.RegisterActor( - new TTabletLookupActor(nodeId, ev->Sender, timeout) - ); - return; - } - catch (yexception&) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing NodeId")); - return; - } - } - TStringStream str; - - HTML(str) { - NTracing::NHttp::OutputStaticPart(str); - AddTabletLookupForm(str); - AddTraceLookupForm(str); - } - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); -} - -STFUNC(TTabletInfoActor::StateWork) { - switch (ev->GetTypeRewrite()) { - HFunc(NMon::TEvHttpInfo, Handle); - // HFunc(TEvents::TEvPoisonPill, Handle); // we do not need PoisonPill for the actor - } -} - -IActor* CreateTabletInfo() { - return new TTabletInfoActor(); -} - -} } // end of the NKikimr::NTabletInfo namespace + ui32 timeout = FromStringWithDefault<ui32>(cgi.Get("timeout"), 10000); + + bool hasNodeIdParam = HasProperty(cgi, "NodeID"); + bool hasTabletIdParam = HasProperty(cgi, "TabletID"); + bool hasTraceIdParam = HasProperty(cgi, "RandomID") && HasProperty(cgi, "CreationTime"); + + if (cgi.Has("SignalID")) { + // SignalID => Signal body direct request + const TString& signalID = cgi.Get("SignalID"); + if (!hasNodeIdParam || !hasTabletIdParam || !hasTraceIdParam || !signalID.length()) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes( + "Need NodeID, TabletID, TraceID and signal address to search for the signal body." + )); + return; + } + const TString& nodeIdParam = cgi.Get("NodeID"); + const TString& tabletIdParam = cgi.Get("TabletID"); + const TString& randomIdParam = cgi.Get("RandomID"); + const TString& creationTimeParam = cgi.Get("CreationTime"); + try { + NTracing::TTraceInfo traceInfo = { + FromString<ui32>(nodeIdParam), + TryParseTabletId(tabletIdParam), + { + FromString<ui64>(randomIdParam), + FromString<ui64>(creationTimeParam) + }, + NTracing::TTimestampInfo( + HasProperty(cgi, "TMode") ? + static_cast<NTracing::TTimestampInfo::EMode>(FromString<ui32>(cgi.Get("TMode"))) + : NTracing::TTimestampInfo::ModeDefault, + HasProperty(cgi, "TPrecision") ? + static_cast<NTracing::TTimestampInfo::EPrecision>(FromString<ui32>(cgi.Get("TPrecision"))) + : NTracing::TTimestampInfo::PrecisionDefault + ) + }; + ctx.ExecutorThread.RegisterActor( + new TSignalBodyRequestActor(traceInfo, signalID, ev->Sender) + ); + return; + } + catch (yexception&) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); + return; + } + } + + if (hasTraceIdParam) { + if (!hasNodeIdParam || !hasTabletIdParam) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Need NodeID and TabletID to search for the trace.")); + return; + } + // NodeID && TabletID && TraceID => Direct trace request + const TString& nodeIdParam = cgi.Get("NodeID"); + const TString& tabletIdParam = cgi.Get("TabletID"); + const TString& randomIdParam = cgi.Get("RandomID"); + const TString& creationTimeParam = cgi.Get("CreationTime"); + try { + NTracing::TTraceInfo traceInfo = { + FromString<ui32>(nodeIdParam), + TryParseTabletId(tabletIdParam), + { + FromString<ui64>(randomIdParam), + FromString<ui64>(creationTimeParam) + }, + NTracing::TTimestampInfo( + HasProperty(cgi, "TMode") ? + static_cast<NTracing::TTimestampInfo::EMode>(FromString<ui32>(cgi.Get("TMode"))) + : NTracing::TTimestampInfo::ModeDefault, + HasProperty(cgi, "TPrecision") ? + static_cast<NTracing::TTimestampInfo::EPrecision>(FromString<ui32>(cgi.Get("TPrecision"))) + : NTracing::TTimestampInfo::PrecisionDefault + ) + }; + ctx.ExecutorThread.RegisterActor( + new TTraceRequestActor(traceInfo, ev->Sender) + ); + return; + } + catch (yexception&) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); + return; + } + } + + if (hasTabletIdParam) { + // TabletID && !TraceID => TraceLookup + const TString& tabletIdParam = cgi.Get("TabletID"); + try { + ui32 nodeId = hasNodeIdParam ? FromString<ui32>(cgi.Get("NodeID")) : 0; + ui64 tabletId = TryParseTabletId(tabletIdParam); + ctx.ExecutorThread.RegisterActor( + new TTraceLookupActor(nodeId, tabletId, ev->Sender, timeout) + ); + return; + } + catch (yexception&) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); + return; + } + } + + if (HasProperty(cgi, "iTabletID")) { + const TString& tabletIdParam = cgi.Get("iTabletID"); + ui64 tabletId; + try { + tabletId = TryParseTabletId(tabletIdParam); + } + catch (yexception&) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing IDs")); + return; + } + RenderTabletPage(ev, ctx, tabletId); + return; + } + + // NodeID && !TabletID && !TraceID => TabletLookup + if (hasNodeIdParam) { + try { + ui32 nodeId = FromString<ui32>(cgi.Get("NodeID")); + ctx.ExecutorThread.RegisterActor( + new TTabletLookupActor(nodeId, ev->Sender, timeout) + ); + return; + } + catch (yexception&) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Error parsing NodeId")); + return; + } + } + TStringStream str; + + HTML(str) { + NTracing::NHttp::OutputStaticPart(str); + AddTabletLookupForm(str); + AddTraceLookupForm(str); + } + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); +} + +STFUNC(TTabletInfoActor::StateWork) { + switch (ev->GetTypeRewrite()) { + HFunc(NMon::TEvHttpInfo, Handle); + // HFunc(TEvents::TEvPoisonPill, Handle); // we do not need PoisonPill for the actor + } +} + +IActor* CreateTabletInfo() { + return new TTabletInfoActor(); +} + +} } // end of the NKikimr::NTabletInfo namespace diff --git a/ydb/core/tracing/tablet_info.h b/ydb/core/tracing/tablet_info.h index 8c7d6f3938..c3ebf10225 100644 --- a/ydb/core/tracing/tablet_info.h +++ b/ydb/core/tracing/tablet_info.h @@ -1,18 +1,18 @@ -#pragma once +#pragma once #include <ydb/core/base/defs.h> #include <library/cpp/actors/core/defs.h> #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/event.h> - -namespace NKikimr { -namespace NTabletInfo { - + +namespace NKikimr { +namespace NTabletInfo { + inline TActorId MakeTabletInfoID(ui32 node = 0) { - char x[12] = {'t','a','b','l','e','t','i','n','f','o','r','m'}; + char x[12] = {'t','a','b','l','e','t','i','n','f','o','r','m'}; return TActorId(node, TStringBuf(x, 12)); -} - -IActor* CreateTabletInfo(); - -} } // end of the NKikimr::NTabletInfo namespace - +} + +IActor* CreateTabletInfo(); + +} } // end of the NKikimr::NTabletInfo namespace + diff --git a/ydb/core/tracing/trace.cpp b/ydb/core/tracing/trace.cpp index d84c57223c..e243c047d7 100644 --- a/ydb/core/tracing/trace.cpp +++ b/ydb/core/tracing/trace.cpp @@ -1,195 +1,195 @@ -#include "trace.h" -#include "http.h" - +#include "trace.h" +#include "http.h" + #include <ydb/core/protos/tracing.pb.h> #include <util/string/cast.h> -namespace NKikimr { -namespace NTracing { - -TTrace::TTrace(ITrace::EType type, ITrace* parent) - : Type(type) - , SelfID(TTraceID::GenerateNew()) - , ParentID(parent ? parent->GetSelfID() : SelfID) - , RootID(parent ? parent->GetRootID() : SelfID) -{} - -TTraceID GetPbTraceID(const NKikimrTracing::TTraceID& proto) { - return TTraceID(proto.GetRandomID(), proto.GetCreationTime()); -} - -TTrace::TTrace(const TString& serializedTrace) { - NKikimrTracing::TTraceSignal pbTraceSignal; +namespace NKikimr { +namespace NTracing { + +TTrace::TTrace(ITrace::EType type, ITrace* parent) + : Type(type) + , SelfID(TTraceID::GenerateNew()) + , ParentID(parent ? parent->GetSelfID() : SelfID) + , RootID(parent ? parent->GetRootID() : SelfID) +{} + +TTraceID GetPbTraceID(const NKikimrTracing::TTraceID& proto) { + return TTraceID(proto.GetRandomID(), proto.GetCreationTime()); +} + +TTrace::TTrace(const TString& serializedTrace) { + NKikimrTracing::TTraceSignal pbTraceSignal; Y_PROTOBUF_SUPPRESS_NODISCARD pbTraceSignal.ParseFromString(serializedTrace); - Type = static_cast<ITrace::EType>(pbTraceSignal.GetType()); - SelfID = GetPbTraceID(pbTraceSignal.GetSelfID()); - ParentID = GetPbTraceID(pbTraceSignal.GetParentID()); - RootID = GetPbTraceID(pbTraceSignal.GetRootID()); - for (size_t i = 0; i < pbTraceSignal.SignalsSize(); ++i) { - auto& subSignal = pbTraceSignal.signals(i); - Signals.push_back({ subSignal.GetType(), std::move(subSignal.GetSignal()) }); - } -} - -ITrace* TTrace::CreateTrace(ITrace::EType type) { - return new TTrace(type, this); -} - -bool TTrace::Attach(THolder<ITraceSignal> signal) { - TString str; - bool result = signal->SerializeToString(str); - Signals.push_back({ signal->GetType(), std::move(str) }); - return result; -} - -TTraceID TTrace::GetSelfID() const { - return SelfID; -} - -TTraceID TTrace::GetParentID() const { - return ParentID; -} - -TTraceID TTrace::GetRootID() const { - return RootID; -} - -ui64 TTrace::GetSize() const { - ui64 result = sizeof(TTrace); - for (auto& serializedSignal : Signals) - result += serializedSignal.Signal.length(); - return result; -} - -void TTrace::OutHtml(TStringStream& str, TTraceInfo& traceInfo) const { - NHttp::OutputTimeDropdown(str,traceInfo); - OutHtml(str, traceInfo.TimestampInfo, []() { return TString(); }); -} - -void TTrace::OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const { - TSignalFactory& signalFactory = TSignalFactory::Instance(); - ui32 i = 0; - TTimestampData tsData(tsInfo, TInstant::MicroSeconds(SelfID.CreationTime)); - for (auto& serializedSignal : Signals) { + Type = static_cast<ITrace::EType>(pbTraceSignal.GetType()); + SelfID = GetPbTraceID(pbTraceSignal.GetSelfID()); + ParentID = GetPbTraceID(pbTraceSignal.GetParentID()); + RootID = GetPbTraceID(pbTraceSignal.GetRootID()); + for (size_t i = 0; i < pbTraceSignal.SignalsSize(); ++i) { + auto& subSignal = pbTraceSignal.signals(i); + Signals.push_back({ subSignal.GetType(), std::move(subSignal.GetSignal()) }); + } +} + +ITrace* TTrace::CreateTrace(ITrace::EType type) { + return new TTrace(type, this); +} + +bool TTrace::Attach(THolder<ITraceSignal> signal) { + TString str; + bool result = signal->SerializeToString(str); + Signals.push_back({ signal->GetType(), std::move(str) }); + return result; +} + +TTraceID TTrace::GetSelfID() const { + return SelfID; +} + +TTraceID TTrace::GetParentID() const { + return ParentID; +} + +TTraceID TTrace::GetRootID() const { + return RootID; +} + +ui64 TTrace::GetSize() const { + ui64 result = sizeof(TTrace); + for (auto& serializedSignal : Signals) + result += serializedSignal.Signal.length(); + return result; +} + +void TTrace::OutHtml(TStringStream& str, TTraceInfo& traceInfo) const { + NHttp::OutputTimeDropdown(str,traceInfo); + OutHtml(str, traceInfo.TimestampInfo, []() { return TString(); }); +} + +void TTrace::OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const { + TSignalFactory& signalFactory = TSignalFactory::Instance(); + ui32 i = 0; + TTimestampData tsData(tsInfo, TInstant::MicroSeconds(SelfID.CreationTime)); + for (auto& serializedSignal : Signals) { THolder<ITraceSignal> signal(signalFactory.Create(serializedSignal)); - if (signal) { - signal->OutHtmlHeader( - str, - tsData, - [&]() { - TString myId = getMyId(); - return myId.length() ? myId + "_" + ToString(i) : ToString(i); - } - ); - } else { - HTML(str) { - DIV_CLASS("row") { - DIV_CLASS("col-md-12") { - str << "<Unknown signal type:" << serializedSignal.Type << ">"; - } - } - } - } - ++i; - } -} - -void TTrace::OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix) const { - TSignalFactory& signalFactory = TSignalFactory::Instance(); - TTimestampData tsData(tsInfo, TInstant::MicroSeconds(SelfID.CreationTime)); - for (auto& serializedSignal : Signals) { + if (signal) { + signal->OutHtmlHeader( + str, + tsData, + [&]() { + TString myId = getMyId(); + return myId.length() ? myId + "_" + ToString(i) : ToString(i); + } + ); + } else { + HTML(str) { + DIV_CLASS("row") { + DIV_CLASS("col-md-12") { + str << "<Unknown signal type:" << serializedSignal.Type << ">"; + } + } + } + } + ++i; + } +} + +void TTrace::OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix) const { + TSignalFactory& signalFactory = TSignalFactory::Instance(); + TTimestampData tsData(tsInfo, TInstant::MicroSeconds(SelfID.CreationTime)); + for (auto& serializedSignal : Signals) { THolder<ITraceSignal> signal(signalFactory.Create(serializedSignal)); - if (signal) { - signal->OutText( - str, - tsData, - prefix - ); - } else { - str << prefix << "<Unknown signal type:" << serializedSignal.Type << ">" << Endl; - } - } -} - -ITrace::EType TTrace::GetType() const { - return Type; -} - -void SetPbTraceID(NKikimrTracing::TTraceID* proto, const TTraceID& traceId) { - proto->SetRandomID(traceId.RandomID); - proto->SetCreationTime(traceId.CreationTime); -} - -bool TTrace::SerializeToString(TString& str) const { - NKikimrTracing::TTraceSignal pbTraceSignal; - pbTraceSignal.SetType(Type); - SetPbTraceID(pbTraceSignal.MutableSelfID(), SelfID); - SetPbTraceID(pbTraceSignal.MutableParentID(), ParentID); - SetPbTraceID(pbTraceSignal.MutableRootID(), RootID); - auto& pbTraces = *pbTraceSignal.MutableSignals(); - for (auto& signal : Signals) { - auto& pbSignal = *pbTraces.Add(); - pbSignal.SetType(signal.Type); - pbSignal.SetSignal(signal.Signal); - } - return pbTraceSignal.SerializeToString(&str); -} - -void TTrace::OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) { - size_t prevPos = 0; - size_t currPos = signalId.find('_'); - TList<ui64> signalAddress; - while (currPos != TString::npos) { - signalAddress.push_back(FromString<ui64>(signalId.substr(prevPos, currPos))); - prevPos = currPos; - currPos = signalId.find('_', currPos + 1); - } - // Last one - signalAddress.push_back(FromString<ui64>(signalId.substr(prevPos, signalId.length()))); - // Start of recursion - OutSignalHtmlBody(str, tsInfo, []() { return TString(); }, signalAddress); -} - -void TTrace::OutSignalHtmlBody(TStringStream& str - , const TTimestampInfo& tsInfo - , std::function<TString()> getMyId - , TList<ui64>& signalAddress -) { - ui64 signalIndex = signalAddress.front(); - if (signalIndex >= Signals.size()) { - str << "Signal index is out of range. Trace has " << Signals.size() << " signals."; - return; - } - auto signalIterator = Signals.begin(); - for (ui64 i = 0; i < signalIndex; ++i) { - ++signalIterator; - } - signalAddress.pop_front(); - TSignalFactory& signalFactory = TSignalFactory::Instance(); + if (signal) { + signal->OutText( + str, + tsData, + prefix + ); + } else { + str << prefix << "<Unknown signal type:" << serializedSignal.Type << ">" << Endl; + } + } +} + +ITrace::EType TTrace::GetType() const { + return Type; +} + +void SetPbTraceID(NKikimrTracing::TTraceID* proto, const TTraceID& traceId) { + proto->SetRandomID(traceId.RandomID); + proto->SetCreationTime(traceId.CreationTime); +} + +bool TTrace::SerializeToString(TString& str) const { + NKikimrTracing::TTraceSignal pbTraceSignal; + pbTraceSignal.SetType(Type); + SetPbTraceID(pbTraceSignal.MutableSelfID(), SelfID); + SetPbTraceID(pbTraceSignal.MutableParentID(), ParentID); + SetPbTraceID(pbTraceSignal.MutableRootID(), RootID); + auto& pbTraces = *pbTraceSignal.MutableSignals(); + for (auto& signal : Signals) { + auto& pbSignal = *pbTraces.Add(); + pbSignal.SetType(signal.Type); + pbSignal.SetSignal(signal.Signal); + } + return pbTraceSignal.SerializeToString(&str); +} + +void TTrace::OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) { + size_t prevPos = 0; + size_t currPos = signalId.find('_'); + TList<ui64> signalAddress; + while (currPos != TString::npos) { + signalAddress.push_back(FromString<ui64>(signalId.substr(prevPos, currPos))); + prevPos = currPos; + currPos = signalId.find('_', currPos + 1); + } + // Last one + signalAddress.push_back(FromString<ui64>(signalId.substr(prevPos, signalId.length()))); + // Start of recursion + OutSignalHtmlBody(str, tsInfo, []() { return TString(); }, signalAddress); +} + +void TTrace::OutSignalHtmlBody(TStringStream& str + , const TTimestampInfo& tsInfo + , std::function<TString()> getMyId + , TList<ui64>& signalAddress +) { + ui64 signalIndex = signalAddress.front(); + if (signalIndex >= Signals.size()) { + str << "Signal index is out of range. Trace has " << Signals.size() << " signals."; + return; + } + auto signalIterator = Signals.begin(); + for (ui64 i = 0; i < signalIndex; ++i) { + ++signalIterator; + } + signalAddress.pop_front(); + TSignalFactory& signalFactory = TSignalFactory::Instance(); THolder<ITraceSignal> signal(signalFactory.Create(*signalIterator)); - if (signal) { - signal->OutHtmlBody( - str, - tsInfo, - [signalIndex, getMyId]() { - TString myId = getMyId(); - return myId.length() ? myId + "_" + ToString(signalIndex) : ToString(signalIndex); - }, - signalAddress - ); - } else { - str << "<Unknown signal type:" << signalIterator->Type << ">" << Endl; - } -} - -ITrace* CreateTrace(ITrace::EType type) { - return new TTrace(type, nullptr); -} - -ITrace* CreateTrace(const TString& serializedTrace) { - return new TTrace(serializedTrace); -} - -} -} + if (signal) { + signal->OutHtmlBody( + str, + tsInfo, + [signalIndex, getMyId]() { + TString myId = getMyId(); + return myId.length() ? myId + "_" + ToString(signalIndex) : ToString(signalIndex); + }, + signalAddress + ); + } else { + str << "<Unknown signal type:" << signalIterator->Type << ">" << Endl; + } +} + +ITrace* CreateTrace(ITrace::EType type) { + return new TTrace(type, nullptr); +} + +ITrace* CreateTrace(const TString& serializedTrace) { + return new TTrace(serializedTrace); +} + +} +} diff --git a/ydb/core/tracing/trace.h b/ydb/core/tracing/trace.h index aee9ac4f36..ca09071da0 100644 --- a/ydb/core/tracing/trace.h +++ b/ydb/core/tracing/trace.h @@ -1,36 +1,36 @@ -#pragma once +#pragma once #include <ydb/core/base/defs.h> #include <ydb/core/base/tracing.h> - -namespace NKikimr { -namespace NTracing { - -class TTrace : public ITrace { -public: - TTrace(ITrace::EType type, ITrace* parent); - TTrace(const TString& serializedTrace); - - ITrace* CreateTrace(ITrace::EType type) override; - bool Attach(THolder<ITraceSignal> signal) override; - TTraceID GetSelfID() const override; - TTraceID GetParentID() const override; - TTraceID GetRootID() const override; - ui64 GetSize() const override; - void OutHtml(TStringStream& str, TTraceInfo& traceInfo) const override; - void OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const override; - void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) override; - void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId, TList<ui64>& signalAddress) override; - void OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix) const override; - ITrace::EType GetType() const override; - bool SerializeToString(TString& str) const override; - -private: - ITrace::EType Type; - TTraceID SelfID; - TTraceID ParentID; - TTraceID RootID; - TList<TSerializedSignal> Signals; -}; - -} -} + +namespace NKikimr { +namespace NTracing { + +class TTrace : public ITrace { +public: + TTrace(ITrace::EType type, ITrace* parent); + TTrace(const TString& serializedTrace); + + ITrace* CreateTrace(ITrace::EType type) override; + bool Attach(THolder<ITraceSignal> signal) override; + TTraceID GetSelfID() const override; + TTraceID GetParentID() const override; + TTraceID GetRootID() const override; + ui64 GetSize() const override; + void OutHtml(TStringStream& str, TTraceInfo& traceInfo) const override; + void OutHtml(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId) const override; + void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, TString signalId) override; + void OutSignalHtmlBody(TStringStream& str, const TTimestampInfo& tsInfo, std::function<TString()> getMyId, TList<ui64>& signalAddress) override; + void OutText(TStringStream& str, TTimestampInfo& tsInfo, const TString& prefix) const override; + ITrace::EType GetType() const override; + bool SerializeToString(TString& str) const override; + +private: + ITrace::EType Type; + TTraceID SelfID; + TTraceID ParentID; + TTraceID RootID; + TList<TSerializedSignal> Signals; +}; + +} +} diff --git a/ydb/core/tracing/trace_collection.cpp b/ydb/core/tracing/trace_collection.cpp index c50be78395..d9b1645070 100644 --- a/ydb/core/tracing/trace_collection.cpp +++ b/ydb/core/tracing/trace_collection.cpp @@ -1,159 +1,159 @@ -#include "trace_collection.h" - -namespace NKikimr { -namespace NTracing { - -static const ui64 MaxTracesPerTablet = 20; -static const ui64 MaxIntrospectionDataSize = 536870912; // 512 MB - -bool TTabletTraces::AddTrace(ITrace* trace) { - bool incremented = true; - if (Traces.size() >= MaxTracesPerTablet) { - const auto& oldestTrace = *Traces.begin(); - TotalSize -= oldestTrace->GetSize(); - Indexes.erase(oldestTrace->GetSelfID()); - Traces.pop_front(); - incremented = false; - } - Traces.emplace_back(trace); - const auto newElementIterator = --Traces.end(); - Indexes[(*newElementIterator)->GetSelfID()] = newElementIterator; - TotalSize += (*newElementIterator)->GetSize(); - return incremented; -} - -void TTabletTraces::GetTraces(TVector<TTraceID>& result) const { - result.reserve(Traces.size()); - for (auto& trace : Traces) { - result.push_back(trace->GetSelfID()); - } -} - -ITrace* TTabletTraces::GetTrace(TTraceID& traceID) { - auto findResult = Indexes.find(traceID); - if (findResult != Indexes.end()) { - auto introspectionTracesIterator = findResult->second; - - // Move this element to the end of the list - Traces.splice(Traces.end(), Traces, introspectionTracesIterator); - introspectionTracesIterator = --Traces.end(); - Indexes[traceID] = introspectionTracesIterator; - return introspectionTracesIterator->Get(); - } - return nullptr; -} - -ui64 TTabletTraces::GetSize() const { - return TotalSize; -} - -ui64 TTabletTraces::GetCount() const { - return Traces.size(); -} - -TTraceCollection::TTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { - if (counters) { - Reporting = true; - ReportedSize = counters->GetCounter("totalsize"); - ReportedCurrentCount = counters->GetCounter("tracescurrent"); - ReportedTotalCount = counters->GetCounter("tracestotal"); - ReportedTabletCount = counters->GetCounter("tabletscurrent"); - } -} - -void TTraceCollection::AddTrace(ui64 tabletID, ITrace* trace) { - auto findResult = Indexes.find(tabletID); - MainQueueType::iterator mainQueueIterator; - if (findResult != Indexes.end()) { - mainQueueIterator = findResult->second; - SubTotalSize(mainQueueIterator->Traces.GetSize()); - // Move this element to the end of the list - MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); - } else { - MainQueue.push_back({ tabletID, TTabletTraces() }); - ReportedTabletCount->Inc(); - } - mainQueueIterator = --MainQueue.end(); - Indexes[tabletID] = mainQueueIterator; - if (mainQueueIterator->Traces.AddTrace(trace)) { - ReportedCurrentCount->Inc(); - } - ReportedTotalCount->Inc(); - AddTotalSize(mainQueueIterator->Traces.GetSize()); - CheckSizeLimit(); -} - -void TTraceCollection::CheckSizeLimit() { - while (TotalSize > MaxIntrospectionDataSize) { - auto FrontElement = MainQueue.begin(); - RemoveElement(FrontElement); - } -} - -void TTraceCollection::RemoveElement(MainQueueType::iterator element) { - SubTotalSize(element->Traces.GetSize()); - ReportedCurrentCount->Sub(element->Traces.GetCount()); - Indexes.erase(element->TabletID); - MainQueue.erase(element); - ReportedTabletCount->Dec(); -} - -void TTraceCollection::AddTotalSize(ui64 value) { - TotalSize += value; - if (Reporting) { - ReportedSize->Add(value); - } -} - -void TTraceCollection::SubTotalSize(ui64 value) { - TotalSize -= value; - if (Reporting) { - ReportedSize->Sub(value); - } -} - -void TTraceCollection::GetTabletIDs(TVector<ui64>& tabletIDs) const { - tabletIDs.reserve(MainQueue.size()); - for (const auto& tabletEntry : MainQueue) { - tabletIDs.push_back(tabletEntry.TabletID); - } -} - - -bool TTraceCollection::HasTabletID(ui64 tabletID) const { +#include "trace_collection.h" + +namespace NKikimr { +namespace NTracing { + +static const ui64 MaxTracesPerTablet = 20; +static const ui64 MaxIntrospectionDataSize = 536870912; // 512 MB + +bool TTabletTraces::AddTrace(ITrace* trace) { + bool incremented = true; + if (Traces.size() >= MaxTracesPerTablet) { + const auto& oldestTrace = *Traces.begin(); + TotalSize -= oldestTrace->GetSize(); + Indexes.erase(oldestTrace->GetSelfID()); + Traces.pop_front(); + incremented = false; + } + Traces.emplace_back(trace); + const auto newElementIterator = --Traces.end(); + Indexes[(*newElementIterator)->GetSelfID()] = newElementIterator; + TotalSize += (*newElementIterator)->GetSize(); + return incremented; +} + +void TTabletTraces::GetTraces(TVector<TTraceID>& result) const { + result.reserve(Traces.size()); + for (auto& trace : Traces) { + result.push_back(trace->GetSelfID()); + } +} + +ITrace* TTabletTraces::GetTrace(TTraceID& traceID) { + auto findResult = Indexes.find(traceID); + if (findResult != Indexes.end()) { + auto introspectionTracesIterator = findResult->second; + + // Move this element to the end of the list + Traces.splice(Traces.end(), Traces, introspectionTracesIterator); + introspectionTracesIterator = --Traces.end(); + Indexes[traceID] = introspectionTracesIterator; + return introspectionTracesIterator->Get(); + } + return nullptr; +} + +ui64 TTabletTraces::GetSize() const { + return TotalSize; +} + +ui64 TTabletTraces::GetCount() const { + return Traces.size(); +} + +TTraceCollection::TTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { + if (counters) { + Reporting = true; + ReportedSize = counters->GetCounter("totalsize"); + ReportedCurrentCount = counters->GetCounter("tracescurrent"); + ReportedTotalCount = counters->GetCounter("tracestotal"); + ReportedTabletCount = counters->GetCounter("tabletscurrent"); + } +} + +void TTraceCollection::AddTrace(ui64 tabletID, ITrace* trace) { + auto findResult = Indexes.find(tabletID); + MainQueueType::iterator mainQueueIterator; + if (findResult != Indexes.end()) { + mainQueueIterator = findResult->second; + SubTotalSize(mainQueueIterator->Traces.GetSize()); + // Move this element to the end of the list + MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); + } else { + MainQueue.push_back({ tabletID, TTabletTraces() }); + ReportedTabletCount->Inc(); + } + mainQueueIterator = --MainQueue.end(); + Indexes[tabletID] = mainQueueIterator; + if (mainQueueIterator->Traces.AddTrace(trace)) { + ReportedCurrentCount->Inc(); + } + ReportedTotalCount->Inc(); + AddTotalSize(mainQueueIterator->Traces.GetSize()); + CheckSizeLimit(); +} + +void TTraceCollection::CheckSizeLimit() { + while (TotalSize > MaxIntrospectionDataSize) { + auto FrontElement = MainQueue.begin(); + RemoveElement(FrontElement); + } +} + +void TTraceCollection::RemoveElement(MainQueueType::iterator element) { + SubTotalSize(element->Traces.GetSize()); + ReportedCurrentCount->Sub(element->Traces.GetCount()); + Indexes.erase(element->TabletID); + MainQueue.erase(element); + ReportedTabletCount->Dec(); +} + +void TTraceCollection::AddTotalSize(ui64 value) { + TotalSize += value; + if (Reporting) { + ReportedSize->Add(value); + } +} + +void TTraceCollection::SubTotalSize(ui64 value) { + TotalSize -= value; + if (Reporting) { + ReportedSize->Sub(value); + } +} + +void TTraceCollection::GetTabletIDs(TVector<ui64>& tabletIDs) const { + tabletIDs.reserve(MainQueue.size()); + for (const auto& tabletEntry : MainQueue) { + tabletIDs.push_back(tabletEntry.TabletID); + } +} + + +bool TTraceCollection::HasTabletID(ui64 tabletID) const { return Indexes.contains(tabletID); -} - -bool TTraceCollection::GetTraces(ui64 tabletID, TVector<TTraceID>& result) { - auto findResult = Indexes.find(tabletID); - if (findResult != Indexes.end()) { - auto mainQueueIterator = findResult->second; - mainQueueIterator->Traces.GetTraces(result); - // Move this element to the end of the list - MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); - mainQueueIterator = --MainQueue.end(); - Indexes[tabletID] = mainQueueIterator; - return true; - } - return false; -} - -ITrace* TTraceCollection::GetTrace(ui64 tabletID, TTraceID& traceID) { - auto findResult = Indexes.find(tabletID); - if (findResult != Indexes.end()) { - auto mainQueueIterator = findResult->second; - // Move this element to the end of the list - MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); - mainQueueIterator = --MainQueue.end(); - Indexes[tabletID] = mainQueueIterator; - return mainQueueIterator->Traces.GetTrace(traceID); - } - return nullptr; -} - -ITraceCollection* CreateTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { - return new TTraceCollection(counters); -} - -} -} +} + +bool TTraceCollection::GetTraces(ui64 tabletID, TVector<TTraceID>& result) { + auto findResult = Indexes.find(tabletID); + if (findResult != Indexes.end()) { + auto mainQueueIterator = findResult->second; + mainQueueIterator->Traces.GetTraces(result); + // Move this element to the end of the list + MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); + mainQueueIterator = --MainQueue.end(); + Indexes[tabletID] = mainQueueIterator; + return true; + } + return false; +} + +ITrace* TTraceCollection::GetTrace(ui64 tabletID, TTraceID& traceID) { + auto findResult = Indexes.find(tabletID); + if (findResult != Indexes.end()) { + auto mainQueueIterator = findResult->second; + // Move this element to the end of the list + MainQueue.splice(MainQueue.end(), MainQueue, mainQueueIterator); + mainQueueIterator = --MainQueue.end(); + Indexes[tabletID] = mainQueueIterator; + return mainQueueIterator->Traces.GetTrace(traceID); + } + return nullptr; +} + +ITraceCollection* CreateTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { + return new TTraceCollection(counters); +} + +} +} diff --git a/ydb/core/tracing/trace_collection.h b/ydb/core/tracing/trace_collection.h index 39a4f4aa06..6373475f8b 100644 --- a/ydb/core/tracing/trace_collection.h +++ b/ydb/core/tracing/trace_collection.h @@ -1,63 +1,63 @@ -#pragma once +#pragma once #include <ydb/core/base/defs.h> #include <ydb/core/base/tracing.h> - -namespace NKikimr { -namespace NTracing { - -class TTabletTraces { -public: - using ListType = TList<THolder<ITrace>>; - using IndexType = THashMap<TTraceID, ListType::iterator>; - - bool AddTrace(ITrace* trace); - void GetTraces(TVector<TTraceID>& result) const; - ITrace* GetTrace(TTraceID& traceID); - ui64 GetSize() const; - ui64 GetCount() const; - -private: - ListType Traces; - IndexType Indexes; - ui64 TotalSize = 0; -}; - -class TTraceCollection : public ITraceCollection { -public: - struct TTabletData { - ui64 TabletID; - TTabletTraces Traces; - }; - - using MainQueueType = TList<TTabletData>; - using IndexType = THashMap<ui64, MainQueueType::iterator>; - - TTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters); - - void AddTrace(ui64 tabletID, ITrace* trace) override; - // Shows all tablet IDs for which it has data - void GetTabletIDs(TVector<ui64>& tabletIDs) const override; - bool HasTabletID(ui64 tabletID) const override; - // Returns a list of introspection trace ids and creation times for given tabletId - bool GetTraces(ui64 tabletID, TVector<TTraceID>& result) override; - ITrace* GetTrace(ui64 tabletID, TTraceID& traceID) override; - -private: - void RemoveElement(MainQueueType::iterator element); - void CheckSizeLimit(); - void AddTotalSize(ui64 value); - void SubTotalSize(ui64 value); - - MainQueueType MainQueue; - IndexType Indexes; - ui64 TotalSize = 0; - - NMonitoring::TDynamicCounters::TCounterPtr ReportedSize; - NMonitoring::TDynamicCounters::TCounterPtr ReportedCurrentCount; - NMonitoring::TDynamicCounters::TCounterPtr ReportedTotalCount; - NMonitoring::TDynamicCounters::TCounterPtr ReportedTabletCount; - bool Reporting = false; -}; - -} -} + +namespace NKikimr { +namespace NTracing { + +class TTabletTraces { +public: + using ListType = TList<THolder<ITrace>>; + using IndexType = THashMap<TTraceID, ListType::iterator>; + + bool AddTrace(ITrace* trace); + void GetTraces(TVector<TTraceID>& result) const; + ITrace* GetTrace(TTraceID& traceID); + ui64 GetSize() const; + ui64 GetCount() const; + +private: + ListType Traces; + IndexType Indexes; + ui64 TotalSize = 0; +}; + +class TTraceCollection : public ITraceCollection { +public: + struct TTabletData { + ui64 TabletID; + TTabletTraces Traces; + }; + + using MainQueueType = TList<TTabletData>; + using IndexType = THashMap<ui64, MainQueueType::iterator>; + + TTraceCollection(TIntrusivePtr<NMonitoring::TDynamicCounters> counters); + + void AddTrace(ui64 tabletID, ITrace* trace) override; + // Shows all tablet IDs for which it has data + void GetTabletIDs(TVector<ui64>& tabletIDs) const override; + bool HasTabletID(ui64 tabletID) const override; + // Returns a list of introspection trace ids and creation times for given tabletId + bool GetTraces(ui64 tabletID, TVector<TTraceID>& result) override; + ITrace* GetTrace(ui64 tabletID, TTraceID& traceID) override; + +private: + void RemoveElement(MainQueueType::iterator element); + void CheckSizeLimit(); + void AddTotalSize(ui64 value); + void SubTotalSize(ui64 value); + + MainQueueType MainQueue; + IndexType Indexes; + ui64 TotalSize = 0; + + NMonitoring::TDynamicCounters::TCounterPtr ReportedSize; + NMonitoring::TDynamicCounters::TCounterPtr ReportedCurrentCount; + NMonitoring::TDynamicCounters::TCounterPtr ReportedTotalCount; + NMonitoring::TDynamicCounters::TCounterPtr ReportedTabletCount; + bool Reporting = false; +}; + +} +} diff --git a/ydb/core/tracing/ya.make b/ydb/core/tracing/ya.make index 1a5b9edb41..56ca79a1d2 100644 --- a/ydb/core/tracing/ya.make +++ b/ydb/core/tracing/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( ddoarn - pnv1 - g:kikimr -) - -SRCS( - http.cpp - tablet_info.cpp - trace.cpp - trace_collection.cpp -) - -PEERDIR( + pnv1 + g:kikimr +) + +SRCS( + http.cpp + tablet_info.cpp + trace.cpp + trace_collection.cpp +) + +PEERDIR( ydb/core/base ydb/core/protos -) - -END() +) + +END() diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp index 23f1a367f0..edd6ca8e19 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp @@ -169,10 +169,10 @@ void TKqpDatashardComputeContext::SetLockTxId(ui64 lockTxId) { LockTxId = lockTxId; } -ui64 TKqpDatashardComputeContext::GetShardId() const { - return Shard->TabletID(); -} - +ui64 TKqpDatashardComputeContext::GetShardId() const { + return Shard->TabletID(); +} + void TKqpDatashardComputeContext::SetReadVersion(TRowVersion readVersion) { ReadVersion = readVersion; } diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h index 08b169ac98..4bbb8fa0de 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.h +++ b/ydb/core/tx/datashard/datashard_kqp_compute.h @@ -34,7 +34,7 @@ public: void ReadTable(const TTableId& tableId, const TArrayRef<const TCell>& key) const; void BreakSetLocks() const; void SetLockTxId(ui64 lockTxId); - ui64 GetShardId() const; + ui64 GetShardId() const; TVector<std::pair<NScheme::TTypeId, TString>> GetKeyColumnsInfo(const TTableId &tableId) const; THashMap<TString, NScheme::TTypeId> GetKeyColumnsMap(const TTableId &tableId) const; diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 4541dc0ed1..8e778a4a1d 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -1705,9 +1705,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { keyDesc.ColumnInfos.clear(); keyDesc.ColumnInfos.reserve(keyDesc.Columns.size()); for (auto& columnOp : keyDesc.Columns) { - if (IsSystemColumn(columnOp.Column)) { - continue; - } + if (IsSystemColumn(columnOp.Column)) { + continue; + } if (IsSysTable() && columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) { SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::OperationNotSupported); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 60df469ec3..fcb564b606 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -1468,7 +1468,7 @@ bool TTableInfo::TryAddShardToMerge(const TSplitSettings& splitSettings, // Check that total load doesn't exceed the limits float shardLoad = stats->GetCurrentRawCpuUsage() * 0.000001; - if (IsMergeByLoadEnabled()) { + if (IsMergeByLoadEnabled()) { const auto& settings = PartitionConfig().GetPartitioningPolicy().GetSplitByLoadSettings(); i64 cpuPercentage = settings.GetCpuPercentageThreshold(); float cpuUsageThreshold = 0.01 * (cpuPercentage ? cpuPercentage : (i64)splitSettings.FastSplitCpuPercentageThreshold); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 9e428f8865..2e5710c3d3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -563,7 +563,7 @@ public: } bool IsMergeBySizeEnabled() const { - return IsSplitBySizeEnabled() && PartitionConfig().GetPartitioningPolicy().GetMinPartitionsCount() != 0; + return IsSplitBySizeEnabled() && PartitionConfig().GetPartitioningPolicy().GetMinPartitionsCount() != 0; } bool IsSplitByLoadEnabled() const { diff --git a/ydb/core/viewer/json/json.cpp b/ydb/core/viewer/json/json.cpp index d43aa12f37..51d7c9a32d 100644 --- a/ydb/core/viewer/json/json.cpp +++ b/ydb/core/viewer/json/json.cpp @@ -40,7 +40,7 @@ void TProtoToJson::EscapeJsonString(IOutputStream& os, const TString& s) { TString TProtoToJson::EscapeJsonString(const TString& s) { TStringStream str; EscapeJsonString(str, s); - return str.Str(); + return str.Str(); } void TProtoToJson::ProtoToJson(IOutputStream& to, const ::google::protobuf::EnumValueDescriptor* descriptor, const TJsonSettings& jsonSettings) { diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 59f8bb363c..0d10bb8450 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -368,7 +368,7 @@ void FillTableStats(Ydb::Table::DescribeTableResult& out, stats->mutable_modification_time()->CopyFrom(modificationTime); } - ui64 creationTimeMs = in.GetSelf().GetCreateStep(); + ui64 creationTimeMs = in.GetSelf().GetCreateStep(); if (creationTimeMs) { auto creationTime = MillisecToProtoTimeStamp(creationTimeMs); stats->mutable_creation_time()->CopyFrom(creationTime); @@ -546,63 +546,63 @@ void FillAttributes(Ydb::Table::CreateTableRequest& out, template <typename TYdbProto> static void FillDefaultPartitioningSettings(TYdbProto& out) { - // (!) We assume that all partitioning methods are disabled by default. But we don't know it for sure. - auto& outPartSettings = *out.mutable_partitioning_settings(); - outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); - outPartSettings.set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); -} - + // (!) We assume that all partitioning methods are disabled by default. But we don't know it for sure. + auto& outPartSettings = *out.mutable_partitioning_settings(); + outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); + outPartSettings.set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); +} + template <typename TYdbProto> void FillPartitioningSettingsImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - if (!in.HasPartitionConfig()) { - FillDefaultPartitioningSettings(out); - return; - } - - const auto& partConfig = in.GetPartitionConfig(); - if (!partConfig.HasPartitioningPolicy()) { - FillDefaultPartitioningSettings(out); - return; - } - - auto& outPartSettings = *out.mutable_partitioning_settings(); - const auto& inPartPolicy = partConfig.GetPartitioningPolicy(); - if (inPartPolicy.HasSizeToSplit()) { - if (inPartPolicy.GetSizeToSplit()) { - outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); - outPartSettings.set_partition_size_mb(inPartPolicy.GetSizeToSplit() / (1 << 20)); - } else { - outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); - } - } else { - // (!) We assume that partitioning by size is disabled by default. But we don't know it for sure. - outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); - } - + if (!in.HasPartitionConfig()) { + FillDefaultPartitioningSettings(out); + return; + } + + const auto& partConfig = in.GetPartitionConfig(); + if (!partConfig.HasPartitioningPolicy()) { + FillDefaultPartitioningSettings(out); + return; + } + + auto& outPartSettings = *out.mutable_partitioning_settings(); + const auto& inPartPolicy = partConfig.GetPartitioningPolicy(); + if (inPartPolicy.HasSizeToSplit()) { + if (inPartPolicy.GetSizeToSplit()) { + outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::ENABLED); + outPartSettings.set_partition_size_mb(inPartPolicy.GetSizeToSplit() / (1 << 20)); + } else { + outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); + } + } else { + // (!) We assume that partitioning by size is disabled by default. But we don't know it for sure. + outPartSettings.set_partitioning_by_size(Ydb::FeatureFlag::DISABLED); + } + if (inPartPolicy.HasSplitByLoadSettings()) { bool enabled = inPartPolicy.GetSplitByLoadSettings().GetEnabled(); outPartSettings.set_partitioning_by_load(enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); - } else { - // (!) We assume that partitioning by load is disabled by default. But we don't know it for sure. - outPartSettings.set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); + } else { + // (!) We assume that partitioning by load is disabled by default. But we don't know it for sure. + outPartSettings.set_partitioning_by_load(Ydb::FeatureFlag::DISABLED); + } + + if (inPartPolicy.HasMinPartitionsCount() && inPartPolicy.GetMinPartitionsCount()) { + outPartSettings.set_min_partitions_count(inPartPolicy.GetMinPartitionsCount()); } - if (inPartPolicy.HasMinPartitionsCount() && inPartPolicy.GetMinPartitionsCount()) { - outPartSettings.set_min_partitions_count(inPartPolicy.GetMinPartitionsCount()); - } - - if (inPartPolicy.HasMaxPartitionsCount() && inPartPolicy.GetMaxPartitionsCount()) { - outPartSettings.set_max_partitions_count(inPartPolicy.GetMaxPartitionsCount()); - } + if (inPartPolicy.HasMaxPartitionsCount() && inPartPolicy.GetMaxPartitionsCount()) { + outPartSettings.set_max_partitions_count(inPartPolicy.GetMaxPartitionsCount()); + } } - + void FillPartitioningSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { FillPartitioningSettingsImpl(out, in); -} - +} + void FillPartitioningSettings(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { FillPartitioningSettingsImpl(out, in); @@ -629,22 +629,22 @@ template <typename TYdbProto> void FillKeyBloomFilterImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - if (!in.HasPartitionConfig()) { - return; - } - - const auto& partConfig = in.GetPartitionConfig(); - if (!partConfig.HasEnableFilterByKey()) { - return; - } - - if (partConfig.GetEnableFilterByKey()) { - out.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); - } else { - out.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); - } -} - + if (!in.HasPartitionConfig()) { + return; + } + + const auto& partConfig = in.GetPartitionConfig(); + if (!partConfig.HasEnableFilterByKey()) { + return; + } + + if (partConfig.GetEnableFilterByKey()) { + out.set_key_bloom_filter(Ydb::FeatureFlag::ENABLED); + } else { + out.set_key_bloom_filter(Ydb::FeatureFlag::DISABLED); + } +} + void FillKeyBloomFilter(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { FillKeyBloomFilterImpl(out, in); @@ -659,33 +659,33 @@ template <typename TYdbProto> void FillReadReplicasSettingsImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in) { - if (!in.HasPartitionConfig()) { - return; - } - - const auto& partConfig = in.GetPartitionConfig(); + if (!in.HasPartitionConfig()) { + return; + } + + const auto& partConfig = in.GetPartitionConfig(); if (!partConfig.FollowerGroupsSize() && !partConfig.HasCrossDataCenterFollowerCount() && !partConfig.HasFollowerCount()) { - return; - } - + return; + } + if (partConfig.FollowerGroupsSize()) { if (partConfig.FollowerGroupsSize() > 1) { - // Not supported yet - return; - } + // Not supported yet + return; + } const auto& followerGroup = partConfig.GetFollowerGroups(0); if (followerGroup.GetFollowerCountPerDataCenter()) { out.mutable_read_replicas_settings()->set_per_az_read_replicas_count(followerGroup.GetFollowerCount()); - } else { + } else { out.mutable_read_replicas_settings()->set_any_az_read_replicas_count(followerGroup.GetFollowerCount()); - } + } } else if (partConfig.HasCrossDataCenterFollowerCount()) { out.mutable_read_replicas_settings()->set_per_az_read_replicas_count(partConfig.GetCrossDataCenterFollowerCount()); } else if (partConfig.HasFollowerCount()) { out.mutable_read_replicas_settings()->set_any_az_read_replicas_count(partConfig.GetFollowerCount()); - } -} - + } +} + void FillReadReplicasSettings(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in) { FillReadReplicasSettingsImpl(out, in); diff --git a/ydb/core/ydb_convert/table_settings.cpp b/ydb/core/ydb_convert/table_settings.cpp index f44b5381e7..3f3547f1f5 100644 --- a/ydb/core/ydb_convert/table_settings.cpp +++ b/ydb/core/ydb_convert/table_settings.cpp @@ -1,88 +1,88 @@ #include "table_description.h" -#include "table_settings.h" - +#include "table_settings.h" + #include <util/generic/list.h> -#include <util/string/builder.h> - -namespace NKikimr { - -void MEWarning(const TString& settingName, TList<TString>& warnings) { - warnings.push_back(TStringBuilder() << "Table profile and " << settingName << " are set. They are mutually exclusive. " - << "Use either one of them."); -} - -namespace { - const ui32 defaultMinPartitions = 1; - const ui64 defaultSizeToSplit = 2ul << 30; // 2048 Mb - - ui32 CalculateDefaultMinPartitions(const Ydb::Table::CreateTableRequest& proto) { - switch (proto.partitions_case()) { - case Ydb::Table::CreateTableRequest::kUniformPartitions: - return static_cast<ui32>(proto.uniform_partitions()); - case Ydb::Table::CreateTableRequest::kPartitionAtKeys: - return proto.partition_at_keys().split_points().size() + 1; - default: - return defaultMinPartitions; - } - } -} - +#include <util/string/builder.h> + +namespace NKikimr { + +void MEWarning(const TString& settingName, TList<TString>& warnings) { + warnings.push_back(TStringBuilder() << "Table profile and " << settingName << " are set. They are mutually exclusive. " + << "Use either one of them."); +} + +namespace { + const ui32 defaultMinPartitions = 1; + const ui64 defaultSizeToSplit = 2ul << 30; // 2048 Mb + + ui32 CalculateDefaultMinPartitions(const Ydb::Table::CreateTableRequest& proto) { + switch (proto.partitions_case()) { + case Ydb::Table::CreateTableRequest::kUniformPartitions: + return static_cast<ui32>(proto.uniform_partitions()); + case Ydb::Table::CreateTableRequest::kPartitionAtKeys: + return proto.partition_at_keys().split_points().size() + 1; + default: + return defaultMinPartitions; + } + } +} + bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, const Ydb::Table::CreateTableRequest& proto, Ydb::StatusIds::StatusCode& code, TString& error, TList<TString>& warnings, bool tableProfileSet) -{ - auto &partitionConfig = *tableDesc.MutablePartitionConfig(); - - if (proto.has_partitioning_settings()) { - if (tableProfileSet) { - MEWarning("PartitioningSettings", warnings); - } - auto& partitioningSettings = proto.partitioning_settings(); - - switch (partitioningSettings.partitioning_by_size()) { - case Ydb::FeatureFlag::STATUS_UNSPECIFIED: - { - if (partitioningSettings.partition_size_mb()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetSizeToSplit((1 << 20) * partitioningSettings.partition_size_mb()); - if (!partitioningSettings.min_partitions_count()) { - policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); - } - } - } - break; - case Ydb::FeatureFlag::ENABLED: - { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - if (partitioningSettings.partition_size_mb()) { - policy.SetSizeToSplit((1 << 20) * partitioningSettings.partition_size_mb()); - } else { - policy.SetSizeToSplit(defaultSizeToSplit); - } - if (!partitioningSettings.min_partitions_count()) { - policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); - } - break; - } - case Ydb::FeatureFlag::DISABLED: - { - if (partitioningSettings.partition_size_mb()) { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Auto partitioning partition size is set while " - "auto partitioning by size is disabled"; - return false; - } - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetSizeToSplit(0); - break; - } - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown auto partitioning by size feature flag status: '" - << (ui32)partitioningSettings.partitioning_by_size() << "'"; - return false; - } - +{ + auto &partitionConfig = *tableDesc.MutablePartitionConfig(); + + if (proto.has_partitioning_settings()) { + if (tableProfileSet) { + MEWarning("PartitioningSettings", warnings); + } + auto& partitioningSettings = proto.partitioning_settings(); + + switch (partitioningSettings.partitioning_by_size()) { + case Ydb::FeatureFlag::STATUS_UNSPECIFIED: + { + if (partitioningSettings.partition_size_mb()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetSizeToSplit((1 << 20) * partitioningSettings.partition_size_mb()); + if (!partitioningSettings.min_partitions_count()) { + policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); + } + } + } + break; + case Ydb::FeatureFlag::ENABLED: + { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + if (partitioningSettings.partition_size_mb()) { + policy.SetSizeToSplit((1 << 20) * partitioningSettings.partition_size_mb()); + } else { + policy.SetSizeToSplit(defaultSizeToSplit); + } + if (!partitioningSettings.min_partitions_count()) { + policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); + } + break; + } + case Ydb::FeatureFlag::DISABLED: + { + if (partitioningSettings.partition_size_mb()) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Auto partitioning partition size is set while " + "auto partitioning by size is disabled"; + return false; + } + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetSizeToSplit(0); + break; + } + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown auto partitioning by size feature flag status: '" + << (ui32)partitioningSettings.partitioning_by_size() << "'"; + return false; + } + switch (partitioningSettings.partitioning_by_load()) { case Ydb::FeatureFlag::STATUS_UNSPECIFIED: { @@ -92,9 +92,9 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, { auto &policy = *partitionConfig.MutablePartitioningPolicy(); policy.MutableSplitByLoadSettings()->SetEnabled(true); - if (!partitioningSettings.min_partitions_count()) { - policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); - } + if (!partitioningSettings.min_partitions_count()) { + policy.SetMinPartitionsCount(CalculateDefaultMinPartitions(proto)); + } break; } case Ydb::FeatureFlag::DISABLED: @@ -110,147 +110,147 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, return false; } - if (partitioningSettings.min_partitions_count()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetMinPartitionsCount(partitioningSettings.min_partitions_count()); - } - - if (partitioningSettings.max_partitions_count()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetMaxPartitionsCount(partitioningSettings.max_partitions_count()); - } - } - - if (proto.partitions_case() != Ydb::Table::CreateTableRequest::PARTITIONS_NOT_SET && tableProfileSet) { - MEWarning("Partitions", warnings); - } - switch (proto.partitions_case()) { - case Ydb::Table::CreateTableRequest::kUniformPartitions: - tableDesc.SetUniformPartitionsCount(proto.uniform_partitions()); - break; - case Ydb::Table::CreateTableRequest::kPartitionAtKeys: + if (partitioningSettings.min_partitions_count()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetMinPartitionsCount(partitioningSettings.min_partitions_count()); + } + + if (partitioningSettings.max_partitions_count()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetMaxPartitionsCount(partitioningSettings.max_partitions_count()); + } + } + + if (proto.partitions_case() != Ydb::Table::CreateTableRequest::PARTITIONS_NOT_SET && tableProfileSet) { + MEWarning("Partitions", warnings); + } + switch (proto.partitions_case()) { + case Ydb::Table::CreateTableRequest::kUniformPartitions: + tableDesc.SetUniformPartitionsCount(proto.uniform_partitions()); + break; + case Ydb::Table::CreateTableRequest::kPartitionAtKeys: if (!CopyExplicitPartitions(tableDesc, proto.partition_at_keys(), code, error)) { - return false; - } - break; - default: - break; - } - - if (proto.key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED && tableProfileSet) { - MEWarning("KeyBloomFilter", warnings); - } - switch (proto.key_bloom_filter()) { - case Ydb::FeatureFlag::STATUS_UNSPECIFIED: - break; - case Ydb::FeatureFlag::ENABLED: - { - partitionConfig.SetEnableFilterByKey(true); - break; - } - case Ydb::FeatureFlag::DISABLED: - partitionConfig.SetEnableFilterByKey(false); - break; - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown key bloom filter feature flag status: '" - << (ui32)proto.key_bloom_filter() << "'"; - return false; - } - - if (proto.has_read_replicas_settings()) { - if (tableProfileSet) { - MEWarning("ReadReplicasSettings", warnings); + return false; + } + break; + default: + break; + } + + if (proto.key_bloom_filter() != Ydb::FeatureFlag::STATUS_UNSPECIFIED && tableProfileSet) { + MEWarning("KeyBloomFilter", warnings); + } + switch (proto.key_bloom_filter()) { + case Ydb::FeatureFlag::STATUS_UNSPECIFIED: + break; + case Ydb::FeatureFlag::ENABLED: + { + partitionConfig.SetEnableFilterByKey(true); + break; + } + case Ydb::FeatureFlag::DISABLED: + partitionConfig.SetEnableFilterByKey(false); + break; + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown key bloom filter feature flag status: '" + << (ui32)proto.key_bloom_filter() << "'"; + return false; + } + + if (proto.has_read_replicas_settings()) { + if (tableProfileSet) { + MEWarning("ReadReplicasSettings", warnings); partitionConfig.ClearFollowerCount(); partitionConfig.ClearCrossDataCenterFollowerCount(); partitionConfig.ClearAllowFollowerPromotion(); partitionConfig.ClearFollowerGroups(); - } - auto& readReplicasSettings = proto.read_replicas_settings(); - switch (readReplicasSettings.settings_case()) { - case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: - { + } + auto& readReplicasSettings = proto.read_replicas_settings(); + switch (readReplicasSettings.settings_case()) { + case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: + { auto& followerGroup = *partitionConfig.AddFollowerGroups(); followerGroup.SetFollowerCount(readReplicasSettings.per_az_read_replicas_count()); followerGroup.SetRequireAllDataCenters(true); followerGroup.SetFollowerCountPerDataCenter(true); - break; - } - case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: - { + break; + } + case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: + { auto& followerGroup = *partitionConfig.AddFollowerGroups(); followerGroup.SetFollowerCount(readReplicasSettings.any_az_read_replicas_count()); followerGroup.SetRequireAllDataCenters(false); - break; - } - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown read_replicas_settings type"; - return false; - } - } - + break; + } + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown read_replicas_settings type"; + return false; + } + } + if (proto.has_ttl_settings()) { if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.ttl_settings(), code, error)) { return false; } } - return true; -} - + return true; +} + bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, const Ydb::Table::AlterTableRequest& proto, Ydb::StatusIds::StatusCode& code, TString& error, bool changed) -{ - bool hadPartitionConfig = tableDesc.HasPartitionConfig(); - auto &partitionConfig = *tableDesc.MutablePartitionConfig(); - - if (proto.has_alter_partitioning_settings()) { - auto& alterSettings = proto.alter_partitioning_settings(); - - switch (alterSettings.partitioning_by_size()) { - case Ydb::FeatureFlag::STATUS_UNSPECIFIED: - if (alterSettings.partition_size_mb()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetSizeToSplit((1 << 20) * alterSettings.partition_size_mb()); - changed = true; - } - break; - case Ydb::FeatureFlag::ENABLED: - { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - if (alterSettings.partition_size_mb()) { - policy.SetSizeToSplit((1 << 20) * alterSettings.partition_size_mb()); - } else { - policy.SetSizeToSplit(defaultSizeToSplit); - } - if (!alterSettings.min_partitions_count()) { - policy.SetMinPartitionsCount(defaultMinPartitions); - } - changed = true; - break; - } - case Ydb::FeatureFlag::DISABLED: - { - if (alterSettings.partition_size_mb()) { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Auto partitioning partition size is set while " - "auto partitioning by size is disabled"; - return false; - } - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetSizeToSplit(0); - changed = true; - break; - } - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown auto partitioning by size feature flag status: '" - << (ui32)alterSettings.partitioning_by_size() << "'"; - return false; - } - +{ + bool hadPartitionConfig = tableDesc.HasPartitionConfig(); + auto &partitionConfig = *tableDesc.MutablePartitionConfig(); + + if (proto.has_alter_partitioning_settings()) { + auto& alterSettings = proto.alter_partitioning_settings(); + + switch (alterSettings.partitioning_by_size()) { + case Ydb::FeatureFlag::STATUS_UNSPECIFIED: + if (alterSettings.partition_size_mb()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetSizeToSplit((1 << 20) * alterSettings.partition_size_mb()); + changed = true; + } + break; + case Ydb::FeatureFlag::ENABLED: + { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + if (alterSettings.partition_size_mb()) { + policy.SetSizeToSplit((1 << 20) * alterSettings.partition_size_mb()); + } else { + policy.SetSizeToSplit(defaultSizeToSplit); + } + if (!alterSettings.min_partitions_count()) { + policy.SetMinPartitionsCount(defaultMinPartitions); + } + changed = true; + break; + } + case Ydb::FeatureFlag::DISABLED: + { + if (alterSettings.partition_size_mb()) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Auto partitioning partition size is set while " + "auto partitioning by size is disabled"; + return false; + } + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetSizeToSplit(0); + changed = true; + break; + } + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown auto partitioning by size feature flag status: '" + << (ui32)alterSettings.partitioning_by_size() << "'"; + return false; + } + switch (alterSettings.partitioning_by_load()) { case Ydb::FeatureFlag::STATUS_UNSPECIFIED: { @@ -261,9 +261,9 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, { auto &policy = *partitionConfig.MutablePartitioningPolicy(); policy.MutableSplitByLoadSettings()->SetEnabled(true); - if (!alterSettings.min_partitions_count()) { - policy.SetMinPartitionsCount(defaultMinPartitions); - } + if (!alterSettings.min_partitions_count()) { + policy.SetMinPartitionsCount(defaultMinPartitions); + } changed = true; break; } @@ -281,63 +281,63 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, return false; } - if (alterSettings.min_partitions_count()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetMinPartitionsCount(alterSettings.min_partitions_count()); - changed = true; - } - - if (alterSettings.max_partitions_count()) { - auto &policy = *partitionConfig.MutablePartitioningPolicy(); - policy.SetMaxPartitionsCount(alterSettings.max_partitions_count()); - changed = true; - } - } - - switch (proto.set_key_bloom_filter()) { - case Ydb::FeatureFlag::STATUS_UNSPECIFIED: - break; - case Ydb::FeatureFlag::ENABLED: - partitionConfig.SetEnableFilterByKey(true); - changed = true; - break; - case Ydb::FeatureFlag::DISABLED: - partitionConfig.SetEnableFilterByKey(false); - changed = true; - break; - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown key bloom filter feature flag status: '" - << (ui32)proto.set_key_bloom_filter() << "'"; - return false; - } - - if (proto.has_set_read_replicas_settings()) { - auto& readReplicasSettings = proto.set_read_replicas_settings(); - switch (readReplicasSettings.settings_case()) { - case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: - { + if (alterSettings.min_partitions_count()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetMinPartitionsCount(alterSettings.min_partitions_count()); + changed = true; + } + + if (alterSettings.max_partitions_count()) { + auto &policy = *partitionConfig.MutablePartitioningPolicy(); + policy.SetMaxPartitionsCount(alterSettings.max_partitions_count()); + changed = true; + } + } + + switch (proto.set_key_bloom_filter()) { + case Ydb::FeatureFlag::STATUS_UNSPECIFIED: + break; + case Ydb::FeatureFlag::ENABLED: + partitionConfig.SetEnableFilterByKey(true); + changed = true; + break; + case Ydb::FeatureFlag::DISABLED: + partitionConfig.SetEnableFilterByKey(false); + changed = true; + break; + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown key bloom filter feature flag status: '" + << (ui32)proto.set_key_bloom_filter() << "'"; + return false; + } + + if (proto.has_set_read_replicas_settings()) { + auto& readReplicasSettings = proto.set_read_replicas_settings(); + switch (readReplicasSettings.settings_case()) { + case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: + { auto& followerGroup = *partitionConfig.AddFollowerGroups(); followerGroup.SetFollowerCount(readReplicasSettings.per_az_read_replicas_count()); followerGroup.SetRequireAllDataCenters(true); followerGroup.SetFollowerCountPerDataCenter(true); - break; - } - case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: - { + break; + } + case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: + { auto& followerGroup = *partitionConfig.AddFollowerGroups(); followerGroup.SetFollowerCount(readReplicasSettings.any_az_read_replicas_count()); followerGroup.SetRequireAllDataCenters(false); - break; - } - default: - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown read_replicas_settings type"; - return false; - } - changed = true; - } - + break; + } + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Unknown read_replicas_settings type"; + return false; + } + changed = true; + } + if (proto.has_set_ttl_settings()) { if (!FillTtlSettings(*tableDesc.MutableTTLSettings()->MutableEnabled(), proto.set_ttl_settings(), code, error)) { return false; @@ -346,11 +346,11 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, tableDesc.MutableTTLSettings()->MutableDisabled(); } - if (!changed && !hadPartitionConfig) { - tableDesc.ClearPartitionConfig(); - } + if (!changed && !hadPartitionConfig) { + tableDesc.ClearPartitionConfig(); + } + + return true; +} - return true; -} - -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp index 5dde9dd0eb..a32a6c3dab 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp @@ -4003,7 +4003,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["ReplaceMember"] = std::bind(&ExpandReplaceMember, _1, _2); map["RemovePrefixMembers"] = std::bind(&ExpandRemovePrefixMembers, _1, _2); - + map["FlattenByColumns"] = std::bind(&ExpandFlattenByColumns, _1, _2); map["AsStruct"] = std::bind(&OptimizeAsStruct, _1, _2); diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index d420ebd44c..a2ed91d640 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -71,8 +71,8 @@ message TIssuesIds { KIKIMR_INDEX_IS_NOT_READY = 2027; KIKIMR_SCHEME_MISMATCH = 2028; KIKIMR_PRECONDITION_FAILED = 2029; - KIKIMR_UNSUPPORTED = 2030; - KIKIMR_BAD_COLUMN_TYPE = 2031; + KIKIMR_UNSUPPORTED = 2030; + KIKIMR_BAD_COLUMN_TYPE = 2031; KIKIMR_NO_COLUMN_DEFAULT_VALUE = 2032; // kikimr warnings @@ -117,7 +117,7 @@ message TIssuesIds { YQL_MULTIWAY_JOIN_WITH_USING = 4514; YQL_DEPRECATED_V0_SYNTAX = 4515; YQL_UNNAMED_COLUMN = 4516; - YQL_SOURCE_SELECT_COLUMN_MISMATCH = 4517; + YQL_SOURCE_SELECT_COLUMN_MISMATCH = 4517; YQL_DEPRECATED_PRAGMA = 4518; YQL_EMPTY_WINDOW_FRAME = 4520; YQL_RANK_WITHOUT_ORDER_BY = 4521; diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index 782798c72b..897d5d6f9a 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -281,14 +281,14 @@ ids { severity: S_ERROR } ids { - code: KIKIMR_UNSUPPORTED - severity: S_ERROR -} -ids { - code: KIKIMR_BAD_COLUMN_TYPE - severity: S_ERROR -} -ids { + code: KIKIMR_UNSUPPORTED + severity: S_ERROR +} +ids { + code: KIKIMR_BAD_COLUMN_TYPE + severity: S_ERROR +} +ids { code: KIKIMR_NO_COLUMN_DEFAULT_VALUE severity: S_ERROR } @@ -435,10 +435,10 @@ ids { ids { code: YQL_UNNAMED_COLUMN severity: S_WARNING -} -ids { - code: YQL_SOURCE_SELECT_COLUMN_MISMATCH - severity: S_WARNING +} +ids { + code: YQL_SOURCE_SELECT_COLUMN_MISMATCH + severity: S_WARNING } ids { code: YQL_NOT_ALLOWED_IN_DISCOVERY @@ -575,7 +575,7 @@ ids { ids { code: YQL_MIXED_TZ severity: S_WARNING -} +} ids { code: YQL_OPERATION_WILL_RETURN_NULL severity: S_WARNING diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 85b1f8cf51..5846e6cb10 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -1705,20 +1705,20 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus RemovePrefixMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto& firstChild = input->Head(); - auto firstChildType = firstChild.GetTypeAnn(); - - if (HasError(firstChildType, ctx.Expr) || !firstChildType) { - YQL_ENSURE(firstChild.Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(firstChild.Pos()), + return IGraphTransformer::TStatus::Error; + } + auto& firstChild = input->Head(); + auto firstChildType = firstChild.GetTypeAnn(); + + if (HasError(firstChildType, ctx.Expr) || !firstChildType) { + YQL_ENSURE(firstChild.Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue( + ctx.Expr.GetPosition(firstChild.Pos()), TStringBuilder() << "Expected struct, variant, or sequence type, but got lambda" - )); - return IGraphTransformer::TStatus::Error; - } - + )); + return IGraphTransformer::TStatus::Error; + } + auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx); if (status != IGraphTransformer::TStatus::Ok) { return status; @@ -1734,13 +1734,13 @@ namespace NTypeAnnImpl { auto prefixes = input->Child(1); auto rebuildStructType = [&ctx, prefixes](const TTypeAnnotationNode* structType) { - TVector<const TItemExprType*> newItems; + TVector<const TItemExprType*> newItems; for (auto& field : structType->Cast<TStructExprType>()->GetItems()) { if (!AnyOf(prefixes->Children(), [field](const auto& prefixNode) { return field->GetName().StartsWith(prefixNode->Content()); })) { - newItems.push_back(field); - } - } - + newItems.push_back(field); + } + } + return ctx.Expr.MakeType<TStructExprType>(newItems); }; @@ -1749,10 +1749,10 @@ namespace NTypeAnnImpl { if (resultType == itemType) { output = input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; - } + } if (!resultType->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } + return IGraphTransformer::TStatus::Error; + } } else if (itemType->GetKind() == ETypeAnnotationKind::Variant) { auto varType = itemType->Cast<TVariantExprType>(); if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) { @@ -1765,7 +1765,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } newTupleItems.push_back(rebuildStructType(tupleItemType)); - } + } resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TTupleExprType>(newTupleItems)); } else { YQL_ENSURE(varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct); @@ -1780,24 +1780,24 @@ namespace NTypeAnnImpl { newStructItems.push_back(ctx.Expr.MakeType<TItemExprType>(structItemType->GetName(), rebuildStructType(structItemType->GetItemType()))); } resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TStructExprType>(newStructItems)); - } + } if (resultType == itemType) { output = input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; } - } else { + } else { output = input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; - } - + } + if (isSequence) { resultType = MakeSequenceType(firstChildType->GetKind(), *resultType, ctx.Expr); } input->SetTypeAnn(resultType); - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus RemoveSystemMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -12824,7 +12824,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["ForceRemoveMember"] = &RemoveMemberWrapper<true>; Functions["ReplaceMember"] = &ReplaceMemberWrapper; Functions["RemovePrefixMembers"] = &RemovePrefixMembersWrapper; - Functions["RemoveSystemMembers"] = &RemoveSystemMembersWrapper; + Functions["RemoveSystemMembers"] = &RemoveSystemMembersWrapper; Functions["FlatMap"] = &FlatMapWrapper<false>; Functions["OrderedFlatMap"] = &FlatMapWrapper<false>; Functions["OrderedFlatMapWarn"] = &FlatMapWrapper<true>; @@ -13382,8 +13382,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } IGraphTransformer::TStatus ProcessUnknown(const TExprNode::TPtr& input, TExprContext& ctx) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() - << "(Core type annotation) Unsupported function: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + << "(Core type annotation) Unsupported function: " << input->Content())); return TStatus::Error; } diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index 108b52206d..f2b793af8d 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -4815,8 +4815,8 @@ TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& bool IsSystemMember(const TStringBuf& memberName) { return memberName.StartsWith(TStringBuf("_yql_")); -} - +} + IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, bool deduplicate) { diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h index 36db02d2c5..a865aa27cd 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.h +++ b/ydb/library/yql/core/yql_expr_type_annotation.h @@ -268,7 +268,7 @@ TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool IsSystemMember(const TStringBuf& memberName); - + IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, bool deduplicte = true); IGraphTransformer::TStatus NormalizeKeyValueTuples(const TExprNode::TPtr& input, ui32 startIndex, TExprNode::TPtr& output, diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp index 57570bf594..69aa84a500 100644 --- a/ydb/library/yql/core/yql_opt_utils.cpp +++ b/ydb/library/yql/core/yql_opt_utils.cpp @@ -591,24 +591,24 @@ TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprCont auto rebuildStruct = [&](const TExprNode::TPtr& srcStruct, const TStructExprType* targetType) -> TExprNode::TPtr { TExprNode::TListType nonSystemMembers; for (auto item : targetType->GetItems()) { - nonSystemMembers.push_back( + nonSystemMembers.push_back( Build<TCoNameValueTuple>(ctx, srcStruct->Pos()) - .Name() - .Value(item->GetName()) + .Name() + .Value(item->GetName()) .Build() .Value<TCoMember>() .Struct(srcStruct) .Name() .Value(item->GetName()) - .Build() - .Build() + .Build() + .Build() .Done().Ptr() - ); - } + ); + } return Build<TCoAsStruct>(ctx, srcStruct->Pos()) - .Add(std::move(nonSystemMembers)) - .Done() - .Ptr(); + .Add(std::move(nonSystemMembers)) + .Done() + .Ptr(); }; if (targetItemType->GetKind() == ETypeAnnotationKind::Struct) { @@ -624,10 +624,10 @@ TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprCont .Add(nonSystemMembers) .Build() .Done().Ptr(); - } - + } + return rebuildStruct(node->HeadPtr(), targetItemType->Cast<TStructExprType>()); - } + } YQL_ENSURE(targetItemType->GetKind() == ETypeAnnotationKind::Variant); @@ -734,8 +734,8 @@ TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprCont }) .Seal() .Build(); -} - +} + TExprNode::TPtr ExpandFlattenMembers(const TExprNode::TPtr& node, TExprContext& ctx) { TExprNode::TListType members; for (auto& child : node->Children()) { diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 67a9759968..aa782ee93f 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -159,9 +159,9 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TVector<TCoNameValueTuple> other; TVector<TCoIndex> indexes; TVector<TCoChangefeed> changefeeds; - TMaybeNode<TExprList> columnFamilies; - TVector<TCoNameValueTuple> tableSettings; - TVector<TCoNameValueTuple> alterActions; + TMaybeNode<TExprList> columnFamilies; + TVector<TCoNameValueTuple> tableSettings; + TVector<TCoNameValueTuple> alterActions; for (auto child : node) { if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) { auto tuple = maybeTuple.Cast(); @@ -223,19 +223,19 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } } changefeeds.push_back(cf.Done()); - } else if (name == "columnFamilies") { - YQL_ENSURE(tuple.Value().Maybe<TExprList>()); - columnFamilies = tuple.Value().Cast<TExprList>(); - } else if (name == "tableSettings") { - YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); - for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { - tableSettings.push_back(item); - } - } else if (name == "actions") { - YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); - for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { - alterActions.push_back(item); - } + } else if (name == "columnFamilies") { + YQL_ENSURE(tuple.Value().Maybe<TExprList>()); + columnFamilies = tuple.Value().Cast<TExprList>(); + } else if (name == "tableSettings") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { + tableSettings.push_back(item); + } + } else if (name == "actions") { + YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>()); + for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { + alterActions.push_back(item); + } } else { other.push_back(tuple); } @@ -254,18 +254,18 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { .Add(changefeeds) .Done(); - const auto& tableProfileSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) - .Add(tableSettings) - .Done(); - - const auto& alterTableActions = Build<TCoNameValueTupleList>(ctx, node.Pos()) - .Add(alterActions) - .Done(); - - if (!columnFamilies.IsValid()) { - columnFamilies = Build<TExprList>(ctx, node.Pos()).Done(); - } - + const auto& tableProfileSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) + .Add(tableSettings) + .Done(); + + const auto& alterTableActions = Build<TCoNameValueTupleList>(ctx, node.Pos()) + .Add(alterActions) + .Done(); + + if (!columnFamilies.IsValid()) { + columnFamilies = Build<TExprList>(ctx, node.Pos()).Done(); + } + TWriteTableSettings ret(otherSettings); ret.Mode = mode; ret.Columns = columns; @@ -276,45 +276,45 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.Update = update; ret.Indexes = idx; ret.Changefeeds = cfs; - ret.ColumnFamilies = columnFamilies; - ret.TableSettings = tableProfileSettings; - ret.AlterActions = alterTableActions; + ret.ColumnFamilies = columnFamilies; + ret.TableSettings = tableProfileSettings; + ret.AlterActions = alterTableActions; + + return ret; +} + +TWriteRoleSettings ParseWriteRoleSettings(TExprList node, TExprContext& ctx) { + TMaybeNode<TCoAtom> mode; + TMaybeNode<TCoAtomList> roles; + TVector<TCoNameValueTuple> other; + for (auto child : node) { + if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) { + auto tuple = maybeTuple.Cast(); + auto name = tuple.Name().Value(); + + if (name == "mode") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + mode = tuple.Value().Cast<TCoAtom>(); + } else if (name == "roles") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>()); + roles = tuple.Value().Cast<TCoAtomList>(); + } else { + other.push_back(tuple); + } + } + } + + const auto& otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) + .Add(other) + .Done(); + + TWriteRoleSettings ret(otherSettings); + ret.Roles = roles; + ret.Mode = mode; return ret; } -TWriteRoleSettings ParseWriteRoleSettings(TExprList node, TExprContext& ctx) { - TMaybeNode<TCoAtom> mode; - TMaybeNode<TCoAtomList> roles; - TVector<TCoNameValueTuple> other; - for (auto child : node) { - if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) { - auto tuple = maybeTuple.Cast(); - auto name = tuple.Name().Value(); - - if (name == "mode") { - YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); - mode = tuple.Value().Cast<TCoAtom>(); - } else if (name == "roles") { - YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>()); - roles = tuple.Value().Cast<TCoAtomList>(); - } else { - other.push_back(tuple); - } - } - } - - const auto& otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos()) - .Add(other) - .Done(); - - TWriteRoleSettings ret(otherSettings); - ret.Roles = roles; - ret.Mode = mode; - - return ret; -} - TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx) { if (!node.Settings()) { return TCommitSettings(Build<TCoNameValueTupleList>(ctx, node.Pos()).Done()); diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index f565ab99ac..9e80cfd135 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -43,23 +43,23 @@ struct TWriteTableSettings { NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes; NNodes::TMaybeNode<NNodes::TCoChangefeedList> Changefeeds; NNodes::TCoNameValueTupleList Other; - NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies; - NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings; - NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> AlterActions; + NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies; + NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings; + NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> AlterActions; TWriteTableSettings(const NNodes::TCoNameValueTupleList& other) : Other(other) {} }; -struct TWriteRoleSettings { - NNodes::TMaybeNode<NNodes::TCoAtom> Mode; - NNodes::TMaybeNode<NNodes::TCoAtomList> Roles; - NNodes::TCoNameValueTupleList Other; - - TWriteRoleSettings(const NNodes::TCoNameValueTupleList& other) - : Other(other) {} -}; - +struct TWriteRoleSettings { + NNodes::TMaybeNode<NNodes::TCoAtom> Mode; + NNodes::TMaybeNode<NNodes::TCoAtomList> Roles; + NNodes::TCoNameValueTupleList Other; + + TWriteRoleSettings(const NNodes::TCoNameValueTupleList& other) + : Other(other) {} +}; + struct TCommitSettings { TPositionHandle Pos; @@ -87,8 +87,8 @@ TVector<TString> GetResOrPullColumnHints(const TExprNode& node); TWriteTableSettings ParseWriteTableSettings(NNodes::TExprList node, TExprContext& ctx); -TWriteRoleSettings ParseWriteRoleSettings(NNodes::TExprList node, TExprContext& ctx); - +TWriteRoleSettings ParseWriteRoleSettings(NNodes::TExprList node, TExprContext& ctx); + TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx); TString FullTableName(const TStringBuf& cluster, const TStringBuf& table); diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index 947853559c..6952e96cf0 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -657,12 +657,12 @@ namespace { } Types.DiscoveryMode = true; } - else if (name == "EnableSystemColumns") { - if (args.size() != 0) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size())); - return false; - } - } + else if (name == "EnableSystemColumns") { + if (args.size() != 0) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size())); + return false; + } + } else if (name == "DqEngine") { if (args.size() != 1) { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size())); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.cpp index 3d4a4995a1..80a9e5c14d 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.cpp @@ -28,18 +28,18 @@ bool TYdbKey::Extract(const TExprNode& key, TExprContext& ctx) { return false; } - Target = nameNode->Head().Content(); + Target = nameNode->Head().Content(); } else if (tagName == "tablescheme") { KeyType = Type::TableScheme; - Target = key.Head().Child(1)->Head().Content(); + Target = key.Head().Child(1)->Head().Content(); } else if (tagName == "tablelist") { KeyType = Type::TableList; - Target = key.Head().Child(1)->Head().Content(); - } else if (tagName == "role") { - KeyType = Type::Role; - Target = key.Head().Child(1)->Head().Content(); + Target = key.Head().Child(1)->Head().Content(); + } else if (tagName == "role") { + KeyType = Type::Role; + Target = key.Head().Child(1)->Head().Content(); } else { - ctx.AddError(TIssue(ctx.GetPosition(key.Head().Pos()), TString("Unexpected tag for YDB key: ") += tagName)); + ctx.AddError(TIssue(ctx.GetPosition(key.Head().Pos()), TString("Unexpected tag for YDB key: ") += tagName)); return false; } @@ -63,7 +63,7 @@ bool TYdbKey::Extract(const TExprNode& key, TExprContext& ctx) { View = viewNode->Head().Content(); } else { - ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), TString("Unexpected tag for YDB key child: ") += tag.Content())); + ctx.AddError(TIssue(ctx.GetPosition(tag.Pos()), TString("Unexpected tag for YDB key child: ") += tag.Content())); return false; } } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h index f3f05615f1..454c019d58 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h @@ -31,8 +31,8 @@ public: enum class Type { Table, TableList, - TableScheme, - Role + TableScheme, + Role }; public: @@ -44,12 +44,12 @@ public: std::string_view GetTablePath() const { Y_VERIFY_DEBUG(KeyType == Type::Table || KeyType == Type::TableScheme); - return Target; + return Target; } std::string_view GetFolderPath() const { Y_VERIFY_DEBUG(KeyType == Type::TableList); - return Target; + return Target; } const std::optional<std::string_view>& GetView() const { @@ -60,7 +60,7 @@ public: private: std::optional<Type> KeyType; - std::string_view Target; + std::string_view Target; std::optional<std::string_view> View; }; diff --git a/ydb/library/yql/sql/v0/context.h b/ydb/library/yql/sql/v0/context.h index 39299e418f..45f40fae61 100644 --- a/ydb/library/yql/sql/v0/context.h +++ b/ydb/library/yql/sql/v0/context.h @@ -149,7 +149,7 @@ namespace NSQLTranslationV0 { bool PragmaYsonStrict = false; bool PragmaClassicDivision = true; bool PragmaPullUpFlatMapOverJoin = true; - bool EnableSystemColumns = true; + bool EnableSystemColumns = true; ui32 ResultRowsLimit = 0; ui64 ResultSizeLimit = 0; ui32 PragmaGroupByLimit = 1 << 5; diff --git a/ydb/library/yql/sql/v0/insert.cpp b/ydb/library/yql/sql/v0/insert.cpp index 5b18d69952..1d8aec7134 100644 --- a/ydb/library/yql/sql/v0/insert.cpp +++ b/ydb/library/yql/sql/v0/insert.cpp @@ -194,30 +194,30 @@ public: const auto& sourceColumns = Source->GetColumns(); const auto numColumns = !ColumnsHint.empty() && sourceColumns ? sourceColumns->List.size() : 0; if (ColumnsHint.size() != numColumns) { - ctx.Error(Pos) << "SELECT has " << numColumns << " columns, " << OperationHumanName << " expects: " << ColumnsHint.size(); + ctx.Error(Pos) << "SELECT has " << numColumns << " columns, " << OperationHumanName << " expects: " << ColumnsHint.size(); return false; } - if (numColumns) { - TStringStream str; - bool mismatchFound = false; - for (size_t i = 0; i < numColumns; ++i) { - bool hasName = sourceColumns->NamedColumns[i]; - if (hasName) { - const auto& hintColumn = ColumnsHint[i]; - const auto& sourceColumn = sourceColumns->List[i]; - if (hintColumn != sourceColumn) { - if (!mismatchFound) { - str << "Column names in SELECT don't match column specification in parenthesis"; - mismatchFound = true; - } - str << ". \"" << hintColumn << "\" doesn't match \"" << sourceColumn << "\""; - } - } - } - if (mismatchFound) { - ctx.Warning(Pos, TIssuesIds::YQL_SOURCE_SELECT_COLUMN_MISMATCH) << str.Str(); - } - } + if (numColumns) { + TStringStream str; + bool mismatchFound = false; + for (size_t i = 0; i < numColumns; ++i) { + bool hasName = sourceColumns->NamedColumns[i]; + if (hasName) { + const auto& hintColumn = ColumnsHint[i]; + const auto& sourceColumn = sourceColumns->List[i]; + if (hintColumn != sourceColumn) { + if (!mismatchFound) { + str << "Column names in SELECT don't match column specification in parenthesis"; + mismatchFound = true; + } + str << ". \"" << hintColumn << "\" doesn't match \"" << sourceColumn << "\""; + } + } + } + if (mismatchFound) { + ctx.Warning(Pos, TIssuesIds::YQL_SOURCE_SELECT_COLUMN_MISMATCH) << str.Str(); + } + } return true; } diff --git a/ydb/library/yql/sql/v0/join.cpp b/ydb/library/yql/sql/v0/join.cpp index a08aefe304..78f5affb49 100644 --- a/ydb/library/yql/sql/v0/join.cpp +++ b/ydb/library/yql/sql/v0/join.cpp @@ -513,9 +513,9 @@ public: if (extraMembers) { sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row")); } - if (ctx.EnableSystemColumns && source->IsTableSource()) { - sourceNode = Y("RemoveSystemMembers", sourceNode); - } + if (ctx.EnableSystemColumns && source->IsTableSource()) { + sourceNode = Y("RemoveSystemMembers", sourceNode); + } equiJoin = L(equiJoin, Q(Y(sourceNode, BuildQuotedAtom(source->GetPos(), source->GetLabel())))); } TNodePtr removeMembers; diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp index efe13245e3..57f0c6a561 100644 --- a/ydb/library/yql/sql/v0/node.cpp +++ b/ydb/library/yql/sql/v0/node.cpp @@ -687,7 +687,7 @@ TNodePtr BuildQuotedAtom(TPosition pos, const TString& content, ui32 flags) { return new TQuotedAtomNode(pos, content, flags); } -bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, bool isReliable, bool hasName) { +bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, bool isReliable, bool hasName) { if (!column || *column == "*") { if (!countHint) { SetAll(); @@ -708,7 +708,7 @@ bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, boo } if (std::find(List.begin(), List.end(), *column) == List.end()) { List.push_back(*column); - NamedColumns.push_back(hasName); + NamedColumns.push_back(hasName); } return inserted; } @@ -1656,10 +1656,10 @@ bool ISource::IsSelect() const { return true; } -bool ISource::IsTableSource() const { - return false; -} - +bool ISource::IsTableSource() const { + return false; +} + bool ISource::ShouldUseSourceAsColumn(const TString& source) { Y_UNUSED(source); return false; diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index 86d65ceb96..4c7cfc273b 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -458,12 +458,12 @@ namespace NSQLTranslationV0 { TSet<TString> Real; TSet<TString> Artificial; TVector<TString> List; - TVector<bool> NamedColumns; + TVector<bool> NamedColumns; bool All = false; bool QualifiedAll = false; bool HasUnreliable = false; - bool Add(const TString* column, bool countHint, bool isArtificial = false, bool isReliable = true, bool hasName = true); + bool Add(const TString* column, bool countHint, bool isArtificial = false, bool isReliable = true, bool hasName = true); void Merge(const TColumns& columns); void SetPrefix(const TString& prefix); void SetAll(); @@ -742,7 +742,7 @@ namespace NSQLTranslationV0 { virtual IJoin* GetJoin(); virtual ISource* GetCompositeSource(); virtual bool IsSelect() const; - virtual bool IsTableSource() const; + virtual bool IsTableSource() const; virtual bool ShouldUseSourceAsColumn(const TString& source); virtual bool IsJoinKeysInitializing() const; virtual const TString* GetWindowName() const; diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 77d3af5464..7752b28b01 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -595,16 +595,16 @@ public: } keys = ctx.GroundBlockShortcutsForExpr(keys); - auto actions = Y(); - + auto actions = Y(); + if (Intent == EAlterTableIntentnt::DropColumn) { - auto columns = Y(); - for (auto& col : Columns) { - columns = L(columns, BuildQuotedAtom(Pos, col.Name)); + auto columns = Y(); + for (auto& col : Columns) { + columns = L(columns, BuildQuotedAtom(Pos, col.Name)); } - actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); + actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); } else { - auto columns = Y(); + auto columns = Y(); for (auto& col: Columns) { auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos); if (!type) { @@ -632,13 +632,13 @@ public: } columns = L(columns, Q(Y(BuildQuotedAtom(Pos, col.Name), AstNode(type)))); } - actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); + actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); } - auto opts = Y(); - + auto opts = Y(); + opts = L(opts, Q(Y(Q("mode"), Q("alter")))); - opts = L(opts, Q(Y(Q("actions"), Q(actions)))); + opts = L(opts, Q(Y(Q("actions"), Q(actions)))); Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.ServiceName(ctx)), BuildQuotedAtom(Pos, Table.Cluster))), diff --git a/ydb/library/yql/sql/v0/select.cpp b/ydb/library/yql/sql/v0/select.cpp index 227686b7c8..ef47023be1 100644 --- a/ydb/library/yql/sql/v0/select.cpp +++ b/ydb/library/yql/sql/v0/select.cpp @@ -644,10 +644,10 @@ public: TPtr DoClone() const final { return new TTableSource(Pos, Table, Stream, GetLabel()); } - - bool IsTableSource() const override { - return true; - } + + bool IsTableSource() const override { + return true; + } protected: TTableRef Table; const bool Stream; @@ -1428,7 +1428,7 @@ private: GroupBy.push_back(BuildColumn(Pos, columnName)); } TString label(term->GetLabel()); - bool hasName = true; + bool hasName = true; if (label.empty()) { auto source = term->GetSourceName(); if (term->IsAsterisk() && !source->empty()) { @@ -1438,10 +1438,10 @@ private: label = isJoin && source && *source ? DotJoin(*source, *column) : *column; } else { label = TStringBuilder() << "column" << Columns.List.size(); - hasName = false; + hasName = false; } } - if (!Columns.Add(&label, false, false, true, hasName)) { + if (!Columns.Add(&label, false, false, true, hasName)) { ctx.Error(Pos) << "Duplicate column: " << label; hasError = true; continue; @@ -1518,11 +1518,11 @@ private: if (Columns.All) { Y_VERIFY_DEBUG(Columns.List.empty()); terms = PrepareWithout(Y()); - if (ctx.EnableSystemColumns) { - terms = L(terms, Y("let", "res", Y("AsList", Y("RemoveSystemMembers", "row")))); - } else { - terms = L(terms, (Y("let", "res", Y("AsList", "row")))); - } + if (ctx.EnableSystemColumns) { + terms = L(terms, Y("let", "res", Y("AsList", Y("RemoveSystemMembers", "row")))); + } else { + terms = L(terms, (Y("let", "res", Y("AsList", "row")))); + } } else if (!Columns.List.empty()) { Y_VERIFY_DEBUG(Columns.List.size() == Terms.size()); const bool isJoin = Source->GetJoin(); diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index 9cd0893222..b028fc0439 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -4815,13 +4815,13 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disablepullupflatmapoverjoin") { Ctx.PragmaPullUpFlatMapOverJoin = false; Ctx.IncrementMonCounter("sql_pragma", "DisablePullUpFlatMapOverJoin"); - } else if (normalizedPragma == "enablesystemcolumns") { - if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { - Error() << "Expected single boolean argument for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - Ctx.IncrementMonCounter("sql_pragma", "EnableSystemColumns"); + } else if (normalizedPragma == "enablesystemcolumns") { + if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { + Error() << "Expected single boolean argument for: " << pragma; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + Ctx.IncrementMonCounter("sql_pragma", "EnableSystemColumns"); } else { Error() << "Unknown pragma: " << pragma; Ctx.IncrementMonCounter("sql_errors", "UnknownPragma"); diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 483603acaf..3a70aef93c 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -1715,12 +1715,12 @@ select FormatType($f()); UNIT_ASSERT(res.Root); } - Y_UNIT_TEST(WarnSourceColumnMismatch) { - NYql::TAstParseResult res = SqlToYql( - "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); - } + Y_UNIT_TEST(WarnSourceColumnMismatch) { + NYql::TAstParseResult res = SqlToYql( + "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); + } Y_UNIT_TEST(YtCaseInsensitive) { NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 2ab1a7cc57..ef394c7a00 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -448,8 +448,8 @@ values_source_row: LPAREN expr_list RPAREN; simple_values_source: expr_list | select_stmt; -create_table_stmt: CREATE TABLE simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN - with_table_settings?; +create_table_stmt: CREATE TABLE simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN + with_table_settings?; create_table_entry: column_schema | table_constraint @@ -458,16 +458,16 @@ create_table_entry: | changefeed ; -with_table_settings: WITH LPAREN table_settings_entry (COMMA table_settings_entry)* RPAREN; -table_settings_entry: an_id EQUALS table_setting_value; +with_table_settings: WITH LPAREN table_settings_entry (COMMA table_settings_entry)* RPAREN; +table_settings_entry: an_id EQUALS table_setting_value; -alter_table_stmt: ALTER TABLE simple_table_ref alter_table_action (COMMA alter_table_action)*; -alter_table_action: - alter_table_add_column - | alter_table_drop_column - | alter_table_alter_column - | alter_table_add_column_family - | alter_table_alter_column_family +alter_table_stmt: ALTER TABLE simple_table_ref alter_table_action (COMMA alter_table_action)*; +alter_table_action: + alter_table_add_column + | alter_table_drop_column + | alter_table_alter_column + | alter_table_add_column_family + | alter_table_alter_column_family | alter_table_set_table_setting_uncompat | alter_table_set_table_setting_compat | alter_table_reset_table_setting @@ -478,12 +478,12 @@ alter_table_action: | alter_table_alter_changefeed | alter_table_drop_changefeed ; - -alter_table_add_column: ADD COLUMN? column_schema; + +alter_table_add_column: ADD COLUMN? column_schema; alter_table_drop_column: DROP COLUMN? an_id; -alter_table_alter_column: ALTER COLUMN an_id SET family_relation; -alter_table_add_column_family: ADD family_entry; -alter_table_alter_column_family: ALTER FAMILY an_id SET an_id family_setting_value; +alter_table_alter_column: ALTER COLUMN an_id SET family_relation; +alter_table_add_column_family: ADD family_entry; +alter_table_alter_column_family: ALTER FAMILY an_id SET an_id family_setting_value; alter_table_set_table_setting_uncompat: SET an_id table_setting_value; alter_table_set_table_setting_compat: SET LPAREN alter_table_setting_entry (COMMA alter_table_setting_entry)* RPAREN; alter_table_reset_table_setting: RESET LPAREN an_id (COMMA an_id)* RPAREN; @@ -494,8 +494,8 @@ alter_table_add_changefeed: ADD changefeed; alter_table_alter_changefeed: ALTER CHANGEFEED an_id changefeed_alter_settings; alter_table_drop_changefeed: DROP CHANGEFEED an_id; -column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; -family_relation: FAMILY an_id; +column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; +family_relation: FAMILY an_id; column_order_by_specification: an_id (ASC | DESC)?; table_constraint: @@ -528,26 +528,26 @@ changefeed_alter_settings: alter_table_setting_entry: an_id EQUALS table_setting_value; -table_setting_value: - id - | STRING_VALUE - | integer - | split_boundaries +table_setting_value: + id + | STRING_VALUE + | integer + | split_boundaries | expr ON an_id -; - -family_entry: FAMILY an_id family_settings?; -family_settings: LPAREN family_settings_entry (COMMA family_settings_entry)* RPAREN; -family_settings_entry: an_id EQUALS family_setting_value; -family_setting_value: STRING_VALUE; - -split_boundaries: - LPAREN literal_value_list (COMMA literal_value_list)* RPAREN - | literal_value_list -; - -literal_value_list: LPAREN literal_value (COMMA literal_value)* RPAREN; - +; + +family_entry: FAMILY an_id family_settings?; +family_settings: LPAREN family_settings_entry (COMMA family_settings_entry)* RPAREN; +family_settings_entry: an_id EQUALS family_setting_value; +family_setting_value: STRING_VALUE; + +split_boundaries: + LPAREN literal_value_list (COMMA literal_value_list)* RPAREN + | literal_value_list +; + +literal_value_list: LPAREN literal_value (COMMA literal_value)* RPAREN; + drop_table_stmt: DROP TABLE (IF EXISTS)? simple_table_ref; create_user_stmt: CREATE USER role_name create_user_option?; @@ -1193,7 +1193,7 @@ EXPLAIN: E X P L A I N; EXPORT: E X P O R T; EXTERNAL: E X T E R N A L; FAIL: F A I L; -FAMILY: F A M I L Y; +FAMILY: F A M I L Y; FILTER: F I L T E R; FLATTEN: F L A T T E N; FLOW: F L O W; diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index 3cf640971d..8951757486 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -233,7 +233,7 @@ namespace NSQLTranslationV1 { bool PragmaPullUpFlatMapOverJoin = true; bool WarnUnnamedColumns = false; bool DiscoveryMode = false; - bool EnableSystemColumns = true; + bool EnableSystemColumns = true; bool DqEngineEnable = false; bool DqEngineForce = false; TMaybe<bool> JsonQueryReturnsJsonDocument; diff --git a/ydb/library/yql/sql/v1/insert.cpp b/ydb/library/yql/sql/v1/insert.cpp index b361e8e976..4f4c863018 100644 --- a/ydb/library/yql/sql/v1/insert.cpp +++ b/ydb/library/yql/sql/v1/insert.cpp @@ -196,27 +196,27 @@ public: ctx.Error(Pos) << "SELECT have " << numColumns << " columns, " << OperationHumanName << " expects: " << ColumnsHint.size(); return false; } - if (numColumns) { - TStringStream str; - bool mismatchFound = false; - for (size_t i = 0; i < numColumns; ++i) { - bool hasName = sourceColumns->NamedColumns[i]; - if (hasName) { - const auto& hintColumn = ColumnsHint[i]; - const auto& sourceColumn = sourceColumns->List[i]; - if (hintColumn != sourceColumn) { - if (!mismatchFound) { - str << "Column names in SELECT don't match column specification in parenthesis"; - mismatchFound = true; - } - str << ". \"" << hintColumn << "\" doesn't match \"" << sourceColumn << "\""; - } - } - } - if (mismatchFound) { - ctx.Warning(Pos, TIssuesIds::YQL_SOURCE_SELECT_COLUMN_MISMATCH) << str.Str(); - } - } + if (numColumns) { + TStringStream str; + bool mismatchFound = false; + for (size_t i = 0; i < numColumns; ++i) { + bool hasName = sourceColumns->NamedColumns[i]; + if (hasName) { + const auto& hintColumn = ColumnsHint[i]; + const auto& sourceColumn = sourceColumns->List[i]; + if (hintColumn != sourceColumn) { + if (!mismatchFound) { + str << "Column names in SELECT don't match column specification in parenthesis"; + mismatchFound = true; + } + str << ". \"" << hintColumn << "\" doesn't match \"" << sourceColumn << "\""; + } + } + } + if (mismatchFound) { + ctx.Warning(Pos, TIssuesIds::YQL_SOURCE_SELECT_COLUMN_MISMATCH) << str.Str(); + } + } return true; } diff --git a/ydb/library/yql/sql/v1/join.cpp b/ydb/library/yql/sql/v1/join.cpp index 6707905a41..29f5b3ec4b 100644 --- a/ydb/library/yql/sql/v1/join.cpp +++ b/ydb/library/yql/sql/v1/join.cpp @@ -567,9 +567,9 @@ public: if (extraMembers) { sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row")); } - if (ctx.EnableSystemColumns && source->IsTableSource()) { - sourceNode = Y("RemoveSystemMembers", sourceNode); - } + if (ctx.EnableSystemColumns && source->IsTableSource()) { + sourceNode = Y("RemoveSystemMembers", sourceNode); + } equiJoin = L(equiJoin, Q(Y(sourceNode, BuildQuotedAtom(source->GetPos(), source->GetLabel())))); } TNodePtr removeMembers; diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index c54045faab..870bbd9cc7 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -44,13 +44,13 @@ TString TTableRef::ShortName() const { return TString(); } -TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TNodePtr& type, bool nullable, - TVector<TIdentifier> families) +TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TNodePtr& type, bool nullable, + TVector<TIdentifier> families) : Pos(pos) , Name(name) , Type(type) , Nullable(nullable) - , Families(families) + , Families(families) { } @@ -867,7 +867,7 @@ TNodePtr BuildQuotedAtom(TPosition pos, const TString& content, ui32 flags) { return new TQuotedAtomNode(pos, content, flags); } -bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, bool isReliable, bool hasName) { +bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, bool isReliable, bool hasName) { if (!column || *column == "*") { if (!countHint) { SetAll(); @@ -888,7 +888,7 @@ bool TColumns::Add(const TString* column, bool countHint, bool isArtificial, boo } if (std::find(List.begin(), List.end(), *column) == List.end()) { List.push_back(*column); - NamedColumns.push_back(hasName); + NamedColumns.push_back(hasName); } return inserted; } @@ -2005,10 +2005,10 @@ bool ISource::IsSelect() const { return true; } -bool ISource::IsTableSource() const { - return false; -} - +bool ISource::IsTableSource() const { + return false; +} + bool ISource::ShouldUseSourceAsColumn(const TString& source) const { Y_UNUSED(source); return false; diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 571294ecdf..de82b45a23 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -533,22 +533,22 @@ namespace NSQLTranslationV1 { TString Name; TNodePtr Type; bool Nullable; - TVector<TIdentifier> Families; + TVector<TIdentifier> Families; - TColumnSchema(TPosition pos, const TString& name, const TNodePtr& type, bool nullable, - TVector<TIdentifier> families); + TColumnSchema(TPosition pos, const TString& name, const TNodePtr& type, bool nullable, + TVector<TIdentifier> families); }; struct TColumns: public TSimpleRefCount<TColumns> { TSet<TString> Real; TSet<TString> Artificial; TVector<TString> List; - TVector<bool> NamedColumns; + TVector<bool> NamedColumns; bool All = false; bool QualifiedAll = false; bool HasUnreliable = false; - bool Add(const TString* column, bool countHint, bool isArtificial = false, bool isReliable = true, bool hasName = true); + bool Add(const TString* column, bool countHint, bool isArtificial = false, bool isReliable = true, bool hasName = true); void Merge(const TColumns& columns); void SetPrefix(const TString& prefix); void SetAll(); @@ -864,7 +864,7 @@ namespace NSQLTranslationV1 { virtual IJoin* GetJoin(); virtual ISource* GetCompositeSource(); virtual bool IsSelect() const; - virtual bool IsTableSource() const; + virtual bool IsTableSource() const; virtual bool ShouldUseSourceAsColumn(const TString& source) const; virtual bool IsJoinKeysInitializing() const; virtual const TString* GetWindowName() const; @@ -1051,7 +1051,7 @@ namespace NSQLTranslationV1 { TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input); TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input); - + struct TTtlSettings { TIdentifier ColumnName; TNodePtr Expr; @@ -1059,36 +1059,36 @@ namespace NSQLTranslationV1 { TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr); }; - struct TTableSettings { - TNodePtr CompactionPolicy; - TMaybe<TIdentifier> AutoPartitioningBySize; - TNodePtr PartitionSizeMb; - TMaybe<TIdentifier> AutoPartitioningByLoad; - TNodePtr MinPartitions; - TNodePtr MaxPartitions; - TNodePtr UniformPartitions; - TVector<TVector<TNodePtr>> PartitionAtKeys; - TMaybe<TIdentifier> KeyBloomFilter; - TNodePtr ReadReplicasSettings; + struct TTableSettings { + TNodePtr CompactionPolicy; + TMaybe<TIdentifier> AutoPartitioningBySize; + TNodePtr PartitionSizeMb; + TMaybe<TIdentifier> AutoPartitioningByLoad; + TNodePtr MinPartitions; + TNodePtr MaxPartitions; + TNodePtr UniformPartitions; + TVector<TVector<TNodePtr>> PartitionAtKeys; + TMaybe<TIdentifier> KeyBloomFilter; + TNodePtr ReadReplicasSettings; NYql::TResetableSetting<TTtlSettings, void> TtlSettings; - bool IsSet() const { - return CompactionPolicy || AutoPartitioningBySize || PartitionSizeMb || AutoPartitioningByLoad - || MinPartitions || MaxPartitions || UniformPartitions || PartitionAtKeys || KeyBloomFilter + bool IsSet() const { + return CompactionPolicy || AutoPartitioningBySize || PartitionSizeMb || AutoPartitioningByLoad + || MinPartitions || MaxPartitions || UniformPartitions || PartitionAtKeys || KeyBloomFilter || ReadReplicasSettings || TtlSettings; - } - }; - - struct TFamilyEntry { - TFamilyEntry(const TIdentifier& name) - :Name(name) - {} - - TIdentifier Name; - TNodePtr Data; - TNodePtr Compression; - }; - + } + }; + + struct TFamilyEntry { + TFamilyEntry(const TIdentifier& name) + :Name(name) + {} + + TIdentifier Name; + TNodePtr Data; + TNodePtr Compression; + }; + struct TIndexDescription { enum class EType { GlobalSync, @@ -1127,24 +1127,24 @@ namespace NSQLTranslationV1 { bool Disable; }; - struct TCreateTableParameters { - TVector<TColumnSchema> Columns; - TVector<TIdentifier> PkColumns; - TVector<TIdentifier> PartitionByColumns; - TVector<std::pair<TIdentifier, bool>> OrderByColumns; + struct TCreateTableParameters { + TVector<TColumnSchema> Columns; + TVector<TIdentifier> PkColumns; + TVector<TIdentifier> PartitionByColumns; + TVector<std::pair<TIdentifier, bool>> OrderByColumns; TVector<TIndexDescription> Indexes; - TVector<TFamilyEntry> ColumnFamilies; + TVector<TFamilyEntry> ColumnFamilies; TVector<TChangefeedDescription> Changefeeds; - TTableSettings TableSettings; - }; - - struct TAlterTableParameters { - TVector<TColumnSchema> AddColumns; - TVector<TString> DropColumns; - TVector<TColumnSchema> AlterColumns; - TVector<TFamilyEntry> AddColumnFamilies; - TVector<TFamilyEntry> AlterColumnFamilies; - TTableSettings TableSettings; + TTableSettings TableSettings; + }; + + struct TAlterTableParameters { + TVector<TColumnSchema> AddColumns; + TVector<TString> DropColumns; + TVector<TColumnSchema> AlterColumns; + TVector<TFamilyEntry> AddColumnFamilies; + TVector<TFamilyEntry> AlterColumnFamilies; + TTableSettings TableSettings; TVector<TIndexDescription> AddIndexes; TVector<TIdentifier> DropIndexes; TMaybe<TIdentifier> RenameTo; @@ -1160,8 +1160,8 @@ namespace NSQLTranslationV1 { && !RenameTo.Defined() && AddChangefeeds.empty() && AlterChangefeeds.empty() && DropChangefeeds.empty(); } - }; - + }; + struct TRoleParameters { TMaybe<TDeferredAtom> Password; bool IsPasswordEncrypted = false; @@ -1321,8 +1321,8 @@ namespace NSQLTranslationV1 { TNodePtr BuildTableKeys(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TString& func, const TVector<TTableArg>& args); TNodePtr BuildInputOptions(TPosition pos, const TTableHints& hints); TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery, TScopedStatePtr scoped); - TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped); - TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); + TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped); + TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, TScopedStatePtr scoped); TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped); TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 5598f6c907..725356b9f5 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -562,10 +562,10 @@ TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubque class TCreateTableNode final: public TAstListNode { public: - TCreateTableNode(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped) + TCreateTableNode(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped) : TAstListNode(pos) , Table(tr) - , Params(params) + , Params(params) , Scoped(scoped) { scoped->UseCluster(Table.Service, Table.Cluster); @@ -584,23 +584,23 @@ public: || !Params.Changefeeds.empty()) { THashSet<TString> columnsSet; - for (auto& col : Params.Columns) { + for (auto& col : Params.Columns) { columnsSet.insert(col.Name); } - for (auto& keyColumn : Params.PkColumns) { + for (auto& keyColumn : Params.PkColumns) { if (!columnsSet.contains(keyColumn.Name)) { ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; return false; } } - for (auto& keyColumn : Params.PartitionByColumns) { + for (auto& keyColumn : Params.PartitionByColumns) { if (!columnsSet.contains(keyColumn.Name)) { ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name; return false; } } - for (auto& keyColumn : Params.OrderByColumns) { + for (auto& keyColumn : Params.OrderByColumns) { if (!columnsSet.contains(keyColumn.first.Name)) { ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name; return false; @@ -638,22 +638,22 @@ public: } } auto columns = Y(); - for (auto& col : Params.Columns) { - auto columnDesc = Y(); - columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); + for (auto& col : Params.Columns) { + auto columnDesc = Y(); + columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); auto type = col.Type; if (col.Nullable) { type = Y("OptionalType", type); } - columnDesc = L(columnDesc, type); - if (col.Families) { - auto familiesDesc = Y(); - for (const auto& family : col.Families) { - familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); - } - columnDesc = L(columnDesc, Q(familiesDesc)); - } - columns = L(columns, Q(columnDesc)); + columnDesc = L(columnDesc, type); + if (col.Families) { + auto familiesDesc = Y(); + for (const auto& family : col.Families) { + familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); + } + columnDesc = L(columnDesc, Q(familiesDesc)); + } + columns = L(columns, Q(columnDesc)); } auto opts = Y(); @@ -667,43 +667,43 @@ public: opts = L(opts, Q(Y(Q("columns"), Q(columns)))); if (Table.Service == RtmrProviderName) { - if (!Params.PkColumns.empty() && !Params.PartitionByColumns.empty()) { + if (!Params.PkColumns.empty() && !Params.PartitionByColumns.empty()) { ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified"; return false; } } else { - if (!Params.OrderByColumns.empty()) { - ctx.Error() << "ORDER BY is supported only for " << RtmrProviderName << " provider"; + if (!Params.OrderByColumns.empty()) { + ctx.Error() << "ORDER BY is supported only for " << RtmrProviderName << " provider"; return false; } } - if (!Params.PkColumns.empty()) { + if (!Params.PkColumns.empty()) { auto primaryKey = Y(); - for (auto& col : Params.PkColumns) { + for (auto& col : Params.PkColumns) { primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name)); } opts = L(opts, Q(Y(Q("primarykey"), Q(primaryKey)))); - if (!Params.OrderByColumns.empty()) { + if (!Params.OrderByColumns.empty()) { ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead"; return false; } - } - - if (!Params.PartitionByColumns.empty()) { - auto partitionBy = Y(); - for (auto& col : Params.PartitionByColumns) { - partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name)); + } + + if (!Params.PartitionByColumns.empty()) { + auto partitionBy = Y(); + for (auto& col : Params.PartitionByColumns) { + partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name)); } - opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy)))); - } - - if (!Params.OrderByColumns.empty()) { - auto orderBy = Y(); - for (auto& col : Params.OrderByColumns) { - orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0")))); + opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy)))); + } + + if (!Params.OrderByColumns.empty()) { + auto orderBy = Y(); + for (auto& col : Params.OrderByColumns) { + orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0")))); } - opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); + opts = L(opts, Q(Y(Q("orderby"), Q(orderBy)))); } for (const auto& index : Params.Indexes) { @@ -716,66 +716,66 @@ public: opts = L(opts, Q(Y(Q("changefeed"), Q(desc)))); } - if (Params.ColumnFamilies) { - auto columnFamilies = Y(); - for (const auto& family : Params.ColumnFamilies) { - auto familyDesc = Y(); - familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); - if (family.Data) { - familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); - } - if (family.Compression) { - familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); - } - columnFamilies = L(columnFamilies, Q(familyDesc)); - } - opts = L(opts, Q(Y(Q("columnFamilies"), Q(columnFamilies)))); - } - - if (Params.TableSettings.IsSet()) { - auto settings = Y(); - - if (Params.TableSettings.CompactionPolicy) { - settings = L(settings, Q(Y(Q("compactionPolicy"), Params.TableSettings.CompactionPolicy))); - } - if (Params.TableSettings.AutoPartitioningBySize) { - const auto& ref = Params.TableSettings.AutoPartitioningBySize.GetRef(); - settings = L(settings, Q(Y(Q("autoPartitioningBySize"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.UniformPartitions) { - settings = L(settings, Q(Y(Q("uniformPartitions"), Params.TableSettings.UniformPartitions))); - } - if (Params.TableSettings.PartitionAtKeys) { - auto keysDesc = Y(); - for (const auto& key : Params.TableSettings.PartitionAtKeys) { - auto columnsDesc = Y(); - for (auto column : key) { - columnsDesc = L(columnsDesc, column); - } - keysDesc = L(keysDesc, Q(columnsDesc)); - } - settings = L(settings, Q(Y(Q("partitionAtKeys"), Q(keysDesc)))); - } - if (Params.TableSettings.PartitionSizeMb) { - settings = L(settings, Q(Y(Q("partitionSizeMb"), Params.TableSettings.PartitionSizeMb))); - } - if (Params.TableSettings.AutoPartitioningByLoad) { - const auto& ref = Params.TableSettings.AutoPartitioningByLoad.GetRef(); - settings = L(settings, Q(Y(Q("autoPartitioningByLoad"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.MinPartitions) { - settings = L(settings, Q(Y(Q("minPartitions"), Params.TableSettings.MinPartitions))); - } - if (Params.TableSettings.MaxPartitions) { - settings = L(settings, Q(Y(Q("maxPartitions"), Params.TableSettings.MaxPartitions))); - } - if (Params.TableSettings.KeyBloomFilter) { - const auto& ref = Params.TableSettings.KeyBloomFilter.GetRef(); - settings = L(settings, Q(Y(Q("keyBloomFilter"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.ReadReplicasSettings) { - settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); - } + if (Params.ColumnFamilies) { + auto columnFamilies = Y(); + for (const auto& family : Params.ColumnFamilies) { + auto familyDesc = Y(); + familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); + if (family.Data) { + familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); + } + if (family.Compression) { + familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); + } + columnFamilies = L(columnFamilies, Q(familyDesc)); + } + opts = L(opts, Q(Y(Q("columnFamilies"), Q(columnFamilies)))); + } + + if (Params.TableSettings.IsSet()) { + auto settings = Y(); + + if (Params.TableSettings.CompactionPolicy) { + settings = L(settings, Q(Y(Q("compactionPolicy"), Params.TableSettings.CompactionPolicy))); + } + if (Params.TableSettings.AutoPartitioningBySize) { + const auto& ref = Params.TableSettings.AutoPartitioningBySize.GetRef(); + settings = L(settings, Q(Y(Q("autoPartitioningBySize"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.UniformPartitions) { + settings = L(settings, Q(Y(Q("uniformPartitions"), Params.TableSettings.UniformPartitions))); + } + if (Params.TableSettings.PartitionAtKeys) { + auto keysDesc = Y(); + for (const auto& key : Params.TableSettings.PartitionAtKeys) { + auto columnsDesc = Y(); + for (auto column : key) { + columnsDesc = L(columnsDesc, column); + } + keysDesc = L(keysDesc, Q(columnsDesc)); + } + settings = L(settings, Q(Y(Q("partitionAtKeys"), Q(keysDesc)))); + } + if (Params.TableSettings.PartitionSizeMb) { + settings = L(settings, Q(Y(Q("partitionSizeMb"), Params.TableSettings.PartitionSizeMb))); + } + if (Params.TableSettings.AutoPartitioningByLoad) { + const auto& ref = Params.TableSettings.AutoPartitioningByLoad.GetRef(); + settings = L(settings, Q(Y(Q("autoPartitioningByLoad"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.MinPartitions) { + settings = L(settings, Q(Y(Q("minPartitions"), Params.TableSettings.MinPartitions))); + } + if (Params.TableSettings.MaxPartitions) { + settings = L(settings, Q(Y(Q("maxPartitions"), Params.TableSettings.MaxPartitions))); + } + if (Params.TableSettings.KeyBloomFilter) { + const auto& ref = Params.TableSettings.KeyBloomFilter.GetRef(); + settings = L(settings, Q(Y(Q("keyBloomFilter"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.ReadReplicasSettings) { + settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); + } if (const auto& ttl = Params.TableSettings.TtlSettings) { if (ttl.IsSet()) { const auto& ttlSettings = ttl.GetValueSet(); @@ -789,10 +789,10 @@ public: YQL_ENSURE(false, "Can't reset TTL settings"); } } - - opts = L(opts, Q(Y(Q("tableSettings"), Q(settings)))); - } - + + opts = L(opts, Q(Y(Q("tableSettings"), Q(settings)))); + } + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), @@ -807,21 +807,21 @@ public: } private: const TTableRef Table; - const TCreateTableParameters Params; + const TCreateTableParameters Params; TScopedStatePtr Scoped; }; -TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped) +TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped) { - return new TCreateTableNode(pos, tr, params, scoped); + return new TCreateTableNode(pos, tr, params, scoped); } class TAlterTableNode final: public TAstListNode { public: - TAlterTableNode(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) + TAlterTableNode(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) : TAstListNode(pos) , Table(tr) - , Params(params) + , Params(params) , Scoped(scoped) { scoped->UseCluster(Table.Service, Table.Cluster); @@ -833,114 +833,114 @@ public: return false; } - auto actions = Y(); - - if (Params.AddColumns) { - auto columns = Y(); - for (auto& col : Params.AddColumns) { - auto columnDesc = Y(); - columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); + auto actions = Y(); + + if (Params.AddColumns) { + auto columns = Y(); + for (auto& col : Params.AddColumns) { + auto columnDesc = Y(); + columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); auto type = col.Type; if (col.Nullable) { type = Y("OptionalType", type); } - columnDesc = L(columnDesc, type); - if (col.Families) { - auto familiesDesc = Y(); - for (const auto& family : col.Families) { - familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); - } - columnDesc = L(columnDesc, Q(familiesDesc)); - } - columns = L(columns, Q(columnDesc)); - } - actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); - } - - if (Params.DropColumns) { - auto columns = Y(); - for (auto& colName : Params.DropColumns) { - columns = L(columns, BuildQuotedAtom(Pos, colName)); - } - actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); - } - - if (Params.AlterColumns) { - auto columns = Y(); - for (auto& col : Params.AlterColumns) { - auto columnDesc = Y(); - columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); - auto familiesDesc = Y(); - for (const auto& family : col.Families) { - familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); - } - columnDesc = L(columnDesc, Q(familiesDesc)); - columns = L(columns, Q(columnDesc)); - } - actions = L(actions, Q(Y(Q("alterColumns"), Q(columns)))); - } - - if (Params.AddColumnFamilies) { - auto columnFamilies = Y(); - for (const auto& family : Params.AddColumnFamilies) { - auto familyDesc = Y(); - familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); - if (family.Data) { - familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); - } - if (family.Compression) { - familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); - } - columnFamilies = L(columnFamilies, Q(familyDesc)); - } - actions = L(actions, Q(Y(Q("addColumnFamilies"), Q(columnFamilies)))); - } - - if (Params.AlterColumnFamilies) { - auto columnFamilies = Y(); - for (const auto& family : Params.AlterColumnFamilies) { - auto familyDesc = Y(); - familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); - if (family.Data) { - familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); - } - if (family.Compression) { - familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); - } - columnFamilies = L(columnFamilies, Q(familyDesc)); - } - actions = L(actions, Q(Y(Q("alterColumnFamilies"), Q(columnFamilies)))); - } - - if (Params.TableSettings.IsSet()) { - auto settings = Y(); - if (Params.TableSettings.CompactionPolicy) { - settings = L(settings, Q(Y(Q("compactionPolicy"), Params.TableSettings.CompactionPolicy))); - } - if (Params.TableSettings.AutoPartitioningBySize) { - const auto& ref = Params.TableSettings.AutoPartitioningBySize.GetRef(); - settings = L(settings, Q(Y(Q("autoPartitioningBySize"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.PartitionSizeMb) { - settings = L(settings, Q(Y(Q("partitionSizeMb"), Params.TableSettings.PartitionSizeMb))); - } - if (Params.TableSettings.AutoPartitioningByLoad) { - const auto& ref = Params.TableSettings.AutoPartitioningByLoad.GetRef(); - settings = L(settings, Q(Y(Q("autoPartitioningByLoad"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.MinPartitions) { - settings = L(settings, Q(Y(Q("minPartitions"), Params.TableSettings.MinPartitions))); - } - if (Params.TableSettings.MaxPartitions) { - settings = L(settings, Q(Y(Q("maxPartitions"), Params.TableSettings.MaxPartitions))); - } - if (Params.TableSettings.KeyBloomFilter) { - const auto& ref = Params.TableSettings.KeyBloomFilter.GetRef(); - settings = L(settings, Q(Y(Q("keyBloomFilter"), BuildQuotedAtom(ref.Pos, ref.Name)))); - } - if (Params.TableSettings.ReadReplicasSettings) { - settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); - } + columnDesc = L(columnDesc, type); + if (col.Families) { + auto familiesDesc = Y(); + for (const auto& family : col.Families) { + familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); + } + columnDesc = L(columnDesc, Q(familiesDesc)); + } + columns = L(columns, Q(columnDesc)); + } + actions = L(actions, Q(Y(Q("addColumns"), Q(columns)))); + } + + if (Params.DropColumns) { + auto columns = Y(); + for (auto& colName : Params.DropColumns) { + columns = L(columns, BuildQuotedAtom(Pos, colName)); + } + actions = L(actions, Q(Y(Q("dropColumns"), Q(columns)))); + } + + if (Params.AlterColumns) { + auto columns = Y(); + for (auto& col : Params.AlterColumns) { + auto columnDesc = Y(); + columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); + auto familiesDesc = Y(); + for (const auto& family : col.Families) { + familiesDesc = L(familiesDesc, BuildQuotedAtom(family.Pos, family.Name)); + } + columnDesc = L(columnDesc, Q(familiesDesc)); + columns = L(columns, Q(columnDesc)); + } + actions = L(actions, Q(Y(Q("alterColumns"), Q(columns)))); + } + + if (Params.AddColumnFamilies) { + auto columnFamilies = Y(); + for (const auto& family : Params.AddColumnFamilies) { + auto familyDesc = Y(); + familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); + if (family.Data) { + familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); + } + if (family.Compression) { + familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); + } + columnFamilies = L(columnFamilies, Q(familyDesc)); + } + actions = L(actions, Q(Y(Q("addColumnFamilies"), Q(columnFamilies)))); + } + + if (Params.AlterColumnFamilies) { + auto columnFamilies = Y(); + for (const auto& family : Params.AlterColumnFamilies) { + auto familyDesc = Y(); + familyDesc = L(familyDesc, Q(Y(Q("name"), BuildQuotedAtom(family.Name.Pos, family.Name.Name)))); + if (family.Data) { + familyDesc = L(familyDesc, Q(Y(Q("data"), family.Data))); + } + if (family.Compression) { + familyDesc = L(familyDesc, Q(Y(Q("compression"), family.Compression))); + } + columnFamilies = L(columnFamilies, Q(familyDesc)); + } + actions = L(actions, Q(Y(Q("alterColumnFamilies"), Q(columnFamilies)))); + } + + if (Params.TableSettings.IsSet()) { + auto settings = Y(); + if (Params.TableSettings.CompactionPolicy) { + settings = L(settings, Q(Y(Q("compactionPolicy"), Params.TableSettings.CompactionPolicy))); + } + if (Params.TableSettings.AutoPartitioningBySize) { + const auto& ref = Params.TableSettings.AutoPartitioningBySize.GetRef(); + settings = L(settings, Q(Y(Q("autoPartitioningBySize"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.PartitionSizeMb) { + settings = L(settings, Q(Y(Q("partitionSizeMb"), Params.TableSettings.PartitionSizeMb))); + } + if (Params.TableSettings.AutoPartitioningByLoad) { + const auto& ref = Params.TableSettings.AutoPartitioningByLoad.GetRef(); + settings = L(settings, Q(Y(Q("autoPartitioningByLoad"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.MinPartitions) { + settings = L(settings, Q(Y(Q("minPartitions"), Params.TableSettings.MinPartitions))); + } + if (Params.TableSettings.MaxPartitions) { + settings = L(settings, Q(Y(Q("maxPartitions"), Params.TableSettings.MaxPartitions))); + } + if (Params.TableSettings.KeyBloomFilter) { + const auto& ref = Params.TableSettings.KeyBloomFilter.GetRef(); + settings = L(settings, Q(Y(Q("keyBloomFilter"), BuildQuotedAtom(ref.Pos, ref.Name)))); + } + if (Params.TableSettings.ReadReplicasSettings) { + settings = L(settings, Q(Y(Q("readReplicasSettings"), Params.TableSettings.ReadReplicasSettings))); + } if (const auto& ttl = Params.TableSettings.TtlSettings) { if (ttl.IsSet()) { const auto& ttlSettings = ttl.GetValueSet(); @@ -954,9 +954,9 @@ public: settings = L(settings, Q(Y(Q("resetTtlSettings"), Q(Y())))); } } - actions = L(actions, Q(Y(Q("setTableSettings"), Q(settings)))); - } - + actions = L(actions, Q(Y(Q("setTableSettings"), Q(settings)))); + } + for (const auto& index : Params.AddIndexes) { const auto& desc = CreateIndexDesc(index, *this); actions = L(actions, Q(Y(Q("addIndex"), Q(desc)))); @@ -988,11 +988,11 @@ public: actions = L(actions, Q(Y(Q("dropChangefeed"), name))); } - auto opts = Y(); - - opts = L(opts, Q(Y(Q("mode"), Q("alter")))); - opts = L(opts, Q(Y(Q("actions"), Q(actions)))); - + auto opts = Y(); + + opts = L(opts, Q(Y(Q("mode"), Q("alter")))); + opts = L(opts, Q(Y(Q("actions"), Q(actions)))); + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), @@ -1006,13 +1006,13 @@ public: } private: TTableRef Table; - const TAlterTableParameters Params; + const TAlterTableParameters Params; TScopedStatePtr Scoped; }; -TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) +TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped) { - return new TAlterTableNode(pos, tr, params, scoped); + return new TAlterTableNode(pos, tr, params, scoped); } class TDropTableNode final: public TAstListNode { diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp index 9bb868d265..2841f05a5b 100644 --- a/ydb/library/yql/sql/v1/select.cpp +++ b/ydb/library/yql/sql/v1/select.cpp @@ -746,10 +746,10 @@ public: TPtr DoClone() const final { return new TTableSource(Pos, Table, GetLabel()); } - - bool IsTableSource() const override { - return true; - } + + bool IsTableSource() const override { + return true; + } protected: TTableRef Table; private: @@ -1805,7 +1805,7 @@ private: } auto column = term->GetColumnName(); TString label(term->GetLabel()); - bool hasName = true; + bool hasName = true; if (label.empty()) { auto source = term->GetSourceName(); if (term->IsAsterisk() && !source->empty()) { @@ -1815,14 +1815,14 @@ private: label = isJoin && source && *source ? DotJoin(*source, *column) : *column; } else { label = TStringBuilder() << "column" << Columns.List.size(); - hasName = false; + hasName = false; if (ctx.WarnUnnamedColumns) { ctx.Warning(term->GetPos(), TIssuesIds::YQL_UNNAMED_COLUMN) << "Autogenerated column name " << label << " will be used for expression"; } } } - if (!Columns.Add(&label, false, false, true, hasName)) { + if (!Columns.Add(&label, false, false, true, hasName)) { ctx.Error(Pos) << "Duplicate column: " << label; hasError = true; continue; diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 5f40ed1ddb..ab5a864788 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -840,17 +840,17 @@ protected: bool ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding); TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node); - bool CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params); + bool CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params); - bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family); - bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family); - bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings); + bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family); + bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family); + bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings); bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, bool alter, bool reset); - bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, - bool alter = false); + bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, + bool alter = false); bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings); - + TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); TNodePtr TypeDecimal(const TRule_type_name_decimal& node); TNodePtr AddOptionals(const TNodePtr& node, size_t optionalCount); @@ -1790,100 +1790,100 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) { } TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) { - const bool nullable = !node.HasBlock4() || !node.GetBlock4().HasBlock1(); + const bool nullable = !node.HasBlock4() || !node.GetBlock4().HasBlock1(); const TString name(Id(node.GetRule_an_id_schema1(), *this)); const TPosition pos(Context().Pos()); const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2()); if (!type) { return {}; } - TVector<TIdentifier> families; - if (node.HasBlock3()) { - const auto& familyRelation = node.GetBlock3().GetRule_family_relation1(); - families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this)); - } - return TColumnSchema(pos, name, type, nullable, families); -} - -bool TSqlTranslation::FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family) { - TIdentifier id = IdEx(settingNode.GetRule_an_id1(), *this); - const TRule_family_setting_value& value = settingNode.GetRule_family_setting_value3(); - if (to_lower(id.Name) == "data") { - const TString stringValue(Ctx.Token(value.GetToken1())); - family.Data = BuildLiteralSmartString(Ctx, stringValue); - } else if (to_lower(id.Name) == "compression") { - const TString stringValue(Ctx.Token(value.GetToken1())); - family.Compression = BuildLiteralSmartString(Ctx, stringValue); - } else { - Ctx.Error() << "Unknown table setting: " << id.Name; - return false; - } - return true; -} - -bool TSqlTranslation::FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family) { - if (!FillFamilySettingsEntry(settingsNode.GetRule_family_settings_entry2(), family)) { - return false; - } - for (auto& block : settingsNode.GetBlock3()) { - if (!FillFamilySettingsEntry(block.GetRule_family_settings_entry2(), family)) { - return false; - } - } - return true; -} - -bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params) + TVector<TIdentifier> families; + if (node.HasBlock3()) { + const auto& familyRelation = node.GetBlock3().GetRule_family_relation1(); + families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this)); + } + return TColumnSchema(pos, name, type, nullable, families); +} + +bool TSqlTranslation::FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family) { + TIdentifier id = IdEx(settingNode.GetRule_an_id1(), *this); + const TRule_family_setting_value& value = settingNode.GetRule_family_setting_value3(); + if (to_lower(id.Name) == "data") { + const TString stringValue(Ctx.Token(value.GetToken1())); + family.Data = BuildLiteralSmartString(Ctx, stringValue); + } else if (to_lower(id.Name) == "compression") { + const TString stringValue(Ctx.Token(value.GetToken1())); + family.Compression = BuildLiteralSmartString(Ctx, stringValue); + } else { + Ctx.Error() << "Unknown table setting: " << id.Name; + return false; + } + return true; +} + +bool TSqlTranslation::FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family) { + if (!FillFamilySettingsEntry(settingsNode.GetRule_family_settings_entry2(), family)) { + return false; + } + for (auto& block : settingsNode.GetBlock3()) { + if (!FillFamilySettingsEntry(block.GetRule_family_settings_entry2(), family)) { + return false; + } + } + return true; +} + +bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params) { switch (node.Alt_case()) { case TRule_create_table_entry::kAltCreateTableEntry1: { - // column_schema + // column_schema auto columnSchema = ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1()); if (!columnSchema) { return false; } - if (columnSchema->Families.size() > 1) { - Ctx.Error() << "Several column families for a single column are not yet supported"; - return false; - } - params.Columns.push_back(*columnSchema); + if (columnSchema->Families.size() > 1) { + Ctx.Error() << "Several column families for a single column are not yet supported"; + return false; + } + params.Columns.push_back(*columnSchema); break; } case TRule_create_table_entry::kAltCreateTableEntry2: { - // table_constraint + // table_constraint auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1(); switch (constraint.Alt_case()) { case TRule_table_constraint::kAltTableConstraint1: { - if (!params.PkColumns.empty()) { - Ctx.Error() << "PRIMARY KEY statement must be specified only once"; - return false; - } + if (!params.PkColumns.empty()) { + Ctx.Error() << "PRIMARY KEY statement must be specified only once"; + return false; + } auto& pkConstraint = constraint.GetAlt_table_constraint1(); - params.PkColumns.push_back(IdEx(pkConstraint.GetRule_an_id4(), *this)); + params.PkColumns.push_back(IdEx(pkConstraint.GetRule_an_id4(), *this)); for (auto& block : pkConstraint.GetBlock5()) { - params.PkColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); + params.PkColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); } break; } case TRule_table_constraint::kAltTableConstraint2: { - if (!params.PartitionByColumns.empty()) { - Ctx.Error() << "PARTITION BY statement must be specified only once"; - return false; - } + if (!params.PartitionByColumns.empty()) { + Ctx.Error() << "PARTITION BY statement must be specified only once"; + return false; + } auto& pbConstraint = constraint.GetAlt_table_constraint2(); - params.PartitionByColumns.push_back(IdEx(pbConstraint.GetRule_an_id4(), *this)); + params.PartitionByColumns.push_back(IdEx(pbConstraint.GetRule_an_id4(), *this)); for (auto& block : pbConstraint.GetBlock5()) { - params.PartitionByColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); + params.PartitionByColumns.push_back(IdEx(block.GetRule_an_id2(), *this)); } break; } case TRule_table_constraint::kAltTableConstraint3: { - if (!params.OrderByColumns.empty()) { - Ctx.Error() << "ORDER BY statement must be specified only once"; - return false; - } + if (!params.OrderByColumns.empty()) { + Ctx.Error() << "ORDER BY statement must be specified only once"; + return false; + } auto& obConstraint = constraint.GetAlt_table_constraint3(); auto extractDirection = [this] (const TRule_column_order_by_specification& spec, bool& desc) { desc = false; @@ -1909,14 +1909,14 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr if (!extractDirection(obSpec, desc)) { return false; } - params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); + params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); for (auto& block : obConstraint.GetBlock5()) { auto& obSpec = block.GetRule_column_order_by_specification2(); if (!extractDirection(obSpec, desc)) { return false; } - params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); + params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc)); } break; } @@ -1928,26 +1928,26 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr } case TRule_create_table_entry::kAltCreateTableEntry3: { - // table_index + // table_index auto& table_index = node.GetAlt_create_table_entry3().GetRule_table_index1(); - if (!CreateTableIndex(table_index, *this, params.Indexes)) { + if (!CreateTableIndex(table_index, *this, params.Indexes)) { return false; } break; } - case TRule_create_table_entry::kAltCreateTableEntry4: - { - // family_entry - auto& family_entry = node.GetAlt_create_table_entry4().GetRule_family_entry1(); - TFamilyEntry family(IdEx(family_entry.GetRule_an_id2(), *this)); - if (family_entry.HasBlock3()) { - if (!FillFamilySettings(family_entry.GetBlock3().GetRule_family_settings1(), family)) { - return false; - } - } - params.ColumnFamilies.push_back(family); - break; - } + case TRule_create_table_entry::kAltCreateTableEntry4: + { + // family_entry + auto& family_entry = node.GetAlt_create_table_entry4().GetRule_family_entry1(); + TFamilyEntry family(IdEx(family_entry.GetRule_an_id2(), *this)); + if (family_entry.HasBlock3()) { + if (!FillFamilySettings(family_entry.GetBlock3().GetRule_family_settings1(), family)) { + return false; + } + } + params.ColumnFamilies.push_back(family); + break; + } case TRule_create_table_entry::kAltCreateTableEntry5: { // changefeed @@ -1964,140 +1964,140 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr return true; } -TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node); - -namespace { - bool StoreId(const TRule_table_setting_value& from, TMaybe<TIdentifier>& to, TTranslation& ctx) { - switch (from.Alt_case()) { - case TRule_table_setting_value::kAltTableSettingValue1: { - // id - to = IdEx(from.GetAlt_table_setting_value1().GetRule_id1(), ctx); - break; - } - default: - return false; - } - return true; - } - - bool StoreString(const TRule_table_setting_value& from, TNodePtr& to, TContext& ctx) { - switch (from.Alt_case()) { - case TRule_table_setting_value::kAltTableSettingValue2: { - // STRING_VALUE - const TString stringValue(ctx.Token(from.GetAlt_table_setting_value2().GetToken1())); - to = BuildLiteralSmartString(ctx, stringValue); - break; - } - default: - return false; - } - return true; - } - - bool StoreInt(const TRule_table_setting_value& from, TNodePtr& to, TContext& ctx) { - switch (from.Alt_case()) { - case TRule_table_setting_value::kAltTableSettingValue3: { - // integer - to = LiteralNumber(ctx, from.GetAlt_table_setting_value3().GetRule_integer1()); - break; - } - default: - return false; - } - return true; - } - - bool StoreSplitBoundary(const TRule_literal_value_list& boundary, TVector<TVector<TNodePtr>>& to, - TSqlExpression& expr, TContext& ctx) { - TVector<TNodePtr> boundaryKeys; - auto first_key = expr.LiteralExpr(boundary.GetRule_literal_value2()); - if (!first_key) { - ctx.Error() << "Empty key in partition at keys"; - return false; - } - if (!first_key->Expr) { - ctx.Error() << "Identifier is not expected in partition at keys"; - return false; - } - boundaryKeys.emplace_back(first_key->Expr); - for (auto& key : boundary.GetBlock3()) { - auto keyExprOrIdent = expr.LiteralExpr(key.GetRule_literal_value2()); - if (!keyExprOrIdent) { - ctx.Error() << "Empty key in partition at keys"; - return false; - } - if (!keyExprOrIdent->Expr) { - ctx.Error() << "Identifier is not expected in partition at keys"; - return false; - } - boundaryKeys.emplace_back(keyExprOrIdent->Expr); - } - to.push_back(boundaryKeys); - return true; - } - - bool StoreSplitBoundaries(const TRule_table_setting_value& from, TVector<TVector<TNodePtr>>& to, - TSqlExpression& expr, TContext& ctx) { - switch (from.Alt_case()) { - case TRule_table_setting_value::kAltTableSettingValue4: { - // split_boundaries - const auto& boundariesNode = from.GetAlt_table_setting_value4().GetRule_split_boundaries1(); - switch (boundariesNode.Alt_case()) { - case TRule_split_boundaries::kAltSplitBoundaries1: { - // literal_value_list (COMMA literal_value_list)* - auto& complexBoundaries = boundariesNode.GetAlt_split_boundaries1(); - - auto& first_boundary = complexBoundaries.GetRule_literal_value_list2(); - if (!StoreSplitBoundary(first_boundary, to, expr, ctx)) { - return false; - } - - for (auto& boundary : complexBoundaries.GetBlock3()) { - if (!StoreSplitBoundary(boundary.GetRule_literal_value_list2(), to, expr, ctx)) { - return false; - } - } - break; - } - case TRule_split_boundaries::kAltSplitBoundaries2: { - // literal_value_list - auto& simpleBoundaries = boundariesNode.GetAlt_split_boundaries2().GetRule_literal_value_list1(); - auto first_key = expr.LiteralExpr(simpleBoundaries.GetRule_literal_value2()); - if (!first_key) { - ctx.Error() << "Empty key in partition at keys"; - return false; - } - if (!first_key->Expr) { - ctx.Error() << "Identifier is not expected in partition at keys"; - return false; - } - to.push_back(TVector<TNodePtr>(1, first_key->Expr)); - for (auto& key : simpleBoundaries.GetBlock3()) { - auto keyExprOrIdent = expr.LiteralExpr(key.GetRule_literal_value2()); - if (!keyExprOrIdent) { - ctx.Error() << "Empty key in partition at keys"; - return false; - } - if (!first_key->Expr) { - ctx.Error() << "Identifier is not expected in partition at keys"; - return false; - } - to.push_back( - TVector<TNodePtr>(1, keyExprOrIdent->Expr) - ); - } - break; - } - default: - return false; - } - break; - } - default: - return false; - } - return true; - } +TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node); + +namespace { + bool StoreId(const TRule_table_setting_value& from, TMaybe<TIdentifier>& to, TTranslation& ctx) { + switch (from.Alt_case()) { + case TRule_table_setting_value::kAltTableSettingValue1: { + // id + to = IdEx(from.GetAlt_table_setting_value1().GetRule_id1(), ctx); + break; + } + default: + return false; + } + return true; + } + + bool StoreString(const TRule_table_setting_value& from, TNodePtr& to, TContext& ctx) { + switch (from.Alt_case()) { + case TRule_table_setting_value::kAltTableSettingValue2: { + // STRING_VALUE + const TString stringValue(ctx.Token(from.GetAlt_table_setting_value2().GetToken1())); + to = BuildLiteralSmartString(ctx, stringValue); + break; + } + default: + return false; + } + return true; + } + + bool StoreInt(const TRule_table_setting_value& from, TNodePtr& to, TContext& ctx) { + switch (from.Alt_case()) { + case TRule_table_setting_value::kAltTableSettingValue3: { + // integer + to = LiteralNumber(ctx, from.GetAlt_table_setting_value3().GetRule_integer1()); + break; + } + default: + return false; + } + return true; + } + + bool StoreSplitBoundary(const TRule_literal_value_list& boundary, TVector<TVector<TNodePtr>>& to, + TSqlExpression& expr, TContext& ctx) { + TVector<TNodePtr> boundaryKeys; + auto first_key = expr.LiteralExpr(boundary.GetRule_literal_value2()); + if (!first_key) { + ctx.Error() << "Empty key in partition at keys"; + return false; + } + if (!first_key->Expr) { + ctx.Error() << "Identifier is not expected in partition at keys"; + return false; + } + boundaryKeys.emplace_back(first_key->Expr); + for (auto& key : boundary.GetBlock3()) { + auto keyExprOrIdent = expr.LiteralExpr(key.GetRule_literal_value2()); + if (!keyExprOrIdent) { + ctx.Error() << "Empty key in partition at keys"; + return false; + } + if (!keyExprOrIdent->Expr) { + ctx.Error() << "Identifier is not expected in partition at keys"; + return false; + } + boundaryKeys.emplace_back(keyExprOrIdent->Expr); + } + to.push_back(boundaryKeys); + return true; + } + + bool StoreSplitBoundaries(const TRule_table_setting_value& from, TVector<TVector<TNodePtr>>& to, + TSqlExpression& expr, TContext& ctx) { + switch (from.Alt_case()) { + case TRule_table_setting_value::kAltTableSettingValue4: { + // split_boundaries + const auto& boundariesNode = from.GetAlt_table_setting_value4().GetRule_split_boundaries1(); + switch (boundariesNode.Alt_case()) { + case TRule_split_boundaries::kAltSplitBoundaries1: { + // literal_value_list (COMMA literal_value_list)* + auto& complexBoundaries = boundariesNode.GetAlt_split_boundaries1(); + + auto& first_boundary = complexBoundaries.GetRule_literal_value_list2(); + if (!StoreSplitBoundary(first_boundary, to, expr, ctx)) { + return false; + } + + for (auto& boundary : complexBoundaries.GetBlock3()) { + if (!StoreSplitBoundary(boundary.GetRule_literal_value_list2(), to, expr, ctx)) { + return false; + } + } + break; + } + case TRule_split_boundaries::kAltSplitBoundaries2: { + // literal_value_list + auto& simpleBoundaries = boundariesNode.GetAlt_split_boundaries2().GetRule_literal_value_list1(); + auto first_key = expr.LiteralExpr(simpleBoundaries.GetRule_literal_value2()); + if (!first_key) { + ctx.Error() << "Empty key in partition at keys"; + return false; + } + if (!first_key->Expr) { + ctx.Error() << "Identifier is not expected in partition at keys"; + return false; + } + to.push_back(TVector<TNodePtr>(1, first_key->Expr)); + for (auto& key : simpleBoundaries.GetBlock3()) { + auto keyExprOrIdent = expr.LiteralExpr(key.GetRule_literal_value2()); + if (!keyExprOrIdent) { + ctx.Error() << "Empty key in partition at keys"; + return false; + } + if (!first_key->Expr) { + ctx.Error() << "Identifier is not expected in partition at keys"; + return false; + } + to.push_back( + TVector<TNodePtr>(1, keyExprOrIdent->Expr) + ); + } + break; + } + default: + return false; + } + break; + } + default: + return false; + } + return true; + } bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, TSqlExpression& expr, TContext& ctx, TTranslation& txc) { @@ -2168,101 +2168,101 @@ namespace { result.Suffix = std::move(current); return result; } -} - +} + bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, bool alter, bool reset) { YQL_ENSURE(value || reset); YQL_ENSURE(!reset || reset & alter); - if (to_lower(id.Name) == "compaction_policy") { + if (to_lower(id.Name) == "compaction_policy") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreString(*value, settings.CompactionPolicy, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; return false; } - } else if (to_lower(id.Name) == "auto_partitioning_by_size") { + } else if (to_lower(id.Name) == "auto_partitioning_by_size") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreId(*value, settings.AutoPartitioningBySize, *this)) { Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - } else if (to_lower(id.Name) == "auto_partitioning_partition_size_mb") { + } else if (to_lower(id.Name) == "auto_partitioning_partition_size_mb") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreInt(*value, settings.PartitionSizeMb, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - } else if (to_lower(id.Name) == "auto_partitioning_by_load") { + } else if (to_lower(id.Name) == "auto_partitioning_by_load") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreId(*value, settings.AutoPartitioningByLoad, *this)) { Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - } else if (to_lower(id.Name) == "auto_partitioning_min_partitions_count") { + } else if (to_lower(id.Name) == "auto_partitioning_min_partitions_count") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreInt(*value, settings.MinPartitions, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - } else if (to_lower(id.Name) == "auto_partitioning_max_partitions_count") { + } else if (to_lower(id.Name) == "auto_partitioning_max_partitions_count") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreInt(*value, settings.MaxPartitions, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be an integer"; return false; } - } else if (to_lower(id.Name) == "uniform_partitions") { - if (alter) { + } else if (to_lower(id.Name) == "uniform_partitions") { + if (alter) { Ctx.Error() << to_upper(id.Name) << " alter is not supported"; - return false; - } + return false; + } if (!StoreInt(*value, settings.UniformPartitions, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be an integer"; - return false; - } - } else if (to_lower(id.Name) == "partition_at_keys") { - if (alter) { + return false; + } + } else if (to_lower(id.Name) == "partition_at_keys") { + if (alter) { Ctx.Error() << to_upper(id.Name) << " alter is not supported"; - return false; - } - TSqlExpression expr(Ctx, Mode); + return false; + } + TSqlExpression expr(Ctx, Mode); if (!StoreSplitBoundaries(*value, settings.PartitionAtKeys, expr, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be a list of keys. " - << "Example1: (10, 1000) Example2: ((10), (1000, \"abc\"))"; - return false; - } - } else if (to_lower(id.Name) == "key_bloom_filter") { + << "Example1: (10, 1000) Example2: ((10), (1000, \"abc\"))"; + return false; + } + } else if (to_lower(id.Name) == "key_bloom_filter") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreId(*value, settings.KeyBloomFilter, *this)) { Ctx.Error() << to_upper(id.Name) << " value should be an identifier"; return false; } - } else if (to_lower(id.Name) == "read_replicas_settings") { + } else if (to_lower(id.Name) == "read_replicas_settings") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; - return false; - } + return false; + } if (!StoreString(*value, settings.ReadReplicasSettings, Ctx)) { Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; return false; @@ -2277,13 +2277,13 @@ bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule } else { settings.TtlSettings.Reset(); } - } else { - Ctx.Error() << "Unknown table setting: " << id.Name; - return false; - } - return true; -} - + } else { + Ctx.Error() << "Unknown table setting: " << id.Name; + return false; + } + return true; +} + bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, bool alter) { return StoreTableSettingsEntry(id, &value, settings, alter, false); @@ -2293,21 +2293,21 @@ bool TSqlTranslation::ResetTableSettingsEntry(const TIdentifier& id, TTableSetti return StoreTableSettingsEntry(id, nullptr, settings, true, true); } -bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings) { - const auto& firstEntry = settingsNode.GetRule_table_settings_entry3(); - if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), - settings)) { - return false; - } - for (auto& block : settingsNode.GetBlock4()) { - const auto& entry = block.GetRule_table_settings_entry2(); - if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), settings)) { - return false; - } - } - return true; -} - +bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings) { + const auto& firstEntry = settingsNode.GetRule_table_settings_entry3(); + if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), + settings)) { + return false; + } + for (auto& block : settingsNode.GetBlock4()) { + const auto& entry = block.GetRule_table_settings_entry2(); + if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), settings)) { + return false; + } + } + return true; +} + bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& suffix); TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) { @@ -8148,12 +8148,12 @@ public: private: bool DeclareStatement(const TRule_declare_stmt& stmt); bool ExportStatement(const TRule_export_stmt& stmt); - bool AlterTableAction(const TRule_alter_table_action& node, TAlterTableParameters& params); - bool AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params); - bool AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params); - bool AlterTableAlterColumn(const TRule_alter_table_alter_column& node, TAlterTableParameters& params); - bool AlterTableAddFamily(const TRule_family_entry& node, TAlterTableParameters& params); - bool AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, TAlterTableParameters& params); + bool AlterTableAction(const TRule_alter_table_action& node, TAlterTableParameters& params); + bool AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params); + bool AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params); + bool AlterTableAlterColumn(const TRule_alter_table_alter_column& node, TAlterTableParameters& params); + bool AlterTableAddFamily(const TRule_family_entry& node, TAlterTableParameters& params); + bool AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, TAlterTableParameters& params); bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, TAlterTableParameters& params); bool AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_compat& node, TAlterTableParameters& params); bool AlterTableResetTableSetting(const TRule_alter_table_reset_table_setting& node, TAlterTableParameters& params); @@ -8381,24 +8381,24 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } - TCreateTableParameters params; + TCreateTableParameters params; - if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) { + if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) { return false; } for (auto& block: rule.GetBlock6()) { - if (!CreateTableEntry(block.GetRule_create_table_entry2(), params)) { + if (!CreateTableEntry(block.GetRule_create_table_entry2(), params)) { + return false; + } + } + + if (rule.HasBlock8()) { + if (!CreateTableSettings(rule.GetBlock8().GetRule_with_table_settings1(), params.TableSettings)) { return false; } } - if (rule.HasBlock8()) { - if (!CreateTableSettings(rule.GetBlock8().GetRule_with_table_settings1(), params.TableSettings)) { - return false; - } - } - - AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, params, Ctx.Scoped)); + AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, params, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore5: { @@ -8491,18 +8491,18 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } - TAlterTableParameters params; - if (!AlterTableAction(rule.GetRule_alter_table_action4(), params)) { - return false; - } + TAlterTableParameters params; + if (!AlterTableAction(rule.GetRule_alter_table_action4(), params)) { + return false; + } - for (auto& block : rule.GetBlock5()) { - if (!AlterTableAction(block.GetRule_alter_table_action2(), params)) { - return false; + for (auto& block : rule.GetBlock5()) { + if (!AlterTableAction(block.GetRule_alter_table_action2(), params)) { + return false; } } - - AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, params, Ctx.Scoped)); + + AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, params, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore16: { @@ -8827,63 +8827,63 @@ bool TSqlQuery::ExportStatement(const TRule_export_stmt& stmt) { return true; } -bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTableParameters& params) { +bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTableParameters& params) { if (params.RenameTo) { // rename action is followed by some other actions Error() << "RENAME TO can not be used together with another table action"; return false; } - switch (node.Alt_case()) { - case TRule_alter_table_action::kAltAlterTableAction1: { - // ADD COLUMN - const auto& addRule = node.GetAlt_alter_table_action1().GetRule_alter_table_add_column1(); - if (!AlterTableAddColumn(addRule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction2: { - // DROP COLUMN - const auto& dropRule = node.GetAlt_alter_table_action2().GetRule_alter_table_drop_column1(); - if (!AlterTableDropColumn(dropRule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction3: { - // ALTER COLUMN - const auto& alterRule = node.GetAlt_alter_table_action3().GetRule_alter_table_alter_column1(); - if (!AlterTableAlterColumn(alterRule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction4: { - // ADD FAMILY - const auto& familyEntry = node.GetAlt_alter_table_action4().GetRule_alter_table_add_column_family1() - .GetRule_family_entry2(); - if (!AlterTableAddFamily(familyEntry, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction5: { - // ALTER FAMILY - const auto& alterRule = node.GetAlt_alter_table_action5().GetRule_alter_table_alter_column_family1(); - if (!AlterTableAlterFamily(alterRule, params)) { - return false; - } - break; - } - case TRule_alter_table_action::kAltAlterTableAction6: { + switch (node.Alt_case()) { + case TRule_alter_table_action::kAltAlterTableAction1: { + // ADD COLUMN + const auto& addRule = node.GetAlt_alter_table_action1().GetRule_alter_table_add_column1(); + if (!AlterTableAddColumn(addRule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction2: { + // DROP COLUMN + const auto& dropRule = node.GetAlt_alter_table_action2().GetRule_alter_table_drop_column1(); + if (!AlterTableDropColumn(dropRule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction3: { + // ALTER COLUMN + const auto& alterRule = node.GetAlt_alter_table_action3().GetRule_alter_table_alter_column1(); + if (!AlterTableAlterColumn(alterRule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction4: { + // ADD FAMILY + const auto& familyEntry = node.GetAlt_alter_table_action4().GetRule_alter_table_add_column_family1() + .GetRule_family_entry2(); + if (!AlterTableAddFamily(familyEntry, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction5: { + // ALTER FAMILY + const auto& alterRule = node.GetAlt_alter_table_action5().GetRule_alter_table_alter_column_family1(); + if (!AlterTableAlterFamily(alterRule, params)) { + return false; + } + break; + } + case TRule_alter_table_action::kAltAlterTableAction6: { // SET (uncompat) const auto& setRule = node.GetAlt_alter_table_action6().GetRule_alter_table_set_table_setting_uncompat1(); - if (!AlterTableSetTableSetting(setRule, params)) { - return false; - } - break; - } + if (!AlterTableSetTableSetting(setRule, params)) { + return false; + } + break; + } case TRule_alter_table_action::kAltAlterTableAction7: { // SET (compat) const auto& setRule = node.GetAlt_alter_table_action7().GetRule_alter_table_set_table_setting_compat1(); @@ -8949,101 +8949,101 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab break; } - default: - AltNotImplemented("alter_table_action", node); - return false; - } - return true; -} + default: + AltNotImplemented("alter_table_action", node); + return false; + } + return true; +} -bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params) { - auto columnSchema = ColumnSchemaImpl(node.GetRule_column_schema3()); +bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params) { + auto columnSchema = ColumnSchemaImpl(node.GetRule_column_schema3()); if (!columnSchema) { return false; } - if (columnSchema->Families.size() > 1) { - Ctx.Error() << "Several column families for a single column are not yet supported"; - return false; - } - params.AddColumns.push_back(*columnSchema); - return true; -} - -bool TSqlQuery::AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params) { - TString name = Id(node.GetRule_an_id3(), *this); - params.DropColumns.push_back(name); - return true; -} - -bool TSqlQuery::AlterTableAlterColumn(const TRule_alter_table_alter_column& node, - TAlterTableParameters& params) -{ - TString name = Id(node.GetRule_an_id3(), *this); - const TPosition pos(Context().Pos()); - TVector<TIdentifier> families; - const auto& familyRelation = node.GetRule_family_relation5(); - families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this)); - params.AlterColumns.emplace_back(pos, name, nullptr, false, families); - return true; -} - -bool TSqlQuery::AlterTableAddFamily(const TRule_family_entry& node, TAlterTableParameters& params) { - TFamilyEntry family(IdEx(node.GetRule_an_id2(), *this)); - if (node.HasBlock3()) { - if (!FillFamilySettings(node.GetBlock3().GetRule_family_settings1(), family)) { + if (columnSchema->Families.size() > 1) { + Ctx.Error() << "Several column families for a single column are not yet supported"; + return false; + } + params.AddColumns.push_back(*columnSchema); + return true; +} + +bool TSqlQuery::AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params) { + TString name = Id(node.GetRule_an_id3(), *this); + params.DropColumns.push_back(name); + return true; +} + +bool TSqlQuery::AlterTableAlterColumn(const TRule_alter_table_alter_column& node, + TAlterTableParameters& params) +{ + TString name = Id(node.GetRule_an_id3(), *this); + const TPosition pos(Context().Pos()); + TVector<TIdentifier> families; + const auto& familyRelation = node.GetRule_family_relation5(); + families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this)); + params.AlterColumns.emplace_back(pos, name, nullptr, false, families); + return true; +} + +bool TSqlQuery::AlterTableAddFamily(const TRule_family_entry& node, TAlterTableParameters& params) { + TFamilyEntry family(IdEx(node.GetRule_an_id2(), *this)); + if (node.HasBlock3()) { + if (!FillFamilySettings(node.GetBlock3().GetRule_family_settings1(), family)) { + return false; + } + } + params.AddColumnFamilies.push_back(family); + return true; +} + +bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, + TAlterTableParameters& params) +{ + TFamilyEntry* entry = nullptr; + TIdentifier name = IdEx(node.GetRule_an_id3(), *this); + for (auto& family : params.AlterColumnFamilies) { + if (family.Name.Name == name.Name) { + entry = &family; + break; + } + } + if (!entry) { + entry = ¶ms.AlterColumnFamilies.emplace_back(name); + } + TIdentifier settingName = IdEx(node.GetRule_an_id5(), *this); + const TRule_family_setting_value& value = node.GetRule_family_setting_value6(); + if (to_lower(settingName.Name) == "data") { + if (entry->Data) { + Ctx.Error() << "Redefinition of 'data' setting for column family '" << name.Name + << "' in one alter"; + return false; + } + const TString stringValue(Ctx.Token(value.GetToken1())); + entry->Data = BuildLiteralSmartString(Ctx, stringValue); + } else if (to_lower(settingName.Name) == "compression") { + if (entry->Compression) { + Ctx.Error() << "Redefinition of 'compression' setting for column family '" << name.Name + << "' in one alter"; return false; } + const TString stringValue(Ctx.Token(value.GetToken1())); + entry->Compression = BuildLiteralSmartString(Ctx, stringValue); + } else { + Ctx.Error() << "Unknown table setting: " << settingName.Name; + return false; } - params.AddColumnFamilies.push_back(family); - return true; -} - -bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_family& node, - TAlterTableParameters& params) -{ - TFamilyEntry* entry = nullptr; - TIdentifier name = IdEx(node.GetRule_an_id3(), *this); - for (auto& family : params.AlterColumnFamilies) { - if (family.Name.Name == name.Name) { - entry = &family; - break; - } - } - if (!entry) { - entry = ¶ms.AlterColumnFamilies.emplace_back(name); - } - TIdentifier settingName = IdEx(node.GetRule_an_id5(), *this); - const TRule_family_setting_value& value = node.GetRule_family_setting_value6(); - if (to_lower(settingName.Name) == "data") { - if (entry->Data) { - Ctx.Error() << "Redefinition of 'data' setting for column family '" << name.Name - << "' in one alter"; - return false; - } - const TString stringValue(Ctx.Token(value.GetToken1())); - entry->Data = BuildLiteralSmartString(Ctx, stringValue); - } else if (to_lower(settingName.Name) == "compression") { - if (entry->Compression) { - Ctx.Error() << "Redefinition of 'compression' setting for column family '" << name.Name - << "' in one alter"; - return false; - } - const TString stringValue(Ctx.Token(value.GetToken1())); - entry->Compression = BuildLiteralSmartString(Ctx, stringValue); - } else { - Ctx.Error() << "Unknown table setting: " << settingName.Name; - return false; - } return true; } bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_setting_uncompat& node, - TAlterTableParameters& params) -{ - if (!StoreTableSettingsEntry(IdEx(node.GetRule_an_id2(), *this), node.GetRule_table_setting_value3(), - params.TableSettings, true)) { - return false; - } + TAlterTableParameters& params) +{ + if (!StoreTableSettingsEntry(IdEx(node.GetRule_an_id2(), *this), node.GetRule_table_setting_value3(), + params.TableSettings, true)) { + return false; + } return true; } @@ -9460,13 +9460,13 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "discoverymode") { Ctx.DiscoveryMode = true; Ctx.IncrementMonCounter("sql_pragma", "DiscoveryMode"); - } else if (normalizedPragma == "enablesystemcolumns") { + } else if (normalizedPragma == "enablesystemcolumns") { if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { Error() << "Expected boolean literal as a single argument for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - Ctx.IncrementMonCounter("sql_pragma", "EnableSystemColumns"); + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + Ctx.IncrementMonCounter("sql_pragma", "EnableSystemColumns"); } else if (normalizedPragma == "ansiinforemptyornullableitemscollections") { Ctx.AnsiInForEmptyOrNullableItemsCollections = true; Ctx.IncrementMonCounter("sql_pragma", "AnsiInForEmptyOrNullableItemsCollections"); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 3a641df16e..85b6e0dfcd 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -196,13 +196,13 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); } - Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { + Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { // FIXME: check if we can get old behaviour //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); - } - + } + Y_UNIT_TEST(ResetKeywordNotReservedForNames) { UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE RESET (RESET Uint32, PRIMARY KEY (RESET));").IsOk()); UNIT_ASSERT(SqlToYql("USE plato; SELECT RESET FROM RESET").IsOk()); @@ -1329,69 +1329,69 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { auto expected = "(Apply (lambda '(\"$foo bar\" \"$x\")"; UNIT_ASSERT(programm.find(expected) != TString::npos); } - - Y_UNIT_TEST(CompactionPolicyParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( COMPACTION_POLICY = "SomeCompactionPreset" );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compactionPolicy")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SomeCompactionPreset")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(AutoPartitioningBySizeParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( AUTO_PARTITIONING_BY_SIZE = ENABLED );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("autoPartitioningBySize")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } - - Y_UNIT_TEST(UniformPartitionsParseCorrect) { - NYql::TAstParseResult res = SqlToYql( - R"( USE plato; - CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) - WITH ( UNIFORM_PARTITIONS = 16 );)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "Write") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("uniformPartitions")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("16")); - } - }; - - TWordCountHive elementStat = { {TString("Write"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); - } + + Y_UNIT_TEST(CompactionPolicyParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( COMPACTION_POLICY = "SomeCompactionPreset" );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("compactionPolicy")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("SomeCompactionPreset")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(AutoPartitioningBySizeParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( AUTO_PARTITIONING_BY_SIZE = ENABLED );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("autoPartitioningBySize")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("ENABLED")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(UniformPartitionsParseCorrect) { + NYql::TAstParseResult res = SqlToYql( + R"( USE plato; + CREATE TABLE tableName (Key Uint32, Value String, PRIMARY KEY (Key)) + WITH ( UNIFORM_PARTITIONS = 16 );)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("uniformPartitions")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("16")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } Y_UNIT_TEST(TtlParseCorrect) { NYql::TAstParseResult res = SqlToYql( @@ -2682,13 +2682,13 @@ select FormatType($f()); UNIT_ASSERT(res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); } - - Y_UNIT_TEST(WarnSourceColumnMismatch) { - NYql::TAstParseResult res = SqlToYql( - "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); - } + + Y_UNIT_TEST(WarnSourceColumnMismatch) { + NYql::TAstParseResult res = SqlToYql( + "insert into plato.Output (key, subkey, new_value, one_more_value) select key as Key, subkey, value, \"x\" from plato.Input;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:51: Warning: Column names in SELECT don't match column specification in parenthesis. \"key\" doesn't match \"Key\". \"new_value\" doesn't match \"value\", code: 4517\n"); + } Y_UNIT_TEST(YtCaseInsensitive) { NYql::TAstParseResult res = SqlToYql("select * from PlatO.foo;"); diff --git a/ydb/public/api/grpc/ydb_discovery_v1.proto b/ydb/public/api/grpc/ydb_discovery_v1.proto index e39a6df68b..dc06a4f678 100644 --- a/ydb/public/api/grpc/ydb_discovery_v1.proto +++ b/ydb/public/api/grpc/ydb_discovery_v1.proto @@ -7,5 +7,5 @@ import "ydb/public/api/protos/ydb_discovery.proto"; service DiscoveryService { rpc ListEndpoints(Ydb.Discovery.ListEndpointsRequest) returns (Ydb.Discovery.ListEndpointsResponse); - rpc WhoAmI(Ydb.Discovery.WhoAmIRequest) returns (Ydb.Discovery.WhoAmIResponse); + rpc WhoAmI(Ydb.Discovery.WhoAmIRequest) returns (Ydb.Discovery.WhoAmIResponse); } diff --git a/ydb/public/api/grpc/ydb_scripting_v1.proto b/ydb/public/api/grpc/ydb_scripting_v1.proto index b7fefd93c6..c8743a07de 100644 --- a/ydb/public/api/grpc/ydb_scripting_v1.proto +++ b/ydb/public/api/grpc/ydb_scripting_v1.proto @@ -7,9 +7,9 @@ import "ydb/public/api/protos/ydb_scripting.proto"; service ScriptingService { rpc ExecuteYql(Scripting.ExecuteYqlRequest) returns (Scripting.ExecuteYqlResponse); - - // Executes yql request with streaming result. - rpc StreamExecuteYql(Scripting.ExecuteYqlRequest) returns (stream Scripting.ExecuteYqlPartialResponse); - + + // Executes yql request with streaming result. + rpc StreamExecuteYql(Scripting.ExecuteYqlRequest) returns (stream Scripting.ExecuteYqlPartialResponse); + rpc ExplainYql(Scripting.ExplainYqlRequest) returns (Scripting.ExplainYqlResponse); } diff --git a/ydb/public/api/protos/ydb_discovery.proto b/ydb/public/api/protos/ydb_discovery.proto index d7b8f0a681..5577dafe36 100644 --- a/ydb/public/api/protos/ydb_discovery.proto +++ b/ydb/public/api/protos/ydb_discovery.proto @@ -44,19 +44,19 @@ message ListEndpointsResult { message ListEndpointsResponse { Ydb.Operations.Operation operation = 1; } - -message WhoAmIRequest { - // Include user groups in response - bool include_groups = 1; -} - -message WhoAmIResult { - // User SID (Security ID) - string user = 1; - // List of group SIDs (Security IDs) for the user - repeated string groups = 2; -} - -message WhoAmIResponse { - Ydb.Operations.Operation operation = 1; -} + +message WhoAmIRequest { + // Include user groups in response + bool include_groups = 1; +} + +message WhoAmIResult { + // User SID (Security ID) + string user = 1; + // List of group SIDs (Security IDs) for the user + repeated string groups = 2; +} + +message WhoAmIResponse { + Ydb.Operations.Operation operation = 1; +} diff --git a/ydb/public/api/protos/ydb_scheme.proto b/ydb/public/api/protos/ydb_scheme.proto index 1d99d277da..d66b765725 100644 --- a/ydb/public/api/protos/ydb_scheme.proto +++ b/ydb/public/api/protos/ydb_scheme.proto @@ -40,7 +40,7 @@ message ListDirectoryResponse { } message Permissions { - // SID (Security ID) of user or group + // SID (Security ID) of user or group string subject = 1; repeated string permission_names = 2; } @@ -58,10 +58,10 @@ message Entry { SEQUENCE = 15; REPLICATION = 16; } - - // Name of scheme entry (dir2 of /dir1/dir2) + + // Name of scheme entry (dir2 of /dir1/dir2) string name = 1; - // SID (Security ID) of user or group + // SID (Security ID) of user or group string owner = 2; Type type = 5; repeated Permissions effective_permissions = 6; diff --git a/ydb/public/api/protos/ydb_scripting.proto b/ydb/public/api/protos/ydb_scripting.proto index eaab4c0455..4c1a295418 100644 --- a/ydb/public/api/protos/ydb_scripting.proto +++ b/ydb/public/api/protos/ydb_scripting.proto @@ -30,34 +30,34 @@ message ExecuteYqlResult { Ydb.TableStats.QueryStats query_stats = 2; } -// Response for StreamExecuteYql is a stream of ExecuteYqlPartialResponse messages. -// These responses can contain ExecuteYqlPartialResult messages with -// results (or result parts) for data or scan queries in the script. -// YqlScript can have multiple results (result sets). -// Each result set has an index (starting at 0). -message ExecuteYqlPartialResponse { - StatusIds.StatusCode status = 1; - repeated Ydb.Issue.IssueMessage issues = 2; - ExecuteYqlPartialResult result = 3; -} - -// Contains result set (or a result set part) for one data or scan query in the script. -// One result set can be split into several responses with same result_index. -// Only the final response can contain query stats. -message ExecuteYqlPartialResult { - // Index of current result - uint32 result_set_index = 1; - // Result set (or a result set part) for one data or scan query - Ydb.ResultSet result_set = 2; - Ydb.TableStats.QueryStats query_stats = 3; -} - +// Response for StreamExecuteYql is a stream of ExecuteYqlPartialResponse messages. +// These responses can contain ExecuteYqlPartialResult messages with +// results (or result parts) for data or scan queries in the script. +// YqlScript can have multiple results (result sets). +// Each result set has an index (starting at 0). +message ExecuteYqlPartialResponse { + StatusIds.StatusCode status = 1; + repeated Ydb.Issue.IssueMessage issues = 2; + ExecuteYqlPartialResult result = 3; +} + +// Contains result set (or a result set part) for one data or scan query in the script. +// One result set can be split into several responses with same result_index. +// Only the final response can contain query stats. +message ExecuteYqlPartialResult { + // Index of current result + uint32 result_set_index = 1; + // Result set (or a result set part) for one data or scan query + Ydb.ResultSet result_set = 2; + Ydb.TableStats.QueryStats query_stats = 3; +} + message ExplainYqlRequest { enum Mode { - MODE_UNSPECIFIED = 0; - // PARSE = 1; - VALIDATE = 2; - PLAN = 3; + MODE_UNSPECIFIED = 0; + // PARSE = 1; + VALIDATE = 2; + PLAN = 3; } Ydb.Operations.OperationParams operation_params = 1; diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 4dd25ba8ae..c8063d9475 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -352,47 +352,47 @@ message ColumnFamily { Ydb.FeatureFlag.Status keep_in_memory = 4; } -message PartitioningSettings { - // List of columns (primary key prefix) to partition by - reserved 1; // partition_by - // Enable auto partitioning on reaching upper or lower partition size bound - Ydb.FeatureFlag.Status partitioning_by_size = 2; - // Preferred partition size for auto partitioning by size, Mb - uint64 partition_size_mb = 3; - // Enable auto partitioning based on load on each partition +message PartitioningSettings { + // List of columns (primary key prefix) to partition by + reserved 1; // partition_by + // Enable auto partitioning on reaching upper or lower partition size bound + Ydb.FeatureFlag.Status partitioning_by_size = 2; + // Preferred partition size for auto partitioning by size, Mb + uint64 partition_size_mb = 3; + // Enable auto partitioning based on load on each partition Ydb.FeatureFlag.Status partitioning_by_load = 4; - reserved 5; // partitioning_by_load settings - // Minimum partitions count auto merge would stop working at - uint64 min_partitions_count = 6; - // Maximum partitions count auto split would stop working at - uint64 max_partitions_count = 7; -} - -message AzReadReplicasSettings { - // AZ name - string name = 1; - // Read replicas count in this AZ - uint64 read_replicas_count = 2; -} - -message ClusterReplicasSettings { - // List of read replicas settings for each AZ - repeated AzReadReplicasSettings az_read_replicas_settings = 2; -} - -message ReadReplicasSettings { - oneof settings { - // Set equal read replicas count for every AZ - uint64 per_az_read_replicas_count = 1; - - // Set total replicas count between all AZs - uint64 any_az_read_replicas_count = 2; - } - - // Specify read replicas count for each AZ in cluster - reserved 3; // cluster_replicas_settings (part of oneof settings) -} - + reserved 5; // partitioning_by_load settings + // Minimum partitions count auto merge would stop working at + uint64 min_partitions_count = 6; + // Maximum partitions count auto split would stop working at + uint64 max_partitions_count = 7; +} + +message AzReadReplicasSettings { + // AZ name + string name = 1; + // Read replicas count in this AZ + uint64 read_replicas_count = 2; +} + +message ClusterReplicasSettings { + // List of read replicas settings for each AZ + repeated AzReadReplicasSettings az_read_replicas_settings = 2; +} + +message ReadReplicasSettings { + oneof settings { + // Set equal read replicas count for every AZ + uint64 per_az_read_replicas_count = 1; + + // Set total replicas count between all AZs + uint64 any_az_read_replicas_count = 2; + } + + // Specify read replicas count for each AZ in cluster + reserved 3; // cluster_replicas_settings (part of oneof settings) +} + message CreateTableRequest { // Session identifier string session_id = 1; @@ -415,22 +415,22 @@ message CreateTableRequest { repeated ColumnFamily column_families = 10; // Attributes. Total size is limited to 10 KB. map<string, string> attributes = 11 [(map_key).length.range = {min: 1, max: 100}, (length).range = {min: 1, max: 4096}]; - // Predefined named set of settings for table compaction ["default", "small_table", "log_table"]. - string compaction_policy = 12; - // Either one of the following partitions options can be specified - oneof partitions { - // Enable uniform partitioning using given partitions count. - // The first components of primary key must have Uint32/Uint64 type. - uint64 uniform_partitions = 13; - // Explicitly specify key values which are used as borders for created partitions. - ExplicitPartitions partition_at_keys = 14; - } - // Partitioning settings for table - PartitioningSettings partitioning_settings = 15; - // Bloom filter by key - Ydb.FeatureFlag.Status key_bloom_filter = 16; - // Read replicas settings for table - ReadReplicasSettings read_replicas_settings = 17; + // Predefined named set of settings for table compaction ["default", "small_table", "log_table"]. + string compaction_policy = 12; + // Either one of the following partitions options can be specified + oneof partitions { + // Enable uniform partitioning using given partitions count. + // The first components of primary key must have Uint32/Uint64 type. + uint64 uniform_partitions = 13; + // Explicitly specify key values which are used as borders for created partitions. + ExplicitPartitions partition_at_keys = 14; + } + // Partitioning settings for table + PartitioningSettings partitioning_settings = 15; + // Bloom filter by key + Ydb.FeatureFlag.Status key_bloom_filter = 16; + // Read replicas settings for table + ReadReplicasSettings read_replicas_settings = 17; } message CreateTableResponse { @@ -481,15 +481,15 @@ message AlterTableRequest { // Alter attributes. Leave the value blank to drop an attribute. // Cannot be used in combination with other fields (except session_id and path) at the moment. map<string, string> alter_attributes = 14 [(map_key).length.range = {min: 1, max: 100}, (length).le = 4096]; - // Set predefined named set of settings for table compaction ["default", "small_table", "log_table"]. - // Set "default" to use default preset. - string set_compaction_policy = 15; - // Change table partitioning settings - PartitioningSettings alter_partitioning_settings = 16; - // Enable/disable bloom filter by key - Ydb.FeatureFlag.Status set_key_bloom_filter = 17; - // Set read replicas settings for table - ReadReplicasSettings set_read_replicas_settings = 18; + // Set predefined named set of settings for table compaction ["default", "small_table", "log_table"]. + // Set "default" to use default preset. + string set_compaction_policy = 15; + // Change table partitioning settings + PartitioningSettings alter_partitioning_settings = 16; + // Enable/disable bloom filter by key + Ydb.FeatureFlag.Status set_key_bloom_filter = 17; + // Set read replicas settings for table + ReadReplicasSettings set_read_replicas_settings = 18; } message AlterTableResponse { @@ -596,14 +596,14 @@ message DescribeTableResult { repeated ColumnFamily column_families = 9; // Attributes map<string, string> attributes = 10; - // Predefined named set of settings for table compaction - reserved 11; // compaction_policy - // Partitioning settings for table - PartitioningSettings partitioning_settings = 12; - // Bloom filter by key - Ydb.FeatureFlag.Status key_bloom_filter = 13; - // Read replicas settings for table - ReadReplicasSettings read_replicas_settings = 14; + // Predefined named set of settings for table compaction + reserved 11; // compaction_policy + // Partitioning settings for table + PartitioningSettings partitioning_settings = 12; + // Bloom filter by key + Ydb.FeatureFlag.Status key_bloom_filter = 13; + // Read replicas settings for table + ReadReplicasSettings read_replicas_settings = 14; } message Query { diff --git a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp index 55a9639a8e..cb7dc3ed6d 100644 --- a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp +++ b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp @@ -301,7 +301,7 @@ Y_UNIT_TEST_SUITE(ClientLibSchema) { } static void CreateVeryPartitionedTable(NClient::TKikimr& kikimr) { - const ui64 numParts = NValgrind::PlainOrUnderValgrind(3000, 10); + const ui64 numParts = NValgrind::PlainOrUnderValgrind(3000, 10); auto dc = kikimr.GetSchemaRoot("dc-1"); auto zoo = dc.MakeDirectory("Zoo"); diff --git a/ydb/public/lib/json_value/ut/ya.make b/ydb/public/lib/json_value/ut/ya.make index 05006ccb26..d1c93eecf5 100644 --- a/ydb/public/lib/json_value/ut/ya.make +++ b/ydb/public/lib/json_value/ut/ya.make @@ -1,24 +1,24 @@ UNITTEST_FOR(ydb/public/lib/json_value) - -OWNER( - pnv1 - g:kikimr -) - -TIMEOUT(600) -SIZE(MEDIUM) - -FORK_SUBTESTS() - -SRCS( - ydb_json_value_ut.cpp -) - -PEERDIR( +OWNER( + pnv1 + g:kikimr +) + +TIMEOUT(600) + +SIZE(MEDIUM) + +FORK_SUBTESTS() + +SRCS( + ydb_json_value_ut.cpp +) + +PEERDIR( library/cpp/json library/cpp/testing/unittest ydb/public/sdk/cpp/client/ydb_proto -) - -END() +) + +END() diff --git a/ydb/public/lib/json_value/ya.make b/ydb/public/lib/json_value/ya.make index a2416a64c1..b42d76c2c0 100644 --- a/ydb/public/lib/json_value/ya.make +++ b/ydb/public/lib/json_value/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - ydb_json_value.cpp - ydb_json_value_ut.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + ydb_json_value.cpp + ydb_json_value_ut.cpp +) + +PEERDIR( library/cpp/json/writer - library/cpp/string_utils/base64 + library/cpp/string_utils/base64 ydb/public/sdk/cpp/client/ydb_result ydb/public/sdk/cpp/client/ydb_value -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/lib/json_value/ydb_json_value.cpp b/ydb/public/lib/json_value/ydb_json_value.cpp index 23369c6777..af126501a7 100644 --- a/ydb/public/lib/json_value/ydb_json_value.cpp +++ b/ydb/public/lib/json_value/ydb_json_value.cpp @@ -1,813 +1,813 @@ -#include "ydb_json_value.h" - -#include <library/cpp/string_utils/base64/base64.h> -#include <util/string/builder.h> -#include <util/string/printf.h> -#include <library/cpp/json/json_reader.h> - -namespace NYdb { - - namespace { - - class TUtf8Transcoder - { - public: +#include "ydb_json_value.h" + +#include <library/cpp/string_utils/base64/base64.h> +#include <util/string/builder.h> +#include <util/string/printf.h> +#include <library/cpp/json/json_reader.h> + +namespace NYdb { + + namespace { + + class TUtf8Transcoder + { + public: explicit TUtf8Transcoder() - { - }; - - TStringBuf Encode(TStringBuf str) { - Buffer.clear(); - IsAscii = true; - for (CurPos = 0; CurPos < str.size(); ++CurPos) { - ui8 c = str[CurPos]; - if (c == '"' || c == '\\') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back(c); - } else if (c == '\b') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back('b'); - } else if (c == '\t') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back('t'); - } else if (c == '\f') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back('f'); - } else if (c == '\r') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back('r'); - } else if (c == '\n') { - SwitchToNonAscii(str); - Buffer.push_back('\\'); - Buffer.push_back('n'); - } else if (c < '\x20' || c > '\x7E') { - SwitchToNonAscii(str); - TString tmp = Sprintf("\\u%04X", c); - for (unsigned char c : tmp) { - Buffer.push_back(c); - } - } else { - if (!IsAscii) { - Buffer.push_back(c); - } - } - } - if (IsAscii) { - return str; - } else { - return TStringBuf(Buffer.data(), Buffer.size()); - } - } - - TStringBuf Decode(TStringBuf str) { - Buffer.clear(); - IsAscii = true; - for (size_t i = 0; i < str.size(); ++i) { - char c = str[i]; - if (ui8(c) < 128) { - if (!IsAscii) { - Buffer.push_back(c); - } - } else if ((c & '\xFC') == '\xC0') { - if (IsAscii) { - Buffer.resize(i); - std::copy(str.data(), str.data() + i, Buffer.data()); - IsAscii = false; - } - Buffer.push_back(((c & '\x03') << 6) | (str[i + 1] & '\x3F')); - i += 1; - } else { + { + }; + + TStringBuf Encode(TStringBuf str) { + Buffer.clear(); + IsAscii = true; + for (CurPos = 0; CurPos < str.size(); ++CurPos) { + ui8 c = str[CurPos]; + if (c == '"' || c == '\\') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back(c); + } else if (c == '\b') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('b'); + } else if (c == '\t') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('t'); + } else if (c == '\f') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('f'); + } else if (c == '\r') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('r'); + } else if (c == '\n') { + SwitchToNonAscii(str); + Buffer.push_back('\\'); + Buffer.push_back('n'); + } else if (c < '\x20' || c > '\x7E') { + SwitchToNonAscii(str); + TString tmp = Sprintf("\\u%04X", c); + for (unsigned char c : tmp) { + Buffer.push_back(c); + } + } else { + if (!IsAscii) { + Buffer.push_back(c); + } + } + } + if (IsAscii) { + return str; + } else { + return TStringBuf(Buffer.data(), Buffer.size()); + } + } + + TStringBuf Decode(TStringBuf str) { + Buffer.clear(); + IsAscii = true; + for (size_t i = 0; i < str.size(); ++i) { + char c = str[i]; + if (ui8(c) < 128) { + if (!IsAscii) { + Buffer.push_back(c); + } + } else if ((c & '\xFC') == '\xC0') { + if (IsAscii) { + Buffer.resize(i); + std::copy(str.data(), str.data() + i, Buffer.data()); + IsAscii = false; + } + Buffer.push_back(((c & '\x03') << 6) | (str[i + 1] & '\x3F')); + i += 1; + } else { ThrowFatalError("Unicode symbols with codes greater than 255 are not supported."); - } - } - if (IsAscii) { - return str; - } else { - return TStringBuf(Buffer.data(), Buffer.size()); - } - } - - private: - void SwitchToNonAscii(TStringBuf& str) { - if (IsAscii) { - Buffer.resize(CurPos); - std::copy(str.data(), str.data() + CurPos, Buffer.data()); - IsAscii = false; - } - } - - private: - bool IsAscii; - std::vector<char> Buffer; - size_t CurPos; - }; - - class TYdbToJsonConverter { - public: + } + } + if (IsAscii) { + return str; + } else { + return TStringBuf(Buffer.data(), Buffer.size()); + } + } + + private: + void SwitchToNonAscii(TStringBuf& str) { + if (IsAscii) { + Buffer.resize(CurPos); + std::copy(str.data(), str.data() + CurPos, Buffer.data()); + IsAscii = false; + } + } + + private: + bool IsAscii; + std::vector<char> Buffer; + size_t CurPos; + }; + + class TYdbToJsonConverter { + public: TYdbToJsonConverter(TValueParser& parser, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) - : Parser(parser) - , Writer(writer) - , Encoding(encoding) - { - } - - void Convert() { - ParseValue(); - } - - private: - void ParsePrimitiveValue(EPrimitiveType type) { - switch (type) { - case EPrimitiveType::Bool: - Writer.WriteBool(Parser.GetBool()); - break; - case EPrimitiveType::Int8: - Writer.WriteInt(Parser.GetInt8()); - break; - case EPrimitiveType::Uint8: - Writer.WriteInt(Parser.GetUint8()); - break; - case EPrimitiveType::Int16: - Writer.WriteInt(Parser.GetInt16()); - break; - case EPrimitiveType::Uint16: - Writer.WriteInt(Parser.GetUint16()); - break; - case EPrimitiveType::Int32: - Writer.WriteInt(Parser.GetInt32()); - break; - case EPrimitiveType::Uint32: - Writer.WriteULongLong(Parser.GetUint32()); - break; - case EPrimitiveType::Int64: - Writer.WriteLongLong(Parser.GetInt64()); - break; - case EPrimitiveType::Uint64: - Writer.WriteULongLong(Parser.GetUint64()); - break; - case EPrimitiveType::Float: - Writer.WriteFloat(Parser.GetFloat(), PREC_AUTO); - break; - case EPrimitiveType::Double: - Writer.WriteDouble(Parser.GetDouble(), PREC_AUTO); - break; - case EPrimitiveType::Date: - Writer.WriteString(Parser.GetDate().FormatGmTime("%Y-%m-%d")); - break; - case EPrimitiveType::Datetime: - Writer.WriteString(Parser.GetDatetime().ToStringUpToSeconds()); - break; - case EPrimitiveType::Timestamp: - Writer.WriteString(Parser.GetTimestamp().ToString()); - break; - case EPrimitiveType::Interval: - Writer.WriteLongLong(Parser.GetInterval()); - break; - case EPrimitiveType::TzDate: - Writer.WriteString(Parser.GetTzDate()); - break; - case EPrimitiveType::TzDatetime: - Writer.WriteString(Parser.GetTzDatetime()); - break; - case EPrimitiveType::TzTimestamp: - Writer.WriteString(Parser.GetTzTimestamp()); - break; - case EPrimitiveType::String: - Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetString())); - break; - case EPrimitiveType::Utf8: - Writer.WriteString(Parser.GetUtf8()); - break; - case EPrimitiveType::Yson: - Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetYson())); - break; - case EPrimitiveType::Json: - Writer.WriteString(Parser.GetJson()); - break; + : Parser(parser) + , Writer(writer) + , Encoding(encoding) + { + } + + void Convert() { + ParseValue(); + } + + private: + void ParsePrimitiveValue(EPrimitiveType type) { + switch (type) { + case EPrimitiveType::Bool: + Writer.WriteBool(Parser.GetBool()); + break; + case EPrimitiveType::Int8: + Writer.WriteInt(Parser.GetInt8()); + break; + case EPrimitiveType::Uint8: + Writer.WriteInt(Parser.GetUint8()); + break; + case EPrimitiveType::Int16: + Writer.WriteInt(Parser.GetInt16()); + break; + case EPrimitiveType::Uint16: + Writer.WriteInt(Parser.GetUint16()); + break; + case EPrimitiveType::Int32: + Writer.WriteInt(Parser.GetInt32()); + break; + case EPrimitiveType::Uint32: + Writer.WriteULongLong(Parser.GetUint32()); + break; + case EPrimitiveType::Int64: + Writer.WriteLongLong(Parser.GetInt64()); + break; + case EPrimitiveType::Uint64: + Writer.WriteULongLong(Parser.GetUint64()); + break; + case EPrimitiveType::Float: + Writer.WriteFloat(Parser.GetFloat(), PREC_AUTO); + break; + case EPrimitiveType::Double: + Writer.WriteDouble(Parser.GetDouble(), PREC_AUTO); + break; + case EPrimitiveType::Date: + Writer.WriteString(Parser.GetDate().FormatGmTime("%Y-%m-%d")); + break; + case EPrimitiveType::Datetime: + Writer.WriteString(Parser.GetDatetime().ToStringUpToSeconds()); + break; + case EPrimitiveType::Timestamp: + Writer.WriteString(Parser.GetTimestamp().ToString()); + break; + case EPrimitiveType::Interval: + Writer.WriteLongLong(Parser.GetInterval()); + break; + case EPrimitiveType::TzDate: + Writer.WriteString(Parser.GetTzDate()); + break; + case EPrimitiveType::TzDatetime: + Writer.WriteString(Parser.GetTzDatetime()); + break; + case EPrimitiveType::TzTimestamp: + Writer.WriteString(Parser.GetTzTimestamp()); + break; + case EPrimitiveType::String: + Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetString())); + break; + case EPrimitiveType::Utf8: + Writer.WriteString(Parser.GetUtf8()); + break; + case EPrimitiveType::Yson: + Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetYson())); + break; + case EPrimitiveType::Json: + Writer.WriteString(Parser.GetJson()); + break; case EPrimitiveType::JsonDocument: - Writer.WriteString(Parser.GetJsonDocument()); + Writer.WriteString(Parser.GetJsonDocument()); break; case EPrimitiveType::DyNumber: - Writer.WriteString(Parser.GetDyNumber()); + Writer.WriteString(Parser.GetDyNumber()); break; - default: + default: ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); - } - } - - void ParseValue() { - switch (Parser.GetKind()) { - case TTypeParser::ETypeKind::Primitive: - ParsePrimitiveValue(Parser.GetPrimitiveType()); - break; - - case TTypeParser::ETypeKind::Decimal: - Writer.WriteString(Parser.GetDecimal().ToString()); - break; - - case TTypeParser::ETypeKind::Optional: - Parser.OpenOptional(); - if (Parser.IsNull()) { - Writer.WriteNull(); - } else { - ParseValue(); - } - Parser.CloseOptional(); - break; - - case TTypeParser::ETypeKind::List: - Parser.OpenList(); - Writer.BeginList(); - - while (Parser.TryNextListItem()) { - ParseValue(); - } - - Parser.CloseList(); - Writer.EndList(); - break; - - case TTypeParser::ETypeKind::Struct: - Parser.OpenStruct(); - Writer.BeginObject(); - - while (Parser.TryNextMember()) { - Writer.WriteKey(Parser.GetMemberName()); - ParseValue(); - } - - Parser.CloseStruct(); - Writer.EndObject(); - break; - - case TTypeParser::ETypeKind::Tuple: - Parser.OpenTuple(); - Writer.BeginList(); - - while (Parser.TryNextElement()) { - ParseValue(); - } - - Parser.CloseTuple(); - Writer.EndList(); - break; - - case TTypeParser::ETypeKind::Dict: - Parser.OpenDict(); - Writer.BeginList(); - - while (Parser.TryNextDictItem()) { - Writer.BeginList(); - Parser.DictKey(); - ParseValue(); - Parser.DictPayload(); - ParseValue(); - Writer.EndList(); - } - Parser.CloseDict(); - Writer.EndList(); - break; - - default: + } + } + + void ParseValue() { + switch (Parser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + ParsePrimitiveValue(Parser.GetPrimitiveType()); + break; + + case TTypeParser::ETypeKind::Decimal: + Writer.WriteString(Parser.GetDecimal().ToString()); + break; + + case TTypeParser::ETypeKind::Optional: + Parser.OpenOptional(); + if (Parser.IsNull()) { + Writer.WriteNull(); + } else { + ParseValue(); + } + Parser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::List: + Parser.OpenList(); + Writer.BeginList(); + + while (Parser.TryNextListItem()) { + ParseValue(); + } + + Parser.CloseList(); + Writer.EndList(); + break; + + case TTypeParser::ETypeKind::Struct: + Parser.OpenStruct(); + Writer.BeginObject(); + + while (Parser.TryNextMember()) { + Writer.WriteKey(Parser.GetMemberName()); + ParseValue(); + } + + Parser.CloseStruct(); + Writer.EndObject(); + break; + + case TTypeParser::ETypeKind::Tuple: + Parser.OpenTuple(); + Writer.BeginList(); + + while (Parser.TryNextElement()) { + ParseValue(); + } + + Parser.CloseTuple(); + Writer.EndList(); + break; + + case TTypeParser::ETypeKind::Dict: + Parser.OpenDict(); + Writer.BeginList(); + + while (Parser.TryNextDictItem()) { + Writer.BeginList(); + Parser.DictKey(); + ParseValue(); + Parser.DictPayload(); + ParseValue(); + Writer.EndList(); + } + Parser.CloseDict(); + Writer.EndList(); + break; + + default: ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << Parser.GetKind()); - } - } - - TString BinaryStringToJsonString(const TString& s) { - TStringStream str; - str << "\""; - switch (Encoding) { - case EBinaryStringEncoding::Unicode: - str << Utf8Transcoder.Encode(s); - break; - case EBinaryStringEncoding::Base64: - str << Base64Encode(s); - break; - default: + } + } + + TString BinaryStringToJsonString(const TString& s) { + TStringStream str; + str << "\""; + switch (Encoding) { + case EBinaryStringEncoding::Unicode: + str << Utf8Transcoder.Encode(s); + break; + case EBinaryStringEncoding::Base64: + str << Base64Encode(s); + break; + default: ThrowFatalError(TStringBuilder() << "Unknown binary string encode mode: " - << static_cast<size_t>(Encoding)); - } - str << "\""; - return str.Str(); - } - - private: - TValueParser& Parser; - NJsonWriter::TBuf& Writer; - EBinaryStringEncoding Encoding; - TUtf8Transcoder Utf8Transcoder; - }; -} - -void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, + << static_cast<size_t>(Encoding)); + } + str << "\""; + return str.Str(); + } + + private: + TValueParser& Parser; + NJsonWriter::TBuf& Writer; + EBinaryStringEncoding Encoding; + TUtf8Transcoder Utf8Transcoder; + }; +} + +void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) -{ - TValueParser typeParser(value); +{ + TValueParser typeParser(value); TYdbToJsonConverter converter(typeParser, writer, encoding); - converter.Convert(); -} - + converter.Convert(); +} + TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding) -{ - TStringStream out; - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); - +{ + TStringStream out; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); + FormatValueJson(value, writer, encoding); - - return out.Str(); -} - -void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, + + return out.Str(); +} + +void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding) -{ - writer.BeginObject(); - for (ui32 i = 0; i < columns.size(); ++i) { - writer.WriteKey(columns[i].Name); +{ + writer.BeginObject(); + for (ui32 i = 0; i < columns.size(); ++i) { + writer.WriteKey(columns[i].Name); TYdbToJsonConverter converter(parser.ColumnParser(i), writer, encoding); - converter.Convert(); - } - writer.EndObject(); -} - -TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, + converter.Convert(); + } + writer.EndObject(); +} + +TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, EBinaryStringEncoding encoding) -{ - TStringStream out; - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); - +{ + TStringStream out; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &out); + FormatResultRowJson(parser, columns, writer, encoding); - - return out.Str(); -} - + + return out.Str(); +} + void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding) -{ - auto columns = result.GetColumnsMeta(); - - TResultSetParser parser(result); - - while (parser.TryNextRow()) { - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, out); +{ + auto columns = result.GetColumnsMeta(); + + TResultSetParser parser(result); + + while (parser.TryNextRow()) { + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, out); FormatResultRowJson(parser, columns, writer, encoding); - *out << Endl; - } -} - + *out << Endl; + } +} + TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding) -{ - TStringStream out; - +{ + TStringStream out; + FormatResultSetJson(result, &out, encoding); - - return out.Str(); -} - -namespace { - class TJsonToYdbConverter { - public: - TJsonToYdbConverter(TValueBuilder& valueBuilder, const NJson::TJsonValue& jsonValue, TTypeParser& typeParser, + + return out.Str(); +} + +namespace { + class TJsonToYdbConverter { + public: + TJsonToYdbConverter(TValueBuilder& valueBuilder, const NJson::TJsonValue& jsonValue, TTypeParser& typeParser, EBinaryStringEncoding encoding) - : ValueBuilder(valueBuilder) - , JsonValue(jsonValue) - , TypeParser(typeParser) - , Encoding(encoding) - { - } - - void Convert() { - ParseValue(JsonValue); - } - - private: - void ParsePrimitiveValue(const NJson::TJsonValue& jsonValue, EPrimitiveType type) { - switch (type) { - case EPrimitiveType::Bool: - EnsureType(jsonValue, NJson::JSON_BOOLEAN); - ValueBuilder.Bool(jsonValue.GetBoolean()); - break; - case EPrimitiveType::Int8: - { - EnsureType(jsonValue, NJson::JSON_INTEGER); - long long intValue = jsonValue.GetInteger(); - if (intValue > std::numeric_limits<i8>::max() || intValue < std::numeric_limits<i8>::min()) { + : ValueBuilder(valueBuilder) + , JsonValue(jsonValue) + , TypeParser(typeParser) + , Encoding(encoding) + { + } + + void Convert() { + ParseValue(JsonValue); + } + + private: + void ParsePrimitiveValue(const NJson::TJsonValue& jsonValue, EPrimitiveType type) { + switch (type) { + case EPrimitiveType::Bool: + EnsureType(jsonValue, NJson::JSON_BOOLEAN); + ValueBuilder.Bool(jsonValue.GetBoolean()); + break; + case EPrimitiveType::Int8: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits<i8>::max() || intValue < std::numeric_limits<i8>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int8 type"); - } - ValueBuilder.Int8(intValue); - break; - } - case EPrimitiveType::Uint8: - { - EnsureType(jsonValue, NJson::JSON_UINTEGER); - unsigned long long intValue = jsonValue.GetUInteger(); - if (intValue > std::numeric_limits<ui8>::max() || intValue < std::numeric_limits<ui8>::min()) { + } + ValueBuilder.Int8(intValue); + break; + } + case EPrimitiveType::Uint8: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits<ui8>::max() || intValue < std::numeric_limits<ui8>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt8 type"); - } - ValueBuilder.Uint8(intValue); - break; - } - case EPrimitiveType::Int16: - { - EnsureType(jsonValue, NJson::JSON_INTEGER); - long long intValue = jsonValue.GetInteger(); - if (intValue > std::numeric_limits<i16>::max() || intValue < std::numeric_limits<i16>::min()) { + } + ValueBuilder.Uint8(intValue); + break; + } + case EPrimitiveType::Int16: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits<i16>::max() || intValue < std::numeric_limits<i16>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int16 type"); - } - ValueBuilder.Int16(intValue); - break; - } - case EPrimitiveType::Uint16: - { - EnsureType(jsonValue, NJson::JSON_UINTEGER); - unsigned long long intValue = jsonValue.GetUInteger(); - if (intValue > std::numeric_limits<ui16>::max() || intValue < std::numeric_limits<ui16>::min()) { + } + ValueBuilder.Int16(intValue); + break; + } + case EPrimitiveType::Uint16: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits<ui16>::max() || intValue < std::numeric_limits<ui16>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt16 type"); - } - ValueBuilder.Uint16(intValue); - break; - } - case EPrimitiveType::Int32: - { - EnsureType(jsonValue, NJson::JSON_INTEGER); - long long intValue = jsonValue.GetInteger(); - if (intValue > std::numeric_limits<i32>::max() || intValue < std::numeric_limits<i32>::min()) { + } + ValueBuilder.Uint16(intValue); + break; + } + case EPrimitiveType::Int32: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits<i32>::max() || intValue < std::numeric_limits<i32>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int32 type"); - } - ValueBuilder.Int32(intValue); - break; - } - case EPrimitiveType::Uint32: - { - EnsureType(jsonValue, NJson::JSON_UINTEGER); - unsigned long long intValue = jsonValue.GetUInteger(); - if (intValue > std::numeric_limits<ui32>::max() || intValue < std::numeric_limits<ui32>::min()) { + } + ValueBuilder.Int32(intValue); + break; + } + case EPrimitiveType::Uint32: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits<ui32>::max() || intValue < std::numeric_limits<ui32>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt32 type"); - } - ValueBuilder.Uint32(intValue); - break; - } - case EPrimitiveType::Int64: - { - EnsureType(jsonValue, NJson::JSON_INTEGER); - long long intValue = jsonValue.GetInteger(); - if (intValue > std::numeric_limits<i64>::max() || intValue < std::numeric_limits<i64>::min()) { + } + ValueBuilder.Uint32(intValue); + break; + } + case EPrimitiveType::Int64: + { + EnsureType(jsonValue, NJson::JSON_INTEGER); + long long intValue = jsonValue.GetInteger(); + if (intValue > std::numeric_limits<i64>::max() || intValue < std::numeric_limits<i64>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in Int64 type"); - } - ValueBuilder.Int64(intValue); - break; - } - case EPrimitiveType::Uint64: - { - EnsureType(jsonValue, NJson::JSON_UINTEGER); - unsigned long long intValue = jsonValue.GetUInteger(); - if (intValue > std::numeric_limits<ui64>::max() || intValue < std::numeric_limits<ui64>::min()) { + } + ValueBuilder.Int64(intValue); + break; + } + case EPrimitiveType::Uint64: + { + EnsureType(jsonValue, NJson::JSON_UINTEGER); + unsigned long long intValue = jsonValue.GetUInteger(); + if (intValue > std::numeric_limits<ui64>::max() || intValue < std::numeric_limits<ui64>::min()) { ThrowFatalError(TStringBuilder() << "Value \"" << intValue << "\" doesn't fit in UInt64 type"); - } - ValueBuilder.Uint64(intValue); - break; - } - case EPrimitiveType::Float: - EnsureType(jsonValue, NJson::JSON_DOUBLE); - ValueBuilder.Float(jsonValue.GetDouble()); - break; - case EPrimitiveType::Double: - EnsureType(jsonValue, NJson::JSON_DOUBLE); - ValueBuilder.Double(jsonValue.GetDouble()); - break; - case EPrimitiveType::Date: - { - EnsureType(jsonValue, NJson::JSON_STRING); - TInstant date; - if (!TInstant::TryParseIso8601(jsonValue.GetString(), date)) { + } + ValueBuilder.Uint64(intValue); + break; + } + case EPrimitiveType::Float: + EnsureType(jsonValue, NJson::JSON_DOUBLE); + ValueBuilder.Float(jsonValue.GetDouble()); + break; + case EPrimitiveType::Double: + EnsureType(jsonValue, NJson::JSON_DOUBLE); + ValueBuilder.Double(jsonValue.GetDouble()); + break; + case EPrimitiveType::Date: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant date; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), date)) { ThrowFatalError(TStringBuilder() << "Can't parse date from string \"" << jsonValue.GetString() << "\""); - } - ValueBuilder.Date(date); - break; - } - case EPrimitiveType::Datetime: - { - EnsureType(jsonValue, NJson::JSON_STRING); - TInstant dateTime; - if (!TInstant::TryParseIso8601(jsonValue.GetString(), dateTime)) { + } + ValueBuilder.Date(date); + break; + } + case EPrimitiveType::Datetime: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant dateTime; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), dateTime)) { ThrowFatalError(TStringBuilder() << "Can't parse dateTime from string \"" << jsonValue.GetString() << "\""); - } - ValueBuilder.Datetime(dateTime); - break; - } - case EPrimitiveType::Timestamp: - { - EnsureType(jsonValue, NJson::JSON_STRING); - TInstant timestamp; - if (!TInstant::TryParseIso8601(jsonValue.GetString(), timestamp)) { + } + ValueBuilder.Datetime(dateTime); + break; + } + case EPrimitiveType::Timestamp: + { + EnsureType(jsonValue, NJson::JSON_STRING); + TInstant timestamp; + if (!TInstant::TryParseIso8601(jsonValue.GetString(), timestamp)) { ThrowFatalError(TStringBuilder() << "Can't parse timestamp from string \"" << jsonValue.GetString() << "\""); - } - ValueBuilder.Timestamp(timestamp); - break; - } - case EPrimitiveType::Interval: - EnsureType(jsonValue, NJson::JSON_INTEGER); - ValueBuilder.Interval(jsonValue.GetInteger()); - break; - case EPrimitiveType::TzDate: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.TzDate(jsonValue.GetString()); - break; - case EPrimitiveType::TzDatetime: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.TzDatetime(jsonValue.GetString()); - break; - case EPrimitiveType::TzTimestamp: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.TzTimestamp(jsonValue.GetString()); - break; - case EPrimitiveType::String: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.String(JsonStringToBinaryString(jsonValue.GetString())); - break; - case EPrimitiveType::Utf8: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.Utf8(jsonValue.GetString()); - break; - case EPrimitiveType::Yson: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.Yson(JsonStringToBinaryString(jsonValue.GetString())); - break; - case EPrimitiveType::Json: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.Json(jsonValue.GetString()); - break; - case EPrimitiveType::JsonDocument: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.JsonDocument(jsonValue.GetString()); - break; - case EPrimitiveType::DyNumber: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.DyNumber(jsonValue.GetString()); - break; - default: + } + ValueBuilder.Timestamp(timestamp); + break; + } + case EPrimitiveType::Interval: + EnsureType(jsonValue, NJson::JSON_INTEGER); + ValueBuilder.Interval(jsonValue.GetInteger()); + break; + case EPrimitiveType::TzDate: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzDate(jsonValue.GetString()); + break; + case EPrimitiveType::TzDatetime: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzDatetime(jsonValue.GetString()); + break; + case EPrimitiveType::TzTimestamp: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.TzTimestamp(jsonValue.GetString()); + break; + case EPrimitiveType::String: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.String(JsonStringToBinaryString(jsonValue.GetString())); + break; + case EPrimitiveType::Utf8: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Utf8(jsonValue.GetString()); + break; + case EPrimitiveType::Yson: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Yson(JsonStringToBinaryString(jsonValue.GetString())); + break; + case EPrimitiveType::Json: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Json(jsonValue.GetString()); + break; + case EPrimitiveType::JsonDocument: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.JsonDocument(jsonValue.GetString()); + break; + case EPrimitiveType::DyNumber: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.DyNumber(jsonValue.GetString()); + break; + default: ThrowFatalError(TStringBuilder() << "Unsupported primitive type: " << type); - } - } - - void BuildTypeInner(TTypeBuilder& typeBuilder) { - switch (TypeParser.GetKind()) { - case TTypeParser::ETypeKind::Primitive: - typeBuilder.Primitive(TypeParser.GetPrimitive()); - break; - case TTypeParser::ETypeKind::Decimal: - typeBuilder.Decimal(TypeParser.GetDecimal()); - break; - case TTypeParser::ETypeKind::Optional: - TypeParser.OpenOptional(); - typeBuilder.BeginOptional(); - BuildTypeInner(typeBuilder); - typeBuilder.EndOptional(); - TypeParser.CloseOptional(); - break; - case TTypeParser::ETypeKind::List: - TypeParser.OpenList(); - typeBuilder.BeginList(); - BuildTypeInner(typeBuilder); - TypeParser.CloseList(); - typeBuilder.EndList(); - break; - case TTypeParser::ETypeKind::Struct: - TypeParser.OpenStruct(); - typeBuilder.BeginStruct(); - while (TypeParser.TryNextMember()) { - typeBuilder.AddMember(TypeParser.GetMemberName()); - BuildTypeInner(typeBuilder); - } - TypeParser.CloseStruct(); - typeBuilder.EndStruct(); - break; - case TTypeParser::ETypeKind::Tuple: - TypeParser.OpenTuple(); - typeBuilder.BeginTuple(); - while (TypeParser.TryNextMember()) { - typeBuilder.AddElement(); - BuildTypeInner(typeBuilder); - } - TypeParser.CloseTuple(); - typeBuilder.EndTuple(); - break; - case TTypeParser::ETypeKind::Dict: - TypeParser.OpenDict(); - typeBuilder.BeginDict(); - TypeParser.DictKey(); - typeBuilder.DictKey(); - BuildTypeInner(typeBuilder); - TypeParser.DictPayload(); - typeBuilder.DictPayload(); - BuildTypeInner(typeBuilder); - TypeParser.CloseDict(); - typeBuilder.EndDict(); - break; - default: + } + } + + void BuildTypeInner(TTypeBuilder& typeBuilder) { + switch (TypeParser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + typeBuilder.Primitive(TypeParser.GetPrimitive()); + break; + case TTypeParser::ETypeKind::Decimal: + typeBuilder.Decimal(TypeParser.GetDecimal()); + break; + case TTypeParser::ETypeKind::Optional: + TypeParser.OpenOptional(); + typeBuilder.BeginOptional(); + BuildTypeInner(typeBuilder); + typeBuilder.EndOptional(); + TypeParser.CloseOptional(); + break; + case TTypeParser::ETypeKind::List: + TypeParser.OpenList(); + typeBuilder.BeginList(); + BuildTypeInner(typeBuilder); + TypeParser.CloseList(); + typeBuilder.EndList(); + break; + case TTypeParser::ETypeKind::Struct: + TypeParser.OpenStruct(); + typeBuilder.BeginStruct(); + while (TypeParser.TryNextMember()) { + typeBuilder.AddMember(TypeParser.GetMemberName()); + BuildTypeInner(typeBuilder); + } + TypeParser.CloseStruct(); + typeBuilder.EndStruct(); + break; + case TTypeParser::ETypeKind::Tuple: + TypeParser.OpenTuple(); + typeBuilder.BeginTuple(); + while (TypeParser.TryNextMember()) { + typeBuilder.AddElement(); + BuildTypeInner(typeBuilder); + } + TypeParser.CloseTuple(); + typeBuilder.EndTuple(); + break; + case TTypeParser::ETypeKind::Dict: + TypeParser.OpenDict(); + typeBuilder.BeginDict(); + TypeParser.DictKey(); + typeBuilder.DictKey(); + BuildTypeInner(typeBuilder); + TypeParser.DictPayload(); + typeBuilder.DictPayload(); + BuildTypeInner(typeBuilder); + TypeParser.CloseDict(); + typeBuilder.EndDict(); + break; + default: ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); - } - } - - TType GetType() { - TTypeBuilder typeBuilder; - BuildTypeInner(typeBuilder); - return typeBuilder.Build(); - } - - void ParseValue(const NJson::TJsonValue& jsonValue) { - switch (TypeParser.GetKind()) { - case TTypeParser::ETypeKind::Primitive: - ParsePrimitiveValue(jsonValue, TypeParser.GetPrimitive()); - break; - - case TTypeParser::ETypeKind::Decimal: - EnsureType(jsonValue, NJson::JSON_STRING); - ValueBuilder.Decimal(jsonValue.GetString()); - break; - - case TTypeParser::ETypeKind::Optional: - TypeParser.OpenOptional(); - if (jsonValue.IsNull() && TypeParser.GetKind() != TTypeParser::ETypeKind::Optional) { - ValueBuilder.EmptyOptional(GetType()); - } else { - ValueBuilder.BeginOptional(); - ParseValue(jsonValue); - ValueBuilder.EndOptional(); - } - TypeParser.CloseOptional(); - break; - - case TTypeParser::ETypeKind::List: - EnsureType(jsonValue, NJson::JSON_ARRAY); - TypeParser.OpenList(); - ValueBuilder.BeginList(); - - for (const auto& element : jsonValue.GetArray()) { - ValueBuilder.AddListItem(); - ParseValue(element); - } - - ValueBuilder.EndList(); - TypeParser.CloseList(); - break; - - case TTypeParser::ETypeKind::Struct: - { - EnsureType(jsonValue, NJson::JSON_MAP); - TypeParser.OpenStruct(); - ValueBuilder.BeginStruct(); - - size_t counter = 0; - const auto& jsonMap = jsonValue.GetMap(); - while (TypeParser.TryNextMember()) { - const TString& memberName = TypeParser.GetMemberName(); - const auto it = jsonMap.find(memberName); - if (it == jsonMap.end()) { + } + } + + TType GetType() { + TTypeBuilder typeBuilder; + BuildTypeInner(typeBuilder); + return typeBuilder.Build(); + } + + void ParseValue(const NJson::TJsonValue& jsonValue) { + switch (TypeParser.GetKind()) { + case TTypeParser::ETypeKind::Primitive: + ParsePrimitiveValue(jsonValue, TypeParser.GetPrimitive()); + break; + + case TTypeParser::ETypeKind::Decimal: + EnsureType(jsonValue, NJson::JSON_STRING); + ValueBuilder.Decimal(jsonValue.GetString()); + break; + + case TTypeParser::ETypeKind::Optional: + TypeParser.OpenOptional(); + if (jsonValue.IsNull() && TypeParser.GetKind() != TTypeParser::ETypeKind::Optional) { + ValueBuilder.EmptyOptional(GetType()); + } else { + ValueBuilder.BeginOptional(); + ParseValue(jsonValue); + ValueBuilder.EndOptional(); + } + TypeParser.CloseOptional(); + break; + + case TTypeParser::ETypeKind::List: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenList(); + ValueBuilder.BeginList(); + + for (const auto& element : jsonValue.GetArray()) { + ValueBuilder.AddListItem(); + ParseValue(element); + } + + ValueBuilder.EndList(); + TypeParser.CloseList(); + break; + + case TTypeParser::ETypeKind::Struct: + { + EnsureType(jsonValue, NJson::JSON_MAP); + TypeParser.OpenStruct(); + ValueBuilder.BeginStruct(); + + size_t counter = 0; + const auto& jsonMap = jsonValue.GetMap(); + while (TypeParser.TryNextMember()) { + const TString& memberName = TypeParser.GetMemberName(); + const auto it = jsonMap.find(memberName); + if (it == jsonMap.end()) { ThrowFatalError(TStringBuilder() << "No member \"" << memberName - << "\" in the map in json string for YDB struct type"); - } - ValueBuilder.AddMember(memberName); - ParseValue(it->second); - ++counter; - } - if (counter != jsonMap.size()) { + << "\" in the map in json string for YDB struct type"); + } + ValueBuilder.AddMember(memberName); + ParseValue(it->second); + ++counter; + } + if (counter != jsonMap.size()) { ThrowFatalError("Map in json string contains more members than YDB struct does"); - } - - ValueBuilder.EndStruct(); - TypeParser.CloseStruct(); - break; - } - - case TTypeParser::ETypeKind::Tuple: - EnsureType(jsonValue, NJson::JSON_ARRAY); - TypeParser.OpenTuple(); - ValueBuilder.BeginTuple(); - - for (const auto& element : jsonValue.GetArray()) { - if (!TypeParser.TryNextElement()) { + } + + ValueBuilder.EndStruct(); + TypeParser.CloseStruct(); + break; + } + + case TTypeParser::ETypeKind::Tuple: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenTuple(); + ValueBuilder.BeginTuple(); + + for (const auto& element : jsonValue.GetArray()) { + if (!TypeParser.TryNextElement()) { ThrowFatalError("Tuple in json string should contain less elements than provided"); - } - ValueBuilder.AddElement(); - ParseValue(element); - } - if (TypeParser.TryNextElement()) { + } + ValueBuilder.AddElement(); + ParseValue(element); + } + if (TypeParser.TryNextElement()) { ThrowFatalError("Tuple in json string should contain more elements than provided"); - } - - ValueBuilder.EndTuple(); - TypeParser.CloseTuple(); - break; - - case TTypeParser::ETypeKind::Dict: - EnsureType(jsonValue, NJson::JSON_ARRAY); - TypeParser.OpenDict(); - - if (jsonValue.GetArray().size()) { - ValueBuilder.BeginDict(); - for (const auto& keyValueElement : jsonValue.GetArray()) { - EnsureType(keyValueElement, NJson::JSON_ARRAY); - const auto& keyValueArray = keyValueElement.GetArray(); - if (keyValueArray.size() != 2) { + } + + ValueBuilder.EndTuple(); + TypeParser.CloseTuple(); + break; + + case TTypeParser::ETypeKind::Dict: + EnsureType(jsonValue, NJson::JSON_ARRAY); + TypeParser.OpenDict(); + + if (jsonValue.GetArray().size()) { + ValueBuilder.BeginDict(); + for (const auto& keyValueElement : jsonValue.GetArray()) { + EnsureType(keyValueElement, NJson::JSON_ARRAY); + const auto& keyValueArray = keyValueElement.GetArray(); + if (keyValueArray.size() != 2) { ThrowFatalError("Each element of a dict type in YDB must be represented with " - "exactly 2 elements in array in json string"); - } - auto it = keyValueArray.begin(); - - ValueBuilder.AddDictItem(); - - TypeParser.DictKey(); - ValueBuilder.DictKey(); - ParseValue(*it); - - TypeParser.DictPayload(); - ValueBuilder.DictPayload(); - ParseValue(*(++it)); - } - ValueBuilder.EndDict(); - } else { - TypeParser.DictKey(); - TType keyType = GetType(); - TypeParser.DictPayload(); - TType payloadType = GetType(); - ValueBuilder.EmptyDict(keyType, payloadType); - } - - TypeParser.CloseDict(); - break; - - default: + "exactly 2 elements in array in json string"); + } + auto it = keyValueArray.begin(); + + ValueBuilder.AddDictItem(); + + TypeParser.DictKey(); + ValueBuilder.DictKey(); + ParseValue(*it); + + TypeParser.DictPayload(); + ValueBuilder.DictPayload(); + ParseValue(*(++it)); + } + ValueBuilder.EndDict(); + } else { + TypeParser.DictKey(); + TType keyType = GetType(); + TypeParser.DictPayload(); + TType payloadType = GetType(); + ValueBuilder.EmptyDict(keyType, payloadType); + } + + TypeParser.CloseDict(); + break; + + default: ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << TypeParser.GetKind()); - } - - } - - TString JsonStringToBinaryString(const TString& s) { - TStringStream str; - switch (Encoding) { - case EBinaryStringEncoding::Unicode: - str << Utf8Transcoder.Decode(s); - break; - case EBinaryStringEncoding::Base64: - str << Base64Decode(s); - break; - default: + } + + } + + TString JsonStringToBinaryString(const TString& s) { + TStringStream str; + switch (Encoding) { + case EBinaryStringEncoding::Unicode: + str << Utf8Transcoder.Decode(s); + break; + case EBinaryStringEncoding::Base64: + str << Base64Decode(s); + break; + default: ThrowFatalError("Unknown binary string encode mode"); - break; - } - return str.Str(); - } - - void EnsureType(const NJson::TJsonValue& value, NJson::EJsonValueType type) { - if (value.GetType() != type) { - if (value.GetType() == NJson::EJsonValueType::JSON_INTEGER && type == NJson::EJsonValueType::JSON_UINTEGER - || value.GetType() == NJson::EJsonValueType::JSON_UINTEGER && type == NJson::EJsonValueType::JSON_INTEGER) { - return; - } - TStringStream str; - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &str); - writer.WriteJsonValue(&value); + break; + } + return str.Str(); + } + + void EnsureType(const NJson::TJsonValue& value, NJson::EJsonValueType type) { + if (value.GetType() != type) { + if (value.GetType() == NJson::EJsonValueType::JSON_INTEGER && type == NJson::EJsonValueType::JSON_UINTEGER + || value.GetType() == NJson::EJsonValueType::JSON_UINTEGER && type == NJson::EJsonValueType::JSON_INTEGER) { + return; + } + TStringStream str; + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &str); + writer.WriteJsonValue(&value); ThrowFatalError(TStringBuilder() << "Wrong type for json value \"" << str.Str() - << "\". Expected type: " << type << ", received type: " << value.GetType() << ". "); - } - } - - private: - TValueBuilder& ValueBuilder; - const NJson::TJsonValue& JsonValue; - TTypeParser& TypeParser; - EBinaryStringEncoding Encoding; - TUtf8Transcoder Utf8Transcoder; - }; -} - + << "\". Expected type: " << type << ", received type: " << value.GetType() << ". "); + } + } + + private: + TValueBuilder& ValueBuilder; + const NJson::TJsonValue& JsonValue; + TTypeParser& TypeParser; + EBinaryStringEncoding Encoding; + TUtf8Transcoder Utf8Transcoder; + }; +} + TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding) { - NJson::TJsonValue jsonValue; - - try { - if (!NJson::ReadJsonTree(jsonString, &jsonValue, true)) { + NJson::TJsonValue jsonValue; + + try { + if (!NJson::ReadJsonTree(jsonString, &jsonValue, true)) { ThrowFatalError(TStringBuilder() << "Can't parse string \"" << jsonString << "\" as json."); - } - } - catch (std::exception& e) { + } + } + catch (std::exception& e) { ThrowFatalError( - TStringBuilder() << "Exception while parsing string \"" << jsonString << "\" as json: " << e.what()); - } + TStringBuilder() << "Exception while parsing string \"" << jsonString << "\" as json: " << e.what()); + } return JsonToYdbValue(jsonValue, type, encoding); } - + TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding) { TValueBuilder builder; - TTypeParser typeParser(type); - + TTypeParser typeParser(type); + TJsonToYdbConverter converter(builder, jsonValue, typeParser, encoding); - converter.Convert(); - - return builder.Build(); -} - -} // namespace NYdb + converter.Convert(); + + return builder.Build(); +} + +} // namespace NYdb diff --git a/ydb/public/lib/json_value/ydb_json_value.h b/ydb/public/lib/json_value/ydb_json_value.h index 3e93644c0f..24fa92d02e 100644 --- a/ydb/public/lib/json_value/ydb_json_value.h +++ b/ydb/public/lib/json_value/ydb_json_value.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> - + #include <library/cpp/json/writer/json.h> - -namespace NYdb { - -enum class EBinaryStringEncoding { - /* For binary strings, encode every byte that is not a printable ascii symbol (codes 32-126) as utf-8. - * Exceptions: \\ \" \b \t \f \r \n - * Example: "Hello\x01" -> "Hello\\u0001" - */ - Unicode = 1, - - /* Encode binary strings to base64 - */ - Base64 -}; - -// ====== YDB to json ====== + +namespace NYdb { + +enum class EBinaryStringEncoding { + /* For binary strings, encode every byte that is not a printable ascii symbol (codes 32-126) as utf-8. + * Exceptions: \\ \" \b \t \f \r \n + * Example: "Hello\x01" -> "Hello\\u0001" + */ + Unicode = 1, + + /* Encode binary strings to base64 + */ + Base64 +}; + +// ====== YDB to json ====== void FormatValueJson(const TValue& value, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); - + TString FormatValueJson(const TValue& value, EBinaryStringEncoding encoding); - -void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, + +void FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, NJsonWriter::TBuf& writer, EBinaryStringEncoding encoding); - -TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, + +TString FormatResultRowJson(TResultSetParser& parser, const TVector<TColumn>& columns, EBinaryStringEncoding encoding); - + void FormatResultSetJson(const TResultSet& result, IOutputStream* out, EBinaryStringEncoding encoding); - + TString FormatResultSetJson(const TResultSet& result, EBinaryStringEncoding encoding); - -// ====== json to YDB ====== + +// ====== json to YDB ====== TValue JsonToYdbValue(const TString& jsonString, const TType& type, EBinaryStringEncoding encoding); TValue JsonToYdbValue(const NJson::TJsonValue& jsonValue, const TType& type, EBinaryStringEncoding encoding); - -} // namespace NYdb + +} // namespace NYdb diff --git a/ydb/public/lib/json_value/ydb_json_value_ut.cpp b/ydb/public/lib/json_value/ydb_json_value_ut.cpp index abb539f8d5..d8c1356338 100644 --- a/ydb/public/lib/json_value/ydb_json_value_ut.cpp +++ b/ydb/public/lib/json_value/ydb_json_value_ut.cpp @@ -1,562 +1,562 @@ -#include "ydb_json_value.h" - +#include "ydb_json_value.h" + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <library/cpp/json/json_reader.h> - -namespace NYdb { - -Y_UNIT_TEST_SUITE(JsonValueTest) { - Y_UNIT_TEST(PrimitiveValueBool) { - TValue value = TValueBuilder() - .Bool(true) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "true"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueInt8) { - TValue value = TValueBuilder() - .Int8(-128) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "-128"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueUint8) { - TValue value = TValueBuilder() - .Uint8(255) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "255"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueInt16) { - TValue value = TValueBuilder() - .Int16(-32768) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "-32768"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueUint16) { - TValue value = TValueBuilder() - .Uint16(65535) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "65535"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueInt32) { - TValue value = TValueBuilder() - .Int32(-2147483648) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "-2147483648"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueUint32) { - TValue value = TValueBuilder() - .Uint32(4294967295) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "4294967295"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueInt64) { - TValue value = TValueBuilder() - .Int64(-9223372036854775807 - 1) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "-9223372036854775808"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueUint64) { - TValue value = TValueBuilder() - .Uint64(18446744073709551615ull) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "18446744073709551615"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueFloat) { - TValue value = TValueBuilder() - .Float(0.1234567890123456789) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "0.12345679"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueDouble) { - TValue value = TValueBuilder() - .Double(0.1234567890123456789) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "0.12345678901234568"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueDate) { - TInstant timestamp = TInstant::ParseIso8601("2000-01-02"); - TValue value = TValueBuilder() - .Date(TInstant::Days(timestamp.Days())) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02")"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueDatetime) { - TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05Z"); - TValue value = TValueBuilder() - .Datetime(TInstant::Seconds(timestamp.Seconds())) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05Z")"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueTimestamp) { - TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05.678901Z"); - TValue value = TValueBuilder() - .Timestamp(TInstant::MicroSeconds(timestamp.MicroSeconds())) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05.678901Z")"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueSimpleString) { - TValue value = TValueBuilder() - .String("Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ") - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - jsonString, - "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" - ); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - namespace { - TString GenerateBinaryString() { - TStringStream str; - for (ui8 i = 0; i < 255; ++i) { - str << static_cast<unsigned char>(i); - } - str << static_cast<unsigned char>(0xff); - return str.Str(); - } - } - - Y_UNIT_TEST(BinaryStringAsciiFollowedByNonAscii) { - TString binaryString = TStringBuilder() << "abc" << static_cast<unsigned char>(0xff) - << static_cast<unsigned char>(0xfe); - TValue value = TValueBuilder() - .String(binaryString) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - jsonString, - R"("abc\u00FF\u00FE")" - ); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(BinaryStringUnicode) { - TString binaryString = GenerateBinaryString(); - TValue value = TValueBuilder() - .String(binaryString) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - TStringBuilder from0To255Utf8; - from0To255Utf8 << "\""; - from0To255Utf8 << - R"(\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F)" - R"(\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F)" - " !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" - R"(\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F)" - R"(\u0090\u0091\u0092\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F)" - R"(\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF)" - R"(\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF)" - R"(\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF)" - R"(\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE\u00DF)" - R"(\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF)" - R"(\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF)"; - from0To255Utf8 << "\""; - UNIT_ASSERT_NO_DIFF( - jsonString, - from0To255Utf8 - ); - - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(BinaryStringBase64) { - TString binaryString = GenerateBinaryString(); - TValue value = TValueBuilder() - .String(binaryString) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); - UNIT_ASSERT_NO_DIFF( - jsonString, - R"("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHS)" - R"(ElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJG)" - R"(Sk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna2)" - R"(9zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==")" - ); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(EmptyBinaryStringUnicode) { - TString binaryString; - TValue value = TValueBuilder() - .String(binaryString) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(EmptyBinaryStringBase64) { - TString binaryString; - TValue value = TValueBuilder() - .String(binaryString) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); - UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - namespace { - TString InvalidJsonToBinaryStringBase(const TString& jsonString) { - TString errorMessage; - try { - TType stringType = TTypeBuilder().Primitive(EPrimitiveType::String).Build(); - TValue resultValue = JsonToYdbValue(jsonString, stringType, EBinaryStringEncoding::Unicode); - UNIT_FAIL("Exception should have been thrown, but it hasn't"); - } - catch (TContractViolation& e) { - errorMessage = e.what(); - } - catch (std::exception& e) { - UNIT_FAIL(TStringBuilder() << "Uncaught exception: " << e.what()); - } - return errorMessage; - } - } - - Y_UNIT_TEST(InvalidJsonToBinaryString1) { - TString jsonString = R"(some string")"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Invalid value" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString2) { - TString jsonString = R"("some string)"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Missing a closing quotation mark in string" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString3) { - TString jsonString = "\"some string \\\""; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Missing a closing quotation mark in string" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString4) { - TString jsonString = R"("some \ string")"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Invalid escape character in string" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString5) { - TString jsonString = R"("some string \u001")"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Incorrect hex digit after \\u escape in string" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString6) { - TString jsonString = R"("some string \u0140")"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Unicode symbols with codes greater than 255 are not supported" - ); - } - - Y_UNIT_TEST(InvalidJsonToBinaryString7) { - TString jsonString = R"("some string \u00AG")"; - TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); - UNIT_ASSERT_STRING_CONTAINS( - errorMessage, - "Incorrect hex digit after \\u escape in string" - ); - } - - Y_UNIT_TEST(PrimitiveValueUtf8String1) { - TString utf8Str = "Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() "; - TValue value = TValueBuilder() - .Utf8(utf8Str) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - - UNIT_ASSERT_NO_DIFF( - jsonString, - "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" - ); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(PrimitiveValueUtf8String2) { - TString utf8Str = "\xD0\xB0\xD0\xB1\xD0\xB2\xD0\xB3\xD0\xB4\xD0\xB5\xD1\x91\xD0\xB6\xD0\xB7\xD0\xB8\xD0\xBA\xD0\xBB" - "\xD0\xBC\xD0\xBD\xD0\xBE\xD0\xBF\xD1\x80\xD1\x81\xD1\x82\xD1\x83\xD1\x84\xD1\x85\xD1\x86\xD1\x87\xD1\x88" - "\xD1\x89\xD1\x8A\xD1\x8B\xD1\x8C\xD1\x8D\xD1\x8E\xD1\x8F\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95" - "\xD0\x81\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2" - "\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF"; - TValue value = TValueBuilder() - .Utf8(utf8Str) - .Build(); - TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - - UNIT_ASSERT_NO_DIFF( - jsonString, - TStringBuilder() << '"' << utf8Str << '"' - ); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(CompositeValueEmptyList) { - TValue value = TValueBuilder() - .EmptyList(TTypeBuilder().Primitive(EPrimitiveType::Uint32).Build()) - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "[]"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(CompositeValueIntList) { - TValue value = TValueBuilder() - .BeginList() - .AddListItem() - .Int32(1) - .AddListItem() - .Int32(10) - .AddListItem() - .Int32(100) - .EndList() - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "[1,10,100]"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(CompositeValueStruct) { - TValue value = TValueBuilder() - .BeginStruct() - .AddMember("Id") - .Uint32(1) - .AddMember("Name") - .String("Anna") - .AddMember("Value") - .Int32(-100) - .AddMember("Description") - .EmptyOptional(EPrimitiveType::Utf8) - .EndStruct() - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, R"({"Id":1,"Name":"Anna","Value":-100,"Description":null})"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(CompositeValueTuple) { - TValue value = TValueBuilder() - .BeginTuple() - .AddElement() - .BeginOptional() - .BeginOptional() - .Int32(10) - .EndOptional() - .EndOptional() - .AddElement() - .BeginOptional() - .BeginOptional() - .BeginOptional() - .Int64(-1) - .EndOptional() - .EndOptional() - .EndOptional() - .AddElement() - .BeginOptional() - .EmptyOptional(TTypeBuilder().Primitive(EPrimitiveType::String).Build()) - .EndOptional() - .AddElement() - .BeginOptional() - .BeginOptional() - .EmptyOptional(EPrimitiveType::Utf8) - .EndOptional() - .EndOptional() - .EndTuple() - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, "[10,-1,null,null]"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } - - Y_UNIT_TEST(CompositeValueDict) { - TValue value = TValueBuilder() - .BeginDict() - .AddDictItem() - .DictKey() - .Int64(1) - .DictPayload() - .String("Value1") - .AddDictItem() - .DictKey() - .Int64(2) - .DictPayload() - .String("Value2") - .EndDict() - .Build(); - const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF(jsonString, R"([[1,"Value1"],[2,"Value2"]])"); - TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); - UNIT_ASSERT_NO_DIFF( - TProtoAccessor::GetProto(value).DebugString(), - TProtoAccessor::GetProto(resultValue).DebugString() - ); - } -} - -} // namespace NYdb + +namespace NYdb { + +Y_UNIT_TEST_SUITE(JsonValueTest) { + Y_UNIT_TEST(PrimitiveValueBool) { + TValue value = TValueBuilder() + .Bool(true) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "true"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt8) { + TValue value = TValueBuilder() + .Int8(-128) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-128"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint8) { + TValue value = TValueBuilder() + .Uint8(255) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "255"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt16) { + TValue value = TValueBuilder() + .Int16(-32768) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-32768"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint16) { + TValue value = TValueBuilder() + .Uint16(65535) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "65535"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt32) { + TValue value = TValueBuilder() + .Int32(-2147483648) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-2147483648"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint32) { + TValue value = TValueBuilder() + .Uint32(4294967295) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "4294967295"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueInt64) { + TValue value = TValueBuilder() + .Int64(-9223372036854775807 - 1) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "-9223372036854775808"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUint64) { + TValue value = TValueBuilder() + .Uint64(18446744073709551615ull) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "18446744073709551615"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueFloat) { + TValue value = TValueBuilder() + .Float(0.1234567890123456789) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "0.12345679"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDouble) { + TValue value = TValueBuilder() + .Double(0.1234567890123456789) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "0.12345678901234568"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDate) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02"); + TValue value = TValueBuilder() + .Date(TInstant::Days(timestamp.Days())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueDatetime) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05Z"); + TValue value = TValueBuilder() + .Datetime(TInstant::Seconds(timestamp.Seconds())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueTimestamp) { + TInstant timestamp = TInstant::ParseIso8601("2000-01-02T03:04:05.678901Z"); + TValue value = TValueBuilder() + .Timestamp(TInstant::MicroSeconds(timestamp.MicroSeconds())) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"("2000-01-02T03:04:05.678901Z")"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueSimpleString) { + TValue value = TValueBuilder() + .String("Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ") + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + jsonString, + "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + namespace { + TString GenerateBinaryString() { + TStringStream str; + for (ui8 i = 0; i < 255; ++i) { + str << static_cast<unsigned char>(i); + } + str << static_cast<unsigned char>(0xff); + return str.Str(); + } + } + + Y_UNIT_TEST(BinaryStringAsciiFollowedByNonAscii) { + TString binaryString = TStringBuilder() << "abc" << static_cast<unsigned char>(0xff) + << static_cast<unsigned char>(0xfe); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + jsonString, + R"("abc\u00FF\u00FE")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(BinaryStringUnicode) { + TString binaryString = GenerateBinaryString(); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + TStringBuilder from0To255Utf8; + from0To255Utf8 << "\""; + from0To255Utf8 << + R"(\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F)" + R"(\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F)" + " !\\\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + R"(\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F)" + R"(\u0090\u0091\u0092\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F)" + R"(\u00A0\u00A1\u00A2\u00A3\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF)" + R"(\u00B0\u00B1\u00B2\u00B3\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF)" + R"(\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF)" + R"(\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE\u00DF)" + R"(\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF)" + R"(\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF)"; + from0To255Utf8 << "\""; + UNIT_ASSERT_NO_DIFF( + jsonString, + from0To255Utf8 + ); + + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(BinaryStringBase64) { + TString binaryString = GenerateBinaryString(); + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + jsonString, + R"("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHS)" + R"(ElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJG)" + R"(Sk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna2)" + R"(9zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(EmptyBinaryStringUnicode) { + TString binaryString; + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(EmptyBinaryStringBase64) { + TString binaryString; + TValue value = TValueBuilder() + .String(binaryString) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF(jsonString, "\"\""); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Base64); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + namespace { + TString InvalidJsonToBinaryStringBase(const TString& jsonString) { + TString errorMessage; + try { + TType stringType = TTypeBuilder().Primitive(EPrimitiveType::String).Build(); + TValue resultValue = JsonToYdbValue(jsonString, stringType, EBinaryStringEncoding::Unicode); + UNIT_FAIL("Exception should have been thrown, but it hasn't"); + } + catch (TContractViolation& e) { + errorMessage = e.what(); + } + catch (std::exception& e) { + UNIT_FAIL(TStringBuilder() << "Uncaught exception: " << e.what()); + } + return errorMessage; + } + } + + Y_UNIT_TEST(InvalidJsonToBinaryString1) { + TString jsonString = R"(some string")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Invalid value" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString2) { + TString jsonString = R"("some string)"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Missing a closing quotation mark in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString3) { + TString jsonString = "\"some string \\\""; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Missing a closing quotation mark in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString4) { + TString jsonString = R"("some \ string")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Invalid escape character in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString5) { + TString jsonString = R"("some string \u001")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Incorrect hex digit after \\u escape in string" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString6) { + TString jsonString = R"("some string \u0140")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Unicode symbols with codes greater than 255 are not supported" + ); + } + + Y_UNIT_TEST(InvalidJsonToBinaryString7) { + TString jsonString = R"("some string \u00AG")"; + TString errorMessage = InvalidJsonToBinaryStringBase(jsonString); + UNIT_ASSERT_STRING_CONTAINS( + errorMessage, + "Incorrect hex digit after \\u escape in string" + ); + } + + Y_UNIT_TEST(PrimitiveValueUtf8String1) { + TString utf8Str = "Escape characters: \" \\ \f \b \t \r\nNon-escaped characters: / ' < > & []() "; + TValue value = TValueBuilder() + .Utf8(utf8Str) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + + UNIT_ASSERT_NO_DIFF( + jsonString, + "\"Escape characters: \\\" " R"(\\ \f \b \t \r\nNon-escaped characters: / ' < > & []() ")" + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(PrimitiveValueUtf8String2) { + TString utf8Str = "\xD0\xB0\xD0\xB1\xD0\xB2\xD0\xB3\xD0\xB4\xD0\xB5\xD1\x91\xD0\xB6\xD0\xB7\xD0\xB8\xD0\xBA\xD0\xBB" + "\xD0\xBC\xD0\xBD\xD0\xBE\xD0\xBF\xD1\x80\xD1\x81\xD1\x82\xD1\x83\xD1\x84\xD1\x85\xD1\x86\xD1\x87\xD1\x88" + "\xD1\x89\xD1\x8A\xD1\x8B\xD1\x8C\xD1\x8D\xD1\x8E\xD1\x8F\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95" + "\xD0\x81\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2" + "\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF"; + TValue value = TValueBuilder() + .Utf8(utf8Str) + .Build(); + TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + + UNIT_ASSERT_NO_DIFF( + jsonString, + TStringBuilder() << '"' << utf8Str << '"' + ); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueEmptyList) { + TValue value = TValueBuilder() + .EmptyList(TTypeBuilder().Primitive(EPrimitiveType::Uint32).Build()) + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueIntList) { + TValue value = TValueBuilder() + .BeginList() + .AddListItem() + .Int32(1) + .AddListItem() + .Int32(10) + .AddListItem() + .Int32(100) + .EndList() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[1,10,100]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueStruct) { + TValue value = TValueBuilder() + .BeginStruct() + .AddMember("Id") + .Uint32(1) + .AddMember("Name") + .String("Anna") + .AddMember("Value") + .Int32(-100) + .AddMember("Description") + .EmptyOptional(EPrimitiveType::Utf8) + .EndStruct() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"({"Id":1,"Name":"Anna","Value":-100,"Description":null})"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueTuple) { + TValue value = TValueBuilder() + .BeginTuple() + .AddElement() + .BeginOptional() + .BeginOptional() + .Int32(10) + .EndOptional() + .EndOptional() + .AddElement() + .BeginOptional() + .BeginOptional() + .BeginOptional() + .Int64(-1) + .EndOptional() + .EndOptional() + .EndOptional() + .AddElement() + .BeginOptional() + .EmptyOptional(TTypeBuilder().Primitive(EPrimitiveType::String).Build()) + .EndOptional() + .AddElement() + .BeginOptional() + .BeginOptional() + .EmptyOptional(EPrimitiveType::Utf8) + .EndOptional() + .EndOptional() + .EndTuple() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, "[10,-1,null,null]"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } + + Y_UNIT_TEST(CompositeValueDict) { + TValue value = TValueBuilder() + .BeginDict() + .AddDictItem() + .DictKey() + .Int64(1) + .DictPayload() + .String("Value1") + .AddDictItem() + .DictKey() + .Int64(2) + .DictPayload() + .String("Value2") + .EndDict() + .Build(); + const TString jsonString = FormatValueJson(value, EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF(jsonString, R"([[1,"Value1"],[2,"Value2"]])"); + TValue resultValue = JsonToYdbValue(jsonString, value.GetType(), EBinaryStringEncoding::Unicode); + UNIT_ASSERT_NO_DIFF( + TProtoAccessor::GetProto(value).DebugString(), + TProtoAccessor::GetProto(resultValue).DebugString() + ); + } +} + +} // namespace NYdb diff --git a/ydb/public/lib/ya.make b/ydb/public/lib/ya.make index 438394e5f8..a4f529b44b 100644 --- a/ydb/public/lib/ya.make +++ b/ydb/public/lib/ya.make @@ -5,7 +5,7 @@ RECURSE( deprecated experimental idx_test - json_value + json_value jwt operation_id scheme_types diff --git a/ydb/public/lib/ydb_cli/commands/ya.make b/ydb/public/lib/ydb_cli/commands/ya.make index 8986628e6f..d453592849 100644 --- a/ydb/public/lib/ydb_cli/commands/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ya.make @@ -1,29 +1,29 @@ -LIBRARY(commands) - -OWNER(g:kikimr) - -SRCS( +LIBRARY(commands) + +OWNER(g:kikimr) + +SRCS( stock_workload.cpp - ydb_command.cpp - ydb_profile.cpp - ydb_root_common.cpp - ydb_service_discovery.cpp - ydb_service_export.cpp - ydb_service_import.cpp - ydb_service_operation.cpp - ydb_service_scheme.cpp - ydb_service_scripting.cpp - ydb_service_stream.cpp - ydb_service_table.cpp - ydb_tools.cpp + ydb_command.cpp + ydb_profile.cpp + ydb_root_common.cpp + ydb_service_discovery.cpp + ydb_service_export.cpp + ydb_service_import.cpp + ydb_service_operation.cpp + ydb_service_scheme.cpp + ydb_service_scripting.cpp + ydb_service_stream.cpp + ydb_service_table.cpp + ydb_tools.cpp ydb_workload.cpp - ydb_yql.cpp -) - -PEERDIR( + ydb_yql.cpp +) + +PEERDIR( library/cpp/histogram/hdr - library/cpp/protobuf/json - library/cpp/regex/pcre + library/cpp/protobuf/json + library/cpp/regex/pcre library/cpp/threading/local_executor ydb/library/backup ydb/library/workload @@ -40,6 +40,6 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_proto ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_table -) - -END() +) + +END() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_command.cpp b/ydb/public/lib/ydb_cli/commands/ydb_command.cpp index 3d4065e549..e1b8fab3e6 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_command.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_command.cpp @@ -1,63 +1,63 @@ -#include "ydb_command.h" -#include "ydb_common.h" - -namespace NYdb { -namespace NConsoleClient { - -TYdbCommand::TYdbCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) - :TClientCommand(name, aliases, description) -{} - -TDriver TYdbCommand::CreateDriver(TConfig& config) { - auto driverConfig = TDriverConfig() - .SetEndpoint(config.Address) +#include "ydb_command.h" +#include "ydb_common.h" + +namespace NYdb { +namespace NConsoleClient { + +TYdbCommand::TYdbCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) + :TClientCommand(name, aliases, description) +{} + +TDriver TYdbCommand::CreateDriver(TConfig& config) { + auto driverConfig = TDriverConfig() + .SetEndpoint(config.Address) .SetDatabase(config.Database) - .SetCredentialsProviderFactory(config.CredentialsGetter(config)); - if (config.EnableSsl) { - driverConfig.UseSecureConnection(config.CaCerts); - } - return TDriver(driverConfig); -} - -TYdbSimpleCommand::TYdbSimpleCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) - :TYdbCommand(name, aliases, description) -{} - -void TYdbSimpleCommand::Config(TConfig& config) { - TClientCommand::Config(config); - - NLastGetopt::TOpts& opts = *config.Opts; - opts.AddLongOption("timeout", "Client timeout. There is no point waiting for the result after this long.") - .RequiredArgument("ms").StoreResult(&ClientTimeout); -} - -TYdbOperationCommand::TYdbOperationCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) - :TYdbCommand(name, aliases, description) -{} - -void TYdbOperationCommand::Config(TConfig& config) { - TYdbCommand::Config(config); - - NLastGetopt::TOpts& opts = *config.Opts; - opts.AddLongOption("timeout", "Operation timeout. Operation should be executed on server within this timeout. " - "There could also be a delay up to 200ms to receive timeout error from server.") - .RequiredArgument("ms").StoreResult(&OperationTimeout); -} - -NScripting::TExplainYqlResult TYdbOperationCommand::ExplainQuery(TClientCommand::TConfig& config, const TString& queryText, - NScripting::ExplainYqlRequestMode mode) { - NScripting::TScriptingClient client(CreateDriver(config)); - - NScripting::TExplainYqlRequestSettings explainSettings; - explainSettings.Mode(mode); - - auto result = client.ExplainYqlScript( - queryText, - explainSettings - ).GetValueSync(); - ThrowOnError(result); - return result; -} - -} -} + .SetCredentialsProviderFactory(config.CredentialsGetter(config)); + if (config.EnableSsl) { + driverConfig.UseSecureConnection(config.CaCerts); + } + return TDriver(driverConfig); +} + +TYdbSimpleCommand::TYdbSimpleCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) + :TYdbCommand(name, aliases, description) +{} + +void TYdbSimpleCommand::Config(TConfig& config) { + TClientCommand::Config(config); + + NLastGetopt::TOpts& opts = *config.Opts; + opts.AddLongOption("timeout", "Client timeout. There is no point waiting for the result after this long.") + .RequiredArgument("ms").StoreResult(&ClientTimeout); +} + +TYdbOperationCommand::TYdbOperationCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) + :TYdbCommand(name, aliases, description) +{} + +void TYdbOperationCommand::Config(TConfig& config) { + TYdbCommand::Config(config); + + NLastGetopt::TOpts& opts = *config.Opts; + opts.AddLongOption("timeout", "Operation timeout. Operation should be executed on server within this timeout. " + "There could also be a delay up to 200ms to receive timeout error from server.") + .RequiredArgument("ms").StoreResult(&OperationTimeout); +} + +NScripting::TExplainYqlResult TYdbOperationCommand::ExplainQuery(TClientCommand::TConfig& config, const TString& queryText, + NScripting::ExplainYqlRequestMode mode) { + NScripting::TScriptingClient client(CreateDriver(config)); + + NScripting::TExplainYqlRequestSettings explainSettings; + explainSettings.Mode(mode); + + auto result = client.ExplainYqlScript( + queryText, + explainSettings + ).GetValueSync(); + ThrowOnError(result); + return result; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_command.h b/ydb/public/lib/ydb_cli/commands/ydb_command.h index 0c223999a2..95b967ec44 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_command.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_command.h @@ -1,69 +1,69 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/common/command.h> - + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> - -namespace NYdb { -namespace NConsoleClient { - -class TYdbCommand : public TClientCommand { -public: - TYdbCommand( - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ); - TDriver CreateDriver(TConfig& config); -}; - -class TYdbSimpleCommand : public TYdbCommand { -public: - TYdbSimpleCommand( - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ); - virtual void Config(TConfig& config) override; - -protected: - template<typename TSettingsType> - TSettingsType&& FillSettings(TSettingsType&& settings) { - if (ClientTimeout) { - settings.ClientTimeout(TDuration::MilliSeconds(FromString<ui64>(ClientTimeout))); - } - return std::forward<TSettingsType>(settings); - } - - TString ClientTimeout; -}; - -class TYdbOperationCommand : public TYdbCommand { -public: - TYdbOperationCommand( - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ); - virtual void Config(TConfig& config) override; - -protected: - template<typename TSettingsType> - TSettingsType&& FillSettings(TSettingsType&& settings) { - if (OperationTimeout) { - ui64 operationTimeout = FromString<ui64>(OperationTimeout); - settings.OperationTimeout(TDuration::MilliSeconds(operationTimeout)); - settings.ClientTimeout(TDuration::MilliSeconds(operationTimeout + 200)); - } - return std::forward<TSettingsType>(settings); - } - - NScripting::TExplainYqlResult ExplainQuery(TClientCommand::TConfig& config, const TString& queryText, - NScripting::ExplainYqlRequestMode mode); - - TString OperationTimeout; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +class TYdbCommand : public TClientCommand { +public: + TYdbCommand( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); + TDriver CreateDriver(TConfig& config); +}; + +class TYdbSimpleCommand : public TYdbCommand { +public: + TYdbSimpleCommand( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); + virtual void Config(TConfig& config) override; + +protected: + template<typename TSettingsType> + TSettingsType&& FillSettings(TSettingsType&& settings) { + if (ClientTimeout) { + settings.ClientTimeout(TDuration::MilliSeconds(FromString<ui64>(ClientTimeout))); + } + return std::forward<TSettingsType>(settings); + } + + TString ClientTimeout; +}; + +class TYdbOperationCommand : public TYdbCommand { +public: + TYdbOperationCommand( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); + virtual void Config(TConfig& config) override; + +protected: + template<typename TSettingsType> + TSettingsType&& FillSettings(TSettingsType&& settings) { + if (OperationTimeout) { + ui64 operationTimeout = FromString<ui64>(OperationTimeout); + settings.OperationTimeout(TDuration::MilliSeconds(operationTimeout)); + settings.ClientTimeout(TDuration::MilliSeconds(operationTimeout + 200)); + } + return std::forward<TSettingsType>(settings); + } + + NScripting::TExplainYqlResult ExplainQuery(TClientCommand::TConfig& config, const TString& queryText, + NScripting::ExplainYqlRequestMode mode); + + TString OperationTimeout; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_common.h b/ydb/public/lib/ydb_cli/commands/ydb_common.h index a504bf86b7..7561c6476e 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_common.h @@ -1,47 +1,47 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> - -namespace NYdb { -namespace NConsoleClient { - -class TYdbErrorException : public yexception { -public: - TYdbErrorException(NYdb::TStatus status) - : Status(std::move(status)) - { } - - friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { - out << "Status: " << e.Status.GetStatus() << Endl; - if (e.Status.GetIssues()) { - out << "Issues: " << Endl; - e.Status.GetIssues().PrintTo(out); - } - return out; - } - -private: - NYdb::TStatus Status; -}; - -inline void ThrowOnError(NYdb::TStatus status) { - if (status.IsSuccess()) { - if (status.GetIssues()) { - Cerr << "Status: " << status.GetStatus() << Endl; - Cerr << "Issues: " << Endl; - status.GetIssues().PrintTo(Cerr); - } - } else { - throw TYdbErrorException(std::move(status)); - } -} - + +namespace NYdb { +namespace NConsoleClient { + +class TYdbErrorException : public yexception { +public: + TYdbErrorException(NYdb::TStatus status) + : Status(std::move(status)) + { } + + friend IOutputStream& operator<<(IOutputStream& out, const TYdbErrorException& e) { + out << "Status: " << e.Status.GetStatus() << Endl; + if (e.Status.GetIssues()) { + out << "Issues: " << Endl; + e.Status.GetIssues().PrintTo(out); + } + return out; + } + +private: + NYdb::TStatus Status; +}; + +inline void ThrowOnError(NYdb::TStatus status) { + if (status.IsSuccess()) { + if (status.GetIssues()) { + Cerr << "Status: " << status.GetStatus() << Endl; + Cerr << "Issues: " << Endl; + status.GetIssues().PrintTo(Cerr); + } + } else { + throw TYdbErrorException(std::move(status)); + } +} + inline void ThrowOnError(const NYdb::TOperation& operation) { if (!operation.Ready()) return; ThrowOnError(operation.Status()); -} +} -} +} } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp index d0c5a6a17e..f9f2a16e85 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_profile.cpp @@ -1,668 +1,668 @@ -#include "ydb_profile.h" - -#include <util/folder/dirut.h> - -#if defined(_win32_) -#include <util/system/env.h> -#endif - -namespace NYdb::NConsoleClient { - -namespace { -#if defined(_darwin_) - const TString HomeDir = GetHomeDir(); -#elif defined(_win32_) - const TString HomeDir = GetEnv("USERPROFILE"); -#else - const TString HomeDir = GetHomeDir(); -#endif -} - -std::shared_ptr<IProfileManager> CreateYdbProfileManager(const TString& ydbDir) { - return CreateProfileManager(TStringBuilder() << HomeDir << '/' << ydbDir << "/config/config.yaml"); -} - -TCommandConfig::TCommandConfig() - : TClientCommandTree("config", {}, "Manage YDB CLI configuration") -{ +#include "ydb_profile.h" + +#include <util/folder/dirut.h> + +#if defined(_win32_) +#include <util/system/env.h> +#endif + +namespace NYdb::NConsoleClient { + +namespace { +#if defined(_darwin_) + const TString HomeDir = GetHomeDir(); +#elif defined(_win32_) + const TString HomeDir = GetEnv("USERPROFILE"); +#else + const TString HomeDir = GetHomeDir(); +#endif +} + +std::shared_ptr<IProfileManager> CreateYdbProfileManager(const TString& ydbDir) { + return CreateProfileManager(TStringBuilder() << HomeDir << '/' << ydbDir << "/config/config.yaml"); +} + +TCommandConfig::TCommandConfig() + : TClientCommandTree("config", {}, "Manage YDB CLI configuration") +{ AddCommand(std::make_unique<TCommandProfile>()); -} - -TCommandProfile::TCommandProfile() - : TClientCommandTree("profile", {}, "Manage configuration profiles") -{ +} + +TCommandProfile::TCommandProfile() + : TClientCommandTree("profile", {}, "Manage configuration profiles") +{ AddCommand(std::make_unique<TCommandCreateProfile>()); AddCommand(std::make_unique<TCommandDeleteProfile>()); AddCommand(std::make_unique<TCommandActivateProfile>()); AddCommand(std::make_unique<TCommandListProfiles>()); AddCommand(std::make_unique<TCommandGetProfile>()); -} - -namespace { - size_t MakeNumericChoice(const size_t optCount) { - Cout << "Please enter your numeric choice: "; - size_t numericChoice = 0; - for (;;) { - try { - Cin >> numericChoice; - } - catch (yexception&) { - } - if (numericChoice > 0 && numericChoice <= optCount) { - break; - } - Cerr << "Please enter a value between 1 and " << optCount << ": "; - } - return numericChoice; - } - - class TNumericOptionsPicker { - public: - using TPickableAction = std::function<void()>; - TNumericOptionsPicker() {}; - - void AddOption(const TString& description, TPickableAction&& action) { - Cout << " [" << ++OptionsCount << "] " << description << Endl; - Options.emplace(OptionsCount, std::move(action)); - } - - void PickOptionAndDoAction() { - size_t numericChoice = MakeNumericChoice(OptionsCount); - auto optionIt = Options.find(numericChoice); - if (optionIt != Options.end()) { - // Do action - optionIt->second(); - } else { - Cerr << "Can't find action with index " << numericChoice << Endl; - } - } - - private: - size_t OptionsCount = 0; - THashMap<size_t, TPickableAction> Options; - - }; - - TString BlurSecret(const TString& in) { - TString out(in); - size_t clearSymbolsCount = Min(size_t(10), out.length() / 4); - for (size_t i = clearSymbolsCount; i < out.length() - clearSymbolsCount; ++i) { - out[i] = '*'; - } - return out; - } - - TString ReplaceWithAsterisks(const TString& in) { - return TString(in.length(), '*'); - } - - void SetupProfileName(TString& profileName, std::shared_ptr<IProfileManager> profileManager) { - if (profileName) { - return; - } - const auto profileNames = profileManager->ListProfiles(); - TString activeProfileName = profileManager->GetActiveProfileName(); - if (profileNames.size()) { - Cout << "Please choose profile to configure:" << Endl; - TNumericOptionsPicker picker; - picker.AddOption( - "Create a new profile", - [&profileName]() { - Cout << "Please enter name for a new profile: "; - Cin >> profileName; - } - ); - for (size_t i = 0; i < profileNames.size(); ++i) { - TStringBuilder description; - TString currentProfile = profileNames[i]; - description << currentProfile; - if (currentProfile == activeProfileName) { - description << " (active)"; - } - picker.AddOption( - description, - [&profileName, currentProfile]() { - profileName = currentProfile; - } - ); - } - picker.PickOptionAndDoAction(); - } else { - Cout << "You have no existing profiles yet." << Endl; - Cout << "Please enter name for a new profile: "; - Cin >> profileName; - } - } - - void SetupProfileSetting(const TString& name, bool existingProfile, const TString& profileName, - std::shared_ptr<IProfile> profile) { - Cout << Endl << "Pick desired action to configure " << name << " in profile \"" << profileName << "\":" << Endl; - - TNumericOptionsPicker picker; - picker.AddOption( - TStringBuilder() << "Set a new " << name << " value", - [&name, &profileName, &profile]() { - Cout << "Please enter new " << name << " value: "; - TString newValue; - Cin >> newValue; - if (newValue) { - Cout << "Setting " << name << " value \"" << newValue << "\" for profile \"" << profileName - << "\"" << Endl; - profile->SetValue(name, newValue); - } - } - ); - picker.AddOption( - TStringBuilder() << "Don't save " << name << " for profile \"" << profileName << "\"", - [&name, &profile]() { - profile->RemoveValue(name); - } - ); - if (existingProfile && profile->Has(name)) { - picker.AddOption( - TStringBuilder() << "Use current " << name << " value \"" << profile->GetValue(name).as<TString>() << "\"", - []() {} - ); - } - picker.PickOptionAndDoAction(); - } - - void SetAuthMethod(const TString& id, const TString& fullName, std::shared_ptr<IProfile> profile, - const TString& profileName) { - Cout << "Please enter " << fullName << " (" << id << "): "; - TString newValue; - Cin >> newValue; - if (newValue) { - Cout << "Setting " << fullName << " for profile \"" << profileName << "\"" << Endl; - profile->RemoveValue("authentication"); - YAML::Node authValue; - authValue["method"] = id; - authValue["data"] = newValue; - profile->SetValue("authentication", authValue); - } - } - - void SetStaticCredentials(std::shared_ptr<IProfile> profile, const TString& profileName) { - Cout << "Please enter user name: "; - TString userName; - Cin >> userName; - Cout << "Please enter password: "; - TString userPassword = InputPassword(); - if (userName && userPassword) { - Cout << "Setting user & password for profile \"" << profileName << "\"" << Endl; - profile->RemoveValue("authentication"); - YAML::Node authValue; - authValue["method"] = "static-credentials"; - auto authData = authValue["data"]; - authData["user"] = userName; - authData["password"] = userPassword; - profile->SetValue("authentication", authValue); - } - } - - void SetupProfileAuthentication(bool existingProfile, const TString& profileName, std::shared_ptr<IProfile> profile, - TClientCommand::TConfig& config) { - Cout << Endl << "Pick desired action to configure authentication method:" << Endl; - TNumericOptionsPicker picker; - if (config.UseStaticCredentials) { - picker.AddOption( - "Use static credentials (user & password)", - [&profile, &profileName]() { - SetStaticCredentials(profile, profileName); - } - ); - } - if (config.UseIamAuth) { - picker.AddOption( - "Use IAM token (iam-token) cloud.yandex.ru/docs/iam/concepts/authorization/iam-token", - [&profile, &profileName]() { - SetAuthMethod("iam-token", "IAM token", profile, profileName); - } - ); - picker.AddOption( - "Use OAuth token of a Yandex Passport user (yc-token). Doesn't work with federative accounts." - " cloud.yandex.ru/docs/iam/concepts/authorization/oauth-token", - [&profile, &profileName]() { - SetAuthMethod("yc-token", "OAuth token of a Yandex Passport user", profile, profileName); - } - ); - picker.AddOption( - "Use metadata service on a virtual machine (use-metadata-credentials)" - " cloud.yandex.ru/docs/compute/operations/vm-connect/auth-inside-vm", - [&profile, &profileName]() { - Cout << "Setting metadata service usage for profile \"" << profileName << "\"" << Endl; - profile->RemoveValue("authentication"); - YAML::Node authValue; - authValue["method"] = "use-metadata-credentials"; - profile->SetValue("authentication", authValue); - } - ); - picker.AddOption( - "Use service account key file (sa-key-file)" - " cloud.yandex.ru/docs/iam/operations/iam-token/create-for-sa", - [&profile, &profileName]() { - SetAuthMethod("sa-key-file", "Path to service account key file", profile, profileName); - } - ); - } - if (config.UseOAuthToken) { - picker.AddOption( - "Set new OAuth token (ydb-token)", - [&profile, &profileName]() { - SetAuthMethod("ydb-token", "OAuth YDB token", profile, profileName); - } - ); - } - picker.AddOption( - TStringBuilder() << "Don't save authentication data for profile \"" << profileName << "\"", - [&profile]() { - profile->RemoveValue("authentication"); - } - ); - if (existingProfile && profile->Has("authentication")) { - auto& authValue = profile->GetValue("authentication"); - if (authValue["method"]) { - TString method = authValue["method"].as<TString>(); - TStringBuilder description; - description << "Use current settings with method \"" << method << "\""; - if (method == "iam-token" || method == "yc-token" || method == "ydb-token") { - description << " and value \"" << BlurSecret(authValue["data"].as<TString>()) << "\""; - } else if (method == "sa-key-file") { - description << " and value \"" << authValue["data"].as<TString>() << "\""; - } - picker.AddOption( - description, - []() {} - ); - } - } - - picker.PickOptionAndDoAction(); - } - - bool AskYesOrNo() { - TString input; - for (;;) { - Cin >> input; - if (to_lower(input) == "y" || to_lower(input) == "yes") { - return true; - } else if (to_lower(input) == "n" || to_lower(input) == "n") { - return false; - } else { - Cout << "Type \"y\" (yes) or \"n\" (no): "; - } - } - return false; - } - - void ConfigureProfile(const TString& profileName, std::shared_ptr<IProfileManager> profileManager, - TClientCommand::TConfig& config) { - bool existingProfile = profileManager->HasProfile(profileName); - auto profile = profileManager->GetProfile(profileName); - Cout << "Configuring " << (existingProfile ? "existing" : "new") - << " profile \"" << profileName << "\"." << Endl; - - SetupProfileSetting("endpoint", existingProfile, profileName, profile); - SetupProfileSetting("database", existingProfile, profileName, profile); - SetupProfileAuthentication(existingProfile, profileName, profile, config); - - TString activeProfileName = profileManager->GetActiveProfileName(); - if (profileName != activeProfileName) { - Cout << Endl << "Activate profile \"" << profileName << "\" to use by default? (current active profile is "; - TString currentActiveProfile = profileManager->GetActiveProfileName(); - if (currentActiveProfile) { - Cout << "\"" << currentActiveProfile << "\""; - } else { - Cout << "not set"; - } - Cout << ") y/n: "; - if (AskYesOrNo()) { - profileManager->SetActiveProfile(profileName); - Cout << "Profile \"" << profileName << "\" was set as active." << Endl; - } - } - Cout << "Configuration process for profile \"" << profileName << "\" is complete." << Endl; - } - - void PrintProfileContent(std::shared_ptr<IProfile> profile) { - if (profile->Has("endpoint")) { - Cout << " endpoint: " << profile->GetValue("endpoint").as<TString>() << Endl; - } - if (profile->Has("database")) { - Cout << " database: " << profile->GetValue("database").as<TString>() << Endl; - } - if (profile->Has("authentication")) { - auto authValue = profile->GetValue("authentication"); - TString authMethod = authValue["method"].as<TString>(); - Cout << " " << authMethod; - if (authMethod == "ydb-token" ||authMethod == "iam-token" - || authMethod == "yc-token" || authMethod == "sa-key-file") - { - TString authData = authValue["data"].as<TString>(); - if (authMethod == "sa-key-file") { - Cout << ": " << authData; - } else { - Cout << ": " << BlurSecret(authData); - } - } else if (authMethod == "static-credentials") { - auto authData = authValue["data"]; - if (authData) { - if (authData["user"]) { - Cout << Endl << " user: " << authData["user"].as<TString>(); - } - if (authData["password"]) { - Cout << Endl << " password: " << ReplaceWithAsterisks(authData["password"].as<TString>()); - } - } - } - Cout << Endl; - } - } -} - -TCommandInit::TCommandInit() - : TClientCommand("init", {}, "YDB CLI initialization") -{} - -void TCommandInit::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsNum(0); -} - -int TCommandInit::Run(TConfig& config) { - Y_UNUSED(config); - Cout << "Welcome! This command will take you through the configuration process." << Endl; - auto profileManager = CreateYdbProfileManager(config.YdbDir); - TString profileName; - SetupProfileName(profileName, profileManager); - ConfigureProfile(profileName, profileManager, config); - return EXIT_SUCCESS; -} - -TCommandCreateProfile::TCommandCreateProfile() - : TClientCommand("create", {}, "Create new configuration profile or re-configure existing one") -{} - -void TCommandCreateProfile::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsMax(1); - SetFreeArgTitle(0, "<name>", "Profile name"); -} - -void TCommandCreateProfile::Parse(TConfig& config) { - TClientCommand::Parse(config); - if (config.ParseResult->GetFreeArgCount()) { - ProfileName = config.ParseResult->GetFreeArgs()[0]; - } -} - -int TCommandCreateProfile::Run(TConfig& config) { - Y_UNUSED(config); - Cout << "Welcome! This command will take you through configuration profile creation process." << Endl; - TString profileName = ProfileName; - if (!profileName) { - Cout << "Please enter configuration profile name to create or re-configure: "; - Cin >> profileName; - } - auto profileManager = CreateYdbProfileManager(config.YdbDir); - ConfigureProfile(profileName, profileManager, config); - return EXIT_SUCCESS; -} - -TCommandDeleteProfile::TCommandDeleteProfile() - : TClientCommand("delete", {"remove"}, "Delete specified configuration profile") -{} - -void TCommandDeleteProfile::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsMax(1); - SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); -} - -void TCommandDeleteProfile::Parse(TConfig& config) { - TClientCommand::Parse(config); - if (config.ParseResult->GetFreeArgCount()) { - ProfileName = config.ParseResult->GetFreeArgs()[0]; - } -} - -int TCommandDeleteProfile::Run(TConfig& config) { - Y_UNUSED(config); - auto profileManager = CreateYdbProfileManager(config.YdbDir); - const auto profileNames = profileManager->ListProfiles(); - if (ProfileName) { - if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { - Cerr << "No existing profile \"" << ProfileName << "\". " - << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; - return EXIT_FAILURE; - } - } - if (!ProfileName) { - if (profileNames.size()) { - TMaybe<int> returnCode; - Cout << "Please choose profile to remove:" << Endl; - TNumericOptionsPicker picker; - picker.AddOption( - "Don't remove anything, just exit", - [&returnCode]() { - Cout << "Nothing is done." << Endl; - returnCode = EXIT_SUCCESS; - } - ); - for (size_t i = 0; i < profileNames.size(); ++i) { - TString currentProfileName = profileNames[i]; - picker.AddOption( - currentProfileName, - [this, currentProfileName]() { - ProfileName = currentProfileName; - } - ); - } - picker.PickOptionAndDoAction(); - if (returnCode.Defined()) { - return returnCode.GetRef(); - } - } else { - Cerr << "You have no existing profiles yet." << Endl; - return EXIT_FAILURE; - } - } - - Cout << "Profile \"" << ProfileName << "\" will be permanently removed. Continue? (y/n): "; - if (AskYesOrNo()) { - if (profileManager->RemoveProfile(ProfileName)) { - Cout << "Profile \"" << ProfileName << "\" was removed." << Endl; - } else { - Cerr << "Profile \"" << ProfileName << "\" was not removed." << Endl; - return EXIT_FAILURE; - } - } else { - Cout << "Nothing is done." << Endl; - return EXIT_SUCCESS; - } - - return EXIT_SUCCESS; -} - -TCommandActivateProfile::TCommandActivateProfile() - : TClientCommand("activate", {"set"}, "Activate specified configuration profile") -{} - -void TCommandActivateProfile::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsMax(1); - SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); -} - -void TCommandActivateProfile::Parse(TConfig& config) { - TClientCommand::Parse(config); - if (config.ParseResult->GetFreeArgCount()) { - ProfileName = config.ParseResult->GetFreeArgs()[0]; - } -} - -int TCommandActivateProfile::Run(TConfig& config) { - Y_UNUSED(config); - auto profileManager = CreateYdbProfileManager(config.YdbDir); - const auto profileNames = profileManager->ListProfiles(); - if (ProfileName) { - if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { - Cerr << "No existing profile \"" << ProfileName << "\". " - << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; - return EXIT_FAILURE; - } - } - TString currentActiveProfileName = profileManager->GetActiveProfileName(); - if (!ProfileName) { - const auto profileNames = profileManager->ListProfiles(); - if (profileNames.size()) { - TMaybe<int> returnCode; - Cout << "Please choose profile to activate:" << Endl; - TNumericOptionsPicker picker; - picker.AddOption( - "Don't do anything, just exit", - [&returnCode]() { - Cout << "Nothing is done." << Endl; - returnCode = EXIT_SUCCESS; - } - ); - if (currentActiveProfileName) { - picker.AddOption( - TStringBuilder() << "Deactivate current active profile \"" << currentActiveProfileName << "\"", - [&returnCode, &profileManager, ¤tActiveProfileName]() { - profileManager->DeactivateProfile(); - Cout << "Current active profile \"" << currentActiveProfileName << "\" was deactivated." << Endl; - returnCode = EXIT_SUCCESS; - } - ); - } - for (size_t i = 0; i < profileNames.size(); ++i) { - TString currentProfileName = profileNames[i]; - TStringBuilder description; - description << currentProfileName; - if (currentProfileName == currentActiveProfileName) { - description << " (active)"; - } - picker.AddOption( - description, - [this, currentProfileName]() { - ProfileName = currentProfileName; - } - ); - } - picker.PickOptionAndDoAction(); - if (returnCode.Defined()) { - return returnCode.GetRef(); - } - } else { - Cerr << "You have no existing profiles yet. Run \"ydb init\" to create one" << Endl; - return EXIT_FAILURE; - } - } - if (currentActiveProfileName != ProfileName) { - profileManager->SetActiveProfile(ProfileName); - Cout << "Profile \"" << ProfileName << "\" was activated." << Endl; - } else { - Cout << "Profile \"" << ProfileName << "\" is already active." << Endl; - } - return EXIT_SUCCESS; -} - -TCommandListProfiles::TCommandListProfiles() - : TClientCommand("list", {}, "List configuration profiles") -{} - -void TCommandListProfiles::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsNum(0); - - config.Opts->AddLongOption("with-content", "Show content in each profile.") - .StoreTrue(&WithContent); -} - -int TCommandListProfiles::Run(TConfig& config) { - Y_UNUSED(config); - auto profileManager = CreateYdbProfileManager(config.YdbDir); - const auto profileNames = profileManager->ListProfiles(); - TString activeProfileName = profileManager->GetActiveProfileName(); - for (const auto& profileName : profileNames) { - Cout << profileName; - if (activeProfileName == profileName) { - Cout << " (active)"; - } - if (WithContent) { - Cout << ":"; - } - Cout << Endl; - if (WithContent) { - PrintProfileContent(profileManager->GetProfile(profileName)); - } - } - return EXIT_SUCCESS; -} - -TCommandGetProfile::TCommandGetProfile() - : TClientCommand("get", {}, "List values for specified configuration profile") -{} - -void TCommandGetProfile::Config(TConfig& config) { - TClientCommand::Config(config); - - config.SetFreeArgsMax(1); - SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); -} - -void TCommandGetProfile::Parse(TConfig& config) { - TClientCommand::Parse(config); - if (config.ParseResult->GetFreeArgCount()) { - ProfileName = config.ParseResult->GetFreeArgs()[0]; - } -} - -int TCommandGetProfile::Run(TConfig& config) { - Y_UNUSED(config); - auto profileManager = CreateYdbProfileManager(config.YdbDir); - const auto profileNames = profileManager->ListProfiles(); - if (ProfileName) { - if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { - Cerr << "No existing profile \"" << ProfileName << "\". " - << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; - return EXIT_FAILURE; - } - } - if (!ProfileName) { - const auto profileNames = profileManager->ListProfiles(); - if (profileNames.size()) { - Cout << "Please choose profile to list its values:" << Endl; - TNumericOptionsPicker picker; - TString activeProfileName = profileManager->GetActiveProfileName(); - for (size_t i = 0; i < profileNames.size(); ++i) { - TString currentProfileName = profileNames[i]; - TStringBuilder description; - description << currentProfileName; - if (currentProfileName == activeProfileName) { - description << " (active)"; - } - picker.AddOption( - description, - [this, currentProfileName]() { - ProfileName = currentProfileName; - } - ); - } - picker.PickOptionAndDoAction(); - } else { - Cerr << "You have no existing profiles yet. Run \"ydb init\" to create one" << Endl; - return EXIT_FAILURE; - } - } - PrintProfileContent(profileManager->GetProfile(ProfileName)); - return EXIT_SUCCESS; -} - -} // NYdb::NConsoleClient +} + +namespace { + size_t MakeNumericChoice(const size_t optCount) { + Cout << "Please enter your numeric choice: "; + size_t numericChoice = 0; + for (;;) { + try { + Cin >> numericChoice; + } + catch (yexception&) { + } + if (numericChoice > 0 && numericChoice <= optCount) { + break; + } + Cerr << "Please enter a value between 1 and " << optCount << ": "; + } + return numericChoice; + } + + class TNumericOptionsPicker { + public: + using TPickableAction = std::function<void()>; + TNumericOptionsPicker() {}; + + void AddOption(const TString& description, TPickableAction&& action) { + Cout << " [" << ++OptionsCount << "] " << description << Endl; + Options.emplace(OptionsCount, std::move(action)); + } + + void PickOptionAndDoAction() { + size_t numericChoice = MakeNumericChoice(OptionsCount); + auto optionIt = Options.find(numericChoice); + if (optionIt != Options.end()) { + // Do action + optionIt->second(); + } else { + Cerr << "Can't find action with index " << numericChoice << Endl; + } + } + + private: + size_t OptionsCount = 0; + THashMap<size_t, TPickableAction> Options; + + }; + + TString BlurSecret(const TString& in) { + TString out(in); + size_t clearSymbolsCount = Min(size_t(10), out.length() / 4); + for (size_t i = clearSymbolsCount; i < out.length() - clearSymbolsCount; ++i) { + out[i] = '*'; + } + return out; + } + + TString ReplaceWithAsterisks(const TString& in) { + return TString(in.length(), '*'); + } + + void SetupProfileName(TString& profileName, std::shared_ptr<IProfileManager> profileManager) { + if (profileName) { + return; + } + const auto profileNames = profileManager->ListProfiles(); + TString activeProfileName = profileManager->GetActiveProfileName(); + if (profileNames.size()) { + Cout << "Please choose profile to configure:" << Endl; + TNumericOptionsPicker picker; + picker.AddOption( + "Create a new profile", + [&profileName]() { + Cout << "Please enter name for a new profile: "; + Cin >> profileName; + } + ); + for (size_t i = 0; i < profileNames.size(); ++i) { + TStringBuilder description; + TString currentProfile = profileNames[i]; + description << currentProfile; + if (currentProfile == activeProfileName) { + description << " (active)"; + } + picker.AddOption( + description, + [&profileName, currentProfile]() { + profileName = currentProfile; + } + ); + } + picker.PickOptionAndDoAction(); + } else { + Cout << "You have no existing profiles yet." << Endl; + Cout << "Please enter name for a new profile: "; + Cin >> profileName; + } + } + + void SetupProfileSetting(const TString& name, bool existingProfile, const TString& profileName, + std::shared_ptr<IProfile> profile) { + Cout << Endl << "Pick desired action to configure " << name << " in profile \"" << profileName << "\":" << Endl; + + TNumericOptionsPicker picker; + picker.AddOption( + TStringBuilder() << "Set a new " << name << " value", + [&name, &profileName, &profile]() { + Cout << "Please enter new " << name << " value: "; + TString newValue; + Cin >> newValue; + if (newValue) { + Cout << "Setting " << name << " value \"" << newValue << "\" for profile \"" << profileName + << "\"" << Endl; + profile->SetValue(name, newValue); + } + } + ); + picker.AddOption( + TStringBuilder() << "Don't save " << name << " for profile \"" << profileName << "\"", + [&name, &profile]() { + profile->RemoveValue(name); + } + ); + if (existingProfile && profile->Has(name)) { + picker.AddOption( + TStringBuilder() << "Use current " << name << " value \"" << profile->GetValue(name).as<TString>() << "\"", + []() {} + ); + } + picker.PickOptionAndDoAction(); + } + + void SetAuthMethod(const TString& id, const TString& fullName, std::shared_ptr<IProfile> profile, + const TString& profileName) { + Cout << "Please enter " << fullName << " (" << id << "): "; + TString newValue; + Cin >> newValue; + if (newValue) { + Cout << "Setting " << fullName << " for profile \"" << profileName << "\"" << Endl; + profile->RemoveValue("authentication"); + YAML::Node authValue; + authValue["method"] = id; + authValue["data"] = newValue; + profile->SetValue("authentication", authValue); + } + } + + void SetStaticCredentials(std::shared_ptr<IProfile> profile, const TString& profileName) { + Cout << "Please enter user name: "; + TString userName; + Cin >> userName; + Cout << "Please enter password: "; + TString userPassword = InputPassword(); + if (userName && userPassword) { + Cout << "Setting user & password for profile \"" << profileName << "\"" << Endl; + profile->RemoveValue("authentication"); + YAML::Node authValue; + authValue["method"] = "static-credentials"; + auto authData = authValue["data"]; + authData["user"] = userName; + authData["password"] = userPassword; + profile->SetValue("authentication", authValue); + } + } + + void SetupProfileAuthentication(bool existingProfile, const TString& profileName, std::shared_ptr<IProfile> profile, + TClientCommand::TConfig& config) { + Cout << Endl << "Pick desired action to configure authentication method:" << Endl; + TNumericOptionsPicker picker; + if (config.UseStaticCredentials) { + picker.AddOption( + "Use static credentials (user & password)", + [&profile, &profileName]() { + SetStaticCredentials(profile, profileName); + } + ); + } + if (config.UseIamAuth) { + picker.AddOption( + "Use IAM token (iam-token) cloud.yandex.ru/docs/iam/concepts/authorization/iam-token", + [&profile, &profileName]() { + SetAuthMethod("iam-token", "IAM token", profile, profileName); + } + ); + picker.AddOption( + "Use OAuth token of a Yandex Passport user (yc-token). Doesn't work with federative accounts." + " cloud.yandex.ru/docs/iam/concepts/authorization/oauth-token", + [&profile, &profileName]() { + SetAuthMethod("yc-token", "OAuth token of a Yandex Passport user", profile, profileName); + } + ); + picker.AddOption( + "Use metadata service on a virtual machine (use-metadata-credentials)" + " cloud.yandex.ru/docs/compute/operations/vm-connect/auth-inside-vm", + [&profile, &profileName]() { + Cout << "Setting metadata service usage for profile \"" << profileName << "\"" << Endl; + profile->RemoveValue("authentication"); + YAML::Node authValue; + authValue["method"] = "use-metadata-credentials"; + profile->SetValue("authentication", authValue); + } + ); + picker.AddOption( + "Use service account key file (sa-key-file)" + " cloud.yandex.ru/docs/iam/operations/iam-token/create-for-sa", + [&profile, &profileName]() { + SetAuthMethod("sa-key-file", "Path to service account key file", profile, profileName); + } + ); + } + if (config.UseOAuthToken) { + picker.AddOption( + "Set new OAuth token (ydb-token)", + [&profile, &profileName]() { + SetAuthMethod("ydb-token", "OAuth YDB token", profile, profileName); + } + ); + } + picker.AddOption( + TStringBuilder() << "Don't save authentication data for profile \"" << profileName << "\"", + [&profile]() { + profile->RemoveValue("authentication"); + } + ); + if (existingProfile && profile->Has("authentication")) { + auto& authValue = profile->GetValue("authentication"); + if (authValue["method"]) { + TString method = authValue["method"].as<TString>(); + TStringBuilder description; + description << "Use current settings with method \"" << method << "\""; + if (method == "iam-token" || method == "yc-token" || method == "ydb-token") { + description << " and value \"" << BlurSecret(authValue["data"].as<TString>()) << "\""; + } else if (method == "sa-key-file") { + description << " and value \"" << authValue["data"].as<TString>() << "\""; + } + picker.AddOption( + description, + []() {} + ); + } + } + + picker.PickOptionAndDoAction(); + } + + bool AskYesOrNo() { + TString input; + for (;;) { + Cin >> input; + if (to_lower(input) == "y" || to_lower(input) == "yes") { + return true; + } else if (to_lower(input) == "n" || to_lower(input) == "n") { + return false; + } else { + Cout << "Type \"y\" (yes) or \"n\" (no): "; + } + } + return false; + } + + void ConfigureProfile(const TString& profileName, std::shared_ptr<IProfileManager> profileManager, + TClientCommand::TConfig& config) { + bool existingProfile = profileManager->HasProfile(profileName); + auto profile = profileManager->GetProfile(profileName); + Cout << "Configuring " << (existingProfile ? "existing" : "new") + << " profile \"" << profileName << "\"." << Endl; + + SetupProfileSetting("endpoint", existingProfile, profileName, profile); + SetupProfileSetting("database", existingProfile, profileName, profile); + SetupProfileAuthentication(existingProfile, profileName, profile, config); + + TString activeProfileName = profileManager->GetActiveProfileName(); + if (profileName != activeProfileName) { + Cout << Endl << "Activate profile \"" << profileName << "\" to use by default? (current active profile is "; + TString currentActiveProfile = profileManager->GetActiveProfileName(); + if (currentActiveProfile) { + Cout << "\"" << currentActiveProfile << "\""; + } else { + Cout << "not set"; + } + Cout << ") y/n: "; + if (AskYesOrNo()) { + profileManager->SetActiveProfile(profileName); + Cout << "Profile \"" << profileName << "\" was set as active." << Endl; + } + } + Cout << "Configuration process for profile \"" << profileName << "\" is complete." << Endl; + } + + void PrintProfileContent(std::shared_ptr<IProfile> profile) { + if (profile->Has("endpoint")) { + Cout << " endpoint: " << profile->GetValue("endpoint").as<TString>() << Endl; + } + if (profile->Has("database")) { + Cout << " database: " << profile->GetValue("database").as<TString>() << Endl; + } + if (profile->Has("authentication")) { + auto authValue = profile->GetValue("authentication"); + TString authMethod = authValue["method"].as<TString>(); + Cout << " " << authMethod; + if (authMethod == "ydb-token" ||authMethod == "iam-token" + || authMethod == "yc-token" || authMethod == "sa-key-file") + { + TString authData = authValue["data"].as<TString>(); + if (authMethod == "sa-key-file") { + Cout << ": " << authData; + } else { + Cout << ": " << BlurSecret(authData); + } + } else if (authMethod == "static-credentials") { + auto authData = authValue["data"]; + if (authData) { + if (authData["user"]) { + Cout << Endl << " user: " << authData["user"].as<TString>(); + } + if (authData["password"]) { + Cout << Endl << " password: " << ReplaceWithAsterisks(authData["password"].as<TString>()); + } + } + } + Cout << Endl; + } + } +} + +TCommandInit::TCommandInit() + : TClientCommand("init", {}, "YDB CLI initialization") +{} + +void TCommandInit::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsNum(0); +} + +int TCommandInit::Run(TConfig& config) { + Y_UNUSED(config); + Cout << "Welcome! This command will take you through the configuration process." << Endl; + auto profileManager = CreateYdbProfileManager(config.YdbDir); + TString profileName; + SetupProfileName(profileName, profileManager); + ConfigureProfile(profileName, profileManager, config); + return EXIT_SUCCESS; +} + +TCommandCreateProfile::TCommandCreateProfile() + : TClientCommand("create", {}, "Create new configuration profile or re-configure existing one") +{} + +void TCommandCreateProfile::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsMax(1); + SetFreeArgTitle(0, "<name>", "Profile name"); +} + +void TCommandCreateProfile::Parse(TConfig& config) { + TClientCommand::Parse(config); + if (config.ParseResult->GetFreeArgCount()) { + ProfileName = config.ParseResult->GetFreeArgs()[0]; + } +} + +int TCommandCreateProfile::Run(TConfig& config) { + Y_UNUSED(config); + Cout << "Welcome! This command will take you through configuration profile creation process." << Endl; + TString profileName = ProfileName; + if (!profileName) { + Cout << "Please enter configuration profile name to create or re-configure: "; + Cin >> profileName; + } + auto profileManager = CreateYdbProfileManager(config.YdbDir); + ConfigureProfile(profileName, profileManager, config); + return EXIT_SUCCESS; +} + +TCommandDeleteProfile::TCommandDeleteProfile() + : TClientCommand("delete", {"remove"}, "Delete specified configuration profile") +{} + +void TCommandDeleteProfile::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsMax(1); + SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); +} + +void TCommandDeleteProfile::Parse(TConfig& config) { + TClientCommand::Parse(config); + if (config.ParseResult->GetFreeArgCount()) { + ProfileName = config.ParseResult->GetFreeArgs()[0]; + } +} + +int TCommandDeleteProfile::Run(TConfig& config) { + Y_UNUSED(config); + auto profileManager = CreateYdbProfileManager(config.YdbDir); + const auto profileNames = profileManager->ListProfiles(); + if (ProfileName) { + if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { + Cerr << "No existing profile \"" << ProfileName << "\". " + << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; + return EXIT_FAILURE; + } + } + if (!ProfileName) { + if (profileNames.size()) { + TMaybe<int> returnCode; + Cout << "Please choose profile to remove:" << Endl; + TNumericOptionsPicker picker; + picker.AddOption( + "Don't remove anything, just exit", + [&returnCode]() { + Cout << "Nothing is done." << Endl; + returnCode = EXIT_SUCCESS; + } + ); + for (size_t i = 0; i < profileNames.size(); ++i) { + TString currentProfileName = profileNames[i]; + picker.AddOption( + currentProfileName, + [this, currentProfileName]() { + ProfileName = currentProfileName; + } + ); + } + picker.PickOptionAndDoAction(); + if (returnCode.Defined()) { + return returnCode.GetRef(); + } + } else { + Cerr << "You have no existing profiles yet." << Endl; + return EXIT_FAILURE; + } + } + + Cout << "Profile \"" << ProfileName << "\" will be permanently removed. Continue? (y/n): "; + if (AskYesOrNo()) { + if (profileManager->RemoveProfile(ProfileName)) { + Cout << "Profile \"" << ProfileName << "\" was removed." << Endl; + } else { + Cerr << "Profile \"" << ProfileName << "\" was not removed." << Endl; + return EXIT_FAILURE; + } + } else { + Cout << "Nothing is done." << Endl; + return EXIT_SUCCESS; + } + + return EXIT_SUCCESS; +} + +TCommandActivateProfile::TCommandActivateProfile() + : TClientCommand("activate", {"set"}, "Activate specified configuration profile") +{} + +void TCommandActivateProfile::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsMax(1); + SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); +} + +void TCommandActivateProfile::Parse(TConfig& config) { + TClientCommand::Parse(config); + if (config.ParseResult->GetFreeArgCount()) { + ProfileName = config.ParseResult->GetFreeArgs()[0]; + } +} + +int TCommandActivateProfile::Run(TConfig& config) { + Y_UNUSED(config); + auto profileManager = CreateYdbProfileManager(config.YdbDir); + const auto profileNames = profileManager->ListProfiles(); + if (ProfileName) { + if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { + Cerr << "No existing profile \"" << ProfileName << "\". " + << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; + return EXIT_FAILURE; + } + } + TString currentActiveProfileName = profileManager->GetActiveProfileName(); + if (!ProfileName) { + const auto profileNames = profileManager->ListProfiles(); + if (profileNames.size()) { + TMaybe<int> returnCode; + Cout << "Please choose profile to activate:" << Endl; + TNumericOptionsPicker picker; + picker.AddOption( + "Don't do anything, just exit", + [&returnCode]() { + Cout << "Nothing is done." << Endl; + returnCode = EXIT_SUCCESS; + } + ); + if (currentActiveProfileName) { + picker.AddOption( + TStringBuilder() << "Deactivate current active profile \"" << currentActiveProfileName << "\"", + [&returnCode, &profileManager, ¤tActiveProfileName]() { + profileManager->DeactivateProfile(); + Cout << "Current active profile \"" << currentActiveProfileName << "\" was deactivated." << Endl; + returnCode = EXIT_SUCCESS; + } + ); + } + for (size_t i = 0; i < profileNames.size(); ++i) { + TString currentProfileName = profileNames[i]; + TStringBuilder description; + description << currentProfileName; + if (currentProfileName == currentActiveProfileName) { + description << " (active)"; + } + picker.AddOption( + description, + [this, currentProfileName]() { + ProfileName = currentProfileName; + } + ); + } + picker.PickOptionAndDoAction(); + if (returnCode.Defined()) { + return returnCode.GetRef(); + } + } else { + Cerr << "You have no existing profiles yet. Run \"ydb init\" to create one" << Endl; + return EXIT_FAILURE; + } + } + if (currentActiveProfileName != ProfileName) { + profileManager->SetActiveProfile(ProfileName); + Cout << "Profile \"" << ProfileName << "\" was activated." << Endl; + } else { + Cout << "Profile \"" << ProfileName << "\" is already active." << Endl; + } + return EXIT_SUCCESS; +} + +TCommandListProfiles::TCommandListProfiles() + : TClientCommand("list", {}, "List configuration profiles") +{} + +void TCommandListProfiles::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption("with-content", "Show content in each profile.") + .StoreTrue(&WithContent); +} + +int TCommandListProfiles::Run(TConfig& config) { + Y_UNUSED(config); + auto profileManager = CreateYdbProfileManager(config.YdbDir); + const auto profileNames = profileManager->ListProfiles(); + TString activeProfileName = profileManager->GetActiveProfileName(); + for (const auto& profileName : profileNames) { + Cout << profileName; + if (activeProfileName == profileName) { + Cout << " (active)"; + } + if (WithContent) { + Cout << ":"; + } + Cout << Endl; + if (WithContent) { + PrintProfileContent(profileManager->GetProfile(profileName)); + } + } + return EXIT_SUCCESS; +} + +TCommandGetProfile::TCommandGetProfile() + : TClientCommand("get", {}, "List values for specified configuration profile") +{} + +void TCommandGetProfile::Config(TConfig& config) { + TClientCommand::Config(config); + + config.SetFreeArgsMax(1); + SetFreeArgTitle(0, "<name>", "Profile name (case sensitive)"); +} + +void TCommandGetProfile::Parse(TConfig& config) { + TClientCommand::Parse(config); + if (config.ParseResult->GetFreeArgCount()) { + ProfileName = config.ParseResult->GetFreeArgs()[0]; + } +} + +int TCommandGetProfile::Run(TConfig& config) { + Y_UNUSED(config); + auto profileManager = CreateYdbProfileManager(config.YdbDir); + const auto profileNames = profileManager->ListProfiles(); + if (ProfileName) { + if (find(profileNames.begin(), profileNames.end(), ProfileName) == profileNames.end()) { + Cerr << "No existing profile \"" << ProfileName << "\". " + << "Run \"ydb config profile list\" without arguments to see existing profiles" << Endl; + return EXIT_FAILURE; + } + } + if (!ProfileName) { + const auto profileNames = profileManager->ListProfiles(); + if (profileNames.size()) { + Cout << "Please choose profile to list its values:" << Endl; + TNumericOptionsPicker picker; + TString activeProfileName = profileManager->GetActiveProfileName(); + for (size_t i = 0; i < profileNames.size(); ++i) { + TString currentProfileName = profileNames[i]; + TStringBuilder description; + description << currentProfileName; + if (currentProfileName == activeProfileName) { + description << " (active)"; + } + picker.AddOption( + description, + [this, currentProfileName]() { + ProfileName = currentProfileName; + } + ); + } + picker.PickOptionAndDoAction(); + } else { + Cerr << "You have no existing profiles yet. Run \"ydb init\" to create one" << Endl; + return EXIT_FAILURE; + } + } + PrintProfileContent(profileManager->GetProfile(ProfileName)); + return EXIT_SUCCESS; +} + +} // NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/ydb_profile.h b/ydb/public/lib/ydb_cli/commands/ydb_profile.h index e8df3eb5ee..1d556f5585 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_profile.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_profile.h @@ -1,83 +1,83 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/common/command.h> #include <ydb/public/lib/ydb_cli/common/profile_manager.h> - -namespace NYdb { -namespace NConsoleClient { - -std::shared_ptr<IProfileManager> CreateYdbProfileManager(const TString& ydbDir); - -class TCommandConfig : public TClientCommandTree { -public: - TCommandConfig(); -}; - -class TCommandProfile : public TClientCommandTree { -public: - TCommandProfile(); -}; - -class TCommandInit : public TClientCommand { -public: - TCommandInit(); - virtual void Config(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -class TCommandCreateProfile : public TClientCommand { -public: - TCommandCreateProfile(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString ProfileName; -}; - -class TCommandDeleteProfile : public TClientCommand { -public: - TCommandDeleteProfile(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString ProfileName; -}; - -class TCommandActivateProfile : public TClientCommand { -public: - TCommandActivateProfile(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString ProfileName; -}; - -class TCommandListProfiles : public TClientCommand { -public: - TCommandListProfiles(); - virtual void Config(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - bool WithContent; -}; - -class TCommandGetProfile : public TClientCommand { -public: - TCommandGetProfile(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString ProfileName; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +std::shared_ptr<IProfileManager> CreateYdbProfileManager(const TString& ydbDir); + +class TCommandConfig : public TClientCommandTree { +public: + TCommandConfig(); +}; + +class TCommandProfile : public TClientCommandTree { +public: + TCommandProfile(); +}; + +class TCommandInit : public TClientCommand { +public: + TCommandInit(); + virtual void Config(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +class TCommandCreateProfile : public TClientCommand { +public: + TCommandCreateProfile(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString ProfileName; +}; + +class TCommandDeleteProfile : public TClientCommand { +public: + TCommandDeleteProfile(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString ProfileName; +}; + +class TCommandActivateProfile : public TClientCommand { +public: + TCommandActivateProfile(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString ProfileName; +}; + +class TCommandListProfiles : public TClientCommand { +public: + TCommandListProfiles(); + virtual void Config(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + bool WithContent; +}; + +class TCommandGetProfile : public TClientCommand { +public: + TCommandGetProfile(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString ProfileName; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 7d7b2df411..2cccf42a41 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -1,31 +1,31 @@ -#include "ydb_root_common.h" -#include "ydb_profile.h" -#include "ydb_service_discovery.h" -#include "ydb_service_export.h" -#include "ydb_service_import.h" -#include "ydb_service_operation.h" -#include "ydb_service_scheme.h" -#include "ydb_service_scripting.h" -#include "ydb_service_table.h" -#include "ydb_service_stream.h" -#include "ydb_tools.h" -#include "ydb_yql.h" - +#include "ydb_root_common.h" +#include "ydb_profile.h" +#include "ydb_service_discovery.h" +#include "ydb_service_export.h" +#include "ydb_service_import.h" +#include "ydb_service_operation.h" +#include "ydb_service_scheme.h" +#include "ydb_service_scripting.h" +#include "ydb_service_table.h" +#include "ydb_service_stream.h" +#include "ydb_tools.h" +#include "ydb_yql.h" + #include "ydb_workload.h" -#include <util/folder/path.h> -#include <util/folder/dirut.h> -#include <util/string/strip.h> -#include <util/system/env.h> - -namespace NYdb { -namespace NConsoleClient { - -TClientCommandRootCommon::TClientCommandRootCommon(const TClientSettings& settings) - : TClientCommandRootBase("ydb") - , Settings (settings) -{ - ValidateSettings(); +#include <util/folder/path.h> +#include <util/folder/dirut.h> +#include <util/string/strip.h> +#include <util/system/env.h> + +namespace NYdb { +namespace NConsoleClient { + +TClientCommandRootCommon::TClientCommandRootCommon(const TClientSettings& settings) + : TClientCommandRootBase("ydb") + , Settings (settings) +{ + ValidateSettings(); AddCommand(std::make_unique<TCommandDiscovery>()); AddCommand(std::make_unique<TCommandScheme>()); AddCommand(std::make_unique<TCommandScripting>()); @@ -39,522 +39,522 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TClientSettings& settin AddCommand(std::make_unique<TCommandYql>()); AddCommand(std::make_unique<TCommandStream>()); AddCommand(std::make_unique<TCommandWorkload>()); -} - -void TClientCommandRootCommon::ValidateSettings() { - if (!Settings.EnableSsl.Defined()) { - Cerr << "Missing ssl enabling flag in client settings" << Endl; - } else if (!Settings.UseOAuthToken.Defined()) { - Cerr << "Missing OAuth token usage flag in client settings" << Endl; - } else if (!Settings.UseDefaultTokenFile.Defined()) { - Cerr << "Missing default token file usage flag in client settings" << Endl; - } else if (!Settings.UseIamAuth.Defined()) { - Cerr << "Missing IAM authentication usage flag in client settings" << Endl; - } else if (!Settings.UseExportToYt.Defined()) { - Cerr << "Missing export to YT command usage flag in client settings" << Endl; - } else if (!Settings.UseStaticCredentials.Defined()) { - Cerr << "Missing static credentials usage flag in client settings" << Endl; - } else if (!Settings.MentionUserAccount.Defined()) { - Cerr << "Missing user account mentioning flag in client settings" << Endl; - } else if (!Settings.YdbDir) { - Cerr << "Missing YDB directory in client settings" << Endl; - } else { - return; - } - exit(EXIT_FAILURE); -} - -void TClientCommandRootCommon::FillConfig(TConfig& config) { - config.YdbDir = Settings.YdbDir; - config.UseOAuthToken = Settings.UseOAuthToken.GetRef(); - config.UseIamAuth = Settings.UseIamAuth.GetRef(); - config.UseStaticCredentials = Settings.UseStaticCredentials.GetRef(); - config.UseExportToYt = Settings.UseExportToYt.GetRef(); - SetCredentialsGetter(config); -} - -// Default CredentialsGetter that can be overridden in different CLI versions -void TClientCommandRootCommon::SetCredentialsGetter(TConfig& config) { - config.CredentialsGetter = [](const TClientCommand::TConfig& config) { - if (config.SecurityToken) { - return CreateOAuthCredentialsProviderFactory(config.SecurityToken); - } - - if (config.UseStaticCredentials) { - if (config.StaticCredentials.User) { - return CreateLoginCredentialsProviderFactory(config.StaticCredentials); - } - } - - return CreateInsecureCredentialsProviderFactory(); - }; -} - -void TClientCommandRootCommon::Config(TConfig& config) { - FillConfig(config); - NLastGetopt::TOpts& opts = *config.Opts; - - TStringBuilder endpointHelp; - endpointHelp << "[Required] Endpoint to connect. Protocols: grpc, grpcs (Default: " - << (Settings.EnableSsl.GetRef() ? "grpcs" : "grpc" ) << ")." << Endl; - - endpointHelp << " Endpoint search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. Active configuration profile"; - TStringBuilder databaseHelp; - databaseHelp << "[Required] Database to work with." << Endl - << " Database search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. Active configuration profile"; - - opts.AddLongOption('e', "endpoint", endpointHelp) - .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); - opts.AddLongOption('d', "database", databaseHelp) - .RequiredArgument("PATH").StoreResult(&Database); - opts.AddLongOption('v', "verbose", "Increase verbosity of operations") - .Optional().StoreTrue(&IsVerbose); - - TClientCommandRootBase::Config(config); - - if (config.UseIamAuth) { - const TString docsUrl = "cloud.yandex.ru/docs"; - TStringBuilder iamTokenHelp; - iamTokenHelp << "IAM token file. Note: IAM tokens expire in 12 hours." << Endl - << " For more info go to: " << docsUrl << "/iam/concepts/authorization/iam-token" << Endl - << " Token search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"IAM_TOKEN\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("iam-token-file", iamTokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); - - TStringBuilder ycTokenHelp; - ycTokenHelp << "YC token file. It should contain OAuth token of a Yandex Passport user to get IAM token with." << Endl - << " For more info go to: " << docsUrl << "/iam/concepts/authorization/oauth-token" << Endl - << " Token search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"YC_TOKEN\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("yc-token-file", ycTokenHelp).RequiredArgument("PATH").StoreResult(&YCTokenFile); - - TStringBuilder metadataHelp; - metadataHelp << "Use metadata service on a virtual machine to get credentials" << Endl - << " For more info go to: " << docsUrl << "/compute/operations/vm-connect/auth-inside-vm" << Endl - << " Definition priority:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"USE_METADATA_CREDENTIALS\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("use-metadata-credentials", metadataHelp).Optional().StoreTrue(&UseMetadataCredentials); - - TStringBuilder saKeyHelp; - saKeyHelp << "Service account"; - if (Settings.MentionUserAccount.GetRef()) { - saKeyHelp << " (or user account)"; - } - saKeyHelp << " key file" << Endl - << " For more info go to: " << docsUrl << "/iam/operations/iam-token/create-for-sa" << Endl - << " Definition priority:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"SA_KEY_FILE\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("sa-key-file", saKeyHelp).RequiredArgument("PATH").StoreResult(&SaKeyFile); - } - - if (config.UseOAuthToken) { - TStringBuilder tokenHelp; - tokenHelp << "OAuth token file" << Endl - << " Token search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"YDB_TOKEN\" environment variable" << Endl - << " 4. Active configuration profile"; - if (Settings.UseDefaultTokenFile.GetRef()) { - tokenHelp << Endl << " 5. Default token file \"" << defaultTokenFile << "\""; - } - opts.AddLongOption("token-file", tokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); - } - - if (config.UseStaticCredentials) { - TStringBuilder userHelp; - userHelp << "User name to authenticate with" << Endl - << " User name search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"YDB_USER\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("user", userHelp).RequiredArgument("STR").StoreResult(&UserName); - - TStringBuilder passwordHelp; - passwordHelp << "File with password to authenticate with" << Endl - << " Password search order:" << Endl - << " 1. This option" << Endl - << " 2. Profile specified with --profile option" << Endl - << " 3. \"YDB_PASSWORD\" environment variable" << Endl - << " 4. Active configuration profile"; - opts.AddLongOption("password-file", passwordHelp).RequiredArgument("PATH").StoreResult(&PasswordFile); - } - - if (config.UseIamAuth) { - opts.AddLongOption("iam-endpoint", "Endpoint of IAM service") - .RequiredArgument("STR") - .StoreResult(&IamEndpoint) - .DefaultValue(config.IamEndpoint); - } - - opts.AddLongOption("profile", "Profile name to use configuration parameters from.") - .RequiredArgument("NAME").StoreResult(&ProfileName); - - TStringStream stream; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - stream << " [options...] <subcommand>" << Endl << Endl - << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; - RenderCommandsDescription(stream, colors); - opts.SetCmdLineDescr(stream.Str()); - - opts.GetLongOption("time").Hidden(); - opts.GetLongOption("progress").Hidden(); -} - -void TClientCommandRootCommon::Parse(TConfig& config) { - ProfileManager = CreateYdbProfileManager(Settings.YdbDir); - ParseProfile(); - - TClientCommandRootBase::Parse(config); - ParseDatabase(config); - ParseCaCerts(config); - config.IsVerbose = IsVerbose; -} - -void TClientCommandRootCommon::ParseAddress(TConfig& config) { - TString hostname; - TString port = "2135"; - - if (Address.empty()) { - auto profile = Profile; - if (!profile) { - profile = ProfileManager->GetActiveProfile(); - } - if (profile && profile->Has("endpoint")) { - Address = profile->GetValue("endpoint").as<TString>(); - } - } - - if (Address.empty()) { - if (config.IsSystemCommand()) { - return; - } else { - throw TMissUseException() - << "Missing required option 'endpoint'."; - } - } else { - config.EnableSsl = Settings.EnableSsl.GetRef(); - ParseProtocol(config); - auto colon_pos = Address.find(":"); - if (colon_pos == TString::npos) { - config.Address = Address + ":" + port; - } else { - if (colon_pos == Address.rfind(":")) { - config.Address = Address; - } else { - throw TMissUseException() << "Wrong format for option 'endpoint': more than one colon found."; - } - } - } -} - -void TClientCommandRootCommon::ParseProfile() { - if (ProfileName) { - if (ProfileManager->HasProfile(ProfileName)) { - Profile = ProfileManager->GetProfile(ProfileName); - } else { - throw TMissUseException() << "Profile " << ProfileName << " does not exist." << Endl - << "Run \"ydb config profile list\" to see existing profiles"; - } - } -} - -void TClientCommandRootCommon::ParseDatabase(TConfig& config) { - if (Database.empty()) { - auto profile = Profile; - if (!profile) { - profile = ProfileManager->GetActiveProfile(); - } - if (profile && profile->Has("database")) { - Database = profile->GetValue("database").as<TString>(); - } - } - - if (Database.empty()) { - if (!config.IsSystemCommand()) { - throw TMissUseException() - << "Missing required option 'database'."; - } - } else if (!Database.StartsWith('/')) { - if (!config.IsSystemCommand()) { - throw TMissUseException() << "Path to a database \"" << Database - << "\" is incorrect. It must be absolute and thus must begin with '/'."; - } - } - config.Database = Database; -} - -namespace { - inline void PrintSettingFromProfile(const TString& setting, std::shared_ptr<IProfile> profile, bool explicitOption) { - Cout << "Using " << setting << " due to configuration in" << (explicitOption ? "" : " active") << " profile \"" - << profile->GetName() << "\"" << (explicitOption ? " from explicit --profile option" : "") << Endl; - } -} - -bool TClientCommandRootCommon::GetCredentialsFromProfile(std::shared_ptr<IProfile> profile, TConfig& config, bool explicitOption) { - if (!profile || !profile->Has("authentication")) { - return false; - } - auto authValue = profile->GetValue("authentication"); - if (!authValue["method"]) { - throw TMissUseException() - << "Configuration profile has \"authentication\" but does not has \"method\" in it"; - } - TString authMethod = authValue["method"].as<TString>(); - - if (authMethod == "use-metadata-credentials") { - if (IsVerbose) { - PrintSettingFromProfile("metadata service", profile, explicitOption); - } - config.UseMetadataCredentials = true; - return true; - } - bool knownMethod = false; - if (config.UseIamAuth) { - knownMethod |= (authMethod == "iam-token" || authMethod == "yc-token" || authMethod == "sa-key-file"); - } - if (config.UseOAuthToken) { - knownMethod |= (authMethod == "ydb-token"); - } - if (config.UseStaticCredentials) { - knownMethod |= (authMethod == "static-credentials"); - } - if (!knownMethod) { - throw TMissUseException() << "Unknown authentication method in configuration profile: \"" << authMethod << "\""; - } - if (!authValue["data"]) { - throw TMissUseException() << "Active configuration profile has \"authentication\" with method \"" - << authMethod << "\" in it, but no \"data\""; - } - auto authData = authValue["data"]; - - if (authMethod == "iam-token") { - if (IsVerbose) { - PrintSettingFromProfile("iam token", profile, explicitOption); - } - config.SecurityToken = authData.as<TString>(); - } else if (authMethod == "yc-token") { - if (IsVerbose) { - PrintSettingFromProfile("Yandex.Cloud Passport token (yc-token)", profile, explicitOption); - } - config.YCToken = authData.as<TString>(); - } else if (authMethod == "sa-key-file") { - if (IsVerbose) { - PrintSettingFromProfile("service account key file (sa-key-file)", profile, explicitOption); - } - config.SaKeyFile = authData.as<TString>(); - } else if (authMethod == "ydb-token") { - if (IsVerbose) { - PrintSettingFromProfile("OAuth token (ydb-token)", profile, explicitOption); - } - config.SecurityToken = authData.as<TString>(); - } else if (authMethod == "static-credentials") { - if (IsVerbose) { - PrintSettingFromProfile("user name & password", profile, explicitOption); - } - if (authData["user"]) { - config.StaticCredentials.User = authData["user"].as<TString>(); - } - if (authData["password"]) { - config.StaticCredentials.Password = authData["password"].as<TString>(); - } - } else { - return false; - } - return true; -} - -void TClientCommandRootCommon::ParseCredentials(TConfig& config) { - size_t explicitAuthMethodCount = (size_t)(!TokenFile.empty()) + (size_t)(!YCTokenFile.empty()) - + (size_t)UseMetadataCredentials + (size_t)(!SaKeyFile.empty()) - + (size_t)(!UserName.empty() || !PasswordFile.empty()); - - switch (explicitAuthMethodCount) { - case 0: - { - // Priority 2. No explicit auth methods. Checking configuration profile given via --profile option. - if (GetCredentialsFromProfile(Profile, config, true)) { - break; - } - - // Priority 3. No auth methods from --profile either. Checking environment variables. - if (config.UseIamAuth) { - TString envIamToken = GetEnv("IAM_TOKEN"); - if (!envIamToken.empty()) { - if (IsVerbose) { - Cout << "Using iam token from IAM_TOKEN env variable" << Endl; - } - config.SecurityToken = envIamToken; - break; - } - TString envYcToken = GetEnv("YC_TOKEN"); - if (!envYcToken.empty()) { - if (IsVerbose) { - Cout << "Using Yandex.Cloud Passport token from YC_TOKEN env variable" << Endl; - } - config.YCToken = envYcToken; - break; - } - if (GetEnv("USE_METADATA_CREDENTIALS") == "1") { - if (IsVerbose) { - Cout << "Using metadata service due to USE_METADATA_CREDENTIALS=\"1\" env variable" << Endl; - } - config.UseMetadataCredentials = true; - break; - } - TString envSaKeyFile = GetEnv("SA_KEY_FILE"); - if (!envSaKeyFile.empty()) { - if (IsVerbose) { - Cout << "Using service account key file from SA_KEY_FILE env variable" << Endl; - } - config.SaKeyFile = envSaKeyFile; - break; - } - } - if (config.UseOAuthToken) { - TString envYdbToken = GetEnv("YDB_TOKEN"); - if (!envYdbToken.empty()) { - if (IsVerbose) { - Cout << "Using OAuth token from YDB_TOKEN env variable" << Endl; - } - config.SecurityToken = envYdbToken; - break; - } - } - if (config.UseStaticCredentials) { - TString userName = GetEnv("YDB_USER"); - if (!userName.empty()) { - if (IsVerbose) { - Cout << "Using user name from YDB_USER env variable" << Endl; - } - config.StaticCredentials.User = userName; - } - - TString password = GetEnv("YDB_PASSWORD"); - if (!password.empty()) { - if (IsVerbose) { - Cout << "Using user password from YDB_PASSWORD env variable" << Endl; - } - config.StaticCredentials.Password = password; - } - if (!userName.empty() || !password.empty()) { - break; - } - } - - // Priority 4. No auth methods from environment variables too. Checking active configuration profile. - // (if --profile option is not set) - if (!Profile && GetCredentialsFromProfile(ProfileManager->GetActiveProfile(), config, false)) { - break; - } - - if (Settings.UseDefaultTokenFile.GetRef()) { - // Priority 5. No auth methods from active configuration profile. Checking default token file. - TString tokenFile = defaultTokenFile; - if (ReadFromFileIfExists(tokenFile, "default token", config.SecurityToken)) { - if (IsVerbose) { - Cout << "Using auth token from default token file " << defaultTokenFile << Endl; - } - } else { - if (IsVerbose) { - Cout << "No authentication methods were found. Going without authentication" << Endl; - } - } - } - break; - } - case 1: - // Priority 1. Exactly one explicit auth method. Using it. - if (TokenFile) { - if (IsVerbose) { - Cout << "Using token from file provided with explicit option" << Endl; - } - config.SecurityToken = ReadFromFile(TokenFile, "token"); - } else if (YCTokenFile) { - if (IsVerbose) { - Cout << "Using Yandex.Cloud Passport token from file provided with --yc-token-file option" << Endl; - } - config.YCToken = ReadFromFile(YCTokenFile, "token"); - } else if (UseMetadataCredentials) { - if (IsVerbose) { - Cout << "Using metadata service due to --use-metadata-credentials option" << Endl; - } - config.UseMetadataCredentials = true; - } else if (SaKeyFile) { - if (IsVerbose) { - Cout << "Using service account key file provided with --sa-key-file option" << Endl; - } - config.SaKeyFile = SaKeyFile; - } else if (UserName || PasswordFile) { - if (UserName) { - if (IsVerbose) { - Cout << "Using user name provided with --user option" << Endl; - } - config.StaticCredentials.User = UserName; - } - if (PasswordFile) { - if (IsVerbose) { - Cout << "Using user password from file provided with --password-file option" << Endl; - } - config.StaticCredentials.Password = ReadFromFile(PasswordFile, "password"); - } - } - break; - default: - TStringBuilder str; - str << explicitAuthMethodCount << " methods were provided via options:"; - if (!TokenFile.empty()) { - str << " TokenFile (" << TokenFile << ")"; - } - if (!YCTokenFile.empty()) { - str << " YCTokenFile (" << YCTokenFile << ")"; - } - if (!SaKeyFile.empty()) { - str << " SaKeyFile (" << SaKeyFile << ")"; - } - if (UseMetadataCredentials) { - str << " UseMetadataCredentials (true)"; - } - - throw TMissUseException() << str << ". Choose exactly one of them"; - } - - if (config.UseStaticCredentials) { - if (config.StaticCredentials.User) { - if (!config.StaticCredentials.Password) { - Cout << "Enter password for user " << config.StaticCredentials.User << ": "; - config.StaticCredentials.Password = InputPassword(); - } - } else { - if (config.StaticCredentials.Password) { - throw TMissUseException() << "User password was provided without user name"; - } - } - } - - if (config.UseIamAuth && IamEndpoint) { - config.IamEndpoint = IamEndpoint; - } -} - -} -} +} + +void TClientCommandRootCommon::ValidateSettings() { + if (!Settings.EnableSsl.Defined()) { + Cerr << "Missing ssl enabling flag in client settings" << Endl; + } else if (!Settings.UseOAuthToken.Defined()) { + Cerr << "Missing OAuth token usage flag in client settings" << Endl; + } else if (!Settings.UseDefaultTokenFile.Defined()) { + Cerr << "Missing default token file usage flag in client settings" << Endl; + } else if (!Settings.UseIamAuth.Defined()) { + Cerr << "Missing IAM authentication usage flag in client settings" << Endl; + } else if (!Settings.UseExportToYt.Defined()) { + Cerr << "Missing export to YT command usage flag in client settings" << Endl; + } else if (!Settings.UseStaticCredentials.Defined()) { + Cerr << "Missing static credentials usage flag in client settings" << Endl; + } else if (!Settings.MentionUserAccount.Defined()) { + Cerr << "Missing user account mentioning flag in client settings" << Endl; + } else if (!Settings.YdbDir) { + Cerr << "Missing YDB directory in client settings" << Endl; + } else { + return; + } + exit(EXIT_FAILURE); +} + +void TClientCommandRootCommon::FillConfig(TConfig& config) { + config.YdbDir = Settings.YdbDir; + config.UseOAuthToken = Settings.UseOAuthToken.GetRef(); + config.UseIamAuth = Settings.UseIamAuth.GetRef(); + config.UseStaticCredentials = Settings.UseStaticCredentials.GetRef(); + config.UseExportToYt = Settings.UseExportToYt.GetRef(); + SetCredentialsGetter(config); +} + +// Default CredentialsGetter that can be overridden in different CLI versions +void TClientCommandRootCommon::SetCredentialsGetter(TConfig& config) { + config.CredentialsGetter = [](const TClientCommand::TConfig& config) { + if (config.SecurityToken) { + return CreateOAuthCredentialsProviderFactory(config.SecurityToken); + } + + if (config.UseStaticCredentials) { + if (config.StaticCredentials.User) { + return CreateLoginCredentialsProviderFactory(config.StaticCredentials); + } + } + + return CreateInsecureCredentialsProviderFactory(); + }; +} + +void TClientCommandRootCommon::Config(TConfig& config) { + FillConfig(config); + NLastGetopt::TOpts& opts = *config.Opts; + + TStringBuilder endpointHelp; + endpointHelp << "[Required] Endpoint to connect. Protocols: grpc, grpcs (Default: " + << (Settings.EnableSsl.GetRef() ? "grpcs" : "grpc" ) << ")." << Endl; + + endpointHelp << " Endpoint search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. Active configuration profile"; + TStringBuilder databaseHelp; + databaseHelp << "[Required] Database to work with." << Endl + << " Database search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. Active configuration profile"; + + opts.AddLongOption('e', "endpoint", endpointHelp) + .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); + opts.AddLongOption('d', "database", databaseHelp) + .RequiredArgument("PATH").StoreResult(&Database); + opts.AddLongOption('v', "verbose", "Increase verbosity of operations") + .Optional().StoreTrue(&IsVerbose); + + TClientCommandRootBase::Config(config); + + if (config.UseIamAuth) { + const TString docsUrl = "cloud.yandex.ru/docs"; + TStringBuilder iamTokenHelp; + iamTokenHelp << "IAM token file. Note: IAM tokens expire in 12 hours." << Endl + << " For more info go to: " << docsUrl << "/iam/concepts/authorization/iam-token" << Endl + << " Token search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"IAM_TOKEN\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("iam-token-file", iamTokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); + + TStringBuilder ycTokenHelp; + ycTokenHelp << "YC token file. It should contain OAuth token of a Yandex Passport user to get IAM token with." << Endl + << " For more info go to: " << docsUrl << "/iam/concepts/authorization/oauth-token" << Endl + << " Token search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"YC_TOKEN\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("yc-token-file", ycTokenHelp).RequiredArgument("PATH").StoreResult(&YCTokenFile); + + TStringBuilder metadataHelp; + metadataHelp << "Use metadata service on a virtual machine to get credentials" << Endl + << " For more info go to: " << docsUrl << "/compute/operations/vm-connect/auth-inside-vm" << Endl + << " Definition priority:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"USE_METADATA_CREDENTIALS\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("use-metadata-credentials", metadataHelp).Optional().StoreTrue(&UseMetadataCredentials); + + TStringBuilder saKeyHelp; + saKeyHelp << "Service account"; + if (Settings.MentionUserAccount.GetRef()) { + saKeyHelp << " (or user account)"; + } + saKeyHelp << " key file" << Endl + << " For more info go to: " << docsUrl << "/iam/operations/iam-token/create-for-sa" << Endl + << " Definition priority:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"SA_KEY_FILE\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("sa-key-file", saKeyHelp).RequiredArgument("PATH").StoreResult(&SaKeyFile); + } + + if (config.UseOAuthToken) { + TStringBuilder tokenHelp; + tokenHelp << "OAuth token file" << Endl + << " Token search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"YDB_TOKEN\" environment variable" << Endl + << " 4. Active configuration profile"; + if (Settings.UseDefaultTokenFile.GetRef()) { + tokenHelp << Endl << " 5. Default token file \"" << defaultTokenFile << "\""; + } + opts.AddLongOption("token-file", tokenHelp).RequiredArgument("PATH").StoreResult(&TokenFile); + } + + if (config.UseStaticCredentials) { + TStringBuilder userHelp; + userHelp << "User name to authenticate with" << Endl + << " User name search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"YDB_USER\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("user", userHelp).RequiredArgument("STR").StoreResult(&UserName); + + TStringBuilder passwordHelp; + passwordHelp << "File with password to authenticate with" << Endl + << " Password search order:" << Endl + << " 1. This option" << Endl + << " 2. Profile specified with --profile option" << Endl + << " 3. \"YDB_PASSWORD\" environment variable" << Endl + << " 4. Active configuration profile"; + opts.AddLongOption("password-file", passwordHelp).RequiredArgument("PATH").StoreResult(&PasswordFile); + } + + if (config.UseIamAuth) { + opts.AddLongOption("iam-endpoint", "Endpoint of IAM service") + .RequiredArgument("STR") + .StoreResult(&IamEndpoint) + .DefaultValue(config.IamEndpoint); + } + + opts.AddLongOption("profile", "Profile name to use configuration parameters from.") + .RequiredArgument("NAME").StoreResult(&ProfileName); + + TStringStream stream; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << " [options...] <subcommand>" << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + RenderCommandsDescription(stream, colors); + opts.SetCmdLineDescr(stream.Str()); + + opts.GetLongOption("time").Hidden(); + opts.GetLongOption("progress").Hidden(); +} + +void TClientCommandRootCommon::Parse(TConfig& config) { + ProfileManager = CreateYdbProfileManager(Settings.YdbDir); + ParseProfile(); + + TClientCommandRootBase::Parse(config); + ParseDatabase(config); + ParseCaCerts(config); + config.IsVerbose = IsVerbose; +} + +void TClientCommandRootCommon::ParseAddress(TConfig& config) { + TString hostname; + TString port = "2135"; + + if (Address.empty()) { + auto profile = Profile; + if (!profile) { + profile = ProfileManager->GetActiveProfile(); + } + if (profile && profile->Has("endpoint")) { + Address = profile->GetValue("endpoint").as<TString>(); + } + } + + if (Address.empty()) { + if (config.IsSystemCommand()) { + return; + } else { + throw TMissUseException() + << "Missing required option 'endpoint'."; + } + } else { + config.EnableSsl = Settings.EnableSsl.GetRef(); + ParseProtocol(config); + auto colon_pos = Address.find(":"); + if (colon_pos == TString::npos) { + config.Address = Address + ":" + port; + } else { + if (colon_pos == Address.rfind(":")) { + config.Address = Address; + } else { + throw TMissUseException() << "Wrong format for option 'endpoint': more than one colon found."; + } + } + } +} + +void TClientCommandRootCommon::ParseProfile() { + if (ProfileName) { + if (ProfileManager->HasProfile(ProfileName)) { + Profile = ProfileManager->GetProfile(ProfileName); + } else { + throw TMissUseException() << "Profile " << ProfileName << " does not exist." << Endl + << "Run \"ydb config profile list\" to see existing profiles"; + } + } +} + +void TClientCommandRootCommon::ParseDatabase(TConfig& config) { + if (Database.empty()) { + auto profile = Profile; + if (!profile) { + profile = ProfileManager->GetActiveProfile(); + } + if (profile && profile->Has("database")) { + Database = profile->GetValue("database").as<TString>(); + } + } + + if (Database.empty()) { + if (!config.IsSystemCommand()) { + throw TMissUseException() + << "Missing required option 'database'."; + } + } else if (!Database.StartsWith('/')) { + if (!config.IsSystemCommand()) { + throw TMissUseException() << "Path to a database \"" << Database + << "\" is incorrect. It must be absolute and thus must begin with '/'."; + } + } + config.Database = Database; +} + +namespace { + inline void PrintSettingFromProfile(const TString& setting, std::shared_ptr<IProfile> profile, bool explicitOption) { + Cout << "Using " << setting << " due to configuration in" << (explicitOption ? "" : " active") << " profile \"" + << profile->GetName() << "\"" << (explicitOption ? " from explicit --profile option" : "") << Endl; + } +} + +bool TClientCommandRootCommon::GetCredentialsFromProfile(std::shared_ptr<IProfile> profile, TConfig& config, bool explicitOption) { + if (!profile || !profile->Has("authentication")) { + return false; + } + auto authValue = profile->GetValue("authentication"); + if (!authValue["method"]) { + throw TMissUseException() + << "Configuration profile has \"authentication\" but does not has \"method\" in it"; + } + TString authMethod = authValue["method"].as<TString>(); + + if (authMethod == "use-metadata-credentials") { + if (IsVerbose) { + PrintSettingFromProfile("metadata service", profile, explicitOption); + } + config.UseMetadataCredentials = true; + return true; + } + bool knownMethod = false; + if (config.UseIamAuth) { + knownMethod |= (authMethod == "iam-token" || authMethod == "yc-token" || authMethod == "sa-key-file"); + } + if (config.UseOAuthToken) { + knownMethod |= (authMethod == "ydb-token"); + } + if (config.UseStaticCredentials) { + knownMethod |= (authMethod == "static-credentials"); + } + if (!knownMethod) { + throw TMissUseException() << "Unknown authentication method in configuration profile: \"" << authMethod << "\""; + } + if (!authValue["data"]) { + throw TMissUseException() << "Active configuration profile has \"authentication\" with method \"" + << authMethod << "\" in it, but no \"data\""; + } + auto authData = authValue["data"]; + + if (authMethod == "iam-token") { + if (IsVerbose) { + PrintSettingFromProfile("iam token", profile, explicitOption); + } + config.SecurityToken = authData.as<TString>(); + } else if (authMethod == "yc-token") { + if (IsVerbose) { + PrintSettingFromProfile("Yandex.Cloud Passport token (yc-token)", profile, explicitOption); + } + config.YCToken = authData.as<TString>(); + } else if (authMethod == "sa-key-file") { + if (IsVerbose) { + PrintSettingFromProfile("service account key file (sa-key-file)", profile, explicitOption); + } + config.SaKeyFile = authData.as<TString>(); + } else if (authMethod == "ydb-token") { + if (IsVerbose) { + PrintSettingFromProfile("OAuth token (ydb-token)", profile, explicitOption); + } + config.SecurityToken = authData.as<TString>(); + } else if (authMethod == "static-credentials") { + if (IsVerbose) { + PrintSettingFromProfile("user name & password", profile, explicitOption); + } + if (authData["user"]) { + config.StaticCredentials.User = authData["user"].as<TString>(); + } + if (authData["password"]) { + config.StaticCredentials.Password = authData["password"].as<TString>(); + } + } else { + return false; + } + return true; +} + +void TClientCommandRootCommon::ParseCredentials(TConfig& config) { + size_t explicitAuthMethodCount = (size_t)(!TokenFile.empty()) + (size_t)(!YCTokenFile.empty()) + + (size_t)UseMetadataCredentials + (size_t)(!SaKeyFile.empty()) + + (size_t)(!UserName.empty() || !PasswordFile.empty()); + + switch (explicitAuthMethodCount) { + case 0: + { + // Priority 2. No explicit auth methods. Checking configuration profile given via --profile option. + if (GetCredentialsFromProfile(Profile, config, true)) { + break; + } + + // Priority 3. No auth methods from --profile either. Checking environment variables. + if (config.UseIamAuth) { + TString envIamToken = GetEnv("IAM_TOKEN"); + if (!envIamToken.empty()) { + if (IsVerbose) { + Cout << "Using iam token from IAM_TOKEN env variable" << Endl; + } + config.SecurityToken = envIamToken; + break; + } + TString envYcToken = GetEnv("YC_TOKEN"); + if (!envYcToken.empty()) { + if (IsVerbose) { + Cout << "Using Yandex.Cloud Passport token from YC_TOKEN env variable" << Endl; + } + config.YCToken = envYcToken; + break; + } + if (GetEnv("USE_METADATA_CREDENTIALS") == "1") { + if (IsVerbose) { + Cout << "Using metadata service due to USE_METADATA_CREDENTIALS=\"1\" env variable" << Endl; + } + config.UseMetadataCredentials = true; + break; + } + TString envSaKeyFile = GetEnv("SA_KEY_FILE"); + if (!envSaKeyFile.empty()) { + if (IsVerbose) { + Cout << "Using service account key file from SA_KEY_FILE env variable" << Endl; + } + config.SaKeyFile = envSaKeyFile; + break; + } + } + if (config.UseOAuthToken) { + TString envYdbToken = GetEnv("YDB_TOKEN"); + if (!envYdbToken.empty()) { + if (IsVerbose) { + Cout << "Using OAuth token from YDB_TOKEN env variable" << Endl; + } + config.SecurityToken = envYdbToken; + break; + } + } + if (config.UseStaticCredentials) { + TString userName = GetEnv("YDB_USER"); + if (!userName.empty()) { + if (IsVerbose) { + Cout << "Using user name from YDB_USER env variable" << Endl; + } + config.StaticCredentials.User = userName; + } + + TString password = GetEnv("YDB_PASSWORD"); + if (!password.empty()) { + if (IsVerbose) { + Cout << "Using user password from YDB_PASSWORD env variable" << Endl; + } + config.StaticCredentials.Password = password; + } + if (!userName.empty() || !password.empty()) { + break; + } + } + + // Priority 4. No auth methods from environment variables too. Checking active configuration profile. + // (if --profile option is not set) + if (!Profile && GetCredentialsFromProfile(ProfileManager->GetActiveProfile(), config, false)) { + break; + } + + if (Settings.UseDefaultTokenFile.GetRef()) { + // Priority 5. No auth methods from active configuration profile. Checking default token file. + TString tokenFile = defaultTokenFile; + if (ReadFromFileIfExists(tokenFile, "default token", config.SecurityToken)) { + if (IsVerbose) { + Cout << "Using auth token from default token file " << defaultTokenFile << Endl; + } + } else { + if (IsVerbose) { + Cout << "No authentication methods were found. Going without authentication" << Endl; + } + } + } + break; + } + case 1: + // Priority 1. Exactly one explicit auth method. Using it. + if (TokenFile) { + if (IsVerbose) { + Cout << "Using token from file provided with explicit option" << Endl; + } + config.SecurityToken = ReadFromFile(TokenFile, "token"); + } else if (YCTokenFile) { + if (IsVerbose) { + Cout << "Using Yandex.Cloud Passport token from file provided with --yc-token-file option" << Endl; + } + config.YCToken = ReadFromFile(YCTokenFile, "token"); + } else if (UseMetadataCredentials) { + if (IsVerbose) { + Cout << "Using metadata service due to --use-metadata-credentials option" << Endl; + } + config.UseMetadataCredentials = true; + } else if (SaKeyFile) { + if (IsVerbose) { + Cout << "Using service account key file provided with --sa-key-file option" << Endl; + } + config.SaKeyFile = SaKeyFile; + } else if (UserName || PasswordFile) { + if (UserName) { + if (IsVerbose) { + Cout << "Using user name provided with --user option" << Endl; + } + config.StaticCredentials.User = UserName; + } + if (PasswordFile) { + if (IsVerbose) { + Cout << "Using user password from file provided with --password-file option" << Endl; + } + config.StaticCredentials.Password = ReadFromFile(PasswordFile, "password"); + } + } + break; + default: + TStringBuilder str; + str << explicitAuthMethodCount << " methods were provided via options:"; + if (!TokenFile.empty()) { + str << " TokenFile (" << TokenFile << ")"; + } + if (!YCTokenFile.empty()) { + str << " YCTokenFile (" << YCTokenFile << ")"; + } + if (!SaKeyFile.empty()) { + str << " SaKeyFile (" << SaKeyFile << ")"; + } + if (UseMetadataCredentials) { + str << " UseMetadataCredentials (true)"; + } + + throw TMissUseException() << str << ". Choose exactly one of them"; + } + + if (config.UseStaticCredentials) { + if (config.StaticCredentials.User) { + if (!config.StaticCredentials.Password) { + Cout << "Enter password for user " << config.StaticCredentials.User << ": "; + config.StaticCredentials.Password = InputPassword(); + } + } else { + if (config.StaticCredentials.Password) { + throw TMissUseException() << "User password was provided without user name"; + } + } + } + + if (config.UseIamAuth && IamEndpoint) { + config.IamEndpoint = IamEndpoint; + } +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h index 7e3024a773..a9433e3d7f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h @@ -1,65 +1,65 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/ydb_cli/common/profile_manager.h> #include <ydb/public/lib/ydb_cli/common/root.h> - -namespace NYdb { -namespace NConsoleClient { - -struct TClientSettings { - // Whether to use secure connection or not - TMaybe<bool> EnableSsl; - // Whether to use OAuth token in auth options or not - TMaybe<bool> UseOAuthToken; - // Whether to use default token file in auth options or not - TMaybe<bool> UseDefaultTokenFile; - // Whether to use IAM authentication (Yandex.Cloud) or not - TMaybe<bool> UseIamAuth; - // Whether to use static credentials (user/password) or not - TMaybe<bool> UseStaticCredentials; - // Whether to use export to YT command or not - TMaybe<bool> UseExportToYt; - // Whether to mention user account in --help command or not - TMaybe<bool> MentionUserAccount; - // Name of a directory in user home directory to save profile config - TString YdbDir; -}; - -class TClientCommandRootCommon : public TClientCommandRootBase { -public: - TClientCommandRootCommon(const TClientSettings& settings); - void Config(TConfig& config) override; - void Parse(TConfig& config) override; - void ParseAddress(TConfig& config) override; - void ParseCredentials(TConfig& config) override; - -protected: - virtual void FillConfig(TConfig& config); - virtual void SetCredentialsGetter(TConfig& config); - -private: - void ValidateSettings(); - bool GetCredentialsFromProfile(std::shared_ptr<IProfile> profile, TConfig& config, bool explicitOption); - - void ParseProfile(); - void ParseDatabase(TConfig& config); - - TString Database; - bool IsVerbose = false; - TString ProfileName; - std::shared_ptr<IProfile> Profile; - std::shared_ptr<IProfileManager> ProfileManager; - - TString UserName; - TString PasswordFile; - - bool UseMetadataCredentials = false; - TString YCToken; - TString YCTokenFile; - TString SaKeyFile; - TString IamEndpoint; - const TClientSettings& Settings; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +struct TClientSettings { + // Whether to use secure connection or not + TMaybe<bool> EnableSsl; + // Whether to use OAuth token in auth options or not + TMaybe<bool> UseOAuthToken; + // Whether to use default token file in auth options or not + TMaybe<bool> UseDefaultTokenFile; + // Whether to use IAM authentication (Yandex.Cloud) or not + TMaybe<bool> UseIamAuth; + // Whether to use static credentials (user/password) or not + TMaybe<bool> UseStaticCredentials; + // Whether to use export to YT command or not + TMaybe<bool> UseExportToYt; + // Whether to mention user account in --help command or not + TMaybe<bool> MentionUserAccount; + // Name of a directory in user home directory to save profile config + TString YdbDir; +}; + +class TClientCommandRootCommon : public TClientCommandRootBase { +public: + TClientCommandRootCommon(const TClientSettings& settings); + void Config(TConfig& config) override; + void Parse(TConfig& config) override; + void ParseAddress(TConfig& config) override; + void ParseCredentials(TConfig& config) override; + +protected: + virtual void FillConfig(TConfig& config); + virtual void SetCredentialsGetter(TConfig& config); + +private: + void ValidateSettings(); + bool GetCredentialsFromProfile(std::shared_ptr<IProfile> profile, TConfig& config, bool explicitOption); + + void ParseProfile(); + void ParseDatabase(TConfig& config); + + TString Database; + bool IsVerbose = false; + TString ProfileName; + std::shared_ptr<IProfile> Profile; + std::shared_ptr<IProfileManager> ProfileManager; + + TString UserName; + TString PasswordFile; + + bool UseMetadataCredentials = false; + TString YCToken; + TString YCTokenFile; + TString SaKeyFile; + TString IamEndpoint; + const TClientSettings& Settings; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp index 4dca60ba04..18aa3a65e5 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.cpp @@ -1,96 +1,96 @@ -#include "ydb_service_discovery.h" - -namespace NYdb { -namespace NConsoleClient { - -TCommandDiscovery::TCommandDiscovery() - : TClientCommandTree("discovery", {}, "Discovery service operations") -{ +#include "ydb_service_discovery.h" + +namespace NYdb { +namespace NConsoleClient { + +TCommandDiscovery::TCommandDiscovery() + : TClientCommandTree("discovery", {}, "Discovery service operations") +{ AddCommand(std::make_unique<TCommandListEndpoints>()); AddCommand(std::make_unique<TCommandWhoAmI>()); -} - -TCommandListEndpoints::TCommandListEndpoints() - : TYdbSimpleCommand("list", {}, "List endpoints") -{} - -void TCommandListEndpoints::Config(TConfig& config) { - TYdbSimpleCommand::Config(config); - config.SetFreeArgsNum(0); -} - -int TCommandListEndpoints::Run(TConfig& config) { - NDiscovery::TDiscoveryClient client(CreateDriver(config)); - NDiscovery::TListEndpointsResult result = client.ListEndpoints( - FillSettings(NDiscovery::TListEndpointsSettings()) - ).GetValueSync(); - ThrowOnError(result); - PrintResponse(result); - return EXIT_SUCCESS; -} - -void TCommandListEndpoints::PrintResponse(NDiscovery::TListEndpointsResult& result) { - const TVector<NDiscovery::TEndpointInfo>& endpoints = result.GetEndpointsInfo(); - if (endpoints.size()) { - for (auto& endpoint : endpoints) { +} + +TCommandListEndpoints::TCommandListEndpoints() + : TYdbSimpleCommand("list", {}, "List endpoints") +{} + +void TCommandListEndpoints::Config(TConfig& config) { + TYdbSimpleCommand::Config(config); + config.SetFreeArgsNum(0); +} + +int TCommandListEndpoints::Run(TConfig& config) { + NDiscovery::TDiscoveryClient client(CreateDriver(config)); + NDiscovery::TListEndpointsResult result = client.ListEndpoints( + FillSettings(NDiscovery::TListEndpointsSettings()) + ).GetValueSync(); + ThrowOnError(result); + PrintResponse(result); + return EXIT_SUCCESS; +} + +void TCommandListEndpoints::PrintResponse(NDiscovery::TListEndpointsResult& result) { + const TVector<NDiscovery::TEndpointInfo>& endpoints = result.GetEndpointsInfo(); + if (endpoints.size()) { + for (auto& endpoint : endpoints) { if (endpoint.Ssl) { Cout << "grpcs://"; } else { Cout << "grpc://"; } - Cout << endpoint.Address << ":" << endpoint.Port; - if (endpoint.Location) { - Cout << " [" << endpoint.Location << "]"; - } - for (const auto& service : endpoint.Services) { - Cout << " #" << service; - } - Cout << Endl; - } - } else { - Cout << "Endpoint list Is empty." << Endl; - } -} - -TCommandWhoAmI::TCommandWhoAmI() - : TYdbSimpleCommand("whoami", {}, "Who am I?") -{} - -void TCommandWhoAmI::Config(TConfig& config) { - TYdbSimpleCommand::Config(config); - config.Opts->AddLongOption('g', "groups", "With groups").NoArgument().SetFlag(&WithGroups); - config.SetFreeArgsNum(0); -} - -int TCommandWhoAmI::Run(TConfig& config) { - auto driver = CreateDriver(config); - NDiscovery::TDiscoveryClient client(driver); - NDiscovery::TWhoAmIResult result = client.WhoAmI( - FillSettings(NDiscovery::TWhoAmISettings().WithGroups(WithGroups)) - ).GetValueSync(); - ThrowOnError(result); - PrintResponse(result); - driver.Stop(true); - return EXIT_SUCCESS; -} - -void TCommandWhoAmI::PrintResponse(NDiscovery::TWhoAmIResult& result) { - const TString& userName = result.GetUserName(); - if (userName) { - Cout << "User SID: " << userName << Endl; - if (WithGroups) { - const TVector<TString>& groups = result.GetGroups(); - if (groups.size() > 0) { - Cout << Endl << "Group SIDs:" << Endl; - for (const TString& group : groups) { - Cout << group << Endl; - } - } else { - Cout << Endl << "User has no groups" << Endl; - } - } - } -} - -} -} + Cout << endpoint.Address << ":" << endpoint.Port; + if (endpoint.Location) { + Cout << " [" << endpoint.Location << "]"; + } + for (const auto& service : endpoint.Services) { + Cout << " #" << service; + } + Cout << Endl; + } + } else { + Cout << "Endpoint list Is empty." << Endl; + } +} + +TCommandWhoAmI::TCommandWhoAmI() + : TYdbSimpleCommand("whoami", {}, "Who am I?") +{} + +void TCommandWhoAmI::Config(TConfig& config) { + TYdbSimpleCommand::Config(config); + config.Opts->AddLongOption('g', "groups", "With groups").NoArgument().SetFlag(&WithGroups); + config.SetFreeArgsNum(0); +} + +int TCommandWhoAmI::Run(TConfig& config) { + auto driver = CreateDriver(config); + NDiscovery::TDiscoveryClient client(driver); + NDiscovery::TWhoAmIResult result = client.WhoAmI( + FillSettings(NDiscovery::TWhoAmISettings().WithGroups(WithGroups)) + ).GetValueSync(); + ThrowOnError(result); + PrintResponse(result); + driver.Stop(true); + return EXIT_SUCCESS; +} + +void TCommandWhoAmI::PrintResponse(NDiscovery::TWhoAmIResult& result) { + const TString& userName = result.GetUserName(); + if (userName) { + Cout << "User SID: " << userName << Endl; + if (WithGroups) { + const TVector<TString>& groups = result.GetGroups(); + if (groups.size() > 0) { + Cout << Endl << "Group SIDs:" << Endl; + for (const TString& group : groups) { + Cout << group << Endl; + } + } else { + Cout << Endl << "User has no groups" << Endl; + } + } + } +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h index 469de80a99..e7b826b8f6 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_discovery.h @@ -1,39 +1,39 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/ydb_discovery/discovery.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandDiscovery : public TClientCommandTree { -public: - TCommandDiscovery(); -}; - -class TCommandListEndpoints : public TYdbSimpleCommand { -public: - TCommandListEndpoints(); - void Config(TConfig& config) override; - int Run(TConfig& config) override; - -private: - void PrintResponse(NDiscovery::TListEndpointsResult& result); -}; - -class TCommandWhoAmI : public TYdbSimpleCommand { -public: - TCommandWhoAmI(); - void Config(TConfig& config) override; - int Run(TConfig& config) override; - -private: - void PrintResponse(NDiscovery::TWhoAmIResult& result); - - bool WithGroups = false; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +class TCommandDiscovery : public TClientCommandTree { +public: + TCommandDiscovery(); +}; + +class TCommandListEndpoints : public TYdbSimpleCommand { +public: + TCommandListEndpoints(); + void Config(TConfig& config) override; + int Run(TConfig& config) override; + +private: + void PrintResponse(NDiscovery::TListEndpointsResult& result); +}; + +class TCommandWhoAmI : public TYdbSimpleCommand { +public: + TCommandWhoAmI(); + void Config(TConfig& config) override; + int Run(TConfig& config) override; + +private: + void PrintResponse(NDiscovery::TWhoAmIResult& result); + + bool WithGroups = false; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp index e37895b690..f997343dc8 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp @@ -68,12 +68,12 @@ namespace { } // anonymous namespace -TCommandExport::TCommandExport(bool useExportToYt) +TCommandExport::TCommandExport(bool useExportToYt) : TClientCommandTree("export", {}, "Export service operations") { - if (useExportToYt) { + if (useExportToYt) { AddCommand(std::make_unique<TCommandExportToYt>()); - } + } AddCommand(std::make_unique<TCommandExportToS3>()); } @@ -132,13 +132,13 @@ void TCommandExportToYt::Config(TConfig& config) { .NoArgument().StoreTrue(&UseTypeV3); AddJsonOption(config); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); } void TCommandExportToYt::Parse(TConfig& config) { TClientCommand::Parse(config); - ParseFormats(); + ParseFormats(); ParseYtProxy(config, "proxy"); ParseYtToken(config, "token"); @@ -263,13 +263,13 @@ void TCommandExportToS3::Config(TConfig& config) { .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries); AddJsonOption(config); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); } void TCommandExportToS3::Parse(TConfig& config) { TClientCommand::Parse(config); - ParseFormats(); + ParseFormats(); ParseAwsAccessKey(config, "access-key"); ParseAwsSecretKey(config, "secret-key"); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h index 944eb8de73..6f842321ec 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h @@ -16,7 +16,7 @@ namespace NConsoleClient { class TCommandExport : public TClientCommandTree { public: - TCommandExport(bool useExportToYt); + TCommandExport(bool useExportToYt); }; class TCommandExportToYt : public TYdbOperationCommand, diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp index a3429c4173..45b694cf94 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.cpp @@ -73,15 +73,15 @@ TCommandGetOperation::TCommandGetOperation() void TCommandGetOperation::Config(TConfig& config) { TCommandWithOperationId::Config(config); AddJsonOption(config); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); +} + +void TCommandGetOperation::Parse(TConfig& config) { + TCommandWithOperationId::Parse(config); + ParseFormats(); } -void TCommandGetOperation::Parse(TConfig& config) { - TCommandWithOperationId::Parse(config); - ParseFormats(); -} - int TCommandGetOperation::Run(TConfig& config) { NOperation::TOperationClient client(CreateDriver(config)); @@ -129,17 +129,17 @@ int TCommandForgetOperation::Run(TConfig& config) { return EXIT_SUCCESS; } -void TCommandListOperations::InitializeKindToHandler(TConfig& config) { - KindToHandler = { - {"export/s3", &ListOperations<NExport::TExportToS3Response>}, - {"import/s3", &ListOperations<NImport::TImportFromS3Response>}, - {"buildindex", &ListOperations<NTable::TBuildIndexOperation>}, - }; - if (config.UseExportToYt) { - KindToHandler.emplace("export", &ListOperations<NExport::TExportToYtResponse>); // deprecated - KindToHandler.emplace("export/yt", &ListOperations<NExport::TExportToYtResponse>); - } -} +void TCommandListOperations::InitializeKindToHandler(TConfig& config) { + KindToHandler = { + {"export/s3", &ListOperations<NExport::TExportToS3Response>}, + {"import/s3", &ListOperations<NImport::TImportFromS3Response>}, + {"buildindex", &ListOperations<NTable::TBuildIndexOperation>}, + }; + if (config.UseExportToYt) { + KindToHandler.emplace("export", &ListOperations<NExport::TExportToYtResponse>); // deprecated + KindToHandler.emplace("export/yt", &ListOperations<NExport::TExportToYtResponse>); + } +} TString TCommandListOperations::KindChoices() { TStringBuilder help; @@ -157,22 +157,22 @@ TString TCommandListOperations::KindChoices() { } TCommandListOperations::TCommandListOperations() - : TYdbCommand("list", {}, "List operations of specified kind") + : TYdbCommand("list", {}, "List operations of specified kind") { } void TCommandListOperations::Config(TConfig& config) { TYdbCommand::Config(config); - InitializeKindToHandler(config); - + InitializeKindToHandler(config); + config.Opts->AddLongOption('s', "page-size", "Page size") .RequiredArgument("NUM").StoreResult(&PageSize); config.Opts->AddLongOption('t', "page-token", "Page token") .RequiredArgument("STRING").StoreResult(&PageToken); AddJsonOption(config); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<kind>", KindChoices()); @@ -180,7 +180,7 @@ void TCommandListOperations::Config(TConfig& config) { void TCommandListOperations::Parse(TConfig& config) { TYdbCommand::Parse(config); - ParseFormats(); + ParseFormats(); Kind = config.ParseResult->GetFreeArgs()[0]; if (!KindToHandler.contains(Kind)) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h index dd6326ee80..238901a88b 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_operation.h @@ -32,7 +32,7 @@ class TCommandGetOperation : public TCommandWithOperationId, public: TCommandGetOperation(); virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; + virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; }; @@ -53,8 +53,8 @@ class TCommandListOperations : public TYdbCommand, using THandler = std::function<void(NOperation::TOperationClient&, ui64, const TString&, EOutputFormat)>; - void InitializeKindToHandler(TConfig& config); - TString KindChoices(); + void InitializeKindToHandler(TConfig& config); + TString KindChoices(); public: TCommandListOperations(); @@ -66,7 +66,7 @@ private: TString Kind; ui64 PageSize = 0; TString PageToken; - THashMap<TString, THandler> KindToHandler; + THashMap<TString, THandler> KindToHandler; }; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index 09efe1ac41..a6f81fe432 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -1,192 +1,192 @@ -#include "ydb_service_scheme.h" - +#include "ydb_service_scheme.h" + #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/lib/ydb_cli/common/tabbed_table.h> #include <ydb/public/lib/ydb_cli/common/print_utils.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> - -namespace NYdb { -namespace NConsoleClient { - -TCommandScheme::TCommandScheme() - : TClientCommandTree("scheme", {}, "Scheme service operations") -{ + +namespace NYdb { +namespace NConsoleClient { + +TCommandScheme::TCommandScheme() + : TClientCommandTree("scheme", {}, "Scheme service operations") +{ AddCommand(std::make_unique<TCommandMakeDirectory>()); AddCommand(std::make_unique<TCommandRemoveDirectory>()); AddCommand(std::make_unique<TCommandDescribe>()); AddCommand(std::make_unique<TCommandList>()); AddCommand(std::make_unique<TCommandPermissions>()); -} - -TCommandMakeDirectory::TCommandMakeDirectory() - : TYdbOperationCommand("mkdir", std::initializer_list<TString>(), "Make directory") -{} - -void TCommandMakeDirectory::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<path>", "Path to create"); -} - -void TCommandMakeDirectory::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandMakeDirectory::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.MakeDirectory( - Path, - FillSettings(NScheme::TMakeDirectorySettings()) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandRemoveDirectory::TCommandRemoveDirectory() - : TYdbOperationCommand("rmdir", std::initializer_list<TString>(), "Remove directory") -{} - -void TCommandRemoveDirectory::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<path>", "Path to remove"); -} - -void TCommandRemoveDirectory::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandRemoveDirectory::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.RemoveDirectory( - Path, - FillSettings(NScheme::TRemoveDirectorySettings()) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -namespace { - TString EntryTypeToString(NScheme::ESchemeEntryType entry) { - switch (entry) { - case NScheme::ESchemeEntryType::Directory: - return "dir"; - case NScheme::ESchemeEntryType::Table: - return "table"; - case NScheme::ESchemeEntryType::PqGroup: - return "pq-group"; - case NScheme::ESchemeEntryType::SubDomain: - return "sub-domain"; - case NScheme::ESchemeEntryType::RtmrVolume: - return "rtmr-volume"; - case NScheme::ESchemeEntryType::BlockStoreVolume: - return "block-store-volume"; - case NScheme::ESchemeEntryType::CoordinationNode: - return "coordination-node"; - default: - return "unknown"; - } - } - - void PrintPermissions(const TVector<NScheme::TPermissions>& permissions) { - if (permissions.size()) { - for (const NScheme::TPermissions& permission : permissions) { - Cout << permission.Subject << ":"; - for (const TString& name : permission.PermissionNames) { - if (name != *permission.PermissionNames.begin()) { - Cout << ","; - } - Cout << name; - } - Cout << Endl; - } - } else { - Cout << "none" << Endl; - } - } - - void PrintAllPermissions( - const TString& owner, - const TVector<NScheme::TPermissions>& permissions, - const TVector<NScheme::TPermissions>& effectivePermissions - ) { - Cout << "Owner: " << owner << Endl << Endl << "Permissions: " << Endl; - PrintPermissions(permissions); - Cout << Endl << "Effective permissions: " << Endl; - PrintPermissions(effectivePermissions); - } - - void PrintEntryVerbose(const NScheme::TSchemeEntry& entry, bool permissions) { - Cout << "<" << EntryTypeToString(entry.Type) << "> " << entry.Name << Endl; - if (permissions) { - Cout << Endl; - PrintAllPermissions(entry.Owner, entry.Permissions, entry.EffectivePermissions); - } - } -} - -TCommandDescribe::TCommandDescribe() - : TYdbOperationCommand("describe", std::initializer_list<TString>(), "Show information about object at given object") -{} - -void TCommandDescribe::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - // Common options - config.Opts->AddLongOption("permissions", "Show owner and permissions").StoreTrue(&ShowPermissions); - - // Table options - config.Opts->AddLongOption("partition-boundaries", "[Table] Show partition key boundaries").StoreTrue(&ShowKeyShardBoundaries) - .AddLongName("shard-boundaries"); - config.Opts->AddLongOption("stats", "[Table] Show table statistics").StoreTrue(&ShowTableStats); - config.Opts->AddLongOption("partition-stats", "[Table] Show partition statistics").StoreTrue(&ShowPartitionStats); - - AddJsonOption(config, "(Deprecated, will be removed soon. Use --format option instead) [Table] Output in json format"); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); - config.Opts->MutuallyExclusive("json", "format"); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<path>", "Path to an object to describe"); -} - -void TCommandDescribe::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParseFormats(); - ParsePath(config, 0); -} - -int TCommandDescribe::Run(TConfig& config) { - TDriver driver = CreateDriver(config); - NScheme::TSchemeClient client(driver); - NScheme::TDescribePathResult result = client.DescribePath( - Path, - FillSettings(NScheme::TDescribePathSettings()) - ).GetValueSync(); - ThrowOnError(result); - return PrintPathResponse(driver, result); -} - -int TCommandDescribe::PrintPathResponse(TDriver& driver, const NScheme::TDescribePathResult& result) { - NScheme::TSchemeEntry entry = result.GetEntry(); - switch (entry.Type) { - case NScheme::ESchemeEntryType::Table: - return DescribeTable(driver); +} + +TCommandMakeDirectory::TCommandMakeDirectory() + : TYdbOperationCommand("mkdir", std::initializer_list<TString>(), "Make directory") +{} + +void TCommandMakeDirectory::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<path>", "Path to create"); +} + +void TCommandMakeDirectory::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandMakeDirectory::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.MakeDirectory( + Path, + FillSettings(NScheme::TMakeDirectorySettings()) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandRemoveDirectory::TCommandRemoveDirectory() + : TYdbOperationCommand("rmdir", std::initializer_list<TString>(), "Remove directory") +{} + +void TCommandRemoveDirectory::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<path>", "Path to remove"); +} + +void TCommandRemoveDirectory::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandRemoveDirectory::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.RemoveDirectory( + Path, + FillSettings(NScheme::TRemoveDirectorySettings()) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +namespace { + TString EntryTypeToString(NScheme::ESchemeEntryType entry) { + switch (entry) { + case NScheme::ESchemeEntryType::Directory: + return "dir"; + case NScheme::ESchemeEntryType::Table: + return "table"; + case NScheme::ESchemeEntryType::PqGroup: + return "pq-group"; + case NScheme::ESchemeEntryType::SubDomain: + return "sub-domain"; + case NScheme::ESchemeEntryType::RtmrVolume: + return "rtmr-volume"; + case NScheme::ESchemeEntryType::BlockStoreVolume: + return "block-store-volume"; + case NScheme::ESchemeEntryType::CoordinationNode: + return "coordination-node"; + default: + return "unknown"; + } + } + + void PrintPermissions(const TVector<NScheme::TPermissions>& permissions) { + if (permissions.size()) { + for (const NScheme::TPermissions& permission : permissions) { + Cout << permission.Subject << ":"; + for (const TString& name : permission.PermissionNames) { + if (name != *permission.PermissionNames.begin()) { + Cout << ","; + } + Cout << name; + } + Cout << Endl; + } + } else { + Cout << "none" << Endl; + } + } + + void PrintAllPermissions( + const TString& owner, + const TVector<NScheme::TPermissions>& permissions, + const TVector<NScheme::TPermissions>& effectivePermissions + ) { + Cout << "Owner: " << owner << Endl << Endl << "Permissions: " << Endl; + PrintPermissions(permissions); + Cout << Endl << "Effective permissions: " << Endl; + PrintPermissions(effectivePermissions); + } + + void PrintEntryVerbose(const NScheme::TSchemeEntry& entry, bool permissions) { + Cout << "<" << EntryTypeToString(entry.Type) << "> " << entry.Name << Endl; + if (permissions) { + Cout << Endl; + PrintAllPermissions(entry.Owner, entry.Permissions, entry.EffectivePermissions); + } + } +} + +TCommandDescribe::TCommandDescribe() + : TYdbOperationCommand("describe", std::initializer_list<TString>(), "Show information about object at given object") +{} + +void TCommandDescribe::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + // Common options + config.Opts->AddLongOption("permissions", "Show owner and permissions").StoreTrue(&ShowPermissions); + + // Table options + config.Opts->AddLongOption("partition-boundaries", "[Table] Show partition key boundaries").StoreTrue(&ShowKeyShardBoundaries) + .AddLongName("shard-boundaries"); + config.Opts->AddLongOption("stats", "[Table] Show table statistics").StoreTrue(&ShowTableStats); + config.Opts->AddLongOption("partition-stats", "[Table] Show partition statistics").StoreTrue(&ShowPartitionStats); + + AddJsonOption(config, "(Deprecated, will be removed soon. Use --format option instead) [Table] Output in json format"); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 }); + config.Opts->MutuallyExclusive("json", "format"); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<path>", "Path to an object to describe"); +} + +void TCommandDescribe::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParseFormats(); + ParsePath(config, 0); +} + +int TCommandDescribe::Run(TConfig& config) { + TDriver driver = CreateDriver(config); + NScheme::TSchemeClient client(driver); + NScheme::TDescribePathResult result = client.DescribePath( + Path, + FillSettings(NScheme::TDescribePathSettings()) + ).GetValueSync(); + ThrowOnError(result); + return PrintPathResponse(driver, result); +} + +int TCommandDescribe::PrintPathResponse(TDriver& driver, const NScheme::TDescribePathResult& result) { + NScheme::TSchemeEntry entry = result.GetEntry(); + switch (entry.Type) { + case NScheme::ESchemeEntryType::Table: + return DescribeTable(driver); case NScheme::ESchemeEntryType::PqGroup: return DescribeStream(driver); - default: - WarnAboutTableOptions(); - PrintEntryVerbose(entry, ShowPermissions); - } - return EXIT_SUCCESS; -} - + default: + WarnAboutTableOptions(); + PrintEntryVerbose(entry, ShowPermissions); + } + return EXIT_SUCCESS; +} + namespace { TString FormatCodecs(const TVector<NYdb::NPersQueue::ECodec> codecs) { if (codecs.empty()) { @@ -282,77 +282,77 @@ int TCommandDescribe::DescribeStream(TDriver& driver) { return PrintStreamResponse(describeResult); } -int TCommandDescribe::DescribeTable(TDriver& driver) { - NTable::TTableClient client(driver); - NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(sessionResult); - NTable::TDescribeTableResult result = sessionResult.GetSession().DescribeTable( - Path, - FillSettings( - NTable::TDescribeTableSettings() - .WithKeyShardBoundary(ShowKeyShardBoundaries) - .WithTableStatistics(ShowTableStats || ShowPartitionStats) - .WithPartitionStatistics(ShowPartitionStats) - ) - ).GetValueSync(); - ThrowOnError(result); - return PrintTableResponse(result); -} - -namespace { - void PrintColumns(const NTable::TTableDescription& tableDescription) { - if (!tableDescription.GetTableColumns().size()) { - return; - } - TPrettyTable table({ "Name", "Type", "Family", "Key" }, TPrettyTableConfig().WithoutRowDelimiters()); - - const TVector<TString>& keyColumns = tableDescription.GetPrimaryKeyColumns(); - for (const NTable::TTableColumn& column : tableDescription.GetTableColumns()) { - TString key = ""; - auto itKey = std::find(keyColumns.begin(), keyColumns.end(), column.Name); - if (itKey != keyColumns.end()) { - key = TStringBuilder() << "K" << itKey - keyColumns.begin(); - } - TString columnType; - try { - columnType = FormatType(column.Type); - } catch (yexception) { - columnType = "<unknown_type>"; - } - table.AddRow() - .Column(0, column.Name) - .Column(1, columnType) - .Column(2, column.Family) - .Column(3, key); - } - - Cout << table; - } - - // Temporary hack until KIKIMR-8635 goes to prod - TInstant CorrectTime(const TInstant& time) { - ui64 timeUs = time.MicroSeconds(); - if (timeUs < 2000000000000) { - timeUs *= 1000; - } - return TInstant::MicroSeconds(timeUs); - } - - void PrintIndexes(const NTable::TTableDescription& tableDescription) { - const TVector<NTable::TIndexDescription>& indexes = tableDescription.GetIndexDescriptions(); - if (!indexes.size()) { - return; - } - Cout << Endl << "Indexes: " << Endl; +int TCommandDescribe::DescribeTable(TDriver& driver) { + NTable::TTableClient client(driver); + NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); + ThrowOnError(sessionResult); + NTable::TDescribeTableResult result = sessionResult.GetSession().DescribeTable( + Path, + FillSettings( + NTable::TDescribeTableSettings() + .WithKeyShardBoundary(ShowKeyShardBoundaries) + .WithTableStatistics(ShowTableStats || ShowPartitionStats) + .WithPartitionStatistics(ShowPartitionStats) + ) + ).GetValueSync(); + ThrowOnError(result); + return PrintTableResponse(result); +} + +namespace { + void PrintColumns(const NTable::TTableDescription& tableDescription) { + if (!tableDescription.GetTableColumns().size()) { + return; + } + TPrettyTable table({ "Name", "Type", "Family", "Key" }, TPrettyTableConfig().WithoutRowDelimiters()); + + const TVector<TString>& keyColumns = tableDescription.GetPrimaryKeyColumns(); + for (const NTable::TTableColumn& column : tableDescription.GetTableColumns()) { + TString key = ""; + auto itKey = std::find(keyColumns.begin(), keyColumns.end(), column.Name); + if (itKey != keyColumns.end()) { + key = TStringBuilder() << "K" << itKey - keyColumns.begin(); + } + TString columnType; + try { + columnType = FormatType(column.Type); + } catch (yexception) { + columnType = "<unknown_type>"; + } + table.AddRow() + .Column(0, column.Name) + .Column(1, columnType) + .Column(2, column.Family) + .Column(3, key); + } + + Cout << table; + } + + // Temporary hack until KIKIMR-8635 goes to prod + TInstant CorrectTime(const TInstant& time) { + ui64 timeUs = time.MicroSeconds(); + if (timeUs < 2000000000000) { + timeUs *= 1000; + } + return TInstant::MicroSeconds(timeUs); + } + + void PrintIndexes(const NTable::TTableDescription& tableDescription) { + const TVector<NTable::TIndexDescription>& indexes = tableDescription.GetIndexDescriptions(); + if (!indexes.size()) { + return; + } + Cout << Endl << "Indexes: " << Endl; for (const auto& index : indexes) { Cout << index.GetIndexName() << " [" << index.GetIndexType() << "] Index columns: ("; const auto& columns = index.GetIndexColumns(); - for (auto colIt = columns.begin(); colIt != columns.end();) { - Cout << (*colIt); - if (++colIt != columns.end()) { - Cout << ","; - } - } + for (auto colIt = columns.begin(); colIt != columns.end();) { + Cout << (*colIt); + if (++colIt != columns.end()) { + Cout << ","; + } + } const auto& cover = index.GetDataColumns(); if (!cover) { @@ -367,101 +367,101 @@ namespace { } Cout << ")" << Endl; } - } - } - - void PrintStorageSettings(const NTable::TTableDescription& tableDescription) { - const NTable::TStorageSettings& settings = tableDescription.GetStorageSettings(); - const auto commitLog0 = settings.GetTabletCommitLog0(); - const auto commitLog1 = settings.GetTabletCommitLog1(); - const auto external = settings.GetExternal(); - const auto storeExternalBlobs = settings.GetStoreExternalBlobs(); - if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.Defined()) { - return; - } - Cout << Endl << "Storage settings: " << Endl; - if (commitLog0) { - Cout << "Internal channel 0 commit log storage pool: " << commitLog0.GetRef() << Endl; - } - if (commitLog1) { - Cout << "Internal channel 1 commit log storage pool: " << commitLog1.GetRef() << Endl; - } - if (external) { - Cout << "External blobs storage pool: " << external.GetRef() << Endl; - } - if (storeExternalBlobs) { - Cout << "Store large values in \"external blobs\": " - << (storeExternalBlobs.GetRef() ? "true" : "false") << Endl; - } - } - - void PrintColumnFamilies(const NTable::TTableDescription& tableDescription) { - if (!tableDescription.GetColumnFamilies()) { - return; - } - TPrettyTable table({ "Name", "Data", "Compression", "Keep in memory" }, - TPrettyTableConfig().WithoutRowDelimiters()); - - for (const NTable::TColumnFamilyDescription& family : tableDescription.GetColumnFamilies()) { - TMaybe<TString> data = family.GetData(); - TString compression; - if (family.GetCompression()) { - switch (family.GetCompression().GetRef()) { - case NTable::EColumnFamilyCompression::None: - compression = "None"; - break; - case NTable::EColumnFamilyCompression::LZ4: - compression = "LZ4"; - break; - default: - compression = TStringBuilder() << "unknown(" - << static_cast<size_t>(family.GetCompression().GetRef()) << ")"; - } - } - TStringBuilder keepInMemory; - if (family.GetKeepInMemory().Defined()) { - keepInMemory << keepInMemory << family.GetKeepInMemory().GetRef(); - } - table.AddRow() - .Column(0, family.GetName()) - .Column(1, data ? data.GetRef() : "") - .Column(2, compression) - .Column(3, keepInMemory); - } - Cout << Endl << "Column families: " << Endl; - Cout << table; - } - - void PrintAttributes(const NTable::TTableDescription& tableDescription) { - if (!tableDescription.GetAttributes().size()) { - return; - } - TPrettyTable table({ "Name", "Value" }, TPrettyTableConfig().WithoutRowDelimiters()); - - for (const auto& [name, value] : tableDescription.GetAttributes()) { - table.AddRow() - .Column(0, name) - .Column(1, value); - } - Cout << Endl << "Attributes: " << Endl; - Cout << table; - } - - void PrintTtlSettings(const NTable::TTableDescription& tableDescription) { - const auto& settings = tableDescription.GetTtlSettings(); - if (!settings) { - return; - } + } + } + + void PrintStorageSettings(const NTable::TTableDescription& tableDescription) { + const NTable::TStorageSettings& settings = tableDescription.GetStorageSettings(); + const auto commitLog0 = settings.GetTabletCommitLog0(); + const auto commitLog1 = settings.GetTabletCommitLog1(); + const auto external = settings.GetExternal(); + const auto storeExternalBlobs = settings.GetStoreExternalBlobs(); + if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.Defined()) { + return; + } + Cout << Endl << "Storage settings: " << Endl; + if (commitLog0) { + Cout << "Internal channel 0 commit log storage pool: " << commitLog0.GetRef() << Endl; + } + if (commitLog1) { + Cout << "Internal channel 1 commit log storage pool: " << commitLog1.GetRef() << Endl; + } + if (external) { + Cout << "External blobs storage pool: " << external.GetRef() << Endl; + } + if (storeExternalBlobs) { + Cout << "Store large values in \"external blobs\": " + << (storeExternalBlobs.GetRef() ? "true" : "false") << Endl; + } + } + + void PrintColumnFamilies(const NTable::TTableDescription& tableDescription) { + if (!tableDescription.GetColumnFamilies()) { + return; + } + TPrettyTable table({ "Name", "Data", "Compression", "Keep in memory" }, + TPrettyTableConfig().WithoutRowDelimiters()); + + for (const NTable::TColumnFamilyDescription& family : tableDescription.GetColumnFamilies()) { + TMaybe<TString> data = family.GetData(); + TString compression; + if (family.GetCompression()) { + switch (family.GetCompression().GetRef()) { + case NTable::EColumnFamilyCompression::None: + compression = "None"; + break; + case NTable::EColumnFamilyCompression::LZ4: + compression = "LZ4"; + break; + default: + compression = TStringBuilder() << "unknown(" + << static_cast<size_t>(family.GetCompression().GetRef()) << ")"; + } + } + TStringBuilder keepInMemory; + if (family.GetKeepInMemory().Defined()) { + keepInMemory << keepInMemory << family.GetKeepInMemory().GetRef(); + } + table.AddRow() + .Column(0, family.GetName()) + .Column(1, data ? data.GetRef() : "") + .Column(2, compression) + .Column(3, keepInMemory); + } + Cout << Endl << "Column families: " << Endl; + Cout << table; + } + + void PrintAttributes(const NTable::TTableDescription& tableDescription) { + if (!tableDescription.GetAttributes().size()) { + return; + } + TPrettyTable table({ "Name", "Value" }, TPrettyTableConfig().WithoutRowDelimiters()); + + for (const auto& [name, value] : tableDescription.GetAttributes()) { + table.AddRow() + .Column(0, name) + .Column(1, value); + } + Cout << Endl << "Attributes: " << Endl; + Cout << table; + } + + void PrintTtlSettings(const NTable::TTableDescription& tableDescription) { + const auto& settings = tableDescription.GetTtlSettings(); + if (!settings) { + return; + } Cout << Endl << "Ttl settings "; - switch (settings->GetMode()) { - case NTable::TTtlSettings::EMode::DateTypeColumn: - { + switch (settings->GetMode()) { + case NTable::TTtlSettings::EMode::DateTypeColumn: + { Cout << "(date type column):" << Endl; - const auto& dateTypeColumn = settings->GetDateTypeColumn(); - Cout << "Column name: " << dateTypeColumn.GetColumnName() << Endl; - Cout << "Expire after: " << dateTypeColumn.GetExpireAfter() << Endl; - break; - } + const auto& dateTypeColumn = settings->GetDateTypeColumn(); + Cout << "Column name: " << dateTypeColumn.GetColumnName() << Endl; + Cout << "Expire after: " << dateTypeColumn.GetExpireAfter() << Endl; + break; + } case NTable::TTtlSettings::EMode::ValueSinceUnixEpoch: { Cout << "(value since unix epoch):" << Endl; @@ -471,621 +471,621 @@ namespace { Cout << "Expire after: " << valueSinceEpoch.GetExpireAfter() << Endl; break; } - default: - NColorizer::TColors colors = NColorizer::AutoColors(Cout); + default: + NColorizer::TColors colors = NColorizer::AutoColors(Cout); Cout << "(unknown):" << Endl << colors.RedColor() << "Unknown ttl settings mode. Please update your version of YDB cli" - << colors.OldColor() << Endl; - } - } - - void PrintPartitioningSettings(const NTable::TTableDescription& tableDescription) { - const auto& settings = tableDescription.GetPartitioningSettings(); - const auto partBySize = settings.GetPartitioningBySize(); + << colors.OldColor() << Endl; + } + } + + void PrintPartitioningSettings(const NTable::TTableDescription& tableDescription) { + const auto& settings = tableDescription.GetPartitioningSettings(); + const auto partBySize = settings.GetPartitioningBySize(); const auto partByLoad = settings.GetPartitioningByLoad(); if (!partBySize.Defined() && !partByLoad.Defined()) { - return; - } - const auto partitionSizeMb = settings.GetPartitionSizeMb(); - const auto minPartitions = settings.GetMinPartitionsCount(); - const auto maxPartitions = settings.GetMaxPartitionsCount(); - Cout << Endl << "Auto partitioning settings: " << Endl; - Cout << "Partitioning by size: " << (partBySize.GetRef() ? "true" : "false") << Endl; + return; + } + const auto partitionSizeMb = settings.GetPartitionSizeMb(); + const auto minPartitions = settings.GetMinPartitionsCount(); + const auto maxPartitions = settings.GetMaxPartitionsCount(); + Cout << Endl << "Auto partitioning settings: " << Endl; + Cout << "Partitioning by size: " << (partBySize.GetRef() ? "true" : "false") << Endl; Cout << "Partitioning by load: " << (partByLoad.GetRef() ? "true" : "false") << Endl; if (partBySize.Defined() && partitionSizeMb) { - Cout << "Preferred partition size (Mb): " << partitionSizeMb << Endl; - } - if (minPartitions) { - Cout << "Min partitions count: " << minPartitions << Endl; - } - if (maxPartitions) { - Cout << "Max partitions count: " << maxPartitions << Endl; - } - } - - void PrintReadReplicasSettings(const NTable::TTableDescription& tableDescription) { - const auto& settings = tableDescription.GetReadReplicasSettings(); - if (!settings) { - return; - } - Cout << Endl << "Read replicas settings: " << Endl; - switch (settings->GetMode()) { - case NTable::TReadReplicasSettings::EMode::PerAz: - Cout << "Read replicas count in each AZ: " << settings->GetReadReplicasCount() << Endl; - break; - case NTable::TReadReplicasSettings::EMode::AnyAz: - Cout << "Read replicas total count in all AZs: " << settings->GetReadReplicasCount() << Endl; - break; - default: - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - Cout << colors.RedColor() << "Unknown read replicas settings mode. Please update your version of YDB cli" - << colors.OldColor() << Endl; - } - } - - void PrintStatistics(const NTable::TTableDescription& tableDescription) { - Cout << Endl << "Table stats:" << Endl; - Cout << "Partitions count: " << tableDescription.GetPartitionsCount() << Endl; - Cout << "Approximate number of rows: " << tableDescription.GetTableRows() << Endl; - Cout << "Approximate size of table: " << PrettySize(tableDescription.GetTableSize()) << Endl; - Cout << "Last modified: " << FormatTime(tableDescription.GetModificationTime()) << Endl; - Cout << "Created: " << FormatTime(CorrectTime(tableDescription.GetCreationTime())) << Endl; - } - - void PrintPartitionInfo(const NTable::TTableDescription& tableDescription, bool showBoundaries, bool showStats) { - const TVector<NTable::TKeyRange>& ranges = tableDescription.GetKeyRanges(); - const TVector<NTable::TPartitionStats>& stats = tableDescription.GetPartitionStats(); - if (showBoundaries) { - if (showStats) { - Cout << Endl << "Partitions info:" << Endl; - if (ranges.empty() && stats.empty()) { - Cout << "No data." << Endl; - return; - } - } else { - Cout << Endl << "Partitions key boundaries:" << Endl; - if (ranges.empty()) { - Cout << "No data." << Endl; - return; - } - } - } else { - Cout << Endl << "Partitions stats:" << Endl; - if (stats.empty()) { - Cout << "No data." << Endl; - return; - } - } - size_t rowsCount; - if (showBoundaries && showStats && ranges.size() != stats.size()) { - Cerr << "(!) Warning: partitions key boundaries size (" << ranges.size() - << ") mismatches partitions stats size (" << stats.size() << ")." << Endl; - rowsCount = Min(ranges.size(), stats.size()); - } else { - rowsCount = Max(ranges.size(), stats.size()); - } - TVector<TString> columnNames = { "#" }; - if (showBoundaries) { - columnNames.push_back(""); - columnNames.push_back("From"); - columnNames.push_back("To"); - columnNames.push_back(""); - } - if (showStats) { - columnNames.push_back("Rows"); - columnNames.push_back("Size"); - } - TPrettyTable table(columnNames, TPrettyTableConfig().WithoutRowDelimiters()); - for (size_t i = 0; i < rowsCount; ++i) { - auto& row = table.AddRow(); - size_t j = 0; - row.Column(j++, i + 1); - if (showBoundaries) { - const NTable::TKeyRange& keyRange = ranges[i]; - const TMaybe<NTable::TKeyBound>& from = keyRange.From(); - const TMaybe<NTable::TKeyBound>& to = keyRange.To(); - if (from.Defined()) { - const NTable::TKeyBound& bound = from.GetRef(); - if (bound.IsInclusive()) { - row.Column(j++, "["); - } else { - row.Column(j++, "("); - } - row.Column(j++, FormatValueJson(bound.GetValue(), EBinaryStringEncoding::Unicode)); - } else { - row.Column(j++, "("); - row.Column(j++, "-Inf"); - } - if (to.Defined()) { - const NTable::TKeyBound& bound = to.GetRef(); - row.Column(j++, FormatValueJson(bound.GetValue(), EBinaryStringEncoding::Unicode)); - if (bound.IsInclusive()) { - row.Column(j++, "]"); - } else { - row.Column(j++, ")"); - } - } else { - row.Column(j++, "+Inf"); - row.Column(j++, ")"); - } - } - if (showStats) { - const NTable::TPartitionStats& partStats = stats[i]; - row.Column(j++, partStats.Rows); - row.Column(j++, PrettySize(partStats.Size)); - } - } - Cout << table; - } -} - -int TCommandDescribe::PrintTableResponse(NTable::TDescribeTableResult& result) { - NTable::TTableDescription tableDescription = result.GetTableDescription(); - switch (OutputFormat) { - case EOutputFormat::Default: - case EOutputFormat::Pretty: - PrintResponsePretty(tableDescription); - break; - case EOutputFormat::Json: - Cerr << "Warning! Option --json is deprecated and will be removed soon. " - << "Use \"--format proto-json-base64\" option instead." << Endl; + Cout << "Preferred partition size (Mb): " << partitionSizeMb << Endl; + } + if (minPartitions) { + Cout << "Min partitions count: " << minPartitions << Endl; + } + if (maxPartitions) { + Cout << "Max partitions count: " << maxPartitions << Endl; + } + } + + void PrintReadReplicasSettings(const NTable::TTableDescription& tableDescription) { + const auto& settings = tableDescription.GetReadReplicasSettings(); + if (!settings) { + return; + } + Cout << Endl << "Read replicas settings: " << Endl; + switch (settings->GetMode()) { + case NTable::TReadReplicasSettings::EMode::PerAz: + Cout << "Read replicas count in each AZ: " << settings->GetReadReplicasCount() << Endl; + break; + case NTable::TReadReplicasSettings::EMode::AnyAz: + Cout << "Read replicas total count in all AZs: " << settings->GetReadReplicasCount() << Endl; + break; + default: + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + Cout << colors.RedColor() << "Unknown read replicas settings mode. Please update your version of YDB cli" + << colors.OldColor() << Endl; + } + } + + void PrintStatistics(const NTable::TTableDescription& tableDescription) { + Cout << Endl << "Table stats:" << Endl; + Cout << "Partitions count: " << tableDescription.GetPartitionsCount() << Endl; + Cout << "Approximate number of rows: " << tableDescription.GetTableRows() << Endl; + Cout << "Approximate size of table: " << PrettySize(tableDescription.GetTableSize()) << Endl; + Cout << "Last modified: " << FormatTime(tableDescription.GetModificationTime()) << Endl; + Cout << "Created: " << FormatTime(CorrectTime(tableDescription.GetCreationTime())) << Endl; + } + + void PrintPartitionInfo(const NTable::TTableDescription& tableDescription, bool showBoundaries, bool showStats) { + const TVector<NTable::TKeyRange>& ranges = tableDescription.GetKeyRanges(); + const TVector<NTable::TPartitionStats>& stats = tableDescription.GetPartitionStats(); + if (showBoundaries) { + if (showStats) { + Cout << Endl << "Partitions info:" << Endl; + if (ranges.empty() && stats.empty()) { + Cout << "No data." << Endl; + return; + } + } else { + Cout << Endl << "Partitions key boundaries:" << Endl; + if (ranges.empty()) { + Cout << "No data." << Endl; + return; + } + } + } else { + Cout << Endl << "Partitions stats:" << Endl; + if (stats.empty()) { + Cout << "No data." << Endl; + return; + } + } + size_t rowsCount; + if (showBoundaries && showStats && ranges.size() != stats.size()) { + Cerr << "(!) Warning: partitions key boundaries size (" << ranges.size() + << ") mismatches partitions stats size (" << stats.size() << ")." << Endl; + rowsCount = Min(ranges.size(), stats.size()); + } else { + rowsCount = Max(ranges.size(), stats.size()); + } + TVector<TString> columnNames = { "#" }; + if (showBoundaries) { + columnNames.push_back(""); + columnNames.push_back("From"); + columnNames.push_back("To"); + columnNames.push_back(""); + } + if (showStats) { + columnNames.push_back("Rows"); + columnNames.push_back("Size"); + } + TPrettyTable table(columnNames, TPrettyTableConfig().WithoutRowDelimiters()); + for (size_t i = 0; i < rowsCount; ++i) { + auto& row = table.AddRow(); + size_t j = 0; + row.Column(j++, i + 1); + if (showBoundaries) { + const NTable::TKeyRange& keyRange = ranges[i]; + const TMaybe<NTable::TKeyBound>& from = keyRange.From(); + const TMaybe<NTable::TKeyBound>& to = keyRange.To(); + if (from.Defined()) { + const NTable::TKeyBound& bound = from.GetRef(); + if (bound.IsInclusive()) { + row.Column(j++, "["); + } else { + row.Column(j++, "("); + } + row.Column(j++, FormatValueJson(bound.GetValue(), EBinaryStringEncoding::Unicode)); + } else { + row.Column(j++, "("); + row.Column(j++, "-Inf"); + } + if (to.Defined()) { + const NTable::TKeyBound& bound = to.GetRef(); + row.Column(j++, FormatValueJson(bound.GetValue(), EBinaryStringEncoding::Unicode)); + if (bound.IsInclusive()) { + row.Column(j++, "]"); + } else { + row.Column(j++, ")"); + } + } else { + row.Column(j++, "+Inf"); + row.Column(j++, ")"); + } + } + if (showStats) { + const NTable::TPartitionStats& partStats = stats[i]; + row.Column(j++, partStats.Rows); + row.Column(j++, PrettySize(partStats.Size)); + } + } + Cout << table; + } +} + +int TCommandDescribe::PrintTableResponse(NTable::TDescribeTableResult& result) { + NTable::TTableDescription tableDescription = result.GetTableDescription(); + switch (OutputFormat) { + case EOutputFormat::Default: + case EOutputFormat::Pretty: + PrintResponsePretty(tableDescription); + break; + case EOutputFormat::Json: + Cerr << "Warning! Option --json is deprecated and will be removed soon. " + << "Use \"--format proto-json-base64\" option instead." << Endl; [[fallthrough]]; - case EOutputFormat::ProtoJsonBase64: - return PrintResponseProtoJsonBase64(tableDescription); - default: - throw TMissUseException() << "This command doesn't support " << OutputFormat << " output format"; - } - return EXIT_SUCCESS; -} - -void TCommandDescribe::PrintResponsePretty(const NTable::TTableDescription& tableDescription) { - PrintColumns(tableDescription); - PrintIndexes(tableDescription); - PrintStorageSettings(tableDescription); - PrintColumnFamilies(tableDescription); - PrintAttributes(tableDescription); - PrintTtlSettings(tableDescription); - PrintPartitioningSettings(tableDescription); - if (tableDescription.GetKeyBloomFilter().Defined()) { - Cout << Endl << "Bloom filter by key: " - << (tableDescription.GetKeyBloomFilter().GetRef() ? "true" : "false") << Endl; - } - PrintReadReplicasSettings(tableDescription); - if (ShowPermissions) { - if (tableDescription.GetColumns().size()) { - Cout << Endl; - } - PrintAllPermissions( - tableDescription.GetOwner(), - tableDescription.GetPermissions(), - tableDescription.GetEffectivePermissions() - ); - } - if (ShowTableStats) { - PrintStatistics(tableDescription); - } - if (ShowKeyShardBoundaries || ShowPartitionStats) { - PrintPartitionInfo(tableDescription, ShowKeyShardBoundaries, ShowPartitionStats); - } -} - -int TCommandDescribe::PrintResponseProtoJsonBase64(const NTable::TTableDescription& tableDescription) { - TString json; - google::protobuf::util::JsonPrintOptions jsonOpts; - jsonOpts.preserve_proto_field_names = true; - auto convertStatus = google::protobuf::util::MessageToJsonString( - NYdb::TProtoAccessor::GetProto(tableDescription), - &json, - jsonOpts - ); + case EOutputFormat::ProtoJsonBase64: + return PrintResponseProtoJsonBase64(tableDescription); + default: + throw TMissUseException() << "This command doesn't support " << OutputFormat << " output format"; + } + return EXIT_SUCCESS; +} + +void TCommandDescribe::PrintResponsePretty(const NTable::TTableDescription& tableDescription) { + PrintColumns(tableDescription); + PrintIndexes(tableDescription); + PrintStorageSettings(tableDescription); + PrintColumnFamilies(tableDescription); + PrintAttributes(tableDescription); + PrintTtlSettings(tableDescription); + PrintPartitioningSettings(tableDescription); + if (tableDescription.GetKeyBloomFilter().Defined()) { + Cout << Endl << "Bloom filter by key: " + << (tableDescription.GetKeyBloomFilter().GetRef() ? "true" : "false") << Endl; + } + PrintReadReplicasSettings(tableDescription); + if (ShowPermissions) { + if (tableDescription.GetColumns().size()) { + Cout << Endl; + } + PrintAllPermissions( + tableDescription.GetOwner(), + tableDescription.GetPermissions(), + tableDescription.GetEffectivePermissions() + ); + } + if (ShowTableStats) { + PrintStatistics(tableDescription); + } + if (ShowKeyShardBoundaries || ShowPartitionStats) { + PrintPartitionInfo(tableDescription, ShowKeyShardBoundaries, ShowPartitionStats); + } +} + +int TCommandDescribe::PrintResponseProtoJsonBase64(const NTable::TTableDescription& tableDescription) { + TString json; + google::protobuf::util::JsonPrintOptions jsonOpts; + jsonOpts.preserve_proto_field_names = true; + auto convertStatus = google::protobuf::util::MessageToJsonString( + NYdb::TProtoAccessor::GetProto(tableDescription), + &json, + jsonOpts + ); if (convertStatus.ok()) { - Cout << json << Endl; - } else { - Cerr << "Error occurred while converting result proto to json" << Endl; - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -void TCommandDescribe::WarnAboutTableOptions() { - if (ShowKeyShardBoundaries || ShowTableStats || ShowPartitionStats || OutputFormat != EOutputFormat::Default) { - TVector<TString> options; - if (ShowKeyShardBoundaries) { - options.emplace_back("\"partition-boundaries\"(\"shard-boundaries\")"); - } - if (ShowTableStats) { - options.emplace_back("\"stats\""); - } - if (ShowPartitionStats) { - options.emplace_back("\"partition-stats\""); - } - if (OutputFormat != EOutputFormat::Default) { - options.emplace_back("\"json\""); - } - Cerr << "Note: \"" << Path << "\" is not a table. Option"; - if (options.size() > 1) { - Cerr << 's'; - } - for (auto& option : options) { - if (option != *options.begin()) { - Cerr << ','; - } - Cerr << ' ' << option; - } - Cerr << (options.size() > 1 ? " are" : " is") - << " used only for tables and thus " - << (options.size() > 1 ? "have" : "has") - << " no effect for this command." << Endl; - } -} - -TCommandList::TCommandList() - : TYdbOperationCommand("ls", std::initializer_list<TString>(), "Show information about objects inside given directory") -{} - -void TCommandList::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.Opts->AddCharOption('l', "List objects with detailed information") - .StoreTrue(&AdvancedMode); - config.Opts->AddCharOption('R', "List subdirectories recursively") - .StoreTrue(&Recursive); - - config.SetFreeArgsMax(1); - SetFreeArgTitle(0, "<path>", "Path to list"); -} - -void TCommandList::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0, true); -} - -int TCommandList::Run(TConfig& config) { - TDriver driver = CreateDriver(config); - NScheme::TSchemeClient client(driver); - NScheme::TListDirectoryResult result = client.ListDirectory( - Path, - FillSettings(NScheme::TListDirectorySettings()) - ).GetValueSync(); - ThrowOnError(result); - if (AdvancedMode) { - PrintResponseAdvanced(result, driver); - } else { - PrintResponse(result, Path, client); - } - Y_UNUSED(Recursive); - return EXIT_SUCCESS; -} - -void TCommandList::PrintResponse( - NScheme::TListDirectoryResult& result, - const TString& path, - NScheme::TSchemeClient& client -) { - TVector<NScheme::TSchemeEntry> children = result.GetChildren(); - if (children.size()) { - if (Recursive) { - Cout << path << ":" << Endl; - } - TAdaptiveTabbedTable table(children); - Cout << table; - if (Recursive) { - for (const auto& child : children) { - TString childPath = path + '/' + child.Name; - if (child.Type == NScheme::ESchemeEntryType::Directory) { - NScheme::TListDirectoryResult child_result = client.ListDirectory( - childPath, - FillSettings(NScheme::TListDirectorySettings()) - ).GetValueSync(); - ThrowOnError(child_result); - Cout << Endl; - PrintResponse(child_result, childPath, client); - } - } - } - } else { - NScheme::TSchemeEntry entry = result.GetEntry(); - if (Recursive) { - Cout << path << ":" << Endl; - } - if (entry.Type != NScheme::ESchemeEntryType::Directory) { - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - PrintSchemeEntry(Cout, entry, colors); - Cout << Endl; - } - } -} - -void TCommandList::PrintResponseAdvanced(NScheme::TListDirectoryResult& result, TDriver& driver) { - TVector<NScheme::TSchemeEntry> entries = result.GetChildren(); - bool oneExactEntry = false; - NTable::TTableClient tableClient(driver); - NScheme::TSchemeClient schemeClient(driver); - if (!entries.size()) { - if (result.GetEntry().Type != NScheme::ESchemeEntryType::Directory) { - entries.push_back(result.GetEntry()); - oneExactEntry = true; - } else { - return; - } - } - TPrettyTable table( - { "Type", "Owner", "Size", "Created", "Modified", "Name" }, - TPrettyTableConfig().WithoutRowDelimiters() - ); - - AddEntriesRecursive(TString(), entries, 0, table, oneExactEntry, tableClient, schemeClient); - - Cout << table; -} - -void TCommandList::AddEntriesRecursive( - const TString& path, - const TVector<NScheme::TSchemeEntry> entries, - size_t depth, - TPrettyTable& table, - bool oneExactEntry, - NTable::TTableClient& tableClient, - NScheme::TSchemeClient& schemeClient -) { - for (const auto& entry : entries) { - TString type = EntryTypeToString(entry.Type); - TString childRalativePath = path; - if (!oneExactEntry) { - childRalativePath += (childRalativePath ? "/" : "") + entry.Name; - } - TString childFullPath = Path + (childRalativePath ? "/" + childRalativePath : ""); - if (oneExactEntry) { - childRalativePath = entry.Name; - } - switch (entry.Type) { - case NScheme::ESchemeEntryType::Table: - { - NTable::TCreateSessionResult sessionResult = tableClient.GetSession( - NTable::TCreateSessionSettings() - ).GetValueSync(); - ThrowOnError(sessionResult); - - NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable( - childFullPath, - FillSettings( - NTable::TDescribeTableSettings().WithTableStatistics(true) - ) - ).GetValueSync(); - ThrowOnError(tableResult); - NTable::TTableDescription tableDescription = tableResult.GetTableDescription(); - - table.AddRow() - .Column(0, EntryTypeToString(entry.Type)) - .Column(1, entry.Owner) - .Column(2, PrettySize(tableDescription.GetTableSize())) - .Column(3, FormatTime(CorrectTime(tableDescription.GetCreationTime()))) - .Column(4, FormatTime(tableDescription.GetModificationTime())) - .Column(5, childRalativePath); - break; - } - default: - { - table.AddRow() - .Column(0, EntryTypeToString(entry.Type)) - .Column(1, entry.Owner) - .Column(2, "") - .Column(3, "") - .Column(4, "") - .Column(5, childRalativePath); - - if (Recursive && entry.Type == NScheme::ESchemeEntryType::Directory) { - NScheme::TListDirectoryResult child_result = schemeClient.ListDirectory( - childFullPath, - FillSettings(NScheme::TListDirectorySettings()) - ).GetValueSync(); - ThrowOnError(child_result); - AddEntriesRecursive(childRalativePath, child_result.GetChildren(), depth + 1, table, false, - tableClient, schemeClient); - } - break; - } // default block - } // switch - } // for -} - -TCommandPermissions::TCommandPermissions() - : TClientCommandTree("permissions", {}, "Modify permissions") -{ + Cout << json << Endl; + } else { + Cerr << "Error occurred while converting result proto to json" << Endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +void TCommandDescribe::WarnAboutTableOptions() { + if (ShowKeyShardBoundaries || ShowTableStats || ShowPartitionStats || OutputFormat != EOutputFormat::Default) { + TVector<TString> options; + if (ShowKeyShardBoundaries) { + options.emplace_back("\"partition-boundaries\"(\"shard-boundaries\")"); + } + if (ShowTableStats) { + options.emplace_back("\"stats\""); + } + if (ShowPartitionStats) { + options.emplace_back("\"partition-stats\""); + } + if (OutputFormat != EOutputFormat::Default) { + options.emplace_back("\"json\""); + } + Cerr << "Note: \"" << Path << "\" is not a table. Option"; + if (options.size() > 1) { + Cerr << 's'; + } + for (auto& option : options) { + if (option != *options.begin()) { + Cerr << ','; + } + Cerr << ' ' << option; + } + Cerr << (options.size() > 1 ? " are" : " is") + << " used only for tables and thus " + << (options.size() > 1 ? "have" : "has") + << " no effect for this command." << Endl; + } +} + +TCommandList::TCommandList() + : TYdbOperationCommand("ls", std::initializer_list<TString>(), "Show information about objects inside given directory") +{} + +void TCommandList::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.Opts->AddCharOption('l', "List objects with detailed information") + .StoreTrue(&AdvancedMode); + config.Opts->AddCharOption('R', "List subdirectories recursively") + .StoreTrue(&Recursive); + + config.SetFreeArgsMax(1); + SetFreeArgTitle(0, "<path>", "Path to list"); +} + +void TCommandList::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0, true); +} + +int TCommandList::Run(TConfig& config) { + TDriver driver = CreateDriver(config); + NScheme::TSchemeClient client(driver); + NScheme::TListDirectoryResult result = client.ListDirectory( + Path, + FillSettings(NScheme::TListDirectorySettings()) + ).GetValueSync(); + ThrowOnError(result); + if (AdvancedMode) { + PrintResponseAdvanced(result, driver); + } else { + PrintResponse(result, Path, client); + } + Y_UNUSED(Recursive); + return EXIT_SUCCESS; +} + +void TCommandList::PrintResponse( + NScheme::TListDirectoryResult& result, + const TString& path, + NScheme::TSchemeClient& client +) { + TVector<NScheme::TSchemeEntry> children = result.GetChildren(); + if (children.size()) { + if (Recursive) { + Cout << path << ":" << Endl; + } + TAdaptiveTabbedTable table(children); + Cout << table; + if (Recursive) { + for (const auto& child : children) { + TString childPath = path + '/' + child.Name; + if (child.Type == NScheme::ESchemeEntryType::Directory) { + NScheme::TListDirectoryResult child_result = client.ListDirectory( + childPath, + FillSettings(NScheme::TListDirectorySettings()) + ).GetValueSync(); + ThrowOnError(child_result); + Cout << Endl; + PrintResponse(child_result, childPath, client); + } + } + } + } else { + NScheme::TSchemeEntry entry = result.GetEntry(); + if (Recursive) { + Cout << path << ":" << Endl; + } + if (entry.Type != NScheme::ESchemeEntryType::Directory) { + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + PrintSchemeEntry(Cout, entry, colors); + Cout << Endl; + } + } +} + +void TCommandList::PrintResponseAdvanced(NScheme::TListDirectoryResult& result, TDriver& driver) { + TVector<NScheme::TSchemeEntry> entries = result.GetChildren(); + bool oneExactEntry = false; + NTable::TTableClient tableClient(driver); + NScheme::TSchemeClient schemeClient(driver); + if (!entries.size()) { + if (result.GetEntry().Type != NScheme::ESchemeEntryType::Directory) { + entries.push_back(result.GetEntry()); + oneExactEntry = true; + } else { + return; + } + } + TPrettyTable table( + { "Type", "Owner", "Size", "Created", "Modified", "Name" }, + TPrettyTableConfig().WithoutRowDelimiters() + ); + + AddEntriesRecursive(TString(), entries, 0, table, oneExactEntry, tableClient, schemeClient); + + Cout << table; +} + +void TCommandList::AddEntriesRecursive( + const TString& path, + const TVector<NScheme::TSchemeEntry> entries, + size_t depth, + TPrettyTable& table, + bool oneExactEntry, + NTable::TTableClient& tableClient, + NScheme::TSchemeClient& schemeClient +) { + for (const auto& entry : entries) { + TString type = EntryTypeToString(entry.Type); + TString childRalativePath = path; + if (!oneExactEntry) { + childRalativePath += (childRalativePath ? "/" : "") + entry.Name; + } + TString childFullPath = Path + (childRalativePath ? "/" + childRalativePath : ""); + if (oneExactEntry) { + childRalativePath = entry.Name; + } + switch (entry.Type) { + case NScheme::ESchemeEntryType::Table: + { + NTable::TCreateSessionResult sessionResult = tableClient.GetSession( + NTable::TCreateSessionSettings() + ).GetValueSync(); + ThrowOnError(sessionResult); + + NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable( + childFullPath, + FillSettings( + NTable::TDescribeTableSettings().WithTableStatistics(true) + ) + ).GetValueSync(); + ThrowOnError(tableResult); + NTable::TTableDescription tableDescription = tableResult.GetTableDescription(); + + table.AddRow() + .Column(0, EntryTypeToString(entry.Type)) + .Column(1, entry.Owner) + .Column(2, PrettySize(tableDescription.GetTableSize())) + .Column(3, FormatTime(CorrectTime(tableDescription.GetCreationTime()))) + .Column(4, FormatTime(tableDescription.GetModificationTime())) + .Column(5, childRalativePath); + break; + } + default: + { + table.AddRow() + .Column(0, EntryTypeToString(entry.Type)) + .Column(1, entry.Owner) + .Column(2, "") + .Column(3, "") + .Column(4, "") + .Column(5, childRalativePath); + + if (Recursive && entry.Type == NScheme::ESchemeEntryType::Directory) { + NScheme::TListDirectoryResult child_result = schemeClient.ListDirectory( + childFullPath, + FillSettings(NScheme::TListDirectorySettings()) + ).GetValueSync(); + ThrowOnError(child_result); + AddEntriesRecursive(childRalativePath, child_result.GetChildren(), depth + 1, table, false, + tableClient, schemeClient); + } + break; + } // default block + } // switch + } // for +} + +TCommandPermissions::TCommandPermissions() + : TClientCommandTree("permissions", {}, "Modify permissions") +{ AddCommand(std::make_unique<TCommandPermissionGrant>()); AddCommand(std::make_unique<TCommandPermissionRevoke>()); AddCommand(std::make_unique<TCommandPermissionSet>()); AddCommand(std::make_unique<TCommandChangeOwner>()); AddCommand(std::make_unique<TCommandPermissionClear>()); -} - -TCommandPermissionGrant::TCommandPermissionGrant() - : TYdbOperationCommand("grant", { "add" }, "Grant permission") -{} - -void TCommandPermissionGrant::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<path>", "Path to grant permissions to"); - SetFreeArgTitle(1, "<subject>", "Subject to grant permissions"); - - config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to grant") - .RequiredArgument("NAME").AppendTo(&PermissionsToGrant); -} - -void TCommandPermissionGrant::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); - Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { - throw TMissUseException() << "Missing required argument <subject>"; - } - if (!PermissionsToGrant.size()) { - throw TMissUseException() << "At least one permission to grant should be provided"; - } -} - -int TCommandPermissionGrant::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.ModifyPermissions( - Path, - FillSettings( - NScheme::TModifyPermissionsSettings() - .AddGrantPermissions({ Subject, PermissionsToGrant }) - ) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandPermissionRevoke::TCommandPermissionRevoke() - : TYdbOperationCommand("revoke", { "remove" }, "Revoke permission") -{} - -void TCommandPermissionRevoke::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<path>", "Path to revoke permissions to"); - SetFreeArgTitle(1, "<subject>", "Subject to revoke permissions"); - - config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to revoke") - .RequiredArgument("NAME").AppendTo(&PermissionsToRevoke); -} - -void TCommandPermissionRevoke::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); - Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { - throw TMissUseException() << "Missing required argument <subject>"; - } - if (!PermissionsToRevoke.size()) { - throw TMissUseException() << "At least one permission to revoke should be provided"; - } -} - -int TCommandPermissionRevoke::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.ModifyPermissions( - Path, - FillSettings( - NScheme::TModifyPermissionsSettings() - .AddRevokePermissions({ Subject, PermissionsToRevoke }) - ) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandPermissionSet::TCommandPermissionSet() - : TYdbOperationCommand("set", std::initializer_list<TString>(), "Set permissions") -{} - -void TCommandPermissionSet::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<path>", "Path to set permissions to"); - SetFreeArgTitle(1, "<subject>", "Subject to set permissions"); - - config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to set") - .RequiredArgument("NAME").AppendTo(&PermissionsToSet); -} - -void TCommandPermissionSet::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); - Subject = config.ParseResult->GetFreeArgs()[1]; - if (!Subject) { - throw TMissUseException() << "Missing required argument <subject>"; - } - if (!PermissionsToSet.size()) { - throw TMissUseException() << "At least one permission to set should be provided"; - } -} - -int TCommandPermissionSet::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.ModifyPermissions( - Path, - FillSettings( - NScheme::TModifyPermissionsSettings() - .AddSetPermissions({ Subject, PermissionsToSet }) - ) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandChangeOwner::TCommandChangeOwner() - : TYdbOperationCommand("chown", std::initializer_list<TString>(), "Change owner") -{} - -void TCommandChangeOwner::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(2); - SetFreeArgTitle(0, "<path>", "Path to change owner for"); - SetFreeArgTitle(1, "<owner>", "Owner to set"); -} - -void TCommandChangeOwner::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); - Owner = config.ParseResult->GetFreeArgs()[1]; - if (!Owner){ - throw TMissUseException() << "Missing required argument <owner>"; - } -} - -int TCommandChangeOwner::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.ModifyPermissions( - Path, - FillSettings( - NScheme::TModifyPermissionsSettings() - .AddChangeOwner(Owner) - ) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandPermissionClear::TCommandPermissionClear() - : TYdbOperationCommand("clear", std::initializer_list<TString>(), "Clear permissions") -{} - -void TCommandPermissionClear::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<path>", "Path to clear permissions to"); -} - -void TCommandPermissionClear::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandPermissionClear::Run(TConfig& config) { - NScheme::TSchemeClient client(CreateDriver(config)); - ThrowOnError( - client.ModifyPermissions( - Path, - FillSettings( - NScheme::TModifyPermissionsSettings() - .AddClearAcl() - ) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -} -} +} + +TCommandPermissionGrant::TCommandPermissionGrant() + : TYdbOperationCommand("grant", { "add" }, "Grant permission") +{} + +void TCommandPermissionGrant::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<path>", "Path to grant permissions to"); + SetFreeArgTitle(1, "<subject>", "Subject to grant permissions"); + + config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to grant") + .RequiredArgument("NAME").AppendTo(&PermissionsToGrant); +} + +void TCommandPermissionGrant::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); + Subject = config.ParseResult->GetFreeArgs()[1]; + if (!Subject) { + throw TMissUseException() << "Missing required argument <subject>"; + } + if (!PermissionsToGrant.size()) { + throw TMissUseException() << "At least one permission to grant should be provided"; + } +} + +int TCommandPermissionGrant::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.ModifyPermissions( + Path, + FillSettings( + NScheme::TModifyPermissionsSettings() + .AddGrantPermissions({ Subject, PermissionsToGrant }) + ) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandPermissionRevoke::TCommandPermissionRevoke() + : TYdbOperationCommand("revoke", { "remove" }, "Revoke permission") +{} + +void TCommandPermissionRevoke::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<path>", "Path to revoke permissions to"); + SetFreeArgTitle(1, "<subject>", "Subject to revoke permissions"); + + config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to revoke") + .RequiredArgument("NAME").AppendTo(&PermissionsToRevoke); +} + +void TCommandPermissionRevoke::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); + Subject = config.ParseResult->GetFreeArgs()[1]; + if (!Subject) { + throw TMissUseException() << "Missing required argument <subject>"; + } + if (!PermissionsToRevoke.size()) { + throw TMissUseException() << "At least one permission to revoke should be provided"; + } +} + +int TCommandPermissionRevoke::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.ModifyPermissions( + Path, + FillSettings( + NScheme::TModifyPermissionsSettings() + .AddRevokePermissions({ Subject, PermissionsToRevoke }) + ) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandPermissionSet::TCommandPermissionSet() + : TYdbOperationCommand("set", std::initializer_list<TString>(), "Set permissions") +{} + +void TCommandPermissionSet::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<path>", "Path to set permissions to"); + SetFreeArgTitle(1, "<subject>", "Subject to set permissions"); + + config.Opts->AddLongOption('p', "permission", "[At least one] Permission(s) to set") + .RequiredArgument("NAME").AppendTo(&PermissionsToSet); +} + +void TCommandPermissionSet::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); + Subject = config.ParseResult->GetFreeArgs()[1]; + if (!Subject) { + throw TMissUseException() << "Missing required argument <subject>"; + } + if (!PermissionsToSet.size()) { + throw TMissUseException() << "At least one permission to set should be provided"; + } +} + +int TCommandPermissionSet::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.ModifyPermissions( + Path, + FillSettings( + NScheme::TModifyPermissionsSettings() + .AddSetPermissions({ Subject, PermissionsToSet }) + ) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandChangeOwner::TCommandChangeOwner() + : TYdbOperationCommand("chown", std::initializer_list<TString>(), "Change owner") +{} + +void TCommandChangeOwner::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(2); + SetFreeArgTitle(0, "<path>", "Path to change owner for"); + SetFreeArgTitle(1, "<owner>", "Owner to set"); +} + +void TCommandChangeOwner::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); + Owner = config.ParseResult->GetFreeArgs()[1]; + if (!Owner){ + throw TMissUseException() << "Missing required argument <owner>"; + } +} + +int TCommandChangeOwner::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.ModifyPermissions( + Path, + FillSettings( + NScheme::TModifyPermissionsSettings() + .AddChangeOwner(Owner) + ) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandPermissionClear::TCommandPermissionClear() + : TYdbOperationCommand("clear", std::initializer_list<TString>(), "Clear permissions") +{} + +void TCommandPermissionClear::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<path>", "Path to clear permissions to"); +} + +void TCommandPermissionClear::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandPermissionClear::Run(TConfig& config) { + NScheme::TSchemeClient client(CreateDriver(config)); + ThrowOnError( + client.ModifyPermissions( + Path, + FillSettings( + NScheme::TModifyPermissionsSettings() + .AddClearAcl() + ) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h index b8fe4f8998..c87e4929f3 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h @@ -1,149 +1,149 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandScheme : public TClientCommandTree { -public: - TCommandScheme(); -}; - -class TCommandMakeDirectory : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandMakeDirectory(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -class TCommandRemoveDirectory : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandRemoveDirectory(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -class TCommandDescribe : public TYdbOperationCommand, public TCommandWithPath, public TCommandWithFormat { -public: - TCommandDescribe(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - int PrintPathResponse(TDriver& driver, const NScheme::TDescribePathResult& result); - int DescribeTable(TDriver& driver); - int PrintTableResponse(NTable::TDescribeTableResult& result); - void PrintResponsePretty(const NTable::TTableDescription& tableDescription); - int PrintResponseProtoJsonBase64(const NTable::TTableDescription& tableDescription); - void WarnAboutTableOptions(); - + +namespace NYdb { +namespace NConsoleClient { + +class TCommandScheme : public TClientCommandTree { +public: + TCommandScheme(); +}; + +class TCommandMakeDirectory : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandMakeDirectory(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +class TCommandRemoveDirectory : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandRemoveDirectory(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +class TCommandDescribe : public TYdbOperationCommand, public TCommandWithPath, public TCommandWithFormat { +public: + TCommandDescribe(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + int PrintPathResponse(TDriver& driver, const NScheme::TDescribePathResult& result); + int DescribeTable(TDriver& driver); + int PrintTableResponse(NTable::TDescribeTableResult& result); + void PrintResponsePretty(const NTable::TTableDescription& tableDescription); + int PrintResponseProtoJsonBase64(const NTable::TTableDescription& tableDescription); + void WarnAboutTableOptions(); + int DescribeStream(TDriver& driver); int PrintStreamResponse(const NYdb::NPersQueue::TDescribeTopicResult& result); int PrintStreamResponsePretty(const NYdb::NPersQueue::TDescribeTopicResult::TTopicSettings& settings); int PrintStreamResponseProtoJsonBase64(const NYdb::NPersQueue::TDescribeTopicResult& result); - // Common options - bool ShowPermissions = false; - // Table options - bool ShowKeyShardBoundaries = false; - bool ShowTableStats = false; - bool ShowPartitionStats = false; -}; - -class TCommandList : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandList(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - void PrintResponse(NScheme::TListDirectoryResult& result, const TString& path, NScheme::TSchemeClient& client); - void PrintResponseAdvanced(NScheme::TListDirectoryResult& result, TDriver& driver); - void AddEntriesRecursive( - const TString& path, - TVector<NScheme::TSchemeEntry> entries, - size_t depth, - TPrettyTable& table, - bool oneExactEntry, - NTable::TTableClient& tableClient, - NScheme::TSchemeClient& schemeClient - ); - - bool AdvancedMode = false; - bool Recursive = false; -}; - -class TCommandPermissions : public TClientCommandTree { -public: - TCommandPermissions(); -}; - -class TCommandPermissionGrant : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandPermissionGrant(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString Subject; - TVector<TString> PermissionsToGrant; -}; - -class TCommandPermissionRevoke : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandPermissionRevoke(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString Subject; - TVector<TString> PermissionsToRevoke; -}; - -class TCommandPermissionSet : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandPermissionSet(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString Subject; - TVector<TString> PermissionsToSet; -}; - -class TCommandChangeOwner : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandChangeOwner(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TString Owner; -}; - -class TCommandPermissionClear : public TYdbOperationCommand, public TCommandWithPath { -public: - TCommandPermissionClear(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -} -} + // Common options + bool ShowPermissions = false; + // Table options + bool ShowKeyShardBoundaries = false; + bool ShowTableStats = false; + bool ShowPartitionStats = false; +}; + +class TCommandList : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandList(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + void PrintResponse(NScheme::TListDirectoryResult& result, const TString& path, NScheme::TSchemeClient& client); + void PrintResponseAdvanced(NScheme::TListDirectoryResult& result, TDriver& driver); + void AddEntriesRecursive( + const TString& path, + TVector<NScheme::TSchemeEntry> entries, + size_t depth, + TPrettyTable& table, + bool oneExactEntry, + NTable::TTableClient& tableClient, + NScheme::TSchemeClient& schemeClient + ); + + bool AdvancedMode = false; + bool Recursive = false; +}; + +class TCommandPermissions : public TClientCommandTree { +public: + TCommandPermissions(); +}; + +class TCommandPermissionGrant : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandPermissionGrant(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString Subject; + TVector<TString> PermissionsToGrant; +}; + +class TCommandPermissionRevoke : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandPermissionRevoke(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString Subject; + TVector<TString> PermissionsToRevoke; +}; + +class TCommandPermissionSet : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandPermissionSet(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString Subject; + TVector<TString> PermissionsToSet; +}; + +class TCommandChangeOwner : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandChangeOwner(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TString Owner; +}; + +class TCommandPermissionClear : public TYdbOperationCommand, public TCommandWithPath { +public: + TCommandPermissionClear(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index 8215a40799..4a25c372ce 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -1,86 +1,86 @@ -#include "ydb_service_scripting.h" - +#include "ydb_service_scripting.h" + #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/lib/ydb_cli/common/query_stats.h> -#include <util/folder/path.h> -#include <util/folder/dirut.h> - -namespace NYdb { -namespace NConsoleClient { - -TCommandScripting::TCommandScripting() -: TClientCommandTree("scripting", {}, "Scripting service operations") -{ +#include <util/folder/path.h> +#include <util/folder/dirut.h> + +namespace NYdb { +namespace NConsoleClient { + +TCommandScripting::TCommandScripting() +: TClientCommandTree("scripting", {}, "Scripting service operations") +{ AddCommand(std::make_unique<TCommandExecuteYqlScript>()); -} - -TCommandExecuteYqlScript::TCommandExecuteYqlScript() - : TYdbOperationCommand("yql", {}, "Execute YQL script") -{} - -void TCommandExecuteYqlScript::Config(TConfig& config) { - AddExamplesOption(config); - TYdbOperationCommand::Config(config); +} + +TCommandExecuteYqlScript::TCommandExecuteYqlScript() + : TYdbOperationCommand("yql", {}, "Execute YQL script") +{} + +void TCommandExecuteYqlScript::Config(TConfig& config) { + AddExamplesOption(config); + TYdbOperationCommand::Config(config); config.Opts->AddLongOption("stats", "Collect statistics mode [none, basic, full]") .RequiredArgument("[String]").StoreResult(&CollectStatsMode); - config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); - config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); + config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); + config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); - AddParametersOption(config); + AddParametersOption(config); + + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); - AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::JsonUnicode, - EOutputFormat::JsonUnicodeArray, - EOutputFormat::JsonBase64, - EOutputFormat::JsonBase64Array + EOutputFormat::JsonUnicodeArray, + EOutputFormat::JsonBase64, + EOutputFormat::JsonBase64Array }); - config.SetFreeArgsNum(0); - - AddCommandExamples( - TExampleSetBuilder() - .BeginExample() - .Title("Execute script text") - .Text("ydb scripting yql -s \"SELECT 1\"") - .EndExample() - .BeginExample() - .Title("Execute script from file") - .Text("ydb scripting yql -f script_file.sql") - .EndExample() - .Build() - ); - - CheckExamples(config); -} - -void TCommandExecuteYqlScript::Parse(TConfig& config) { - TClientCommand::Parse(config); + config.SetFreeArgsNum(0); + + AddCommandExamples( + TExampleSetBuilder() + .BeginExample() + .Title("Execute script text") + .Text("ydb scripting yql -s \"SELECT 1\"") + .EndExample() + .BeginExample() + .Title("Execute script from file") + .Text("ydb scripting yql -f script_file.sql") + .EndExample() + .Build() + ); + + CheckExamples(config); +} + +void TCommandExecuteYqlScript::Parse(TConfig& config) { + TClientCommand::Parse(config); ParseFormats(); - if (!Script && !ScriptFile) { - throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") " - << "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided."; - } - if (Script && ScriptFile) { - throw TMissUseException() << "Both mutually exclusive options \"Text of script\" (\"--script\", \"-s\") " - << "and \"Path to file with script text\" (\"--file\", \"-f\") were provided."; - } - ParseParameters(); -} - -int TCommandExecuteYqlScript::Run(TConfig& config) { - if (ScriptFile) { - Script = ReadFromFile(ScriptFile, "script"); - } - NScripting::TScriptingClient client(CreateDriver(config)); + if (!Script && !ScriptFile) { + throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") " + << "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided."; + } + if (Script && ScriptFile) { + throw TMissUseException() << "Both mutually exclusive options \"Text of script\" (\"--script\", \"-s\") " + << "and \"Path to file with script text\" (\"--file\", \"-f\") were provided."; + } + ParseParameters(); +} + +int TCommandExecuteYqlScript::Run(TConfig& config) { + if (ScriptFile) { + Script = ReadFromFile(ScriptFile, "script"); + } + NScripting::TScriptingClient client(CreateDriver(config)); if (Explain) { NScripting::TExplainYqlRequestSettings settings; @@ -90,7 +90,7 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { ThrowOnError(result); PrintExplainResult(result); - } else { + } else { NScripting::TExecuteYqlRequestSettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); @@ -113,33 +113,33 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { ThrowOnError(result); PrintResponseHeader(result); PrintResponse(result); - } - - return EXIT_SUCCESS; -} - -void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& result) { - { - TResultSetPrinter printer(OutputFormat); - const TVector<TResultSet>& resultSets = result.GetResultSets(); - for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { - if (resultSetIt != resultSets.begin()) { - printer.Reset(); - } - printer.Print(*resultSetIt); - } - } // TResultSetPrinter destructor should be called before printing stats + } + + return EXIT_SUCCESS; +} + +void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& result) { + { + TResultSetPrinter printer(OutputFormat); + const TVector<TResultSet>& resultSets = result.GetResultSets(); + for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { + if (resultSetIt != resultSets.begin()) { + printer.Reset(); + } + printer.Print(*resultSetIt); + } + } // TResultSetPrinter destructor should be called before printing stats const TMaybe<NTable::TQueryStats>& stats = result.GetStats(); if (stats.Defined()) { Cout << Endl << "Statistics:" << Endl << stats->ToString(); } -} - +} + void TCommandExecuteYqlScript::PrintExplainResult(NScripting::TExplainYqlResult& result) { TQueryPlanPrinter queryPlanPrinter(OutputFormat); queryPlanPrinter.Print(result.GetPlan()); -} +} -} +} } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h index f69b561198..9911c6a744 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h @@ -1,37 +1,37 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/parameters.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandScripting : public TClientCommandTree { -public: - TCommandScripting(); -}; - -class TCommandExecuteYqlScript : public TYdbOperationCommand, public TCommandWithFormat, - public TCommandWithResponseHeaders, TCommandWithParameters -{ -public: - TCommandExecuteYqlScript(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - void PrintResponse(NScripting::TExecuteYqlResult& result); + +namespace NYdb { +namespace NConsoleClient { + +class TCommandScripting : public TClientCommandTree { +public: + TCommandScripting(); +}; + +class TCommandExecuteYqlScript : public TYdbOperationCommand, public TCommandWithFormat, + public TCommandWithResponseHeaders, TCommandWithParameters +{ +public: + TCommandExecuteYqlScript(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + void PrintResponse(NScripting::TExecuteYqlResult& result); void PrintExplainResult(NScripting::TExplainYqlResult& result); - -private: + +private: TString CollectStatsMode; - TString Script; - TString ScriptFile; + TString Script; + TString ScriptFile; bool Explain = false; -}; - -} -} +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index a2960e8dab..2f94b2fc03 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -1,26 +1,26 @@ -#include "ydb_service_table.h" - +#include "ydb_service_table.h" + #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> #include <ydb/public/lib/ydb_cli/common/query_stats.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - + #include <library/cpp/json/json_prettifier.h> #include <google/protobuf/util/json_util.h> - -#include <util/string/split.h> -#include <util/folder/path.h> -#include <util/folder/dirut.h> - + +#include <util/string/split.h> +#include <util/folder/path.h> +#include <util/folder/dirut.h> + #include <math.h> -namespace NYdb { -namespace NConsoleClient { - -TCommandTable::TCommandTable() - : TClientCommandTree("table", {}, "Table service operations") -{ +namespace NYdb { +namespace NConsoleClient { + +TCommandTable::TCommandTable() + : TClientCommandTree("table", {}, "Table service operations") +{ //AddCommand(std::make_unique<TCommandCreateTable>()); AddCommand(std::make_unique<TCommandDropTable>()); AddCommand(std::make_unique<TCommandQuery>()); @@ -28,15 +28,15 @@ TCommandTable::TCommandTable() AddCommand(std::make_unique<TCommandIndex>()); AddCommand(std::make_unique<TCommandAttribute>()); AddCommand(std::make_unique<TCommandTtl>()); -} - -TCommandQuery::TCommandQuery() - : TClientCommandTree("query", {}, "Query operations") -{ +} + +TCommandQuery::TCommandQuery() + : TClientCommandTree("query", {}, "Query operations") +{ AddCommand(std::make_unique<TCommandExecuteQuery>()); AddCommand(std::make_unique<TCommandExplain>()); -} - +} + TCommandIndex::TCommandIndex() : TClientCommandTree("index", {}, "Index operations") { @@ -65,417 +65,417 @@ TCommandIndexAdd::TCommandIndexAdd() AddCommand(std::make_unique<TCommandIndexAddGlobalAsync>()); } -TTableCommand::TTableCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) - : TYdbOperationCommand(name, aliases, description) -{} - -void TTableCommand::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - // TODO: Session options? -} - -NTable::TSession TTableCommand::GetSession(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - NTable::TCreateSessionResult result = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(result); - return result.GetSession(); -} - -namespace { - TList<std::pair<TString, EPrimitiveType>> YdbPrimitives = { - {"Bool", EPrimitiveType::Bool}, - {"Int8", EPrimitiveType::Int8}, - {"Uint8", EPrimitiveType::Uint8}, - {"Int16", EPrimitiveType::Int16}, - {"Uint16", EPrimitiveType::Uint16}, - {"Int32", EPrimitiveType::Int32}, - {"Uint32", EPrimitiveType::Uint32}, - {"Int64", EPrimitiveType::Int64}, - {"Uint64", EPrimitiveType::Uint64}, - {"Float", EPrimitiveType::Float}, - {"Double", EPrimitiveType::Double}, - {"Date", EPrimitiveType::Date}, - {"Datetime", EPrimitiveType::Datetime}, - {"Timestamp", EPrimitiveType::Timestamp}, - {"Interval", EPrimitiveType::Interval}, - {"TzDate", EPrimitiveType::TzDate}, - {"TzDatetime", EPrimitiveType::TzDatetime}, - {"TzTimestamp", EPrimitiveType::TzTimestamp}, - {"String", EPrimitiveType::String}, - {"Utf8", EPrimitiveType::Utf8}, - {"Yson", EPrimitiveType::Yson}, - {"Json", EPrimitiveType::Json}, +TTableCommand::TTableCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) + : TYdbOperationCommand(name, aliases, description) +{} + +void TTableCommand::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + // TODO: Session options? +} + +NTable::TSession TTableCommand::GetSession(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + NTable::TCreateSessionResult result = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); + ThrowOnError(result); + return result.GetSession(); +} + +namespace { + TList<std::pair<TString, EPrimitiveType>> YdbPrimitives = { + {"Bool", EPrimitiveType::Bool}, + {"Int8", EPrimitiveType::Int8}, + {"Uint8", EPrimitiveType::Uint8}, + {"Int16", EPrimitiveType::Int16}, + {"Uint16", EPrimitiveType::Uint16}, + {"Int32", EPrimitiveType::Int32}, + {"Uint32", EPrimitiveType::Uint32}, + {"Int64", EPrimitiveType::Int64}, + {"Uint64", EPrimitiveType::Uint64}, + {"Float", EPrimitiveType::Float}, + {"Double", EPrimitiveType::Double}, + {"Date", EPrimitiveType::Date}, + {"Datetime", EPrimitiveType::Datetime}, + {"Timestamp", EPrimitiveType::Timestamp}, + {"Interval", EPrimitiveType::Interval}, + {"TzDate", EPrimitiveType::TzDate}, + {"TzDatetime", EPrimitiveType::TzDatetime}, + {"TzTimestamp", EPrimitiveType::TzTimestamp}, + {"String", EPrimitiveType::String}, + {"Utf8", EPrimitiveType::Utf8}, + {"Yson", EPrimitiveType::Yson}, + {"Json", EPrimitiveType::Json}, {"Uuid", EPrimitiveType::Uuid}, {"JsonDocument", EPrimitiveType::JsonDocument}, {"DyNumber", EPrimitiveType::DyNumber}, - }; - - TString GetAllTypesString() { - TStringBuilder result; - for (auto& type : YdbPrimitives) { - result << type.first; - result << ", "; - } - result << "Decimal:<precision>:<scale>"; - return result; - } - - EPrimitiveType ConvertStringToYdbPrimitive(const TString& type) { - auto result = find_if( - YdbPrimitives.begin(), - YdbPrimitives.end(), - [&type](const std::pair<TString, EPrimitiveType>& it) { return it.first == type; } - ); - if (result == YdbPrimitives.end()) { - throw TMissUseException() << "Unknown type: " << type << Endl << "Allowed types: " << GetAllTypesString(); - } - return result->second; - } -} - -TCommandCreateTable::TCommandCreateTable() - : TTableCommand("create", {}, "Create new table") -{} - -void TCommandCreateTable::Config(TConfig& config) { - TTableCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "New table path"); - - config.Opts->AddLongOption('c', "Column", - TStringBuilder() << "[At least one] Column(s)." << Endl << "Allowed types : " << GetAllTypesString()) - .RequiredArgument("<name>:<type>[:<family>]").AppendTo(&Columns); - config.Opts->AddLongOption('p', "primary-key", "[At least one] Primary key(s)") - .RequiredArgument("NAME").AppendTo(&PrimaryKeys); - config.Opts->AddLongOption('i', "index", "Index(es).") - .RequiredArgument("<name>:<column1>[,<column2>...]").AppendTo(&Indexes); - config.Opts->AddLongOption("preset-name", "Create table preset name") - .RequiredArgument("NAME").StoreResult(&PresetName); - config.Opts->AddLongOption("execution-policy", "Execution policy preset name") - .RequiredArgument("NAME").StoreResult(&ExecutionPolicy); - config.Opts->AddLongOption("compaction-policy", "Compaction policy preset name") - .RequiredArgument("NAME").StoreResult(&CompactionPolicy); - config.Opts->AddLongOption("partitioning-policy", "Partitioning policy preset name") - .RequiredArgument("NAME").StoreResult(&PartitioningPolicy); - config.Opts->AddLongOption("auto-partitioning", "Auto-partitioning policy. [Disabled, AutoSplit, AutoSplitMerge]") - .RequiredArgument("[String]").StoreResult(&AutoPartitioning); - config.Opts->AddLongOption("uniform-partitions", "Enable uniform sharding using given shards number." - "The first components of primary key must have Uint32/Uint64 type.") - .RequiredArgument("[Uint64]").StoreResult(&UniformPartitions); - config.Opts->AddLongOption("replication-policy", "Replication policy preset name") - .RequiredArgument("NAME").StoreResult(&ReplicationPolicy); - config.Opts->AddLongOption("replicas-count", "If value is non-zero then it specifies a number of read-only " - "replicas to create for a table. Zero value means preset setting usage.") - .RequiredArgument("Ui32").StoreResult(&ReplicasCount); - config.Opts->AddLongOption("per-availability-zone", "If this feature in enabled then requested number of replicas " - "will be created in each availability zone.") - .StoreTrue(&CreatePerAvailabilityZone); - config.Opts->AddLongOption("allow-promotion", "If this feature in enabled then read-only replicas can be promoted " + }; + + TString GetAllTypesString() { + TStringBuilder result; + for (auto& type : YdbPrimitives) { + result << type.first; + result << ", "; + } + result << "Decimal:<precision>:<scale>"; + return result; + } + + EPrimitiveType ConvertStringToYdbPrimitive(const TString& type) { + auto result = find_if( + YdbPrimitives.begin(), + YdbPrimitives.end(), + [&type](const std::pair<TString, EPrimitiveType>& it) { return it.first == type; } + ); + if (result == YdbPrimitives.end()) { + throw TMissUseException() << "Unknown type: " << type << Endl << "Allowed types: " << GetAllTypesString(); + } + return result->second; + } +} + +TCommandCreateTable::TCommandCreateTable() + : TTableCommand("create", {}, "Create new table") +{} + +void TCommandCreateTable::Config(TConfig& config) { + TTableCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "New table path"); + + config.Opts->AddLongOption('c', "Column", + TStringBuilder() << "[At least one] Column(s)." << Endl << "Allowed types : " << GetAllTypesString()) + .RequiredArgument("<name>:<type>[:<family>]").AppendTo(&Columns); + config.Opts->AddLongOption('p', "primary-key", "[At least one] Primary key(s)") + .RequiredArgument("NAME").AppendTo(&PrimaryKeys); + config.Opts->AddLongOption('i', "index", "Index(es).") + .RequiredArgument("<name>:<column1>[,<column2>...]").AppendTo(&Indexes); + config.Opts->AddLongOption("preset-name", "Create table preset name") + .RequiredArgument("NAME").StoreResult(&PresetName); + config.Opts->AddLongOption("execution-policy", "Execution policy preset name") + .RequiredArgument("NAME").StoreResult(&ExecutionPolicy); + config.Opts->AddLongOption("compaction-policy", "Compaction policy preset name") + .RequiredArgument("NAME").StoreResult(&CompactionPolicy); + config.Opts->AddLongOption("partitioning-policy", "Partitioning policy preset name") + .RequiredArgument("NAME").StoreResult(&PartitioningPolicy); + config.Opts->AddLongOption("auto-partitioning", "Auto-partitioning policy. [Disabled, AutoSplit, AutoSplitMerge]") + .RequiredArgument("[String]").StoreResult(&AutoPartitioning); + config.Opts->AddLongOption("uniform-partitions", "Enable uniform sharding using given shards number." + "The first components of primary key must have Uint32/Uint64 type.") + .RequiredArgument("[Uint64]").StoreResult(&UniformPartitions); + config.Opts->AddLongOption("replication-policy", "Replication policy preset name") + .RequiredArgument("NAME").StoreResult(&ReplicationPolicy); + config.Opts->AddLongOption("replicas-count", "If value is non-zero then it specifies a number of read-only " + "replicas to create for a table. Zero value means preset setting usage.") + .RequiredArgument("Ui32").StoreResult(&ReplicasCount); + config.Opts->AddLongOption("per-availability-zone", "If this feature in enabled then requested number of replicas " + "will be created in each availability zone.") + .StoreTrue(&CreatePerAvailabilityZone); + config.Opts->AddLongOption("allow-promotion", "If this feature in enabled then read-only replicas can be promoted " "to leader.") - .StoreTrue(&AllowPromotion); -} - -void TCommandCreateTable::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); - if (!Columns.size()) { - throw TMissUseException() << "At least one column should be provided"; - } - if (!PrimaryKeys.size()) { - throw TMissUseException() << "At least one primary key should be provided"; - } -} - -int TCommandCreateTable::Run(TConfig& config) { - NTable::TTableBuilder builder; - for (const TString& column : Columns) { - TVector<TString> parts = StringSplitter(column).Split(':'); - if (parts[1] == "Decimal") { - if (parts.size() != 4 && parts.size() != 5) { - throw TMissUseException() << "Can't parse column \"" << column - << "\". Expected decimal format: \"<name>:Decimal:<precision>:<scale>[:<family>]\""; - } - TString family; - if (parts.size() == 5) { - family = parts[4]; - } - builder.AddNullableColumn( - parts[0], - TDecimalType(FromString<ui8>(parts[2]), FromString<ui8>(parts[3])), - family - ); - } else { - if (parts.size() != 2 && parts.size() != 3) { - throw TMissUseException() - << "Can't parse column \"" << column << "\". Expected format: \"<name>:<type>[:<family>]\""; - } - TString family; - if (parts.size() == 3) { - family = parts[2]; - } - builder.AddNullableColumn(parts[0], ConvertStringToYdbPrimitive(parts[1]), family); - } - } - builder.SetPrimaryKeyColumns(PrimaryKeys); - for (const TString& index : Indexes) { - TVector<TString> parts = StringSplitter(index).Split(':'); - if (parts.size() != 2 || !parts[0] || !parts[1]) { - throw TMissUseException() << "Can't parse index \"" << index - << "\". Need exactly one colon. Expected format: \"<name>:<column1>[,<column2>,...]\""; - } - TVector<TString> columns = StringSplitter(parts[1]).Split(','); - for (TString& column : columns) { - if (!column) { - throw TMissUseException() << "Can't parse index \"" << index - << "\". Empty column names found. Expected format: \"<name>:<column1>[,<column2>,...]\""; - } - } - builder.AddSecondaryIndex(parts[0], columns); - } - - NTable::TCreateTableSettings tableSettings = FillSettings(NTable::TCreateTableSettings()); - if (PresetName) { - tableSettings.PresetName(PresetName); - } - if (ExecutionPolicy) { - tableSettings.ExecutionPolicy(ExecutionPolicy); - } - if (CompactionPolicy) { - tableSettings.CompactionPolicy(CompactionPolicy); - } - - NTable::TPartitioningPolicy partitioningPolicy; - if (PartitioningPolicy) { - partitioningPolicy.PresetName(PartitioningPolicy); - } - if (AutoPartitioning) { - if (AutoPartitioning == "Disabled") { - partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::Disabled); - } else { - if (AutoPartitioning == "AutoSplit") { - partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::AutoSplit); - } else { - if (AutoPartitioning == "AutoSplitMerge") { - partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::AutoSplitMerge); - } else { - throw TMissUseException() << "Unknown auto-partitioning policy."; - } - } - } - } - if (UniformPartitions) { - partitioningPolicy.UniformPartitions(FromString<ui64>(UniformPartitions)); - } - tableSettings.PartitioningPolicy(partitioningPolicy); - - NTable::TReplicationPolicy replicationPolicy; - if (ReplicationPolicy) { - replicationPolicy.PresetName(ReplicationPolicy); - } - if (ReplicasCount) { - replicationPolicy.ReplicasCount(FromString<ui32>(ReplicasCount)); - } - if (CreatePerAvailabilityZone) { - replicationPolicy.CreatePerAvailabilityZone(CreatePerAvailabilityZone); - } - if (AllowPromotion) { - replicationPolicy.AllowPromotion(AllowPromotion); - } - - ThrowOnError( - GetSession(config).CreateTable( - Path, - builder.Build(), - std::move(tableSettings) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -TCommandDropTable::TCommandDropTable() - : TTableCommand("drop", {}, "Drop a table") -{} - -void TCommandDropTable::Config(TConfig& config) { - TTableCommand::Config(config); - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "table to drop path"); -} - -void TCommandDropTable::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParsePath(config, 0); -} - -int TCommandDropTable::Run(TConfig& config) { - ThrowOnError( - GetSession(config).DropTable( - Path, - FillSettings(NTable::TDropTableSettings()) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - -void TCommandQueryBase::CheckQueryOptions() const { - if (!Query && !QueryFile) { - throw TMissUseException() << "Neither \"Text of query\" (\"--query\", \"-q\") " - << "nor \"Path to file with query text\" (\"--file\", \"-f\") were provided."; - } - if (Query && QueryFile) { - throw TMissUseException() << "Both mutually exclusive options \"Text of query\" (\"--query\", \"-q\") " - << "and \"Path to file with query text\" (\"--file\", \"-f\") were provided."; - } -} - -void TCommandQueryBase::CheckQueryFile() { - if (QueryFile) { - Query = ReadFromFile(QueryFile, "query"); - } -} - -TCommandExecuteQuery::TCommandExecuteQuery() - : TTableCommand("execute", {"exec"}, "Execute query") -{} - -void TCommandExecuteQuery::Config(TConfig& config) { - TTableCommand::Config(config); - AddExamplesOption(config); - + .StoreTrue(&AllowPromotion); +} + +void TCommandCreateTable::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); + if (!Columns.size()) { + throw TMissUseException() << "At least one column should be provided"; + } + if (!PrimaryKeys.size()) { + throw TMissUseException() << "At least one primary key should be provided"; + } +} + +int TCommandCreateTable::Run(TConfig& config) { + NTable::TTableBuilder builder; + for (const TString& column : Columns) { + TVector<TString> parts = StringSplitter(column).Split(':'); + if (parts[1] == "Decimal") { + if (parts.size() != 4 && parts.size() != 5) { + throw TMissUseException() << "Can't parse column \"" << column + << "\". Expected decimal format: \"<name>:Decimal:<precision>:<scale>[:<family>]\""; + } + TString family; + if (parts.size() == 5) { + family = parts[4]; + } + builder.AddNullableColumn( + parts[0], + TDecimalType(FromString<ui8>(parts[2]), FromString<ui8>(parts[3])), + family + ); + } else { + if (parts.size() != 2 && parts.size() != 3) { + throw TMissUseException() + << "Can't parse column \"" << column << "\". Expected format: \"<name>:<type>[:<family>]\""; + } + TString family; + if (parts.size() == 3) { + family = parts[2]; + } + builder.AddNullableColumn(parts[0], ConvertStringToYdbPrimitive(parts[1]), family); + } + } + builder.SetPrimaryKeyColumns(PrimaryKeys); + for (const TString& index : Indexes) { + TVector<TString> parts = StringSplitter(index).Split(':'); + if (parts.size() != 2 || !parts[0] || !parts[1]) { + throw TMissUseException() << "Can't parse index \"" << index + << "\". Need exactly one colon. Expected format: \"<name>:<column1>[,<column2>,...]\""; + } + TVector<TString> columns = StringSplitter(parts[1]).Split(','); + for (TString& column : columns) { + if (!column) { + throw TMissUseException() << "Can't parse index \"" << index + << "\". Empty column names found. Expected format: \"<name>:<column1>[,<column2>,...]\""; + } + } + builder.AddSecondaryIndex(parts[0], columns); + } + + NTable::TCreateTableSettings tableSettings = FillSettings(NTable::TCreateTableSettings()); + if (PresetName) { + tableSettings.PresetName(PresetName); + } + if (ExecutionPolicy) { + tableSettings.ExecutionPolicy(ExecutionPolicy); + } + if (CompactionPolicy) { + tableSettings.CompactionPolicy(CompactionPolicy); + } + + NTable::TPartitioningPolicy partitioningPolicy; + if (PartitioningPolicy) { + partitioningPolicy.PresetName(PartitioningPolicy); + } + if (AutoPartitioning) { + if (AutoPartitioning == "Disabled") { + partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::Disabled); + } else { + if (AutoPartitioning == "AutoSplit") { + partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::AutoSplit); + } else { + if (AutoPartitioning == "AutoSplitMerge") { + partitioningPolicy.AutoPartitioning(NTable::EAutoPartitioningPolicy::AutoSplitMerge); + } else { + throw TMissUseException() << "Unknown auto-partitioning policy."; + } + } + } + } + if (UniformPartitions) { + partitioningPolicy.UniformPartitions(FromString<ui64>(UniformPartitions)); + } + tableSettings.PartitioningPolicy(partitioningPolicy); + + NTable::TReplicationPolicy replicationPolicy; + if (ReplicationPolicy) { + replicationPolicy.PresetName(ReplicationPolicy); + } + if (ReplicasCount) { + replicationPolicy.ReplicasCount(FromString<ui32>(ReplicasCount)); + } + if (CreatePerAvailabilityZone) { + replicationPolicy.CreatePerAvailabilityZone(CreatePerAvailabilityZone); + } + if (AllowPromotion) { + replicationPolicy.AllowPromotion(AllowPromotion); + } + + ThrowOnError( + GetSession(config).CreateTable( + Path, + builder.Build(), + std::move(tableSettings) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +TCommandDropTable::TCommandDropTable() + : TTableCommand("drop", {}, "Drop a table") +{} + +void TCommandDropTable::Config(TConfig& config) { + TTableCommand::Config(config); + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "table to drop path"); +} + +void TCommandDropTable::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParsePath(config, 0); +} + +int TCommandDropTable::Run(TConfig& config) { + ThrowOnError( + GetSession(config).DropTable( + Path, + FillSettings(NTable::TDropTableSettings()) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + +void TCommandQueryBase::CheckQueryOptions() const { + if (!Query && !QueryFile) { + throw TMissUseException() << "Neither \"Text of query\" (\"--query\", \"-q\") " + << "nor \"Path to file with query text\" (\"--file\", \"-f\") were provided."; + } + if (Query && QueryFile) { + throw TMissUseException() << "Both mutually exclusive options \"Text of query\" (\"--query\", \"-q\") " + << "and \"Path to file with query text\" (\"--file\", \"-f\") were provided."; + } +} + +void TCommandQueryBase::CheckQueryFile() { + if (QueryFile) { + Query = ReadFromFile(QueryFile, "query"); + } +} + +TCommandExecuteQuery::TCommandExecuteQuery() + : TTableCommand("execute", {"exec"}, "Execute query") +{} + +void TCommandExecuteQuery::Config(TConfig& config) { + TTableCommand::Config(config); + AddExamplesOption(config); + config.Opts->AddLongOption('t', "type", "Query type [data, scheme, scan]") - .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType); - config.Opts->AddLongOption("stats", "Collect statistics mode (for data & scan queries) [none, basic, full]") - .RequiredArgument("[String]").StoreResult(&CollectStatsMode); - config.Opts->AddCharOption('s', "Collect statistics in basic mode").StoreTrue(&BasicStats); - config.Opts->AddLongOption("tx-mode", "Transaction mode (for data queries only) [serializable-rw, online-ro, stale-ro]") - .RequiredArgument("[String]").DefaultValue("serializable-rw").StoreResult(&TxMode); - config.Opts->AddLongOption('q', "query", "Text of query to execute").RequiredArgument("[String]").StoreResult(&Query); - config.Opts->AddLongOption('f', "file", "Path to file with query text to execute") - .RequiredArgument("PATH").StoreResult(&QueryFile); - - AddParametersOption(config, "(for data & scan queries)"); - - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); - - + .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType); + config.Opts->AddLongOption("stats", "Collect statistics mode (for data & scan queries) [none, basic, full]") + .RequiredArgument("[String]").StoreResult(&CollectStatsMode); + config.Opts->AddCharOption('s', "Collect statistics in basic mode").StoreTrue(&BasicStats); + config.Opts->AddLongOption("tx-mode", "Transaction mode (for data queries only) [serializable-rw, online-ro, stale-ro]") + .RequiredArgument("[String]").DefaultValue("serializable-rw").StoreResult(&TxMode); + config.Opts->AddLongOption('q', "query", "Text of query to execute").RequiredArgument("[String]").StoreResult(&Query); + config.Opts->AddLongOption('f', "file", "Path to file with query text to execute") + .RequiredArgument("PATH").StoreResult(&QueryFile); + + AddParametersOption(config, "(for data & scan queries)"); + + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::JsonUnicode, - EOutputFormat::JsonUnicodeArray, - EOutputFormat::JsonBase64, - EOutputFormat::JsonBase64Array + EOutputFormat::JsonUnicodeArray, + EOutputFormat::JsonBase64, + EOutputFormat::JsonBase64Array }); - CheckExamples(config); + CheckExamples(config); + + config.SetFreeArgsNum(0); +} - config.SetFreeArgsNum(0); -} - -void TCommandExecuteQuery::Parse(TConfig& config) { - TClientCommand::Parse(config); +void TCommandExecuteQuery::Parse(TConfig& config) { + TClientCommand::Parse(config); ParseFormats(); - if (BasicStats && CollectStatsMode) { - throw TMissUseException() << "Both mutually exclusive options \"--stats\" and \"-s\" are provided."; - } - if (ParameterOptions.size() && QueryType == "scheme") { - throw TMissUseException() << "Scheme queries does not support parameters."; - } - ParseParameters(); - CheckQueryOptions(); -} - -int TCommandExecuteQuery::Run(TConfig& config) { - CheckQueryFile(); - if (QueryType) { - if (QueryType == "data") { - return ExecuteDataQuery(config); - } - if (QueryType == "scheme") { - return ExecuteSchemeQuery(config); - } + if (BasicStats && CollectStatsMode) { + throw TMissUseException() << "Both mutually exclusive options \"--stats\" and \"-s\" are provided."; + } + if (ParameterOptions.size() && QueryType == "scheme") { + throw TMissUseException() << "Scheme queries does not support parameters."; + } + ParseParameters(); + CheckQueryOptions(); +} + +int TCommandExecuteQuery::Run(TConfig& config) { + CheckQueryFile(); + if (QueryType) { + if (QueryType == "data") { + return ExecuteDataQuery(config); + } + if (QueryType == "scheme") { + return ExecuteSchemeQuery(config); + } if (QueryType == "scan") { return ExecuteScanQuery(config); } - } - throw TMissUseException() << "Unknown query type."; -} - -int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { + } + throw TMissUseException() << "Unknown query type."; +} + +int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { auto defaultStatsMode = BasicStats ? NTable::ECollectQueryStatsMode::Basic : NTable::ECollectQueryStatsMode::None; - NTable::TExecDataQuerySettings settings; + NTable::TExecDataQuerySettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, defaultStatsMode)); - NTable::TTxSettings txSettings; - if (TxMode) { - if (TxMode == "serializable-rw") { - txSettings = NTable::TTxSettings::SerializableRW(); - } else { - if (TxMode == "online-ro") { - txSettings = NTable::TTxSettings::OnlineRO(); - } else { - if (TxMode == "stale-ro") { - txSettings = NTable::TTxSettings::StaleRO(); - } else { - throw TMissUseException() << "Unknown transaction mode."; - } - } - } - } - NTable::TAsyncDataQueryResult asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = GetSession(config).ExecuteDataQuery( - Query, - NTable::TTxControl::BeginTx(txSettings).CommitTx(), - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); - } else { - asyncResult = GetSession(config).ExecuteDataQuery( - Query, - NTable::TTxControl::BeginTx(txSettings).CommitTx(), - FillSettings(settings) - ); - } - NTable::TDataQueryResult result = asyncResult.GetValueSync(); - ThrowOnError(result); - PrintDataQueryResponse(result); - return EXIT_SUCCESS; -} - -void TCommandExecuteQuery::PrintDataQueryResponse(NTable::TDataQueryResult& result) { - { - TResultSetPrinter printer(OutputFormat); - const TVector<TResultSet>& resultSets = result.GetResultSets(); - for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { - if (resultSetIt != resultSets.begin()) { - printer.Reset(); - } - printer.Print(*resultSetIt); - } - } // TResultSetPrinter destructor should be called before printing stats - - const TMaybe<NTable::TQueryStats>& stats = result.GetStats(); - if (stats.Defined()) { - Cout << Endl << "Statistics:" << Endl << stats->ToString(); - } -} - -int TCommandExecuteQuery::ExecuteSchemeQuery(TConfig& config) { - ThrowOnError( - GetSession(config).ExecuteSchemeQuery( - Query, - FillSettings(NTable::TExecSchemeQuerySettings()) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - + NTable::TTxSettings txSettings; + if (TxMode) { + if (TxMode == "serializable-rw") { + txSettings = NTable::TTxSettings::SerializableRW(); + } else { + if (TxMode == "online-ro") { + txSettings = NTable::TTxSettings::OnlineRO(); + } else { + if (TxMode == "stale-ro") { + txSettings = NTable::TTxSettings::StaleRO(); + } else { + throw TMissUseException() << "Unknown transaction mode."; + } + } + } + } + NTable::TAsyncDataQueryResult asyncResult; + if (Parameters.size()) { + auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); + asyncResult = GetSession(config).ExecuteDataQuery( + Query, + NTable::TTxControl::BeginTx(txSettings).CommitTx(), + BuildParams(validateResult.GetParameterTypes(), InputFormat), + FillSettings(settings) + ); + } else { + asyncResult = GetSession(config).ExecuteDataQuery( + Query, + NTable::TTxControl::BeginTx(txSettings).CommitTx(), + FillSettings(settings) + ); + } + NTable::TDataQueryResult result = asyncResult.GetValueSync(); + ThrowOnError(result); + PrintDataQueryResponse(result); + return EXIT_SUCCESS; +} + +void TCommandExecuteQuery::PrintDataQueryResponse(NTable::TDataQueryResult& result) { + { + TResultSetPrinter printer(OutputFormat); + const TVector<TResultSet>& resultSets = result.GetResultSets(); + for (auto resultSetIt = resultSets.begin(); resultSetIt != resultSets.end(); ++resultSetIt) { + if (resultSetIt != resultSets.begin()) { + printer.Reset(); + } + printer.Print(*resultSetIt); + } + } // TResultSetPrinter destructor should be called before printing stats + + const TMaybe<NTable::TQueryStats>& stats = result.GetStats(); + if (stats.Defined()) { + Cout << Endl << "Statistics:" << Endl << stats->ToString(); + } +} + +int TCommandExecuteQuery::ExecuteSchemeQuery(TConfig& config) { + ThrowOnError( + GetSession(config).ExecuteSchemeQuery( + Query, + FillSettings(NTable::TExecSchemeQuerySettings()) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { NTable::TTableClient client(CreateDriver(config)); @@ -483,19 +483,19 @@ int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { NTable::TStreamExecScanQuerySettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, defaultStatsMode)); - NTable::TAsyncScanQueryPartIterator asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.StreamExecuteScanQuery( - Query, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - settings - ); - } else { - asyncResult = client.StreamExecuteScanQuery(Query, settings); - } - - auto result = asyncResult.GetValueSync(); + NTable::TAsyncScanQueryPartIterator asyncResult; + if (Parameters.size()) { + auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); + asyncResult = client.StreamExecuteScanQuery( + Query, + BuildParams(validateResult.GetParameterTypes(), InputFormat), + settings + ); + } else { + asyncResult = client.StreamExecuteScanQuery(Query, settings); + } + + auto result = asyncResult.GetValueSync(); ThrowOnError(result); PrintScanQueryResponse(result); return EXIT_SUCCESS; @@ -503,54 +503,54 @@ int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) { SetInterruptHandlers(); - TStringStream statsStr; - { - TResultSetPrinter printer(OutputFormat, &IsInterrupted); - - while (!IsInterrupted()) { - auto streamPart = result.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - if (streamPart.EOS()) { - break; - } - ThrowOnError(streamPart); + TStringStream statsStr; + { + TResultSetPrinter printer(OutputFormat, &IsInterrupted); + + while (!IsInterrupted()) { + auto streamPart = result.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + if (streamPart.EOS()) { + break; + } + ThrowOnError(streamPart); } - if (streamPart.HasResultSet()) { - printer.Print(streamPart.GetResultSet()); - } + if (streamPart.HasResultSet()) { + printer.Print(streamPart.GetResultSet()); + } - if (streamPart.HasQueryStats()) { - const auto& queryStats = streamPart.GetQueryStats(); - statsStr << Endl << queryStats.ToString(false) << Endl; + if (streamPart.HasQueryStats()) { + const auto& queryStats = streamPart.GetQueryStats(); + statsStr << Endl << queryStats.ToString(false) << Endl; - auto plan = queryStats.GetPlan(); - if (plan) { - statsStr << "Full statistics:" << Endl << *plan << Endl; - } + auto plan = queryStats.GetPlan(); + if (plan) { + statsStr << "Full statistics:" << Endl << *plan << Endl; + } } } - } // TResultSetPrinter destructor should be called before printing stats - - if (statsStr.Size()) { - Cout << Endl << "Statistics:" << statsStr.Str(); + } // TResultSetPrinter destructor should be called before printing stats + + if (statsStr.Size()) { + Cout << Endl << "Statistics:" << statsStr.Str(); } - if (IsInterrupted()) { - Cerr << "<INTERRUPTED>" << Endl; - } + if (IsInterrupted()) { + Cerr << "<INTERRUPTED>" << Endl; + } } -TCommandExplain::TCommandExplain() - : TTableCommand("explain", {}, "Explain query") -{} - -void TCommandExplain::Config(TConfig& config) { - TTableCommand::Config(config); - - config.Opts->AddLongOption('q', "query", "Text of query to explain").RequiredArgument("[String]").StoreResult(&Query); - config.Opts->AddLongOption('f', "file", "Path to file with query text to explain") - .RequiredArgument("PATH").StoreResult(&QueryFile); +TCommandExplain::TCommandExplain() + : TTableCommand("explain", {}, "Explain query") +{} + +void TCommandExplain::Config(TConfig& config) { + TTableCommand::Config(config); + + config.Opts->AddLongOption('q', "query", "Text of query to explain").RequiredArgument("[String]").StoreResult(&Query); + config.Opts->AddLongOption('f', "file", "Path to file with query text to explain") + .RequiredArgument("PATH").StoreResult(&QueryFile); config.Opts->AddLongOption("ast", "Print query AST") .StoreTrue(&PrintAst); @@ -565,17 +565,17 @@ void TCommandExplain::Config(TConfig& config) { EOutputFormat::JsonBase64 }); - config.SetFreeArgsNum(0); -} - -void TCommandExplain::Parse(TConfig& config) { - TClientCommand::Parse(config); + config.SetFreeArgsNum(0); +} + +void TCommandExplain::Parse(TConfig& config) { + TClientCommand::Parse(config); ParseFormats(); - CheckQueryOptions(); -} - -int TCommandExplain::Run(TConfig& config) { - CheckQueryFile(); + CheckQueryOptions(); +} + +int TCommandExplain::Run(TConfig& config) { + CheckQueryFile(); TString planJson; TString ast; @@ -593,7 +593,7 @@ int TCommandExplain::Run(TConfig& config) { ThrowOnError(result); SetInterruptHandlers(); - while (!IsInterrupted()) { + while (!IsInterrupted()) { auto tablePart = result.ReadNext().GetValueSync(); if (!tablePart.IsSuccess()) { if (tablePart.EOS()) { @@ -608,10 +608,10 @@ int TCommandExplain::Run(TConfig& config) { } } - if (IsInterrupted()) { - Cerr << "<INTERRUPTED>" << Endl; - } - + if (IsInterrupted()) { + Cerr << "<INTERRUPTED>" << Endl; + } + } else if (QueryType == "data" && Analyze) { NTable::TExecDataQuerySettings settings; settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full); @@ -648,196 +648,196 @@ int TCommandExplain::Run(TConfig& config) { queryPlanPrinter.Print(planJson); } - return EXIT_SUCCESS; -} - -TCommandReadTable::TCommandReadTable() - : TYdbCommand("readtable", {}, "Stream read table") -{} - -void TCommandReadTable::Config(TConfig& config) { - TYdbCommand::Config(config); - - config.Opts->AddLongOption("ordered", "Result should be ordered by primary key") - .NoArgument().SetFlag(&Ordered); - config.Opts->AddLongOption("limit", "Limit result rows count") - .RequiredArgument("NUM").StoreResult(&RowLimit); - config.Opts->AddLongOption("columns", "Comma separated list of columns to read") - .RequiredArgument("CSV").StoreResult(&Columns); - config.Opts->AddLongOption("count-only", "Print only rows count") - .NoArgument().SetFlag(&CountOnly); - config.Opts->AddLongOption("from", "Key prefix value to start read from.\n" - " Format should be a json-string containing array of elements representing a tuple - key prefix.\n" - " Option \"--input-format\" defines how to parse binary strings.\n" - " Examples:\n" - " 1) using one column from PK:\n" - " --from [10] --to [100]\n" - " 2) using two columns from PK (forwarding strings in command line is OS-specific. Example for linux):\n" - " --from [10,\\\"OneWord\\\"] --to '[100,\"Two Words\"]'") - .RequiredArgument("JSON").StoreResult(&From); - config.Opts->AddLongOption("to", "Key prefix value to read until.\n" - " Same format as for \"--from\" option.") - .RequiredArgument("JSON").StoreResult(&To); - config.Opts->AddLongOption("from-exclusive", "Don't include the left border element into response") - .NoArgument().SetFlag(&FromExclusive); - config.Opts->AddLongOption("to-exclusive", "Don't include the right border element into response") - .NoArgument().SetFlag(&ToExclusive); - - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); - + return EXIT_SUCCESS; +} + +TCommandReadTable::TCommandReadTable() + : TYdbCommand("readtable", {}, "Stream read table") +{} + +void TCommandReadTable::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption("ordered", "Result should be ordered by primary key") + .NoArgument().SetFlag(&Ordered); + config.Opts->AddLongOption("limit", "Limit result rows count") + .RequiredArgument("NUM").StoreResult(&RowLimit); + config.Opts->AddLongOption("columns", "Comma separated list of columns to read") + .RequiredArgument("CSV").StoreResult(&Columns); + config.Opts->AddLongOption("count-only", "Print only rows count") + .NoArgument().SetFlag(&CountOnly); + config.Opts->AddLongOption("from", "Key prefix value to start read from.\n" + " Format should be a json-string containing array of elements representing a tuple - key prefix.\n" + " Option \"--input-format\" defines how to parse binary strings.\n" + " Examples:\n" + " 1) using one column from PK:\n" + " --from [10] --to [100]\n" + " 2) using two columns from PK (forwarding strings in command line is OS-specific. Example for linux):\n" + " --from [10,\\\"OneWord\\\"] --to '[100,\"Two Words\"]'") + .RequiredArgument("JSON").StoreResult(&From); + config.Opts->AddLongOption("to", "Key prefix value to read until.\n" + " Same format as for \"--from\" option.") + .RequiredArgument("JSON").StoreResult(&To); + config.Opts->AddLongOption("from-exclusive", "Don't include the left border element into response") + .NoArgument().SetFlag(&FromExclusive); + config.Opts->AddLongOption("to-exclusive", "Don't include the right border element into response") + .NoArgument().SetFlag(&ToExclusive); + + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::JsonUnicode, - EOutputFormat::JsonUnicodeArray, - EOutputFormat::JsonBase64, - EOutputFormat::JsonBase64Array + EOutputFormat::JsonUnicodeArray, + EOutputFormat::JsonBase64, + EOutputFormat::JsonBase64Array }); - - // TODO: KIKIMR-8675 - // Add csv format - - config.SetFreeArgsNum(1); - SetFreeArgTitle(0, "<table path>", "Path to a table"); -} - -void TCommandReadTable::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParseFormats(); - ParsePath(config, 0); -} - -namespace { - TType GetKeyPrefixTypeFromJson(const TString& jsonString, const TString& optionName - , NTable::TTableDescription& tableDescription) { - NJson::TJsonValue jsonValue; - if (!ReadJsonTree(jsonString, &jsonValue)) { - throw TMissUseException() << "Can't parse string \"" << jsonString << "\" (--" << optionName << " option) as json"; - } - if (!jsonValue.IsArray()) { - throw TMissUseException() << "json string in \"--" << optionName - << "\" should contain array of elements representing tuple with key prefix, but it doesn't"; - } - TTypeBuilder typebuilder; - typebuilder.BeginTuple(); - const auto& pkColumnNames = tableDescription.GetPrimaryKeyColumns(); - auto pkColumnNamesIterator = pkColumnNames.begin(); - for (const auto& element : jsonValue.GetArray()) { - Y_UNUSED(element); - if (pkColumnNamesIterator == pkColumnNames.end()) { - throw TMissUseException() << "json string in \"--" << optionName << "\" option contains more elements (" - << jsonValue.GetArray().size() << ") then columns in table primary key (" << pkColumnNames.size() << ")"; - } - for (const auto& column : tableDescription.GetTableColumns()) { - if (*pkColumnNamesIterator == column.Name) { - typebuilder.AddElement(column.Type); - break; - } - } - ++pkColumnNamesIterator; - } - typebuilder.EndTuple(); - return typebuilder.Build(); - } -} - -int TCommandReadTable::Run(TConfig& config) { - NTable::TTableClient client(CreateDriver(config)); - - NTable::TReadTableSettings readTableSettings; - if (RowLimit) { - readTableSettings.RowLimit(RowLimit); - } - if (Ordered) { - readTableSettings.Ordered(Ordered); - } - if (Columns) { - readTableSettings.Columns_ = StringSplitter(Columns).Split(',').ToList<TString>(); - } - - if (From || To) { - NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); - ThrowOnError(sessionResult); - NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable(Path).GetValueSync(); - NTable::TTableDescription tableDescription = tableResult.GetTableDescription(); - - EBinaryStringEncoding encoding; - switch (InputFormat) { - case EOutputFormat::Default: - case EOutputFormat::JsonUnicode: - encoding = EBinaryStringEncoding::Unicode; - break; - case EOutputFormat::JsonBase64: - encoding = EBinaryStringEncoding::Base64; - break; - default: - throw TMissUseException() << "Unknown input format: " << InputFormat; - } - - if (From) { - TValue fromValue = JsonToYdbValue(From, GetKeyPrefixTypeFromJson(From, "from", tableDescription), encoding); - readTableSettings.From(FromExclusive - ? NTable::TKeyBound::Exclusive(fromValue) - : NTable::TKeyBound::Inclusive(fromValue)); - } - - if (To) { - TValue toValue = JsonToYdbValue(To, GetKeyPrefixTypeFromJson(To, "to", tableDescription), encoding); - readTableSettings.To(ToExclusive - ? NTable::TKeyBound::Exclusive(toValue) - : NTable::TKeyBound::Inclusive(toValue)); - } - } - - TMaybe<NTable::TTablePartIterator> tableIterator; - - ThrowOnError(client.RetryOperationSync([this, &readTableSettings, &tableIterator](NTable::TSession session) { - NTable::TTablePartIterator result = session.ReadTable(Path, readTableSettings).GetValueSync(); - - if (result.IsSuccess()) { - tableIterator = result; - } - - return result; - })); - - PrintResponse(tableIterator.GetRef()); - return EXIT_SUCCESS; -} - -void TCommandReadTable::PrintResponse(NTable::TTablePartIterator& result) { - size_t totalRows = 0; + + // TODO: KIKIMR-8675 + // Add csv format + + config.SetFreeArgsNum(1); + SetFreeArgTitle(0, "<table path>", "Path to a table"); +} + +void TCommandReadTable::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParseFormats(); + ParsePath(config, 0); +} + +namespace { + TType GetKeyPrefixTypeFromJson(const TString& jsonString, const TString& optionName + , NTable::TTableDescription& tableDescription) { + NJson::TJsonValue jsonValue; + if (!ReadJsonTree(jsonString, &jsonValue)) { + throw TMissUseException() << "Can't parse string \"" << jsonString << "\" (--" << optionName << " option) as json"; + } + if (!jsonValue.IsArray()) { + throw TMissUseException() << "json string in \"--" << optionName + << "\" should contain array of elements representing tuple with key prefix, but it doesn't"; + } + TTypeBuilder typebuilder; + typebuilder.BeginTuple(); + const auto& pkColumnNames = tableDescription.GetPrimaryKeyColumns(); + auto pkColumnNamesIterator = pkColumnNames.begin(); + for (const auto& element : jsonValue.GetArray()) { + Y_UNUSED(element); + if (pkColumnNamesIterator == pkColumnNames.end()) { + throw TMissUseException() << "json string in \"--" << optionName << "\" option contains more elements (" + << jsonValue.GetArray().size() << ") then columns in table primary key (" << pkColumnNames.size() << ")"; + } + for (const auto& column : tableDescription.GetTableColumns()) { + if (*pkColumnNamesIterator == column.Name) { + typebuilder.AddElement(column.Type); + break; + } + } + ++pkColumnNamesIterator; + } + typebuilder.EndTuple(); + return typebuilder.Build(); + } +} + +int TCommandReadTable::Run(TConfig& config) { + NTable::TTableClient client(CreateDriver(config)); + + NTable::TReadTableSettings readTableSettings; + if (RowLimit) { + readTableSettings.RowLimit(RowLimit); + } + if (Ordered) { + readTableSettings.Ordered(Ordered); + } + if (Columns) { + readTableSettings.Columns_ = StringSplitter(Columns).Split(',').ToList<TString>(); + } + + if (From || To) { + NTable::TCreateSessionResult sessionResult = client.GetSession(NTable::TCreateSessionSettings()).GetValueSync(); + ThrowOnError(sessionResult); + NTable::TDescribeTableResult tableResult = sessionResult.GetSession().DescribeTable(Path).GetValueSync(); + NTable::TTableDescription tableDescription = tableResult.GetTableDescription(); + + EBinaryStringEncoding encoding; + switch (InputFormat) { + case EOutputFormat::Default: + case EOutputFormat::JsonUnicode: + encoding = EBinaryStringEncoding::Unicode; + break; + case EOutputFormat::JsonBase64: + encoding = EBinaryStringEncoding::Base64; + break; + default: + throw TMissUseException() << "Unknown input format: " << InputFormat; + } + + if (From) { + TValue fromValue = JsonToYdbValue(From, GetKeyPrefixTypeFromJson(From, "from", tableDescription), encoding); + readTableSettings.From(FromExclusive + ? NTable::TKeyBound::Exclusive(fromValue) + : NTable::TKeyBound::Inclusive(fromValue)); + } + + if (To) { + TValue toValue = JsonToYdbValue(To, GetKeyPrefixTypeFromJson(To, "to", tableDescription), encoding); + readTableSettings.To(ToExclusive + ? NTable::TKeyBound::Exclusive(toValue) + : NTable::TKeyBound::Inclusive(toValue)); + } + } + + TMaybe<NTable::TTablePartIterator> tableIterator; + + ThrowOnError(client.RetryOperationSync([this, &readTableSettings, &tableIterator](NTable::TSession session) { + NTable::TTablePartIterator result = session.ReadTable(Path, readTableSettings).GetValueSync(); + + if (result.IsSuccess()) { + tableIterator = result; + } + + return result; + })); + + PrintResponse(tableIterator.GetRef()); + return EXIT_SUCCESS; +} + +void TCommandReadTable::PrintResponse(NTable::TTablePartIterator& result) { + size_t totalRows = 0; SetInterruptHandlers(); - TResultSetPrinter printer(OutputFormat, &IsInterrupted); - - while (!IsInterrupted()) { - auto tablePart = result.ReadNext().GetValueSync(); - if (!tablePart.IsSuccess()) { - if (tablePart.EOS()) { - break; - } - ThrowOnError(tablePart); - } - if (CountOnly) { - TResultSetParser parser(tablePart.ExtractPart()); - while (parser.TryNextRow()) { - ++totalRows; - } - continue; - } - printer.Print(tablePart.GetPart()); - } - if (CountOnly) { - Cout << totalRows << Endl; - } - - if (IsInterrupted()) { - Cerr << "<INTERRUPTED>" << Endl; - } -} - + TResultSetPrinter printer(OutputFormat, &IsInterrupted); + + while (!IsInterrupted()) { + auto tablePart = result.ReadNext().GetValueSync(); + if (!tablePart.IsSuccess()) { + if (tablePart.EOS()) { + break; + } + ThrowOnError(tablePart); + } + if (CountOnly) { + TResultSetParser parser(tablePart.ExtractPart()); + while (parser.TryNextRow()) { + ++totalRows; + } + continue; + } + printer.Print(tablePart.GetPart()); + } + if (CountOnly) { + Cout << totalRows << Endl; + } + + if (IsInterrupted()) { + Cerr << "<INTERRUPTED>" << Endl; + } +} + TCommandIndexAddGlobal::TCommandIndexAddGlobal( NTable::EIndexType type, const TString& name, @@ -859,14 +859,14 @@ void TCommandIndexAddGlobal::Config(TConfig& config) { config.SetFreeArgsNum(1); SetFreeArgTitle(0, "<table path>", "Path to a table"); -} +} void TCommandIndexAddGlobal::Parse(TConfig& config) { TClientCommand::Parse(config); ParseFormats(); ParsePath(config, 0); -} - +} + int TCommandIndexAddGlobal::Run(TConfig& config) { NTable::TTableClient client(CreateDriver(config)); auto columns = StringSplitter(Columns).Split(',').ToList<TString>(); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index 1125e8d951..5cd2514aa0 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -1,28 +1,28 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/interruptible.h> #include <ydb/public/lib/ydb_cli/common/parameters.h> #include <ydb/public/lib/json_value/ydb_json_value.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandTable : public TClientCommandTree { -public: - TCommandTable(); -}; - -class TCommandQuery : public TClientCommandTree { -public: - TCommandQuery(); -}; - + +namespace NYdb { +namespace NConsoleClient { + +class TCommandTable : public TClientCommandTree { +public: + TCommandTable(); +}; + +class TCommandQuery : public TClientCommandTree { +public: + TCommandQuery(); +}; + class TCommandIndex : public TClientCommandTree { public: TCommandIndex(); @@ -38,119 +38,119 @@ public: TCommandTtl(); }; -class TTableCommand : public TYdbOperationCommand { -public: - TTableCommand( - const TString& name, - const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), - const TString& description = TString() - ); - - virtual void Config(TConfig& config) override; - NYdb::NTable::TSession GetSession(TConfig& config); -}; - -class TCommandCreateTable : public TTableCommand, public TCommandWithPath { -public: - TCommandCreateTable(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - TVector<TString> Columns; - TVector<TString> PrimaryKeys; - TVector<TString> Indexes; - TString PresetName; - TString ExecutionPolicy; - TString CompactionPolicy; - TString PartitioningPolicy; - TString AutoPartitioning; - TString UniformPartitions; - TString ReplicationPolicy; - TString ReplicasCount; - bool CreatePerAvailabilityZone = false; - bool AllowPromotion = false; -}; - -class TCommandDropTable : public TTableCommand, public TCommandWithPath { -public: - TCommandDropTable(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; -}; - -class TCommandQueryBase { -protected: - void CheckQueryOptions() const; - void CheckQueryFile(); - +class TTableCommand : public TYdbOperationCommand { +public: + TTableCommand( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); + + virtual void Config(TConfig& config) override; + NYdb::NTable::TSession GetSession(TConfig& config); +}; + +class TCommandCreateTable : public TTableCommand, public TCommandWithPath { +public: + TCommandCreateTable(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + TVector<TString> Columns; + TVector<TString> PrimaryKeys; + TVector<TString> Indexes; + TString PresetName; + TString ExecutionPolicy; + TString CompactionPolicy; + TString PartitioningPolicy; + TString AutoPartitioning; + TString UniformPartitions; + TString ReplicationPolicy; + TString ReplicasCount; + bool CreatePerAvailabilityZone = false; + bool AllowPromotion = false; +}; + +class TCommandDropTable : public TTableCommand, public TCommandWithPath { +public: + TCommandDropTable(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; +}; + +class TCommandQueryBase { +protected: + void CheckQueryOptions() const; + void CheckQueryFile(); + protected: - TString Query; - TString QueryFile; -}; - -class TCommandExecuteQuery : public TTableCommand, TCommandQueryBase, TCommandWithParameters, + TString Query; + TString QueryFile; +}; + +class TCommandExecuteQuery : public TTableCommand, TCommandQueryBase, TCommandWithParameters, public TCommandWithFormat, public TInterruptibleCommand { -public: - TCommandExecuteQuery(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; +public: + TCommandExecuteQuery(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; - int ExecuteDataQuery(TConfig& config); - void PrintDataQueryResponse(NTable::TDataQueryResult& result); + int ExecuteDataQuery(TConfig& config); + void PrintDataQueryResponse(NTable::TDataQueryResult& result); + + int ExecuteSchemeQuery(TConfig& config); - int ExecuteSchemeQuery(TConfig& config); - int ExecuteScanQuery(TConfig& config); void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result); -private: - TString CollectStatsMode; - TString TxMode; - TString QueryType; - bool BasicStats = false; -}; - +private: + TString CollectStatsMode; + TString TxMode; + TString QueryType; + bool BasicStats = false; +}; + class TCommandExplain : public TTableCommand, public TCommandWithFormat, TCommandQueryBase, TInterruptibleCommand { -public: - TCommandExplain(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; +public: + TCommandExplain(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; private: bool PrintAst = false; TString QueryType; bool Analyze = false; -}; - +}; + class TCommandReadTable : public TYdbCommand, public TCommandWithPath, public TCommandWithFormat, public TInterruptibleCommand { -public: - TCommandReadTable(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - void PrintResponse(NTable::TTablePartIterator& result); - - bool Ordered = false; - bool CountOnly = false; - bool FromExclusive = false; - bool ToExclusive = false; - ui64 RowLimit = 0; - TString Columns; - TString From; - TString To; -}; - +public: + TCommandReadTable(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + void PrintResponse(NTable::TTablePartIterator& result); + + bool Ordered = false; + bool CountOnly = false; + bool FromExclusive = false; + bool ToExclusive = false; + ui64 RowLimit = 0; + TString Columns; + TString From; + TString To; +}; + class TCommandIndexAdd : public TClientCommandTree { public: TCommandIndexAdd(); @@ -236,5 +236,5 @@ public: virtual int Run(TConfig& config) override; }; -} -} +} +} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index 54417fff19..db4ec14d30 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -31,7 +31,7 @@ void TToolsCommand::Config(TConfig& config) { // Dump //////////////////////////////////////////////////////////////////////////////// TCommandDump::TCommandDump() - : TToolsCommand("dump", {}, "Dump specified database directory or table into local directory") + : TToolsCommand("dump", {}, "Dump specified database directory or table into local directory") {} void TCommandDump::Config(TConfig& config) { @@ -39,41 +39,41 @@ void TCommandDump::Config(TConfig& config) { config.SetFreeArgsNum(0); - config.Opts->AddLongOption('p', "path", "Database path to a directory or a table to be dumped.") - .DefaultValue(".").StoreResult(&Path); - config.Opts->AddLongOption("exclude", "Pattern(s) (PCRE) for paths excluded from dump." - " Option can be used several times - one for each pattern.") + config.Opts->AddLongOption('p', "path", "Database path to a directory or a table to be dumped.") + .DefaultValue(".").StoreResult(&Path); + config.Opts->AddLongOption("exclude", "Pattern(s) (PCRE) for paths excluded from dump." + " Option can be used several times - one for each pattern.") .RequiredArgument("STRING").Handler1T<TString>([this](const TString& arg) { ExclusionPatterns.emplace_back(TRegExMatch(arg)); }); - config.Opts->AddLongOption('o', "output", "[Required] Path in a local filesystem to a directory to place dump into." - " Directory should either not exist or be empty.") + config.Opts->AddLongOption('o', "output", "[Required] Path in a local filesystem to a directory to place dump into." + " Directory should either not exist or be empty.") .StoreResult(&FilePath); - config.Opts->AddLongOption("scheme-only", "Dump only scheme") + config.Opts->AddLongOption("scheme-only", "Dump only scheme") .StoreTrue(&IsSchemeOnly); - config.Opts->AddLongOption("avoid-copy", "Avoid copying." - " By default, YDB makes a copy of a table before dumping it to reduce impact on workload and ensure consistency.\n" - "In some cases (e.g. for tables with external blobs) copying should be disabled.") + config.Opts->AddLongOption("avoid-copy", "Avoid copying." + " By default, YDB makes a copy of a table before dumping it to reduce impact on workload and ensure consistency.\n" + "In some cases (e.g. for tables with external blobs) copying should be disabled.") .StoreTrue(&AvoidCopy); - config.Opts->AddLongOption("save-partial-result", "Do not remove partial dump result." - " If this option is not enabled, all files that have already been created will be removed in case of error.") + config.Opts->AddLongOption("save-partial-result", "Do not remove partial dump result." + " If this option is not enabled, all files that have already been created will be removed in case of error.") .StoreTrue(&SavePartialResult); - config.Opts->AddLongOption("preserve-pool-kinds", "Preserve storage pool kind settings." - " If this option is enabled, storage pool kind will be saved to dump." - " In this case, if there will be no such storage pool kind in database on restore, error will occur." - " By default this option is disabled and any existing storage pool kind will be used on restore.") + config.Opts->AddLongOption("preserve-pool-kinds", "Preserve storage pool kind settings." + " If this option is enabled, storage pool kind will be saved to dump." + " In this case, if there will be no such storage pool kind in database on restore, error will occur." + " By default this option is disabled and any existing storage pool kind will be used on restore.") .StoreTrue(&PreservePoolKinds); - config.Opts->AddLongOption("consistency-level", "Consistency level." - " Options: database, table\n" - "database - take one consistent snapshot of all tables specified for dump." - " Takes more time and is more likely to impact workload;\n" - "table - take consistent snapshot per each table independently.") + config.Opts->AddLongOption("consistency-level", "Consistency level." + " Options: database, table\n" + "database - take one consistent snapshot of all tables specified for dump." + " Takes more time and is more likely to impact workload;\n" + "table - take consistent snapshot per each table independently.") .DefaultValue("database").StoreResult(&ConsistencyLevel); } void TCommandDump::Parse(TConfig& config) { TClientCommand::Parse(config); - AdjustPath(config); + AdjustPath(config); } int TCommandDump::Run(TConfig& config) { @@ -84,7 +84,7 @@ int TCommandDump::Run(TConfig& config) { } else if (ConsistencyLevel == "table") { useConsistentCopyTable = false; } else { - throw yexception() << "Incorrect consistency level. Available options: \"database\", \"table\"" << Endl; + throw yexception() << "Incorrect consistency level. Available options: \"database\", \"table\"" << Endl; } NYdb::SetVerbosity(config.IsVerbose); @@ -116,16 +116,16 @@ void TCommandRestore::Config(TConfig& config) { config.SetFreeArgsNum(0); config.Opts->AddLongOption('p', "path", - "[Required] Database path to a destination directory where restored directory or table will be placed.") + "[Required] Database path to a destination directory where restored directory or table will be placed.") .StoreResult(&Path); config.Opts->AddLongOption('i', "input", - "[Required] Path in a local filesystem to a directory with dump.") + "[Required] Path in a local filesystem to a directory with dump.") .StoreResult(&FilePath); config.Opts->AddLongOption("dry-run", TStringBuilder() << "Do not restore tables, only check that:" << Endl - << " - all dumped tables exist in database;" << Endl - << " - all dumped table schemes are the same as in database.") + << " - all dumped tables exist in database;" << Endl + << " - all dumped table schemes are the same as in database.") .StoreTrue(&IsDryRun); NDump::TRestoreSettings defaults; @@ -142,9 +142,9 @@ void TCommandRestore::Config(TConfig& config) { .DefaultValue(defaults.SkipDocumentTables_).StoreResult(&SkipDocumentTables) .Hidden(); // Deprecated - config.Opts->AddLongOption("save-partial-result", "Do not remove partial restore result." - " If this option is not enabled, all changes in database that have already been applied during restore" - " will be reverted in case of error.") + config.Opts->AddLongOption("save-partial-result", "Do not remove partial restore result." + " If this option is not enabled, all changes in database that have already been applied during restore" + " will be reverted in case of error.") .StoreTrue(&SavePartialResult); config.Opts->AddLongOption("bandwidth", "Limit data upload bandwidth, bytes per second (example: 2MiB)") @@ -162,16 +162,16 @@ void TCommandRestore::Config(TConfig& config) { config.Opts->AddLongOption("upload-batch-rus", "Limit upload batch size in request units (example: 100)") .DefaultValue(defaults.RequestUnitsPerRequest_).StoreResult(&RequestUnitsPerRequest); - config.Opts->AddLongOption("in-flight", "Limit in-flight request count") + config.Opts->AddLongOption("in-flight", "Limit in-flight request count") .DefaultValue(defaults.InFly_).StoreResult(&InFly); - config.Opts->AddLongOption("bulk-upsert", "Use BulkUpsert - a more efficient way to upload data with lower consistency level." - " Global secondary indexes are not supported in this mode.") - .StoreTrue(&UseBulkUpsert) - .Hidden(); // Deprecated. Using ImportData should be more effective. + config.Opts->AddLongOption("bulk-upsert", "Use BulkUpsert - a more efficient way to upload data with lower consistency level." + " Global secondary indexes are not supported in this mode.") + .StoreTrue(&UseBulkUpsert) + .Hidden(); // Deprecated. Using ImportData should be more effective. - config.Opts->AddLongOption("import-data", "Use ImportData - a more efficient way to upload data with lower consistency level." - " Global secondary indexes are not supported in this mode.") + config.Opts->AddLongOption("import-data", "Use ImportData - a more efficient way to upload data with lower consistency level." + " Global secondary indexes are not supported in this mode.") .StoreTrue(&UseImportData); config.Opts->MutuallyExclusive("bandwidth", "rps"); @@ -180,7 +180,7 @@ void TCommandRestore::Config(TConfig& config) { void TCommandRestore::Parse(TConfig& config) { TClientCommand::Parse(config); - AdjustPath(config); + AdjustPath(config); } int TCommandRestore::Run(TConfig& config) { @@ -227,61 +227,61 @@ int TCommandRestore::Run(TConfig& config) { return EXIT_SUCCESS; } -//////////////////////////////////////////////////////////////////////////////// -// Copy -//////////////////////////////////////////////////////////////////////////////// - -TCommandCopy::TCommandCopy() - : TTableCommand("copy", {}, "Copy table(s)") +//////////////////////////////////////////////////////////////////////////////// +// Copy +//////////////////////////////////////////////////////////////////////////////// + +TCommandCopy::TCommandCopy() + : TTableCommand("copy", {}, "Copy table(s)") { TItem::DefineFields({ {"Source", {{"source", "src", "s"}, "Source table path", true}}, {"Destination", {{"destination", "dst", "d"}, "Destination table path", true}} }); } - -void TCommandCopy::Config(TConfig& config) { - TTableCommand::Config(config); - - config.SetFreeArgsNum(0); - - TStringBuilder itemHelp; - itemHelp << "[At least one] Item specification" << Endl - << " Possible property names:" << Endl + +void TCommandCopy::Config(TConfig& config) { + TTableCommand::Config(config); + + config.SetFreeArgsNum(0); + + TStringBuilder itemHelp; + itemHelp << "[At least one] Item specification" << Endl + << " Possible property names:" << Endl << TItem::FormatHelp(2); - config.Opts->AddLongOption("item", itemHelp) - .RequiredArgument("PROPERTY=VALUE,..."); -} - -void TCommandCopy::Parse(TConfig& config) { - TClientCommand::Parse(config); - - Items = TItem::Parse(config, "item"); - if (Items.empty()) { - throw TMissUseException() << "At least one item should be provided"; - } + config.Opts->AddLongOption("item", itemHelp) + .RequiredArgument("PROPERTY=VALUE,..."); +} + +void TCommandCopy::Parse(TConfig& config) { + TClientCommand::Parse(config); + + Items = TItem::Parse(config, "item"); + if (Items.empty()) { + throw TMissUseException() << "At least one item should be provided"; + } for (auto& item : Items) { NConsoleClient::AdjustPath(item.Source, config); NConsoleClient::AdjustPath(item.Destination, config); } -} - -int TCommandCopy::Run(TConfig& config) { - TVector<NYdb::NTable::TCopyItem> copyItems; - copyItems.reserve(Items.size()); - for (auto& item : Items) { - copyItems.emplace_back(item.Source, item.Destination); - } - ThrowOnError( - GetSession(config).CopyTables( - copyItems, - FillSettings(NTable::TCopyTablesSettings()) - ).GetValueSync() - ); - return EXIT_SUCCESS; -} - +} + +int TCommandCopy::Run(TConfig& config) { + TVector<NYdb::NTable::TCopyItem> copyItems; + copyItems.reserve(Items.size()); + for (auto& item : Items) { + copyItems.emplace_back(item.Source, item.Destination); + } + ThrowOnError( + GetSession(config).CopyTables( + copyItems, + FillSettings(NTable::TCopyTablesSettings()) + ).GetValueSync() + ); + return EXIT_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////// // Rename diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.h b/ydb/public/lib/ydb_cli/commands/ydb_tools.h index 9937ef77df..3a1218a981 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.h @@ -2,11 +2,11 @@ #include "ydb_command.h" #include "ydb_common.h" -#include "ydb_service_table.h" +#include "ydb_service_table.h" #include <ydb/public/lib/ydb_cli/common/examples.h> #include <ydb/public/lib/ydb_cli/common/parseable_struct.h> - + #include <library/cpp/regex/pcre/regexp.h> namespace NYdb { @@ -28,7 +28,7 @@ public: virtual void Config(TConfig& config) override; }; -class TCommandDump : public TToolsCommand, public TCommandWithPath { +class TCommandDump : public TToolsCommand, public TCommandWithPath { public: TCommandDump(); virtual void Config(TConfig& config) override; @@ -45,7 +45,7 @@ private: bool PreservePoolKinds = false; }; -class TCommandRestore : public TToolsCommand, public TCommandWithPath { +class TCommandRestore : public TToolsCommand, public TCommandWithPath { public: TCommandRestore(); virtual void Config(TConfig& config) override; @@ -69,24 +69,24 @@ private: bool UseImportData = false; }; -class TCommandCopy : public TTableCommand { -public: - TCommandCopy(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - struct TItemFields { - TString Source; - TString Destination; - }; - DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); - - TVector<TItem> Items; - TString DatabaseName; -}; - +class TCommandCopy : public TTableCommand { +public: + TCommandCopy(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + struct TItemFields { + TString Source; + TString Destination; + }; + DEFINE_PARSEABLE_STRUCT(TItem, TItemFields, Source, Destination); + + TVector<TItem> Items; + TString DatabaseName; +}; + class TCommandRename : public TTableCommand, public TCommandWithExamples { public: TCommandRename(); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp index 0b3c6ab126..ac8a256172 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp @@ -1,132 +1,132 @@ -#include "ydb_yql.h" - +#include "ydb_yql.h" + #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> #include <ydb/public/lib/ydb_cli/common/query_stats.h> - -namespace NYdb { -namespace NConsoleClient { - -TCommandYql::TCommandYql() - : TYdbOperationCommand("yql", {}, "Execute YQL script (streaming)") -{} - -void TCommandYql::Config(TConfig& config) { - TYdbOperationCommand::Config(config); - config.Opts->AddLongOption("stats", "Collect statistics mode [none, basic, full]") - .RequiredArgument("[String]").StoreResult(&CollectStatsMode); - config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); - config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); - - AddParametersOption(config); - - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); - - AddFormats(config, { - EOutputFormat::Pretty, - EOutputFormat::JsonUnicode, - EOutputFormat::JsonUnicodeArray, - EOutputFormat::JsonBase64, - EOutputFormat::JsonBase64Array - }); - - config.SetFreeArgsNum(0); -} - -void TCommandYql::Parse(TConfig& config) { - TClientCommand::Parse(config); - ParseFormats(); - if (!Script && !ScriptFile) { - throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") " - << "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided."; - } - if (Script && ScriptFile) { - throw TMissUseException() << "Both mutually exclusive options \"Text of script\" (\"--script\", \"-s\") " - << "and \"Path to file with script text\" (\"--file\", \"-f\") were provided."; - } - if (ScriptFile) { - Script = ReadFromFile(ScriptFile, "script"); - } - ParseParameters(); -} - -int TCommandYql::Run(TConfig& config) { - TDriver driver = CreateDriver(config); - NScripting::TScriptingClient client(driver); - NTable::TTableClient tableClient(driver); - - NScripting::TExecuteYqlRequestSettings settings; - settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); - - NScripting::TAsyncYqlResultPartIterator asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.StreamExecuteYqlScript( - Script, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); - } else { - asyncResult = client.StreamExecuteYqlScript( - Script, - FillSettings(settings) - ); - } - NScripting::TYqlResultPartIterator result = asyncResult.GetValueSync(); - - ThrowOnError(result); - PrintResponse(result); - return EXIT_SUCCESS; -} - -void TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { - SetInterruptHandlers(); - TStringStream statsStr; - { - ui32 currentIndex = 0; - TResultSetPrinter printer(OutputFormat, &IsInterrupted); - - while (!IsInterrupted()) { - auto streamPart = result.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - if (streamPart.EOS()) { - break; - } - ThrowOnError(streamPart); - } - - if (streamPart.HasPartialResult()) { - const auto& partialResult = streamPart.GetPartialResult(); - - ui32 resultSetIndex = partialResult.GetResultSetIndex(); - if (currentIndex != resultSetIndex) { - currentIndex = resultSetIndex; - printer.Reset(); - } - - printer.Print(partialResult.GetResultSet()); - } - - if (streamPart.HasQueryStats()) { - const auto& queryStats = streamPart.GetQueryStats(); - statsStr << Endl << queryStats.ToString() << Endl; - } - } - } // TResultSetPrinter destructor should be called before printing stats - - if (statsStr.Size()) { - Cout << Endl << "Statistics:" << statsStr.Str(); - } - - if (IsInterrupted()) { - Cerr << "<INTERRUPTED>" << Endl; - } -} - -} -} - + +namespace NYdb { +namespace NConsoleClient { + +TCommandYql::TCommandYql() + : TYdbOperationCommand("yql", {}, "Execute YQL script (streaming)") +{} + +void TCommandYql::Config(TConfig& config) { + TYdbOperationCommand::Config(config); + config.Opts->AddLongOption("stats", "Collect statistics mode [none, basic, full]") + .RequiredArgument("[String]").StoreResult(&CollectStatsMode); + config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); + config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); + + AddParametersOption(config); + + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, + EOutputFormat::JsonUnicodeArray, + EOutputFormat::JsonBase64, + EOutputFormat::JsonBase64Array + }); + + config.SetFreeArgsNum(0); +} + +void TCommandYql::Parse(TConfig& config) { + TClientCommand::Parse(config); + ParseFormats(); + if (!Script && !ScriptFile) { + throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") " + << "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided."; + } + if (Script && ScriptFile) { + throw TMissUseException() << "Both mutually exclusive options \"Text of script\" (\"--script\", \"-s\") " + << "and \"Path to file with script text\" (\"--file\", \"-f\") were provided."; + } + if (ScriptFile) { + Script = ReadFromFile(ScriptFile, "script"); + } + ParseParameters(); +} + +int TCommandYql::Run(TConfig& config) { + TDriver driver = CreateDriver(config); + NScripting::TScriptingClient client(driver); + NTable::TTableClient tableClient(driver); + + NScripting::TExecuteYqlRequestSettings settings; + settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); + + NScripting::TAsyncYqlResultPartIterator asyncResult; + if (Parameters.size()) { + auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); + asyncResult = client.StreamExecuteYqlScript( + Script, + BuildParams(validateResult.GetParameterTypes(), InputFormat), + FillSettings(settings) + ); + } else { + asyncResult = client.StreamExecuteYqlScript( + Script, + FillSettings(settings) + ); + } + NScripting::TYqlResultPartIterator result = asyncResult.GetValueSync(); + + ThrowOnError(result); + PrintResponse(result); + return EXIT_SUCCESS; +} + +void TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { + SetInterruptHandlers(); + TStringStream statsStr; + { + ui32 currentIndex = 0; + TResultSetPrinter printer(OutputFormat, &IsInterrupted); + + while (!IsInterrupted()) { + auto streamPart = result.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + if (streamPart.EOS()) { + break; + } + ThrowOnError(streamPart); + } + + if (streamPart.HasPartialResult()) { + const auto& partialResult = streamPart.GetPartialResult(); + + ui32 resultSetIndex = partialResult.GetResultSetIndex(); + if (currentIndex != resultSetIndex) { + currentIndex = resultSetIndex; + printer.Reset(); + } + + printer.Print(partialResult.GetResultSet()); + } + + if (streamPart.HasQueryStats()) { + const auto& queryStats = streamPart.GetQueryStats(); + statsStr << Endl << queryStats.ToString() << Endl; + } + } + } // TResultSetPrinter destructor should be called before printing stats + + if (statsStr.Size()) { + Cout << Endl << "Statistics:" << statsStr.Str(); + } + + if (IsInterrupted()) { + Cerr << "<INTERRUPTED>" << Endl; + } +} + +} +} + diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.h b/ydb/public/lib/ydb_cli/commands/ydb_yql.h index 13ac838c5c..3e238a414b 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.h @@ -1,32 +1,32 @@ -#pragma once - -#include "ydb_command.h" -#include "ydb_common.h" - +#pragma once + +#include "ydb_command.h" +#include "ydb_common.h" + #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> #include <ydb/public/lib/ydb_cli/common/format.h> #include <ydb/public/lib/ydb_cli/common/interruptible.h> #include <ydb/public/lib/ydb_cli/common/parameters.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandYql : public TYdbOperationCommand, public TCommandWithParameters, - public TCommandWithFormat, public TInterruptibleCommand -{ -public: - TCommandYql(); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - -private: - void PrintResponse(NScripting::TYqlResultPartIterator& result); - - TString CollectStatsMode; - TString Script; - TString ScriptFile; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +class TCommandYql : public TYdbOperationCommand, public TCommandWithParameters, + public TCommandWithFormat, public TInterruptibleCommand +{ +public: + TCommandYql(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + void PrintResponse(NScripting::TYqlResultPartIterator& result); + + TString CollectStatsMode; + TString Script; + TString ScriptFile; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 5172ac6864..ce7334d738 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -1,102 +1,102 @@ -#include "command.h" +#include "command.h" #include "normalize_path.h" -namespace NYdb { -namespace NConsoleClient { +namespace NYdb { +namespace NConsoleClient { bool TClientCommand::TIME_REQUESTS = false; // measure time of requests bool TClientCommand::PROGRESS_REQUESTS = false; // display progress of long requests -namespace { - TString FormatOption(const NLastGetopt::TOpt* option, const NColorizer::TColors& colors) { - using namespace NLastGetopt; - TStringStream result; - const TOpt::TShortNames& shorts = option->GetShortNames(); - const TOpt::TLongNames& longs = option->GetLongNames(); - - const size_t nopts = shorts.size() + longs.size(); - const bool multiple = 1 < nopts; - if (multiple) { - result << '{'; - } - for (size_t i = 0; i < nopts; ++i) { - if (multiple && 0 != i) { - result << '|'; - } - - if (i < shorts.size()) { // short - result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); - } else { - result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); - } - } - if (multiple) { - result << '}'; - } - - return result.Str(); - } - - // Option not to show in parent command help - bool NeedToHideOption(const NLastGetopt::TOpt* opt) { - if (opt->IsHidden()) { - return true; - } - for (const char shortName : opt->GetShortNames()) { - if (shortName == 'V' || shortName == 'h') - return true; - } - return false; - } - - void PrintOptionsDescription(IOutputStream& os, const NLastGetopt::TOpts* opts, NColorizer::TColors& colors) { - using namespace NLastGetopt; - NColorizer::TColors disabledColors(false); - os << " "; - bool firstPrintedOption = true; - for (size_t i = 0; i < opts->Opts_.size(); i++) { - const TOpt* opt = opts->Opts_[i].Get(); - if (NeedToHideOption(opt)) { - continue; - } - if (!firstPrintedOption) { - os << ", "; - } - os << FormatOption(opt, colors); - firstPrintedOption = false; - } - - os << Endl << " To get full description of these options run 'ydb --help'."; - } - - void PrintParentOptions(TStringStream& stream, TClientCommand::TConfig& config, NColorizer::TColors& colors) { - bool foundRootParent = false; - for (const auto& parentCommand : config.ParentCommands) { - if (parentCommand.Options) { - if (!foundRootParent) { - foundRootParent = true; - stream << colors.BoldColor() << "Global options" << colors.OldColor() << ":" << Endl; - PrintOptionsDescription(stream, parentCommand.Options, colors); - } else { - throw yexception() << "More than two tree commands have options"; - } - } - } - } -} - +namespace { + TString FormatOption(const NLastGetopt::TOpt* option, const NColorizer::TColors& colors) { + using namespace NLastGetopt; + TStringStream result; + const TOpt::TShortNames& shorts = option->GetShortNames(); + const TOpt::TLongNames& longs = option->GetLongNames(); + + const size_t nopts = shorts.size() + longs.size(); + const bool multiple = 1 < nopts; + if (multiple) { + result << '{'; + } + for (size_t i = 0; i < nopts; ++i) { + if (multiple && 0 != i) { + result << '|'; + } + + if (i < shorts.size()) { // short + result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); + } else { + result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); + } + } + if (multiple) { + result << '}'; + } + + return result.Str(); + } + + // Option not to show in parent command help + bool NeedToHideOption(const NLastGetopt::TOpt* opt) { + if (opt->IsHidden()) { + return true; + } + for (const char shortName : opt->GetShortNames()) { + if (shortName == 'V' || shortName == 'h') + return true; + } + return false; + } + + void PrintOptionsDescription(IOutputStream& os, const NLastGetopt::TOpts* opts, NColorizer::TColors& colors) { + using namespace NLastGetopt; + NColorizer::TColors disabledColors(false); + os << " "; + bool firstPrintedOption = true; + for (size_t i = 0; i < opts->Opts_.size(); i++) { + const TOpt* opt = opts->Opts_[i].Get(); + if (NeedToHideOption(opt)) { + continue; + } + if (!firstPrintedOption) { + os << ", "; + } + os << FormatOption(opt, colors); + firstPrintedOption = false; + } + + os << Endl << " To get full description of these options run 'ydb --help'."; + } + + void PrintParentOptions(TStringStream& stream, TClientCommand::TConfig& config, NColorizer::TColors& colors) { + bool foundRootParent = false; + for (const auto& parentCommand : config.ParentCommands) { + if (parentCommand.Options) { + if (!foundRootParent) { + foundRootParent = true; + stream << colors.BoldColor() << "Global options" << colors.OldColor() << ":" << Endl; + PrintOptionsDescription(stream, parentCommand.Options, colors); + } else { + throw yexception() << "More than two tree commands have options"; + } + } + } + } +} + TClientCommand::TClientCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) : Name(name) , Aliases(aliases) , Description(description) , Parent(nullptr) , Opts(NLastGetopt::TOpts::Default()) -{ - HideOption("svnrevision"); - Opts.AddHelpOption('h'); - ChangeOptionDescription("help", "Print usage"); +{ + HideOption("svnrevision"); + Opts.AddHelpOption('h'); + ChangeOptionDescription("help", "Print usage"); Opts.SetWrap(Max(Opts.Wrap_, static_cast<ui32>(TermWidth()))); -} +} TClientCommand::TOptsParseOneLevelResult::TOptsParseOneLevelResult(const NLastGetopt::TOpts* options, int argc, char** argv) { @@ -129,12 +129,12 @@ TClientCommand::TOptsParseOneLevelResult::TOptsParseOneLevelResult(const NLastGe void TClientCommand::Config(TConfig& config) { config.Opts = &Opts; - TStringStream stream; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - stream << Endl << Endl - << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl; - PrintParentOptions(stream, config, colors); - config.Opts->SetCmdLineDescr(stream.Str()); + TStringStream stream; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << Endl << Endl + << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl; + PrintParentOptions(stream, config, colors); + config.Opts->SetCmdLineDescr(stream.Str()); } void TClientCommand::Parse(TConfig& config) { @@ -144,63 +144,63 @@ void TClientCommand::Parse(TConfig& config) { int TClientCommand::Run(TConfig& config) { Y_UNUSED(config); // TODO: invalid usage ? error? help? - return EXIT_FAILURE; + return EXIT_FAILURE; } int TClientCommand::Process(TConfig& config) { - config.ArgsSettings.Reset(new TConfig::TArgSettings()); + config.ArgsSettings.Reset(new TConfig::TArgSettings()); config.Opts = &Opts; Config(config); - config.CheckParamsCount(); - SetCustomUsage(config); + config.CheckParamsCount(); + SetCustomUsage(config); NLastGetopt::TOptsParseResult parseResult(config.Opts, config.ArgC, config.ArgV); config.ParseResult = &parseResult; Parse(config); return Run(config); } -void TClientCommand::SetCustomUsage(TConfig& config) { - // command1 [global options...] command2 ... lastCommand [options...] - TStringBuilder fullName; - bool foundRootParent = false; - for (const auto& parent : config.ParentCommands) { - fullName << parent.Name; - if (parent.Options) { - if (!foundRootParent) { - foundRootParent = true; - fullName << " [global options...]"; - } else { - throw yexception() << "More than two tree commands have options"; - } - } - fullName << " "; - } - fullName << config.ArgV[0] << " [options...]"; - for (auto& arg : Args) { - fullName << ' ' << arg.second; - } - config.Opts->SetCustomUsage(fullName); -} - -void TClientCommand::HideOption(const TString& name) { - NLastGetopt::TOpt* opt = Opts.FindLongOption(name); - if (opt) { - opt->Hidden_ = true; - } -} - -void TClientCommand::ChangeOptionDescription(const TString& name, const TString& description) { - NLastGetopt::TOpt* opt = Opts.FindLongOption(name); - if (opt) { - opt->Help_ = description; - } -} - -void TClientCommand::SetFreeArgTitle(size_t pos, const TString& title, const TString& help) { - Args[pos] = title; - Opts.SetFreeArgTitle(pos, title, help); -} - +void TClientCommand::SetCustomUsage(TConfig& config) { + // command1 [global options...] command2 ... lastCommand [options...] + TStringBuilder fullName; + bool foundRootParent = false; + for (const auto& parent : config.ParentCommands) { + fullName << parent.Name; + if (parent.Options) { + if (!foundRootParent) { + foundRootParent = true; + fullName << " [global options...]"; + } else { + throw yexception() << "More than two tree commands have options"; + } + } + fullName << " "; + } + fullName << config.ArgV[0] << " [options...]"; + for (auto& arg : Args) { + fullName << ' ' << arg.second; + } + config.Opts->SetCustomUsage(fullName); +} + +void TClientCommand::HideOption(const TString& name) { + NLastGetopt::TOpt* opt = Opts.FindLongOption(name); + if (opt) { + opt->Hidden_ = true; + } +} + +void TClientCommand::ChangeOptionDescription(const TString& name, const TString& description) { + NLastGetopt::TOpt* opt = Opts.FindLongOption(name); + if (opt) { + opt->Help_ = description; + } +} + +void TClientCommand::SetFreeArgTitle(size_t pos, const TString& title, const TString& help) { + Args[pos] = title; + Opts.SetFreeArgTitle(pos, title, help); +} + TString TClientCommand::Ends2Prefix(const std::basic_string<bool>& ends) { TString prefix; if (!ends.empty()) { @@ -218,40 +218,40 @@ TString TClientCommand::Ends2Prefix(const std::basic_string<bool>& ends) { return prefix; } -void TClientCommand::RenderCommandsDescription( - TStringStream& stream, - const NColorizer::TColors& colors, - const std::basic_string<bool>& ends -) { - TString prefix = Ends2Prefix(ends); - TString line = prefix + Name; - stream << prefix << colors.BoldColor() << Name << colors.OldColor(); - if (!Description.empty()) { - int namePartLength = GetNumberOfUTF8Chars(line); - if (namePartLength < DESCRIPTION_ALIGNMENT) - stream << TString(DESCRIPTION_ALIGNMENT - namePartLength, ' '); - else - stream << ' '; - stream << Description; - if (!Aliases.empty()) { - stream << " (aliases: "; - for (auto it = Aliases.begin(); it != Aliases.end(); ++it) { - if (it != Aliases.begin()) - stream << ", "; - stream << colors.BoldColor() << *it << colors.OldColor(); - } - stream << ')'; - } - } - stream << '\n'; -} - +void TClientCommand::RenderCommandsDescription( + TStringStream& stream, + const NColorizer::TColors& colors, + const std::basic_string<bool>& ends +) { + TString prefix = Ends2Prefix(ends); + TString line = prefix + Name; + stream << prefix << colors.BoldColor() << Name << colors.OldColor(); + if (!Description.empty()) { + int namePartLength = GetNumberOfUTF8Chars(line); + if (namePartLength < DESCRIPTION_ALIGNMENT) + stream << TString(DESCRIPTION_ALIGNMENT - namePartLength, ' '); + else + stream << ' '; + stream << Description; + if (!Aliases.empty()) { + stream << " (aliases: "; + for (auto it = Aliases.begin(); it != Aliases.end(); ++it) { + if (it != Aliases.begin()) + stream << ", "; + stream << colors.BoldColor() << *it << colors.OldColor(); + } + stream << ')'; + } + } + stream << '\n'; +} + TClientCommandTree::TClientCommandTree(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) : TClientCommand(name, aliases, description) , SelectedCommand(nullptr) -{ - Args[0] = "<subcommand>"; -} +{ + Args[0] = "<subcommand>"; +} void TClientCommandTree::AddCommand(std::unique_ptr<TClientCommand> command) { for (const TString& a : command->Aliases) { @@ -263,22 +263,22 @@ void TClientCommandTree::AddCommand(std::unique_ptr<TClientCommand> command) { void TClientCommandTree::Config(TConfig& config) { TClientCommand::Config(config); - SetFreeArgs(config); + SetFreeArgs(config); TString commands; for (auto it = SubCommands.begin(); it != SubCommands.end(); ++it) { if (!commands.empty()) commands += ','; commands += it->first; } - SetFreeArgTitle(0, "<subcommand>", commands); + SetFreeArgTitle(0, "<subcommand>", commands); TStringStream stream; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - stream << Endl << Endl - << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl - << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; - RenderCommandsDescription(stream, colors); - stream << Endl; - PrintParentOptions(stream, config, colors); + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << Endl << Endl + << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + RenderCommandsDescription(stream, colors); + stream << Endl; + PrintParentOptions(stream, config, colors); config.Opts->SetCmdLineDescr(stream.Str()); } @@ -316,66 +316,66 @@ int TClientCommandTree::Run(TConfig& config) { } int TClientCommandTree::Process(TConfig& config) { - config.ArgsSettings.Reset(new TConfig::TArgSettings()); + config.ArgsSettings.Reset(new TConfig::TArgSettings()); config.Opts = &Opts; Config(config); - config.CheckParamsCount(); - SetCustomUsage(config); + config.CheckParamsCount(); + SetCustomUsage(config); TOptsParseOneLevelResult parseResult(config.Opts, config.ArgC, config.ArgV); config.ParseResult = &parseResult; Parse(config); - config.ParentCommands.push_back({ Name, HasOptionsToShow() ? &Opts : nullptr }); + config.ParentCommands.push_back({ Name, HasOptionsToShow() ? &Opts : nullptr }); return Run(config); } -void TClientCommandTree::SetFreeArgs(TConfig& config) { - config.SetFreeArgsMin(1); -} - -bool TClientCommandTree::HasOptionsToShow() { - for (auto opt : Opts.Opts_) { - if (!NeedToHideOption(opt.Get())) { - return true; +void TClientCommandTree::SetFreeArgs(TConfig& config) { + config.SetFreeArgsMin(1); +} + +bool TClientCommandTree::HasOptionsToShow() { + for (auto opt : Opts.Opts_) { + if (!NeedToHideOption(opt.Get())) { + return true; } } - return false; + return false; } -void TClientCommandTree::RenderCommandsDescription( - TStringStream& stream, - const NColorizer::TColors& colors, - const std::basic_string<bool>& ends -) { - TClientCommand::RenderCommandsDescription(stream, colors, ends); +void TClientCommandTree::RenderCommandsDescription( + TStringStream& stream, + const NColorizer::TColors& colors, + const std::basic_string<bool>& ends +) { + TClientCommand::RenderCommandsDescription(stream, colors, ends); for (auto it = SubCommands.begin(); it != SubCommands.end(); ++it) { bool lastCommand = (std::next(it) == SubCommands.end()); - it->second->RenderCommandsDescription(stream, colors, ends + lastCommand); + it->second->RenderCommandsDescription(stream, colors, ends + lastCommand); + } +} + +void TCommandWithPath::ParsePath(const TClientCommand::TConfig& config, const size_t argPos, bool isPathOptional) { + if (config.ParseResult->GetFreeArgCount() < argPos + 1 && isPathOptional) { + if (isPathOptional) { + Path = "."; + } + } else { + Path = config.ParseResult->GetFreeArgs()[argPos]; } + + AdjustPath(config); } -void TCommandWithPath::ParsePath(const TClientCommand::TConfig& config, const size_t argPos, bool isPathOptional) { - if (config.ParseResult->GetFreeArgCount() < argPos + 1 && isPathOptional) { - if (isPathOptional) { - Path = "."; - } - } else { - Path = config.ParseResult->GetFreeArgs()[argPos]; - } - - AdjustPath(config); -} - -void TCommandWithPath::AdjustPath(const TClientCommand::TConfig& config) { - if (!Path) { - throw TMissUseException() << "Missing required argument <path>"; - } - +void TCommandWithPath::AdjustPath(const TClientCommand::TConfig& config) { + if (!Path) { + throw TMissUseException() << "Missing required argument <path>"; + } + NConsoleClient::AdjustPath(Path, config); } - + void TCommandWithStreamName::ParseStreamName(const TClientCommand::TConfig &config, const size_t argPos) { StreamName = config.ParseResult->GetFreeArgs()[argPos]; -} +} -} +} } diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index 532f36cdce..441206cdf7 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -1,360 +1,360 @@ -#pragma once - -#include "common.h" - +#pragma once + +#include "common.h" + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/colorizer/colors.h> -#include <util/generic/strbuf.h> -#include <util/generic/vector.h> -#include <util/charset/utf8.h> -#include <util/string/type.h> -#include <string> - -namespace NYdb { -namespace NConsoleClient { - -class TClientCommand { -public: - static bool TIME_REQUESTS; // measure time of requests - static bool PROGRESS_REQUESTS; // display progress of long requests - TString Name; - TVector<TString> Aliases; - TString Description; - const TClientCommand* Parent; - NLastGetopt::TOpts Opts; - TString Argument; - TMap<ui32, TString> Args; - - TClientCommand(const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString()); - - class TOptsParseOneLevelResult : public NLastGetopt::TOptsParseResult { - public: - TOptsParseOneLevelResult(const NLastGetopt::TOpts* options, int argc, char** argv); - }; - - class TConfig { - struct TCommandInfo { - TString Name; - NLastGetopt::TOpts* Options; - }; - - public: - using TCredentialsGetter = std::function<std::shared_ptr<ICredentialsProviderFactory>(const TClientCommand::TConfig&)>; - - class TArgSetting { - public: - void Set(size_t value) { - Value = value; - IsSet = true; - } - - size_t Get() const { - return Value; - } - - bool GetIsSet() const { - return IsSet; - } - - private: - size_t Value = 0; - bool IsSet = false; - }; - - struct TArgSettings { - TArgSetting Min; - TArgSetting Max; - }; - - int ArgC; - char** ArgV; - NLastGetopt::TOpts* Opts; - const NLastGetopt::TOptsParseResult* ParseResult; - TVector<TString> Tokens; - TString SecurityToken; - TList<TCommandInfo> ParentCommands; - TString Path; - THolder<TArgSettings> ArgsSettings; - TString Address; - TString Database; - TString CaCerts; - bool IsVerbose = false; - bool EnableSsl = false; - +#include <util/generic/strbuf.h> +#include <util/generic/vector.h> +#include <util/charset/utf8.h> +#include <util/string/type.h> +#include <string> + +namespace NYdb { +namespace NConsoleClient { + +class TClientCommand { +public: + static bool TIME_REQUESTS; // measure time of requests + static bool PROGRESS_REQUESTS; // display progress of long requests + TString Name; + TVector<TString> Aliases; + TString Description; + const TClientCommand* Parent; + NLastGetopt::TOpts Opts; + TString Argument; + TMap<ui32, TString> Args; + + TClientCommand(const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString()); + + class TOptsParseOneLevelResult : public NLastGetopt::TOptsParseResult { + public: + TOptsParseOneLevelResult(const NLastGetopt::TOpts* options, int argc, char** argv); + }; + + class TConfig { + struct TCommandInfo { + TString Name; + NLastGetopt::TOpts* Options; + }; + + public: + using TCredentialsGetter = std::function<std::shared_ptr<ICredentialsProviderFactory>(const TClientCommand::TConfig&)>; + + class TArgSetting { + public: + void Set(size_t value) { + Value = value; + IsSet = true; + } + + size_t Get() const { + return Value; + } + + bool GetIsSet() const { + return IsSet; + } + + private: + size_t Value = 0; + bool IsSet = false; + }; + + struct TArgSettings { + TArgSetting Min; + TArgSetting Max; + }; + + int ArgC; + char** ArgV; + NLastGetopt::TOpts* Opts; + const NLastGetopt::TOptsParseResult* ParseResult; + TVector<TString> Tokens; + TString SecurityToken; + TList<TCommandInfo> ParentCommands; + TString Path; + THolder<TArgSettings> ArgsSettings; + TString Address; + TString Database; + TString CaCerts; + bool IsVerbose = false; + bool EnableSsl = false; + bool JsonUi64AsText = false; bool JsonBinaryAsBase64 = false; - - ui64 TabletId; // admin tablet # - ui32 NodeId; // admin node # - TString Tenant; // admin tenant name - TString SlotId; // admin slot id - - TLoginCredentialsParams StaticCredentials; - - TString YCToken; + + ui64 TabletId; // admin tablet # + ui32 NodeId; // admin node # + TString Tenant; // admin tenant name + TString SlotId; // admin slot id + + TLoginCredentialsParams StaticCredentials; + + TString YCToken; bool UseMetadataCredentials = false; - TString SaKeyFile; + TString SaKeyFile; TString IamEndpoint; TString YScope; - - TString YdbDir; - bool UseOAuthToken = true; - bool UseIamAuth = false; - bool UseStaticCredentials = false; - bool UseExportToYt = true; - - TCredentialsGetter CredentialsGetter; - - TConfig(int argc, char** argv) - : ArgC(argc) - , ArgV(argv) - , Opts(nullptr) - , ParseResult(nullptr) - , TabletId(0) - { - CredentialsGetter = [](const TClientCommand::TConfig& config) { - if (config.SecurityToken) { - return CreateOAuthCredentialsProviderFactory(config.SecurityToken); - } - return CreateInsecureCredentialsProviderFactory(); - }; - } - - bool IsHelpCommand() { - TString lastArg = ArgV[ArgC - 1]; - return lastArg == "--help" || lastArg == "-h" || lastArg == "-?"; - } - - bool IsSvnVersionCommand() { - TString lastArg = ArgV[ArgC - 1]; - return lastArg == "--svnrevision" || lastArg == "-V"; - } - - bool IsVersionCommand() { - TString lastArg = ArgV[ArgC - 1]; - if (lastArg == "version") { - return true; - } - if (ArgC > 1) { - TString penultimateArg = ArgV[ArgC - 2]; - if (penultimateArg == "version" && lastArg == "--semantic") { - return true; - } - } - return false; - } - - bool IsUpdateCommand() { - TString lastArg = ArgV[ArgC - 1]; - if (lastArg == "update") { - return true; - } - if (ArgC > 1) { - TString penultimateArg = ArgV[ArgC - 2]; - if (penultimateArg == "update" && (lastArg == "--force" || lastArg == "-f")) { - return true; - } - } - return false; - } - - bool IsInitCommand() { - TString lastArg = ArgV[ArgC - 1]; - if (lastArg == "init") { - return true; - } - return false; - } - - bool IsProfileCommand() { - for (int i = 1; i < 3; ++i) { - if (ArgC <= i) { - return false; - } - TString currentArg = ArgV[ArgC - i - 1]; - if (currentArg == "profile") { - return true; - } - } - return false; - } - - bool IsLicenseCommand() { - TString lastArg = ArgV[ArgC - 1]; - return lastArg == "--license"; - } - - bool IsCreditsCommand() { - TString lastArg = ArgV[ArgC - 1]; - return lastArg == "--credits"; - } - - bool IsHelpExCommand() { - TString lastArg = ArgV[ArgC - 1]; - return lastArg == "--help-ex"; - } - - bool IsSystemCommand() { - return IsHelpCommand() || IsSvnVersionCommand() || IsUpdateCommand() || IsVersionCommand() - || IsInitCommand() || IsProfileCommand() || IsLicenseCommand() || IsCreditsCommand() - || IsHelpExCommand(); - } - - void SetFreeArgsMin(size_t value) { - ArgsSettings->Min.Set(value); - Opts->SetFreeArgsMin(value); - } - - void SetFreeArgsMax(size_t value) { - ArgsSettings->Max.Set(value); - Opts->SetFreeArgsMax(value); - } - - void SetFreeArgsNum(size_t minValue, size_t maxValue) { - ArgsSettings->Min.Set(minValue); - ArgsSettings->Max.Set(maxValue); - Opts->SetFreeArgsNum(minValue, maxValue); - } - - void SetFreeArgsNum(size_t value) { - SetFreeArgsNum(value, value); - } - - void CheckParamsCount() { - size_t count = GetParamsCount(); - if (IsSystemCommand()) { - return; - } - bool minSet = ArgsSettings->Min.GetIsSet(); - size_t minValue = ArgsSettings->Min.Get(); - bool maxSet = ArgsSettings->Max.GetIsSet(); - size_t maxValue = ArgsSettings->Max.Get(); - bool minFailed = minSet && count < minValue; - bool maxFailed = maxSet && count > maxValue; - if (minFailed || maxFailed) { - if (minSet && maxSet) { - if (minValue == maxValue) { - throw TMissUseException() << "Command " << ArgV[0] - << " requires exactly " << minValue << " free arg(s)."; - } - throw TMissUseException() << "Command " << ArgV[0] - << " requires from " << minValue << " to " << maxValue << " free arg(s)."; - } - if (minFailed) { - throw TMissUseException() << "Command " << ArgV[0] - << " requires at least " << minValue << " free arg(s)."; - } - throw TMissUseException() << "Command " << ArgV[0] - << " requires at most " << maxValue << " free arg(s)."; - } - } - - private: - size_t GetParamsCount() { - size_t result = 0; - bool optionArgument = false; - for (int i = 1; i < ArgC; ++i) { - char* pos = ArgV[i]; - while (*pos == '\"' || *pos == '\'') { - ++pos; - } - if (*pos == '-') { - optionArgument = true; - // Exclude --opt=value case - while (*pos != '\0') { - if (*pos == '=') { - optionArgument = false; - break; - } - ++pos; - } - // Exclude opts with no arguments - const NLastGetopt::TOpt* opt; - pos = ArgV[i] + 1; - if (*pos == '-') { - char* end = pos + 1; - while (*end != '\0' && *end != '\'' && *end != '\"') { - ++end; - } - opt = Opts->FindLongOption(TString(pos + 1, end)); - } else { - opt = Opts->FindCharOption(*pos); - } - if (opt && opt->GetHasArg() == NLastGetopt::NO_ARGUMENT) { - optionArgument = false; - } - } else { - if (optionArgument) { - optionArgument = false; - } else { - ++result; - } - } - } - return result; - } - }; - - virtual ~TClientCommand() {} - - virtual void Config(TConfig& config); - virtual void Parse(TConfig& config); - virtual int Run(TConfig& config); - virtual int Process(TConfig& config); - - virtual void RenderCommandsDescription( - TStringStream& stream, - const NColorizer::TColors& colors = NColorizer::TColors(false), - const std::basic_string<bool>& ends = std::basic_string<bool>() - ); - -protected: - void SetFreeArgTitle(size_t pos, const TString& title, const TString& help); - virtual void SetCustomUsage(TConfig& config); - -private: - void HideOption(const TString& name); - void ChangeOptionDescription(const TString& name, const TString& description); - - constexpr static int DESCRIPTION_ALIGNMENT = 28; - - static TString Ends2Prefix(const std::basic_string<bool>& ends); -}; - -class TClientCommandTree : public TClientCommand { -public: + + TString YdbDir; + bool UseOAuthToken = true; + bool UseIamAuth = false; + bool UseStaticCredentials = false; + bool UseExportToYt = true; + + TCredentialsGetter CredentialsGetter; + + TConfig(int argc, char** argv) + : ArgC(argc) + , ArgV(argv) + , Opts(nullptr) + , ParseResult(nullptr) + , TabletId(0) + { + CredentialsGetter = [](const TClientCommand::TConfig& config) { + if (config.SecurityToken) { + return CreateOAuthCredentialsProviderFactory(config.SecurityToken); + } + return CreateInsecureCredentialsProviderFactory(); + }; + } + + bool IsHelpCommand() { + TString lastArg = ArgV[ArgC - 1]; + return lastArg == "--help" || lastArg == "-h" || lastArg == "-?"; + } + + bool IsSvnVersionCommand() { + TString lastArg = ArgV[ArgC - 1]; + return lastArg == "--svnrevision" || lastArg == "-V"; + } + + bool IsVersionCommand() { + TString lastArg = ArgV[ArgC - 1]; + if (lastArg == "version") { + return true; + } + if (ArgC > 1) { + TString penultimateArg = ArgV[ArgC - 2]; + if (penultimateArg == "version" && lastArg == "--semantic") { + return true; + } + } + return false; + } + + bool IsUpdateCommand() { + TString lastArg = ArgV[ArgC - 1]; + if (lastArg == "update") { + return true; + } + if (ArgC > 1) { + TString penultimateArg = ArgV[ArgC - 2]; + if (penultimateArg == "update" && (lastArg == "--force" || lastArg == "-f")) { + return true; + } + } + return false; + } + + bool IsInitCommand() { + TString lastArg = ArgV[ArgC - 1]; + if (lastArg == "init") { + return true; + } + return false; + } + + bool IsProfileCommand() { + for (int i = 1; i < 3; ++i) { + if (ArgC <= i) { + return false; + } + TString currentArg = ArgV[ArgC - i - 1]; + if (currentArg == "profile") { + return true; + } + } + return false; + } + + bool IsLicenseCommand() { + TString lastArg = ArgV[ArgC - 1]; + return lastArg == "--license"; + } + + bool IsCreditsCommand() { + TString lastArg = ArgV[ArgC - 1]; + return lastArg == "--credits"; + } + + bool IsHelpExCommand() { + TString lastArg = ArgV[ArgC - 1]; + return lastArg == "--help-ex"; + } + + bool IsSystemCommand() { + return IsHelpCommand() || IsSvnVersionCommand() || IsUpdateCommand() || IsVersionCommand() + || IsInitCommand() || IsProfileCommand() || IsLicenseCommand() || IsCreditsCommand() + || IsHelpExCommand(); + } + + void SetFreeArgsMin(size_t value) { + ArgsSettings->Min.Set(value); + Opts->SetFreeArgsMin(value); + } + + void SetFreeArgsMax(size_t value) { + ArgsSettings->Max.Set(value); + Opts->SetFreeArgsMax(value); + } + + void SetFreeArgsNum(size_t minValue, size_t maxValue) { + ArgsSettings->Min.Set(minValue); + ArgsSettings->Max.Set(maxValue); + Opts->SetFreeArgsNum(minValue, maxValue); + } + + void SetFreeArgsNum(size_t value) { + SetFreeArgsNum(value, value); + } + + void CheckParamsCount() { + size_t count = GetParamsCount(); + if (IsSystemCommand()) { + return; + } + bool minSet = ArgsSettings->Min.GetIsSet(); + size_t minValue = ArgsSettings->Min.Get(); + bool maxSet = ArgsSettings->Max.GetIsSet(); + size_t maxValue = ArgsSettings->Max.Get(); + bool minFailed = minSet && count < minValue; + bool maxFailed = maxSet && count > maxValue; + if (minFailed || maxFailed) { + if (minSet && maxSet) { + if (minValue == maxValue) { + throw TMissUseException() << "Command " << ArgV[0] + << " requires exactly " << minValue << " free arg(s)."; + } + throw TMissUseException() << "Command " << ArgV[0] + << " requires from " << minValue << " to " << maxValue << " free arg(s)."; + } + if (minFailed) { + throw TMissUseException() << "Command " << ArgV[0] + << " requires at least " << minValue << " free arg(s)."; + } + throw TMissUseException() << "Command " << ArgV[0] + << " requires at most " << maxValue << " free arg(s)."; + } + } + + private: + size_t GetParamsCount() { + size_t result = 0; + bool optionArgument = false; + for (int i = 1; i < ArgC; ++i) { + char* pos = ArgV[i]; + while (*pos == '\"' || *pos == '\'') { + ++pos; + } + if (*pos == '-') { + optionArgument = true; + // Exclude --opt=value case + while (*pos != '\0') { + if (*pos == '=') { + optionArgument = false; + break; + } + ++pos; + } + // Exclude opts with no arguments + const NLastGetopt::TOpt* opt; + pos = ArgV[i] + 1; + if (*pos == '-') { + char* end = pos + 1; + while (*end != '\0' && *end != '\'' && *end != '\"') { + ++end; + } + opt = Opts->FindLongOption(TString(pos + 1, end)); + } else { + opt = Opts->FindCharOption(*pos); + } + if (opt && opt->GetHasArg() == NLastGetopt::NO_ARGUMENT) { + optionArgument = false; + } + } else { + if (optionArgument) { + optionArgument = false; + } else { + ++result; + } + } + } + return result; + } + }; + + virtual ~TClientCommand() {} + + virtual void Config(TConfig& config); + virtual void Parse(TConfig& config); + virtual int Run(TConfig& config); + virtual int Process(TConfig& config); + + virtual void RenderCommandsDescription( + TStringStream& stream, + const NColorizer::TColors& colors = NColorizer::TColors(false), + const std::basic_string<bool>& ends = std::basic_string<bool>() + ); + +protected: + void SetFreeArgTitle(size_t pos, const TString& title, const TString& help); + virtual void SetCustomUsage(TConfig& config); + +private: + void HideOption(const TString& name); + void ChangeOptionDescription(const TString& name, const TString& description); + + constexpr static int DESCRIPTION_ALIGNMENT = 28; + + static TString Ends2Prefix(const std::basic_string<bool>& ends); +}; + +class TClientCommandTree : public TClientCommand { +public: TMap<TString, std::unique_ptr<TClientCommand>> SubCommands; - TMap<TString, TString> Aliases; - TClientCommand* SelectedCommand; - - TClientCommandTree(const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString()); + TMap<TString, TString> Aliases; + TClientCommand* SelectedCommand; + + TClientCommandTree(const TString& name, const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), const TString& description = TString()); void AddCommand(std::unique_ptr<TClientCommand> command); - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - virtual int Run(TConfig& config) override; - virtual int Process(TConfig& config) override; - virtual void RenderCommandsDescription( - TStringStream& stream, - const NColorizer::TColors& colors = NColorizer::TColors(false), - const std::basic_string<bool>& ends = std::basic_string<bool>() - ) override; - virtual void SetFreeArgs(TConfig& config); - -private: - bool HasOptionsToShow(); -}; - -class TCommandWithPath { -protected: - // Get path from free argument and adjust it - void ParsePath(const TClientCommand::TConfig& config, const size_t argPos, bool isPathOptional = false); - void AdjustPath(const TClientCommand::TConfig& config); - - TString Path; -}; - + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + virtual int Process(TConfig& config) override; + virtual void RenderCommandsDescription( + TStringStream& stream, + const NColorizer::TColors& colors = NColorizer::TColors(false), + const std::basic_string<bool>& ends = std::basic_string<bool>() + ) override; + virtual void SetFreeArgs(TConfig& config); + +private: + bool HasOptionsToShow(); +}; + +class TCommandWithPath { +protected: + // Get path from free argument and adjust it + void ParsePath(const TClientCommand::TConfig& config, const size_t argPos, bool isPathOptional = false); + void AdjustPath(const TClientCommand::TConfig& config); + + TString Path; +}; + class TCommandWithStreamName { protected: void ParseStreamName(const TClientCommand::TConfig& config, const size_t argPos); @@ -362,5 +362,5 @@ protected: TString StreamName; }; -} -} +} +} diff --git a/ydb/public/lib/ydb_cli/common/common.cpp b/ydb/public/lib/ydb_cli/common/common.cpp index f96a393fca..cf20e68ae2 100644 --- a/ydb/public/lib/ydb_cli/common/common.cpp +++ b/ydb/public/lib/ydb_cli/common/common.cpp @@ -1,167 +1,167 @@ -#include "common.h" - -#include <util/folder/dirut.h> -#include <util/folder/path.h> -#include <util/string/strip.h> - -#ifdef _unix_ -#include <sys/ioctl.h> -#include <termios.h> - -#elif defined(_win_) -#include <windows.h> -#endif - -namespace NYdb { -namespace NConsoleClient { - -TProfileConfig::TProfileConfig(const TString& profileName) - : ProfileName(profileName) -{ -} - -bool TProfileConfig::GetVariable(const TString& name, TString& value) { - if (!InMemory) { - ReadFromFile(); - } - const auto it = Values.find(name); - if (it != Values.end()) { - value = it->second; - return true; - } - return false; -} - -void TProfileConfig::ReadFromFile() { - TString profileConfigFile = "~/.ydb/" + ProfileName + "/config"; - correctpath(profileConfigFile); - TFsPath configFile(profileConfigFile); - if (!configFile.Exists()) { - throw yexception() << "Can't find provided config file \"" << profileConfigFile << "\" for given profile."; - } - TFileInput config(profileConfigFile); - TString line; - while (config.ReadLine(line)) { - auto it = line.find('='); - if (it != TString::npos) { - TString name = Strip(line.substr(0, it)); - TString value = Strip(line.substr(it + 1)); - if (name && value) { - Values[name] = value; - continue; - } - } - throw yexception() << "Error parsing config file \"" << profileConfigFile << "\" for given profile."; - } - if (Values.empty()) { - throw yexception() << "Empty config file \"" << profileConfigFile << "\" for given profile."; - } - InMemory = true; -} - -bool ReadFromFileIfExists(TString& filePath, const TString& fileName, TString& output) { - TFsPath fsPath(filePath); - if (!fsPath.Exists()) { - correctpath(filePath); - fsPath = TFsPath(filePath); - } - TString content; - if (fsPath.Exists()) { - content = Strip(TUnbufferedFileInput(fsPath).ReadAll()); - if (!content) { - throw yexception() << "Empty " << fileName << " file provided: \"" << filePath << "\"."; - } else { - output = content; - return true; - } - } - return false; -} - -TString ReadFromFile(TString& filePath, const TString& fileName) { - TString content; - if (ReadFromFileIfExists(filePath, fileName, content)) { - return content; - } else { - throw yexception() << "Can't find " << fileName << " file \"" << filePath << "\"."; - } -} - -size_t TermWidth() { -#ifdef _unix_ - struct winsize ws; - if (!ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)) { - return ws.ws_col; - } -#endif - - return Max<size_t>(); -} - -TString InputPassword() { - // Disable echoing characters and enable per-symbol input handling -#if defined(_unix_) - termios oldState; - tcgetattr(STDIN_FILENO, &oldState); - termios newState = oldState; - newState.c_lflag &= ~ECHO; - newState.c_lflag &= VEOF; - tcsetattr(STDIN_FILENO, TCSANOW, &newState); -#elif defined(_win_) - HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); - DWORD mode; - DWORD count; - GetConsoleMode(hStdin, &mode); - SetConsoleMode(hStdin, mode & (~ENABLE_ECHO_INPUT) & (~ENABLE_LINE_INPUT)); -#endif - - TString password; - char c; - - for ( -#if defined(_unix_) - Cin >> c; -#elif defined(_win_) - // Cin doesn't handle "Enter" press properly on Windows - ReadConsoleA(hStdin, &c, 1, &count, NULL); -#endif - c != '\r' && c != '\n'; -#if defined(_unix_) - Cin >> c -#elif defined(_win_) - ReadConsoleA(hStdin, &c, 1, &count, NULL) -#endif - ) { - if (c == '\b' || c == 0x7F) { - // Backspace. Remove last char if there is any - if (password.size()) { - Cout << "\b \b"; - password.pop_back(); - } - } else if (c == 0x03) { - // Ctrl + C - Cerr << Endl << "Input interrupted" << Endl; -#if defined(_unix_) - tcsetattr(STDIN_FILENO, TCSANOW, &oldState); -#elif defined(_win_) - SetConsoleMode(hStdin, mode); -#endif - exit(EXIT_FAILURE); - } else { - Cout << '*'; - password.push_back(c); - } - } - Cout << Endl; - -#if defined(_unix_) - tcsetattr(STDIN_FILENO, TCSANOW, &oldState); -#elif defined(_win_) - SetConsoleMode(hStdin, mode); -#endif - - return password; -} - -} -} +#include "common.h" + +#include <util/folder/dirut.h> +#include <util/folder/path.h> +#include <util/string/strip.h> + +#ifdef _unix_ +#include <sys/ioctl.h> +#include <termios.h> + +#elif defined(_win_) +#include <windows.h> +#endif + +namespace NYdb { +namespace NConsoleClient { + +TProfileConfig::TProfileConfig(const TString& profileName) + : ProfileName(profileName) +{ +} + +bool TProfileConfig::GetVariable(const TString& name, TString& value) { + if (!InMemory) { + ReadFromFile(); + } + const auto it = Values.find(name); + if (it != Values.end()) { + value = it->second; + return true; + } + return false; +} + +void TProfileConfig::ReadFromFile() { + TString profileConfigFile = "~/.ydb/" + ProfileName + "/config"; + correctpath(profileConfigFile); + TFsPath configFile(profileConfigFile); + if (!configFile.Exists()) { + throw yexception() << "Can't find provided config file \"" << profileConfigFile << "\" for given profile."; + } + TFileInput config(profileConfigFile); + TString line; + while (config.ReadLine(line)) { + auto it = line.find('='); + if (it != TString::npos) { + TString name = Strip(line.substr(0, it)); + TString value = Strip(line.substr(it + 1)); + if (name && value) { + Values[name] = value; + continue; + } + } + throw yexception() << "Error parsing config file \"" << profileConfigFile << "\" for given profile."; + } + if (Values.empty()) { + throw yexception() << "Empty config file \"" << profileConfigFile << "\" for given profile."; + } + InMemory = true; +} + +bool ReadFromFileIfExists(TString& filePath, const TString& fileName, TString& output) { + TFsPath fsPath(filePath); + if (!fsPath.Exists()) { + correctpath(filePath); + fsPath = TFsPath(filePath); + } + TString content; + if (fsPath.Exists()) { + content = Strip(TUnbufferedFileInput(fsPath).ReadAll()); + if (!content) { + throw yexception() << "Empty " << fileName << " file provided: \"" << filePath << "\"."; + } else { + output = content; + return true; + } + } + return false; +} + +TString ReadFromFile(TString& filePath, const TString& fileName) { + TString content; + if (ReadFromFileIfExists(filePath, fileName, content)) { + return content; + } else { + throw yexception() << "Can't find " << fileName << " file \"" << filePath << "\"."; + } +} + +size_t TermWidth() { +#ifdef _unix_ + struct winsize ws; + if (!ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)) { + return ws.ws_col; + } +#endif + + return Max<size_t>(); +} + +TString InputPassword() { + // Disable echoing characters and enable per-symbol input handling +#if defined(_unix_) + termios oldState; + tcgetattr(STDIN_FILENO, &oldState); + termios newState = oldState; + newState.c_lflag &= ~ECHO; + newState.c_lflag &= VEOF; + tcsetattr(STDIN_FILENO, TCSANOW, &newState); +#elif defined(_win_) + HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + DWORD mode; + DWORD count; + GetConsoleMode(hStdin, &mode); + SetConsoleMode(hStdin, mode & (~ENABLE_ECHO_INPUT) & (~ENABLE_LINE_INPUT)); +#endif + + TString password; + char c; + + for ( +#if defined(_unix_) + Cin >> c; +#elif defined(_win_) + // Cin doesn't handle "Enter" press properly on Windows + ReadConsoleA(hStdin, &c, 1, &count, NULL); +#endif + c != '\r' && c != '\n'; +#if defined(_unix_) + Cin >> c +#elif defined(_win_) + ReadConsoleA(hStdin, &c, 1, &count, NULL) +#endif + ) { + if (c == '\b' || c == 0x7F) { + // Backspace. Remove last char if there is any + if (password.size()) { + Cout << "\b \b"; + password.pop_back(); + } + } else if (c == 0x03) { + // Ctrl + C + Cerr << Endl << "Input interrupted" << Endl; +#if defined(_unix_) + tcsetattr(STDIN_FILENO, TCSANOW, &oldState); +#elif defined(_win_) + SetConsoleMode(hStdin, mode); +#endif + exit(EXIT_FAILURE); + } else { + Cout << '*'; + password.push_back(c); + } + } + Cout << Endl; + +#if defined(_unix_) + tcsetattr(STDIN_FILENO, TCSANOW, &oldState); +#elif defined(_win_) + SetConsoleMode(hStdin, mode); +#endif + + return password; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/common.h b/ydb/public/lib/ydb_cli/common/common.h index f6f801d871..ffb8e4ea78 100644 --- a/ydb/public/lib/ydb_cli/common/common.h +++ b/ydb/public/lib/ydb_cli/common/common.h @@ -1,33 +1,33 @@ -#pragma once - -#include <util/stream/file.h> -#include <util/string/builder.h> -#include <util/generic/string.h> -#include <util/generic/yexception.h> -#include <util/generic/map.h> - -namespace NYdb { -namespace NConsoleClient { - -class TMissUseException : public yexception {}; - -class TProfileConfig { -public: - TProfileConfig(const TString& profileName); - bool GetVariable(const TString& name, TString& value); - -private: - void ReadFromFile(); - - TString ProfileName; - bool InMemory = false; - TMap<TString, TString> Values; -}; - -bool ReadFromFileIfExists(TString& filePath, const TString& fileName, TString& output); -TString ReadFromFile(TString& filePath, const TString& fileName); -size_t TermWidth(); -TString InputPassword(); - -} -} +#pragma once + +#include <util/stream/file.h> +#include <util/string/builder.h> +#include <util/generic/string.h> +#include <util/generic/yexception.h> +#include <util/generic/map.h> + +namespace NYdb { +namespace NConsoleClient { + +class TMissUseException : public yexception {}; + +class TProfileConfig { +public: + TProfileConfig(const TString& profileName); + bool GetVariable(const TString& name, TString& value); + +private: + void ReadFromFile(); + + TString ProfileName; + bool InMemory = false; + TMap<TString, TString> Values; +}; + +bool ReadFromFileIfExists(TString& filePath, const TString& fileName, TString& output); +TString ReadFromFile(TString& filePath, const TString& fileName); +size_t TermWidth(); +TString InputPassword(); + +} +} diff --git a/ydb/public/lib/ydb_cli/common/examples.cpp b/ydb/public/lib/ydb_cli/common/examples.cpp index a5421e7c9e..dc7e0733b2 100644 --- a/ydb/public/lib/ydb_cli/common/examples.cpp +++ b/ydb/public/lib/ydb_cli/common/examples.cpp @@ -1,128 +1,128 @@ -#include "examples.h" - -namespace NYdb { -namespace NConsoleClient { - -TExampleBuilder::TExampleBuilder(TExampleSetBuilder& parent) - : Parent(parent) -{} - -TExampleBuilder& TExampleBuilder::Title(const TString& title) { - Example.Title = title; - return *this; -} - -TExampleBuilder& TExampleBuilder::Text(const TString& text) { - Example.Text = text; - return *this; -} - -TExampleSetBuilder& TExampleBuilder::EndExample() { - return Parent.AddExample(std::move(Example)); -} - -///////////////////////////////////////////////////////////////// - -TExampleSetBuilder& TExampleSetBuilder::Title(const TString& title) { - ExampleSet.Title = title; - return *this; -} - -TExampleSetBuilder& TExampleSetBuilder::AddExample(TExample&& example) { - ExampleSet.Examples.push_back(std::move(example)); - return *this; -} - -TExampleBuilder TExampleSetBuilder::BeginExample() { - return TExampleBuilder(*this); -} - -TExampleSet TExampleSetBuilder::Build() { - return std::move(ExampleSet); -} - -///////////////////////////////////////////////////////////////// - -void TCommandWithExamples::AddExamplesOption(TClientCommand::TConfig& config) { - config.Opts->AddLongOption("help-ex", "Print usage with examples") - .HasArg(NLastGetopt::NO_ARGUMENT) - .IfPresentDisableCompletion() - .Handler(&NLastGetopt::PrintUsageAndExit); -} - -void TCommandWithExamples::AddOptionExamples(const TString& optionName, TExampleSet&& examples) { - OptionExamples[optionName] = std::move(examples); -} - -void TCommandWithExamples::AddCommandExamples(TExampleSet&& examples) { - CommandExamples = std::move(examples); -} - -namespace { - void PrintExamples(TStringStream& descr, const TVector<TExample>& examples) { - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - if (examples.size() == 1) { - if (examples[0].Title) { - descr << examples[0].Title << ':' << Endl; - } - descr << examples[0].Text << Endl; - } else { - size_t orderNum = 0; - for (const auto& example : examples) { - descr << colors.BoldColor() << ++orderNum << ") " << colors.OldColor(); - if (example.Title) { - descr << example.Title << Endl; - } - descr << example.Text << Endl; - } - } - } -} - -void TCommandWithExamples::CheckExamples(const TClientCommand::TConfig& config) { - if (!IsOptionCalled(config)) { - return; - } - - for (const auto& [optionName, examples] : OptionExamples) { - NLastGetopt::TOpt* opt = config.Opts->FindLongOption(optionName); - if (!opt) { - continue; - } - TStringStream descr; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - descr << opt->Help_ << Endl << Endl << colors.BoldColor(); - if (examples.Title) { - descr << examples.Title; - } else { - descr << "Example"; - if (examples.Examples.size() > 1) { - descr << 's'; - } - } - descr << colors.OldColor() << ":" << Endl; - PrintExamples(descr, examples.Examples); - opt->Help_ = descr.Str(); - } - - if (CommandExamples.Examples.size()) { - TStringStream descr; - PrintExamples(descr, CommandExamples.Examples); - config.Opts->AddSection(CommandExamples.Title ? CommandExamples.Title - : (CommandExamples.Examples.size() == 1 ? "Example" : "Examples"), - descr.Str()); - } -} - -bool TCommandWithExamples::IsOptionCalled(const TClientCommand::TConfig& config) const { - for (int i = 0; i < config.ArgC; ++i) { - TString argName = config.ArgV[i]; - if (argName == "--help-ex") { - return true; - } - } - return false; -} - -} -} +#include "examples.h" + +namespace NYdb { +namespace NConsoleClient { + +TExampleBuilder::TExampleBuilder(TExampleSetBuilder& parent) + : Parent(parent) +{} + +TExampleBuilder& TExampleBuilder::Title(const TString& title) { + Example.Title = title; + return *this; +} + +TExampleBuilder& TExampleBuilder::Text(const TString& text) { + Example.Text = text; + return *this; +} + +TExampleSetBuilder& TExampleBuilder::EndExample() { + return Parent.AddExample(std::move(Example)); +} + +///////////////////////////////////////////////////////////////// + +TExampleSetBuilder& TExampleSetBuilder::Title(const TString& title) { + ExampleSet.Title = title; + return *this; +} + +TExampleSetBuilder& TExampleSetBuilder::AddExample(TExample&& example) { + ExampleSet.Examples.push_back(std::move(example)); + return *this; +} + +TExampleBuilder TExampleSetBuilder::BeginExample() { + return TExampleBuilder(*this); +} + +TExampleSet TExampleSetBuilder::Build() { + return std::move(ExampleSet); +} + +///////////////////////////////////////////////////////////////// + +void TCommandWithExamples::AddExamplesOption(TClientCommand::TConfig& config) { + config.Opts->AddLongOption("help-ex", "Print usage with examples") + .HasArg(NLastGetopt::NO_ARGUMENT) + .IfPresentDisableCompletion() + .Handler(&NLastGetopt::PrintUsageAndExit); +} + +void TCommandWithExamples::AddOptionExamples(const TString& optionName, TExampleSet&& examples) { + OptionExamples[optionName] = std::move(examples); +} + +void TCommandWithExamples::AddCommandExamples(TExampleSet&& examples) { + CommandExamples = std::move(examples); +} + +namespace { + void PrintExamples(TStringStream& descr, const TVector<TExample>& examples) { + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + if (examples.size() == 1) { + if (examples[0].Title) { + descr << examples[0].Title << ':' << Endl; + } + descr << examples[0].Text << Endl; + } else { + size_t orderNum = 0; + for (const auto& example : examples) { + descr << colors.BoldColor() << ++orderNum << ") " << colors.OldColor(); + if (example.Title) { + descr << example.Title << Endl; + } + descr << example.Text << Endl; + } + } + } +} + +void TCommandWithExamples::CheckExamples(const TClientCommand::TConfig& config) { + if (!IsOptionCalled(config)) { + return; + } + + for (const auto& [optionName, examples] : OptionExamples) { + NLastGetopt::TOpt* opt = config.Opts->FindLongOption(optionName); + if (!opt) { + continue; + } + TStringStream descr; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + descr << opt->Help_ << Endl << Endl << colors.BoldColor(); + if (examples.Title) { + descr << examples.Title; + } else { + descr << "Example"; + if (examples.Examples.size() > 1) { + descr << 's'; + } + } + descr << colors.OldColor() << ":" << Endl; + PrintExamples(descr, examples.Examples); + opt->Help_ = descr.Str(); + } + + if (CommandExamples.Examples.size()) { + TStringStream descr; + PrintExamples(descr, CommandExamples.Examples); + config.Opts->AddSection(CommandExamples.Title ? CommandExamples.Title + : (CommandExamples.Examples.size() == 1 ? "Example" : "Examples"), + descr.Str()); + } +} + +bool TCommandWithExamples::IsOptionCalled(const TClientCommand::TConfig& config) const { + for (int i = 0; i < config.ArgC; ++i) { + TString argName = config.ArgV[i]; + if (argName == "--help-ex") { + return true; + } + } + return false; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/examples.h b/ydb/public/lib/ydb_cli/common/examples.h index 9145a23494..44b5de5ca4 100644 --- a/ydb/public/lib/ydb_cli/common/examples.h +++ b/ydb/public/lib/ydb_cli/common/examples.h @@ -1,98 +1,98 @@ -#pragma once - -#include "command.h" - -namespace NYdb { -namespace NConsoleClient { - - -// Format description: -// -// --option1 <Option1 description> -// -// Option1 examples title: -// 1) Example 1 title -// Example 1 text -// 2) Example 2 title -// Example 2 text -// ... -// -// --option2 <Option2 description> -// -// Option2 examples title: -// 1) Example 1 title -// Example 1 text -// 2) Example 2 title -// Example 2 text -// ... -// -// ... -// -// Command examples title: -// 1) Example 1 title -// Example 1 text -// 2) Example 2 title -// Example 2 text -// - -struct TExample { - TString Title; - TString Text; -}; - -struct TExampleSet { - TString Title; - TVector<TExample> Examples; -}; - -class TExampleSetBuilder; - -class TExampleBuilder { -public: - explicit TExampleBuilder(TExampleSetBuilder& parent); - - // Optional - TExampleBuilder& Title(const TString& title); - TExampleBuilder& Text(const TString& text); - TExampleSetBuilder& EndExample(); - -private: - TExample Example; - TExampleSetBuilder& Parent; -}; - -class TExampleSetBuilder { -public: - TExampleSetBuilder() = default; - - // Optional. Default value : "Examples" ("Example" in case of a single example) - TExampleSetBuilder& Title(const TString& title); - TExampleBuilder BeginExample(); - - TExampleSetBuilder& AddExample(TExample&& example); - TExampleSet Build(); - -private: - TExampleSet ExampleSet; -}; - -class TCommandWithExamples { -protected: - void AddExamplesOption(TClientCommand::TConfig& config); - - void AddOptionExamples(const TString& optionName, TExampleSet&& examples); - void AddCommandExamples(TExampleSet&& examples); - - // Should be called at the end of Config method of inherited commands when all other options are already added - // This method adds example texts to option descriptions if --help-ex option was used - void CheckExamples(const TClientCommand::TConfig& config); - -private: - bool IsOptionCalled(const TClientCommand::TConfig& config) const; - - THashMap<TString, TExampleSet> OptionExamples; - TExampleSet CommandExamples; -}; - -} -} +#pragma once + +#include "command.h" + +namespace NYdb { +namespace NConsoleClient { + + +// Format description: +// +// --option1 <Option1 description> +// +// Option1 examples title: +// 1) Example 1 title +// Example 1 text +// 2) Example 2 title +// Example 2 text +// ... +// +// --option2 <Option2 description> +// +// Option2 examples title: +// 1) Example 1 title +// Example 1 text +// 2) Example 2 title +// Example 2 text +// ... +// +// ... +// +// Command examples title: +// 1) Example 1 title +// Example 1 text +// 2) Example 2 title +// Example 2 text +// + +struct TExample { + TString Title; + TString Text; +}; + +struct TExampleSet { + TString Title; + TVector<TExample> Examples; +}; + +class TExampleSetBuilder; + +class TExampleBuilder { +public: + explicit TExampleBuilder(TExampleSetBuilder& parent); + + // Optional + TExampleBuilder& Title(const TString& title); + TExampleBuilder& Text(const TString& text); + TExampleSetBuilder& EndExample(); + +private: + TExample Example; + TExampleSetBuilder& Parent; +}; + +class TExampleSetBuilder { +public: + TExampleSetBuilder() = default; + + // Optional. Default value : "Examples" ("Example" in case of a single example) + TExampleSetBuilder& Title(const TString& title); + TExampleBuilder BeginExample(); + + TExampleSetBuilder& AddExample(TExample&& example); + TExampleSet Build(); + +private: + TExampleSet ExampleSet; +}; + +class TCommandWithExamples { +protected: + void AddExamplesOption(TClientCommand::TConfig& config); + + void AddOptionExamples(const TString& optionName, TExampleSet&& examples); + void AddCommandExamples(TExampleSet&& examples); + + // Should be called at the end of Config method of inherited commands when all other options are already added + // This method adds example texts to option descriptions if --help-ex option was used + void CheckExamples(const TClientCommand::TConfig& config); + +private: + bool IsOptionCalled(const TClientCommand::TConfig& config) const; + + THashMap<TString, TExampleSet> OptionExamples; + TExampleSet CommandExamples; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 372d4ee6e3..7184ab41d1 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -9,31 +9,31 @@ namespace NYdb { namespace NConsoleClient { -namespace { - THashMap<EOutputFormat, TString> InputFormatDescriptions = { - { EOutputFormat::JsonUnicode, "Input in json format, binary strings are decoded with unicode characters" }, +namespace { + THashMap<EOutputFormat, TString> InputFormatDescriptions = { + { EOutputFormat::JsonUnicode, "Input in json format, binary strings are decoded with unicode characters" }, { EOutputFormat::JsonBase64, "Input in json format, binary strings are decoded with base64" }, { EOutputFormat::Csv, "Input in csv format" }, { EOutputFormat::Tsv, "Input in tsv format" } - }; - - THashMap<EOutputFormat, TString> FormatDescriptions = { - { EOutputFormat::Pretty, "Human readable output" }, - { EOutputFormat::JsonUnicode, "Output in json format, binary strings are encoded with unicode characters. " - "Every row is a separate json on a separate line." }, - { EOutputFormat::JsonUnicodeArray, "Output in json format, binary strings are encoded with unicode characters. " - "Every resultset is a json array of rows. " - "Every row is a separate json on a separate line." }, - { EOutputFormat::JsonBase64, "Output in json format, binary strings are encoded with base64. " - "Every row is a separate json on a separate line." }, - { EOutputFormat::JsonBase64Array, "Output in json format, binary strings are encoded with base64. " - "Every resultset is a json array of rows. " - "Every row is a separate json on a separate line." }, - { EOutputFormat::ProtoJsonBase64, "Output result protobuf in json format, binary strings are encoded with base64" }, + }; + + THashMap<EOutputFormat, TString> FormatDescriptions = { + { EOutputFormat::Pretty, "Human readable output" }, + { EOutputFormat::JsonUnicode, "Output in json format, binary strings are encoded with unicode characters. " + "Every row is a separate json on a separate line." }, + { EOutputFormat::JsonUnicodeArray, "Output in json format, binary strings are encoded with unicode characters. " + "Every resultset is a json array of rows. " + "Every row is a separate json on a separate line." }, + { EOutputFormat::JsonBase64, "Output in json format, binary strings are encoded with base64. " + "Every row is a separate json on a separate line." }, + { EOutputFormat::JsonBase64Array, "Output in json format, binary strings are encoded with base64. " + "Every resultset is a json array of rows. " + "Every row is a separate json on a separate line." }, + { EOutputFormat::ProtoJsonBase64, "Output result protobuf in json format, binary strings are encoded with base64" }, { EOutputFormat::Csv, "Output in csv format" }, - }; -} - + }; +} + void TCommandWithResponseHeaders::PrintResponseHeader(const TStatus& status) { if (!ShowHeaders) return; @@ -57,59 +57,59 @@ void TCommandWithResponseHeaders::PrintResponseHeaderPretty(const TStatus& statu const TString TCommandWithResponseHeaders::ResponseHeadersHelp = "Show response metadata for ydb call"; -// Deprecated -void TCommandWithFormat::AddJsonOption(TClientCommand::TConfig& config, const TString& description) { - config.Opts->AddLongOption("json", description).NoArgument() - .StoreValue(&OutputFormat, EOutputFormat::Json).StoreValue(&DeprecatedOptionUsed, true); -} - -void TCommandWithFormat::AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { - TStringStream description; - description << "Input format. Available options: "; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - for (const auto& format : allowedFormats) { - auto findResult = InputFormatDescriptions.find(format); - Y_VERIFY(findResult != InputFormatDescriptions.end(), +// Deprecated +void TCommandWithFormat::AddJsonOption(TClientCommand::TConfig& config, const TString& description) { + config.Opts->AddLongOption("json", description).NoArgument() + .StoreValue(&OutputFormat, EOutputFormat::Json).StoreValue(&DeprecatedOptionUsed, true); +} + +void TCommandWithFormat::AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { + TStringStream description; + description << "Input format. Available options: "; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + for (const auto& format : allowedFormats) { + auto findResult = InputFormatDescriptions.find(format); + Y_VERIFY(findResult != InputFormatDescriptions.end(), "Couldn't find description for %s input format", (TStringBuilder() << format).c_str()); - description << "\n " << colors.BoldColor() << format << colors.OldColor() - << "\n " << findResult->second; - } - config.Opts->AddLongOption("input-format", description.Str()) - .RequiredArgument("STRING").StoreResult(&InputFormat); - AllowedInputFormats = allowedFormats; -} - -void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { - TStringStream description; - description << "Output format. Available options: "; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - for (const auto& format : allowedFormats) { - auto findResult = FormatDescriptions.find(format); - Y_VERIFY(findResult != FormatDescriptions.end(), - "Couldn't find description for %s output format", (TStringBuilder() << format).c_str()); description << "\n " << colors.BoldColor() << format << colors.OldColor() << "\n " << findResult->second; - } - config.Opts->AddLongOption("format", description.Str()) - .RequiredArgument("STRING").StoreResult(&OutputFormat); - AllowedFormats = allowedFormats; + } + config.Opts->AddLongOption("input-format", description.Str()) + .RequiredArgument("STRING").StoreResult(&InputFormat); + AllowedInputFormats = allowedFormats; } - -void TCommandWithFormat::ParseFormats() { - if (InputFormat != EOutputFormat::Default - && std::find(AllowedInputFormats.begin(), AllowedInputFormats.end(), InputFormat) == AllowedInputFormats.end()) { - throw TMissUseException() << "Input format " << InputFormat << " is not available for this command"; - } - - - if (OutputFormat == EOutputFormat::Default || DeprecatedOptionUsed) { - return; - } - if (std::find(AllowedFormats.begin(), AllowedFormats.end(), OutputFormat) == AllowedFormats.end()) { - throw TMissUseException() << "Output format " << OutputFormat << " is not available for this command"; - } + +void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { + TStringStream description; + description << "Output format. Available options: "; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + for (const auto& format : allowedFormats) { + auto findResult = FormatDescriptions.find(format); + Y_VERIFY(findResult != FormatDescriptions.end(), + "Couldn't find description for %s output format", (TStringBuilder() << format).c_str()); + description << "\n " << colors.BoldColor() << format << colors.OldColor() + << "\n " << findResult->second; + } + config.Opts->AddLongOption("format", description.Str()) + .RequiredArgument("STRING").StoreResult(&OutputFormat); + AllowedFormats = allowedFormats; } - + +void TCommandWithFormat::ParseFormats() { + if (InputFormat != EOutputFormat::Default + && std::find(AllowedInputFormats.begin(), AllowedInputFormats.end(), InputFormat) == AllowedInputFormats.end()) { + throw TMissUseException() << "Input format " << InputFormat << " is not available for this command"; + } + + + if (OutputFormat == EOutputFormat::Default || DeprecatedOptionUsed) { + return; + } + if (std::find(AllowedFormats.begin(), AllowedFormats.end(), OutputFormat) == AllowedFormats.end()) { + throw TMissUseException() << "Output format " << OutputFormat << " is not available for this command"; + } +} + void TQueryPlanPrinter::Print(const TString& plan) { switch (Format) { case EOutputFormat::Default: @@ -259,144 +259,144 @@ TString TQueryPlanPrinter::JsonToString(const NJson::TJsonValue& jsonValue) { return str; } - -TResultSetPrinter::TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted) - : Format(format) - , IsInterrupted(isInterrupted) -{} - -TResultSetPrinter::~TResultSetPrinter() { - if (PrintedSomething && !IsInterrupted()) { - EndResultSet(); - } + +TResultSetPrinter::TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted) + : Format(format) + , IsInterrupted(isInterrupted) +{} + +TResultSetPrinter::~TResultSetPrinter() { + if (PrintedSomething && !IsInterrupted()) { + EndResultSet(); + } +} + +void TResultSetPrinter::Print(const TResultSet& resultSet) { + if (FirstPart) { + BeginResultSet(); + } else { + EndLineBeforeNextResult(); + } + PrintedSomething = true; + + switch (Format) { + case EOutputFormat::Default: + case EOutputFormat::Pretty: + PrintPretty(resultSet); + break; + case EOutputFormat::JsonUnicodeArray: + PrintJsonArray(resultSet, EBinaryStringEncoding::Unicode); + break; + case EOutputFormat::JsonUnicode: + FormatResultSetJson(resultSet, &Cout, EBinaryStringEncoding::Unicode); + break; + case EOutputFormat::JsonBase64Array: + PrintJsonArray(resultSet, EBinaryStringEncoding::Base64); + break; + case EOutputFormat::JsonBase64: + FormatResultSetJson(resultSet, &Cout, EBinaryStringEncoding::Base64); + break; + case EOutputFormat::Csv: + PrintCsv(resultSet); + break; + default: + throw TMissUseException() << "This command doesn't support " << Format << " output format"; + } +} + +void TResultSetPrinter::Reset() { + if (PrintedSomething) { + EndResultSet(); + FirstPart = true; + } } - -void TResultSetPrinter::Print(const TResultSet& resultSet) { - if (FirstPart) { - BeginResultSet(); - } else { - EndLineBeforeNextResult(); - } - PrintedSomething = true; - - switch (Format) { - case EOutputFormat::Default: - case EOutputFormat::Pretty: - PrintPretty(resultSet); - break; - case EOutputFormat::JsonUnicodeArray: - PrintJsonArray(resultSet, EBinaryStringEncoding::Unicode); - break; - case EOutputFormat::JsonUnicode: - FormatResultSetJson(resultSet, &Cout, EBinaryStringEncoding::Unicode); - break; - case EOutputFormat::JsonBase64Array: - PrintJsonArray(resultSet, EBinaryStringEncoding::Base64); - break; - case EOutputFormat::JsonBase64: - FormatResultSetJson(resultSet, &Cout, EBinaryStringEncoding::Base64); - break; - case EOutputFormat::Csv: - PrintCsv(resultSet); - break; - default: - throw TMissUseException() << "This command doesn't support " << Format << " output format"; - } + +void TResultSetPrinter::BeginResultSet() { + switch (Format) { + case EOutputFormat::JsonUnicodeArray: + case EOutputFormat::JsonBase64Array: + Cout << '['; + break; + default: + break; + } } - -void TResultSetPrinter::Reset() { - if (PrintedSomething) { - EndResultSet(); - FirstPart = true; - } -} - -void TResultSetPrinter::BeginResultSet() { - switch (Format) { - case EOutputFormat::JsonUnicodeArray: - case EOutputFormat::JsonBase64Array: - Cout << '['; + +void TResultSetPrinter::EndResultSet() { + switch (Format) { + case EOutputFormat::JsonUnicodeArray: + case EOutputFormat::JsonBase64Array: + Cout << ']' << Endl; break; - default: - break; - } -} - -void TResultSetPrinter::EndResultSet() { - switch (Format) { - case EOutputFormat::JsonUnicodeArray: - case EOutputFormat::JsonBase64Array: - Cout << ']' << Endl; + default: break; - default: - break; - } -} - -void TResultSetPrinter::EndLineBeforeNextResult() { - switch (Format) { - case EOutputFormat::JsonUnicodeArray: - case EOutputFormat::JsonBase64Array: - Cout << ',' << Endl; + } +} + +void TResultSetPrinter::EndLineBeforeNextResult() { + switch (Format) { + case EOutputFormat::JsonUnicodeArray: + case EOutputFormat::JsonBase64Array: + Cout << ',' << Endl; + break; + default: break; - default: - break; - } -} - -void TResultSetPrinter::PrintPretty(const TResultSet& resultSet) { - const TVector<TColumn>& columns = resultSet.GetColumnsMeta(); - TResultSetParser parser(resultSet); - TVector<TString> columnNames; - for (const auto& column : columns) { - columnNames.push_back(column.Name); - } - - TPrettyTableConfig tableConfig; - if (!FirstPart) { - tableConfig.WithoutHeader(); - } - TPrettyTable table(columnNames, tableConfig); - - while (parser.TryNextRow()) { - auto& row = table.AddRow(); - for (ui32 i = 0; i < columns.size(); ++i) { - row.Column(i, FormatValueJson(parser.GetValue(i), EBinaryStringEncoding::Unicode)); - } - } - - Cout << table; -} - -void TResultSetPrinter::PrintJsonArray(const TResultSet& resultSet, EBinaryStringEncoding encoding) { - auto columns = resultSet.GetColumnsMeta(); - - TResultSetParser parser(resultSet); - bool firstRow = true; - while (parser.TryNextRow()) { - if (!firstRow) { - EndLineBeforeNextResult(); - } else { - firstRow = false; - } - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &Cout); - FormatResultRowJson(parser, columns, writer, encoding); - } -} - -void TResultSetPrinter::PrintCsv(const TResultSet& resultSet) { - const TVector<TColumn>& columns = resultSet.GetColumnsMeta(); - TResultSetParser parser(resultSet); - while (parser.TryNextRow()) { - for (ui32 i = 0; i < columns.size(); ++i) { - Cout << FormatValueJson(parser.GetValue(i), EBinaryStringEncoding::Unicode); - if (i < columns.size() - 1) { - Cout << ","; - } - } - Cout << Endl; - } -} - -} -} + } +} + +void TResultSetPrinter::PrintPretty(const TResultSet& resultSet) { + const TVector<TColumn>& columns = resultSet.GetColumnsMeta(); + TResultSetParser parser(resultSet); + TVector<TString> columnNames; + for (const auto& column : columns) { + columnNames.push_back(column.Name); + } + + TPrettyTableConfig tableConfig; + if (!FirstPart) { + tableConfig.WithoutHeader(); + } + TPrettyTable table(columnNames, tableConfig); + + while (parser.TryNextRow()) { + auto& row = table.AddRow(); + for (ui32 i = 0; i < columns.size(); ++i) { + row.Column(i, FormatValueJson(parser.GetValue(i), EBinaryStringEncoding::Unicode)); + } + } + + Cout << table; +} + +void TResultSetPrinter::PrintJsonArray(const TResultSet& resultSet, EBinaryStringEncoding encoding) { + auto columns = resultSet.GetColumnsMeta(); + + TResultSetParser parser(resultSet); + bool firstRow = true; + while (parser.TryNextRow()) { + if (!firstRow) { + EndLineBeforeNextResult(); + } else { + firstRow = false; + } + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE, &Cout); + FormatResultRowJson(parser, columns, writer, encoding); + } +} + +void TResultSetPrinter::PrintCsv(const TResultSet& resultSet) { + const TVector<TColumn>& columns = resultSet.GetColumnsMeta(); + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + for (ui32 i = 0; i < columns.size(); ++i) { + Cout << FormatValueJson(parser.GetValue(i), EBinaryStringEncoding::Unicode); + if (i < columns.size() - 1) { + Cout << ","; + } + } + Cout << Endl; + } +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index 2225263c83..2d8d6a2e38 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -21,50 +21,50 @@ protected: class TCommandWithFormat { protected: - void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); - void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); - void ParseFormats(); + void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); + void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); + void ParseFormats(); - // Deprecated - void AddJsonOption(TClientCommand::TConfig& config, - const TString& description = "(Deprecated, will be removed soon. Use --format option instead)" - " Output in json format"); + // Deprecated + void AddJsonOption(TClientCommand::TConfig& config, + const TString& description = "(Deprecated, will be removed soon. Use --format option instead)" + " Output in json format"); protected: - EOutputFormat OutputFormat = EOutputFormat::Default; - EOutputFormat InputFormat = EOutputFormat::Default; - -private: - TVector<EOutputFormat> AllowedInputFormats; - TVector<EOutputFormat> AllowedFormats; - bool DeprecatedOptionUsed = false; + EOutputFormat OutputFormat = EOutputFormat::Default; + EOutputFormat InputFormat = EOutputFormat::Default; + +private: + TVector<EOutputFormat> AllowedInputFormats; + TVector<EOutputFormat> AllowedFormats; + bool DeprecatedOptionUsed = false; +}; + +class TResultSetPrinter { +public: + TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted = []() { return false; }); + ~TResultSetPrinter(); + + void Print(const TResultSet& resultSet); + + // Should be called between result sets + void Reset(); + +private: + void BeginResultSet(); + void EndResultSet(); + void EndLineBeforeNextResult(); + + void PrintPretty(const TResultSet& resultSet); + void PrintJsonArray(const TResultSet& resultSet, EBinaryStringEncoding encoding); + void PrintCsv(const TResultSet& resultSet); + + bool FirstPart = true; + bool PrintedSomething = false; + EOutputFormat Format; + std::function<bool()> IsInterrupted; }; -class TResultSetPrinter { -public: - TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted = []() { return false; }); - ~TResultSetPrinter(); - - void Print(const TResultSet& resultSet); - - // Should be called between result sets - void Reset(); - -private: - void BeginResultSet(); - void EndResultSet(); - void EndLineBeforeNextResult(); - - void PrintPretty(const TResultSet& resultSet); - void PrintJsonArray(const TResultSet& resultSet, EBinaryStringEncoding encoding); - void PrintCsv(const TResultSet& resultSet); - - bool FirstPart = true; - bool PrintedSomething = false; - EOutputFormat Format; - std::function<bool()> IsInterrupted; -}; - class TQueryPlanPrinter { public: TQueryPlanPrinter(EOutputFormat format, bool analyzeMode = false) diff --git a/ydb/public/lib/ydb_cli/common/formats.h b/ydb/public/lib/ydb_cli/common/formats.h index 43b6380b24..46e83d943d 100644 --- a/ydb/public/lib/ydb_cli/common/formats.h +++ b/ydb/public/lib/ydb_cli/common/formats.h @@ -4,14 +4,14 @@ namespace NYdb { namespace NConsoleClient { enum class EOutputFormat { - Default /* "default" */, - Pretty /* "pretty" */, - Json /* "json" (deprecated) */, - JsonUnicode /* "json-unicode" */, - JsonUnicodeArray /* "json-unicode-array" */, - JsonBase64 /* "json-base64" */, - JsonBase64Array /* "json-base64-array" */, - ProtoJsonBase64 /* "proto-json-base64" */, + Default /* "default" */, + Pretty /* "pretty" */, + Json /* "json" (deprecated) */, + JsonUnicode /* "json-unicode" */, + JsonUnicodeArray /* "json-unicode-array" */, + JsonBase64 /* "json-base64" */, + JsonBase64Array /* "json-base64-array" */, + ProtoJsonBase64 /* "proto-json-base64" */, Csv /* "csv" */, Tsv /* "tsv" */, }; diff --git a/ydb/public/lib/ydb_cli/common/interruptible.cpp b/ydb/public/lib/ydb_cli/common/interruptible.cpp index 2baca73263..eb6fafadb1 100644 --- a/ydb/public/lib/ydb_cli/common/interruptible.cpp +++ b/ydb/public/lib/ydb_cli/common/interruptible.cpp @@ -1,22 +1,22 @@ -#include "interruptible.h" - -namespace NYdb { -namespace NConsoleClient { - -TAtomic TInterruptibleCommand::Interrupted = false; - -void TInterruptibleCommand::OnTerminate(int) { - AtomicSet(Interrupted, true); -} - -void TInterruptibleCommand::SetInterruptHandlers() { - signal(SIGINT, &TInterruptibleCommand::OnTerminate); - signal(SIGTERM, &TInterruptibleCommand::OnTerminate); -} - -bool TInterruptibleCommand::IsInterrupted() { - return AtomicGet(Interrupted); -} - -} -} +#include "interruptible.h" + +namespace NYdb { +namespace NConsoleClient { + +TAtomic TInterruptibleCommand::Interrupted = false; + +void TInterruptibleCommand::OnTerminate(int) { + AtomicSet(Interrupted, true); +} + +void TInterruptibleCommand::SetInterruptHandlers() { + signal(SIGINT, &TInterruptibleCommand::OnTerminate); + signal(SIGTERM, &TInterruptibleCommand::OnTerminate); +} + +bool TInterruptibleCommand::IsInterrupted() { + return AtomicGet(Interrupted); +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/interruptible.h b/ydb/public/lib/ydb_cli/common/interruptible.h index a8dd3af66b..84647316c5 100644 --- a/ydb/public/lib/ydb_cli/common/interruptible.h +++ b/ydb/public/lib/ydb_cli/common/interruptible.h @@ -1,23 +1,23 @@ -#pragma once - -#include "command.h" -#include "formats.h" - +#pragma once + +#include "command.h" +#include "formats.h" + #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - -namespace NYdb { -namespace NConsoleClient { - -class TInterruptibleCommand { -protected: - static void OnTerminate(int); - static void SetInterruptHandlers(); - static bool IsInterrupted(); - -private: - static TAtomic Interrupted; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +class TInterruptibleCommand { +protected: + static void OnTerminate(int); + static void SetInterruptHandlers(); + static bool IsInterrupted(); + +private: + static TAtomic Interrupted; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/parameters.cpp b/ydb/public/lib/ydb_cli/common/parameters.cpp index ede5c1d2bc..f34620d1ea 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.cpp +++ b/ydb/public/lib/ydb_cli/common/parameters.cpp @@ -1,92 +1,92 @@ -#include "parameters.h" - +#include "parameters.h" + #include <ydb/public/lib/json_value/ydb_json_value.h> - -namespace NYdb { -namespace NConsoleClient { - -void TCommandWithParameters::ParseParameters() { - for (const auto& parameterOption : ParameterOptions) { - auto equalPos = parameterOption.find("="); - if (equalPos == TString::npos) { - throw TMissUseException() << "Wrong parameter format for \"" << parameterOption - << "\". Parameter option should have equal sign. Example (for linux):\n" - << "--param '$input=1'"; - } - - auto paramName = parameterOption.substr(0, equalPos); - - if (!paramName.StartsWith('$') || equalPos <= 1) { - throw TMissUseException() << "Wrong parameter name \"" << paramName << "\" at option \"" - << parameterOption << "\". " << "Parameter name should start with '$' sign. Example (for linux):\n" - << "--param '$input=1'"; - } - - Parameters[paramName] = parameterOption.substr(equalPos + 1); - } -} - -void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config, const TString& clarification) { - TStringStream descr; - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - descr << "Query parameter[s]."; - if (clarification) { - descr << ' ' << clarification; - } - descr << Endl << "Several parameter options can be specified. " << Endl - << "To change input format use --input-format option." << Endl - << "Escaping depends on operating system."; - config.Opts->AddLongOption('p', "param", descr.Str()) - .RequiredArgument("$name=value").AppendTo(&ParameterOptions); - - AddOptionExamples( - "param", - TExampleSetBuilder() - .Title("Examples (with default json-unicode format)") - .BeginExample() - .Title("One parameter of type Uint64") - .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$id=3" << colors.OldColor() << Endl - << "How to pass in linux: " << colors.BoldColor() << "--param '$id=3'" << colors.OldColor()) - .EndExample() - .BeginExample() - .Title("Two parameters of types Uint64 and Utf8") - .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$key=1 $value=\"One\"" << colors.OldColor() << Endl - << "How to pass in linux: " << colors.BoldColor() << "--param '$key=3' --param '$value=\"One\"'" << colors.OldColor()) - .EndExample() - .BeginExample() - .Title("More complex parameter of type List<Struct<key:Uint64, value:Utf8>>") - .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]" << colors.OldColor() << Endl - << "How to pass in linux: " << colors.BoldColor() << "--param '$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]'" << colors.OldColor()) - .EndExample() - .Build() - ); -} - -TParams TCommandWithParameters::BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat) { - EBinaryStringEncoding encoding; - switch (inputFormat) { - case EOutputFormat::Default: - case EOutputFormat::JsonUnicode: - encoding = EBinaryStringEncoding::Unicode; - break; - case EOutputFormat::JsonBase64: - encoding = EBinaryStringEncoding::Base64; - break; - default: - throw TMissUseException() << "Unknown input format: " << inputFormat; - } - - TParamsBuilder paramBuilder; - for (const auto&[name, value] : Parameters) { - auto paramIt = paramTypes.find(name); - if (paramIt == paramTypes.end()) { - throw TMissUseException() << "Query does not contain parameter \"" << name << "\"."; - } - const TType& type = (*paramIt).second; - paramBuilder.AddParam(name, JsonToYdbValue(value, type, encoding)); - } - return paramBuilder.Build(); -} - -} -} + +namespace NYdb { +namespace NConsoleClient { + +void TCommandWithParameters::ParseParameters() { + for (const auto& parameterOption : ParameterOptions) { + auto equalPos = parameterOption.find("="); + if (equalPos == TString::npos) { + throw TMissUseException() << "Wrong parameter format for \"" << parameterOption + << "\". Parameter option should have equal sign. Example (for linux):\n" + << "--param '$input=1'"; + } + + auto paramName = parameterOption.substr(0, equalPos); + + if (!paramName.StartsWith('$') || equalPos <= 1) { + throw TMissUseException() << "Wrong parameter name \"" << paramName << "\" at option \"" + << parameterOption << "\". " << "Parameter name should start with '$' sign. Example (for linux):\n" + << "--param '$input=1'"; + } + + Parameters[paramName] = parameterOption.substr(equalPos + 1); + } +} + +void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config, const TString& clarification) { + TStringStream descr; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + descr << "Query parameter[s]."; + if (clarification) { + descr << ' ' << clarification; + } + descr << Endl << "Several parameter options can be specified. " << Endl + << "To change input format use --input-format option." << Endl + << "Escaping depends on operating system."; + config.Opts->AddLongOption('p', "param", descr.Str()) + .RequiredArgument("$name=value").AppendTo(&ParameterOptions); + + AddOptionExamples( + "param", + TExampleSetBuilder() + .Title("Examples (with default json-unicode format)") + .BeginExample() + .Title("One parameter of type Uint64") + .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$id=3" << colors.OldColor() << Endl + << "How to pass in linux: " << colors.BoldColor() << "--param '$id=3'" << colors.OldColor()) + .EndExample() + .BeginExample() + .Title("Two parameters of types Uint64 and Utf8") + .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$key=1 $value=\"One\"" << colors.OldColor() << Endl + << "How to pass in linux: " << colors.BoldColor() << "--param '$key=3' --param '$value=\"One\"'" << colors.OldColor()) + .EndExample() + .BeginExample() + .Title("More complex parameter of type List<Struct<key:Uint64, value:Utf8>>") + .Text(TStringBuilder() << "What cli expects: " << colors.BoldColor() << "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]" << colors.OldColor() << Endl + << "How to pass in linux: " << colors.BoldColor() << "--param '$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]'" << colors.OldColor()) + .EndExample() + .Build() + ); +} + +TParams TCommandWithParameters::BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat) { + EBinaryStringEncoding encoding; + switch (inputFormat) { + case EOutputFormat::Default: + case EOutputFormat::JsonUnicode: + encoding = EBinaryStringEncoding::Unicode; + break; + case EOutputFormat::JsonBase64: + encoding = EBinaryStringEncoding::Base64; + break; + default: + throw TMissUseException() << "Unknown input format: " << inputFormat; + } + + TParamsBuilder paramBuilder; + for (const auto&[name, value] : Parameters) { + auto paramIt = paramTypes.find(name); + if (paramIt == paramTypes.end()) { + throw TMissUseException() << "Query does not contain parameter \"" << name << "\"."; + } + const TType& type = (*paramIt).second; + paramBuilder.AddParam(name, JsonToYdbValue(value, type, encoding)); + } + return paramBuilder.Build(); +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/parameters.h b/ydb/public/lib/ydb_cli/common/parameters.h index 503075a5a5..2a33734e16 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.h +++ b/ydb/public/lib/ydb_cli/common/parameters.h @@ -1,24 +1,24 @@ -#pragma once - -#include "examples.h" -#include "command.h" -#include "formats.h" - +#pragma once + +#include "examples.h" +#include "command.h" +#include "formats.h" + #include <ydb/public/sdk/cpp/client/ydb_params/params.h> - -namespace NYdb { -namespace NConsoleClient { - -class TCommandWithParameters : public TCommandWithExamples { -protected: - void ParseParameters(); - void AddParametersOption(TClientCommand::TConfig& config, const TString& clarification = ""); - TParams BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat); - -protected: - TVector<TString> ParameterOptions; - TMap<TString, TString> Parameters; -}; - -} -} + +namespace NYdb { +namespace NConsoleClient { + +class TCommandWithParameters : public TCommandWithExamples { +protected: + void ParseParameters(); + void AddParametersOption(TClientCommand::TConfig& config, const TString& clarification = ""); + TParams BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat); + +protected: + TVector<TString> ParameterOptions; + TMap<TString, TString> Parameters; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.cpp b/ydb/public/lib/ydb_cli/common/pretty_table.cpp index 58924c8482..d376454198 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.cpp +++ b/ydb/public/lib/ydb_cli/common/pretty_table.cpp @@ -1,5 +1,5 @@ #include "pretty_table.h" -#include "common.h" +#include "common.h" #include <util/generic/algorithm.h> #include <util/generic/xrange.h> diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.h b/ydb/public/lib/ydb_cli/common/pretty_table.h index b11cefbfbe..f39411d902 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.h +++ b/ydb/public/lib/ydb_cli/common/pretty_table.h @@ -33,7 +33,7 @@ public: class TRow { friend class TPrettyTable; - explicit TRow(const TVector<TString>& columnNames) { + explicit TRow(const TVector<TString>& columnNames) { for (const auto& name : columnNames) { Columns.push_back({name}); } @@ -75,7 +75,7 @@ public: }; public: - explicit TPrettyTable(const TVector<TString>& columnNames, const TPrettyTableConfig& config = TPrettyTableConfig()) + explicit TPrettyTable(const TVector<TString>& columnNames, const TPrettyTableConfig& config = TPrettyTableConfig()) : Columns(columnNames.size()) , Config(config) { diff --git a/ydb/public/lib/ydb_cli/common/print_operation.cpp b/ydb/public/lib/ydb_cli/common/print_operation.cpp index 9a9ef883f0..28a84375dc 100644 --- a/ydb/public/lib/ydb_cli/common/print_operation.cpp +++ b/ydb/public/lib/ydb_cli/common/print_operation.cpp @@ -221,8 +221,8 @@ namespace { template <typename T> void PrintOperationImpl(const T& operation, EOutputFormat format) { switch (format) { - case EOutputFormat::Default: - case EOutputFormat::Pretty: + case EOutputFormat::Default: + case EOutputFormat::Pretty: { auto table = MakeTable(operation); PrettyPrint(operation, table); @@ -231,10 +231,10 @@ namespace { } case EOutputFormat::Json: - Cerr << "Warning! Option --json is deprecated and will be removed soon. " - << "Use \"--format proto-json-base64\" option instead." << Endl; + Cerr << "Warning! Option --json is deprecated and will be removed soon. " + << "Use \"--format proto-json-base64\" option instead." << Endl; [[fallthrough]]; - case EOutputFormat::ProtoJsonBase64: + case EOutputFormat::ProtoJsonBase64: Cout << operation.ToJsonString() << Endl; break; @@ -246,8 +246,8 @@ namespace { template <typename T> void PrintOperationsListImpl(const T& operations, EOutputFormat format) { switch (format) { - case EOutputFormat::Default: - case EOutputFormat::Pretty: + case EOutputFormat::Default: + case EOutputFormat::Pretty: if (operations.GetList()) { auto table = MakeTable(operations.GetList().front()); for (const auto& operation : operations.GetList()) { @@ -259,10 +259,10 @@ namespace { break; case EOutputFormat::Json: - Cerr << "Warning! Option --json is deprecated and will be removed soon. " - << "Use \"--format proto-json-base64\" option instead." << Endl; + Cerr << "Warning! Option --json is deprecated and will be removed soon. " + << "Use \"--format proto-json-base64\" option instead." << Endl; [[fallthrough]]; - case EOutputFormat::ProtoJsonBase64: + case EOutputFormat::ProtoJsonBase64: Cout << operations.ToJsonString() << Endl; break; diff --git a/ydb/public/lib/ydb_cli/common/profile_manager.cpp b/ydb/public/lib/ydb_cli/common/profile_manager.cpp index b8165ba237..2dc09ecf61 100644 --- a/ydb/public/lib/ydb_cli/common/profile_manager.cpp +++ b/ydb/public/lib/ydb_cli/common/profile_manager.cpp @@ -1,173 +1,173 @@ -#include "profile_manager.h" - -#include <util/folder/path.h> -#include <util/stream/file.h> -#include <util/string/builder.h> -#include <util/system/env.h> - -namespace NYdb { -namespace NConsoleClient { - -class TProfile : public IProfile { -public: - TProfile(const TString& name, YAML::Node yamlProfile, bool& profilesChanged, std::shared_ptr<IProfileManager> manager) - : Name(name) - , YamlProfile(yamlProfile) - , ProfilesChanged(profilesChanged) - , ProfileManager(manager) - { - } - - TString GetName() const override { - return Name; - } - - bool Has(const TString& key) const noexcept override { - return YamlProfile[key]; - } - - void SetValue(const TString& key, const YAML::Node& value) override { - ProfilesChanged = true; - YamlProfile[key] = value; - } - - void SetValue(const TString& key, const TString& value) override { - ProfilesChanged = true; - YamlProfile[key] = value; - } - - const YAML::Node GetValue(const TString& key) const override { - return YamlProfile[key]; - } - - void RemoveValue(const TString& key) override { - ProfilesChanged = true; - YamlProfile.remove(key); - } - -private: - TString Name; - YAML::Node YamlProfile; - bool& ProfilesChanged; - std::shared_ptr<IProfileManager> ProfileManager; -}; - -class TProfileManager : public IProfileManager { -public: - TProfileManager(const TString& configPath) - : ConfigPath(configPath) - { - LoadConfig(); - } - - ~TProfileManager() { - if (ProfilesChanged) { - SaveConfig(); - } - } - - bool HasProfile(const TString& profileName) const override { - return Config["profiles"] && Config["profiles"][profileName]; - } - - std::shared_ptr<IProfile> GetProfile(const TString& profileName) override { - return std::make_shared<TProfile>(profileName, Config["profiles"][profileName], ProfilesChanged, shared_from_this()); - } - - TVector<TString> ListProfiles() const override { - const auto profilesNode = Config["profiles"]; - TVector<TString> result; - if (Config["profiles"]) { - result.reserve(Config["profiles"].size()); - for (const auto& profile : Config["profiles"]) { - result.push_back(profile.first.as<TString>()); - } - } - return result; - } - - bool RemoveProfile(const TString& profileName) override { - if (Config["profiles"] && Config["profiles"][profileName]) { - if (GetActiveProfileName() == profileName) { - DeactivateProfile(); - } - Config["profiles"].remove(profileName); - ProfilesChanged = true; - return true; - } - return false; - } - - void SetActiveProfile(const TString& profileName) override { - Config["active_profile"] = profileName; - ProfilesChanged = true; - } - - void DeactivateProfile() override { - if (Config["active_profile"]) { - Config.remove("active_profile"); - ProfilesChanged = true; - } - } - - TString GetActiveProfileName() const override { - if (Config["active_profile"]) { - return Config["active_profile"].as<TString>(); - } else { - return TString(); - } - } - - std::shared_ptr<IProfile> GetActiveProfile() override { - const TString activeProfileName = GetActiveProfileName(); - if (activeProfileName && Config["profiles"][activeProfileName]) { - return std::make_shared<TProfile>( - activeProfileName, Config["profiles"][activeProfileName], ProfilesChanged, shared_from_this()); - } else { - return nullptr; - } - } - -private: - void LoadConfig() { - TFsPath configFilePath(ConfigPath); - configFilePath.Fix(); - try { - if (configFilePath.Exists()) { - Config = YAML::LoadFile(configFilePath.GetPath()); - return; - } - } - catch (const std::exception& e) { - Cerr << "(!) Couldn't load profiles from config file \"" << configFilePath.GetPath() - << "\". " << e.what() << Endl; - } - } - - void SaveConfig() { - TFsPath configFilePath(ConfigPath); - configFilePath.Fix(); - try { - if (!configFilePath.Parent().Exists()) { - configFilePath.Parent().MkDirs(); - } - TFileOutput resultConfigFile(configFilePath); - resultConfigFile << YAML::Dump(Config); - } - catch (const std::exception& e) { - Cerr << "(!) Couldn't save profiles to config file \"" << configFilePath.GetPath() - << "\". " << e.what() << Endl; - } - } - - TString ConfigPath; - YAML::Node Config; - bool ProfilesChanged = false; -}; - -std::shared_ptr<IProfileManager> CreateProfileManager(const TString& configPath) { - return std::make_shared<TProfileManager>(configPath); -} - -} -} +#include "profile_manager.h" + +#include <util/folder/path.h> +#include <util/stream/file.h> +#include <util/string/builder.h> +#include <util/system/env.h> + +namespace NYdb { +namespace NConsoleClient { + +class TProfile : public IProfile { +public: + TProfile(const TString& name, YAML::Node yamlProfile, bool& profilesChanged, std::shared_ptr<IProfileManager> manager) + : Name(name) + , YamlProfile(yamlProfile) + , ProfilesChanged(profilesChanged) + , ProfileManager(manager) + { + } + + TString GetName() const override { + return Name; + } + + bool Has(const TString& key) const noexcept override { + return YamlProfile[key]; + } + + void SetValue(const TString& key, const YAML::Node& value) override { + ProfilesChanged = true; + YamlProfile[key] = value; + } + + void SetValue(const TString& key, const TString& value) override { + ProfilesChanged = true; + YamlProfile[key] = value; + } + + const YAML::Node GetValue(const TString& key) const override { + return YamlProfile[key]; + } + + void RemoveValue(const TString& key) override { + ProfilesChanged = true; + YamlProfile.remove(key); + } + +private: + TString Name; + YAML::Node YamlProfile; + bool& ProfilesChanged; + std::shared_ptr<IProfileManager> ProfileManager; +}; + +class TProfileManager : public IProfileManager { +public: + TProfileManager(const TString& configPath) + : ConfigPath(configPath) + { + LoadConfig(); + } + + ~TProfileManager() { + if (ProfilesChanged) { + SaveConfig(); + } + } + + bool HasProfile(const TString& profileName) const override { + return Config["profiles"] && Config["profiles"][profileName]; + } + + std::shared_ptr<IProfile> GetProfile(const TString& profileName) override { + return std::make_shared<TProfile>(profileName, Config["profiles"][profileName], ProfilesChanged, shared_from_this()); + } + + TVector<TString> ListProfiles() const override { + const auto profilesNode = Config["profiles"]; + TVector<TString> result; + if (Config["profiles"]) { + result.reserve(Config["profiles"].size()); + for (const auto& profile : Config["profiles"]) { + result.push_back(profile.first.as<TString>()); + } + } + return result; + } + + bool RemoveProfile(const TString& profileName) override { + if (Config["profiles"] && Config["profiles"][profileName]) { + if (GetActiveProfileName() == profileName) { + DeactivateProfile(); + } + Config["profiles"].remove(profileName); + ProfilesChanged = true; + return true; + } + return false; + } + + void SetActiveProfile(const TString& profileName) override { + Config["active_profile"] = profileName; + ProfilesChanged = true; + } + + void DeactivateProfile() override { + if (Config["active_profile"]) { + Config.remove("active_profile"); + ProfilesChanged = true; + } + } + + TString GetActiveProfileName() const override { + if (Config["active_profile"]) { + return Config["active_profile"].as<TString>(); + } else { + return TString(); + } + } + + std::shared_ptr<IProfile> GetActiveProfile() override { + const TString activeProfileName = GetActiveProfileName(); + if (activeProfileName && Config["profiles"][activeProfileName]) { + return std::make_shared<TProfile>( + activeProfileName, Config["profiles"][activeProfileName], ProfilesChanged, shared_from_this()); + } else { + return nullptr; + } + } + +private: + void LoadConfig() { + TFsPath configFilePath(ConfigPath); + configFilePath.Fix(); + try { + if (configFilePath.Exists()) { + Config = YAML::LoadFile(configFilePath.GetPath()); + return; + } + } + catch (const std::exception& e) { + Cerr << "(!) Couldn't load profiles from config file \"" << configFilePath.GetPath() + << "\". " << e.what() << Endl; + } + } + + void SaveConfig() { + TFsPath configFilePath(ConfigPath); + configFilePath.Fix(); + try { + if (!configFilePath.Parent().Exists()) { + configFilePath.Parent().MkDirs(); + } + TFileOutput resultConfigFile(configFilePath); + resultConfigFile << YAML::Dump(Config); + } + catch (const std::exception& e) { + Cerr << "(!) Couldn't save profiles to config file \"" << configFilePath.GetPath() + << "\". " << e.what() << Endl; + } + } + + TString ConfigPath; + YAML::Node Config; + bool ProfilesChanged = false; +}; + +std::shared_ptr<IProfileManager> CreateProfileManager(const TString& configPath) { + return std::make_shared<TProfileManager>(configPath); +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/profile_manager.h b/ydb/public/lib/ydb_cli/common/profile_manager.h index 65a55ccee9..44bf9aa79b 100644 --- a/ydb/public/lib/ydb_cli/common/profile_manager.h +++ b/ydb/public/lib/ydb_cli/common/profile_manager.h @@ -1,39 +1,39 @@ -#pragma once -#include <library/cpp/yaml/as/tstring.h> - -namespace NYdb { -namespace NConsoleClient { - -extern const char* VersionResourceName; - -class IProfile { -public: - virtual ~IProfile() = default; - - virtual TString GetName() const = 0; - virtual bool Has(const TString& key) const noexcept = 0; - virtual void SetValue(const TString& key, const YAML::Node& value) = 0; - virtual void SetValue(const TString& key, const TString& value) = 0; - virtual const YAML::Node GetValue(const TString& key) const = 0; - virtual void RemoveValue(const TString& key) = 0; -}; - -class IProfileManager : public std::enable_shared_from_this<IProfileManager> { -public: - virtual ~IProfileManager() = default; - - virtual bool HasProfile(const TString& profileName) const = 0; - // Get existing profile or create new - virtual std::shared_ptr<IProfile> GetProfile(const TString& profileName) = 0; - virtual TVector<TString> ListProfiles() const = 0; - virtual bool RemoveProfile(const TString& profileName) = 0; - virtual void SetActiveProfile(const TString& profileName) = 0; - virtual void DeactivateProfile() = 0; - virtual TString GetActiveProfileName() const = 0; - virtual std::shared_ptr<IProfile> GetActiveProfile() = 0; -}; - -std::shared_ptr<IProfileManager> CreateProfileManager(const TString& configPath); - -} -} +#pragma once +#include <library/cpp/yaml/as/tstring.h> + +namespace NYdb { +namespace NConsoleClient { + +extern const char* VersionResourceName; + +class IProfile { +public: + virtual ~IProfile() = default; + + virtual TString GetName() const = 0; + virtual bool Has(const TString& key) const noexcept = 0; + virtual void SetValue(const TString& key, const YAML::Node& value) = 0; + virtual void SetValue(const TString& key, const TString& value) = 0; + virtual const YAML::Node GetValue(const TString& key) const = 0; + virtual void RemoveValue(const TString& key) = 0; +}; + +class IProfileManager : public std::enable_shared_from_this<IProfileManager> { +public: + virtual ~IProfileManager() = default; + + virtual bool HasProfile(const TString& profileName) const = 0; + // Get existing profile or create new + virtual std::shared_ptr<IProfile> GetProfile(const TString& profileName) = 0; + virtual TVector<TString> ListProfiles() const = 0; + virtual bool RemoveProfile(const TString& profileName) = 0; + virtual void SetActiveProfile(const TString& profileName) = 0; + virtual void DeactivateProfile() = 0; + virtual TString GetActiveProfileName() const = 0; + virtual std::shared_ptr<IProfile> GetActiveProfile() = 0; +}; + +std::shared_ptr<IProfileManager> CreateProfileManager(const TString& configPath); + +} +} diff --git a/ydb/public/lib/ydb_cli/common/root.cpp b/ydb/public/lib/ydb_cli/common/root.cpp index 57a74d6b7f..e57c93a643 100644 --- a/ydb/public/lib/ydb_cli/common/root.cpp +++ b/ydb/public/lib/ydb_cli/common/root.cpp @@ -1,109 +1,109 @@ -#include "root.h" -#include <util/folder/path.h> -#include <util/folder/dirut.h> -#include <util/string/strip.h> -#include <util/system/env.h> - -namespace NYdb { -namespace NConsoleClient { - -const TString defaultTokenFile = "~/.ydb/token"; - -TClientCommandRootBase::TClientCommandRootBase(const TString& name) - : TClientCommandTree(name) -{} - -void TClientCommandRootBase::Config(TConfig& config) { - TClientCommandTree::Config(config); - TimeRequests = false; - ProgressRequests = false; - - NLastGetopt::TOpts& opts = *config.Opts; - opts.AddLongOption('t', "time", "Show request execution time").NoArgument().SetFlag(&TimeRequests); - opts.AddLongOption('o', "progress", "Show progress of long requests").NoArgument().SetFlag(&ProgressRequests); - opts.AddLongOption("ca-file", - "Path to a file containing the PEM encoding of the server root certificates for tls connections.\n" - "If this parameter is empty, the default roots will be used.") - .RequiredArgument("PATH").StoreResult(&CaCertsFile); - - opts.SetCustomUsage(config.ArgV[0]); - config.SetFreeArgsMin(1); - opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; -} - -void TClientCommandRootBase::SetCustomUsage(TConfig& config) { - Y_UNUSED(config); -} - -void TClientCommandRootBase::Parse(TConfig& config) { - TClientCommandTree::Parse(config); - if (!config.IsSystemCommand()) { - ParseCredentials(config); - ParseAddress(config); - } - - TClientCommand::TIME_REQUESTS = TimeRequests; - TClientCommand::PROGRESS_REQUESTS = ProgressRequests; -} - -void TClientCommandRootBase::ParseToken(TString& token, TString& tokenFile, const TString& envName, bool useDefaultToken) { - if (token.empty()) { - if (tokenFile) { - // 1. command line token-file option - token = ReadFromFile(tokenFile, "token"); - } else { - // 2. Environment variable - TString ydbToken = GetEnv(envName); - if (!ydbToken.empty()) { - token = ydbToken; - } else { - if (useDefaultToken && token.empty()) { - // 3. Default token file - tokenFile = defaultTokenFile; - ReadFromFileIfExists(tokenFile, "default token", token); - } - } - } - } -} - -void TClientCommandRootBase::ParseProtocol(TConfig& config) { - auto separator_pos = Address.find("://"); - if (separator_pos != TString::npos) { - TString protocol = Address.substr(0, separator_pos); - protocol.to_lower(); - if (protocol == "grpcs") { - config.EnableSsl = true; - } else if (protocol == "grpc") { - config.EnableSsl = false; - } else { - throw TMissUseException() << "Unknown protocol \"" << protocol << "\"."; - } - Address = Address.substr(separator_pos + 3); - } -} - -void TClientCommandRootBase::ParseCaCerts(TConfig& config) { - if (CaCertsFile.empty()) { - return; - } - if (!config.EnableSsl) { - throw TMissUseException() - << "\"ca-file\" option provided for a non-ssl connection. Use grpcs:// prefix for host to connect using SSL."; - } - config.CaCerts = ReadFromFile(CaCertsFile, "CA certificates"); -} - -void TClientCommandRootBase::ParseCredentials(TConfig& config) { - ParseToken(Token, TokenFile, "YDB_TOKEN", true); - if (!Token.empty()) { - config.SecurityToken = Token; - } -} - -void TClientCommandRootBase::SetFreeArgs(TConfig& config) { - Y_UNUSED(config); -} - -} -} +#include "root.h" +#include <util/folder/path.h> +#include <util/folder/dirut.h> +#include <util/string/strip.h> +#include <util/system/env.h> + +namespace NYdb { +namespace NConsoleClient { + +const TString defaultTokenFile = "~/.ydb/token"; + +TClientCommandRootBase::TClientCommandRootBase(const TString& name) + : TClientCommandTree(name) +{} + +void TClientCommandRootBase::Config(TConfig& config) { + TClientCommandTree::Config(config); + TimeRequests = false; + ProgressRequests = false; + + NLastGetopt::TOpts& opts = *config.Opts; + opts.AddLongOption('t', "time", "Show request execution time").NoArgument().SetFlag(&TimeRequests); + opts.AddLongOption('o', "progress", "Show progress of long requests").NoArgument().SetFlag(&ProgressRequests); + opts.AddLongOption("ca-file", + "Path to a file containing the PEM encoding of the server root certificates for tls connections.\n" + "If this parameter is empty, the default roots will be used.") + .RequiredArgument("PATH").StoreResult(&CaCertsFile); + + opts.SetCustomUsage(config.ArgV[0]); + config.SetFreeArgsMin(1); + opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; +} + +void TClientCommandRootBase::SetCustomUsage(TConfig& config) { + Y_UNUSED(config); +} + +void TClientCommandRootBase::Parse(TConfig& config) { + TClientCommandTree::Parse(config); + if (!config.IsSystemCommand()) { + ParseCredentials(config); + ParseAddress(config); + } + + TClientCommand::TIME_REQUESTS = TimeRequests; + TClientCommand::PROGRESS_REQUESTS = ProgressRequests; +} + +void TClientCommandRootBase::ParseToken(TString& token, TString& tokenFile, const TString& envName, bool useDefaultToken) { + if (token.empty()) { + if (tokenFile) { + // 1. command line token-file option + token = ReadFromFile(tokenFile, "token"); + } else { + // 2. Environment variable + TString ydbToken = GetEnv(envName); + if (!ydbToken.empty()) { + token = ydbToken; + } else { + if (useDefaultToken && token.empty()) { + // 3. Default token file + tokenFile = defaultTokenFile; + ReadFromFileIfExists(tokenFile, "default token", token); + } + } + } + } +} + +void TClientCommandRootBase::ParseProtocol(TConfig& config) { + auto separator_pos = Address.find("://"); + if (separator_pos != TString::npos) { + TString protocol = Address.substr(0, separator_pos); + protocol.to_lower(); + if (protocol == "grpcs") { + config.EnableSsl = true; + } else if (protocol == "grpc") { + config.EnableSsl = false; + } else { + throw TMissUseException() << "Unknown protocol \"" << protocol << "\"."; + } + Address = Address.substr(separator_pos + 3); + } +} + +void TClientCommandRootBase::ParseCaCerts(TConfig& config) { + if (CaCertsFile.empty()) { + return; + } + if (!config.EnableSsl) { + throw TMissUseException() + << "\"ca-file\" option provided for a non-ssl connection. Use grpcs:// prefix for host to connect using SSL."; + } + config.CaCerts = ReadFromFile(CaCertsFile, "CA certificates"); +} + +void TClientCommandRootBase::ParseCredentials(TConfig& config) { + ParseToken(Token, TokenFile, "YDB_TOKEN", true); + if (!Token.empty()) { + config.SecurityToken = Token; + } +} + +void TClientCommandRootBase::SetFreeArgs(TConfig& config) { + Y_UNUSED(config); +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/root.h b/ydb/public/lib/ydb_cli/common/root.h index d2d509de73..36da3f5cc1 100644 --- a/ydb/public/lib/ydb_cli/common/root.h +++ b/ydb/public/lib/ydb_cli/common/root.h @@ -1,35 +1,35 @@ -#pragma once - -#include "command.h" - -namespace NYdb { -namespace NConsoleClient { - -extern const TString defaultTokenFile; - -class TClientCommandRootBase : public TClientCommandTree { -public: - TClientCommandRootBase(const TString& name); - - bool TimeRequests; - bool ProgressRequests; - TString Address; - TString Token; - TString TokenFile; - TString CaCertsFile; - - virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; - void SetCustomUsage(TConfig& config) override; - void SetFreeArgs(TConfig& config) override; - -protected: - void ParseToken(TString& token, TString& tokenFile, const TString& envName, bool useDefaultToken = false); - void ParseProtocol(TConfig& config); - void ParseCaCerts(TConfig& config); - virtual void ParseCredentials(TConfig& config); - virtual void ParseAddress(TConfig& config) = 0; -}; - -} -} +#pragma once + +#include "command.h" + +namespace NYdb { +namespace NConsoleClient { + +extern const TString defaultTokenFile; + +class TClientCommandRootBase : public TClientCommandTree { +public: + TClientCommandRootBase(const TString& name); + + bool TimeRequests; + bool ProgressRequests; + TString Address; + TString Token; + TString TokenFile; + TString CaCertsFile; + + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + void SetCustomUsage(TConfig& config) override; + void SetFreeArgs(TConfig& config) override; + +protected: + void ParseToken(TString& token, TString& tokenFile, const TString& envName, bool useDefaultToken = false); + void ParseProtocol(TConfig& config); + void ParseCaCerts(TConfig& config); + virtual void ParseCredentials(TConfig& config); + virtual void ParseAddress(TConfig& config) = 0; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp index 9e4592eb21..f5371d22a2 100644 --- a/ydb/public/lib/ydb_cli/common/tabbed_table.cpp +++ b/ydb/public/lib/ydb_cli/common/tabbed_table.cpp @@ -1,110 +1,110 @@ -#include "tabbed_table.h" - -#include "common.h" +#include "tabbed_table.h" + +#include "common.h" #include "print_utils.h" - -namespace NYdb { -namespace NConsoleClient { - - -TAdaptiveTabbedTable::TAdaptiveTabbedTable(const TVector<NScheme::TSchemeEntry>& entries) - : Entries(entries) -{ - CalculateColumns(); -} - -void TAdaptiveTabbedTable::Print(IOutputStream& o) const { - - /* Calculate the number of rows that will be in each column except possibly - for a short column on the right. */ - size_t rows = Entries.size() / ColumnCount + (Entries.size() % ColumnCount != 0); - - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - for (size_t row = 0; row < rows; ++row) { - for (size_t column = 0; column < ColumnCount; ++column) { - size_t idx = column * rows + row; - if (idx < Entries.size()) { - PrintSchemeEntry(o, Entries[idx], colors); - if (column != ColumnCount - 1) { - size_t columnWidth = ColumnInfo[ColumnCount - 1].LengthValues[column]; - o << TString(columnWidth - Entries[idx].Name.length(), ' '); - } - } - } - o << Endl; - } -} - -void TAdaptiveTabbedTable::InitializeColumnInfo(size_t maxCols, size_t minColumnWidth) { - ColumnInfo = TVector<TColumnInfo>(maxCols); - - for (size_t i = 0; i < maxCols; ++i) { - ColumnInfo[i].LineLength = (i + 1) * minColumnWidth; - for (size_t j = 0; j <= i; ++j) { - ColumnInfo[i].LengthValues.push_back(minColumnWidth); - } - } -} - -void TAdaptiveTabbedTable::CalculateColumns() { - size_t lineLength = TermWidth(); - size_t max_length = 0; - for (auto entry : Entries) { - if (entry.Name.length() > max_length) { - max_length = entry.Name.length(); - } - } - if (lineLength < max_length + 1) { - lineLength = Max<size_t>(); - } - size_t minColumnWidth = 3; - - /* Determine the max possible number of display columns. */ - size_t maxIdx = lineLength / minColumnWidth; - - /* Account for first display column not having a separator, - or lineLength shorter than minColumnWidth. */ - maxIdx += lineLength % minColumnWidth != 0; - - /* Normally the maximum number of columns is determined by the - screen width. But if few files are available this might limit it - as well. */ - size_t maxCols = Min(maxIdx, Entries.size()); - - InitializeColumnInfo(maxCols, minColumnWidth); - - for (size_t filesno = 0; filesno < Entries.size(); ++filesno) { - size_t name_length = Entries[filesno].Name.length(); - - for (size_t i = 0; i < maxCols; ++i) { - if (ColumnInfo[i].ValidLen) - { - size_t idx = filesno / ((Entries.size() + i) / (i + 1)); - size_t real_length = name_length + (idx == i ? 0 : 2); - - if (ColumnInfo[i].LengthValues[idx] < real_length) { - ColumnInfo[i].LineLength += (real_length - ColumnInfo[i].LengthValues[idx]); - ColumnInfo[i].LengthValues[idx] = real_length; - ColumnInfo[i].ValidLen = (ColumnInfo[i].LineLength < lineLength); - if (name_length > ColumnInfo[i].ColumnWidth) { - ColumnInfo[i].ColumnWidth = name_length; - } - } - } - } - } - - /* Find maximum allowed columns. */ - for (size_t cols = maxCols; 0 < cols; --cols) { - if (ColumnInfo[cols - 1].ValidLen) { - ColumnCount = cols; - return; - } - } - - Cerr << "Can't find valid column count to display" << Endl; - return; -} - -} -} + +namespace NYdb { +namespace NConsoleClient { + + +TAdaptiveTabbedTable::TAdaptiveTabbedTable(const TVector<NScheme::TSchemeEntry>& entries) + : Entries(entries) +{ + CalculateColumns(); +} + +void TAdaptiveTabbedTable::Print(IOutputStream& o) const { + + /* Calculate the number of rows that will be in each column except possibly + for a short column on the right. */ + size_t rows = Entries.size() / ColumnCount + (Entries.size() % ColumnCount != 0); + + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + for (size_t row = 0; row < rows; ++row) { + for (size_t column = 0; column < ColumnCount; ++column) { + size_t idx = column * rows + row; + if (idx < Entries.size()) { + PrintSchemeEntry(o, Entries[idx], colors); + if (column != ColumnCount - 1) { + size_t columnWidth = ColumnInfo[ColumnCount - 1].LengthValues[column]; + o << TString(columnWidth - Entries[idx].Name.length(), ' '); + } + } + } + o << Endl; + } +} + +void TAdaptiveTabbedTable::InitializeColumnInfo(size_t maxCols, size_t minColumnWidth) { + ColumnInfo = TVector<TColumnInfo>(maxCols); + + for (size_t i = 0; i < maxCols; ++i) { + ColumnInfo[i].LineLength = (i + 1) * minColumnWidth; + for (size_t j = 0; j <= i; ++j) { + ColumnInfo[i].LengthValues.push_back(minColumnWidth); + } + } +} + +void TAdaptiveTabbedTable::CalculateColumns() { + size_t lineLength = TermWidth(); + size_t max_length = 0; + for (auto entry : Entries) { + if (entry.Name.length() > max_length) { + max_length = entry.Name.length(); + } + } + if (lineLength < max_length + 1) { + lineLength = Max<size_t>(); + } + size_t minColumnWidth = 3; + + /* Determine the max possible number of display columns. */ + size_t maxIdx = lineLength / minColumnWidth; + + /* Account for first display column not having a separator, + or lineLength shorter than minColumnWidth. */ + maxIdx += lineLength % minColumnWidth != 0; + + /* Normally the maximum number of columns is determined by the + screen width. But if few files are available this might limit it + as well. */ + size_t maxCols = Min(maxIdx, Entries.size()); + + InitializeColumnInfo(maxCols, minColumnWidth); + + for (size_t filesno = 0; filesno < Entries.size(); ++filesno) { + size_t name_length = Entries[filesno].Name.length(); + + for (size_t i = 0; i < maxCols; ++i) { + if (ColumnInfo[i].ValidLen) + { + size_t idx = filesno / ((Entries.size() + i) / (i + 1)); + size_t real_length = name_length + (idx == i ? 0 : 2); + + if (ColumnInfo[i].LengthValues[idx] < real_length) { + ColumnInfo[i].LineLength += (real_length - ColumnInfo[i].LengthValues[idx]); + ColumnInfo[i].LengthValues[idx] = real_length; + ColumnInfo[i].ValidLen = (ColumnInfo[i].LineLength < lineLength); + if (name_length > ColumnInfo[i].ColumnWidth) { + ColumnInfo[i].ColumnWidth = name_length; + } + } + } + } + } + + /* Find maximum allowed columns. */ + for (size_t cols = maxCols; 0 < cols; --cols) { + if (ColumnInfo[cols - 1].ValidLen) { + ColumnCount = cols; + return; + } + } + + Cerr << "Can't find valid column count to display" << Endl; + return; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/tabbed_table.h b/ydb/public/lib/ydb_cli/common/tabbed_table.h index f3a73581cf..32e7daea4e 100644 --- a/ydb/public/lib/ydb_cli/common/tabbed_table.h +++ b/ydb/public/lib/ydb_cli/common/tabbed_table.h @@ -1,38 +1,38 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> - -namespace NYdb { -namespace NConsoleClient { - -class TAdaptiveTabbedTable { -public: - TAdaptiveTabbedTable(const TVector<NScheme::TSchemeEntry>& entries); - void Print(IOutputStream& o) const; - -private: - struct TColumnInfo { - bool ValidLen = true; - TVector<size_t> LengthValues; - size_t LineLength = 0; - size_t ColumnWidth = 0; - }; - - void InitializeColumnInfo(size_t maxCols, size_t minColumnWidth); - void CalculateColumns(); - - const TVector<NScheme::TSchemeEntry>& Entries; - TVector<TColumnInfo> ColumnInfo; - size_t ColumnCount; -}; - -} -} - -template <> -inline void Out<NYdb::NConsoleClient::TAdaptiveTabbedTable>( - IOutputStream& o, - const NYdb::NConsoleClient::TAdaptiveTabbedTable& x -) { - return x.Print(o); -} + +namespace NYdb { +namespace NConsoleClient { + +class TAdaptiveTabbedTable { +public: + TAdaptiveTabbedTable(const TVector<NScheme::TSchemeEntry>& entries); + void Print(IOutputStream& o) const; + +private: + struct TColumnInfo { + bool ValidLen = true; + TVector<size_t> LengthValues; + size_t LineLength = 0; + size_t ColumnWidth = 0; + }; + + void InitializeColumnInfo(size_t maxCols, size_t minColumnWidth); + void CalculateColumns(); + + const TVector<NScheme::TSchemeEntry>& Entries; + TVector<TColumnInfo> ColumnInfo; + size_t ColumnCount; +}; + +} +} + +template <> +inline void Out<NYdb::NConsoleClient::TAdaptiveTabbedTable>( + IOutputStream& o, + const NYdb::NConsoleClient::TAdaptiveTabbedTable& x +) { + return x.Print(o); +} diff --git a/ydb/public/lib/ydb_cli/common/ya.make b/ydb/public/lib/ydb_cli/common/ya.make index 9addabb7fa..35523bfb4c 100644 --- a/ydb/public/lib/ydb_cli/common/ya.make +++ b/ydb/public/lib/ydb_cli/common/ya.make @@ -1,32 +1,32 @@ -LIBRARY(common) - -OWNER(g:kikimr) - -SRCS( +LIBRARY(common) + +OWNER(g:kikimr) + +SRCS( aws.cpp - command.cpp - common.cpp - examples.cpp + command.cpp + common.cpp + examples.cpp format.cpp - interruptible.cpp + interruptible.cpp normalize_path.cpp - parameters.cpp + parameters.cpp pretty_table.cpp print_operation.cpp print_utils.cpp - profile_manager.cpp + profile_manager.cpp query_stats.cpp recursive_list.cpp - root.cpp - tabbed_table.cpp - ydb_updater.cpp + root.cpp + tabbed_table.cpp + ydb_updater.cpp yt.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/getopt - library/cpp/json/writer - library/cpp/yaml/as + library/cpp/json/writer + library/cpp/yaml/as ydb/public/lib/json_value ydb/public/lib/operation_id ydb/public/lib/yson_value @@ -34,11 +34,11 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_result ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_types/credentials -) - +) + GENERATE_ENUM_SERIALIZATION(formats.h) -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/public/lib/ydb_cli/common/ydb_updater.cpp b/ydb/public/lib/ydb_cli/common/ydb_updater.cpp index 3e0860614e..ae27e34da2 100644 --- a/ydb/public/lib/ydb_cli/common/ydb_updater.cpp +++ b/ydb/public/lib/ydb_cli/common/ydb_updater.cpp @@ -1,200 +1,200 @@ -#include "ydb_updater.h" - -#include <library/cpp/json/writer/json.h> -#include <library/cpp/resource/resource.h> -#include <util/datetime/base.h> -#include <util/folder/dirut.h> -#include <util/folder/path.h> -#include <util/stream/file.h> -#include <util/string/builder.h> -#include <util/string/strip.h> -#include <util/system/env.h> -#include <util/system/execpath.h> -#include <util/system/shellcommand.h> - -namespace NYdb { -namespace NConsoleClient { - -const char* VersionResourceName = "version.txt"; - -namespace { -#if defined(_darwin_) - const TString OsVersion = "darwin"; - const TString BinaryName = "ydb"; - const TString HomeDir = GetHomeDir(); -#elif defined(_win32_) - const TString OsVersion = "windows"; - const TString BinaryName = "ydb.exe"; - const TString HomeDir = GetEnv("USERPROFILE"); -#else - const TString OsVersion = "linux"; - const TString BinaryName = "ydb"; - const TString HomeDir = GetHomeDir(); -#endif - const TString DefaultConfigFile = TStringBuilder() << HomeDir << "/ydb/bin/config.json"; - const TString DefaultTempFile = TStringBuilder() << HomeDir << "/ydb/install/" << BinaryName; - const TString StorageUrl = "https://storage.yandexcloud.net/yandexcloud-ydb/release/"; - const TString VersionUrl = TStringBuilder() << StorageUrl << "stable"; -} - -TYdbUpdater::TYdbUpdater() - : MyVersion(StripString(NResource::Find(TStringBuf(VersionResourceName)))) -{ - LoadConfig(); -} - -TYdbUpdater::~TYdbUpdater() { - if (ConfigChanged) { - SaveConfig(); - } -} - -bool TYdbUpdater::CheckIfUpdateNeeded(bool forceRequest) { - if (!forceRequest && Config.Has("outdated") && Config["outdated"].GetBoolean()) { - return true; - } - - if (!forceRequest && !IsTimeToCheckForUpdate()) { - return false; - } - - Config["last_check"] = TInstant::Now().Seconds(); - ConfigChanged = true; - - if (GetLatestVersion() && MyVersion != LatestVersion) { - Config["outdated"] = true; - ConfigChanged = true; - return true; - } - return false; -} - -int TYdbUpdater::Update(bool forceUpdate) { - if (!GetLatestVersion()) { - return EXIT_FAILURE; - } - if (!CheckIfUpdateNeeded(/*forceRequest*/ true) && !forceUpdate) { - Cerr << "Current version: \"" << MyVersion << "\". Latest version Available: \"" << LatestVersion - << "\". No need to update. Use '--force' option to update anyway." << Endl; - return EXIT_FAILURE; - } - - TFsPath tmpPathToBinary(DefaultTempFile); - tmpPathToBinary.Fix(); - TString corrPath = tmpPathToBinary.GetPath(); - if (!tmpPathToBinary.Parent().Exists()) { - tmpPathToBinary.Parent().MkDirs(); - } - const TString DownloadUrl = TStringBuilder() << StorageUrl << LatestVersion << "/" << OsVersion << "/amd64/" - << BinaryName; - Cout << "Downloading binary from url " << DownloadUrl << Endl; - TShellCommand curlCmd(TStringBuilder() << "curl " << DownloadUrl << " -o " << tmpPathToBinary.GetPath()); - curlCmd.Run().Wait(); - if (curlCmd.GetExitCode() != 0) { - Cerr << "Failed to download from url \"" << DownloadUrl << "\". " << curlCmd.GetError() << Endl; - return EXIT_FAILURE; - } - Cout << "Downloaded to " << tmpPathToBinary.GetPath() << Endl; - -#ifndef _win32_ - int chmodResult = Chmod(tmpPathToBinary.GetPath().data(), MODE0777); - if (chmodResult != 0) { - Cerr << "Couldn't make binary \"" << tmpPathToBinary.GetPath() << "\" executable. Error code: " - << chmodResult << Endl; - } -#endif - - // Check new binary - Cout << "Checking downloaded binary by calling 'version' command..." << Endl; - TShellCommand checkCmd(TStringBuilder() << tmpPathToBinary.GetPath() << " version"); - checkCmd.Run().Wait(); - if (checkCmd.GetExitCode() != 0) { - Cerr << "Failed to check downloaded binary. " << checkCmd.GetError() << Endl; - return EXIT_FAILURE; - } - Cout << checkCmd.GetOutput(); - - TFsPath fsPathToBinary(GetExecPath()); -#ifdef _win32_ - TFsPath binaryNameOld(TStringBuilder() << fsPathToBinary.GetPath() << "_old"); - binaryNameOld.Fix(); - binaryNameOld.DeleteIfExists(); - fsPathToBinary.RenameTo(binaryNameOld); - Cout << "Old binary renamed to " << binaryNameOld.GetPath() << Endl; -#else - fsPathToBinary.DeleteIfExists(); - Cout << "Old binary removed" << Endl; -#endif - tmpPathToBinary.RenameTo(fsPathToBinary); - Cout << "New binary renamed to " << fsPathToBinary.GetPath() << Endl; - - Config["outdated"] = false; - ConfigChanged = true; - return EXIT_SUCCESS; -} - -void TYdbUpdater::LoadConfig() { - TFsPath configFilePath(DefaultConfigFile); - configFilePath.Fix(); - try { - if (configFilePath.Exists()) { - TUnbufferedFileInput input(configFilePath); - TString rawConfig = input.ReadAll(); - if (NJson::ReadJsonTree(rawConfig, &Config, /* throwOnError */ true)) { - return; - } - } - } - catch (const yexception& e) { - Cerr << "(!) Couldn't load config from file \"" << configFilePath.GetPath() - << "\". Using default. " << e.what() << Endl; - } - Config = NJson::TJsonValue(); -} - -void TYdbUpdater::SaveConfig() { - try { - TFsPath configFilePath(DefaultConfigFile); - configFilePath.Fix(); - if (!configFilePath.Parent().Exists()) { - configFilePath.Parent().MkDirs(); - } - TFileOutput resultConfigFile(configFilePath); - NJsonWriter::TBuf buf; - buf.WriteJsonValue(&Config); - resultConfigFile << buf.Str(); - } - catch (const yexception& e) { - Cerr << "(!) Couldn't save config to file. " << e.what() << Endl; - } -} - -bool TYdbUpdater::IsTimeToCheckForUpdate() { - if (Config.Has("last_check")) { - TInstant lastCheck = TInstant::Seconds(Config["last_check"].GetUInteger()); - // Do not check for updates more than once every 24h - if (lastCheck + TDuration::Days(1) > TInstant::Now()) { - return false; - } - } - return true; -} - -bool TYdbUpdater::GetLatestVersion() { - if (LatestVersion) { - return true; - } - - TShellCommand curlCmd(TStringBuilder() << "curl --silent " << VersionUrl); - curlCmd.Run().Wait(); - - if (curlCmd.GetExitCode() == 0) { - LatestVersion = StripString(curlCmd.GetOutput()); - return true; - } - Cerr << "(!) Couldn't get latest version from url \"" << VersionUrl << "\". " << curlCmd.GetError() << Endl; - return false; -} - -} -} +#include "ydb_updater.h" + +#include <library/cpp/json/writer/json.h> +#include <library/cpp/resource/resource.h> +#include <util/datetime/base.h> +#include <util/folder/dirut.h> +#include <util/folder/path.h> +#include <util/stream/file.h> +#include <util/string/builder.h> +#include <util/string/strip.h> +#include <util/system/env.h> +#include <util/system/execpath.h> +#include <util/system/shellcommand.h> + +namespace NYdb { +namespace NConsoleClient { + +const char* VersionResourceName = "version.txt"; + +namespace { +#if defined(_darwin_) + const TString OsVersion = "darwin"; + const TString BinaryName = "ydb"; + const TString HomeDir = GetHomeDir(); +#elif defined(_win32_) + const TString OsVersion = "windows"; + const TString BinaryName = "ydb.exe"; + const TString HomeDir = GetEnv("USERPROFILE"); +#else + const TString OsVersion = "linux"; + const TString BinaryName = "ydb"; + const TString HomeDir = GetHomeDir(); +#endif + const TString DefaultConfigFile = TStringBuilder() << HomeDir << "/ydb/bin/config.json"; + const TString DefaultTempFile = TStringBuilder() << HomeDir << "/ydb/install/" << BinaryName; + const TString StorageUrl = "https://storage.yandexcloud.net/yandexcloud-ydb/release/"; + const TString VersionUrl = TStringBuilder() << StorageUrl << "stable"; +} + +TYdbUpdater::TYdbUpdater() + : MyVersion(StripString(NResource::Find(TStringBuf(VersionResourceName)))) +{ + LoadConfig(); +} + +TYdbUpdater::~TYdbUpdater() { + if (ConfigChanged) { + SaveConfig(); + } +} + +bool TYdbUpdater::CheckIfUpdateNeeded(bool forceRequest) { + if (!forceRequest && Config.Has("outdated") && Config["outdated"].GetBoolean()) { + return true; + } + + if (!forceRequest && !IsTimeToCheckForUpdate()) { + return false; + } + + Config["last_check"] = TInstant::Now().Seconds(); + ConfigChanged = true; + + if (GetLatestVersion() && MyVersion != LatestVersion) { + Config["outdated"] = true; + ConfigChanged = true; + return true; + } + return false; +} + +int TYdbUpdater::Update(bool forceUpdate) { + if (!GetLatestVersion()) { + return EXIT_FAILURE; + } + if (!CheckIfUpdateNeeded(/*forceRequest*/ true) && !forceUpdate) { + Cerr << "Current version: \"" << MyVersion << "\". Latest version Available: \"" << LatestVersion + << "\". No need to update. Use '--force' option to update anyway." << Endl; + return EXIT_FAILURE; + } + + TFsPath tmpPathToBinary(DefaultTempFile); + tmpPathToBinary.Fix(); + TString corrPath = tmpPathToBinary.GetPath(); + if (!tmpPathToBinary.Parent().Exists()) { + tmpPathToBinary.Parent().MkDirs(); + } + const TString DownloadUrl = TStringBuilder() << StorageUrl << LatestVersion << "/" << OsVersion << "/amd64/" + << BinaryName; + Cout << "Downloading binary from url " << DownloadUrl << Endl; + TShellCommand curlCmd(TStringBuilder() << "curl " << DownloadUrl << " -o " << tmpPathToBinary.GetPath()); + curlCmd.Run().Wait(); + if (curlCmd.GetExitCode() != 0) { + Cerr << "Failed to download from url \"" << DownloadUrl << "\". " << curlCmd.GetError() << Endl; + return EXIT_FAILURE; + } + Cout << "Downloaded to " << tmpPathToBinary.GetPath() << Endl; + +#ifndef _win32_ + int chmodResult = Chmod(tmpPathToBinary.GetPath().data(), MODE0777); + if (chmodResult != 0) { + Cerr << "Couldn't make binary \"" << tmpPathToBinary.GetPath() << "\" executable. Error code: " + << chmodResult << Endl; + } +#endif + + // Check new binary + Cout << "Checking downloaded binary by calling 'version' command..." << Endl; + TShellCommand checkCmd(TStringBuilder() << tmpPathToBinary.GetPath() << " version"); + checkCmd.Run().Wait(); + if (checkCmd.GetExitCode() != 0) { + Cerr << "Failed to check downloaded binary. " << checkCmd.GetError() << Endl; + return EXIT_FAILURE; + } + Cout << checkCmd.GetOutput(); + + TFsPath fsPathToBinary(GetExecPath()); +#ifdef _win32_ + TFsPath binaryNameOld(TStringBuilder() << fsPathToBinary.GetPath() << "_old"); + binaryNameOld.Fix(); + binaryNameOld.DeleteIfExists(); + fsPathToBinary.RenameTo(binaryNameOld); + Cout << "Old binary renamed to " << binaryNameOld.GetPath() << Endl; +#else + fsPathToBinary.DeleteIfExists(); + Cout << "Old binary removed" << Endl; +#endif + tmpPathToBinary.RenameTo(fsPathToBinary); + Cout << "New binary renamed to " << fsPathToBinary.GetPath() << Endl; + + Config["outdated"] = false; + ConfigChanged = true; + return EXIT_SUCCESS; +} + +void TYdbUpdater::LoadConfig() { + TFsPath configFilePath(DefaultConfigFile); + configFilePath.Fix(); + try { + if (configFilePath.Exists()) { + TUnbufferedFileInput input(configFilePath); + TString rawConfig = input.ReadAll(); + if (NJson::ReadJsonTree(rawConfig, &Config, /* throwOnError */ true)) { + return; + } + } + } + catch (const yexception& e) { + Cerr << "(!) Couldn't load config from file \"" << configFilePath.GetPath() + << "\". Using default. " << e.what() << Endl; + } + Config = NJson::TJsonValue(); +} + +void TYdbUpdater::SaveConfig() { + try { + TFsPath configFilePath(DefaultConfigFile); + configFilePath.Fix(); + if (!configFilePath.Parent().Exists()) { + configFilePath.Parent().MkDirs(); + } + TFileOutput resultConfigFile(configFilePath); + NJsonWriter::TBuf buf; + buf.WriteJsonValue(&Config); + resultConfigFile << buf.Str(); + } + catch (const yexception& e) { + Cerr << "(!) Couldn't save config to file. " << e.what() << Endl; + } +} + +bool TYdbUpdater::IsTimeToCheckForUpdate() { + if (Config.Has("last_check")) { + TInstant lastCheck = TInstant::Seconds(Config["last_check"].GetUInteger()); + // Do not check for updates more than once every 24h + if (lastCheck + TDuration::Days(1) > TInstant::Now()) { + return false; + } + } + return true; +} + +bool TYdbUpdater::GetLatestVersion() { + if (LatestVersion) { + return true; + } + + TShellCommand curlCmd(TStringBuilder() << "curl --silent " << VersionUrl); + curlCmd.Run().Wait(); + + if (curlCmd.GetExitCode() == 0) { + LatestVersion = StripString(curlCmd.GetOutput()); + return true; + } + Cerr << "(!) Couldn't get latest version from url \"" << VersionUrl << "\". " << curlCmd.GetError() << Endl; + return false; +} + +} +} diff --git a/ydb/public/lib/ydb_cli/common/ydb_updater.h b/ydb/public/lib/ydb_cli/common/ydb_updater.h index 49a8f8dd42..9707223fe9 100644 --- a/ydb/public/lib/ydb_cli/common/ydb_updater.h +++ b/ydb/public/lib/ydb_cli/common/ydb_updater.h @@ -1,31 +1,31 @@ -#pragma once -#include <library/cpp/json/json_reader.h> - -namespace NYdb { -namespace NConsoleClient { - -extern const char* VersionResourceName; - -class TYdbUpdater { -public: - TYdbUpdater(); - ~TYdbUpdater(); - - bool CheckIfUpdateNeeded(bool forceRequest = false); - int Update(bool forceUpdate); - -private: - void LoadConfig(); - void SaveConfig(); - bool IsTimeToCheckForUpdate(); - bool IsVersionUpToDate(); - bool GetLatestVersion(); - - NJson::TJsonValue Config; - TString MyVersion; - TString LatestVersion; - bool ConfigChanged = false; -}; - -} -} +#pragma once +#include <library/cpp/json/json_reader.h> + +namespace NYdb { +namespace NConsoleClient { + +extern const char* VersionResourceName; + +class TYdbUpdater { +public: + TYdbUpdater(); + ~TYdbUpdater(); + + bool CheckIfUpdateNeeded(bool forceRequest = false); + int Update(bool forceUpdate); + +private: + void LoadConfig(); + void SaveConfig(); + bool IsTimeToCheckForUpdate(); + bool IsVersionUpToDate(); + bool GetLatestVersion(); + + NJson::TJsonValue Config; + TString MyVersion; + TString LatestVersion; + bool ConfigChanged = false; +}; + +} +} diff --git a/ydb/public/lib/ydb_cli/common/yt.cpp b/ydb/public/lib/ydb_cli/common/yt.cpp index fdcaf1748f..91c37eaec2 100644 --- a/ydb/public/lib/ydb_cli/common/yt.cpp +++ b/ydb/public/lib/ydb_cli/common/yt.cpp @@ -34,7 +34,7 @@ const TString TCommandWithYtToken::YtTokenFile = "~/.yt/token"; void TCommandWithYtToken::ReadYtToken() { TString tokenFile = YtTokenFile; - YtToken = ReadFromFile(tokenFile, "YT token"); + YtToken = ReadFromFile(tokenFile, "YT token"); } } diff --git a/ydb/public/lib/ydb_cli/dump/ya.make b/ydb/public/lib/ydb_cli/dump/ya.make index 81c5559feb..b85766bb39 100644 --- a/ydb/public/lib/ydb_cli/dump/ya.make +++ b/ydb/public/lib/ydb_cli/dump/ya.make @@ -6,17 +6,17 @@ OWNER( ) SRCS( - dump.cpp - dump_impl.cpp - restore_impl.cpp - restore_import_data.cpp - restore_compat.cpp + dump.cpp + dump_impl.cpp + restore_impl.cpp + restore_import_data.cpp + restore_compat.cpp ) PEERDIR( - library/cpp/bucket_quoter - library/cpp/string_utils/quote - ydb/library/backup + library/cpp/bucket_quoter + library/cpp/string_utils/quote + ydb/library/backup ydb/public/api/protos ydb/public/lib/ydb_cli/common ydb/public/lib/ydb_cli/dump/util diff --git a/ydb/public/lib/ydb_cli/import/ya.make b/ydb/public/lib/ydb_cli/import/ya.make index d5046da94b..82518d2816 100644 --- a/ydb/public/lib/ydb_cli/import/ya.make +++ b/ydb/public/lib/ydb_cli/import/ya.make @@ -6,7 +6,7 @@ OWNER( ) SRCS( - import.cpp + import.cpp ) PEERDIR( diff --git a/ydb/public/lib/ydb_cli/ya.make b/ydb/public/lib/ydb_cli/ya.make index 2b6ac7c24a..c976e9e52e 100644 --- a/ydb/public/lib/ydb_cli/ya.make +++ b/ydb/public/lib/ydb_cli/ya.make @@ -1,10 +1,10 @@ - -OWNER(g:kikimr) - -RECURSE( - commands - common + +OWNER(g:kikimr) + +RECURSE( + commands + common dump import -) - +) + diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp index 3bd5128572..38b8c8123d 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp @@ -42,100 +42,100 @@ const TMaybe<NTable::TQueryStats>& TExecuteYqlResult::GetStats() const { //////////////////////////////////////////////////////////////////////////////// -class TYqlResultPartIterator::TReaderImpl { -public: - using TSelf = TYqlResultPartIterator::TReaderImpl; - using TResponse = Ydb::Scripting::ExecuteYqlPartialResponse; - using TStreamProcessorPtr = NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; - using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; - using TGRpcStatus = NGrpc::TGrpcStatus; - +class TYqlResultPartIterator::TReaderImpl { +public: + using TSelf = TYqlResultPartIterator::TReaderImpl; + using TResponse = Ydb::Scripting::ExecuteYqlPartialResponse; + using TStreamProcessorPtr = NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; + using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; + using TGRpcStatus = NGrpc::TGrpcStatus; + TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) - : StreamProcessor_(streamProcessor) - , Finished_(false) - , Endpoint_(endpoint) - {} - - ~TReaderImpl() { - StreamProcessor_->Cancel(); - } - - bool IsFinished() const { - return Finished_; - } - - TAsyncYqlResultPart ReadNext(std::shared_ptr<TSelf> self) { - auto promise = NThreading::NewPromise<TYqlResultPart>(); - // Capture self - guarantee no dtor call during the read - auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { - if (!grpcStatus.Ok()) { - self->Finished_ = true; + : StreamProcessor_(streamProcessor) + , Finished_(false) + , Endpoint_(endpoint) + {} + + ~TReaderImpl() { + StreamProcessor_->Cancel(); + } + + bool IsFinished() const { + return Finished_; + } + + TAsyncYqlResultPart ReadNext(std::shared_ptr<TSelf> self) { + auto promise = NThreading::NewPromise<TYqlResultPart>(); + // Capture self - guarantee no dtor call during the read + auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable { + if (!grpcStatus.Ok()) { + self->Finished_ = true; promise.SetValue({ TStatus(TPlainStatus(grpcStatus, self->Endpoint_)) }); - } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); - EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); - TPlainStatus plainStatus{ clientStatus, std::move(issues), self->Endpoint_, {} }; + } else { + NYql::TIssues issues; + NYql::IssuesFromMessage(self->Response_.issues(), issues); + EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); + TPlainStatus plainStatus{ clientStatus, std::move(issues), self->Endpoint_, {} }; TStatus status{ std::move(plainStatus) }; - TMaybe<NTable::TQueryStats> queryStats; - - if (self->Response_.result().has_query_stats()) { - queryStats = NTable::TQueryStats(self->Response_.result().query_stats()); - } - if (self->Response_.result().has_result_set()) { - promise.SetValue( - { - std::move(status), - TYqlPartialResult( - self->Response_.result().Getresult_set_index(), - TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())) - ), - queryStats - } - ); - } else { - promise.SetValue({ std::move(status), queryStats }); - } - } - }; - StreamProcessor_->Read(&Response_, readCb); - return promise.GetFuture(); - } -private: - TStreamProcessorPtr StreamProcessor_; - TResponse Response_; - bool Finished_; - TString Endpoint_; -}; - -TYqlResultPartIterator::TYqlResultPartIterator( - std::shared_ptr<TReaderImpl> impl, + TMaybe<NTable::TQueryStats> queryStats; + + if (self->Response_.result().has_query_stats()) { + queryStats = NTable::TQueryStats(self->Response_.result().query_stats()); + } + if (self->Response_.result().has_result_set()) { + promise.SetValue( + { + std::move(status), + TYqlPartialResult( + self->Response_.result().Getresult_set_index(), + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())) + ), + queryStats + } + ); + } else { + promise.SetValue({ std::move(status), queryStats }); + } + } + }; + StreamProcessor_->Read(&Response_, readCb); + return promise.GetFuture(); + } +private: + TStreamProcessorPtr StreamProcessor_; + TResponse Response_; + bool Finished_; + TString Endpoint_; +}; + +TYqlResultPartIterator::TYqlResultPartIterator( + std::shared_ptr<TReaderImpl> impl, TPlainStatus&& status) : TStatus(std::move(status)) - , ReaderImpl_(impl) -{} - -TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() { - if (ReaderImpl_->IsFinished()) - RaiseError("Attempt to perform read on invalid or finished stream"); - return ReaderImpl_->ReadNext(ReaderImpl_); -} - -//////////////////////////////////////////////////////////////////////////////// - -TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan) + , ReaderImpl_(impl) +{} + +TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() { + if (ReaderImpl_->IsFinished()) + RaiseError("Attempt to perform read on invalid or finished stream"); + return ReaderImpl_->ReadNext(ReaderImpl_); +} + +//////////////////////////////////////////////////////////////////////////////// + +TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan) : TStatus(std::move(status)) - , ParameterTypes_(std::move(types)) + , ParameterTypes_(std::move(types)) , Plan_(plan) {} -std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const { - std::map<TString, TType> typesMap; - for (const auto& param : ParameterTypes_) { - typesMap.emplace(param.first, TType(param.second)); - } - return typesMap; -} - +std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const { + std::map<TString, TType> typesMap; + for (const auto& param : ParameterTypes_) { + typesMap.emplace(param.first, TType(param.second)); + } + return typesMap; +} + const TString& TExplainYqlResult::GetPlan() const { return Plan_; } @@ -144,8 +144,8 @@ const TString& TExplainYqlResult::GetPlan() const { class TScriptingClient::TImpl : public TClientImplCommon<TScriptingClient::TImpl> { public: - using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr; - + using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr; + TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) : TClientImplCommon(std::move(connections), settings) {} @@ -195,56 +195,56 @@ public: return promise.GetFuture(); } - template<typename TParamsType> - TFuture<std::pair<TPlainStatus, TYqlScriptProcessorPtr>> StreamExecuteYqlScriptInternal(const TString& script, - TParamsType params, const TExecuteYqlRequestSettings& settings) - { - auto request = MakeOperationRequest<Ydb::Scripting::ExecuteYqlRequest>(settings); - request.set_script(script); - SetParams(params, &request); - request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); - - auto promise = NewPromise<std::pair<TPlainStatus, TYqlScriptProcessorPtr>>(); - - Connections_->StartReadStream< - Ydb::Scripting::V1::ScriptingService, - Ydb::Scripting::ExecuteYqlRequest, - Ydb::Scripting::ExecuteYqlPartialResponse> - ( - std::move(request), - [promise](TPlainStatus status, TYqlScriptProcessorPtr processor) mutable { - promise.SetValue(std::make_pair(status, processor)); - }, - &Ydb::Scripting::V1::ScriptingService::Stub::AsyncStreamExecuteYql, - DbDriverState_, - TRpcRequestSettings::Make(settings) - ); - - return promise.GetFuture(); - } - - template<typename TParamsType> - TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& query, TParamsType params, - const TExecuteYqlRequestSettings& settings) - { - auto promise = NewPromise<TYqlResultPartIterator>(); - + template<typename TParamsType> + TFuture<std::pair<TPlainStatus, TYqlScriptProcessorPtr>> StreamExecuteYqlScriptInternal(const TString& script, + TParamsType params, const TExecuteYqlRequestSettings& settings) + { + auto request = MakeOperationRequest<Ydb::Scripting::ExecuteYqlRequest>(settings); + request.set_script(script); + SetParams(params, &request); + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + + auto promise = NewPromise<std::pair<TPlainStatus, TYqlScriptProcessorPtr>>(); + + Connections_->StartReadStream< + Ydb::Scripting::V1::ScriptingService, + Ydb::Scripting::ExecuteYqlRequest, + Ydb::Scripting::ExecuteYqlPartialResponse> + ( + std::move(request), + [promise](TPlainStatus status, TYqlScriptProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Scripting::V1::ScriptingService::Stub::AsyncStreamExecuteYql, + DbDriverState_, + TRpcRequestSettings::Make(settings) + ); + + return promise.GetFuture(); + } + + template<typename TParamsType> + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& query, TParamsType params, + const TExecuteYqlRequestSettings& settings) + { + auto promise = NewPromise<TYqlResultPartIterator>(); + auto iteratorCallback = [promise](TFuture<std::pair<TPlainStatus, - TScriptingClient::TImpl::TYqlScriptProcessorPtr>> future) mutable - { - Y_ASSERT(future.HasValue()); - auto pair = future.ExtractValue(); - promise.SetValue(TYqlResultPartIterator( - pair.second + TScriptingClient::TImpl::TYqlScriptProcessorPtr>> future) mutable + { + Y_ASSERT(future.HasValue()); + auto pair = future.ExtractValue(); + promise.SetValue(TYqlResultPartIterator( + pair.second ? std::make_shared<TYqlResultPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint) - : nullptr, + : nullptr, std::move(pair.first))); - }; - - StreamExecuteYqlScriptInternal(query, params, settings).Subscribe(iteratorCallback); - return promise.GetFuture(); - } - + }; + + StreamExecuteYqlScriptInternal(query, params, settings).Subscribe(iteratorCallback); + return promise.GetFuture(); + } + TAsyncExplainYqlResult ExplainYqlScript(const TString& script, const TExplainYqlRequestSettings& settings) { @@ -252,10 +252,10 @@ public: request.set_script(script); switch (settings.Mode_) { - // KIKIMR-10990 - //case ExplainYqlRequestMode::Parse: - // request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PARSE); - // break; + // KIKIMR-10990 + //case ExplainYqlRequestMode::Parse: + // request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PARSE); + // break; case ExplainYqlRequestMode::Validate: request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE); break; @@ -269,17 +269,17 @@ public: auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { TString plan; - ::google::protobuf::Map<TString, Ydb::Type> types; + ::google::protobuf::Map<TString, Ydb::Type> types; if (any) { Ydb::Scripting::ExplainYqlResult result; any->UnpackTo(&result); plan = result.plan(); - types = result.parameters_types(); + types = result.parameters_types(); } TExplainYqlResult explainResult(TStatus(std::move(status)), - std::move(types), std::move(plan)); + std::move(types), std::move(plan)); promise.SetValue(std::move(explainResult)); }; @@ -297,15 +297,15 @@ public: } private: - template<typename TRequest> - static void SetParams(::google::protobuf::Map<TString, Ydb::TypedValue>* params, TRequest* request) { + template<typename TRequest> + static void SetParams(::google::protobuf::Map<TString, Ydb::TypedValue>* params, TRequest* request) { if (params) { request->mutable_parameters()->swap(*params); } } - template<typename TRequest> - static void SetParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& params, TRequest* request) { + template<typename TRequest> + static void SetParams(const ::google::protobuf::Map<TString, Ydb::TypedValue>& params, TRequest* request) { *request->mutable_parameters() = params; } @@ -343,36 +343,36 @@ TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, } } -TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &script, +TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &script, const TExecuteYqlRequestSettings &settings) { - return Impl_->ExecuteYqlScript(script, nullptr, settings); + return Impl_->ExecuteYqlScript(script, nullptr, settings); +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, + const TExecuteYqlRequestSettings& settings) +{ + return Impl_->StreamExecuteYqlScript(script, nullptr, settings); +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, const TParams& params, + const TExecuteYqlRequestSettings& settings) +{ + if (params.Empty()) { + return Impl_->StreamExecuteYqlScript(script, nullptr, settings); + } else { + using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; + return Impl_->StreamExecuteYqlScript<TProtoParamsType&>(script, params.GetProtoMap(), settings); + } +} + +TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, TParams&& params, + const TExecuteYqlRequestSettings& settings) +{ + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + return Impl_->StreamExecuteYqlScript(script, paramsPtr, settings); } -TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, - const TExecuteYqlRequestSettings& settings) -{ - return Impl_->StreamExecuteYqlScript(script, nullptr, settings); -} - -TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, const TParams& params, - const TExecuteYqlRequestSettings& settings) -{ - if (params.Empty()) { - return Impl_->StreamExecuteYqlScript(script, nullptr, settings); - } else { - using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; - return Impl_->StreamExecuteYqlScript<TProtoParamsType&>(script, params.GetProtoMap(), settings); - } -} - -TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TString& script, TParams&& params, - const TExecuteYqlRequestSettings& settings) -{ - auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); - return Impl_->StreamExecuteYqlScript(script, paramsPtr, settings); -} - TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script, const TExplainYqlRequestSettings& settings) { diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h index b8a3e55fdf..e99f05daa2 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h @@ -23,95 +23,95 @@ private: TMaybe<NTable::TQueryStats> QueryStats_; }; -class TYqlPartialResult { -public: - TYqlPartialResult(ui32 resultSetIndex, TResultSet&& resultSet) - : ResultSetIndex_(resultSetIndex) - , ResultSet_(std::move(resultSet)) - {} - - ui32 GetResultSetIndex() const { return ResultSetIndex_; } - const TResultSet& GetResultSet() const { return ResultSet_; } - -private: - ui32 ResultSetIndex_; - TResultSet ResultSet_; -}; - -class TYqlResultPart : public TStreamPartStatus { -public: - bool HasPartialResult() const { return PartialResult_.Defined(); } - const TYqlPartialResult& GetPartialResult() const { return *PartialResult_; } - - bool HasQueryStats() const { return QueryStats_.Defined(); } - const NTable::TQueryStats& GetQueryStats() const { return *QueryStats_; } - NTable::TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } - - TYqlResultPart(TStatus&& status) - : TStreamPartStatus(std::move(status)) - {} - - TYqlResultPart(TStatus&& status, const TMaybe<NTable::TQueryStats> &queryStats) - : TStreamPartStatus(std::move(status)) - , QueryStats_(queryStats) - {} - - TYqlResultPart(TStatus&& status, TYqlPartialResult&& partialResult, const TMaybe<NTable::TQueryStats> &queryStats) - : TStreamPartStatus(std::move(status)) - , PartialResult_(std::move(partialResult)) - , QueryStats_(queryStats) - {} - -private: - TMaybe<TYqlPartialResult> PartialResult_; - TMaybe<NTable::TQueryStats> QueryStats_; -}; - -using TAsyncYqlResultPart = NThreading::TFuture<TYqlResultPart>; - -class TYqlResultPartIterator : public TStatus { - friend class TScriptingClient; -public: - TAsyncYqlResultPart ReadNext(); - class TReaderImpl; -private: - TYqlResultPartIterator( - std::shared_ptr<TReaderImpl> impl, +class TYqlPartialResult { +public: + TYqlPartialResult(ui32 resultSetIndex, TResultSet&& resultSet) + : ResultSetIndex_(resultSetIndex) + , ResultSet_(std::move(resultSet)) + {} + + ui32 GetResultSetIndex() const { return ResultSetIndex_; } + const TResultSet& GetResultSet() const { return ResultSet_; } + +private: + ui32 ResultSetIndex_; + TResultSet ResultSet_; +}; + +class TYqlResultPart : public TStreamPartStatus { +public: + bool HasPartialResult() const { return PartialResult_.Defined(); } + const TYqlPartialResult& GetPartialResult() const { return *PartialResult_; } + + bool HasQueryStats() const { return QueryStats_.Defined(); } + const NTable::TQueryStats& GetQueryStats() const { return *QueryStats_; } + NTable::TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } + + TYqlResultPart(TStatus&& status) + : TStreamPartStatus(std::move(status)) + {} + + TYqlResultPart(TStatus&& status, const TMaybe<NTable::TQueryStats> &queryStats) + : TStreamPartStatus(std::move(status)) + , QueryStats_(queryStats) + {} + + TYqlResultPart(TStatus&& status, TYqlPartialResult&& partialResult, const TMaybe<NTable::TQueryStats> &queryStats) + : TStreamPartStatus(std::move(status)) + , PartialResult_(std::move(partialResult)) + , QueryStats_(queryStats) + {} + +private: + TMaybe<TYqlPartialResult> PartialResult_; + TMaybe<NTable::TQueryStats> QueryStats_; +}; + +using TAsyncYqlResultPart = NThreading::TFuture<TYqlResultPart>; + +class TYqlResultPartIterator : public TStatus { + friend class TScriptingClient; +public: + TAsyncYqlResultPart ReadNext(); + class TReaderImpl; +private: + TYqlResultPartIterator( + std::shared_ptr<TReaderImpl> impl, TPlainStatus&& status - ); - std::shared_ptr<TReaderImpl> ReaderImpl_; -}; - + ); + std::shared_ptr<TReaderImpl> ReaderImpl_; +}; + class TExplainYqlResult : public TStatus { public: - TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan); + TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan); - std::map<TString, TType> GetParameterTypes() const; + std::map<TString, TType> GetParameterTypes() const; const TString& GetPlan() const; private: - ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_; + ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_; TString Plan_; }; using TAsyncExecuteYqlResult = NThreading::TFuture<TExecuteYqlResult>; -using TAsyncYqlResultPartIterator = NThreading::TFuture<TYqlResultPartIterator>; +using TAsyncYqlResultPartIterator = NThreading::TFuture<TYqlResultPartIterator>; using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>; //////////////////////////////////////////////////////////////////////////////// struct TExecuteYqlRequestSettings : public TOperationRequestSettings<TExecuteYqlRequestSettings> { - FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); + FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); }; enum class ExplainYqlRequestMode { - // Parse = 1, + // Parse = 1, Validate = 2, Plan = 3, }; struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYqlRequestSettings> { - FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); + FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); }; //////////////////////////////////////////////////////////////////////////////// @@ -134,15 +134,15 @@ public: TAsyncExecuteYqlResult ExecuteYqlScript(const TString& script, TParams&& params, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, - const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - - TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, const TParams& params, - const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - - TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, TParams&& params, - const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, const TParams& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + + TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, TParams&& params, + const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); + TAsyncExplainYqlResult ExplainYqlScript(const TString& script, const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings()); diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 619c47a447..c43e9aab69 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -19,7 +19,7 @@ public: const TMaybe<TString>& discoveryEndpoint, const TMaybe<EDiscoveryMode>& discoveryMode, const TMaybe<bool>& enableSsl, - const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory) + const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory) : Connections_(std::move(connections)) , DbDriverState_(Connections_->GetDriverState(database, discoveryEndpoint, discoveryMode, enableSsl, credentialsProviderFactory)) { diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp index 26344eb4d6..6957a558ec 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp @@ -37,25 +37,25 @@ const TVector<TEndpointInfo>& TListEndpointsResult::GetEndpointsInfo() const { return Info_; } -TWhoAmIResult::TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto) - : TStatus(std::move(status)) -{ - UserName_ = proto.user(); - const auto& groups = proto.groups(); - Groups_.reserve(groups.size()); - for (const auto& group : groups) { - Groups_.emplace_back(group); - } -} - -const TString& TWhoAmIResult::GetUserName() const { - return UserName_; -} - -const TVector<TString>& TWhoAmIResult::GetGroups() const { - return Groups_; -} - +TWhoAmIResult::TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto) + : TStatus(std::move(status)) +{ + UserName_ = proto.user(); + const auto& groups = proto.groups(); + Groups_.reserve(groups.size()); + for (const auto& group : groups) { + Groups_.emplace_back(group); + } +} + +const TString& TWhoAmIResult::GetUserName() const { + return UserName_; +} + +const TVector<TString>& TWhoAmIResult::GetGroups() const { + return Groups_; +} + class TDiscoveryClient::TImpl : public TClientImplCommon<TDiscoveryClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) @@ -89,36 +89,36 @@ public: return promise.GetFuture(); } - - TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings) { - Ydb::Discovery::WhoAmIRequest request; - if (settings.WithGroups_) { - request.set_include_groups(true); - } - - auto promise = NThreading::NewPromise<TWhoAmIResult>(); - + + TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings) { + Ydb::Discovery::WhoAmIRequest request; + if (settings.WithGroups_) { + request.set_include_groups(true); + } + + auto promise = NThreading::NewPromise<TWhoAmIResult>(); + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Discovery::WhoAmIResult result; - if (any) { - any->UnpackTo(&result); - } + Ydb::Discovery::WhoAmIResult result; + if (any) { + any->UnpackTo(&result); + } TWhoAmIResult val{ TStatus(std::move(status)), result }; - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::WhoAmIRequest, Ydb::Discovery::WhoAmIResponse>( - std::move(request), - extractor, - &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncWhoAmI, - DbDriverState_, + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::WhoAmIRequest, Ydb::Discovery::WhoAmIResponse>( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncWhoAmI, + DbDriverState_, INITIAL_DEFERRED_CALL_DELAY, TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return promise.GetFuture(); - } + settings.ClientTimeout_); + + return promise.GetFuture(); + } }; TDiscoveryClient::TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings) @@ -129,9 +129,9 @@ TAsyncListEndpointsResult TDiscoveryClient::ListEndpoints(const TListEndpointsSe return Impl_->ListEndpoints(settings); } -TAsyncWhoAmIResult TDiscoveryClient::WhoAmI(const TWhoAmISettings& settings) { - return Impl_->WhoAmI(settings); -} - +TAsyncWhoAmIResult TDiscoveryClient::WhoAmI(const TWhoAmISettings& settings) { + return Impl_->WhoAmI(settings); +} + } // namespace NDiscovery } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h index feb861690c..d81275b7f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h @@ -16,10 +16,10 @@ namespace NDiscovery { class TListEndpointsSettings : public TSimpleRequestSettings<TListEndpointsSettings> {}; -struct TWhoAmISettings : public TSimpleRequestSettings<TWhoAmISettings> { - FLUENT_SETTING_DEFAULT(bool, WithGroups, false); -}; - +struct TWhoAmISettings : public TSimpleRequestSettings<TWhoAmISettings> { + FLUENT_SETTING_DEFAULT(bool, WithGroups, false); +}; + struct TEndpointInfo { TString Address; ui32 Port = 0; @@ -42,18 +42,18 @@ private: using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>; -class TWhoAmIResult : public TStatus { -public: - TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto); - const TString& GetUserName() const; - const TVector<TString>& GetGroups() const; -private: - TString UserName_; - TVector<TString> Groups_; -}; - -using TAsyncWhoAmIResult = NThreading::TFuture<TWhoAmIResult>; - +class TWhoAmIResult : public TStatus { +public: + TWhoAmIResult(TStatus&& status, const Ydb::Discovery::WhoAmIResult& proto); + const TString& GetUserName() const; + const TVector<TString>& GetGroups() const; +private: + TString UserName_; + TVector<TString> Groups_; +}; + +using TAsyncWhoAmIResult = NThreading::TFuture<TWhoAmIResult>; + //////////////////////////////////////////////////////////////////////////////// class TDiscoveryClient { @@ -61,7 +61,7 @@ public: explicit TDiscoveryClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); TAsyncListEndpointsResult ListEndpoints(const TListEndpointsSettings& settings = TListEndpointsSettings()); - TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings = TWhoAmISettings()); + TAsyncWhoAmIResult WhoAmI(const TWhoAmISettings& settings = TWhoAmISettings()); private: class TImpl; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp index c5fd437bbd..de17c6e679 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp @@ -32,10 +32,10 @@ public: size_t GetNetworkThreadsNum() const override { return NetworkThreadsNum; } size_t GetClientThreadsNum() const override { return ClientThreadsNum; } size_t GetMaxQueuedResponses() const override { return MaxQueuedResponses; } - bool IsSslEnabled() const override { return EnableSsl; } + bool IsSslEnabled() const override { return EnableSsl; } TStringType GetCaCert() const override { return CaCert; } TStringType GetDatabase() const override { return Database; } - std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const override { return CredentialsProviderFactory; } + std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const override { return CredentialsProviderFactory; } EDiscoveryMode GetDiscoveryMode() const override { return DiscoveryMode; } size_t GetMaxQueuedRequests() const override { return MaxQueuedRequests; } TTcpKeepAliveSettings GetTcpKeepAliveSettings() const override { return TcpKeepAliveSettings; } @@ -51,10 +51,10 @@ public: size_t NetworkThreadsNum = 2; size_t ClientThreadsNum = 0; size_t MaxQueuedResponses = 0; - bool EnableSsl = false; + bool EnableSsl = false; TStringType CaCert; TStringType Database; - std::shared_ptr<ICredentialsProviderFactory> CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); + std::shared_ptr<ICredentialsProviderFactory> CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; size_t MaxQueuedRequests = 100; NGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings = @@ -104,13 +104,13 @@ TDriverConfig& TDriverConfig::SetMaxClientQueueSize(size_t sz) { } TDriverConfig& TDriverConfig::UseSecureConnection(const TStringType& cert) { - Impl_->EnableSsl = true; + Impl_->EnableSsl = true; Impl_->CaCert = cert; return *this; } TDriverConfig& TDriverConfig::SetAuthToken(const TStringType& token) { - return SetCredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token)); + return SetCredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token)); } TDriverConfig& TDriverConfig::SetDatabase(const TStringType& database) { @@ -119,11 +119,11 @@ TDriverConfig& TDriverConfig::SetDatabase(const TStringType& database) { return *this; } -TDriverConfig& TDriverConfig::SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory) { - Impl_->CredentialsProviderFactory = credentialsProviderFactory; - return *this; -} - +TDriverConfig& TDriverConfig::SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory) { + Impl_->CredentialsProviderFactory = credentialsProviderFactory; + return *this; +} + TDriverConfig& TDriverConfig::SetDiscoveryMode(EDiscoveryMode discoveryMode) { Impl_->DiscoveryMode = discoveryMode; return *this; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.h b/ydb/public/sdk/cpp/client/ydb_driver/driver.h index cf5189bacd..39c91d8852 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.h +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.h @@ -37,26 +37,26 @@ public: //! of this pool is blocked somewhere in user code. //! default: 0 TDriverConfig& SetClientThreadsNum(size_t sz); - //! Warning: not recommended to change + //! Warning: not recommended to change //! Set max number of queued responses. 0 - no limit //! There is a queue to perform async calls to user code, //! if this queue is full, attempts to enqueue responses inside sdk will be blocked - //! Size of this queue must be greater than max size of all requests inflight + //! Size of this queue must be greater than max size of all requests inflight //! Note: if this limit is reached network threads will be blocked. //! Note: set of this limit can cause deadlock in some case of using async interface //! This value doesn't make sense if SetClientThreadsNum is 0 //! default: 0 TDriverConfig& SetMaxClientQueueSize(size_t sz); - //! Enable Ssl. - //! caCerts - The buffer containing the PEM encoding of the server root certificates. - //! If this parameter is empty, the default roots will be used. + //! Enable Ssl. + //! caCerts - The buffer containing the PEM encoding of the server root certificates. + //! If this parameter is empty, the default roots will be used. TDriverConfig& UseSecureConnection(const TStringType& caCerts = TStringType()); - //! Set token, this option can be overridden for client by ClientSettings + //! Set token, this option can be overridden for client by ClientSettings TDriverConfig& SetAuthToken(const TStringType& token); - //! Set database, this option can be overridden for client by ClientSettings + //! Set database, this option can be overridden for client by ClientSettings TDriverConfig& SetDatabase(const TStringType& database); - //! Set credentials data, this option can be overridden for client by ClientSettings - TDriverConfig& SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory); + //! Set credentials data, this option can be overridden for client by ClientSettings + TDriverConfig& SetCredentialsProviderFactory(std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory); //! Set behaviour of discovery routine //! See EDiscoveryMode enum comments //! default: EDiscoveryMode::Sync @@ -71,7 +71,7 @@ public: //! Specify tcp keep alive settings //! This option allows to adjust tcp keep alive settings, useful to work //! with balancers or to detect unexpected connectivity problem. - //! enable - if true enable tcp keep alive and use following settings + //! enable - if true enable tcp keep alive and use following settings //! - if false disable tcp keep alive //! idle - (Linux only) the interval between the last data packet sent and the first keepalive probe, sec //! if zero use OS default @@ -85,11 +85,11 @@ public: //! Enable or disable drain of client logic (e.g. session pool drain) during dtor call TDriverConfig& SetDrainOnDtors(bool allowed); //! Set policy for balancing - //! Params is a optionally field to set policy settings + //! Params is a optionally field to set policy settings //! default: EBalancingPolicy::UsePreferableLocation TDriverConfig& SetBalancingPolicy(EBalancingPolicy policy, const TStringType& params = TStringType()); //! !!! EXPERIMENTAL !!! - //! Set grpc level keep alive. If keepalive ping was delayed more than given timeout + //! Set grpc level keep alive. If keepalive ping was delayed more than given timeout //! internal grpc routine fails request with TRANSIENT_FAILURE or TRANSPORT_UNAVAILABLE error //! Note: this timeout should not be too small to prevent fail due to //! network buffers delay. I.e. values less than 5 seconds may cause request failure diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h index b8cb120611..be0165d74b 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h @@ -33,7 +33,7 @@ public: static const Ydb::Value& GetProto(const TValue& value); static const Ydb::ResultSet& GetProto(const TResultSet& resultSet); static const Ydb::TableStats::QueryStats& GetProto(const NTable::TQueryStats& queryStats); - static const Ydb::Table::DescribeTableResult& GetProto(const NTable::TTableDescription& tableDescription); + static const Ydb::Table::DescribeTableResult& GetProto(const NTable::TTableDescription& tableDescription); static const Ydb::PersQueue::V1::DescribeTopicResult& GetProto(const NYdb::NPersQueue::TDescribeTopicResult& topicDescription); static NTable::TQueryStats FromProto(const Ydb::TableStats::QueryStats& queryStats); diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h index bb35e4f5fe..355c271dc2 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h @@ -24,7 +24,7 @@ class TProtoAccessor; namespace NScripting { class TScriptingClient; -class TYqlResultPartIterator; +class TYqlResultPartIterator; } // namespace NScripting @@ -40,7 +40,7 @@ class TQueryStats { friend class TTableClient; friend class NYdb::TProtoAccessor; friend class NYdb::NScripting::TScriptingClient; - friend class NYdb::NScripting::TYqlResultPartIterator; + friend class NYdb::NScripting::TYqlResultPartIterator; friend class TScanQueryPartIterator; public: diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 46d1565bdb..610fec3884 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -173,41 +173,41 @@ const TBuildIndexOperation::TMetadata& TBuildIndexOperation::Metadata() const { //////////////////////////////////////////////////////////////////////////////// -class TPartitioningSettings::TImpl { -public: - TImpl() { } - - explicit TImpl(const Ydb::Table::PartitioningSettings& proto) - : Proto_(proto) - { } - -public: - const Ydb::Table::PartitioningSettings Proto_; -}; - -TPartitioningSettings::TPartitioningSettings() - : Impl_(std::make_shared<TImpl>()) -{ } - -TPartitioningSettings::TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto) - : Impl_(std::make_shared<TImpl>(proto)) -{ } - -const Ydb::Table::PartitioningSettings& TPartitioningSettings::GetProto() const { - return Impl_->Proto_; -} - -TMaybe<bool> TPartitioningSettings::GetPartitioningBySize() const { - switch (GetProto().partitioning_by_size()) { - case Ydb::FeatureFlag::ENABLED: - return true; - case Ydb::FeatureFlag::DISABLED: - return false; - default: - return { }; - } -} - +class TPartitioningSettings::TImpl { +public: + TImpl() { } + + explicit TImpl(const Ydb::Table::PartitioningSettings& proto) + : Proto_(proto) + { } + +public: + const Ydb::Table::PartitioningSettings Proto_; +}; + +TPartitioningSettings::TPartitioningSettings() + : Impl_(std::make_shared<TImpl>()) +{ } + +TPartitioningSettings::TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto) + : Impl_(std::make_shared<TImpl>(proto)) +{ } + +const Ydb::Table::PartitioningSettings& TPartitioningSettings::GetProto() const { + return Impl_->Proto_; +} + +TMaybe<bool> TPartitioningSettings::GetPartitioningBySize() const { + switch (GetProto().partitioning_by_size()) { + case Ydb::FeatureFlag::ENABLED: + return true; + case Ydb::FeatureFlag::DISABLED: + return false; + default: + return { }; + } +} + TMaybe<bool> TPartitioningSettings::GetPartitioningByLoad() const { switch (GetProto().partitioning_by_load()) { case Ydb::FeatureFlag::ENABLED: @@ -219,20 +219,20 @@ TMaybe<bool> TPartitioningSettings::GetPartitioningByLoad() const { } } -ui64 TPartitioningSettings::GetPartitionSizeMb() const { - return GetProto().partition_size_mb(); -} - -ui64 TPartitioningSettings::GetMinPartitionsCount() const { - return GetProto().min_partitions_count(); -} - -ui64 TPartitioningSettings::GetMaxPartitionsCount() const { - return GetProto().max_partitions_count(); -} - -//////////////////////////////////////////////////////////////////////////////// - +ui64 TPartitioningSettings::GetPartitionSizeMb() const { + return GetProto().partition_size_mb(); +} + +ui64 TPartitioningSettings::GetMinPartitionsCount() const { + return GetProto().min_partitions_count(); +} + +ui64 TPartitioningSettings::GetMaxPartitionsCount() const { + return GetProto().max_partitions_count(); +} + +//////////////////////////////////////////////////////////////////////////////// + struct TTableStats { ui64 Rows = 0; ui64 Size = 0; @@ -256,7 +256,7 @@ class TTableDescription::TImpl { , PartitioningSettings_(proto.partitioning_settings()) , HasStorageSettings_(proto.has_storage_settings()) , HasPartitioningSettings_(proto.has_partitioning_settings()) - { + { // primary key for (const auto& pk : proto.primary_key()) { PrimaryKey_.push_back(pk); @@ -342,12 +342,12 @@ public: { Proto_ = std::move(desc); - Owner_ = Proto_.self().owner(); - PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); - PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); + Owner_ = Proto_.self().owner(); + PermissionToSchemeEntry(Proto_.self().permissions(), &Permissions_); + PermissionToSchemeEntry(Proto_.self().effective_permissions(), &EffectivePermissions_); TMaybe<TValue> leftValue; - for (const auto& bound : Proto_.shard_key_bounds()) { + for (const auto& bound : Proto_.shard_key_bounds()) { TMaybe<TKeyBound> fromBound = leftValue ? TKeyBound::Inclusive(*leftValue) : TMaybe<TKeyBound>(); @@ -359,18 +359,18 @@ public: leftValue = value; } - for (const auto& shardStats : Proto_.table_stats().partition_stats()) { + for (const auto& shardStats : Proto_.table_stats().partition_stats()) { PartitionStats_.emplace_back( TPartitionStats{shardStats.rows_estimate(), shardStats.store_size()} ); } - TableStats.Rows = Proto_.table_stats().rows_estimate(); - TableStats.Size = Proto_.table_stats().store_size(); - TableStats.Partitions = Proto_.table_stats().partitions(); + TableStats.Rows = Proto_.table_stats().rows_estimate(); + TableStats.Size = Proto_.table_stats().store_size(); + TableStats.Partitions = Proto_.table_stats().partitions(); - TableStats.ModificationTime = ProtobufTimestampToTInstant(Proto_.table_stats().modification_time()); - TableStats.CreationTime = ProtobufTimestampToTInstant(Proto_.table_stats().creation_time()); + TableStats.ModificationTime = ProtobufTimestampToTInstant(Proto_.table_stats().modification_time()); + TableStats.CreationTime = ProtobufTimestampToTInstant(Proto_.table_stats().creation_time()); if (describeSettings.WithKeyShardBoundary_) { Ranges_.emplace_back(TKeyRange( @@ -387,12 +387,12 @@ public: if (request.compaction_policy()) { SetCompactionPolicy(request.compaction_policy()); } - + switch (request.partitions_case()) { case Ydb::Table::CreateTableRequest::kUniformPartitions: SetUniformPartitions(request.uniform_partitions()); break; - + case Ydb::Table::CreateTableRequest::kPartitionAtKeys: { TExplicitPartitions partitionAtKeys; for (const auto& splitPoint : request.partition_at_keys().split_points()) { @@ -401,20 +401,20 @@ public: } SetPartitionAtKeys(partitionAtKeys); - break; + break; } - default: - break; - } + default: + break; + } } TImpl() = default; - const Ydb::Table::DescribeTableResult& GetProto() const { - return Proto_; - } - + const Ydb::Table::DescribeTableResult& GetProto() const { + return Proto_; + } + void AddColumn(const TString& name, const Ydb::Type& type, const TString& family) { Columns_.emplace_back(name, type, family); } @@ -460,31 +460,31 @@ public: Attributes_ = std::move(attrs); } - void SetCompactionPolicy(const TString& name) { - CompactionPolicy_ = name; - } - - void SetUniformPartitions(ui64 partitionsCount) { - UniformPartitions_ = partitionsCount; - } - - void SetPartitionAtKeys(const TExplicitPartitions& keys) { - PartitionAtKeys_ = keys; - } - - void SetPartitioningSettings(const TPartitioningSettings& settings) { - PartitioningSettings_ = settings; - HasPartitioningSettings_ = true; - } - - void SetKeyBloomFilter(bool enabled) { - KeyBloomFilter_ = enabled; - } - - void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { - ReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); - } - + void SetCompactionPolicy(const TString& name) { + CompactionPolicy_ = name; + } + + void SetUniformPartitions(ui64 partitionsCount) { + UniformPartitions_ = partitionsCount; + } + + void SetPartitionAtKeys(const TExplicitPartitions& keys) { + PartitionAtKeys_ = keys; + } + + void SetPartitioningSettings(const TPartitioningSettings& settings) { + PartitioningSettings_ = settings; + HasPartitioningSettings_ = true; + } + + void SetKeyBloomFilter(bool enabled) { + KeyBloomFilter_ = enabled; + } + + void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { + ReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); + } + const TVector<TString>& GetPrimaryKeyColumns() const { return PrimaryKey_; } @@ -505,10 +505,10 @@ public: return Owner_; } - const TVector<NScheme::TPermissions>& GetPermissions() const { - return Permissions_; - } - + const TVector<NScheme::TPermissions>& GetPermissions() const { + return Permissions_; + } + const TVector<NScheme::TPermissions>& GetEffectivePermissions() const { return EffectivePermissions_; } @@ -557,41 +557,41 @@ public: return HasPartitioningSettings_; } - const TPartitioningSettings& GetPartitioningSettings() const { - return PartitioningSettings_; - } - - TMaybe<bool> GetKeyBloomFilter() const { - return KeyBloomFilter_; - } - - const TMaybe<TReadReplicasSettings>& GetReadReplicasSettings() const { - return ReadReplicasSettings_; - } - + const TPartitioningSettings& GetPartitioningSettings() const { + return PartitioningSettings_; + } + + TMaybe<bool> GetKeyBloomFilter() const { + return KeyBloomFilter_; + } + + const TMaybe<TReadReplicasSettings>& GetReadReplicasSettings() const { + return ReadReplicasSettings_; + } + private: - Ydb::Table::DescribeTableResult Proto_; + Ydb::Table::DescribeTableResult Proto_; TStorageSettings StorageSettings_; TVector<TString> PrimaryKey_; TVector<TTableColumn> Columns_; TVector<TIndexDescription> Indexes_; TMaybe<TTtlSettings> TtlSettings_; TString Owner_; - TVector<NScheme::TPermissions> Permissions_; + TVector<NScheme::TPermissions> Permissions_; TVector<NScheme::TPermissions> EffectivePermissions_; TVector<TKeyRange> Ranges_; TVector<TPartitionStats> PartitionStats_; TTableStats TableStats; TVector<TColumnFamilyDescription> ColumnFamilies_; THashMap<TString, TString> Attributes_; - TString CompactionPolicy_; - TMaybe<ui64> UniformPartitions_; - TMaybe<TExplicitPartitions> PartitionAtKeys_; - TPartitioningSettings PartitioningSettings_; - TMaybe<bool> KeyBloomFilter_; - TMaybe<TReadReplicasSettings> ReadReplicasSettings_; + TString CompactionPolicy_; + TMaybe<ui64> UniformPartitions_; + TMaybe<TExplicitPartitions> PartitionAtKeys_; + TPartitioningSettings PartitioningSettings_; + TMaybe<bool> KeyBloomFilter_; + TMaybe<TReadReplicasSettings> ReadReplicasSettings_; bool HasStorageSettings_ = false; - bool HasPartitioningSettings_ = false; + bool HasPartitioningSettings_ = false; }; TTableDescription::TTableDescription() @@ -640,10 +640,10 @@ const TString& TTableDescription::GetOwner() const { return Impl_->GetOwner(); } -const TVector<NScheme::TPermissions>& TTableDescription::GetPermissions() const { - return Impl_->GetPermissions(); -} - +const TVector<NScheme::TPermissions>& TTableDescription::GetPermissions() const { + return Impl_->GetPermissions(); +} + const TVector<NScheme::TPermissions>& TTableDescription::GetEffectivePermissions() const { return Impl_->GetEffectivePermissions(); } @@ -720,30 +720,30 @@ void TTableDescription::SetAttributes(THashMap<TString, TString>&& attrs) { Impl_->SetAttributes(std::move(attrs)); } -void TTableDescription::SetCompactionPolicy(const TString& name) { - Impl_->SetCompactionPolicy(name); -} - -void TTableDescription::SetUniformPartitions(ui64 partitionsCount) { - Impl_->SetUniformPartitions(partitionsCount); -} - -void TTableDescription::SetPartitionAtKeys(const TExplicitPartitions& keys) { - Impl_->SetPartitionAtKeys(keys); -} - -void TTableDescription::SetPartitioningSettings(const TPartitioningSettings& settings) { - Impl_->SetPartitioningSettings(settings); -} - -void TTableDescription::SetKeyBloomFilter(bool enabled) { - Impl_->SetKeyBloomFilter(enabled); -} - -void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { - Impl_->SetReadReplicasSettings(mode, readReplicasCount); -} - +void TTableDescription::SetCompactionPolicy(const TString& name) { + Impl_->SetCompactionPolicy(name); +} + +void TTableDescription::SetUniformPartitions(ui64 partitionsCount) { + Impl_->SetUniformPartitions(partitionsCount); +} + +void TTableDescription::SetPartitionAtKeys(const TExplicitPartitions& keys) { + Impl_->SetPartitionAtKeys(keys); +} + +void TTableDescription::SetPartitioningSettings(const TPartitioningSettings& settings) { + Impl_->SetPartitioningSettings(settings); +} + +void TTableDescription::SetKeyBloomFilter(bool enabled) { + Impl_->SetKeyBloomFilter(enabled); +} + +void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { + Impl_->SetReadReplicasSettings(mode, readReplicasCount); +} + const TVector<TPartitionStats>& TTableDescription::GetPartitionStats() const { return Impl_->GetPartitionStats(); } @@ -780,22 +780,22 @@ const THashMap<TString, TString>& TTableDescription::GetAttributes() const { return Impl_->GetAttributes(); } -const TPartitioningSettings& TTableDescription::GetPartitioningSettings() const { - return Impl_->GetPartitioningSettings(); -} - -TMaybe<bool> TTableDescription::GetKeyBloomFilter() const { - return Impl_->GetKeyBloomFilter(); -} - -TMaybe<TReadReplicasSettings> TTableDescription::GetReadReplicasSettings() const { - return Impl_->GetReadReplicasSettings(); -} - -const Ydb::Table::DescribeTableResult& TTableDescription::GetProto() const { - return Impl_->GetProto(); -} - +const TPartitioningSettings& TTableDescription::GetPartitioningSettings() const { + return Impl_->GetPartitioningSettings(); +} + +TMaybe<bool> TTableDescription::GetKeyBloomFilter() const { + return Impl_->GetKeyBloomFilter(); +} + +TMaybe<TReadReplicasSettings> TTableDescription::GetReadReplicasSettings() const { + return Impl_->GetReadReplicasSettings(); +} + +const Ydb::Table::DescribeTableResult& TTableDescription::GetProto() const { + return Impl_->GetProto(); +} + void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) const { for (const auto& column : Impl_->GetColumns()) { auto& protoColumn = *request.add_columns(); @@ -914,50 +914,50 @@ TStorageSettings TStorageSettingsBuilder::Build() const { //////////////////////////////////////////////////////////////////////////////// -class TPartitioningSettingsBuilder::TImpl { -public: - Ydb::Table::PartitioningSettings Proto; -}; - -TPartitioningSettingsBuilder::TPartitioningSettingsBuilder() - : Impl_(new TImpl) -{ } - -TPartitioningSettingsBuilder::~TPartitioningSettingsBuilder() { } - -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningBySize(bool enabled) { - Impl_->Proto.set_partitioning_by_size( - enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); - return *this; -} - +class TPartitioningSettingsBuilder::TImpl { +public: + Ydb::Table::PartitioningSettings Proto; +}; + +TPartitioningSettingsBuilder::TPartitioningSettingsBuilder() + : Impl_(new TImpl) +{ } + +TPartitioningSettingsBuilder::~TPartitioningSettingsBuilder() { } + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningBySize(bool enabled) { + Impl_->Proto.set_partitioning_by_size( + enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + return *this; +} + TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningByLoad(bool enabled) { Impl_->Proto.set_partitioning_by_load( enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); return *this; } -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitionSizeMb(ui64 sizeMb) { - Impl_->Proto.set_partition_size_mb(sizeMb); - return *this; -} - -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMinPartitionsCount(ui64 count) { - Impl_->Proto.set_min_partitions_count(count); - return *this; -} - -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMaxPartitionsCount(ui64 count) { - Impl_->Proto.set_max_partitions_count(count); - return *this; -} - -TPartitioningSettings TPartitioningSettingsBuilder::Build() const { - return TPartitioningSettings(Impl_->Proto); -} - -//////////////////////////////////////////////////////////////////////////////// - +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitionSizeMb(ui64 sizeMb) { + Impl_->Proto.set_partition_size_mb(sizeMb); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMinPartitionsCount(ui64 count) { + Impl_->Proto.set_min_partitions_count(count); + return *this; +} + +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMaxPartitionsCount(ui64 count) { + Impl_->Proto.set_max_partitions_count(count); + return *this; +} + +TPartitioningSettings TPartitioningSettingsBuilder::Build() const { + return TPartitioningSettings(Impl_->Proto); +} + +//////////////////////////////////////////////////////////////////////////////// + class TColumnFamilyBuilder::TImpl { public: Ydb::Table::ColumnFamily Proto; @@ -1137,36 +1137,36 @@ TTableBuilder& TTableBuilder::SetAttributes(THashMap<TString, TString>&& attrs) return *this; } -TTableBuilder& TTableBuilder::SetCompactionPolicy(const TString& name) { - TableDescription_.SetCompactionPolicy(name); - return *this; -} - -TTableBuilder& TTableBuilder::SetUniformPartitions(ui64 partitionsCount) { - TableDescription_.SetUniformPartitions(partitionsCount); - return *this; -} - -TTableBuilder& TTableBuilder::SetPartitionAtKeys(const TExplicitPartitions& keys) { - TableDescription_.SetPartitionAtKeys(keys); - return *this; -} - -TTableBuilder& TTableBuilder::SetPartitioningSettings(const TPartitioningSettings& settings) { - TableDescription_.SetPartitioningSettings(settings); - return *this; -} - -TTableBuilder& TTableBuilder::SetKeyBloomFilter(bool enabled) { - TableDescription_.SetKeyBloomFilter(enabled); - return *this; -} - -TTableBuilder& TTableBuilder::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { - TableDescription_.SetReadReplicasSettings(mode, readReplicasCount); - return *this; -} - +TTableBuilder& TTableBuilder::SetCompactionPolicy(const TString& name) { + TableDescription_.SetCompactionPolicy(name); + return *this; +} + +TTableBuilder& TTableBuilder::SetUniformPartitions(ui64 partitionsCount) { + TableDescription_.SetUniformPartitions(partitionsCount); + return *this; +} + +TTableBuilder& TTableBuilder::SetPartitionAtKeys(const TExplicitPartitions& keys) { + TableDescription_.SetPartitionAtKeys(keys); + return *this; +} + +TTableBuilder& TTableBuilder::SetPartitioningSettings(const TPartitioningSettings& settings) { + TableDescription_.SetPartitioningSettings(settings); + return *this; +} + +TTableBuilder& TTableBuilder::SetKeyBloomFilter(bool enabled) { + TableDescription_.SetKeyBloomFilter(enabled); + return *this; +} + +TTableBuilder& TTableBuilder::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { + TableDescription_.SetReadReplicasSettings(mode, readReplicasCount); + return *this; +} + TTableDescription TTableBuilder::Build() { return TableDescription_; } @@ -3632,29 +3632,29 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin } } } - if (settings.ReplicationPolicy_) { - const auto& policy = settings.ReplicationPolicy_.GetRef(); - if (policy.PresetName_) { - proto->mutable_replication_policy()->set_preset_name(policy.PresetName_.GetRef()); - } - if (policy.ReplicasCount_) { - proto->mutable_replication_policy()->set_replicas_count(policy.ReplicasCount_.GetRef()); - } - if (policy.CreatePerAvailabilityZone_) { - proto->mutable_replication_policy()->set_create_per_availability_zone( - policy.CreatePerAvailabilityZone_.GetRef() + if (settings.ReplicationPolicy_) { + const auto& policy = settings.ReplicationPolicy_.GetRef(); + if (policy.PresetName_) { + proto->mutable_replication_policy()->set_preset_name(policy.PresetName_.GetRef()); + } + if (policy.ReplicasCount_) { + proto->mutable_replication_policy()->set_replicas_count(policy.ReplicasCount_.GetRef()); + } + if (policy.CreatePerAvailabilityZone_) { + proto->mutable_replication_policy()->set_create_per_availability_zone( + policy.CreatePerAvailabilityZone_.GetRef() ? Ydb::FeatureFlag_Status::FeatureFlag_Status_ENABLED : Ydb::FeatureFlag_Status::FeatureFlag_Status_DISABLED - ); - } - if (policy.AllowPromotion_) { - proto->mutable_replication_policy()->set_allow_promotion( - policy.AllowPromotion_.GetRef() + ); + } + if (policy.AllowPromotion_) { + proto->mutable_replication_policy()->set_allow_promotion( + policy.AllowPromotion_.GetRef() ? Ydb::FeatureFlag_Status::FeatureFlag_Status_ENABLED : Ydb::FeatureFlag_Status::FeatureFlag_Status_DISABLED - ); - } - } + ); + } + } } //////////////////////////////////////////////////////////////////////////////// @@ -3762,35 +3762,35 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( (*request.mutable_alter_attributes())[key] = value; } - if (settings.SetCompactionPolicy_) { - request.set_set_compaction_policy(settings.SetCompactionPolicy_); - } - - if (settings.AlterPartitioningSettings_) { - request.mutable_alter_partitioning_settings()->CopyFrom(settings.AlterPartitioningSettings_->GetProto()); - } - - if (settings.SetKeyBloomFilter_.Defined()) { - request.set_set_key_bloom_filter( - settings.SetKeyBloomFilter_.GetRef() ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); - } - - if (settings.SetReadReplicasSettings_.Defined()) { - const auto& replSettings = settings.SetReadReplicasSettings_.GetRef(); - switch (replSettings.GetMode()) { - case TReadReplicasSettings::EMode::PerAz: - request.mutable_set_read_replicas_settings()->set_per_az_read_replicas_count( - replSettings.GetReadReplicasCount()); - break; - case TReadReplicasSettings::EMode::AnyAz: - request.mutable_set_read_replicas_settings()->set_any_az_read_replicas_count( - replSettings.GetReadReplicasCount()); - break; - default: - break; - } - } - + if (settings.SetCompactionPolicy_) { + request.set_set_compaction_policy(settings.SetCompactionPolicy_); + } + + if (settings.AlterPartitioningSettings_) { + request.mutable_alter_partitioning_settings()->CopyFrom(settings.AlterPartitioningSettings_->GetProto()); + } + + if (settings.SetKeyBloomFilter_.Defined()) { + request.set_set_key_bloom_filter( + settings.SetKeyBloomFilter_.GetRef() ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED); + } + + if (settings.SetReadReplicasSettings_.Defined()) { + const auto& replSettings = settings.SetReadReplicasSettings_.GetRef(); + switch (replSettings.GetMode()) { + case TReadReplicasSettings::EMode::PerAz: + request.mutable_set_read_replicas_settings()->set_per_az_read_replicas_count( + replSettings.GetReadReplicasCount()); + break; + case TReadReplicasSettings::EMode::AnyAz: + request.mutable_set_read_replicas_settings()->set_any_az_read_replicas_count( + replSettings.GetReadReplicasCount()); + break; + default: + break; + } + } + return request; } @@ -4653,21 +4653,21 @@ const TMaybe<TAlterTtlSettings>& TAlterTableSettings::GetAlterTtlSettings() cons //////////////////////////////////////////////////////////////////////////////// -TReadReplicasSettings::TReadReplicasSettings(EMode mode, ui64 readReplicasCount) - : Mode_(mode) - , ReadReplicasCount_(readReplicasCount) -{} - -TReadReplicasSettings::EMode TReadReplicasSettings::GetMode() const { - return Mode_; -} - -ui64 TReadReplicasSettings::GetReadReplicasCount() const { - return ReadReplicasCount_; -} - -//////////////////////////////////////////////////////////////////////////////// - +TReadReplicasSettings::TReadReplicasSettings(EMode mode, ui64 readReplicasCount) + : Mode_(mode) + , ReadReplicasCount_(readReplicasCount) +{} + +TReadReplicasSettings::EMode TReadReplicasSettings::GetMode() const { + return Mode_; +} + +ui64 TReadReplicasSettings::GetReadReplicasCount() const { + return ReadReplicasCount_; +} + +//////////////////////////////////////////////////////////////////////////////// + TBulkUpsertResult::TBulkUpsertResult(TStatus&& status) : TStatus(std::move(status)) {} diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 5b7ee11acc..42d21d2fec 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -12,23 +12,23 @@ #include <util/generic/maybe.h> #include <util/generic/variant.h> -namespace Ydb { -namespace Table { - +namespace Ydb { +namespace Table { + class StorageSettings; class ColumnFamily; class CreateTableRequest; -class DescribeTableResult; -class PartitioningSettings; +class DescribeTableResult; +class PartitioningSettings; class DateTypeColumnModeSettings; class TtlSettings; class TableIndex; class TableIndexDescription; class ValueSinceUnixEpochModeSettings; - -} -} - + +} +} + namespace NYdb { namespace NScheme { @@ -342,50 +342,50 @@ private: std::shared_ptr<TImpl> Impl_; }; -//! Represents table partitioning settings -class TPartitioningSettings { -public: - TPartitioningSettings(); - explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto); - - const Ydb::Table::PartitioningSettings& GetProto() const; - - TMaybe<bool> GetPartitioningBySize() const; +//! Represents table partitioning settings +class TPartitioningSettings { +public: + TPartitioningSettings(); + explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto); + + const Ydb::Table::PartitioningSettings& GetProto() const; + + TMaybe<bool> GetPartitioningBySize() const; TMaybe<bool> GetPartitioningByLoad() const; - ui64 GetPartitionSizeMb() const; - ui64 GetMinPartitionsCount() const; - ui64 GetMaxPartitionsCount() const; - -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -//! Represents table read replicas settings -class TReadReplicasSettings { -public: - enum class EMode { - PerAz = 0, - AnyAz = 1 - }; - - TReadReplicasSettings(EMode mode, ui64 readReplicasCount); - - EMode GetMode() const; - ui64 GetReadReplicasCount() const; - -private: - EMode Mode_; - ui64 ReadReplicasCount_; -}; - -struct TExplicitPartitions; + ui64 GetPartitionSizeMb() const; + ui64 GetMinPartitionsCount() const; + ui64 GetMaxPartitionsCount() const; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +//! Represents table read replicas settings +class TReadReplicasSettings { +public: + enum class EMode { + PerAz = 0, + AnyAz = 1 + }; + + TReadReplicasSettings(EMode mode, ui64 readReplicasCount); + + EMode GetMode() const; + ui64 GetReadReplicasCount() const; + +private: + EMode Mode_; + ui64 ReadReplicasCount_; +}; + +struct TExplicitPartitions; struct TDescribeTableSettings; - + //! Represents table description class TTableDescription { friend class TTableBuilder; - friend class NYdb::TProtoAccessor; + friend class NYdb::TProtoAccessor; using EUnit = TValueSinceUnixEpochModeSettings::EUnit; @@ -400,7 +400,7 @@ public: TMaybe<TTtlSettings> GetTtlSettings() const; const TString& GetOwner() const; - const TVector<NScheme::TPermissions>& GetPermissions() const; + const TVector<NScheme::TPermissions>& GetPermissions() const; const TVector<NScheme::TPermissions>& GetEffectivePermissions() const; const TVector<TKeyRange>& GetKeyRanges() const; @@ -431,15 +431,15 @@ public: // Attributes const THashMap<TString, TString>& GetAttributes() const; - // Returns partitioning settings of the table - const TPartitioningSettings& GetPartitioningSettings() const; - - // Bloom filter by key - TMaybe<bool> GetKeyBloomFilter() const; - - // Returns read replicas settings of the table - TMaybe<TReadReplicasSettings> GetReadReplicasSettings() const; - + // Returns partitioning settings of the table + const TPartitioningSettings& GetPartitioningSettings() const; + + // Bloom filter by key + TMaybe<bool> GetKeyBloomFilter() const; + + // Returns read replicas settings of the table + TMaybe<TReadReplicasSettings> GetReadReplicasSettings() const; + // Fills CreateTableRequest proto from this description void SerializeTo(Ydb::Table::CreateTableRequest& request) const; @@ -471,13 +471,13 @@ private: void AddAttribute(const TString& key, const TString& value); void SetAttributes(const THashMap<TString, TString>& attrs); void SetAttributes(THashMap<TString, TString>&& attrs); - void SetCompactionPolicy(const TString& name); - void SetUniformPartitions(ui64 partitionsCount); - void SetPartitionAtKeys(const TExplicitPartitions& keys); - void SetPartitioningSettings(const TPartitioningSettings& settings); - void SetKeyBloomFilter(bool enabled); - void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount); - const Ydb::Table::DescribeTableResult& GetProto() const; + void SetCompactionPolicy(const TString& name); + void SetUniformPartitions(ui64 partitionsCount); + void SetPartitionAtKeys(const TExplicitPartitions& keys); + void SetPartitioningSettings(const TPartitioningSettings& settings); + void SetKeyBloomFilter(bool enabled); + void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount); + const Ydb::Table::DescribeTableResult& GetProto() const; class TImpl; std::shared_ptr<TImpl> Impl_; @@ -504,26 +504,26 @@ private: //////////////////////////////////////////////////////////////////////////////// -class TPartitioningSettingsBuilder { -public: - TPartitioningSettingsBuilder(); - ~TPartitioningSettingsBuilder(); - - TPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled); +class TPartitioningSettingsBuilder { +public: + TPartitioningSettingsBuilder(); + ~TPartitioningSettingsBuilder(); + + TPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled); TPartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled); - TPartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb); - TPartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count); - TPartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count); - - TPartitioningSettings Build() const; - -private: - class TImpl; - std::unique_ptr<TImpl> Impl_; -}; - -//////////////////////////////////////////////////////////////////////////////// - + TPartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb); + TPartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count); + TPartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count); + + TPartitioningSettings Build() const; + +private: + class TImpl; + std::unique_ptr<TImpl> Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + class TColumnFamilyBuilder { public: explicit TColumnFamilyBuilder(const TString& name); @@ -600,44 +600,44 @@ private: TColumnFamilyBuilder Builder_; }; -class TTablePartitioningSettingsBuilder { -public: - explicit TTablePartitioningSettingsBuilder(TTableBuilder& parent) - : Parent_(parent) - { } - - TTablePartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { - Builder_.SetPartitioningBySize(enabled); - return *this; - } - +class TTablePartitioningSettingsBuilder { +public: + explicit TTablePartitioningSettingsBuilder(TTableBuilder& parent) + : Parent_(parent) + { } + + TTablePartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { + Builder_.SetPartitioningBySize(enabled); + return *this; + } + TTablePartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled) { Builder_.SetPartitioningByLoad(enabled); return *this; } - TTablePartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb) { - Builder_.SetPartitionSizeMb(sizeMb); - return *this; - } - - TTablePartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count) { - Builder_.SetMinPartitionsCount(count); - return *this; - } - - TTablePartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count) { - Builder_.SetMaxPartitionsCount(count); - return *this; - } - - TTableBuilder& EndPartitioningSettings(); - -private: - TTableBuilder& Parent_; - TPartitioningSettingsBuilder Builder_; -}; - + TTablePartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb) { + Builder_.SetPartitionSizeMb(sizeMb); + return *this; + } + + TTablePartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count) { + Builder_.SetMinPartitionsCount(count); + return *this; + } + + TTablePartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count) { + Builder_.SetMaxPartitionsCount(count); + return *this; + } + + TTableBuilder& EndPartitioningSettings(); + +private: + TTableBuilder& Parent_; + TPartitioningSettingsBuilder Builder_; +}; + //////////////////////////////////////////////////////////////////////////////// class TTableBuilder { @@ -686,18 +686,18 @@ public: TTableBuilder& SetAttributes(const THashMap<TString, TString>& attrs); TTableBuilder& SetAttributes(THashMap<TString, TString>&& attrs); - TTableBuilder& SetCompactionPolicy(const TString& name); - - // UniformPartitions and PartitionAtKeys are mutually exclusive - TTableBuilder& SetUniformPartitions(ui64 partitionsCount); - TTableBuilder& SetPartitionAtKeys(const TExplicitPartitions& keys); - - TTableBuilder& SetPartitioningSettings(const TPartitioningSettings& settings); - - TTableBuilder& SetKeyBloomFilter(bool enabled); - - TTableBuilder& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount); - + TTableBuilder& SetCompactionPolicy(const TString& name); + + // UniformPartitions and PartitionAtKeys are mutually exclusive + TTableBuilder& SetUniformPartitions(ui64 partitionsCount); + TTableBuilder& SetPartitionAtKeys(const TExplicitPartitions& keys); + + TTableBuilder& SetPartitioningSettings(const TPartitioningSettings& settings); + + TTableBuilder& SetKeyBloomFilter(bool enabled); + + TTableBuilder& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount); + TTableStorageSettingsBuilder BeginStorageSettings() { return TTableStorageSettingsBuilder(*this); } @@ -706,10 +706,10 @@ public: return TTableColumnFamilyBuilder(*this, name); } - TTablePartitioningSettingsBuilder BeginPartitioningSettings() { - return TTablePartitioningSettingsBuilder(*this); - } - + TTablePartitioningSettingsBuilder BeginPartitioningSettings() { + return TTablePartitioningSettingsBuilder(*this); + } + TTableDescription Build(); private: @@ -724,10 +724,10 @@ inline TTableBuilder& TTableColumnFamilyBuilder::EndColumnFamily() { return Parent_.AddColumnFamily(Builder_.Build()); } -inline TTableBuilder& TTablePartitioningSettingsBuilder::EndPartitioningSettings() { - return Parent_.SetPartitioningSettings(Builder_.Build()); -} - +inline TTableBuilder& TTablePartitioningSettingsBuilder::EndPartitioningSettings() { + return Parent_.SetPartitioningSettings(Builder_.Build()); +} + //////////////////////////////////////////////////////////////////////////////// class TCopyItem { @@ -1150,18 +1150,18 @@ struct TPartitioningPolicy { FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions); }; -struct TReplicationPolicy { - using TSelf = TReplicationPolicy; - - FLUENT_SETTING_OPTIONAL(TString, PresetName); - - FLUENT_SETTING_OPTIONAL(ui32, ReplicasCount); - - FLUENT_SETTING_OPTIONAL(bool, CreatePerAvailabilityZone); - - FLUENT_SETTING_OPTIONAL(bool, AllowPromotion); -}; - +struct TReplicationPolicy { + using TSelf = TReplicationPolicy; + + FLUENT_SETTING_OPTIONAL(TString, PresetName); + + FLUENT_SETTING_OPTIONAL(ui32, ReplicasCount); + + FLUENT_SETTING_OPTIONAL(bool, CreatePerAvailabilityZone); + + FLUENT_SETTING_OPTIONAL(bool, AllowPromotion); +}; + //////////////////////////////////////////////////////////////////////////////// struct TCreateTableSettings : public TOperationRequestSettings<TCreateTableSettings> { @@ -1176,8 +1176,8 @@ struct TCreateTableSettings : public TOperationRequestSettings<TCreateTableSetti FLUENT_SETTING_OPTIONAL(TPartitioningPolicy, PartitioningPolicy); FLUENT_SETTING_OPTIONAL(TStoragePolicy, StoragePolicy); - - FLUENT_SETTING_OPTIONAL(TReplicationPolicy, ReplicationPolicy); + + FLUENT_SETTING_OPTIONAL(TReplicationPolicy, ReplicationPolicy); }; //////////////////////////////////////////////////////////////////////////////// @@ -1293,44 +1293,44 @@ private: THashMap<TString, TString> AlterAttributes_; }; -class TAlterPartitioningSettingsBuilder { -public: - explicit TAlterPartitioningSettingsBuilder(TAlterTableSettings& parent) - : Parent_(parent) - { } - - TAlterPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { - Builder_.SetPartitioningBySize(enabled); - return *this; - } - +class TAlterPartitioningSettingsBuilder { +public: + explicit TAlterPartitioningSettingsBuilder(TAlterTableSettings& parent) + : Parent_(parent) + { } + + TAlterPartitioningSettingsBuilder& SetPartitioningBySize(bool enabled) { + Builder_.SetPartitioningBySize(enabled); + return *this; + } + TAlterPartitioningSettingsBuilder& SetPartitioningByLoad(bool enabled) { Builder_.SetPartitioningByLoad(enabled); return *this; } - TAlterPartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb) { - Builder_.SetPartitionSizeMb(sizeMb); - return *this; - } - - TAlterPartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count) { - Builder_.SetMinPartitionsCount(count); - return *this; - } - - TAlterPartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count) { - Builder_.SetMaxPartitionsCount(count); - return *this; - } - - TAlterTableSettings& EndAlterPartitioningSettings(); - -private: - TAlterTableSettings& Parent_; - TPartitioningSettingsBuilder Builder_; -}; - + TAlterPartitioningSettingsBuilder& SetPartitionSizeMb(ui64 sizeMb) { + Builder_.SetPartitionSizeMb(sizeMb); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetMinPartitionsCount(ui64 count) { + Builder_.SetMinPartitionsCount(count); + return *this; + } + + TAlterPartitioningSettingsBuilder& SetMaxPartitionsCount(ui64 count) { + Builder_.SetMaxPartitionsCount(count); + return *this; + } + + TAlterTableSettings& EndAlterPartitioningSettings(); + +private: + TAlterTableSettings& Parent_; + TPartitioningSettingsBuilder Builder_; +}; + struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSettings> { using TSelf = TAlterTableSettings; using TAlterAttributes = THashMap<TString, TString>; @@ -1363,18 +1363,18 @@ struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSetting FLUENT_SETTING(TAlterAttributes, AlterAttributes); - FLUENT_SETTING(TString, SetCompactionPolicy); - - FLUENT_SETTING_OPTIONAL(TPartitioningSettings, AlterPartitioningSettings); - - FLUENT_SETTING_OPTIONAL(bool, SetKeyBloomFilter); - - FLUENT_SETTING_OPTIONAL(TReadReplicasSettings, SetReadReplicasSettings); - TSelf& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { - SetReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); - return *this; - } - + FLUENT_SETTING(TString, SetCompactionPolicy); + + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, AlterPartitioningSettings); + + FLUENT_SETTING_OPTIONAL(bool, SetKeyBloomFilter); + + FLUENT_SETTING_OPTIONAL(TReadReplicasSettings, SetReadReplicasSettings); + TSelf& SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { + SetReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); + return *this; + } + TAlterStorageSettingsBuilder BeginAlterStorageSettings() { return TAlterStorageSettingsBuilder(*this); } @@ -1394,10 +1394,10 @@ struct TAlterTableSettings : public TOperationRequestSettings<TAlterTableSetting TAlterAttributesBuilder BeginAlterAttributes() { return TAlterAttributesBuilder(*this); } - - TAlterPartitioningSettingsBuilder BeginAlterPartitioningSettings() { - return TAlterPartitioningSettingsBuilder(*this); - } + + TAlterPartitioningSettingsBuilder BeginAlterPartitioningSettings() { + return TAlterPartitioningSettingsBuilder(*this); + } private: class TImpl; @@ -1420,10 +1420,10 @@ inline TAlterTableSettings& TAlterAttributesBuilder::EndAlterAttributes() { return Parent_.AlterAttributes(AlterAttributes_); } -inline TAlterTableSettings& TAlterPartitioningSettingsBuilder::EndAlterPartitioningSettings() { - return Parent_.AlterPartitioningSettings(Builder_.Build()); -} - +inline TAlterTableSettings& TAlterPartitioningSettingsBuilder::EndAlterPartitioningSettings() { + return Parent_.AlterPartitioningSettings(Builder_.Build()); +} + //////////////////////////////////////////////////////////////////////////////// struct TCopyTableSettings : public TOperationRequestSettings<TCopyTableSettings> {}; diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index 1ea82d4c59..d3f2ca05af 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -72,14 +72,14 @@ float TStatus::GetConsumedRu() const { return Impl_->Status.ConstInfo.consumed_units(); } -//////////////////////////////////////////////////////////////////////////////// - -TStreamPartStatus::TStreamPartStatus(TStatus&& status) - : TStatus(std::move(status)) -{} - -bool TStreamPartStatus::EOS() const { - return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE; -} - +//////////////////////////////////////////////////////////////////////////////// + +TStreamPartStatus::TStreamPartStatus(TStatus&& status) + : TStatus(std::move(status)) +{} + +bool TStreamPartStatus::EOS() const { + return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE; +} + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index 0fd392fb40..7185a03964 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -37,10 +37,10 @@ private: using TAsyncStatus = NThreading::TFuture<TStatus>; -class TStreamPartStatus : public TStatus { -public: - TStreamPartStatus(TStatus&& status); - bool EOS() const; -}; - +class TStreamPartStatus : public TStatus { +public: + TStreamPartStatus(TStatus&& status); + bool EOS() const; +}; + } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp index 488b4bc574..d7508a5f9b 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp @@ -315,8 +315,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[10u;[1000u]];[20u;[2000u]]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[10,{"Member1":"1972-09-27"}],[20,{"Member1":"1975-06-24"}]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[10,{"Member1":"1972-09-27"}],[20,{"Member1":"1975-06-24"}]])"); } Y_UNIT_TEST(ParseValueMaybe) { @@ -539,8 +539,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([17;19;21])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([17,19,21])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([17,19,21])"); } Y_UNIT_TEST(BuildValueListEmpty) { @@ -552,8 +552,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(List<Uint32>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([])"); } Y_UNIT_TEST(BuildValueListEmpty2) { @@ -571,8 +571,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[10u];[]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[10],[]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[10],[]])"); } Y_UNIT_TEST(BuildValueListEmpty3) { @@ -590,8 +590,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[];[10u]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[],[10]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[],[10]])"); } Y_UNIT_TEST(BuildValueBadCall) { @@ -627,8 +627,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(List<Int32?>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[1];#;[57];#])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([1,null,57,null])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([1,null,57,null])"); auto expectedProtoValueStr = "items {\n" @@ -683,8 +683,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(Tuple<Int32??,Int64???,String??,Utf8???>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[[10]];[[[-1]]];[#];[[#]]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([10,-1,null,null])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([10,-1,null,null])"); auto expectedProtoValueStr = "items {\n" @@ -782,11 +782,11 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[1u;"Anna";-100;#];[2u;"Paul";-200;["Some details"]]])"); - UNIT_ASSERT_NO_DIFF( - FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([{"Id":1,"Name":"Anna","Value":-100,"Description":null},)" - R"({"Id":2,"Name":"Paul","Value":-200,"Description":"Some details"}])" - ); + UNIT_ASSERT_NO_DIFF( + FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([{"Id":1,"Name":"Anna","Value":-100,"Description":null},)" + R"({"Id":2,"Name":"Paul","Value":-200,"Description":"Some details"}])" + ); } Y_UNIT_TEST(BuildValueStructMissingMember) { @@ -850,11 +850,11 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(Tuple<Bool,Int8,Uint8,Int16,Uint16,Int32,Uint32,Int64,Uint64,Float,Double,Date,Datetime,Timestamp,Interval,TzDate,TzDatetime,TzTimestamp,String,Utf8,Yson,Json>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([%true;-1;1u;-2;2u;-3;3u;-4;4u;-5.5;6.6;7u;8u;9u;-10;"2018-02-02,Europe/Moscow";"2018-02-03T15:00:00,Europe/Moscow";"2018-02-07T15:00:00,Europe/Moscow";"TestString";"TestUtf8";"[]";"{}"])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([true,-1,1,-2,2,-3,3,-4,4,-5.5,6.6,"1970-01-08","1970-01-01T00:00:08Z",)" - R"("1970-01-01T00:00:00.000009Z",-10,"2018-02-02,Europe/Moscow",)" - R"("2018-02-03T15:00:00,Europe/Moscow","2018-02-07T15:00:00,Europe/Moscow",)" - R"("TestString","TestUtf8","[]","{}"])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([true,-1,1,-2,2,-3,3,-4,4,-5.5,6.6,"1970-01-08","1970-01-01T00:00:08Z",)" + R"("1970-01-01T00:00:00.000009Z",-10,"2018-02-02,Europe/Moscow",)" + R"("2018-02-03T15:00:00,Europe/Moscow","2018-02-07T15:00:00,Europe/Moscow",)" + R"("TestString","TestUtf8","[]","{}"])"); } Y_UNIT_TEST(BuildValueTuple1) { @@ -878,8 +878,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[10;"Str1"];[20;"Str2"]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[10,"Str1"],[20,"Str2"]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[10,"Str1"],[20,"Str2"]])"); } Y_UNIT_TEST(BuildValueTuple2) { @@ -902,8 +902,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[-10;"Utf1"];[-20;"Utf2"]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[-10,"Utf1"],[-20,"Utf2"]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[-10,"Utf1"],[-20,"Utf2"]])"); } Y_UNIT_TEST(BuildValueTupleElementsMismatch1) { @@ -1003,8 +1003,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[1;"Str1"];[2;"Str2"]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[1,"Str1"],[2,"Str2"]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[1,"Str1"],[2,"Str2"]])"); } Y_UNIT_TEST(BuildValueDict2) { @@ -1023,8 +1023,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[1;"Str1"];[2;"Str2"]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[1,"Str1"],[2,"Str2"]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[1,"Str1"],[2,"Str2"]])"); } Y_UNIT_TEST(BuildValueDictTypeMismatch1) { @@ -1080,8 +1080,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(Dict<Uint32,String>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([])"); } Y_UNIT_TEST(BuildValueDictEmpty2) { @@ -1108,8 +1108,8 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(List<Dict<Int32,String>>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([[[1;"Str1"]];[];[]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), - R"([[[1,"Str1"]],[],[]])"); + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + R"([[[1,"Str1"]],[],[]])"); } Y_UNIT_TEST(BuildValueDictEmptyNoType) { @@ -1217,7 +1217,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { R"(Tuple<Struct<'Name':String,'Value':Uint64>,Utf8?,List<Bool>,Dict<Int32,Uint8?>,DyNumber?>)"); UNIT_ASSERT_NO_DIFF(FormatValueYson(value), R"([["Sergey";1u];#;[%true];[[10;#]];["12.345"]])"); - UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), + UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode), R"([{"Name":"Sergey","Value":1},null,[true],[[10,null]],"12.345"])"); } } diff --git a/ydb/public/sdk/python/ydb/_tx_ctx_impl.py b/ydb/public/sdk/python/ydb/_tx_ctx_impl.py index cd038f7e3a..765cd82e72 100644 --- a/ydb/public/sdk/python/ydb/_tx_ctx_impl.py +++ b/ydb/public/sdk/python/ydb/_tx_ctx_impl.py @@ -132,11 +132,11 @@ def execute_request_factory( parameters_types = data_query.parameters_types elif isinstance(query, types.DataQuery): if settings is not None and hasattr(settings, "keep_in_cache"): - keep_in_cache = settings.keep_in_cache - else: - # that is an instance of a data query and we don't know query id for id. - # so let's prepare it to keep in cache - keep_in_cache = True + keep_in_cache = settings.keep_in_cache + else: + # that is an instance of a data query and we don't know query id for id. + # so let's prepare it to keep in cache + keep_in_cache = True yql_text = query.yql_text parameters_types = query.parameters_types else: diff --git a/ydb/public/sdk/python/ydb/connection.py b/ydb/public/sdk/python/ydb/connection.py index 0307051c66..573b276e20 100644 --- a/ydb/public/sdk/python/ydb/connection.py +++ b/ydb/public/sdk/python/ydb/connection.py @@ -186,7 +186,7 @@ def _construct_channel_options(driver_config, endpoint_options=None): getattr(driver_config, "grpc_lb_policy_name", "round_robin"), ), ] - if driver_config.grpc_keep_alive_timeout is not None: + if driver_config.grpc_keep_alive_timeout is not None: _default_connect_options.extend( [ ("grpc.keepalive_time_ms", driver_config.grpc_keep_alive_timeout >> 3), diff --git a/ydb/public/sdk/python/ydb/driver.py b/ydb/public/sdk/python/ydb/driver.py index 2c7c96f17f..e14f90176e 100644 --- a/ydb/public/sdk/python/ydb/driver.py +++ b/ydb/public/sdk/python/ydb/driver.py @@ -132,7 +132,7 @@ class DriverConfig(object): private key should be used. :param certificate_chain: The PEM-encoded certificate chain as a byte string\ to use or or None if no certificate chain should be used. - :param grpc_keep_alive_timeout: GRpc KeepAlive timeout, ms + :param grpc_keep_alive_timeout: GRpc KeepAlive timeout, ms :param ydb.Tracer tracer: ydb.Tracer instance to trace requests in driver.\ If tracing aio ScopeManager must be ContextVarsScopeManager :param grpc_lb_policy_name: A load balancing policy to be used for discovery channel construction. Default value is `round_round` @@ -155,7 +155,7 @@ class DriverConfig(object): self.root_certificates = root_certificates self.certificate_chain = certificate_chain self.private_key = private_key - self.grpc_keep_alive_timeout = grpc_keep_alive_timeout + self.grpc_keep_alive_timeout = grpc_keep_alive_timeout self.table_client_settings = table_client_settings self.primary_user_agent = primary_user_agent self.tracer = tracer if tracer is not None else ydb.Tracer(None) @@ -191,11 +191,11 @@ class DriverConfig(object): **kwargs ) - def set_grpc_keep_alive_timeout(self, timeout): - self.grpc_keep_alive_timeout = timeout - return self + def set_grpc_keep_alive_timeout(self, timeout): + self.grpc_keep_alive_timeout = timeout + return self + - ConnectionParams = DriverConfig diff --git a/ydb/public/sdk/python/ydb/scripting.py b/ydb/public/sdk/python/ydb/scripting.py index de5a17c175..587630f324 100644 --- a/ydb/public/sdk/python/ydb/scripting.py +++ b/ydb/public/sdk/python/ydb/scripting.py @@ -24,10 +24,10 @@ class ScriptingClientSettings(object): class ExplainYqlScriptSettings(settings.BaseRequestSettings): - MODE_UNSPECIFIED = 0 - MODE_PARSE = 1 - MODE_VALIDATE = 2 - MODE_EXPLAIN = 3 + MODE_UNSPECIFIED = 0 + MODE_PARSE = 1 + MODE_VALIDATE = 2 + MODE_EXPLAIN = 3 def __init__(self): super(ExplainYqlScriptSettings, self).__init__() diff --git a/ydb/public/sdk/python/ydb/table.py b/ydb/public/sdk/python/ydb/table.py index 3635579e9f..c49c4660c7 100644 --- a/ydb/public/sdk/python/ydb/table.py +++ b/ydb/public/sdk/python/ydb/table.py @@ -52,15 +52,15 @@ class DescribeTableSettings(settings_impl.BaseRequestSettings): class ExecDataQuerySettings(settings_impl.BaseRequestSettings): - def __init__(self): - super(ExecDataQuerySettings, self).__init__() - self.keep_in_cache = True - - def with_keep_in_cache(self, value): - self.keep_in_cache = value - return self - - + def __init__(self): + super(ExecDataQuerySettings, self).__init__() + self.keep_in_cache = True + + def with_keep_in_cache(self, value): + self.keep_in_cache = value + return self + + class KeyBound(object): __slots__ = ("_equal", "value", "type") @@ -2224,7 +2224,7 @@ class BaseTxContext(ITxContext): :return: A result sets or exception in case of execution errors """ return self._driver( - _tx_ctx_impl.execute_request_factory( + _tx_ctx_impl.execute_request_factory( self._session_state, self._tx_state, query, diff --git a/ydb/services/discovery/grpc_service.cpp b/ydb/services/discovery/grpc_service.cpp index c444ae79a5..010322b5d9 100644 --- a/ydb/services/discovery/grpc_service.cpp +++ b/ydb/services/discovery/grpc_service.cpp @@ -58,9 +58,9 @@ TGRpcDiscoveryService::TGRpcDiscoveryService(NActors::TActorSystem *system, ADD_REQUEST(ListEndpoints, ListEndpointsRequest, ListEndpointsResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TEvListEndpointsRequest(reqCtx)); }) - ADD_REQUEST(WhoAmI, WhoAmIRequest, WhoAmIResponse, { - ActorSystem_->Send(GRpcRequestProxyId_, new TEvWhoAmIRequest(reqCtx)); - }) + ADD_REQUEST(WhoAmI, WhoAmIRequest, WhoAmIResponse, { + ActorSystem_->Send(GRpcRequestProxyId_, new TEvWhoAmIRequest(reqCtx)); + }) #undef ADD_REQUEST } diff --git a/ydb/services/ydb/ydb_scripting.cpp b/ydb/services/ydb/ydb_scripting.cpp index becc39a405..3596b17fd9 100644 --- a/ydb/services/ydb/ydb_scripting.cpp +++ b/ydb/services/ydb/ydb_scripting.cpp @@ -58,12 +58,12 @@ void TGRpcYdbScriptingService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { (ctx, &DoExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Ru, nullptr})); }) - ADD_REQUEST(StreamExecuteYql, ExecuteYqlRequest, ExecuteYqlPartialResponse, { + ADD_REQUEST(StreamExecuteYql, ExecuteYqlRequest, ExecuteYqlPartialResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TGrpcRequestNoOperationCall<ExecuteYqlRequest, ExecuteYqlPartialResponse> (ctx, &DoStreamExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); - }) - + }) + ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TGrpcRequestOperationCall<ExplainYqlRequest, ExplainYqlResponse> diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index 83b92966e8..4861c06b92 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -25,7 +25,7 @@ using namespace NYdb::NTable; TSession CreateSession(TDriver driver, const TString& token = "", const TString& discoveryEndpoint = "") { NYdb::NTable::TClientSettings settings; if (token) - settings.AuthToken(token); + settings.AuthToken(token); if (discoveryEndpoint) settings.DiscoveryEndpoint(discoveryEndpoint); NYdb::NTable::TTableClient client(driver, settings); @@ -60,14 +60,14 @@ static void MultiTenantSDK(bool asyncDiscovery) { NYdb::NTable::TClientSettings settings; - settings.AuthToken("root@builtin"); + settings.AuthToken("root@builtin"); NYdb::NTable::TTableClient clientgood(driver, settings); NYdb::NTable::TTableClient clientbad(driver); //TODO: No discovery in ut /* NYdb::NTable::TClientSettings settings2; - settings2.AuthToken("root@builtin"); + settings2.AuthToken("root@builtin"); settings2.Database_ = "/balabla"; NYdb::NTable::TTableClient clientbad2(driver, settings2); */ @@ -3960,280 +3960,280 @@ R"___(<main>: Error: Transaction not found: , code: 2015 EnsureTablePartitions(client, "/Root/Foo", 3); } - - Y_UNIT_TEST(CreateAndAltertTableWithCompactionPolicy) { - TKikimrWithGrpcAndRootSchema server; - server.Server_->SetupDefaultProfiles(); - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetCompactionPolicy("compaction2"); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TClient client(*server.ServerSettings); - auto describeResult = client.Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 2); - } - { - auto settings = NYdb::NTable::TAlterTableSettings() - .SetCompactionPolicy("default"); - - auto result = session.AlterTable(tableName, settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TClient client(*server.ServerSettings); - auto describeResult = client.Ls(tableName); - UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() - .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 3); - } - } - - Y_UNIT_TEST(CreateTableWithUniformPartitions) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetUniformPartitions(4); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - EnsureTablePartitions(client, tableName, 4); - } - - Y_UNIT_TEST(CreateTableWithUniformPartitionsAndAutoPartitioning) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetUniformPartitions(4) - .BeginPartitioningSettings() - .SetPartitioningBySize(true) - .EndPartitioningSettings(); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); - } - } - - Y_UNIT_TEST(CreateTableWithPartitionAtKeys) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - TExplicitPartitions partitions; - partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build()); - partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build()); - - auto builder = TTableBuilder() - .AddNullableColumn("Value", EPrimitiveType::Utf8) - .AddNullableColumn("SubKey", EPrimitiveType::Utf8) - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .SetPrimaryKeyColumn("Key") - .SetPartitionAtKeys(partitions); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - EnsureTablePartitions(client, tableName, 3); - } - - Y_UNIT_TEST(CreateTableWithPartitionAtKeysAndAutoPartitioning) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - TExplicitPartitions partitions; - partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build()); - partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build()); - - auto builder = TTableBuilder() - .AddNullableColumn("Value", EPrimitiveType::Utf8) - .AddNullableColumn("SubKey", EPrimitiveType::Utf8) - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .SetPrimaryKeyColumn("Key") - .SetPartitionAtKeys(partitions) - .BeginPartitioningSettings() - .SetPartitioningBySize(true) - .EndPartitioningSettings(); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 3); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); - } - } - - Y_UNIT_TEST(CreateAndAltertTableWithPartitioningBySize) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .BeginPartitioningSettings() - .SetPartitioningBySize(true) - .SetPartitionSizeMb(100) - .SetMinPartitionsCount(2) - .SetMaxPartitionsCount(50) - .EndPartitioningSettings(); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 100); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 50); - } - { - auto settings = NYdb::NTable::TAlterTableSettings() - .BeginAlterPartitioningSettings() - .SetPartitionSizeMb(50) - .SetMinPartitionsCount(4) - .SetMaxPartitionsCount(100) - .EndAlterPartitioningSettings(); - - auto result = session.AlterTable(tableName, settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 50); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 100); - } - { - auto settings = NYdb::NTable::TAlterTableSettings() - .BeginAlterPartitioningSettings() - .SetPartitioningBySize(false) - .EndAlterPartitioningSettings(); - - auto result = session.AlterTable(tableName, settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - } + + Y_UNIT_TEST(CreateAndAltertTableWithCompactionPolicy) { + TKikimrWithGrpcAndRootSchema server; + server.Server_->SetupDefaultProfiles(); + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetCompactionPolicy("compaction2"); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TClient client(*server.ServerSettings); + auto describeResult = client.Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 2); + } + { + auto settings = NYdb::NTable::TAlterTableSettings() + .SetCompactionPolicy("default"); + + auto result = session.AlterTable(tableName, settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TClient client(*server.ServerSettings); + auto describeResult = client.Ls(tableName); + UNIT_ASSERT_VALUES_EQUAL(describeResult->Record.GetPathDescription().GetTable() + .GetPartitionConfig().GetCompactionPolicy().GetGeneration().size(), 3); + } + } + + Y_UNIT_TEST(CreateTableWithUniformPartitions) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetUniformPartitions(4); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + EnsureTablePartitions(client, tableName, 4); + } + + Y_UNIT_TEST(CreateTableWithUniformPartitionsAndAutoPartitioning) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetUniformPartitions(4) + .BeginPartitioningSettings() + .SetPartitioningBySize(true) + .EndPartitioningSettings(); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); + } + } + + Y_UNIT_TEST(CreateTableWithPartitionAtKeys) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + TExplicitPartitions partitions; + partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build()); + partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build()); + + auto builder = TTableBuilder() + .AddNullableColumn("Value", EPrimitiveType::Utf8) + .AddNullableColumn("SubKey", EPrimitiveType::Utf8) + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .SetPrimaryKeyColumn("Key") + .SetPartitionAtKeys(partitions); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + EnsureTablePartitions(client, tableName, 3); + } + + Y_UNIT_TEST(CreateTableWithPartitionAtKeysAndAutoPartitioning) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + TExplicitPartitions partitions; + partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build()); + partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build()); + + auto builder = TTableBuilder() + .AddNullableColumn("Value", EPrimitiveType::Utf8) + .AddNullableColumn("SubKey", EPrimitiveType::Utf8) + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .SetPrimaryKeyColumn("Key") + .SetPartitionAtKeys(partitions) + .BeginPartitioningSettings() + .SetPartitioningBySize(true) + .EndPartitioningSettings(); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 3); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); + } + } + + Y_UNIT_TEST(CreateAndAltertTableWithPartitioningBySize) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .BeginPartitioningSettings() + .SetPartitioningBySize(true) + .SetPartitionSizeMb(100) + .SetMinPartitionsCount(2) + .SetMaxPartitionsCount(50) + .EndPartitioningSettings(); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 100); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 50); + } + { + auto settings = NYdb::NTable::TAlterTableSettings() + .BeginAlterPartitioningSettings() + .SetPartitionSizeMb(50) + .SetMinPartitionsCount(4) + .SetMaxPartitionsCount(100) + .EndAlterPartitioningSettings(); + + auto result = session.AlterTable(tableName, settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 50); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMaxPartitionsCount(), 100); + } + { + auto settings = NYdb::NTable::TAlterTableSettings() + .BeginAlterPartitioningSettings() + .SetPartitioningBySize(false) + .EndAlterPartitioningSettings(); + + auto result = session.AlterTable(tableName, settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + } { auto settings = NYdb::NTable::TAlterTableSettings() .BeginAlterPartitioningSettings() @@ -4251,8 +4251,8 @@ R"___(<main>: Error: Transaction not found: , code: 2015 UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); } - } - + } + Y_UNIT_TEST(CreateAndAltertTableWithPartitioningByLoad) { TKikimrWithGrpcAndRootSchema server; @@ -4288,7 +4288,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015 UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); } { auto settings = NYdb::NTable::TAlterTableSettings() @@ -4307,9 +4307,9 @@ R"___(<main>: Error: Transaction not found: , code: 2015 UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); } { auto settings = NYdb::NTable::TAlterTableSettings() @@ -4330,242 +4330,242 @@ R"___(<main>: Error: Transaction not found: , code: 2015 } } - Y_UNIT_TEST(CheckDefaultTableSettings1) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key"); - - auto desc = builder.Build(); - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - } - } - - Y_UNIT_TEST(CheckDefaultTableSettings2) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .BeginPartitioningSettings() - .SetPartitioningBySize(true) - .EndPartitioningSettings(); - - auto desc = builder.Build(); - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); - } - } - - Y_UNIT_TEST(CheckDefaultTableSettings3) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .BeginPartitioningSettings() - .SetPartitioningByLoad(true) - .EndPartitioningSettings(); - - auto desc = builder.Build(); - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); - UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); - UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); - UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); - } - } - - Y_UNIT_TEST(CreateAndAltertTableWithKeyBloomFilter) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetKeyBloomFilter(true); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), true); - } - { - auto settings = NYdb::NTable::TAlterTableSettings() - .SetKeyBloomFilter(false); - - auto result = session.AlterTable(tableName, settings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); - UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), false); - } - } - - Y_UNIT_TEST(CreateAndAltertTableWithReadReplicasSettings) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - const TString tableName = "Root/Test"; - - { - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetReadReplicasSettings(TReadReplicasSettings::EMode::AnyAz, 2); - - auto desc = builder.Build(); - - auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() - == TReadReplicasSettings::EMode::AnyAz); - UNIT_ASSERT_VALUES_EQUAL( - describeResult.GetTableDescription().GetReadReplicasSettings()->GetReadReplicasCount(), 2); - } - { - auto settings = NYdb::NTable::TAlterTableSettings() - .SetReadReplicasSettings(TReadReplicasSettings::EMode::PerAz, 1); - - auto result = session.AlterTable(tableName, settings).ExtractValueSync(); - - UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - { - TDescribeTableResult describeResult = session.DescribeTable(tableName) - .GetValueSync(); - UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); - UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() - == TReadReplicasSettings::EMode::PerAz); - UNIT_ASSERT_VALUES_EQUAL( - describeResult.GetTableDescription().GetReadReplicasSettings()->GetReadReplicasCount(), 1); - } - } - - Y_UNIT_TEST(CreateTableWithMESettings) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - - NYdb::NTable::TTableClient client(driver); - auto getSessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); - auto session = getSessionResult.GetSession(); - auto builder = TTableBuilder() - .AddNullableColumn("key", EPrimitiveType::Uint64) - .AddNullableColumn("value", EPrimitiveType::Utf8) - .SetPrimaryKeyColumn("key") - .SetReadReplicasSettings(TReadReplicasSettings::EMode::AnyAz, 1); - - auto desc = builder.Build(); - - TCreateTableSettings createTableSettings = - TCreateTableSettings() - .ReplicationPolicy(TReplicationPolicy().ReplicasCount(1).CreatePerAvailabilityZone(false)); - auto result = session.CreateTable("Root/Test", std::move(desc), createTableSettings).GetValueSync(); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_STRING_CONTAINS_C( - result.GetIssues().ToString(), - "Warning: Table profile and ReadReplicasSettings are set. They are mutually exclusive. Use either one of them.", - "Unexpected error message"); - } + Y_UNIT_TEST(CheckDefaultTableSettings1) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key"); + + auto desc = builder.Build(); + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + } + } + + Y_UNIT_TEST(CheckDefaultTableSettings2) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .BeginPartitioningSettings() + .SetPartitioningBySize(true) + .EndPartitioningSettings(); + + auto desc = builder.Build(); + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), false); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), true); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitionSizeMb(), 2048); + } + } + + Y_UNIT_TEST(CheckDefaultTableSettings3) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .BeginPartitioningSettings() + .SetPartitioningByLoad(true) + .EndPartitioningSettings(); + + auto desc = builder.Build(); + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + const auto& partSettings = describeResult.GetTableDescription().GetPartitioningSettings(); + UNIT_ASSERT(partSettings.GetPartitioningByLoad().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningByLoad().GetRef(), true); + UNIT_ASSERT(partSettings.GetPartitioningBySize().Defined()); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetPartitioningBySize().GetRef(), false); + UNIT_ASSERT_VALUES_EQUAL(partSettings.GetMinPartitionsCount(), 1); + } + } + + Y_UNIT_TEST(CreateAndAltertTableWithKeyBloomFilter) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetKeyBloomFilter(true); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), true); + } + { + auto settings = NYdb::NTable::TAlterTableSettings() + .SetKeyBloomFilter(false); + + auto result = session.AlterTable(tableName, settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(describeResult.GetTableDescription().GetKeyBloomFilter().Defined()); + UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetKeyBloomFilter().GetRef(), false); + } + } + + Y_UNIT_TEST(CreateAndAltertTableWithReadReplicasSettings) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + const TString tableName = "Root/Test"; + + { + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetReadReplicasSettings(TReadReplicasSettings::EMode::AnyAz, 2); + + auto desc = builder.Build(); + + auto result = session.CreateTable(tableName, std::move(desc)).GetValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() + == TReadReplicasSettings::EMode::AnyAz); + UNIT_ASSERT_VALUES_EQUAL( + describeResult.GetTableDescription().GetReadReplicasSettings()->GetReadReplicasCount(), 2); + } + { + auto settings = NYdb::NTable::TAlterTableSettings() + .SetReadReplicasSettings(TReadReplicasSettings::EMode::PerAz, 1); + + auto result = session.AlterTable(tableName, settings).ExtractValueSync(); + + UNIT_ASSERT_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + { + TDescribeTableResult describeResult = session.DescribeTable(tableName) + .GetValueSync(); + UNIT_ASSERT_EQUAL(describeResult.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings().Defined()); + UNIT_ASSERT(describeResult.GetTableDescription().GetReadReplicasSettings()->GetMode() + == TReadReplicasSettings::EMode::PerAz); + UNIT_ASSERT_VALUES_EQUAL( + describeResult.GetTableDescription().GetReadReplicasSettings()->GetReadReplicasCount(), 1); + } + } + + Y_UNIT_TEST(CreateTableWithMESettings) { + TKikimrWithGrpcAndRootSchema server; + + NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); + + NYdb::NTable::TTableClient client(driver); + auto getSessionResult = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(getSessionResult.GetStatus(), EStatus::SUCCESS, getSessionResult.GetIssues().ToString()); + auto session = getSessionResult.GetSession(); + auto builder = TTableBuilder() + .AddNullableColumn("key", EPrimitiveType::Uint64) + .AddNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .SetReadReplicasSettings(TReadReplicasSettings::EMode::AnyAz, 1); + + auto desc = builder.Build(); + + TCreateTableSettings createTableSettings = + TCreateTableSettings() + .ReplicationPolicy(TReplicationPolicy().ReplicasCount(1).CreatePerAvailabilityZone(false)); + auto result = session.CreateTable("Root/Test", std::move(desc), createTableSettings).GetValueSync(); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_STRING_CONTAINS_C( + result.GetIssues().ToString(), + "Warning: Table profile and ReadReplicasSettings are set. They are mutually exclusive. Use either one of them.", + "Unexpected error message"); + } Y_UNIT_TEST(TableKeyRangesSinglePartition) { TKikimrWithGrpcAndRootSchema server; diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index f2c8392993..2d37063079 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -1265,10 +1265,10 @@ columns { } } primary_key: "Key" -partitioning_settings { - partitioning_by_size: DISABLED - partitioning_by_load: DISABLED -} +partitioning_settings { + partitioning_by_size: DISABLED + partitioning_by_load: DISABLED +} )___"; UNIT_ASSERT_NO_DIFF(tmp, expected); } @@ -1606,10 +1606,10 @@ indexes { } status: STATUS_READY } -partitioning_settings { - partitioning_by_size: DISABLED - partitioning_by_load: DISABLED -} +partitioning_settings { + partitioning_by_size: DISABLED + partitioning_by_load: DISABLED +} )___"; UNIT_ASSERT_NO_DIFF(tmp, expected); } @@ -3661,14 +3661,14 @@ void Apply(const NKikimrConfig::TReplicationPolicy &policy, if (policy.HasFollowerCount()) { auto& followerGroup = *partition.AddFollowerGroups(); followerGroup.SetFollowerCount(policy.GetFollowerCount()); - if (policy.GetCrossDataCenter()) { + if (policy.GetCrossDataCenter()) { followerGroup.SetRequireAllDataCenters(true); - } else { + } else { followerGroup.SetRequireAllDataCenters(false); - } + } if (policy.HasAllowFollowerPromotion()) { followerGroup.SetAllowLeaderPromotion(policy.GetAllowFollowerPromotion()); - } + } } } diff --git a/ydb/tests/functional/scheme_shard/test_copy_ops.py b/ydb/tests/functional/scheme_shard/test_copy_ops.py index b1043cdf68..e55a985fd1 100644 --- a/ydb/tests/functional/scheme_shard/test_copy_ops.py +++ b/ydb/tests/functional/scheme_shard/test_copy_ops.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os -from hamcrest import assert_that, has_length, has_property, equal_to +from hamcrest import assert_that, has_length, has_property, equal_to from ydb.tests.library.common.protobuf_ss import SchemeDescribeRequest from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory @@ -91,14 +91,14 @@ class TestSchemeShardCopyOps(object): assert_that( description.PathDescription.Table.PartitionConfig.FollowerGroups, - has_length( - equal_to( - 1 - ) - ) - ) - - assert_that( + has_length( + equal_to( + 1 + ) + ) + ) + + assert_that( description.PathDescription.Table.PartitionConfig.FollowerGroups[0], has_property( 'FollowerCount', @@ -113,17 +113,17 @@ class TestSchemeShardCopyOps(object): description = self.cluster.client.send( SchemeDescribeRequest(destination_table_full_name).with_partition_config().protobuf, 'SchemeDescribe' ) - + assert_that( description.PathDescription.Table.PartitionConfig.FollowerGroups, - has_length( - equal_to( - 1 - ) - ) - ) - - assert_that( + has_length( + equal_to( + 1 + ) + ) + ) + + assert_that( description.PathDescription.Table.PartitionConfig.FollowerGroups[0], has_property( 'FollowerCount', diff --git a/ydb/tests/functional/ydb_cli/canondata/result.json b/ydb/tests/functional/ydb_cli/canondata/result.json index 689205382d..6388558475 100644 --- a/ydb/tests/functional/ydb_cli/canondata/result.json +++ b/ydb/tests/functional/ydb_cli/canondata/result.json @@ -1,110 +1,110 @@ -{ - "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64": { +{ + "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64_array": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64_array": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_base64_array/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode_array": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode_array": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_json_unicode_array/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_pretty": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_pretty": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_stream_yql_script_pretty/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64_array": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64_array": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_base64_array/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode_array": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode_array": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_json_unicode_array/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_pretty": { + }, + "test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_pretty": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithFormats.test_yql_script_pretty/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithParams.test_list": { + }, + "test_ydb_scripting.TestExecuteScriptWithParams.test_list": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithParams.test_list/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithParams.test_struct": { + }, + "test_ydb_scripting.TestExecuteScriptWithParams.test_struct": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithParams.test_struct/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithParams.test_uint32": { + }, + "test_ydb_scripting.TestExecuteScriptWithParams.test_uint32": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithParams.test_uint32/result.output" - }, - "test_ydb_scripting.TestExecuteScriptWithParams.test_uint64_and_string": { + }, + "test_ydb_scripting.TestExecuteScriptWithParams.test_uint64_and_string": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithParams.test_uint64_and_string/result.output" - }, - "test_ydb_scripting.TestScriptingServiceHelp.test_help": { + }, + "test_ydb_scripting.TestScriptingServiceHelp.test_help": { "uri": "file://test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output" - }, - "test_ydb_scripting.TestScriptingServiceHelp.test_help_ex": { + }, + "test_ydb_scripting.TestScriptingServiceHelp.test_help_ex": { "uri": "file://test_ydb_scripting.TestScriptingServiceHelp.test_help_ex/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_base64_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_data_query_json_unicode_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_pretty": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_data_query_pretty": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_data_query_pretty/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_base64_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_read_table_json_unicode_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_pretty": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_read_table_pretty": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_read_table_pretty/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_base64_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode_array": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode_array": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_json_unicode_array/result.output" - }, - "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_pretty": { + }, + "test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_pretty": { "uri": "file://test_ydb_table.TestExecuteQueryWithFormats.test_scan_query_pretty/result.output" - }, - "test_ydb_table.TestExecuteQueryWithParams.test_list": { + }, + "test_ydb_table.TestExecuteQueryWithParams.test_list": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_list/result.output" - }, - "test_ydb_table.TestExecuteQueryWithParams.test_scan_query_with_parameters": { + }, + "test_ydb_table.TestExecuteQueryWithParams.test_scan_query_with_parameters": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_scan_query_with_parameters/result.output" - }, - "test_ydb_table.TestExecuteQueryWithParams.test_struct": { + }, + "test_ydb_table.TestExecuteQueryWithParams.test_struct": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_struct/result.output" - }, - "test_ydb_table.TestExecuteQueryWithParams.test_uint32": { + }, + "test_ydb_table.TestExecuteQueryWithParams.test_uint32": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_uint32/result.output" - }, - "test_ydb_table.TestExecuteQueryWithParams.test_uint64_and_string": { + }, + "test_ydb_table.TestExecuteQueryWithParams.test_uint64_and_string": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_uint64_and_string/result.output" - } -} + } +} diff --git a/ydb/tests/functional/ydb_cli/test_ydb_scripting.py b/ydb/tests/functional/ydb_cli/test_ydb_scripting.py index 8985aa7f86..73ab7d9339 100644 --- a/ydb/tests/functional/ydb_cli/test_ydb_scripting.py +++ b/ydb/tests/functional/ydb_cli/test_ydb_scripting.py @@ -1,195 +1,195 @@ -# -*- coding: utf-8 -*- - +# -*- coding: utf-8 -*- + from ydb.tests.library.common import yatest_common from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory import ydb -import os -import logging - - -logger = logging.getLogger(__name__) - - -def ydb_bin(): +import os +import logging + + +logger = logging.getLogger(__name__) + + +def ydb_bin(): return yatest_common.binary_path("ydb/apps/ydb/ydb") - - -def upsert_simple(session, full_path): - path, table = os.path.split(full_path) - session.transaction().execute( - """ - PRAGMA TablePathPrefix("{0}"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (1, 1111, "one"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (2, 2222, "two"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (3, 3333, "three"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (5, 5555, "five"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (7, 7777, "seven"); - """.format(path, table), - commit_tx=True, - ) - - -def create_table_with_data(session, path): - session.create_table( - path, - ydb.TableDescription() - .with_column(ydb.Column("key", ydb.OptionalType(ydb.PrimitiveType.Uint32))) - .with_column(ydb.Column("id", ydb.OptionalType(ydb.PrimitiveType.Uint64))) - .with_column(ydb.Column("value", ydb.OptionalType(ydb.PrimitiveType.String))) - .with_primary_keys("key") - ) - - upsert_simple(session, path) - - -class BaseTestScriptingService(object): - @classmethod - def execute_ydb_cli_command(cls, args): - execution = yatest_common.execute([ydb_bin()] + args) - result = execution.std_out + + +def upsert_simple(session, full_path): + path, table = os.path.split(full_path) + session.transaction().execute( + """ + PRAGMA TablePathPrefix("{0}"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (1, 1111, "one"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (2, 2222, "two"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (3, 3333, "three"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (5, 5555, "five"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (7, 7777, "seven"); + """.format(path, table), + commit_tx=True, + ) + + +def create_table_with_data(session, path): + session.create_table( + path, + ydb.TableDescription() + .with_column(ydb.Column("key", ydb.OptionalType(ydb.PrimitiveType.Uint32))) + .with_column(ydb.Column("id", ydb.OptionalType(ydb.PrimitiveType.Uint64))) + .with_column(ydb.Column("value", ydb.OptionalType(ydb.PrimitiveType.String))) + .with_primary_keys("key") + ) + + upsert_simple(session, path) + + +class BaseTestScriptingService(object): + @classmethod + def execute_ydb_cli_command(cls, args): + execution = yatest_common.execute([ydb_bin()] + args) + result = execution.std_out logger.debug("std_out:\n" + result.decode('utf-8')) - return result - - @staticmethod - def canonical_result(output_result): - output_file_name = "result.output" - with open(output_file_name, "w") as f: + return result + + @staticmethod + def canonical_result(output_result): + output_file_name = "result.output" + with open(output_file_name, "w") as f: f.write(output_result.decode('utf-8')) return yatest_common.canonical_file(output_file_name, local=True, universal_lines=True) - - -class BaseTestScriptingServiceWithDatabase(BaseTestScriptingService): - @classmethod - def setup_class(cls): + + +class BaseTestScriptingServiceWithDatabase(BaseTestScriptingService): + @classmethod + def setup_class(cls): cls.cluster = kikimr_cluster_factory() - cls.cluster.start() + cls.cluster.start() cls.root_dir = "/Root" driver_config = ydb.DriverConfig( database="/Root", endpoint="%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port)) - cls.driver = ydb.Driver(driver_config) - cls.driver.wait(timeout=4) - - @classmethod - def teardown_class(cls): + cls.driver = ydb.Driver(driver_config) + cls.driver.wait(timeout=4) + + @classmethod + def teardown_class(cls): cls.cluster.stop() - - @classmethod - def execute_ydb_cli_command_with_db(cls, args): - return cls.execute_ydb_cli_command( - [ + + @classmethod + def execute_ydb_cli_command_with_db(cls, args): + return cls.execute_ydb_cli_command( + [ "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port, - "--database", cls.root_dir - ] + - args - ) - - -class TestExecuteScriptWithParams(BaseTestScriptingServiceWithDatabase): - @classmethod - def setup_class(cls): - BaseTestScriptingServiceWithDatabase.setup_class() - - session = cls.driver.table_client.session().create() - cls.table_path = cls.root_dir + "/scripting_params" - create_table_with_data(session, cls.table_path) - - def test_uint32(self): - script = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) - output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--param", "$par1=1"]) - return self.canonical_result(output) - - def test_uint64_and_string(self): - script = "DECLARE $id AS Uint64; "\ - "DECLARE $value AS String; "\ - "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) - output = self.execute_ydb_cli_command_with_db( - ["scripting", "yql", "-s", script, "--param", "$id=2222", - "--param", "$value=\"seven\""] - ) - return self.canonical_result(output) - - def test_list(self): - script = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" - output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--param", "$values=[1,2,3]"]) - return self.canonical_result(output) - - def test_struct(self): - script = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ - "SELECT "\ - "Table.key AS key, "\ - "Table.value AS value "\ - "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" - output = self.execute_ydb_cli_command_with_db( - ["scripting", "yql", "-s", script, "--param", - "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]"] - ) - return self.canonical_result(output) - - -class TestScriptingServiceHelp(BaseTestScriptingService): - def test_help(self): - output = self.execute_ydb_cli_command(["scripting", "yql", "--help"]) - return self.canonical_result(output) - - def test_help_ex(self): - output = self.execute_ydb_cli_command(["scripting", "yql", "--help-ex"]) - return self.canonical_result(output) - - -class TestExecuteScriptWithFormats(BaseTestScriptingServiceWithDatabase): - @classmethod - def setup_class(cls): - BaseTestScriptingServiceWithDatabase.setup_class() - - session = cls.driver.table_client.session().create() - cls.table_path = cls.root_dir + "/scripting_formats" - create_table_with_data(session, cls.table_path) - - def yql_script(self, format): - script = "SELECT * FROM `{path}` WHERE key < 4;" \ - "SELECT id FROM `{path}` WHERE key = 4;" \ - "SELECT value FROM `{path}` WHERE key > 5".format(path=self.table_path) - output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--format", format]) - return self.canonical_result(output) - - def stream_yql_script(self, format): - script = "SELECT * FROM `{path}` WHERE key < 4;" \ - "SELECT id FROM `{path}` WHERE key = 4;" \ - "SELECT value FROM `{path}` WHERE key > 5".format(path=self.table_path) - output = self.execute_ydb_cli_command_with_db(["yql", "-s", script, "--format", format]) - return self.canonical_result(output) - - # YqlScript - - def test_yql_script_pretty(self): - return self.yql_script('pretty') - - def test_yql_script_json_base64(self): - return self.yql_script('json-base64') - - def test_yql_script_json_base64_array(self): - return self.yql_script('json-base64-array') - - def test_yql_script_json_unicode(self): - return self.yql_script('json-unicode') - - def test_yql_script_json_unicode_array(self): - return self.yql_script('json-unicode-array') - - # StreamYqlScript - - def test_stream_yql_script_pretty(self): - return self.stream_yql_script('pretty') - - def test_stream_yql_script_json_base64(self): - return self.stream_yql_script('json-base64') - - def test_stream_yql_script_json_base64_array(self): - return self.stream_yql_script('json-base64-array') - - def test_stream_yql_script_json_unicode(self): - return self.stream_yql_script('json-unicode') - - def test_stream_yql_script_json_unicode_array(self): - return self.stream_yql_script('json-unicode-array') + "--database", cls.root_dir + ] + + args + ) + + +class TestExecuteScriptWithParams(BaseTestScriptingServiceWithDatabase): + @classmethod + def setup_class(cls): + BaseTestScriptingServiceWithDatabase.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/scripting_params" + create_table_with_data(session, cls.table_path) + + def test_uint32(self): + script = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) + output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--param", "$par1=1"]) + return self.canonical_result(output) + + def test_uint64_and_string(self): + script = "DECLARE $id AS Uint64; "\ + "DECLARE $value AS String; "\ + "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) + output = self.execute_ydb_cli_command_with_db( + ["scripting", "yql", "-s", script, "--param", "$id=2222", + "--param", "$value=\"seven\""] + ) + return self.canonical_result(output) + + def test_list(self): + script = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" + output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--param", "$values=[1,2,3]"]) + return self.canonical_result(output) + + def test_struct(self): + script = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ + "SELECT "\ + "Table.key AS key, "\ + "Table.value AS value "\ + "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" + output = self.execute_ydb_cli_command_with_db( + ["scripting", "yql", "-s", script, "--param", + "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]"] + ) + return self.canonical_result(output) + + +class TestScriptingServiceHelp(BaseTestScriptingService): + def test_help(self): + output = self.execute_ydb_cli_command(["scripting", "yql", "--help"]) + return self.canonical_result(output) + + def test_help_ex(self): + output = self.execute_ydb_cli_command(["scripting", "yql", "--help-ex"]) + return self.canonical_result(output) + + +class TestExecuteScriptWithFormats(BaseTestScriptingServiceWithDatabase): + @classmethod + def setup_class(cls): + BaseTestScriptingServiceWithDatabase.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/scripting_formats" + create_table_with_data(session, cls.table_path) + + def yql_script(self, format): + script = "SELECT * FROM `{path}` WHERE key < 4;" \ + "SELECT id FROM `{path}` WHERE key = 4;" \ + "SELECT value FROM `{path}` WHERE key > 5".format(path=self.table_path) + output = self.execute_ydb_cli_command_with_db(["scripting", "yql", "-s", script, "--format", format]) + return self.canonical_result(output) + + def stream_yql_script(self, format): + script = "SELECT * FROM `{path}` WHERE key < 4;" \ + "SELECT id FROM `{path}` WHERE key = 4;" \ + "SELECT value FROM `{path}` WHERE key > 5".format(path=self.table_path) + output = self.execute_ydb_cli_command_with_db(["yql", "-s", script, "--format", format]) + return self.canonical_result(output) + + # YqlScript + + def test_yql_script_pretty(self): + return self.yql_script('pretty') + + def test_yql_script_json_base64(self): + return self.yql_script('json-base64') + + def test_yql_script_json_base64_array(self): + return self.yql_script('json-base64-array') + + def test_yql_script_json_unicode(self): + return self.yql_script('json-unicode') + + def test_yql_script_json_unicode_array(self): + return self.yql_script('json-unicode-array') + + # StreamYqlScript + + def test_stream_yql_script_pretty(self): + return self.stream_yql_script('pretty') + + def test_stream_yql_script_json_base64(self): + return self.stream_yql_script('json-base64') + + def test_stream_yql_script_json_base64_array(self): + return self.stream_yql_script('json-base64-array') + + def test_stream_yql_script_json_unicode(self): + return self.stream_yql_script('json-unicode') + + def test_stream_yql_script_json_unicode_array(self): + return self.stream_yql_script('json-unicode-array') diff --git a/ydb/tests/functional/ydb_cli/test_ydb_table.py b/ydb/tests/functional/ydb_cli/test_ydb_table.py index ed860ee68f..148d33ede2 100644 --- a/ydb/tests/functional/ydb_cli/test_ydb_table.py +++ b/ydb/tests/functional/ydb_cli/test_ydb_table.py @@ -1,208 +1,208 @@ -# -*- coding: utf-8 -*- - +# -*- coding: utf-8 -*- + from ydb.tests.library.common import yatest_common from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory import ydb -import os -import logging - - -logger = logging.getLogger(__name__) - - -def ydb_bin(): +import os +import logging + + +logger = logging.getLogger(__name__) + + +def ydb_bin(): return yatest_common.binary_path("ydb/apps/ydb/ydb") - - -def upsert_simple(session, full_path): - path, table = os.path.split(full_path) - session.transaction().execute( - """ - PRAGMA TablePathPrefix("{0}"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (1, 1111, "one"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (2, 2222, "two"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (3, 3333, "three"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (5, 5555, "five"); - UPSERT INTO {1} (`key`, `id`, `value`) VALUES (7, 7777, "seven"); - """.format(path, table), - commit_tx=True, - ) - - -def create_table_with_data(session, path): - session.create_table( - path, - ydb.TableDescription() - .with_column(ydb.Column("key", ydb.OptionalType(ydb.PrimitiveType.Uint32))) - .with_column(ydb.Column("id", ydb.OptionalType(ydb.PrimitiveType.Uint64))) - .with_column(ydb.Column("value", ydb.OptionalType(ydb.PrimitiveType.String))) - .with_primary_keys("key") - ) - - upsert_simple(session, path) - - -class BaseTestTableService(object): - @classmethod - def setup_class(cls): + + +def upsert_simple(session, full_path): + path, table = os.path.split(full_path) + session.transaction().execute( + """ + PRAGMA TablePathPrefix("{0}"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (1, 1111, "one"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (2, 2222, "two"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (3, 3333, "three"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (5, 5555, "five"); + UPSERT INTO {1} (`key`, `id`, `value`) VALUES (7, 7777, "seven"); + """.format(path, table), + commit_tx=True, + ) + + +def create_table_with_data(session, path): + session.create_table( + path, + ydb.TableDescription() + .with_column(ydb.Column("key", ydb.OptionalType(ydb.PrimitiveType.Uint32))) + .with_column(ydb.Column("id", ydb.OptionalType(ydb.PrimitiveType.Uint64))) + .with_column(ydb.Column("value", ydb.OptionalType(ydb.PrimitiveType.String))) + .with_primary_keys("key") + ) + + upsert_simple(session, path) + + +class BaseTestTableService(object): + @classmethod + def setup_class(cls): cls.cluster = kikimr_cluster_factory() - cls.cluster.start() + cls.cluster.start() cls.root_dir = "/Root" driver_config = ydb.DriverConfig( database="/Root", endpoint="%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port)) - cls.driver = ydb.Driver(driver_config) - cls.driver.wait(timeout=4) - - @classmethod - def teardown_class(cls): - if hasattr(cls, 'cluster'): - cls.cluster.stop() - - @classmethod - def execute_ydb_cli_command(cls, args): - execution = yatest_common.execute( - [ - ydb_bin(), + cls.driver = ydb.Driver(driver_config) + cls.driver.wait(timeout=4) + + @classmethod + def teardown_class(cls): + if hasattr(cls, 'cluster'): + cls.cluster.stop() + + @classmethod + def execute_ydb_cli_command(cls, args): + execution = yatest_common.execute( + [ + ydb_bin(), "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port, - "--database", cls.root_dir - ] + - args - ) - - result = execution.std_out + "--database", cls.root_dir + ] + + args + ) + + result = execution.std_out logger.debug("std_out:\n" + result.decode('utf-8')) - return result - - @staticmethod - def canonical_result(output_result): - output_file_name = "result.output" - with open(output_file_name, "w") as f: + return result + + @staticmethod + def canonical_result(output_result): + output_file_name = "result.output" + with open(output_file_name, "w") as f: f.write(output_result.decode('utf-8')) return yatest_common.canonical_file(output_file_name, local=True, universal_lines=True) - - -class TestExecuteQueryWithParams(BaseTestTableService): - - @classmethod - def setup_class(cls): - BaseTestTableService.setup_class() - - session = cls.driver.table_client.session().create() - cls.table_path = cls.root_dir + "/table_params" - create_table_with_data(session, cls.table_path) - - def test_uint32(self): - query = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) - output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$par1=1"]) - return self.canonical_result(output) - - def test_uint64_and_string(self): - query = "DECLARE $id AS Uint64; "\ - "DECLARE $value AS String; "\ - "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) - output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$id=2222", - "--param", "$value=\"seven\""]) - return self.canonical_result(output) - - def test_list(self): - query = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" - output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$values=[1,2,3]"]) - return self.canonical_result(output) - - def test_struct(self): - query = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ - "SELECT "\ - "Table.key AS key, "\ - "Table.value AS value "\ - "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" - output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", - "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]"]) - return self.canonical_result(output) - - def test_scan_query_with_parameters(self): - query = "DECLARE $id AS Uint64; "\ - "DECLARE $value AS String; "\ - "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) - output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "scan", "-q", query, - "--param", "$id=2222", - "--param", "$value=\"seven\""]) - return self.canonical_result(output) - - -class TestExecuteQueryWithFormats(BaseTestTableService): - - @classmethod - def setup_class(cls): - BaseTestTableService.setup_class() - - session = cls.driver.table_client.session().create() - cls.table_path = cls.root_dir + "/table_formats" - create_table_with_data(session, cls.table_path) - - def execute_data_query(self, format): - query = "SELECT * FROM `{}` WHERE key < 4;".format(self.table_path) - output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "data", "-q", query, - "--format", format]) - return self.canonical_result(output) - - def execute_scan_query(self, format): - query = "SELECT * FROM `{}` WHERE key < 4;".format(self.table_path) - output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "scan", "-q", query, - "--format", format]) - return self.canonical_result(output) - - def execute_read_table(self, format): - output = self.execute_ydb_cli_command(["table", "readtable", self.table_path, "--format", format]) - return self.canonical_result(output) - - # DataQuery - - def test_data_query_pretty(self): - return self.execute_data_query('pretty') - - def test_data_query_json_base64(self): - return self.execute_data_query('json-base64') - - def test_data_query_json_base64_array(self): - return self.execute_data_query('json-base64-array') - - def test_data_query_json_unicode(self): - return self.execute_data_query('json-unicode') - - def test_data_query_json_unicode_array(self): - return self.execute_data_query('json-unicode-array') - - # ScanQuery - - def test_scan_query_pretty(self): - return self.execute_scan_query('pretty') - - def test_scan_query_json_base64(self): - return self.execute_scan_query('json-base64') - - def test_scan_query_json_base64_array(self): - return self.execute_scan_query('json-base64-array') - - def test_scan_query_json_unicode(self): - return self.execute_scan_query('json-unicode') - - def test_scan_query_json_unicode_array(self): - return self.execute_scan_query('json-unicode-array') - - # ReadTable - - def test_read_table_pretty(self): - return self.execute_read_table('pretty') - - def test_read_table_json_base64(self): - return self.execute_read_table('json-base64') - - def test_read_table_json_base64_array(self): - return self.execute_read_table('json-base64-array') - - def test_read_table_json_unicode(self): - return self.execute_read_table('json-unicode') - - def test_read_table_json_unicode_array(self): - return self.execute_read_table('json-unicode-array') + + +class TestExecuteQueryWithParams(BaseTestTableService): + + @classmethod + def setup_class(cls): + BaseTestTableService.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/table_params" + create_table_with_data(session, cls.table_path) + + def test_uint32(self): + query = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$par1=1"]) + return self.canonical_result(output) + + def test_uint64_and_string(self): + query = "DECLARE $id AS Uint64; "\ + "DECLARE $value AS String; "\ + "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$id=2222", + "--param", "$value=\"seven\""]) + return self.canonical_result(output) + + def test_list(self): + query = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" + output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", "$values=[1,2,3]"]) + return self.canonical_result(output) + + def test_struct(self): + query = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ + "SELECT "\ + "Table.key AS key, "\ + "Table.value AS value "\ + "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" + output = self.execute_ydb_cli_command(["table", "query", "execute", "-q", query, "--param", + "$values=[{\"key\":1,\"value\":\"one\"},{\"key\":2,\"value\":\"two\"}]"]) + return self.canonical_result(output) + + def test_scan_query_with_parameters(self): + query = "DECLARE $id AS Uint64; "\ + "DECLARE $value AS String; "\ + "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "scan", "-q", query, + "--param", "$id=2222", + "--param", "$value=\"seven\""]) + return self.canonical_result(output) + + +class TestExecuteQueryWithFormats(BaseTestTableService): + + @classmethod + def setup_class(cls): + BaseTestTableService.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/table_formats" + create_table_with_data(session, cls.table_path) + + def execute_data_query(self, format): + query = "SELECT * FROM `{}` WHERE key < 4;".format(self.table_path) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "data", "-q", query, + "--format", format]) + return self.canonical_result(output) + + def execute_scan_query(self, format): + query = "SELECT * FROM `{}` WHERE key < 4;".format(self.table_path) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", "scan", "-q", query, + "--format", format]) + return self.canonical_result(output) + + def execute_read_table(self, format): + output = self.execute_ydb_cli_command(["table", "readtable", self.table_path, "--format", format]) + return self.canonical_result(output) + + # DataQuery + + def test_data_query_pretty(self): + return self.execute_data_query('pretty') + + def test_data_query_json_base64(self): + return self.execute_data_query('json-base64') + + def test_data_query_json_base64_array(self): + return self.execute_data_query('json-base64-array') + + def test_data_query_json_unicode(self): + return self.execute_data_query('json-unicode') + + def test_data_query_json_unicode_array(self): + return self.execute_data_query('json-unicode-array') + + # ScanQuery + + def test_scan_query_pretty(self): + return self.execute_scan_query('pretty') + + def test_scan_query_json_base64(self): + return self.execute_scan_query('json-base64') + + def test_scan_query_json_base64_array(self): + return self.execute_scan_query('json-base64-array') + + def test_scan_query_json_unicode(self): + return self.execute_scan_query('json-unicode') + + def test_scan_query_json_unicode_array(self): + return self.execute_scan_query('json-unicode-array') + + # ReadTable + + def test_read_table_pretty(self): + return self.execute_read_table('pretty') + + def test_read_table_json_base64(self): + return self.execute_read_table('json-base64') + + def test_read_table_json_base64_array(self): + return self.execute_read_table('json-base64-array') + + def test_read_table_json_unicode(self): + return self.execute_read_table('json-unicode') + + def test_read_table_json_unicode_array(self): + return self.execute_read_table('json-unicode-array') diff --git a/ydb/tests/functional/ydb_cli/ya.make b/ydb/tests/functional/ydb_cli/ya.make index 6e7298b111..cf8d4dae48 100644 --- a/ydb/tests/functional/ydb_cli/ya.make +++ b/ydb/tests/functional/ydb_cli/ya.make @@ -3,8 +3,8 @@ PY3TEST() TEST_SRCS( test_ydb_backup.py - test_ydb_table.py - test_ydb_scripting.py + test_ydb_table.py + test_ydb_scripting.py ) ENV(YDB_TOKEN="root@builtin") diff --git a/ydb/tests/library/harness/kikimr_cluster.py b/ydb/tests/library/harness/kikimr_cluster.py index 134bd40ea2..942e1588ca 100644 --- a/ydb/tests/library/harness/kikimr_cluster.py +++ b/ydb/tests/library/harness/kikimr_cluster.py @@ -3,7 +3,7 @@ import os import itertools import logging -import subprocess +import subprocess import time import pprint from concurrent import futures @@ -71,11 +71,11 @@ class ExternalKiKiMRCluster(KiKiMRClusterInterface): def stop(self): return self - def restart(self): - self._stop() - self._start() - return self - + def restart(self): + self._stop() + self._start() + return self + def _start(self): for inst_set in [self.nodes, self.slots, self.nbs]: self._run_on(inst_set, lambda x: x.start()) @@ -233,37 +233,37 @@ class ExternalKiKiMRCluster(KiKiMRClusterInterface): start += 10 return self._slots - - def _run_discovery_command(self, tenant_name): - discovery_cmd = [ + + def _run_discovery_command(self, tenant_name): + discovery_cmd = [ self.__binary_path, '--server', self.nodes[1].host, 'discovery', 'list', '-d', tenant_name - ] - logger.info('Executing discovery command: %s' % ' '.join(list(discovery_cmd))) - ds_result = subprocess.check_output(discovery_cmd) - logger.info('Discovery command result: "{}"'.format(ds_result)) - return ds_result - - def _get_node(self, host, grpc_port): - nodes = self._slots if self._slots else self.nodes - for node in nodes.values(): - if node.host == host and str(node.grpc_port) == grpc_port: - return node - logger.error('Cant find node with host {} and grpc_port {}'.format(host, grpc_port)) - - def get_active_tenant_nodes(self, tenant_name): - for try_num in range(5): - ds_result = self._run_discovery_command(tenant_name) - if not len(ds_result) or ds_result[:2] != 'OK': - continue - lines = ds_result.splitlines() - nodes = [] - for line_num in range(1, len(lines)): - # 'grpc://ydb-ru-testing-vla-0000.search.yandex.net:31010 [VLA]' - line = lines[line_num] - words = line.split(':') - host = words[1][2:] - port = words[2].split(' ')[0] - node = self._get_node(host, port) - if node is not None: - nodes.append(node) - return nodes + ] + logger.info('Executing discovery command: %s' % ' '.join(list(discovery_cmd))) + ds_result = subprocess.check_output(discovery_cmd) + logger.info('Discovery command result: "{}"'.format(ds_result)) + return ds_result + + def _get_node(self, host, grpc_port): + nodes = self._slots if self._slots else self.nodes + for node in nodes.values(): + if node.host == host and str(node.grpc_port) == grpc_port: + return node + logger.error('Cant find node with host {} and grpc_port {}'.format(host, grpc_port)) + + def get_active_tenant_nodes(self, tenant_name): + for try_num in range(5): + ds_result = self._run_discovery_command(tenant_name) + if not len(ds_result) or ds_result[:2] != 'OK': + continue + lines = ds_result.splitlines() + nodes = [] + for line_num in range(1, len(lines)): + # 'grpc://ydb-ru-testing-vla-0000.search.yandex.net:31010 [VLA]' + line = lines[line_num] + words = line.split(':') + host = words[1][2:] + port = words[2].split(' ')[0] + node = self._get_node(host, port) + if node is not None: + nodes.append(node) + return nodes diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index fb8ca860c7..251f485b5d 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -120,18 +120,18 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface): command.append("--suppress-version-check") if self.__node_broker_port is not None: - command.append("--node-broker=%s%s:%d" % ( - "grpcs://" if self.__configurator.grpc_ssl_enable else "", - self.host, - self.__node_broker_port)) + command.append("--node-broker=%s%s:%d" % ( + "grpcs://" if self.__configurator.grpc_ssl_enable else "", + self.host, + self.__node_broker_port)) else: command.append("--node=%d" % self.node_id) - if self.__configurator.grpc_ssl_enable: - command.append( - "--ca=%s" % self.__configurator.grpc_tls_ca_path - ) - + if self.__configurator.grpc_ssl_enable: + command.append( + "--ca=%s" % self.__configurator.grpc_tls_ca_path + ) + if self.__role == 'slot': command.append( "--tenant=%s" % self._tenant_affiliation @@ -348,10 +348,10 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): def _register_slot(self, tenant_affiliation=None, encryption_key=None): slot_index = next(self._slot_index_allocator) - node_broker_port = ( - self.nodes[1].grpc_ssl_port if self.__configurator.grpc_ssl_enable - else self.nodes[1].grpc_port - ) + node_broker_port = ( + self.nodes[1].grpc_ssl_port if self.__configurator.grpc_ssl_enable + else self.nodes[1].grpc_port + ) self._slots[slot_index] = KiKiMRNode( slot_index, self.config_path, @@ -360,7 +360,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): configurator=self.__configurator, udfs_dir=self.__common_udfs_dir, role='slot', - node_broker_port=node_broker_port, + node_broker_port=node_broker_port, tenant_affiliation=tenant_affiliation, encryption_key=encryption_key, ) |