diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2024-12-17 14:13:00 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-12-17 14:31:07 +0300 |
commit | 3f61c3b13253c98c4d3663c76de3357d3ae04aa7 (patch) | |
tree | 6c3eebed23313391ff13e8914a8e4f0d447044cf | |
parent | 30272794b11546a1e358cac78fc561ad1d4ef01b (diff) | |
download | ydb-3f61c3b13253c98c4d3663c76de3357d3ae04aa7.tar.gz |
Intermediate changes
commit_hash:5d5aa527dd6fcb0dfb1e62748135e5f04f92b1b0
-rw-r--r-- | yql/essentials/tools/yql_facade_run/ya.make | 2 | ||||
-rw-r--r-- | yql/essentials/tools/yql_facade_run/yql_facade_run.cpp | 230 | ||||
-rw-r--r-- | yql/essentials/tools/yql_facade_run/yql_facade_run.h | 49 | ||||
-rw-r--r-- | yql/tools/yqlrun/lib/yqlrun_lib.cpp | 17 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/config.cpp | 3 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/config.h | 2 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/exporter.cpp | 1 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/public.h | 9 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/registry.cpp | 5 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/registry.h | 1 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/tag_registry-inl.h | 31 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/tag_registry.cpp | 164 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/tag_registry.h | 26 | ||||
-rw-r--r-- | yt/yt/library/profiling/unittests/solomon_ut.cpp | 130 |
14 files changed, 546 insertions, 124 deletions
diff --git a/yql/essentials/tools/yql_facade_run/ya.make b/yql/essentials/tools/yql_facade_run/ya.make index e9d93e235d..e67e0b6c7f 100644 --- a/yql/essentials/tools/yql_facade_run/ya.make +++ b/yql/essentials/tools/yql_facade_run/ya.make @@ -21,6 +21,8 @@ PEERDIR( yql/essentials/core/url_lister yql/essentials/core/url_preprocessing yql/essentials/core/peephole_opt + yql/essentials/core/qplayer/storage/interface + yql/essentials/core/qplayer/storage/file yql/essentials/core yql/essentials/minikql/invoke_builtins yql/essentials/minikql diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index 91a3a3eaf8..6e6167582f 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -21,6 +21,7 @@ #include <yql/essentials/core/facade/yql_facade.h> #include <yql/essentials/core/url_lister/url_lister_manager.h> #include <yql/essentials/core/url_preprocessing/url_preprocessing.h> +#include <yql/essentials/core/qplayer/storage/file/yql_qstorage_file.h> #include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h> #include <yql/essentials/minikql/mkql_function_registry.h> #include <yql/essentials/ast/yql_expr.h> @@ -51,9 +52,11 @@ #include <util/stream/file.h> #include <util/stream/null.h> #include <util/system/user.h> +#include <util/system/env.h> #include <util/string/split.h> #include <util/string/join.h> #include <util/string/builder.h> +#include <util/string/strip.h> #include <util/generic/vector.h> #include <util/generic/ptr.h> #include <util/generic/yexception.h> @@ -155,7 +158,6 @@ public: namespace NYql { TFacadeRunOptions::TFacadeRunOptions() { - User = GetUsername(); } TFacadeRunOptions::~TFacadeRunOptions() { @@ -185,11 +187,23 @@ void TFacadeRunOptions::PrintInfo(const TString& msg) { } void TFacadeRunOptions::Parse(int argc, const char *argv[]) { + User = GetUsername(); + + if (EnableCredentials) { + Token = GetEnv("YQL_TOKEN"); + if (!Token) { + const TString home = GetEnv("HOME"); + auto tokenPath = TFsPath(home) / ".yql" / "token"; + if (tokenPath.Exists()) { + Token = StripStringRight(TFileInput(tokenPath).ReadAll()); + } + } + } NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); opts.AddHelpOption(); - opts.AddLongOption('p', "program", "Program file").Required().RequiredArgument("FILE") + opts.AddLongOption('p', "program", "Program file (use - to read from stdin)").Required().RequiredArgument("FILE") .Handler1T<TString>([this](const TString& file) { ProgramFile = file; if (ProgramFile == "-") { @@ -203,7 +217,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption('s', "sql", "Program is SQL query").NoArgument().StoreValue(&ProgramType, EProgramType::Sql); if (PgSupport) { opts.AddLongOption("pg", "Program has PG syntax").NoArgument().StoreValue(&ProgramType, EProgramType::Pg); - opts.AddLongOption("pg-ext", "pg extensions config file").Optional().RequiredArgument("FILE") + opts.AddLongOption("pg-ext", "Pg extensions config file").Optional().RequiredArgument("FILE") .Handler1T<TString>([this](const TString& file) { PgExtConfig = ParseProtoConfig<NProto::TPgExtensions>(file); }); @@ -238,7 +252,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { .Handler1T<TString>([this](const TString& file) { Params = TFileInput(file).ReadAll(); }); - opts.AddLongOption('G', "gateways", "Used gateways").DefaultValue(JoinSeq(",", SupportedGateways_)) + opts.AddLongOption('G', "gateways", TStringBuilder() << "Used gateways, available: " << JoinSeq(",", SupportedGateways_)).DefaultValue(JoinSeq(",", GatewayTypes)) .Handler1T<TString>([this](const TString& gateways) { ::StringSplitter(gateways).Split(',').Consume([&](const TStringBuf& val) { if (!SupportedGateways_.contains(val)) { @@ -256,8 +270,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { FsConfig = MakeHolder<TFileStorageConfig>(); LoadFsConfigFromFile(file, *FsConfig); }); - opts.AddLongOption('u', "udf", "Load shared library with UDF by given path").AppendTo(&UdfsPaths); - opts.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory") + opts.AddLongOption('u', "udf", "Load shared library with UDF by given path").RequiredArgument("PATH").AppendTo(&UdfsPaths); + opts.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory").RequiredArgument("DIR") .Handler1T<TString>([this](const TString& dir) { NKikimr::NMiniKQL::FindUdfsInDir(dir, &UdfsPaths); }); @@ -265,14 +279,14 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption("udf-resolver-filter-syscalls", "Filter syscalls in udf resolver").Optional().NoArgument().SetFlag(&UdfResolverFilterSyscalls); opts.AddLongOption("scan-udfs", "Scan specified udfs with external udf-resolver to use static function registry").NoArgument().SetFlag(&ScanUdfs); - opts.AddLongOption("parse-only", "Exit after program has been parsed").NoArgument().StoreValue(&Mode, ERunMode::Parse); - opts.AddLongOption("compile-only", "Exit after program has been compiled").NoArgument().StoreValue(&Mode, ERunMode::Compile); - opts.AddLongOption("validate", "Exit after program has been validated").NoArgument().StoreValue(&Mode, ERunMode::Validate); - opts.AddLongOption("lineage", "Exit after data lineage has been calculated").NoArgument().StoreValue(&Mode, ERunMode::Lineage); - opts.AddLongOption('O',"optimize", "Optimize expression").NoArgument().StoreValue(&Mode, ERunMode::Optimize); - opts.AddLongOption('R',"run", "Run expression using input/output tables").NoArgument().StoreValue(&Mode, ERunMode::Run); - opts.AddLongOption('D', "discover", "Discover tables in the program").NoArgument().StoreValue(&Mode, ERunMode::Discover); - opts.AddLongOption("peephole", "Perform peephole stage of expression using input/output tables").NoArgument().StoreValue(&Mode, ERunMode::Peephole); + opts.AddLongOption("parse-only", "Parse program and exit").NoArgument().StoreValue(&Mode, ERunMode::Parse); + opts.AddLongOption("compile-only", "Compiled program and exit").NoArgument().StoreValue(&Mode, ERunMode::Compile); + opts.AddLongOption("validate", "Validate program and exit").NoArgument().StoreValue(&Mode, ERunMode::Validate); + opts.AddLongOption("lineage", "Calculate program lineage and exit").NoArgument().StoreValue(&Mode, ERunMode::Lineage); + opts.AddLongOption('O',"optimize", "Optimize program and exir").NoArgument().StoreValue(&Mode, ERunMode::Optimize); + opts.AddLongOption('D', "discover", "Discover tables in the program and exit").NoArgument().StoreValue(&Mode, ERunMode::Discover); + opts.AddLongOption("peephole", "Perform peephole program optimization and exit").NoArgument().StoreValue(&Mode, ERunMode::Peephole); + opts.AddLongOption('R',"run", "Run progrum (use by default)").NoArgument().StoreValue(&Mode, ERunMode::Run); opts.AddLongOption('L', "show-log", "Show transformation log").Optional().NoArgument().SetFlag(&ShowLog); opts.AddLongOption('v', "verbosity", "Log verbosity level").Optional().RequiredArgument("LEVEL").StoreResult(&Verbosity); @@ -290,8 +304,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { }); opts.AddLongOption("expr-file", "Print AST to that file instead of stdout").Optional().RequiredArgument("FILE") .Handler1T<TString>([this](const TString& file) { - ExprStreamHolder = MakeHolder<TFixedBufferFileOutput>(file); - ExprStream = ExprStreamHolder.Get(); + ExprStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file); + ExprStream = ExprStreamHolder_.Get(); }); opts.AddLongOption("print-result", "Print program execution result to stdout").NoArgument() .Handler0([this]() { @@ -317,8 +331,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption("result-file", "Print program execution result to file").Optional().RequiredArgument("FILE") .Handler1T<TString>([this](const TString& file) { - ResultStreamHolder = MakeHolder<TFixedBufferFileOutput>(file); - ResultStream = ResultStreamHolder.Get(); + ResultStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file); + ResultStream = ResultStreamHolder_.Get(); }); opts.AddLongOption('P',"trace-plan", "Print plan before execution").NoArgument() .Handler0([this]() { @@ -328,13 +342,13 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { }); opts.AddLongOption("plan-file", "Print program plan to file").Optional().RequiredArgument("FILE") .Handler1T<TString>([this](const TString& file) { - PlanStreamHolder = MakeHolder<TFixedBufferFileOutput>(file); - PlanStream = PlanStreamHolder.Get(); + PlanStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file); + PlanStream = PlanStreamHolder_.Get(); }); opts.AddLongOption("err-file", "Print validate/optimize/runtime errors to file") .Handler1T<TString>([this](const TString& file) { - ErrStreamHolder = MakeHolder<TFixedBufferFileOutput>(file); - ErrStream = ErrStreamHolder.Get(); + ErrStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file); + ErrStream = ErrStreamHolder_.Get(); }); opts.AddLongOption("full-expr", "Avoid buffering of expr/plan").NoArgument().SetFlag(&FullExpr); opts.AddLongOption("mem-limit", "Set memory limit in megabytes") @@ -366,8 +380,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption("stat", "Print execution statistics").Optional().OptionalArgument("FILE") .Handler1T<TString>([this](const TString& file) { if (file) { - StatStreamHolder = MakeHolder<TFileOutput>(file); - StatStream = StatStreamHolder.Get(); + StatStreamHolder_ = MakeHolder<TFileOutput>(file); + StatStream = StatStreamHolder_.Get(); } else { StatStream = &Cerr; } @@ -396,6 +410,42 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { } }); } + if (EnableCredentials) { + opts.AddLongOption("token", "YQL token") + .Optional() + .RequiredArgument("VALUE") + .StoreResult(&Token); + opts.AddLongOption("custom-tokens", "Custom tokens") + .Optional() + .RequiredArgument("NAME=VALUE or NAME=@PATH") + .KVHandler([this](TString key, TString value) { + if (value.StartsWith('@')) { + value = StripStringRight(TFileInput(value.substr(1)).ReadAll()); + } + Credentials->AddCredential(key, TCredential("custom", "", value)); + }); + } + if (EnableQPlayer) { + opts.AddLongOption("qstorage-dir", "Directory for QStorage").RequiredArgument("DIR") + .Handler1T<TString>([this](const TString& dir) { + QPlayerStorage_ = MakeFileQStorage(dir); + }); + opts.AddLongOption("op-id", "QStorage operation id").StoreResult(&OperationId).DefaultValue("dummy_op"); + opts.AddLongOption("capture", "Write query metadata to QStorage").NoArgument() + .Handler0([this]() { + if (EQPlayerMode::Replay == QPlayerMode) { + throw yexception() << "replay and capture options can't be used simultaneously"; + } + QPlayerMode = EQPlayerMode::Capture; + }); + opts.AddLongOption("replay", "Read query metadata from QStorage").NoArgument() + .Handler0([this]() { + if (EQPlayerMode::Capture == QPlayerMode) { + throw yexception() << "replay and capture options can't be used simultaneously"; + } + QPlayerMode = EQPlayerMode::Replay; + }); + } opts.SetFreeArgsMax(0); @@ -405,8 +455,17 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { auto res = NLastGetopt::TOptsParseResult(&opts, argc, argv); - for (auto& handle: OptHandlers_) { - handle(res); + if (QPlayerMode != EQPlayerMode::None) { + if (!QPlayerStorage_) { + QPlayerStorage_ = MakeFileQStorage("."); + } + if (EQPlayerMode::Replay == QPlayerMode) { + QPlayerContext = TQContext(QPlayerStorage_->MakeReader(OperationId, {})); + ProgramFile = "-replay-"; + ProgramText = ""; + } else if (EQPlayerMode::Capture == QPlayerMode) { + QPlayerContext = TQContext(QPlayerStorage_->MakeWriter(OperationId, {})); + } } if (Mode >= ERunMode::Validate && GatewayTypes.empty()) { @@ -420,6 +479,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { if (GatewaysConfig && GatewaysConfig->HasSqlCore()) { SqlFlags.insert(GatewaysConfig->GetSqlCore().GetTranslationFlags().begin(), GatewaysConfig->GetSqlCore().GetTranslationFlags().end()); } + UpdateSqlFlagsFromQContext(QPlayerContext, SqlFlags); if (!FsConfig) { FsConfig = MakeHolder<TFileStorageConfig>(); @@ -427,6 +487,28 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { LoadFsConfigFromResource("fs.conf", *FsConfig); } } + + if (EnableCredentials && Token) { + for (auto name: SupportedGateways_) { + Credentials->AddCredential(TStringBuilder() << "default_" << name, TCredential(name, "", Token)); + } + } + + for (auto& handle: OptHandlers_) { + handle(res); + } +} + +TFacadeRunner::TFacadeRunner(TString name) + : Name_(std::move(name)) +{ +} + +TFacadeRunner::~TFacadeRunner() { +} + +TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> TFacadeRunner::GetFuncRegistry() { + return FuncRegistry_; } int TFacadeRunner::Main(int argc, const char *argv[]) { @@ -463,7 +545,7 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { if (RunOptions_.PgExtConfig) { TVector<NPg::TExtensionDesc> extensions; PgExtensionsFromProto(*RunOptions_.PgExtConfig, extensions); - NPg::RegisterExtensions(extensions, false, + NPg::RegisterExtensions(extensions, RunOptions_.QPlayerContext.CanRead(), *NSQLTranslationPG::CreateExtensionSqlParser(), NKikimr::NMiniKQL::CreateExtensionLoader().get()); } @@ -503,7 +585,9 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { if (RunOptions_.Mode >= ERunMode::Validate) { std::vector<NFS::IDownloaderPtr> downloaders; for (auto& factory: FsDownloadFactories_) { - downloaders.push_back(factory()); + if (auto download = factory()) { + downloaders.push_back(std::move(download)); + } } FileStorage_ = WithAsync(CreateFileStorage(*RunOptions_.FsConfig, downloaders)); @@ -537,12 +621,16 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { dataProvidersInit.push_back(GetPgDataProviderInitializer()); } for (auto& factory: ProviderFactories_) { - dataProvidersInit.push_back(factory()); + if (auto init = factory()) { + dataProvidersInit.push_back(std::move(init)); + } } TVector<IUrlListerPtr> urlListers; for (auto& factory: UrlListerFactories_) { - urlListers.push_back(factory()); + if (auto listener = factory()) { + urlListers.push_back(std::move(listener)); + } } TProgramFactory factory(RunOptions_.UseRepeatableRandomAndTimeProviders, FuncRegistry_.Get(), ctx.NextUniqueId, dataProvidersInit, Name_); @@ -555,18 +643,22 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) { factory.SetUdfIndex(udfIndex, new TUdfIndexPackageSet()); factory.SetUdfResolver(udfResolver); factory.SetGatewaysConfig(RunOptions_.GatewaysConfig.Get()); - factory.SetCredentials(Credentials_); + factory.SetCredentials(RunOptions_.Credentials); factory.EnableRangeComputeFor(); if (!urlListers.empty()) { factory.SetUrlListerManager(MakeUrlListerManager(urlListers)); } - return RunProgram(factory); + int result = DoRun(factory); + if (result == 0 && EQPlayerMode::Capture == RunOptions_.QPlayerMode) { + RunOptions_.QPlayerContext.GetWriter()->Commit().GetValueSync(); + } + return result; } -int TFacadeRunner::RunProgram(TProgramFactory& factory) { +int TFacadeRunner::DoRun(TProgramFactory& factory) { - TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText);; + TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText, RunOptions_.OperationId, EHiddenMode::Disable, RunOptions_.QPlayerContext);; if (RunOptions_.Params) { program->SetParametersYson(RunOptions_.Params); } @@ -673,40 +765,15 @@ int TFacadeRunner::RunProgram(TProgramFactory& factory) { return 0; } - auto defOptConfig = TOptPipelineConfigurator(program, RunOptions_.FullExpr ? RunOptions_.PlanStream : nullptr, RunOptions_.FullExpr ? RunOptions_.ExprStream : nullptr, RunOptions_.WithTypes); - IPipelineConfigurator* optConfig = OptPipelineConfigurator_ ? OptPipelineConfigurator_ : &defOptConfig; - - TProgram::TStatus status = TProgram::TStatus::Ok; - if (ERunMode::Peephole == RunOptions_.Mode) { - RunOptions_.PrintInfo("Peephole..."); - auto defConfig = TPeepHolePipelineConfigurator(); - IPipelineConfigurator* config = PeepholePipelineConfigurator_ ? PeepholePipelineConfigurator_ : &defConfig; - status = program->OptimizeWithConfig(RunOptions_.User, *config); + TProgram::TStatus status = DoRunProgram(program); - if (RunOptions_.ExprStream && program->ExprRoot()) { - auto ast = ConvertToAst(*program->ExprRoot(), program->ExprCtx(), RunOptions_.WithTypes ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true); - ui32 prettyFlags = TAstPrintFlags::ShortQuote; - if (!RunOptions_.WithTypes) { - prettyFlags |= TAstPrintFlags::PerLine; - } - ast.Root->PrettyPrintTo(*RunOptions_.ExprStream, prettyFlags); + if (ERunMode::Peephole == RunOptions_.Mode && RunOptions_.ExprStream && program->ExprRoot()) { + auto ast = ConvertToAst(*program->ExprRoot(), program->ExprCtx(), RunOptions_.WithTypes ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true); + ui32 prettyFlags = TAstPrintFlags::ShortQuote; + if (!RunOptions_.WithTypes) { + prettyFlags |= TAstPrintFlags::PerLine; } - - } else if (ERunMode::Run == RunOptions_.Mode) { - RunOptions_.PrintInfo("Run program..."); - status = program->RunWithConfig(RunOptions_.User, *optConfig); - } else if (ERunMode::Optimize == RunOptions_.Mode) { - RunOptions_.PrintInfo("Optimize program..."); - status = program->OptimizeWithConfig(RunOptions_.User, *optConfig); - } else if (ERunMode::Validate == RunOptions_.Mode) { - RunOptions_.PrintInfo("Validate program..."); - status = program->Validate(RunOptions_.User, RunOptions_.ExprStream, RunOptions_.WithTypes); - } else if (ERunMode::Discover == RunOptions_.Mode) { - RunOptions_.PrintInfo("Discover program..."); - status = program->Discover(RunOptions_.User); - } else if (ERunMode::Lineage == RunOptions_.Mode) { - RunOptions_.PrintInfo("Calculating lineage in program..."); - status = program->LineageWithConfig(RunOptions_.User, *optConfig); + ast.Root->PrettyPrintTo(*RunOptions_.ExprStream, prettyFlags); } if (RunOptions_.WithFinalIssues) { @@ -776,4 +843,35 @@ int TFacadeRunner::RunProgram(TProgramFactory& factory) { return 0; } +TProgram::TStatus TFacadeRunner::DoRunProgram(TProgramPtr program) { + TProgram::TStatus status = TProgram::TStatus::Ok; + + auto defOptConfig = TOptPipelineConfigurator(program, RunOptions_.FullExpr ? RunOptions_.PlanStream : nullptr, RunOptions_.FullExpr ? RunOptions_.ExprStream : nullptr, RunOptions_.WithTypes); + IPipelineConfigurator* optConfig = OptPipelineConfigurator_ ? OptPipelineConfigurator_ : &defOptConfig; + + if (ERunMode::Peephole == RunOptions_.Mode) { + RunOptions_.PrintInfo("Peephole..."); + auto defConfig = TPeepHolePipelineConfigurator(); + IPipelineConfigurator* config = PeepholePipelineConfigurator_ ? PeepholePipelineConfigurator_ : &defConfig; + status = program->OptimizeWithConfig(RunOptions_.User, *config); + } else if (ERunMode::Run == RunOptions_.Mode) { + RunOptions_.PrintInfo("Run program..."); + status = program->RunWithConfig(RunOptions_.User, *optConfig); + } else if (ERunMode::Optimize == RunOptions_.Mode) { + RunOptions_.PrintInfo("Optimize program..."); + status = program->OptimizeWithConfig(RunOptions_.User, *optConfig); + } else if (ERunMode::Validate == RunOptions_.Mode) { + RunOptions_.PrintInfo("Validate program..."); + status = program->Validate(RunOptions_.User, RunOptions_.ExprStream, RunOptions_.WithTypes); + } else if (ERunMode::Discover == RunOptions_.Mode) { + RunOptions_.PrintInfo("Discover program..."); + status = program->Discover(RunOptions_.User); + } else if (ERunMode::Lineage == RunOptions_.Mode) { + RunOptions_.PrintInfo("Calculating lineage in program..."); + status = program->LineageWithConfig(RunOptions_.User, *optConfig); + } + + return status; +} + } // NYql diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h index 0b7e7a9e8d..f5e1fd5bd2 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h @@ -6,6 +6,8 @@ #include <yql/essentials/core/url_lister/interface/url_lister.h> #include <yql/essentials/core/yql_data_provider.h> #include <yql/essentials/core/yql_user_data.h> +#include <yql/essentials/core/facade/yql_facade.h> +#include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h> #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/yson/public.h> @@ -25,7 +27,6 @@ namespace NKikimr::NMiniKQL { namespace NYql { class TFileStorageConfig; class TGatewaysConfig; - class TProgramFactory; } namespace NYql::NProto { @@ -55,6 +56,12 @@ enum class EProgramType { Pg /* "pg" */, }; +enum class EQPlayerMode { + None /* "none" */, + Capture /* "capture" */, + Replay /* "replay" */, +}; + class TFacadeRunOptions { public: TFacadeRunOptions(); @@ -66,7 +73,11 @@ public: TString ProgramFile; TString ProgramText; TString User; + TString Token; ui64 MemLimit = 0; + EQPlayerMode QPlayerMode = EQPlayerMode::None; + TString OperationId; + TQContext QPlayerContext; THashSet<TString> SqlFlags; ui16 SyntaxVersion = 1; @@ -85,20 +96,16 @@ public: IOutputStream* TraceOptStream = nullptr; IOutputStream* ErrStream = &Cerr; - THolder<IOutputStream> ErrStreamHolder; IOutputStream* PlanStream = nullptr; - THolder<IOutputStream> PlanStreamHolder; IOutputStream* ExprStream = nullptr; - THolder<IOutputStream> ExprStreamHolder; IOutputStream* ResultStream = nullptr; - THolder<IOutputStream> ResultStreamHolder; IOutputStream* StatStream = nullptr; - THolder<IOutputStream> StatStreamHolder; NYql::TUserDataTable DataTable; TVector<TString> UdfsPaths; TString Params; NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::Greedy; + TCredentials::TPtr Credentials = MakeIntrusive<TCredentials>(); THashSet<TString> GatewayTypes; TString UdfResolverPath; @@ -118,6 +125,8 @@ public: bool TestSqlFormat = false; bool ValidateResultFormat = false; bool EnableResultPosition = false; + bool EnableCredentials = false; + bool EnableQPlayer = false; void Parse(int argc, const char *argv[]); @@ -139,14 +148,19 @@ private: std::vector<std::function<void(NLastGetopt::TOpts&)>> OptExtenders_; std::vector<std::function<void(const NLastGetopt::TOptsParseResult&)>> OptHandlers_; THashSet<TString> SupportedGateways_; + THolder<IOutputStream> ErrStreamHolder_; + THolder<IOutputStream> PlanStreamHolder_; + THolder<IOutputStream> ExprStreamHolder_; + THolder<IOutputStream> ResultStreamHolder_; + THolder<IOutputStream> StatStreamHolder_; + IQStoragePtr QPlayerStorage_; }; class TFacadeRunner { public: - TFacadeRunner(TString name) - : Name_(std::move(name)) - { - } + TFacadeRunner(TString name); + ~TFacadeRunner(); + int Main(int argc, const char *argv[]); void AddFsDownloadFactory(std::function<NFS::IDownloaderPtr()> factory) { @@ -180,19 +194,15 @@ public: TFileStoragePtr GetFileStorage() const { return FileStorage_; } - TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> GetFuncRegistry() { - return FuncRegistry_; - } - TCredentials::TPtr GetCredentials() { - return Credentials_; - } + TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> GetFuncRegistry(); TFacadeRunOptions& GetRunOptions() { return RunOptions_; } -private: - int DoMain(int argc, const char *argv[]); - int RunProgram(TProgramFactory& factory); +protected: + virtual int DoMain(int argc, const char *argv[]); + virtual int DoRun(TProgramFactory& factory); + virtual TProgram::TStatus DoRunProgram(TProgramPtr program); private: TString Name_; @@ -203,7 +213,6 @@ private: THolder<TFileStorageConfig> FileStorageConfig_; TFileStoragePtr FileStorage_; TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> FuncRegistry_; - TCredentials::TPtr Credentials_ = MakeIntrusive<TCredentials>(); TOperationProgressWriter ProgressWriter_; IPipelineConfigurator* OptPipelineConfigurator_ = nullptr; IPipelineConfigurator* PeepholePipelineConfigurator_ = nullptr; diff --git a/yql/tools/yqlrun/lib/yqlrun_lib.cpp b/yql/tools/yqlrun/lib/yqlrun_lib.cpp index d9db2a9048..fdb3d28457 100644 --- a/yql/tools/yqlrun/lib/yqlrun_lib.cpp +++ b/yql/tools/yqlrun/lib/yqlrun_lib.cpp @@ -58,8 +58,6 @@ TYqlRunTool::TYqlRunTool() TablesMapping_[name] = path; }, '@'); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("tables-dir", "Table dirs mapping").RequiredArgument("cluster@dir") .KVHandler([&](TString cluster, TString dir) { if (cluster.empty() || dir.empty()) { @@ -75,8 +73,6 @@ TYqlRunTool::TYqlRunTool() } }, '@'); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption('C', "cluster", "Cluster to service mapping").RequiredArgument("name@service") .KVHandler([&](TString cluster, TString provider) { if (cluster.empty() || provider.empty()) { @@ -85,29 +81,16 @@ TYqlRunTool::TYqlRunTool() AddClusterMapping(std::move(cluster), std::move(provider)); }, '@'); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("ndebug", "Do not show debug info in error output").NoArgument().SetFlag(&GetRunOptions().NoDebug); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("keep-temp", "Keep temporary tables").NoArgument().SetFlag(&KeepTemp_); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("show-progress", "Report operation progress").NoArgument() .Handler0([&]() { SetOperationProgressWriter([](const TOperationProgress& progress) { Cerr << "Operation: [" << progress.Category << "] " << progress.Id << ", state: " << progress.State << "\n"; }); }); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("tmp-dir", "Directory for temporary tables").RequiredArgument("DIR").StoreResult(&TmpDir_); - }); - - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("test-format", "Compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported)").NoArgument().SetFlag(&GetRunOptions().TestSqlFormat); - }); - GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) { opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&GetRunOptions().ValidateResultFormat); }); diff --git a/yt/yt/library/profiling/solomon/config.cpp b/yt/yt/library/profiling/solomon/config.cpp index 84c4bf2308..e714548804 100644 --- a/yt/yt/library/profiling/solomon/config.cpp +++ b/yt/yt/library/profiling/solomon/config.cpp @@ -92,6 +92,9 @@ void TSolomonExporterConfig::Register(TRegistrar registrar) .Default(DefaultProducerCollectionBatchSize) .GreaterThan(0); + registrar.Parameter("label_sanitization_policy", &TThis::LabelSanitizationPolicy) + .Default(ELabelSanitizationPolicy::None); + registrar.Postprocessor([] (TThis* config) { if (config->LingerTimeout.GetValue() % config->GridStep.GetValue() != 0) { THROW_ERROR_EXCEPTION("\"linger_timeout\" must be multiple of \"grid_step\""); diff --git a/yt/yt/library/profiling/solomon/config.h b/yt/yt/library/profiling/solomon/config.h index 0c36c1120d..42b5b4548e 100644 --- a/yt/yt/library/profiling/solomon/config.h +++ b/yt/yt/library/profiling/solomon/config.h @@ -74,6 +74,8 @@ struct TSolomonExporterConfig int ProducerCollectionBatchSize; + ELabelSanitizationPolicy LabelSanitizationPolicy; + TShardConfigPtr MatchShard(const std::string& sensorName); ESummaryPolicy GetSummaryPolicy() const; diff --git a/yt/yt/library/profiling/solomon/exporter.cpp b/yt/yt/library/profiling/solomon/exporter.cpp index d039924ab5..aee542660a 100644 --- a/yt/yt/library/profiling/solomon/exporter.cpp +++ b/yt/yt/library/profiling/solomon/exporter.cpp @@ -88,6 +88,7 @@ TSolomonExporter::TSolomonExporter( return shard->GridStep->GetValue() / config->GridStep.GetValue(); }); + Registry_->SetLabelSanitizationPolicy(Config_->LabelSanitizationPolicy); if (Config_->ReportBuildInfo) { TProfiler profiler{registry, ""}; diff --git a/yt/yt/library/profiling/solomon/public.h b/yt/yt/library/profiling/solomon/public.h index d497d9f9b9..a59e20b28e 100644 --- a/yt/yt/library/profiling/solomon/public.h +++ b/yt/yt/library/profiling/solomon/public.h @@ -1,6 +1,7 @@ #pragma once #include <library/cpp/yt/memory/ref_counted.h> +#include <library/cpp/yt/misc/enum.h> namespace NYT::NProfiling { @@ -18,4 +19,12 @@ DECLARE_REFCOUNTED_STRUCT(IEndpointProvider) //////////////////////////////////////////////////////////////////////////////// +DEFINE_ENUM(ELabelSanitizationPolicy, + ((None) (0)) + ((Weak) (1)) // Escape only zero symbol and trim label to 200 symbols + ((Strong) (2)) // Escape all forbidden symbols and trim label to 200 symbols +); + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NProfiling diff --git a/yt/yt/library/profiling/solomon/registry.cpp b/yt/yt/library/profiling/solomon/registry.cpp index 9fe71af15a..7147c2904c 100644 --- a/yt/yt/library/profiling/solomon/registry.cpp +++ b/yt/yt/library/profiling/solomon/registry.cpp @@ -281,6 +281,11 @@ void TSolomonRegistry::SetProducerCollectionBatchSize(int batchSize) Producers_.SetCollectionBatchSize(batchSize); } +void TSolomonRegistry::SetLabelSanitizationPolicy(ELabelSanitizationPolicy LabelSanitizationPolicy) +{ + Tags_.SetLabelSanitizationPolicy(LabelSanitizationPolicy); +} + int TSolomonRegistry::GetWindowSize() const { if (!WindowSize_) { diff --git a/yt/yt/library/profiling/solomon/registry.h b/yt/yt/library/profiling/solomon/registry.h index bddf548b67..cf4c149df0 100644 --- a/yt/yt/library/profiling/solomon/registry.h +++ b/yt/yt/library/profiling/solomon/registry.h @@ -129,6 +129,7 @@ public: void SetGridFactor(std::function<int(const std::string&)> gridFactor); void SetWindowSize(int windowSize); void SetProducerCollectionBatchSize(int batchSize); + void SetLabelSanitizationPolicy(ELabelSanitizationPolicy LabelSanitizationPolicy); void ProcessRegistrations(); void Collect(IInvokerPtr offloadInvoker = GetSyncInvoker()); void ReadSensors( diff --git a/yt/yt/library/profiling/solomon/tag_registry-inl.h b/yt/yt/library/profiling/solomon/tag_registry-inl.h new file mode 100644 index 0000000000..226aa5a1c1 --- /dev/null +++ b/yt/yt/library/profiling/solomon/tag_registry-inl.h @@ -0,0 +1,31 @@ +#ifndef TAG_REGISTRY_INL_H +#error "Direct inclusion of this file is not allowed, include tag_registry.h" +// For the sake of sane code completion. +#include "tag_registry.h" +#endif + +namespace NYT::NProfiling { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TTagPerfect> +TTagId TTagRegistry::EncodeSanitized(TTagPerfect&& tag) +{ + static_assert(std::is_same_v<std::remove_cvref_t<TTagPerfect>, TTag>); + + THashMap<TTag, TTagId>::insert_ctx insertCtx; + if (auto it = TagByName_.find(tag, insertCtx); it != TagByName_.end()) { + return it->second; + } else { + TTagId tagId = TagById_.size() + 1; + + TagByName_.emplace_direct(insertCtx, tag, tagId); + TagById_.push_back(std::forward<TTagPerfect>(tag)); + + return tagId; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NProfiling diff --git a/yt/yt/library/profiling/solomon/tag_registry.cpp b/yt/yt/library/profiling/solomon/tag_registry.cpp index 30d7f0d654..1a66d405a0 100644 --- a/yt/yt/library/profiling/solomon/tag_registry.cpp +++ b/yt/yt/library/profiling/solomon/tag_registry.cpp @@ -10,18 +10,30 @@ namespace NYT::NProfiling { //////////////////////////////////////////////////////////////////////////////// +namespace { + +static constexpr int MaxLabelSize = 200; +static constexpr int HalfMaxLabelSize = MaxLabelSize / 2; + +} + +//////////////////////////////////////////////////////////////////////////////// + +void TTagRegistry::SetLabelSanitizationPolicy(ELabelSanitizationPolicy labelSanitizationPolicy) +{ + LabelSanitizationPolicy_ = labelSanitizationPolicy; +} + +TTagIdList TTagRegistry::Encode(const TTagSet& tags) +{ + return Encode(tags.Tags()); +} + TTagIdList TTagRegistry::Encode(const TTagList& tags) { TTagIdList ids; - for (const auto& tag : tags) { - if (auto it = TagByName_.find(tag); it != TagByName_.end()) { - ids.push_back(it->second); - } else { - TagById_.push_back(tag); - TagByName_[tag] = TagById_.size(); - ids.push_back(TagById_.size()); - } + ids.push_back(Encode(tag)); } return ids; @@ -29,18 +41,17 @@ TTagIdList TTagRegistry::Encode(const TTagList& tags) TTagId TTagRegistry::Encode(const TTag& tag) { - if (auto it = TagByName_.find(tag); it != TagByName_.end()) { - return it->second; - } else { - TagById_.push_back(tag); - TagByName_[tag] = TagById_.size(); - return TagById_.size(); + if (LabelSanitizationPolicy_ == ELabelSanitizationPolicy::None) { + return EncodeSanitized(tag); } -} -TTagIdList TTagRegistry::Encode(const TTagSet& tags) -{ - return Encode(tags.Tags()); + if (auto sanitizeParameters = ScanForSanitize(tag.second); + sanitizeParameters.IsSanitizationRequired()) + { + return EncodeSanitized(SanitizeMonitoringTag(tag, sanitizeParameters.ResultingLength)); + } else { + return EncodeSanitized(tag); + } } TCompactVector<std::optional<TTagId>, TypicalTagCount> TTagRegistry::TryEncode(const TTagList& tags) const @@ -48,10 +59,17 @@ TCompactVector<std::optional<TTagId>, TypicalTagCount> TTagRegistry::TryEncode(c TCompactVector<std::optional<TTagId>, TypicalTagCount> ids; for (const auto& tag : tags) { - if (auto it = TagByName_.find(tag); it != TagByName_.end()) { - ids.push_back(it->second); + if (LabelSanitizationPolicy_ == ELabelSanitizationPolicy::None) { + ids.push_back(TryEncodeSanitized(tag)); + continue; + } + + if (auto sanitizeParameters = ScanForSanitize(tag.second); + sanitizeParameters.IsSanitizationRequired()) + { + ids.push_back(TryEncodeSanitized(SanitizeMonitoringTag(tag, sanitizeParameters.ResultingLength))); } else { - ids.push_back({}); + ids.push_back(TryEncodeSanitized(tag)); } } @@ -95,6 +113,110 @@ void TTagRegistry::DumpTags(NProto::TSensorDump* dump) //////////////////////////////////////////////////////////////////////////////// +bool TTagRegistry::TSanitizeParameters::IsSanitizationRequired() const +{ + return ForbiddenCharCount > 0 || ResultingLength > MaxLabelSize; +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TTagRegistry::IsAllowedMonitoringTagValueChar(unsigned char c) const +{ + switch (LabelSanitizationPolicy_) { + case ELabelSanitizationPolicy::Weak: + return c != '\0'; + + case ELabelSanitizationPolicy::Strong: + return 31 < c && + c < 127 && + c != '|' && + c != '*' && + c != '?' && + c != '"' && + c != '\'' && + c != '\\' && + c != '`'; + + default: + YT_ABORT(); + } +} + +TTagRegistry::TSanitizeParameters TTagRegistry::ScanForSanitize(const std::string& value) const +{ + YT_VERIFY(LabelSanitizationPolicy_ != ELabelSanitizationPolicy::None); + + int forbiddenCharCount = 0; + for (unsigned char c : value) { + forbiddenCharCount += static_cast<int>(!IsAllowedMonitoringTagValueChar(c)); + } + + return { + .ForbiddenCharCount = forbiddenCharCount, + .ResultingLength = static_cast<int>(value.size() + forbiddenCharCount * 2), + }; +} + +std::string TTagRegistry::SanitizeMonitoringTagValue(const std::string& value, int resultingLength) const +{ + bool needTrim = resultingLength > MaxLabelSize; + + std::string result; + result.resize(std::min(resultingLength, MaxLabelSize)); + + int resultIndex = 0; + for (int index = 0; resultIndex < (needTrim ? HalfMaxLabelSize : resultingLength); ++index) { + unsigned char c = value[index]; + + if (IsAllowedMonitoringTagValueChar(c)) { + result[resultIndex++] = c; + } else { + result[resultIndex++] = '%'; + result[resultIndex++] = IntToHexLowercase[c >> 4]; + result[resultIndex++] = IntToHexLowercase[c & 0x0f]; + } + } + + if (!needTrim) { + return result; + } + + resultIndex = MaxLabelSize - 1; + for (int index = ssize(value) - 1; resultIndex > HalfMaxLabelSize + 2; --index) { + unsigned char c = value[index]; + + if (IsAllowedMonitoringTagValueChar(value[index])) { + result[resultIndex--] = c; + } else { + result[resultIndex--] = IntToHexLowercase[c & 0x0f]; + result[resultIndex--] = IntToHexLowercase[c >> 4]; + result[resultIndex--] = '%'; + } + } + + result[HalfMaxLabelSize] = '.'; + result[HalfMaxLabelSize + 1] = '.'; + result[HalfMaxLabelSize + 2] = '.'; + + return result; +} + +TTag TTagRegistry::SanitizeMonitoringTag(const TTag& tag, int resultingLength) const +{ + return {tag.first, SanitizeMonitoringTagValue(tag.second, resultingLength)}; +} + +std::optional<TTagId> TTagRegistry::TryEncodeSanitized(const TTag& tag) const +{ + if (auto it = TagByName_.find(tag); it != TagByName_.end()) { + return it->second; + } else { + return std::nullopt; + } +} + +//////////////////////////////////////////////////////////////////////////////// + void TTagWriter::WriteLabel(TTagId tag) { if (static_cast<size_t>(tag) >= Cache_.size()) { diff --git a/yt/yt/library/profiling/solomon/tag_registry.h b/yt/yt/library/profiling/solomon/tag_registry.h index 8e76ae1175..e733046136 100644 --- a/yt/yt/library/profiling/solomon/tag_registry.h +++ b/yt/yt/library/profiling/solomon/tag_registry.h @@ -1,5 +1,7 @@ #pragma once +#include "public.h" + #include <yt/yt/core/profiling/public.h> #include <yt/yt/library/profiling/tag.h> @@ -16,6 +18,8 @@ namespace NYT::NProfiling { class TTagRegistry { public: + void SetLabelSanitizationPolicy(ELabelSanitizationPolicy labelSanitizationPolicy); + TTagIdList Encode(const TTagSet& tags); TTagIdList Encode(const TTagList& tags); TTagId Encode(const TTag& tag); @@ -30,6 +34,24 @@ public: void DumpTags(NProto::TSensorDump* dump); private: + struct TSanitizeParameters + { + int ForbiddenCharCount; + int ResultingLength; + + bool IsSanitizationRequired() const; + }; + + bool IsAllowedMonitoringTagValueChar(unsigned char c) const; + TSanitizeParameters ScanForSanitize(const std::string& value) const; + std::string SanitizeMonitoringTagValue(const std::string& value, int resultingLength) const; + TTag SanitizeMonitoringTag(const TTag& tag, int resultingLength) const; + + template <class TTagPerfect> + TTagId EncodeSanitized(TTagPerfect&& tag); + std::optional<TTagId> TryEncodeSanitized(const TTag& tag) const; + + ELabelSanitizationPolicy LabelSanitizationPolicy_ = ELabelSanitizationPolicy::None; // TODO(prime@): maybe do something about the fact that tags are never freed. THashMap<TTag, TTagId> TagByName_; std::deque<TTag> TagById_; @@ -60,3 +82,7 @@ private: //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NProfiling + +#define TAG_REGISTRY_INL_H +#include "tag_registry-inl.h" +#undef TAG_REGISTRY_INL_H diff --git a/yt/yt/library/profiling/unittests/solomon_ut.cpp b/yt/yt/library/profiling/unittests/solomon_ut.cpp index c26d478234..8addff92fc 100644 --- a/yt/yt/library/profiling/unittests/solomon_ut.cpp +++ b/yt/yt/library/profiling/unittests/solomon_ut.cpp @@ -17,6 +17,10 @@ namespace { //////////////////////////////////////////////////////////////////////////////// +using namespace std::string_literals; + +//////////////////////////////////////////////////////////////////////////////// + struct TTestMetricConsumer : public NMonitoring::IMetricConsumer { @@ -962,6 +966,132 @@ TEST_P(TOmitNameLabelSuffixTest, GaugeSummary) ASSERT_NEAR(gauges[Format("yt.davg%v{}", omitNameLabelSuffix ? "" : ".avg")], 40 + 1 / 3.0, 1e-6); } +struct TTagInfo +{ + std::string TagValue; + std::string EncodedTagValue; +}; + +using TTagInfoMapping = THashMap<std::string, TTagInfo>; + +TTagInfo GetLongTag() { + TString longTag; + longTag.reserve(210); + for (int index = 0; index < 210; ++index) { + longTag.append('a' + index % 26); + } + TString longTagEncoded; + longTagEncoded.reserve(200); + longTagEncoded.append(longTag.begin(), 100).append("..."); + for (int index = 103; index < 200; ++index) { + longTagEncoded.append('a' + (index - 103 + 9) % 26); + } + + return { + .TagValue = std::move(longTag), + .EncodedTagValue = std::move(longTagEncoded), + }; +} + +void CheckTags(const TSolomonRegistryPtr& registry, const TTagInfoMapping& tagInfoMapping) { + int tagNameLength = tagInfoMapping.begin()->first.size(); + + TTagSet tagSet; + for (const auto& [tagName, tagInfo] : tagInfoMapping) { + ASSERT_EQ(ssize(tagName), tagNameLength); + tagSet.AddTag({tagName, tagInfo.TagValue}); + } + + auto profiler = TProfiler(registry, "/debug") + .WithTags(tagSet); + auto c0 = profiler.Counter("/c"); + c0.Increment(1); + + auto result = CollectSensors(registry); + + for (const auto& label : result.Labels) { + // Label has structure - "labelName=labelValue" + auto tagName = label.substr(0, tagNameLength); + auto tagValue = label.substr(tagNameLength + 1, label.size() - tagNameLength - 1); + + ASSERT_EQ(tagValue, tagInfoMapping.at(tagName).EncodedTagValue); + } +} + +TEST(TSolomonRegistry, IncorrectSolomonLabelsWeakPolicy) +{ + auto impl = New<TSolomonRegistry>(); + impl->SetWindowSize(12); + impl->SetLabelSanitizationPolicy(ELabelSanitizationPolicy::Weak); + + CheckTags( + impl, + { + { + "tag0", + GetLongTag(), + }, + { + "tag1", + { + .TagValue = "a\0aa|*?\"'\\`b\0b\xff"s, + .EncodedTagValue = "a%00aa|*?\"'\\`b%00b\xff", + }, + }, + { + "tag2", + { + .TagValue = TString(98, 'a') + .append('\0') + .append(100, 'a') + .append('\0') + .append('\xff'), + .EncodedTagValue = TString(98, 'a') + .append("%0...") + .append(93, 'a') + .append("%00") + .append('\xff') + }, + }, + }); +} + +TEST(TSolomonRegistry, IncorrectSolomonLabelsStrongPolicy) +{ + auto impl = New<TSolomonRegistry>(); + impl->SetWindowSize(12); + impl->SetLabelSanitizationPolicy(ELabelSanitizationPolicy::Strong); + + CheckTags( + impl, + { + { + "tag0", + GetLongTag(), + }, + { + "tag1", + { + .TagValue = "a\0aa|*?\"'\\`bb\xff\0"s, + .EncodedTagValue = "a%00aa%7c%2a%3f%22%27%5c%60bb%ff%00", + }, + }, + { + "tag2", + { + .TagValue = TString(98, 'a') + .append('\xff') + .append(100, 'a') + .append('\0'), + .EncodedTagValue = TString(98, 'a') + .append("%f...") + .append(94, 'a') + .append("%00"), + }, + }, + }); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace |