aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2024-12-17 14:13:00 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2024-12-17 14:31:07 +0300
commit3f61c3b13253c98c4d3663c76de3357d3ae04aa7 (patch)
tree6c3eebed23313391ff13e8914a8e4f0d447044cf
parent30272794b11546a1e358cac78fc561ad1d4ef01b (diff)
downloadydb-3f61c3b13253c98c4d3663c76de3357d3ae04aa7.tar.gz
Intermediate changes
commit_hash:5d5aa527dd6fcb0dfb1e62748135e5f04f92b1b0
-rw-r--r--yql/essentials/tools/yql_facade_run/ya.make2
-rw-r--r--yql/essentials/tools/yql_facade_run/yql_facade_run.cpp230
-rw-r--r--yql/essentials/tools/yql_facade_run/yql_facade_run.h49
-rw-r--r--yql/tools/yqlrun/lib/yqlrun_lib.cpp17
-rw-r--r--yt/yt/library/profiling/solomon/config.cpp3
-rw-r--r--yt/yt/library/profiling/solomon/config.h2
-rw-r--r--yt/yt/library/profiling/solomon/exporter.cpp1
-rw-r--r--yt/yt/library/profiling/solomon/public.h9
-rw-r--r--yt/yt/library/profiling/solomon/registry.cpp5
-rw-r--r--yt/yt/library/profiling/solomon/registry.h1
-rw-r--r--yt/yt/library/profiling/solomon/tag_registry-inl.h31
-rw-r--r--yt/yt/library/profiling/solomon/tag_registry.cpp164
-rw-r--r--yt/yt/library/profiling/solomon/tag_registry.h26
-rw-r--r--yt/yt/library/profiling/unittests/solomon_ut.cpp130
14 files changed, 546 insertions, 124 deletions
diff --git a/yql/essentials/tools/yql_facade_run/ya.make b/yql/essentials/tools/yql_facade_run/ya.make
index e9d93e235d..e67e0b6c7f 100644
--- a/yql/essentials/tools/yql_facade_run/ya.make
+++ b/yql/essentials/tools/yql_facade_run/ya.make
@@ -21,6 +21,8 @@ PEERDIR(
yql/essentials/core/url_lister
yql/essentials/core/url_preprocessing
yql/essentials/core/peephole_opt
+ yql/essentials/core/qplayer/storage/interface
+ yql/essentials/core/qplayer/storage/file
yql/essentials/core
yql/essentials/minikql/invoke_builtins
yql/essentials/minikql
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
index 91a3a3eaf8..6e6167582f 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
@@ -21,6 +21,7 @@
#include <yql/essentials/core/facade/yql_facade.h>
#include <yql/essentials/core/url_lister/url_lister_manager.h>
#include <yql/essentials/core/url_preprocessing/url_preprocessing.h>
+#include <yql/essentials/core/qplayer/storage/file/yql_qstorage_file.h>
#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h>
#include <yql/essentials/minikql/mkql_function_registry.h>
#include <yql/essentials/ast/yql_expr.h>
@@ -51,9 +52,11 @@
#include <util/stream/file.h>
#include <util/stream/null.h>
#include <util/system/user.h>
+#include <util/system/env.h>
#include <util/string/split.h>
#include <util/string/join.h>
#include <util/string/builder.h>
+#include <util/string/strip.h>
#include <util/generic/vector.h>
#include <util/generic/ptr.h>
#include <util/generic/yexception.h>
@@ -155,7 +158,6 @@ public:
namespace NYql {
TFacadeRunOptions::TFacadeRunOptions() {
- User = GetUsername();
}
TFacadeRunOptions::~TFacadeRunOptions() {
@@ -185,11 +187,23 @@ void TFacadeRunOptions::PrintInfo(const TString& msg) {
}
void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
+ User = GetUsername();
+
+ if (EnableCredentials) {
+ Token = GetEnv("YQL_TOKEN");
+ if (!Token) {
+ const TString home = GetEnv("HOME");
+ auto tokenPath = TFsPath(home) / ".yql" / "token";
+ if (tokenPath.Exists()) {
+ Token = StripStringRight(TFileInput(tokenPath).ReadAll());
+ }
+ }
+ }
NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
opts.AddHelpOption();
- opts.AddLongOption('p', "program", "Program file").Required().RequiredArgument("FILE")
+ opts.AddLongOption('p', "program", "Program file (use - to read from stdin)").Required().RequiredArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
ProgramFile = file;
if (ProgramFile == "-") {
@@ -203,7 +217,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
opts.AddLongOption('s', "sql", "Program is SQL query").NoArgument().StoreValue(&ProgramType, EProgramType::Sql);
if (PgSupport) {
opts.AddLongOption("pg", "Program has PG syntax").NoArgument().StoreValue(&ProgramType, EProgramType::Pg);
- opts.AddLongOption("pg-ext", "pg extensions config file").Optional().RequiredArgument("FILE")
+ opts.AddLongOption("pg-ext", "Pg extensions config file").Optional().RequiredArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
PgExtConfig = ParseProtoConfig<NProto::TPgExtensions>(file);
});
@@ -238,7 +252,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
.Handler1T<TString>([this](const TString& file) {
Params = TFileInput(file).ReadAll();
});
- opts.AddLongOption('G', "gateways", "Used gateways").DefaultValue(JoinSeq(",", SupportedGateways_))
+ opts.AddLongOption('G', "gateways", TStringBuilder() << "Used gateways, available: " << JoinSeq(",", SupportedGateways_)).DefaultValue(JoinSeq(",", GatewayTypes))
.Handler1T<TString>([this](const TString& gateways) {
::StringSplitter(gateways).Split(',').Consume([&](const TStringBuf& val) {
if (!SupportedGateways_.contains(val)) {
@@ -256,8 +270,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
FsConfig = MakeHolder<TFileStorageConfig>();
LoadFsConfigFromFile(file, *FsConfig);
});
- opts.AddLongOption('u', "udf", "Load shared library with UDF by given path").AppendTo(&UdfsPaths);
- opts.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory")
+ opts.AddLongOption('u', "udf", "Load shared library with UDF by given path").RequiredArgument("PATH").AppendTo(&UdfsPaths);
+ opts.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory").RequiredArgument("DIR")
.Handler1T<TString>([this](const TString& dir) {
NKikimr::NMiniKQL::FindUdfsInDir(dir, &UdfsPaths);
});
@@ -265,14 +279,14 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
opts.AddLongOption("udf-resolver-filter-syscalls", "Filter syscalls in udf resolver").Optional().NoArgument().SetFlag(&UdfResolverFilterSyscalls);
opts.AddLongOption("scan-udfs", "Scan specified udfs with external udf-resolver to use static function registry").NoArgument().SetFlag(&ScanUdfs);
- opts.AddLongOption("parse-only", "Exit after program has been parsed").NoArgument().StoreValue(&Mode, ERunMode::Parse);
- opts.AddLongOption("compile-only", "Exit after program has been compiled").NoArgument().StoreValue(&Mode, ERunMode::Compile);
- opts.AddLongOption("validate", "Exit after program has been validated").NoArgument().StoreValue(&Mode, ERunMode::Validate);
- opts.AddLongOption("lineage", "Exit after data lineage has been calculated").NoArgument().StoreValue(&Mode, ERunMode::Lineage);
- opts.AddLongOption('O',"optimize", "Optimize expression").NoArgument().StoreValue(&Mode, ERunMode::Optimize);
- opts.AddLongOption('R',"run", "Run expression using input/output tables").NoArgument().StoreValue(&Mode, ERunMode::Run);
- opts.AddLongOption('D', "discover", "Discover tables in the program").NoArgument().StoreValue(&Mode, ERunMode::Discover);
- opts.AddLongOption("peephole", "Perform peephole stage of expression using input/output tables").NoArgument().StoreValue(&Mode, ERunMode::Peephole);
+ opts.AddLongOption("parse-only", "Parse program and exit").NoArgument().StoreValue(&Mode, ERunMode::Parse);
+ opts.AddLongOption("compile-only", "Compiled program and exit").NoArgument().StoreValue(&Mode, ERunMode::Compile);
+ opts.AddLongOption("validate", "Validate program and exit").NoArgument().StoreValue(&Mode, ERunMode::Validate);
+ opts.AddLongOption("lineage", "Calculate program lineage and exit").NoArgument().StoreValue(&Mode, ERunMode::Lineage);
+ opts.AddLongOption('O',"optimize", "Optimize program and exir").NoArgument().StoreValue(&Mode, ERunMode::Optimize);
+ opts.AddLongOption('D', "discover", "Discover tables in the program and exit").NoArgument().StoreValue(&Mode, ERunMode::Discover);
+ opts.AddLongOption("peephole", "Perform peephole program optimization and exit").NoArgument().StoreValue(&Mode, ERunMode::Peephole);
+ opts.AddLongOption('R',"run", "Run progrum (use by default)").NoArgument().StoreValue(&Mode, ERunMode::Run);
opts.AddLongOption('L', "show-log", "Show transformation log").Optional().NoArgument().SetFlag(&ShowLog);
opts.AddLongOption('v', "verbosity", "Log verbosity level").Optional().RequiredArgument("LEVEL").StoreResult(&Verbosity);
@@ -290,8 +304,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
});
opts.AddLongOption("expr-file", "Print AST to that file instead of stdout").Optional().RequiredArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
- ExprStreamHolder = MakeHolder<TFixedBufferFileOutput>(file);
- ExprStream = ExprStreamHolder.Get();
+ ExprStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file);
+ ExprStream = ExprStreamHolder_.Get();
});
opts.AddLongOption("print-result", "Print program execution result to stdout").NoArgument()
.Handler0([this]() {
@@ -317,8 +331,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
opts.AddLongOption("result-file", "Print program execution result to file").Optional().RequiredArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
- ResultStreamHolder = MakeHolder<TFixedBufferFileOutput>(file);
- ResultStream = ResultStreamHolder.Get();
+ ResultStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file);
+ ResultStream = ResultStreamHolder_.Get();
});
opts.AddLongOption('P',"trace-plan", "Print plan before execution").NoArgument()
.Handler0([this]() {
@@ -328,13 +342,13 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
});
opts.AddLongOption("plan-file", "Print program plan to file").Optional().RequiredArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
- PlanStreamHolder = MakeHolder<TFixedBufferFileOutput>(file);
- PlanStream = PlanStreamHolder.Get();
+ PlanStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file);
+ PlanStream = PlanStreamHolder_.Get();
});
opts.AddLongOption("err-file", "Print validate/optimize/runtime errors to file")
.Handler1T<TString>([this](const TString& file) {
- ErrStreamHolder = MakeHolder<TFixedBufferFileOutput>(file);
- ErrStream = ErrStreamHolder.Get();
+ ErrStreamHolder_ = MakeHolder<TFixedBufferFileOutput>(file);
+ ErrStream = ErrStreamHolder_.Get();
});
opts.AddLongOption("full-expr", "Avoid buffering of expr/plan").NoArgument().SetFlag(&FullExpr);
opts.AddLongOption("mem-limit", "Set memory limit in megabytes")
@@ -366,8 +380,8 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
opts.AddLongOption("stat", "Print execution statistics").Optional().OptionalArgument("FILE")
.Handler1T<TString>([this](const TString& file) {
if (file) {
- StatStreamHolder = MakeHolder<TFileOutput>(file);
- StatStream = StatStreamHolder.Get();
+ StatStreamHolder_ = MakeHolder<TFileOutput>(file);
+ StatStream = StatStreamHolder_.Get();
} else {
StatStream = &Cerr;
}
@@ -396,6 +410,42 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
}
});
}
+ if (EnableCredentials) {
+ opts.AddLongOption("token", "YQL token")
+ .Optional()
+ .RequiredArgument("VALUE")
+ .StoreResult(&Token);
+ opts.AddLongOption("custom-tokens", "Custom tokens")
+ .Optional()
+ .RequiredArgument("NAME=VALUE or NAME=@PATH")
+ .KVHandler([this](TString key, TString value) {
+ if (value.StartsWith('@')) {
+ value = StripStringRight(TFileInput(value.substr(1)).ReadAll());
+ }
+ Credentials->AddCredential(key, TCredential("custom", "", value));
+ });
+ }
+ if (EnableQPlayer) {
+ opts.AddLongOption("qstorage-dir", "Directory for QStorage").RequiredArgument("DIR")
+ .Handler1T<TString>([this](const TString& dir) {
+ QPlayerStorage_ = MakeFileQStorage(dir);
+ });
+ opts.AddLongOption("op-id", "QStorage operation id").StoreResult(&OperationId).DefaultValue("dummy_op");
+ opts.AddLongOption("capture", "Write query metadata to QStorage").NoArgument()
+ .Handler0([this]() {
+ if (EQPlayerMode::Replay == QPlayerMode) {
+ throw yexception() << "replay and capture options can't be used simultaneously";
+ }
+ QPlayerMode = EQPlayerMode::Capture;
+ });
+ opts.AddLongOption("replay", "Read query metadata from QStorage").NoArgument()
+ .Handler0([this]() {
+ if (EQPlayerMode::Capture == QPlayerMode) {
+ throw yexception() << "replay and capture options can't be used simultaneously";
+ }
+ QPlayerMode = EQPlayerMode::Replay;
+ });
+ }
opts.SetFreeArgsMax(0);
@@ -405,8 +455,17 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
auto res = NLastGetopt::TOptsParseResult(&opts, argc, argv);
- for (auto& handle: OptHandlers_) {
- handle(res);
+ if (QPlayerMode != EQPlayerMode::None) {
+ if (!QPlayerStorage_) {
+ QPlayerStorage_ = MakeFileQStorage(".");
+ }
+ if (EQPlayerMode::Replay == QPlayerMode) {
+ QPlayerContext = TQContext(QPlayerStorage_->MakeReader(OperationId, {}));
+ ProgramFile = "-replay-";
+ ProgramText = "";
+ } else if (EQPlayerMode::Capture == QPlayerMode) {
+ QPlayerContext = TQContext(QPlayerStorage_->MakeWriter(OperationId, {}));
+ }
}
if (Mode >= ERunMode::Validate && GatewayTypes.empty()) {
@@ -420,6 +479,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
if (GatewaysConfig && GatewaysConfig->HasSqlCore()) {
SqlFlags.insert(GatewaysConfig->GetSqlCore().GetTranslationFlags().begin(), GatewaysConfig->GetSqlCore().GetTranslationFlags().end());
}
+ UpdateSqlFlagsFromQContext(QPlayerContext, SqlFlags);
if (!FsConfig) {
FsConfig = MakeHolder<TFileStorageConfig>();
@@ -427,6 +487,28 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
LoadFsConfigFromResource("fs.conf", *FsConfig);
}
}
+
+ if (EnableCredentials && Token) {
+ for (auto name: SupportedGateways_) {
+ Credentials->AddCredential(TStringBuilder() << "default_" << name, TCredential(name, "", Token));
+ }
+ }
+
+ for (auto& handle: OptHandlers_) {
+ handle(res);
+ }
+}
+
+TFacadeRunner::TFacadeRunner(TString name)
+ : Name_(std::move(name))
+{
+}
+
+TFacadeRunner::~TFacadeRunner() {
+}
+
+TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> TFacadeRunner::GetFuncRegistry() {
+ return FuncRegistry_;
}
int TFacadeRunner::Main(int argc, const char *argv[]) {
@@ -463,7 +545,7 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
if (RunOptions_.PgExtConfig) {
TVector<NPg::TExtensionDesc> extensions;
PgExtensionsFromProto(*RunOptions_.PgExtConfig, extensions);
- NPg::RegisterExtensions(extensions, false,
+ NPg::RegisterExtensions(extensions, RunOptions_.QPlayerContext.CanRead(),
*NSQLTranslationPG::CreateExtensionSqlParser(),
NKikimr::NMiniKQL::CreateExtensionLoader().get());
}
@@ -503,7 +585,9 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
if (RunOptions_.Mode >= ERunMode::Validate) {
std::vector<NFS::IDownloaderPtr> downloaders;
for (auto& factory: FsDownloadFactories_) {
- downloaders.push_back(factory());
+ if (auto download = factory()) {
+ downloaders.push_back(std::move(download));
+ }
}
FileStorage_ = WithAsync(CreateFileStorage(*RunOptions_.FsConfig, downloaders));
@@ -537,12 +621,16 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
dataProvidersInit.push_back(GetPgDataProviderInitializer());
}
for (auto& factory: ProviderFactories_) {
- dataProvidersInit.push_back(factory());
+ if (auto init = factory()) {
+ dataProvidersInit.push_back(std::move(init));
+ }
}
TVector<IUrlListerPtr> urlListers;
for (auto& factory: UrlListerFactories_) {
- urlListers.push_back(factory());
+ if (auto listener = factory()) {
+ urlListers.push_back(std::move(listener));
+ }
}
TProgramFactory factory(RunOptions_.UseRepeatableRandomAndTimeProviders, FuncRegistry_.Get(), ctx.NextUniqueId, dataProvidersInit, Name_);
@@ -555,18 +643,22 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
factory.SetUdfIndex(udfIndex, new TUdfIndexPackageSet());
factory.SetUdfResolver(udfResolver);
factory.SetGatewaysConfig(RunOptions_.GatewaysConfig.Get());
- factory.SetCredentials(Credentials_);
+ factory.SetCredentials(RunOptions_.Credentials);
factory.EnableRangeComputeFor();
if (!urlListers.empty()) {
factory.SetUrlListerManager(MakeUrlListerManager(urlListers));
}
- return RunProgram(factory);
+ int result = DoRun(factory);
+ if (result == 0 && EQPlayerMode::Capture == RunOptions_.QPlayerMode) {
+ RunOptions_.QPlayerContext.GetWriter()->Commit().GetValueSync();
+ }
+ return result;
}
-int TFacadeRunner::RunProgram(TProgramFactory& factory) {
+int TFacadeRunner::DoRun(TProgramFactory& factory) {
- TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText);;
+ TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText, RunOptions_.OperationId, EHiddenMode::Disable, RunOptions_.QPlayerContext);;
if (RunOptions_.Params) {
program->SetParametersYson(RunOptions_.Params);
}
@@ -673,40 +765,15 @@ int TFacadeRunner::RunProgram(TProgramFactory& factory) {
return 0;
}
- auto defOptConfig = TOptPipelineConfigurator(program, RunOptions_.FullExpr ? RunOptions_.PlanStream : nullptr, RunOptions_.FullExpr ? RunOptions_.ExprStream : nullptr, RunOptions_.WithTypes);
- IPipelineConfigurator* optConfig = OptPipelineConfigurator_ ? OptPipelineConfigurator_ : &defOptConfig;
-
- TProgram::TStatus status = TProgram::TStatus::Ok;
- if (ERunMode::Peephole == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Peephole...");
- auto defConfig = TPeepHolePipelineConfigurator();
- IPipelineConfigurator* config = PeepholePipelineConfigurator_ ? PeepholePipelineConfigurator_ : &defConfig;
- status = program->OptimizeWithConfig(RunOptions_.User, *config);
+ TProgram::TStatus status = DoRunProgram(program);
- if (RunOptions_.ExprStream && program->ExprRoot()) {
- auto ast = ConvertToAst(*program->ExprRoot(), program->ExprCtx(), RunOptions_.WithTypes ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true);
- ui32 prettyFlags = TAstPrintFlags::ShortQuote;
- if (!RunOptions_.WithTypes) {
- prettyFlags |= TAstPrintFlags::PerLine;
- }
- ast.Root->PrettyPrintTo(*RunOptions_.ExprStream, prettyFlags);
+ if (ERunMode::Peephole == RunOptions_.Mode && RunOptions_.ExprStream && program->ExprRoot()) {
+ auto ast = ConvertToAst(*program->ExprRoot(), program->ExprCtx(), RunOptions_.WithTypes ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true);
+ ui32 prettyFlags = TAstPrintFlags::ShortQuote;
+ if (!RunOptions_.WithTypes) {
+ prettyFlags |= TAstPrintFlags::PerLine;
}
-
- } else if (ERunMode::Run == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Run program...");
- status = program->RunWithConfig(RunOptions_.User, *optConfig);
- } else if (ERunMode::Optimize == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Optimize program...");
- status = program->OptimizeWithConfig(RunOptions_.User, *optConfig);
- } else if (ERunMode::Validate == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Validate program...");
- status = program->Validate(RunOptions_.User, RunOptions_.ExprStream, RunOptions_.WithTypes);
- } else if (ERunMode::Discover == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Discover program...");
- status = program->Discover(RunOptions_.User);
- } else if (ERunMode::Lineage == RunOptions_.Mode) {
- RunOptions_.PrintInfo("Calculating lineage in program...");
- status = program->LineageWithConfig(RunOptions_.User, *optConfig);
+ ast.Root->PrettyPrintTo(*RunOptions_.ExprStream, prettyFlags);
}
if (RunOptions_.WithFinalIssues) {
@@ -776,4 +843,35 @@ int TFacadeRunner::RunProgram(TProgramFactory& factory) {
return 0;
}
+TProgram::TStatus TFacadeRunner::DoRunProgram(TProgramPtr program) {
+ TProgram::TStatus status = TProgram::TStatus::Ok;
+
+ auto defOptConfig = TOptPipelineConfigurator(program, RunOptions_.FullExpr ? RunOptions_.PlanStream : nullptr, RunOptions_.FullExpr ? RunOptions_.ExprStream : nullptr, RunOptions_.WithTypes);
+ IPipelineConfigurator* optConfig = OptPipelineConfigurator_ ? OptPipelineConfigurator_ : &defOptConfig;
+
+ if (ERunMode::Peephole == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Peephole...");
+ auto defConfig = TPeepHolePipelineConfigurator();
+ IPipelineConfigurator* config = PeepholePipelineConfigurator_ ? PeepholePipelineConfigurator_ : &defConfig;
+ status = program->OptimizeWithConfig(RunOptions_.User, *config);
+ } else if (ERunMode::Run == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Run program...");
+ status = program->RunWithConfig(RunOptions_.User, *optConfig);
+ } else if (ERunMode::Optimize == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Optimize program...");
+ status = program->OptimizeWithConfig(RunOptions_.User, *optConfig);
+ } else if (ERunMode::Validate == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Validate program...");
+ status = program->Validate(RunOptions_.User, RunOptions_.ExprStream, RunOptions_.WithTypes);
+ } else if (ERunMode::Discover == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Discover program...");
+ status = program->Discover(RunOptions_.User);
+ } else if (ERunMode::Lineage == RunOptions_.Mode) {
+ RunOptions_.PrintInfo("Calculating lineage in program...");
+ status = program->LineageWithConfig(RunOptions_.User, *optConfig);
+ }
+
+ return status;
+}
+
} // NYql
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
index 0b7e7a9e8d..f5e1fd5bd2 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
@@ -6,6 +6,8 @@
#include <yql/essentials/core/url_lister/interface/url_lister.h>
#include <yql/essentials/core/yql_data_provider.h>
#include <yql/essentials/core/yql_user_data.h>
+#include <yql/essentials/core/facade/yql_facade.h>
+#include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h>
#include <library/cpp/getopt/last_getopt.h>
#include <library/cpp/yson/public.h>
@@ -25,7 +27,6 @@ namespace NKikimr::NMiniKQL {
namespace NYql {
class TFileStorageConfig;
class TGatewaysConfig;
- class TProgramFactory;
}
namespace NYql::NProto {
@@ -55,6 +56,12 @@ enum class EProgramType {
Pg /* "pg" */,
};
+enum class EQPlayerMode {
+ None /* "none" */,
+ Capture /* "capture" */,
+ Replay /* "replay" */,
+};
+
class TFacadeRunOptions {
public:
TFacadeRunOptions();
@@ -66,7 +73,11 @@ public:
TString ProgramFile;
TString ProgramText;
TString User;
+ TString Token;
ui64 MemLimit = 0;
+ EQPlayerMode QPlayerMode = EQPlayerMode::None;
+ TString OperationId;
+ TQContext QPlayerContext;
THashSet<TString> SqlFlags;
ui16 SyntaxVersion = 1;
@@ -85,20 +96,16 @@ public:
IOutputStream* TraceOptStream = nullptr;
IOutputStream* ErrStream = &Cerr;
- THolder<IOutputStream> ErrStreamHolder;
IOutputStream* PlanStream = nullptr;
- THolder<IOutputStream> PlanStreamHolder;
IOutputStream* ExprStream = nullptr;
- THolder<IOutputStream> ExprStreamHolder;
IOutputStream* ResultStream = nullptr;
- THolder<IOutputStream> ResultStreamHolder;
IOutputStream* StatStream = nullptr;
- THolder<IOutputStream> StatStreamHolder;
NYql::TUserDataTable DataTable;
TVector<TString> UdfsPaths;
TString Params;
NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::Greedy;
+ TCredentials::TPtr Credentials = MakeIntrusive<TCredentials>();
THashSet<TString> GatewayTypes;
TString UdfResolverPath;
@@ -118,6 +125,8 @@ public:
bool TestSqlFormat = false;
bool ValidateResultFormat = false;
bool EnableResultPosition = false;
+ bool EnableCredentials = false;
+ bool EnableQPlayer = false;
void Parse(int argc, const char *argv[]);
@@ -139,14 +148,19 @@ private:
std::vector<std::function<void(NLastGetopt::TOpts&)>> OptExtenders_;
std::vector<std::function<void(const NLastGetopt::TOptsParseResult&)>> OptHandlers_;
THashSet<TString> SupportedGateways_;
+ THolder<IOutputStream> ErrStreamHolder_;
+ THolder<IOutputStream> PlanStreamHolder_;
+ THolder<IOutputStream> ExprStreamHolder_;
+ THolder<IOutputStream> ResultStreamHolder_;
+ THolder<IOutputStream> StatStreamHolder_;
+ IQStoragePtr QPlayerStorage_;
};
class TFacadeRunner {
public:
- TFacadeRunner(TString name)
- : Name_(std::move(name))
- {
- }
+ TFacadeRunner(TString name);
+ ~TFacadeRunner();
+
int Main(int argc, const char *argv[]);
void AddFsDownloadFactory(std::function<NFS::IDownloaderPtr()> factory) {
@@ -180,19 +194,15 @@ public:
TFileStoragePtr GetFileStorage() const {
return FileStorage_;
}
- TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> GetFuncRegistry() {
- return FuncRegistry_;
- }
- TCredentials::TPtr GetCredentials() {
- return Credentials_;
- }
+ TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> GetFuncRegistry();
TFacadeRunOptions& GetRunOptions() {
return RunOptions_;
}
-private:
- int DoMain(int argc, const char *argv[]);
- int RunProgram(TProgramFactory& factory);
+protected:
+ virtual int DoMain(int argc, const char *argv[]);
+ virtual int DoRun(TProgramFactory& factory);
+ virtual TProgram::TStatus DoRunProgram(TProgramPtr program);
private:
TString Name_;
@@ -203,7 +213,6 @@ private:
THolder<TFileStorageConfig> FileStorageConfig_;
TFileStoragePtr FileStorage_;
TIntrusivePtr<NKikimr::NMiniKQL::IFunctionRegistry> FuncRegistry_;
- TCredentials::TPtr Credentials_ = MakeIntrusive<TCredentials>();
TOperationProgressWriter ProgressWriter_;
IPipelineConfigurator* OptPipelineConfigurator_ = nullptr;
IPipelineConfigurator* PeepholePipelineConfigurator_ = nullptr;
diff --git a/yql/tools/yqlrun/lib/yqlrun_lib.cpp b/yql/tools/yqlrun/lib/yqlrun_lib.cpp
index d9db2a9048..fdb3d28457 100644
--- a/yql/tools/yqlrun/lib/yqlrun_lib.cpp
+++ b/yql/tools/yqlrun/lib/yqlrun_lib.cpp
@@ -58,8 +58,6 @@ TYqlRunTool::TYqlRunTool()
TablesMapping_[name] = path;
}, '@');
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("tables-dir", "Table dirs mapping").RequiredArgument("cluster@dir")
.KVHandler([&](TString cluster, TString dir) {
if (cluster.empty() || dir.empty()) {
@@ -75,8 +73,6 @@ TYqlRunTool::TYqlRunTool()
}
}, '@');
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption('C', "cluster", "Cluster to service mapping").RequiredArgument("name@service")
.KVHandler([&](TString cluster, TString provider) {
if (cluster.empty() || provider.empty()) {
@@ -85,29 +81,16 @@ TYqlRunTool::TYqlRunTool()
AddClusterMapping(std::move(cluster), std::move(provider));
}, '@');
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("ndebug", "Do not show debug info in error output").NoArgument().SetFlag(&GetRunOptions().NoDebug);
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("keep-temp", "Keep temporary tables").NoArgument().SetFlag(&KeepTemp_);
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("show-progress", "Report operation progress").NoArgument()
.Handler0([&]() {
SetOperationProgressWriter([](const TOperationProgress& progress) {
Cerr << "Operation: [" << progress.Category << "] " << progress.Id << ", state: " << progress.State << "\n";
});
});
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("tmp-dir", "Directory for temporary tables").RequiredArgument("DIR").StoreResult(&TmpDir_);
- });
-
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("test-format", "Compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported)").NoArgument().SetFlag(&GetRunOptions().TestSqlFormat);
- });
- GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&GetRunOptions().ValidateResultFormat);
});
diff --git a/yt/yt/library/profiling/solomon/config.cpp b/yt/yt/library/profiling/solomon/config.cpp
index 84c4bf2308..e714548804 100644
--- a/yt/yt/library/profiling/solomon/config.cpp
+++ b/yt/yt/library/profiling/solomon/config.cpp
@@ -92,6 +92,9 @@ void TSolomonExporterConfig::Register(TRegistrar registrar)
.Default(DefaultProducerCollectionBatchSize)
.GreaterThan(0);
+ registrar.Parameter("label_sanitization_policy", &TThis::LabelSanitizationPolicy)
+ .Default(ELabelSanitizationPolicy::None);
+
registrar.Postprocessor([] (TThis* config) {
if (config->LingerTimeout.GetValue() % config->GridStep.GetValue() != 0) {
THROW_ERROR_EXCEPTION("\"linger_timeout\" must be multiple of \"grid_step\"");
diff --git a/yt/yt/library/profiling/solomon/config.h b/yt/yt/library/profiling/solomon/config.h
index 0c36c1120d..42b5b4548e 100644
--- a/yt/yt/library/profiling/solomon/config.h
+++ b/yt/yt/library/profiling/solomon/config.h
@@ -74,6 +74,8 @@ struct TSolomonExporterConfig
int ProducerCollectionBatchSize;
+ ELabelSanitizationPolicy LabelSanitizationPolicy;
+
TShardConfigPtr MatchShard(const std::string& sensorName);
ESummaryPolicy GetSummaryPolicy() const;
diff --git a/yt/yt/library/profiling/solomon/exporter.cpp b/yt/yt/library/profiling/solomon/exporter.cpp
index d039924ab5..aee542660a 100644
--- a/yt/yt/library/profiling/solomon/exporter.cpp
+++ b/yt/yt/library/profiling/solomon/exporter.cpp
@@ -88,6 +88,7 @@ TSolomonExporter::TSolomonExporter(
return shard->GridStep->GetValue() / config->GridStep.GetValue();
});
+ Registry_->SetLabelSanitizationPolicy(Config_->LabelSanitizationPolicy);
if (Config_->ReportBuildInfo) {
TProfiler profiler{registry, ""};
diff --git a/yt/yt/library/profiling/solomon/public.h b/yt/yt/library/profiling/solomon/public.h
index d497d9f9b9..a59e20b28e 100644
--- a/yt/yt/library/profiling/solomon/public.h
+++ b/yt/yt/library/profiling/solomon/public.h
@@ -1,6 +1,7 @@
#pragma once
#include <library/cpp/yt/memory/ref_counted.h>
+#include <library/cpp/yt/misc/enum.h>
namespace NYT::NProfiling {
@@ -18,4 +19,12 @@ DECLARE_REFCOUNTED_STRUCT(IEndpointProvider)
////////////////////////////////////////////////////////////////////////////////
+DEFINE_ENUM(ELabelSanitizationPolicy,
+ ((None) (0))
+ ((Weak) (1)) // Escape only zero symbol and trim label to 200 symbols
+ ((Strong) (2)) // Escape all forbidden symbols and trim label to 200 symbols
+);
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace NYT::NProfiling
diff --git a/yt/yt/library/profiling/solomon/registry.cpp b/yt/yt/library/profiling/solomon/registry.cpp
index 9fe71af15a..7147c2904c 100644
--- a/yt/yt/library/profiling/solomon/registry.cpp
+++ b/yt/yt/library/profiling/solomon/registry.cpp
@@ -281,6 +281,11 @@ void TSolomonRegistry::SetProducerCollectionBatchSize(int batchSize)
Producers_.SetCollectionBatchSize(batchSize);
}
+void TSolomonRegistry::SetLabelSanitizationPolicy(ELabelSanitizationPolicy LabelSanitizationPolicy)
+{
+ Tags_.SetLabelSanitizationPolicy(LabelSanitizationPolicy);
+}
+
int TSolomonRegistry::GetWindowSize() const
{
if (!WindowSize_) {
diff --git a/yt/yt/library/profiling/solomon/registry.h b/yt/yt/library/profiling/solomon/registry.h
index bddf548b67..cf4c149df0 100644
--- a/yt/yt/library/profiling/solomon/registry.h
+++ b/yt/yt/library/profiling/solomon/registry.h
@@ -129,6 +129,7 @@ public:
void SetGridFactor(std::function<int(const std::string&)> gridFactor);
void SetWindowSize(int windowSize);
void SetProducerCollectionBatchSize(int batchSize);
+ void SetLabelSanitizationPolicy(ELabelSanitizationPolicy LabelSanitizationPolicy);
void ProcessRegistrations();
void Collect(IInvokerPtr offloadInvoker = GetSyncInvoker());
void ReadSensors(
diff --git a/yt/yt/library/profiling/solomon/tag_registry-inl.h b/yt/yt/library/profiling/solomon/tag_registry-inl.h
new file mode 100644
index 0000000000..226aa5a1c1
--- /dev/null
+++ b/yt/yt/library/profiling/solomon/tag_registry-inl.h
@@ -0,0 +1,31 @@
+#ifndef TAG_REGISTRY_INL_H
+#error "Direct inclusion of this file is not allowed, include tag_registry.h"
+// For the sake of sane code completion.
+#include "tag_registry.h"
+#endif
+
+namespace NYT::NProfiling {
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class TTagPerfect>
+TTagId TTagRegistry::EncodeSanitized(TTagPerfect&& tag)
+{
+ static_assert(std::is_same_v<std::remove_cvref_t<TTagPerfect>, TTag>);
+
+ THashMap<TTag, TTagId>::insert_ctx insertCtx;
+ if (auto it = TagByName_.find(tag, insertCtx); it != TagByName_.end()) {
+ return it->second;
+ } else {
+ TTagId tagId = TagById_.size() + 1;
+
+ TagByName_.emplace_direct(insertCtx, tag, tagId);
+ TagById_.push_back(std::forward<TTagPerfect>(tag));
+
+ return tagId;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NProfiling
diff --git a/yt/yt/library/profiling/solomon/tag_registry.cpp b/yt/yt/library/profiling/solomon/tag_registry.cpp
index 30d7f0d654..1a66d405a0 100644
--- a/yt/yt/library/profiling/solomon/tag_registry.cpp
+++ b/yt/yt/library/profiling/solomon/tag_registry.cpp
@@ -10,18 +10,30 @@ namespace NYT::NProfiling {
////////////////////////////////////////////////////////////////////////////////
+namespace {
+
+static constexpr int MaxLabelSize = 200;
+static constexpr int HalfMaxLabelSize = MaxLabelSize / 2;
+
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TTagRegistry::SetLabelSanitizationPolicy(ELabelSanitizationPolicy labelSanitizationPolicy)
+{
+ LabelSanitizationPolicy_ = labelSanitizationPolicy;
+}
+
+TTagIdList TTagRegistry::Encode(const TTagSet& tags)
+{
+ return Encode(tags.Tags());
+}
+
TTagIdList TTagRegistry::Encode(const TTagList& tags)
{
TTagIdList ids;
-
for (const auto& tag : tags) {
- if (auto it = TagByName_.find(tag); it != TagByName_.end()) {
- ids.push_back(it->second);
- } else {
- TagById_.push_back(tag);
- TagByName_[tag] = TagById_.size();
- ids.push_back(TagById_.size());
- }
+ ids.push_back(Encode(tag));
}
return ids;
@@ -29,18 +41,17 @@ TTagIdList TTagRegistry::Encode(const TTagList& tags)
TTagId TTagRegistry::Encode(const TTag& tag)
{
- if (auto it = TagByName_.find(tag); it != TagByName_.end()) {
- return it->second;
- } else {
- TagById_.push_back(tag);
- TagByName_[tag] = TagById_.size();
- return TagById_.size();
+ if (LabelSanitizationPolicy_ == ELabelSanitizationPolicy::None) {
+ return EncodeSanitized(tag);
}
-}
-TTagIdList TTagRegistry::Encode(const TTagSet& tags)
-{
- return Encode(tags.Tags());
+ if (auto sanitizeParameters = ScanForSanitize(tag.second);
+ sanitizeParameters.IsSanitizationRequired())
+ {
+ return EncodeSanitized(SanitizeMonitoringTag(tag, sanitizeParameters.ResultingLength));
+ } else {
+ return EncodeSanitized(tag);
+ }
}
TCompactVector<std::optional<TTagId>, TypicalTagCount> TTagRegistry::TryEncode(const TTagList& tags) const
@@ -48,10 +59,17 @@ TCompactVector<std::optional<TTagId>, TypicalTagCount> TTagRegistry::TryEncode(c
TCompactVector<std::optional<TTagId>, TypicalTagCount> ids;
for (const auto& tag : tags) {
- if (auto it = TagByName_.find(tag); it != TagByName_.end()) {
- ids.push_back(it->second);
+ if (LabelSanitizationPolicy_ == ELabelSanitizationPolicy::None) {
+ ids.push_back(TryEncodeSanitized(tag));
+ continue;
+ }
+
+ if (auto sanitizeParameters = ScanForSanitize(tag.second);
+ sanitizeParameters.IsSanitizationRequired())
+ {
+ ids.push_back(TryEncodeSanitized(SanitizeMonitoringTag(tag, sanitizeParameters.ResultingLength)));
} else {
- ids.push_back({});
+ ids.push_back(TryEncodeSanitized(tag));
}
}
@@ -95,6 +113,110 @@ void TTagRegistry::DumpTags(NProto::TSensorDump* dump)
////////////////////////////////////////////////////////////////////////////////
+bool TTagRegistry::TSanitizeParameters::IsSanitizationRequired() const
+{
+ return ForbiddenCharCount > 0 || ResultingLength > MaxLabelSize;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool TTagRegistry::IsAllowedMonitoringTagValueChar(unsigned char c) const
+{
+ switch (LabelSanitizationPolicy_) {
+ case ELabelSanitizationPolicy::Weak:
+ return c != '\0';
+
+ case ELabelSanitizationPolicy::Strong:
+ return 31 < c &&
+ c < 127 &&
+ c != '|' &&
+ c != '*' &&
+ c != '?' &&
+ c != '"' &&
+ c != '\'' &&
+ c != '\\' &&
+ c != '`';
+
+ default:
+ YT_ABORT();
+ }
+}
+
+TTagRegistry::TSanitizeParameters TTagRegistry::ScanForSanitize(const std::string& value) const
+{
+ YT_VERIFY(LabelSanitizationPolicy_ != ELabelSanitizationPolicy::None);
+
+ int forbiddenCharCount = 0;
+ for (unsigned char c : value) {
+ forbiddenCharCount += static_cast<int>(!IsAllowedMonitoringTagValueChar(c));
+ }
+
+ return {
+ .ForbiddenCharCount = forbiddenCharCount,
+ .ResultingLength = static_cast<int>(value.size() + forbiddenCharCount * 2),
+ };
+}
+
+std::string TTagRegistry::SanitizeMonitoringTagValue(const std::string& value, int resultingLength) const
+{
+ bool needTrim = resultingLength > MaxLabelSize;
+
+ std::string result;
+ result.resize(std::min(resultingLength, MaxLabelSize));
+
+ int resultIndex = 0;
+ for (int index = 0; resultIndex < (needTrim ? HalfMaxLabelSize : resultingLength); ++index) {
+ unsigned char c = value[index];
+
+ if (IsAllowedMonitoringTagValueChar(c)) {
+ result[resultIndex++] = c;
+ } else {
+ result[resultIndex++] = '%';
+ result[resultIndex++] = IntToHexLowercase[c >> 4];
+ result[resultIndex++] = IntToHexLowercase[c & 0x0f];
+ }
+ }
+
+ if (!needTrim) {
+ return result;
+ }
+
+ resultIndex = MaxLabelSize - 1;
+ for (int index = ssize(value) - 1; resultIndex > HalfMaxLabelSize + 2; --index) {
+ unsigned char c = value[index];
+
+ if (IsAllowedMonitoringTagValueChar(value[index])) {
+ result[resultIndex--] = c;
+ } else {
+ result[resultIndex--] = IntToHexLowercase[c & 0x0f];
+ result[resultIndex--] = IntToHexLowercase[c >> 4];
+ result[resultIndex--] = '%';
+ }
+ }
+
+ result[HalfMaxLabelSize] = '.';
+ result[HalfMaxLabelSize + 1] = '.';
+ result[HalfMaxLabelSize + 2] = '.';
+
+ return result;
+}
+
+TTag TTagRegistry::SanitizeMonitoringTag(const TTag& tag, int resultingLength) const
+{
+ return {tag.first, SanitizeMonitoringTagValue(tag.second, resultingLength)};
+}
+
+std::optional<TTagId> TTagRegistry::TryEncodeSanitized(const TTag& tag) const
+{
+ if (auto it = TagByName_.find(tag); it != TagByName_.end()) {
+ return it->second;
+ } else {
+ return std::nullopt;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
void TTagWriter::WriteLabel(TTagId tag)
{
if (static_cast<size_t>(tag) >= Cache_.size()) {
diff --git a/yt/yt/library/profiling/solomon/tag_registry.h b/yt/yt/library/profiling/solomon/tag_registry.h
index 8e76ae1175..e733046136 100644
--- a/yt/yt/library/profiling/solomon/tag_registry.h
+++ b/yt/yt/library/profiling/solomon/tag_registry.h
@@ -1,5 +1,7 @@
#pragma once
+#include "public.h"
+
#include <yt/yt/core/profiling/public.h>
#include <yt/yt/library/profiling/tag.h>
@@ -16,6 +18,8 @@ namespace NYT::NProfiling {
class TTagRegistry
{
public:
+ void SetLabelSanitizationPolicy(ELabelSanitizationPolicy labelSanitizationPolicy);
+
TTagIdList Encode(const TTagSet& tags);
TTagIdList Encode(const TTagList& tags);
TTagId Encode(const TTag& tag);
@@ -30,6 +34,24 @@ public:
void DumpTags(NProto::TSensorDump* dump);
private:
+ struct TSanitizeParameters
+ {
+ int ForbiddenCharCount;
+ int ResultingLength;
+
+ bool IsSanitizationRequired() const;
+ };
+
+ bool IsAllowedMonitoringTagValueChar(unsigned char c) const;
+ TSanitizeParameters ScanForSanitize(const std::string& value) const;
+ std::string SanitizeMonitoringTagValue(const std::string& value, int resultingLength) const;
+ TTag SanitizeMonitoringTag(const TTag& tag, int resultingLength) const;
+
+ template <class TTagPerfect>
+ TTagId EncodeSanitized(TTagPerfect&& tag);
+ std::optional<TTagId> TryEncodeSanitized(const TTag& tag) const;
+
+ ELabelSanitizationPolicy LabelSanitizationPolicy_ = ELabelSanitizationPolicy::None;
// TODO(prime@): maybe do something about the fact that tags are never freed.
THashMap<TTag, TTagId> TagByName_;
std::deque<TTag> TagById_;
@@ -60,3 +82,7 @@ private:
////////////////////////////////////////////////////////////////////////////////
} // namespace NYT::NProfiling
+
+#define TAG_REGISTRY_INL_H
+#include "tag_registry-inl.h"
+#undef TAG_REGISTRY_INL_H
diff --git a/yt/yt/library/profiling/unittests/solomon_ut.cpp b/yt/yt/library/profiling/unittests/solomon_ut.cpp
index c26d478234..8addff92fc 100644
--- a/yt/yt/library/profiling/unittests/solomon_ut.cpp
+++ b/yt/yt/library/profiling/unittests/solomon_ut.cpp
@@ -17,6 +17,10 @@ namespace {
////////////////////////////////////////////////////////////////////////////////
+using namespace std::string_literals;
+
+////////////////////////////////////////////////////////////////////////////////
+
struct TTestMetricConsumer
: public NMonitoring::IMetricConsumer
{
@@ -962,6 +966,132 @@ TEST_P(TOmitNameLabelSuffixTest, GaugeSummary)
ASSERT_NEAR(gauges[Format("yt.davg%v{}", omitNameLabelSuffix ? "" : ".avg")], 40 + 1 / 3.0, 1e-6);
}
+struct TTagInfo
+{
+ std::string TagValue;
+ std::string EncodedTagValue;
+};
+
+using TTagInfoMapping = THashMap<std::string, TTagInfo>;
+
+TTagInfo GetLongTag() {
+ TString longTag;
+ longTag.reserve(210);
+ for (int index = 0; index < 210; ++index) {
+ longTag.append('a' + index % 26);
+ }
+ TString longTagEncoded;
+ longTagEncoded.reserve(200);
+ longTagEncoded.append(longTag.begin(), 100).append("...");
+ for (int index = 103; index < 200; ++index) {
+ longTagEncoded.append('a' + (index - 103 + 9) % 26);
+ }
+
+ return {
+ .TagValue = std::move(longTag),
+ .EncodedTagValue = std::move(longTagEncoded),
+ };
+}
+
+void CheckTags(const TSolomonRegistryPtr& registry, const TTagInfoMapping& tagInfoMapping) {
+ int tagNameLength = tagInfoMapping.begin()->first.size();
+
+ TTagSet tagSet;
+ for (const auto& [tagName, tagInfo] : tagInfoMapping) {
+ ASSERT_EQ(ssize(tagName), tagNameLength);
+ tagSet.AddTag({tagName, tagInfo.TagValue});
+ }
+
+ auto profiler = TProfiler(registry, "/debug")
+ .WithTags(tagSet);
+ auto c0 = profiler.Counter("/c");
+ c0.Increment(1);
+
+ auto result = CollectSensors(registry);
+
+ for (const auto& label : result.Labels) {
+ // Label has structure - "labelName=labelValue"
+ auto tagName = label.substr(0, tagNameLength);
+ auto tagValue = label.substr(tagNameLength + 1, label.size() - tagNameLength - 1);
+
+ ASSERT_EQ(tagValue, tagInfoMapping.at(tagName).EncodedTagValue);
+ }
+}
+
+TEST(TSolomonRegistry, IncorrectSolomonLabelsWeakPolicy)
+{
+ auto impl = New<TSolomonRegistry>();
+ impl->SetWindowSize(12);
+ impl->SetLabelSanitizationPolicy(ELabelSanitizationPolicy::Weak);
+
+ CheckTags(
+ impl,
+ {
+ {
+ "tag0",
+ GetLongTag(),
+ },
+ {
+ "tag1",
+ {
+ .TagValue = "a\0aa|*?\"'\\`b\0b\xff"s,
+ .EncodedTagValue = "a%00aa|*?\"'\\`b%00b\xff",
+ },
+ },
+ {
+ "tag2",
+ {
+ .TagValue = TString(98, 'a')
+ .append('\0')
+ .append(100, 'a')
+ .append('\0')
+ .append('\xff'),
+ .EncodedTagValue = TString(98, 'a')
+ .append("%0...")
+ .append(93, 'a')
+ .append("%00")
+ .append('\xff')
+ },
+ },
+ });
+}
+
+TEST(TSolomonRegistry, IncorrectSolomonLabelsStrongPolicy)
+{
+ auto impl = New<TSolomonRegistry>();
+ impl->SetWindowSize(12);
+ impl->SetLabelSanitizationPolicy(ELabelSanitizationPolicy::Strong);
+
+ CheckTags(
+ impl,
+ {
+ {
+ "tag0",
+ GetLongTag(),
+ },
+ {
+ "tag1",
+ {
+ .TagValue = "a\0aa|*?\"'\\`bb\xff\0"s,
+ .EncodedTagValue = "a%00aa%7c%2a%3f%22%27%5c%60bb%ff%00",
+ },
+ },
+ {
+ "tag2",
+ {
+ .TagValue = TString(98, 'a')
+ .append('\xff')
+ .append(100, 'a')
+ .append('\0'),
+ .EncodedTagValue = TString(98, 'a')
+ .append("%f...")
+ .append(94, 'a')
+ .append("%00"),
+ },
+ },
+ });
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace