diff options
author | ziganshinmr <ziganshinmr@yandex-team.com> | 2025-04-24 20:00:40 +0300 |
---|---|---|
committer | ziganshinmr <ziganshinmr@yandex-team.com> | 2025-04-24 20:16:49 +0300 |
commit | b9324f14b0bfe5a47f2faf87ed3080fb4c44158c (patch) | |
tree | 06d52554acf9e3496d6c73c4bed3d662e27ebed6 | |
parent | a9eb2b1429aba8e0632720068e81f51f6e6d944f (diff) | |
download | ydb-b9324f14b0bfe5a47f2faf87ed3080fb4c44158c.tar.gz |
Check YT operations spec not to contain secrets
commit_hash:f3bb066b14d05257ef5fe43c4f80a92a16c10b4a
22 files changed, 152 insertions, 1 deletions
diff --git a/yql/essentials/core/issue/protos/issue_id.proto b/yql/essentials/core/issue/protos/issue_id.proto index a84eb6fac3a..47d09e2c823 100644 --- a/yql/essentials/core/issue/protos/issue_id.proto +++ b/yql/essentials/core/issue/protos/issue_id.proto @@ -113,6 +113,7 @@ message TIssuesIds { YT_MISSING_PROTO_FIELD = 3017; YT_FOLDER_INPUT_IS_NOT_A_FOLDER = 3018; YT_SECURE_DATA_IN_COMMON_TMP = 3019; + YT_OP_SPEC_CONTAINS_SECRETS = 3020; // yql parser warnings YQL_PRAGMA_WARNING_MSG = 4500; diff --git a/yql/essentials/core/issue/yql_issue.txt b/yql/essentials/core/issue/yql_issue.txt index 08e3cfb2e61..a2ac582aa64 100644 --- a/yql/essentials/core/issue/yql_issue.txt +++ b/yql/essentials/core/issue/yql_issue.txt @@ -644,6 +644,10 @@ ids { severity: S_ERROR } ids { + code: YT_OP_SPEC_CONTAINS_SECRETS + severity: S_ERROR +} +ids { code: YQL_OFFSET_WITHOUT_SORT severity: S_WARNING } diff --git a/yt/yql/providers/yt/common/yql_configuration.h b/yt/yql/providers/yt/common/yql_configuration.h index 083504924e0..99bde583d47 100644 --- a/yt/yql/providers/yt/common/yql_configuration.h +++ b/yt/yql/providers/yt/common/yql_configuration.h @@ -134,4 +134,7 @@ constexpr bool DEFAULT_ALLOW_REMOTE_CLUSTER_INPUT = false; constexpr bool DEFAULT_USE_COLUMN_GROUPS_FROM_INPUT_TABLE = false; constexpr bool DEFAULT_USE_NATIVE_DYNAMIC_TABLE_READ = false; + +constexpr bool DEFAULT_FORBID_SENSITIVE_DATA_IN_OPERATION_SPEC = false; + } // NYql diff --git a/yt/yql/providers/yt/common/yql_yt_settings.cpp b/yt/yql/providers/yt/common/yql_yt_settings.cpp index c6d0cb8dec9..a5d888f6d77 100644 --- a/yt/yql/providers/yt/common/yql_yt_settings.cpp +++ b/yt/yql/providers/yt/common/yql_yt_settings.cpp @@ -557,6 +557,7 @@ TYtConfiguration::TYtConfiguration(TTypeAnnotationContext& typeCtx) REGISTER_SETTING(*this, _AllowRemoteClusterInput); REGISTER_SETTING(*this, UseColumnGroupsFromInputTables); REGISTER_SETTING(*this, UseNativeDynamicTableRead); + REGISTER_SETTING(*this, _ForbidSensitiveDataInOperationSpec); } EReleaseTempDataMode GetReleaseTempDataMode(const TYtSettings& settings) { diff --git a/yt/yql/providers/yt/common/yql_yt_settings.h b/yt/yql/providers/yt/common/yql_yt_settings.h index 677472ecb5d..6b8d99fbd86 100644 --- a/yt/yql/providers/yt/common/yql_yt_settings.h +++ b/yt/yql/providers/yt/common/yql_yt_settings.h @@ -123,6 +123,7 @@ struct TYtSettings { NCommon::TConfSetting<bool, false> ForceTmpSecurity; NCommon::TConfSetting<ERuntimeClusterSelectionMode, false> RuntimeClusterSelection; NCommon::TConfSetting<TString, false> DefaultRuntimeCluster; + NCommon::TConfSetting<bool, false> _ForbidSensitiveDataInOperationSpec; // Job runtime NCommon::TConfSetting<TString, true> Pool; diff --git a/yt/yql/providers/yt/gateway/native/ut/ya.make b/yt/yql/providers/yt/gateway/native/ut/ya.make index 8dd7c9a3a7d..2c68c61ab7f 100644 --- a/yt/yql/providers/yt/gateway/native/ut/ya.make +++ b/yt/yql/providers/yt/gateway/native/ut/ya.make @@ -9,6 +9,7 @@ PEERDIR( yt/yql/providers/yt/gateway/file yt/yql/providers/yt/codec/codegen yt/yql/providers/yt/comp_nodes/llvm16 + yt/yql/providers/yt/lib/secret_masker/dummy yt/yql/providers/yt/lib/ut_common library/cpp/testing/mock_server library/cpp/testing/common @@ -21,4 +22,3 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() - diff --git a/yt/yql/providers/yt/gateway/native/ut/yql_yt_native_folders_ut.cpp b/yt/yql/providers/yt/gateway/native/ut/yql_yt_native_folders_ut.cpp index b08db52a500..e767b3237b9 100644 --- a/yt/yql/providers/yt/gateway/native/ut/yql_yt_native_folders_ut.cpp +++ b/yt/yql/providers/yt/gateway/native/ut/yql_yt_native_folders_ut.cpp @@ -1,5 +1,6 @@ #include "library/cpp/testing/unittest/registar.h" #include <library/cpp/yson/node/node_io.h> +#include <yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.h> #include <yt/yql/providers/yt/lib/ut_common/yql_ut_common.h> #include <library/cpp/testing/common/network.h> #include <library/cpp/testing/mock_server/server.h> @@ -190,6 +191,7 @@ std::pair<TIntrusivePtr<TYtState>, IYtGateway::TPtr> InitTest(const NTesting::TP auto gatewaysConfig = MakeGatewaysConfig(port); nativeServices.Config = std::make_shared<TYtGatewayConfig>(gatewaysConfig.GetYt()); nativeServices.FileStorage = CreateFileStorage(TFileStorageConfig{}); + nativeServices.SecretMasker = CreateDummySecretMasker(); auto ytGateway = CreateYtNativeGateway(nativeServices); auto ytState = MakeIntrusive<TYtState>(types); diff --git a/yt/yql/providers/yt/gateway/native/ya.make b/yt/yql/providers/yt/gateway/native/ya.make index c0bad18eebf..967557d118b 100644 --- a/yt/yql/providers/yt/gateway/native/ya.make +++ b/yt/yql/providers/yt/gateway/native/ya.make @@ -52,6 +52,7 @@ PEERDIR( yt/yql/providers/yt/lib/mkql_helpers yt/yql/providers/yt/lib/res_pull yt/yql/providers/yt/lib/schema + yt/yql/providers/yt/lib/secret_masker yt/yql/providers/yt/lib/skiff yt/yql/providers/yt/lib/url_mapper yt/yql/providers/yt/lib/yson_helpers diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.cpp index 9209cf86e67..6164b07004f 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.cpp @@ -45,6 +45,7 @@ TExecContextBase::TExecContextBase(const TYtNativeServices& services, : FunctionRegistry_(services.FunctionRegistry) , FileStorage_(services.FileStorage) , Config_(services.Config) + , SecretMasker(services.SecretMasker) , Clusters_(clusters) , MkqlCompiler_(mkqlCompiler) , Session_(session) diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h index 70459783d26..059af11397f 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h +++ b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h @@ -154,6 +154,7 @@ public: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_ = nullptr; TFileStoragePtr FileStorage_; TYtGatewayConfigPtr Config_; + ISecretMasker::TPtr SecretMasker; TConfigClusters::TPtr Clusters_; TIntrusivePtr<NCommon::TMkqlCommonCallableCompiler> MkqlCompiler_; TSession::TPtr Session_; diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index 5467508a59a..6a65dadebb8 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -2852,6 +2852,7 @@ private: } FillSpec(spec, *execCtx, entry, 0., Nothing(), flags); + CheckSpecForSecrets(spec, execCtx); if (combineChunks) { mergeSpec.CombineChunks(true); @@ -3258,6 +3259,7 @@ private: NYT::TNode spec = execCtx->Session_->CreateSpecWithDesc(execCtx->CodeSnippets_); FillSpec(spec, *execCtx, entry, 0., Nothing(), EYtOpProp::WithMapper); spec["job_count"] = 1; + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -3734,6 +3736,7 @@ private: if (hasNonStrict) { spec["schema_inference_mode"] = "from_output"; // YTADMINREQ-17692 } + CheckSpecForSecrets(spec, execCtx); return execCtx->RunOperation([entry, sortOpSpec = std::move(sortOpSpec), spec = std::move(spec)](){ return entry->Tx->Sort(sortOpSpec, TOperationOptions().StartOperationMode(TOperationOptions::EStartOperationMode::AsyncPrepare).Spec(spec)); @@ -3818,6 +3821,7 @@ private: } PrepareInputQueryForMerge(spec, mergeOpSpec.Inputs_, inputQueryExpr, execCtx->Options_.Config()); + CheckSpecForSecrets(spec, execCtx); return execCtx->RunOperation([entry, mergeOpSpec = std::move(mergeOpSpec), spec = std::move(spec)](){ return entry->Tx->Merge(mergeOpSpec, TOperationOptions().StartOperationMode(TOperationOptions::EStartOperationMode::AsyncPrepare).Spec(spec)); @@ -4009,6 +4013,7 @@ private: } PrepareInputQueryForMap(spec, mapOpSpec, inputQueryExpr, execCtx->Options_.Config(), /*useSystemColumns*/ useSkiff); + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -4220,6 +4225,7 @@ private: if (maxDataSizePerJob) { spec["max_data_size_per_job"] = static_cast<i64>(*maxDataSizePerJob); } + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -4516,6 +4522,7 @@ private: } PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQueryExpr, execCtx->Options_.Config(), /*useSystemColumns*/ useSkiff); + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -4666,6 +4673,7 @@ private: } PrepareInputQueryForMap(spec, mapReduceOpSpec, inputQueryExpr, execCtx->Options_.Config(), /*useSystemColumns*/ useSkiff); + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -4998,6 +5006,7 @@ private: NYT::TNode spec = execCtx->Session_->CreateSpecWithDesc(execCtx->CodeSnippets_); FillSpec(spec, *execCtx, entry, extraUsage.Cpu, Nothing(), EYtOpProp::TemporaryAutoMerge | EYtOpProp::WithMapper | EYtOpProp::WithUserJobs); + CheckSpecForSecrets(spec, execCtx); TOperationOptions opOpts; FillOperationOptions(opOpts, execCtx, entry); @@ -5620,6 +5629,7 @@ private: } NYT::TNode spec = execCtx->Session_->CreateSpecWithDesc(execCtx->CodeSnippets_); FillSpec(spec, *execCtx, entry, extraUsage.Cpu, Nothing(), EYtOpProp::WithMapper); + CheckSpecForSecrets(spec, execCtx); PrepareTempDestination(tmpTable, execCtx, entry, entry->Tx); diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.h b/yt/yql/providers/yt/gateway/native/yql_yt_native.h index 4ae23ac55da..81668c72884 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.h +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.h @@ -1,6 +1,7 @@ #pragma once #include <yt/yql/providers/yt/provider/yql_yt_gateway.h> +#include <yt/yql/providers/yt/lib/secret_masker/secret_masker.h> #include <yql/essentials/core/file_storage/file_storage.h> #include <yql/essentials/minikql/mkql_function_registry.h> @@ -21,6 +22,7 @@ struct TYtNativeServices { // allow anonymous access for tests bool DisableAnonymousClusterAccess = false; IMetricsRegistryPtr Metrics; + ISecretMasker::TPtr SecretMasker; }; IYtGateway::TPtr CreateYtNativeGateway(const TYtNativeServices& services); diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp index dae30d87341..96bb073c780 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp @@ -532,6 +532,33 @@ void FillSpec(NYT::TNode& spec, } } +void CheckSpecForSecretsImpl( + const NYT::TNode& spec, + const ISecretMasker::TPtr& secretMasker, + const TYtSettings::TConstPtr& settings +) { + if (!settings->_ForbidSensitiveDataInOperationSpec.Get().GetOrElse(DEFAULT_FORBID_SENSITIVE_DATA_IN_OPERATION_SPEC)) { + return; + } + + YQL_ENSURE(secretMasker); + + auto maskedSpecStr = NYT::NodeToYsonString(spec); + auto secrets = secretMasker->Mask(maskedSpecStr); + if (!secrets.empty()) { + auto maskedSpecStrBuf = TStringBuf(maskedSpecStr); + + TVector<TString> maskedSecrets; + for (auto& secret : secrets) { + maskedSecrets.push_back(TStringBuilder() << "\"" << maskedSpecStrBuf.substr(secret.From, secret.Len) << "\""); + } + + YQL_LOG_CTX_THROW TErrorException(TIssuesIds::YT_OP_SPEC_CONTAINS_SECRETS) + << "YT operation spec contains sensitive data (masked): " + << JoinSeq(", ", maskedSecrets); + } +} + void FillSecureVault(NYT::TNode& spec, const IYtGateway::TSecureParams& secureParams) { if (secureParams.empty()) { return; diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_spec.h b/yt/yql/providers/yt/gateway/native/yql_yt_spec.h index 7e7941508e0..57132d2e57b 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_spec.h +++ b/yt/yql/providers/yt/gateway/native/yql_yt_spec.h @@ -49,6 +49,12 @@ void FillSpec(NYT::TNode& spec, EYtOpProps opProps = 0, const TSet<TString>& addSecTags = {}); +void CheckSpecForSecretsImpl( + const NYT::TNode& spec, + const ISecretMasker::TPtr& secretMasker, + const TYtSettings::TConstPtr& settings +); + void FillSecureVault(NYT::TNode& spec, const IYtGateway::TSecureParams& secureParams); void FillUserJobSpecImpl(NYT::TUserJobSpec& spec, @@ -124,6 +130,11 @@ inline void FillOperationOptions(NYT::TOperationOptions& opOpts, FillOperationOptionsImpl(opOpts, execCtx->Options_.Config(), entry); } +template <class TExecParamsPtr> +inline void CheckSpecForSecrets(const NYT::TNode& spec, const TExecParamsPtr& execCtx) { + CheckSpecForSecretsImpl(spec, execCtx->SecretMasker, execCtx->Options_.Config()); +} + } // NNative } // NYql diff --git a/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.cpp b/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.cpp new file mode 100644 index 00000000000..40b8e139ec6 --- /dev/null +++ b/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.cpp @@ -0,0 +1,22 @@ +#include "dummy_secret_masker.h" + +namespace NYql { + +class TDummySecretMasker : public ISecretMasker { +public: + TSecretList Search(TStringBuf) override { + // Do nothing + return {}; + } + + TSecretList Mask(TString&) override { + // Do nothing + return {}; + } +}; + +ISecretMasker::TPtr CreateDummySecretMasker() { + return MakeIntrusive<TDummySecretMasker>(); +} + +} // namespace NYql diff --git a/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.h b/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.h new file mode 100644 index 00000000000..f91f2358fa6 --- /dev/null +++ b/yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yt/yql/providers/yt/lib/secret_masker/secret_masker.h> + +namespace NYql { + +ISecretMasker::TPtr CreateDummySecretMasker(); + +} // namespace NYql diff --git a/yt/yql/providers/yt/lib/secret_masker/dummy/ya.make b/yt/yql/providers/yt/lib/secret_masker/dummy/ya.make new file mode 100644 index 00000000000..a3823c82b80 --- /dev/null +++ b/yt/yql/providers/yt/lib/secret_masker/dummy/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + dummy_secret_masker.cpp +) + +PEERDIR( + yt/yql/providers/yt/lib/secret_masker +) + +END() diff --git a/yt/yql/providers/yt/lib/secret_masker/secret_masker.h b/yt/yql/providers/yt/lib/secret_masker/secret_masker.h new file mode 100644 index 00000000000..993b0faef5f --- /dev/null +++ b/yt/yql/providers/yt/lib/secret_masker/secret_masker.h @@ -0,0 +1,25 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NYql { + +struct TSecretPos { + size_t From; + size_t Len; +}; + +using TSecretList = TVector<TSecretPos>; + +class ISecretMasker : public TThrRefBase { +public: + using TPtr = TIntrusivePtr<ISecretMasker>; + + virtual ~ISecretMasker() = default; + virtual TSecretList Search(TStringBuf data) = 0; + virtual TSecretList Mask(TString& data) = 0; +}; + +} // namespace NYql diff --git a/yt/yql/providers/yt/lib/secret_masker/ya.make b/yt/yql/providers/yt/lib/secret_masker/ya.make new file mode 100644 index 00000000000..dec13ca568a --- /dev/null +++ b/yt/yql/providers/yt/lib/secret_masker/ya.make @@ -0,0 +1,10 @@ +LIBRARY() +END() + +RECURSE( + dummy +) + +IF (NOT OPENSOURCE) + INCLUDE(ya_non_opensource.inc) +ENDIF() diff --git a/yt/yql/tools/ytrun/lib/ya.make b/yt/yql/tools/ytrun/lib/ya.make index fc10024b330..8211271e01d 100644 --- a/yt/yql/tools/ytrun/lib/ya.make +++ b/yt/yql/tools/ytrun/lib/ya.make @@ -13,6 +13,7 @@ PEERDIR( yt/yql/providers/yt/lib/yt_download yt/yql/providers/yt/lib/yt_url_lister yt/yql/providers/yt/lib/log + yt/yql/providers/yt/lib/secret_masker/dummy yql/essentials/providers/common/provider yql/essentials/core/cbo diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp index 25bda1b40fc..64eb2116908 100644 --- a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp +++ b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp @@ -6,6 +6,7 @@ #include <yt/yql/providers/yt/lib/yt_download/yt_download.h> #include <yt/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h> #include <yt/yql/providers/yt/lib/log/yt_logger.h> +#include <yt/yql/providers/yt/lib/secret_masker/dummy/dummy_secret_masker.h> #include <yt/yql/providers/yt/gateway/native/yql_yt_native.h> #include <yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.h> #include <yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.h> @@ -186,6 +187,7 @@ IYtGateway::TPtr TYtRunTool::CreateYtGateway() { services.FunctionRegistry = GetFuncRegistry().Get(); services.FileStorage = GetFileStorage(); services.Config = std::make_shared<TYtGatewayConfig>(GetRunOptions().GatewaysConfig->GetYt()); + services.SecretMasker = CreateSecretMasker(); auto ytGateway = CreateYtNativeGateway(services); if (!GetRunOptions().GatewayTypes.contains(NFmr::FastMapReduceGatewayName)) { return ytGateway; @@ -204,6 +206,10 @@ IDqHelper::TPtr TYtRunTool::CreateDqHelper() { return {}; } +ISecretMasker::TPtr TYtRunTool::CreateSecretMasker() { + return CreateDummySecretMasker(); +} + int TYtRunTool::DoMain(int argc, const char *argv[]) { // Init MR/YT for proper work of embedded agent NYT::Initialize(argc, argv); diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.h b/yt/yql/tools/ytrun/lib/ytrun_lib.h index b90fddcb6ba..36563b742ed 100644 --- a/yt/yql/tools/ytrun/lib/ytrun_lib.h +++ b/yt/yql/tools/ytrun/lib/ytrun_lib.h @@ -2,6 +2,7 @@ #include <yt/yql/providers/yt/provider/yql_yt_gateway.h> #include <yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.h> +#include <yt/yql/providers/yt/lib/secret_masker/secret_masker.h> #include <yql/essentials/tools/yql_facade_run/yql_facade_run.h> #include <yql/essentials/core/cbo/cbo_optimizer_new.h> @@ -24,6 +25,7 @@ protected: virtual IYtGateway::TPtr CreateYtGateway(); virtual IOptimizerFactory::TPtr CreateCboFactory(); virtual IDqHelper::TPtr CreateDqHelper(); + virtual ISecretMasker::TPtr CreateSecretMasker(); protected: TString MrJobBin_; |