diff options
author | vvvv <vvvv@ydb.tech> | 2023-11-02 21:26:28 +0300 |
---|---|---|
committer | vvvv <vvvv@ydb.tech> | 2023-11-02 22:00:50 +0300 |
commit | 16ebc933ab60f4738ec54a12255fb512be238ebf (patch) | |
tree | 3205d7aa1ed3dc6e48555127dd39dc59e59c3e59 | |
parent | 638d0f03813f82c4c2df549f9441a5979fc9ea2a (diff) | |
download | ydb-16ebc933ab60f4738ec54a12255fb512be238ebf.tar.gz |
YQL-16922 isolation of files, libs & udfs inside views, prohibit context functions
fixes for custom prefixes
init
46 files changed, 592 insertions, 109 deletions
diff --git a/ydb/library/yql/core/yql_udf_resolver.h b/ydb/library/yql/core/yql_udf_resolver.h index 46a40fe72b..68c0ba0525 100644 --- a/ydb/library/yql/core/yql_udf_resolver.h +++ b/ydb/library/yql/core/yql_udf_resolver.h @@ -68,6 +68,7 @@ public: const TVector<TFunction*>& functions, TExprContext& ctx) const = 0; virtual TResolveResult LoadRichMetadata(const TVector<TImport>& imports) const = 0; + virtual bool ContainsModule(const TStringBuf& moduleName) const = 0; }; TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks); diff --git a/ydb/library/yql/minikql/mkql_function_registry.cpp b/ydb/library/yql/minikql/mkql_function_registry.cpp index 74a80cd9f6..7889d375c9 100644 --- a/ydb/library/yql/minikql/mkql_function_registry.cpp +++ b/ydb/library/yql/minikql/mkql_function_registry.cpp @@ -49,11 +49,13 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry public: TUdfModuleLoader( TUdfModulesMap& modulesMap, + THashSet<TString>* newModules, const TString& libraryPath, const TUdfModuleRemappings& remappings, ui32 abiVersion, const TString& customUdfPrefix = {}) : ModulesMap(modulesMap) + , NewModules(newModules) , LibraryPath(libraryPath) , Remappings(remappings) , AbiVersion(NUdf::AbiVersionToStr(abiVersion)) @@ -86,6 +88,8 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry << "UDF module duplication: name " << TStringBuf(name) << ", already loaded from " << oldModule->LibraryPath << ", trying to load from " << LibraryPath); + } else if (NewModules) { + NewModules->insert(newName); } } } @@ -95,6 +99,7 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry private: TUdfModulesMap& ModulesMap; + THashSet<TString>* NewModules; const TString LibraryPath; const TUdfModuleRemappings& Remappings; const TString AbiVersion; @@ -125,15 +130,14 @@ public: void LoadUdfs( const TString& libraryPath, const TUdfModuleRemappings& remmapings, - ui32 flags /* = 0 */, - const TString& customUdfPrefix = {}) override + ui32 flags = 0, + const TString& customUdfPrefix = {}, + THashSet<TString>* modules = nullptr) override { TUdfLibraryPtr lib; auto libIt = LoadedLibraries_.find(libraryPath); - if (libIt != LoadedLibraries_.end()) { - return; - } else { + if (libIt == LoadedLibraries_.end()) { lib = MakeIntrusive<TUdfLibrary>(); #ifdef _win32_ ui32 loadFlags = 0; @@ -182,6 +186,8 @@ public: } libIt = LoadedLibraries_.insert({ libraryPath, lib }).first; + } else { + lib = libIt->second; } // (2) ensure that Register() func is present @@ -189,13 +195,19 @@ public: lib->Lib.Sym(RegisterFuncName)); // (3) do load + THashSet<TString> newModules; TUdfModuleLoader loader( - UdfModules_, - libraryPath, - remmapings, - lib->AbiVersion, customUdfPrefix); + UdfModules_, + &newModules, + libraryPath, + remmapings, + lib->AbiVersion, customUdfPrefix); registerFunc(loader, flags); Y_ENSURE(!loader.HasError(), loader.GetError()); + + if (modules) { + *modules = std::move(newModules); + } } void AddModule( @@ -211,7 +223,7 @@ public: TUdfModuleRemappings remappings; TUdfModuleLoader loader( - UdfModules_, libraryPathStr, + UdfModules_, nullptr, libraryPathStr, remappings, NUdf::CurrentAbiVersion()); loader.AddModule(moduleName, std::move(module)); @@ -540,8 +552,11 @@ TIntrusivePtr<IFunctionRegistry> CreateFunctionRegistry( // system UDFs loaded with default names TUdfModuleRemappings remappings; + THashSet<TString> usedUdfPaths; for (const TString& udfPath: udfsPaths) { - registry->LoadUdfs(udfPath, remappings, flags); + if (usedUdfPaths.insert(udfPath).second) { + registry->LoadUdfs(udfPath, remappings, flags); + } } return registry.Release(); diff --git a/ydb/library/yql/minikql/mkql_function_registry.h b/ydb/library/yql/minikql/mkql_function_registry.h index 18942cbc97..9b6f67fe32 100644 --- a/ydb/library/yql/minikql/mkql_function_registry.h +++ b/ydb/library/yql/minikql/mkql_function_registry.h @@ -104,7 +104,8 @@ public: const TString& libraryPath, const TUdfModuleRemappings& remmapings, ui32 flags = 0, - const TString& customUdfPrefix = {}) = 0; + const TString& customUdfPrefix = {}, + THashSet<TString>* modules = nullptr) = 0; virtual void AddModule( const TStringBuf& libraryPath, diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index f1f631d49b..42df4d8753 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -135,6 +135,10 @@ public: return MakeMaybe<TFilePathWithMd5>(*path, md5); } + bool ContainsModule(const TStringBuf& moduleName) const override { + return FunctionRegistry_->IsLoadedUdfModule(moduleName); + } + bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override { THashSet<TString> requiredLoadedModules; THashSet<TString> requiredExternalModules; @@ -160,7 +164,7 @@ public: } TResolve request; - THashMap<TString, TImport*> importMap; + TVector<TImport*> usedImports; THoldingFileStorage holdingFileStorage(FileStorage_); THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp(); @@ -187,10 +191,9 @@ public: import->Modules.ConstructInPlace(); } - importMap[import->FileAlias] = import; - try { LoadImport(holdingFileStorage, *filesBox, *import, request); + usedImports.push_back(import); } catch (const std::exception& e) { ctx.AddError(ExceptionToIssue(e)); hasErrors = true; @@ -227,7 +230,7 @@ public: } // extract regardless of hasErrors value - hasErrors = !ExtractMetadata(response, importMap, externalFunctions, ctx) || hasErrors; + hasErrors = !ExtractMetadata(response, usedImports, externalFunctions, ctx) || hasErrors; hasErrors = !LoadFunctionsMetadata(loadedFunctions, *FunctionRegistry_, TypeInfoHelper_, ctx) || hasErrors; if (!hasErrors) { @@ -316,21 +319,19 @@ private: filesBox.MakeLinkFrom(UdfDependencyStubPath_, sharedLibrary); } - static bool ExtractMetadata(const TResolveResult& response, const THashMap<TString, TImport*>& importMap, const TVector<TFunction*>& functions, TExprContext& ctx) { + static bool ExtractMetadata(const TResolveResult& response, const TVector<TImport*>& usedImports, const TVector<TFunction*>& functions, TExprContext& ctx) { bool hasErrors = false; YQL_ENSURE(response.UdfsSize() == functions.size(), "Number of returned udf signatures doesn't match original one"); + YQL_ENSURE(response.ImportsSize() >= usedImports.size(), "Number of returned udf modules is too low"); - for (size_t i = 0; i < response.ImportsSize(); ++i) { + for (size_t i = 0; i < usedImports.size(); ++i) { const TImportResult& importRes = response.GetImports(i); - TImport* import = nullptr; - if (auto p = importMap.FindPtr(importRes.GetFileAlias())) { - import = *p; - } + TImport* import = usedImports[i]; if (importRes.HasError()) { ctx.AddError(TIssue(import ? import->Pos : TPosition(), importRes.GetError())); hasErrors = true; - } else if (import) { + } else { import->Modules.ConstructInPlace(); for (auto& module : importRes.GetModules()) { import->Modules->push_back(module); diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index 031ab4ddc6..01bf58373d 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -68,7 +68,7 @@ public: THoldingFileStorage holdingFileStorage(FileStorage_); auto newRegistry = FunctionRegistry_->Clone(); - THashMap<TString, TImport*> path2import; + THashMap<std::pair<TString, TString>, THashSet<TString>> cachedModules; for (auto import: imports) { if (import->Modules) { bool needLibrary = false; @@ -87,15 +87,22 @@ public: } const TString& customUdfPrefix = import->Block->CustomUdfPrefix; try { + THashSet<TString> modules; if (FileStorage_) { auto link = holdingFileStorage.FreezeFile(*import->Block); auto path = link->GetPath().GetPath(); - newRegistry->LoadUdfs( + auto [it, inserted] = cachedModules.emplace(std::make_pair(path, customUdfPrefix), THashSet<TString>()); + if (inserted) { + newRegistry->LoadUdfs( path, {}, NUdf::IRegistrator::TFlags::TypesOnly, - customUdfPrefix); - path2import[path] = import; + customUdfPrefix, + &modules); + it->second = modules; + } else { + modules = it->second; + } } else { if (import->Block->Type != EUserDataType::PATH) { ctx.AddError(TIssue(import->Pos, TStringBuilder() << @@ -103,13 +110,21 @@ public: hasErrors = true; continue; } - newRegistry->LoadUdfs( + auto [it, inserted] = cachedModules.emplace(std::make_pair(import->Block->Data, customUdfPrefix), THashSet<TString>()); + if (inserted) { + newRegistry->LoadUdfs( import->Block->Data, {}, NUdf::IRegistrator::TFlags::TypesOnly, - customUdfPrefix); - path2import[import->Block->Data] = import; + customUdfPrefix, + &modules); + it->second = modules; + } else { + modules = it->second; + } } + + import->Modules->assign(modules.begin(), modules.end()); } catch (yexception& e) { ctx.AddError(TIssue(import->Pos, TStringBuilder() @@ -119,15 +134,6 @@ public: } } - if (!hasErrors) { - for (auto& m : newRegistry->GetAllModuleNames()) { - auto path = *newRegistry->FindUdfPath(m); - if (auto import = path2import.FindPtr(path)) { - (*import)->Modules->push_back(m); - } - } - } - hasErrors = !LoadFunctionsMetadata(functions, *newRegistry, TypeInfoHelper_, ctx) || hasErrors; return !hasErrors; } @@ -138,6 +144,10 @@ public: ythrow yexception() << "LoadRichMetadata is not supported in SimpleUdfResolver"; } + bool ContainsModule(const TStringBuf& moduleName) const override { + return FunctionRegistry_->IsLoadedUdfModule(moduleName); + } + private: mutable TAdaptiveLock Lock_; const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp index ac43a629b5..9fd436fdc5 100644 --- a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp +++ b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp @@ -113,6 +113,15 @@ public: return Fallback_->LoadRichMetadata(imports); } + bool ContainsModule(const TStringBuf& moduleName) const override { + TString moduleNameStr = TString(moduleName); + if (UdfIndex_->ContainsModule(moduleNameStr)) { + return true; + } + + return Fallback_->ContainsModule(moduleName); + } + private: bool LoadFunctionMetadata(TFunction& function, TExprContext& ctx, TFunction*& fallbackFunction, TImport*& additionalImport) const { TStringBuf moduleName, funcName; diff --git a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp index 008d4e50bd..dc9fb2e84d 100644 --- a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp +++ b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file.cpp @@ -82,12 +82,20 @@ namespace NFile { /////////////////////////////////////////////////////////////////////////////////////////////////////// TIntrusivePtr<IFunctionRegistry> MakeFunctionRegistry( - const IFunctionRegistry& functionRegistry, const TUserDataTable& files) { + const IFunctionRegistry& functionRegistry, const TUserDataTable& files, + const TFileStoragePtr& fileStorage, TVector<TFileLinkPtr>& externalFiles) { auto cloned = functionRegistry.Clone(); for (auto& d : files) { if (d.first.IsFile() && d.second.Usage.Test(EUserDataBlockUsage::Udf)) { - YQL_ENSURE(d.second.Type == EUserDataType::PATH); - cloned->LoadUdfs(d.second.Data, {}, 0, d.second.CustomUdfPrefix); + if (d.second.Type == EUserDataType::PATH) { + cloned->LoadUdfs(d.second.Data, {}, 0, d.second.CustomUdfPrefix); + } else if (fileStorage && d.second.Type == EUserDataType::URL) { + auto externalFile = fileStorage->PutUrl(d.second.Data, ""); + externalFiles.push_back(externalFile); + cloned->LoadUdfs(externalFile->GetPath().GetPath(), {}, 0, d.second.CustomUdfPrefix); + } else { + MKQL_ENSURE(false, "Unsupported block type"); + } } } return cloned; @@ -468,8 +476,9 @@ public: Services_->GetFunctionRegistry()->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); + TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks()), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *Services_->GetFunctionRegistry()); TVector<TRuntimeNode> strings; @@ -786,8 +795,9 @@ public: Services_->GetFunctionRegistry()->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); + TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks()), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); auto nodeFactory = GetYtFileFullFactory(Services_); for (auto& node: nodes) { auto data = builder.BuildLambda(*MkqlCompiler_, node, ctx); @@ -827,8 +837,9 @@ public: TScopedAlloc alloc(__LOCATION__, TAlignedPagePoolCounters(), Services_->GetFunctionRegistry()->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); + TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}), nullptr); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}, Services_->GetFileStorage(), externalFiles), nullptr); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), builder.GetFunctionRegistry()); TMkqlBuildContext ctx(*MkqlCompiler_, pgmBuilder, exprCtx); @@ -1165,8 +1176,9 @@ private: Services_->GetFunctionRegistry()->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); + TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks()), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); auto data = builder.BuildLambda(*MkqlCompiler_, input.Ptr(), exprCtx); auto transform = TFileTransformProvider(Services_, options.UserDataBlocks()); data = builder.TransformAndOptimizeProgram(data, transform); @@ -1243,8 +1255,9 @@ private: Services_->GetFunctionRegistry()->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); + TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks()), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); auto data = builder.BuildLambda(*MkqlCompiler_, node, exprCtx); auto transform = TFileTransformProvider(Services_, options.UserDataBlocks()); data = builder.TransformAndOptimizeProgram(data, transform); diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp index 80062f05c2..9d1fa28fce 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp @@ -103,7 +103,8 @@ public: if (!tableDesc.FillViews( clusterAndTable.first, clusterAndTable.second, ctx, State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider, - State_->Configuration->ViewIsolation.Get().GetOrElse(false) + State_->Configuration->ViewIsolation.Get().GetOrElse(false), + State_->Types->UdfResolver )) { return TStatus::Error; } @@ -231,7 +232,8 @@ public: if (!tableDesc.Fill( cluster, tableName, ctx, State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider, - State_->Configuration->ViewIsolation.Get().GetOrElse(false) + State_->Configuration->ViewIsolation.Get().GetOrElse(false), + State_->Types->UdfResolver )) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp index 59a9ed8863..77301f3da7 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp @@ -15,7 +15,7 @@ namespace NYql { bool TYtTableDescription::Fill( const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool allowViewIsolation) { + bool allowViewIsolation, IUdfResolver::TPtr udfResolver) { const TStructExprType* type = RowSpec ? RowSpec->GetType() : nullptr; if (!type) { TVector<const TItemExprType*> items; @@ -29,7 +29,7 @@ bool TYtTableDescription::Fill( if (!TYtTableDescriptionBase::Fill(TString{YtProviderName}, cluster, table, type, Meta->SqlView, Meta->SqlViewSyntaxVersion, Meta->Attrs, ctx, - moduleResolver, urlListerManager, randomProvider, allowViewIsolation)) { + moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver)) { return false; } if (QB2RowSpec) { @@ -245,10 +245,10 @@ void TYtTableDescription::SetConstraintsReady() { bool TYtTableDescription::FillViews( const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool allowViewIsolation) { + bool allowViewIsolation, IUdfResolver::TPtr udfResolver) { return TYtTableDescriptionBase::FillViews( TString{YtProviderName}, cluster, table, Meta->Attrs, ctx, - moduleResolver, urlListerManager, randomProvider, allowViewIsolation); + moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver); } const TYtTableDescription& TYtTablesData::GetTable(const TString& cluster, const TString& table, TMaybe<ui32> epoch) const { diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_provider.h b/ydb/library/yql/providers/yt/provider/yql_yt_provider.h index 3da522530e..90ad1d58fa 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_provider.h +++ b/ydb/library/yql/providers/yt/provider/yql_yt_provider.h @@ -47,7 +47,7 @@ struct TYtTableDescription: public TYtTableDescriptionBase { bool Fill( const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool allowViewIsolation); + bool allowViewIsolation, IUdfResolver::TPtr udfResolver); void ToYson(NYson::TYsonWriter& writer, const TString& cluster, const TString& table, const TString& view) const; bool Validate(TPosition pos, TStringBuf cluster, TStringBuf tableName, bool withQB, const THashMap<std::pair<TString, TString>, TString>& anonymousLabels, TExprContext& ctx) const; @@ -55,7 +55,7 @@ struct TYtTableDescription: public TYtTableDescriptionBase { bool FillViews( const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool allowViewIsolation); + bool allowViewIsolation, IUdfResolver::TPtr udfResolver); }; // Anonymous tables are kept by labels diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp index c5c7cff252..33f499d01f 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp @@ -150,7 +150,8 @@ TExprNode::TPtr BuildIgnoreTypeV3Remapper(const TStructExprType* rowType, TExprC } TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, - TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation) + TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, + IRandomProvider& randomProvider, bool enableViewIsolation, IUdfResolver::TPtr udfResolver) { NSQLTranslation::TTranslationSettings settings; settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; @@ -179,6 +180,10 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, return {}; } + if (!enableViewIsolation) { + return exprRoot; + } + constexpr TStringBuf OuterFuncs[] = { "SecureParam", "CurrentOperationId", @@ -192,50 +197,70 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, "FolderPath", "Files", "Configure!", + "Udf", + "ScriptUdf", + "SqlCall", }; - bool hasError = false; - VisitExpr(*exprRoot, [&](const TExprNode& node) { + TOptimizeExprSettings optSettings(nullptr); + optSettings.VisitChanges = true; + auto status = OptimizeExpr(exprRoot, exprRoot, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { for (const auto& name : OuterFuncs) { - if (node.IsCallable(name)) { - hasError = true; - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << name << " function can't be used in views")); - return false; + if (node->IsCallable(name)) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << name << " function can't be used in views")); + return nullptr; } } - if (node.IsCallable("FuncCode") && node.ChildrenSize() > 0) { - if (!node.Head().IsCallable("String")) { - hasError = true; - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "FuncCode should have constant function name in views")); - return false; + if (node->IsCallable("FuncCode") && node->ChildrenSize() > 0) { + if (!node->Head().IsCallable("String")) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "FuncCode should have constant function name in views")); + return nullptr; } - if (node.Head().Head().IsAtom()) { + if (node->Head().Head().IsAtom()) { for (const auto& name : OuterFuncs) { - if (node.Head().Head().Content() == name) { - hasError = true; - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << name << " function can't be used in views")); - return false; + if (node->Head().Head().Content() == name) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << name << " function can't be used in views")); + return nullptr; } } for (const auto& name : CodegenFuncs) { - if (node.Head().Head().Content() == name) { - hasError = true; - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << name << " function can't be used inside generated code in views")); - return false; + if (node->Head().Head().Content() == name) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << name << " function can't be used inside generated code in views")); + return nullptr; } } } } - return true; - }); + if (node->IsCallable("ScriptUdf") && node->ChildrenSize() > 0 && node->Head().Content().StartsWith("CustomPython")) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "CustomPython module can't be used in views")); + return nullptr; + } - if (hasError) { - return {}; - } + if (node->IsCallable({"Udf","SqlCall"}) && node->Head().IsAtom()) { + auto origFunc = node->Head().Content(); + TStringBuf moduleName, funcName; + if (!SplitUdfName(origFunc, moduleName, funcName)) { + return node; + } + + if (udfResolver->ContainsModule(TString(moduleName))) { + return node; + } + + return ctx.ChangeChild(*node, 0, + ctx.NewAtom(node->Head().Pos(), settings.FileAliasPrefix + origFunc)); + } + + return node; + }, ctx, optSettings); + + if (status == IGraphTransformer::TStatus::Error) { + return nullptr; + }; return exprRoot; } @@ -244,10 +269,11 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, bool TYtViewDescription::Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx, - IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation) + IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation, + IUdfResolver::TPtr udfResolver) { Sql = sql; - CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation); + CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver); return bool(CompiledSql); } @@ -258,7 +284,8 @@ void TYtViewDescription::CleanupCompiledSQL() bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& cluster, const TString& table, const TStructExprType* type, const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs, - TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation) + TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation, + IUdfResolver::TPtr udfResolver) { // (1) row type RawRowType = type; @@ -337,13 +364,13 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust } // (3) views - if (!FillViews(provider, cluster, table, metaAttrs, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation)) { + if (!FillViews(provider, cluster, table, metaAttrs, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) { return false; } if (viewSql) { if (!View) { - if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation)) { + if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) { ctx.AddError(TIssue(TPosition(), TStringBuilder() << "Can't load sql view, table: " << cluster << '.' << table)); return false; @@ -356,7 +383,7 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString& cluster, const TString& table, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, - IRandomProvider& randomProvider, bool allowViewIsolation) + IRandomProvider& randomProvider, bool allowViewIsolation, IUdfResolver::TPtr udfResolver) { for (auto& view: Views) { TYtViewDescription& viewDesc = view.second; @@ -384,7 +411,7 @@ bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString& } } - if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, allowViewIsolation)) { + if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver)) { ctx.AddError(TIssue(TPosition(), TStringBuilder() << "Can't load sql view " << viewSql.Quote() << ", table: " << cluster << '.' << table diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h index 5acc1067ab..d97389ad10 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h +++ b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h @@ -2,6 +2,7 @@ #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h> +#include <ydb/library/yql/core/yql_udf_resolver.h> #include <library/cpp/random_provider/random_provider.h> @@ -49,7 +50,8 @@ struct TYtViewDescription { const TTypeAnnotationNode* RowType = nullptr; // Filled only if scheme requested bool Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx, - IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation); + IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, + bool enableViewIsolation, IUdfResolver::TPtr udfResolver); void CleanupCompiledSQL(); }; @@ -71,11 +73,11 @@ struct TYtTableDescriptionBase { bool Fill(const TString& provider, const TString& cluster, const TString& table, const TStructExprType* type, const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool enableViewIsolation); + bool enableViewIsolation, IUdfResolver::TPtr udfResolver); void CleanupCompiledSQL(); bool FillViews(const TString& provider, const TString& cluster, const TString& table, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, - bool enableViewIsolation); + bool enableViewIsolation, IUdfResolver::TPtr udfResolver); }; } diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index b2f18df5f5..4675cd149f 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -1429,6 +1429,21 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success return {}; } + if (Ctx.Settings.FileAliasPrefix) { + if (values.size() == 1) { + values.emplace_back(TDeferredAtom(Ctx.Pos(), "")); + } + + TString prefix; + if (!values[1].GetLiteral(prefix, Ctx)) { + Error() << "Expected literal UDF module prefix in views"; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + values[1] = TDeferredAtom(Ctx.Pos(), Ctx.Settings.FileAliasPrefix + prefix); + } + Ctx.IncrementMonCounter("sql_pragma", "udf"); success = true; return BuildPragma(Ctx.Pos(), TString(ConfigProviderName), "ImportUdfs", values, false); diff --git a/ydb/library/yql/tests/sql/dq_file.make b/ydb/library/yql/tests/sql/dq_file.make index 8bf171faf8..e7b3fff36e 100644 --- a/ydb/library/yql/tests/sql/dq_file.make +++ b/ydb/library/yql/tests/sql/dq_file.make @@ -23,6 +23,7 @@ DEPENDS( ydb/library/yql/tools/dqrun ydb/library/yql/tools/yqlrun ydb/library/yql/tests/common/test_framework/udfs_deps + ydb/library/yql/udfs/test/test_import ) DATA( arcadia/ydb/library/yql/tests/sql # python files diff --git a/ydb/library/yql/tests/sql/dq_file/part2/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part2/canondata/result.json index cb2d3b0242..61ad055355 100644 --- a/ydb/library/yql/tests/sql/dq_file/part2/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part2/canondata/result.json @@ -5794,6 +5794,11 @@ "uri": "file://test.test_view-file_outer--Results_/extracted" } ], + "test.test[view-file_outer_library--Results]": [ + { + "uri": "file://test.test_view-file_outer_library--Results_/extracted" + } + ], "test.test[weak_field-yql-7888_mapfieldsubset--Analyze]": [ { "checksum": "794347e345742152325e2c4ad845019f", diff --git a/ydb/library/yql/tests/sql/dq_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted b/ydb/library/yql/tests/sql/dq_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted new file mode 100644 index 0000000000..1bd5f23f60 --- /dev/null +++ b/ydb/library/yql/tests/sql/dq_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted @@ -0,0 +1,8 @@ +<tmp_path>/program.sql:<main>: Error: Table metadata loading + + <tmp_path>/program.sql:<main>: Error: Table Input + + <tmp_path>/program.sql:<main>:1:22: Error: Module 'mylib_module0''view_bb686f68-4245bd5f-2318fa8e-22eb9250/mylib' does not exist + /* postgres can not */ + ^ + <tmp_path>/program.sql:<main>: Error: Can't load sql view "IMPORT mylib SYMBOLS $x;select $x as k, 2 as s, 3 as v", table: plato.Input, view: file_outer_library
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json index fd806afb42..70b05df2ef 100644 --- a/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json @@ -5032,6 +5032,28 @@ } ], "test.test[union_all-union_all_subexpr-default.txt-Results]": [], + "test.test[view-system_udf--Analyze]": [ + { + "checksum": "ea3c648992488b04578c5ac72fcb0576", + "size": 3640, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/2aaad78dd4884290d625cc79ceae25742307829c/resource.tar.gz#test.test_view-system_udf--Analyze_/plan.txt" + } + ], + "test.test[view-system_udf--Debug]": [ + { + "checksum": "c1791f784b0176ea6ffea0cc068d3555", + "size": 1953, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/2aaad78dd4884290d625cc79ceae25742307829c/resource.tar.gz#test.test_view-system_udf--Debug_/opt.yql_patched" + } + ], + "test.test[view-system_udf--Plan]": [ + { + "checksum": "ea3c648992488b04578c5ac72fcb0576", + "size": 3640, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/2aaad78dd4884290d625cc79ceae25742307829c/resource.tar.gz#test.test_view-system_udf--Plan_/plan.txt" + } + ], + "test.test[view-system_udf--Results]": [], "test.test[weak_field-weak_field--Analyze]": [ { "checksum": "7e22acde121c5b6a7069afa01ccae258", diff --git a/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json index 2c975e7565..ae0e52d1b2 100644 --- a/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json @@ -5405,6 +5405,13 @@ } ], "test.test[union_all-union_all_with_top_level_limits_ansi-default.txt-Results]": [], + "test.test[view-file_inner_udf--Analyze]": [ + { + "checksum": "1e0bc616fa564e7c0c72a2af6e583b93", + "size": 5517, + "uri": "https://storage.yandex-team.ru/get-devtools/1775059/4ce689cacc1b04e7b955e62a2269c8180fca36bb/resource.tar.gz#test.test_view-file_inner_udf--Analyze_/plan.txt" + } + ], "test.test[view-trivial_view--Analyze]": [ { "checksum": "6eee21387ac9a19187d0bfda16852d86", diff --git a/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json index 4ecd152c10..d04aec767d 100644 --- a/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json @@ -5286,6 +5286,28 @@ } ], "test.test[view-all_from_view--Results]": [], + "test.test[view-file_inner_library--Analyze]": [ + { + "checksum": "212be881133a20b5b73ef1250dbeda51", + "size": 960, + "uri": "https://storage.yandex-team.ru/get-devtools/1942100/d0c70441f77c59d324dc5d88b00985f8d51475a9/resource.tar.gz#test.test_view-file_inner_library--Analyze_/plan.txt" + } + ], + "test.test[view-file_inner_library--Debug]": [ + { + "checksum": "00eea88e4f7e62409a6f728ffe5cb0a0", + "size": 484, + "uri": "https://storage.yandex-team.ru/get-devtools/1942100/d0c70441f77c59d324dc5d88b00985f8d51475a9/resource.tar.gz#test.test_view-file_inner_library--Debug_/opt.yql_patched" + } + ], + "test.test[view-file_inner_library--Plan]": [ + { + "checksum": "212be881133a20b5b73ef1250dbeda51", + "size": 960, + "uri": "https://storage.yandex-team.ru/get-devtools/1942100/d0c70441f77c59d324dc5d88b00985f8d51475a9/resource.tar.gz#test.test_view-file_inner_library--Plan_/plan.txt" + } + ], + "test.test[view-file_inner_library--Results]": [], "test.test[view-secure_eval_dyn--Results]": [ { "uri": "file://test.test_view-secure_eval_dyn--Results_/extracted" diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index b1c9b63f3c..01f4d6a8fe 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -15798,6 +15798,20 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1936273/12b0fce12ec386ac74f561a826f56b64f91391eb/resource.tar.gz#test_sql2yql.test_view-file_inner_/sql.yql" } ], + "test_sql2yql.test[view-file_inner_library]": [ + { + "checksum": "3195311b43069c7cea58a30529a787da", + "size": 1504, + "uri": "https://storage.yandex-team.ru/get-devtools/1936947/963d1704490452e3088d0f61e8e57b3b23b54160/resource.tar.gz#test_sql2yql.test_view-file_inner_library_/sql.yql" + } + ], + "test_sql2yql.test[view-file_inner_udf]": [ + { + "checksum": "e56a9a74c5b5d27c249b8708832bf8d7", + "size": 2376, + "uri": "https://storage.yandex-team.ru/get-devtools/1871182/78c93a3a7026d654a695a4da702ab87241ec4ff7/resource.tar.gz#test_sql2yql.test_view-file_inner_udf_/sql.yql" + } + ], "test_sql2yql.test[view-file_outer]": [ { "checksum": "cb7f58e243fc2b161e9c2ad2783c4e03", @@ -15805,6 +15819,13 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1936273/12b0fce12ec386ac74f561a826f56b64f91391eb/resource.tar.gz#test_sql2yql.test_view-file_outer_/sql.yql" } ], + "test_sql2yql.test[view-file_outer_library]": [ + { + "checksum": "fbc64bb959d66bcbc998a9ec6a2fc58c", + "size": 1526, + "uri": "https://storage.yandex-team.ru/get-devtools/1871182/78c93a3a7026d654a695a4da702ab87241ec4ff7/resource.tar.gz#test_sql2yql.test_view-file_outer_library_/sql.yql" + } + ], "test_sql2yql.test[view-init_view_after_eval]": [ { "checksum": "db1af9f900a72e1f750beed31c5e898b", @@ -15814,23 +15835,30 @@ ], "test_sql2yql.test[view-secure]": [ { - "checksum": "c40d481aea3b60246c76b0ca3c3eda88", - "size": 1401, - "uri": "https://storage.yandex-team.ru/get-devtools/1871002/192508ba8ad4749d5bbae0501a749b4bb1f0d76d/resource.tar.gz#test_sql2yql.test_view-secure_/sql.yql" + "checksum": "fdc8ee2514c8ab4574b9146ebed49d95", + "size": 1492, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql2yql.test_view-secure_/sql.yql" } ], "test_sql2yql.test[view-secure_eval]": [ { - "checksum": "65da6f4a9b70da321d82347b20c668ae", - "size": 1406, - "uri": "https://storage.yandex-team.ru/get-devtools/1942415/7da5301d8953b4fe62e551695771727982e93185/resource.tar.gz#test_sql2yql.test_view-secure_eval_/sql.yql" + "checksum": "57b11f22da4e21e09954912dd50f0937", + "size": 1497, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql2yql.test_view-secure_eval_/sql.yql" } ], "test_sql2yql.test[view-secure_eval_dyn]": [ { - "checksum": "4ab7d66e4f81830c5e8f553b7483157c", - "size": 1414, - "uri": "https://storage.yandex-team.ru/get-devtools/1942415/7da5301d8953b4fe62e551695771727982e93185/resource.tar.gz#test_sql2yql.test_view-secure_eval_dyn_/sql.yql" + "checksum": "419ded953f8f9b8017fc42f0f7b5b7ba", + "size": 1505, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql2yql.test_view-secure_eval_dyn_/sql.yql" + } + ], + "test_sql2yql.test[view-system_udf]": [ + { + "checksum": "ffcddc5abbce83f3c5258d83b31e4e6b", + "size": 1496, + "uri": "https://storage.yandex-team.ru/get-devtools/1931696/42a6f80fb9a2cfd80b7c1295017e9eb6eae3c0b6/resource.tar.gz#test_sql2yql.test_view-system_udf_/sql.yql" } ], "test_sql2yql.test[view-trivial_view]": [ @@ -30435,6 +30463,20 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1936273/12b0fce12ec386ac74f561a826f56b64f91391eb/resource.tar.gz#test_sql_format.test_view-file_inner_/formatted.sql" } ], + "test_sql_format.test[view-file_inner_library]": [ + { + "checksum": "2d1e41c015c6b640e4c89654b3807517", + "size": 158, + "uri": "https://storage.yandex-team.ru/get-devtools/1936947/963d1704490452e3088d0f61e8e57b3b23b54160/resource.tar.gz#test_sql_format.test_view-file_inner_library_/formatted.sql" + } + ], + "test_sql_format.test[view-file_inner_udf]": [ + { + "checksum": "886157e34fa2543bbbf0f2a8bc462950", + "size": 228, + "uri": "https://storage.yandex-team.ru/get-devtools/1871182/78c93a3a7026d654a695a4da702ab87241ec4ff7/resource.tar.gz#test_sql_format.test_view-file_inner_udf_/formatted.sql" + } + ], "test_sql_format.test[view-file_outer]": [ { "checksum": "f4927ace78d10a4e90b80f463a5129a5", @@ -30442,6 +30484,13 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1936273/12b0fce12ec386ac74f561a826f56b64f91391eb/resource.tar.gz#test_sql_format.test_view-file_outer_/formatted.sql" } ], + "test_sql_format.test[view-file_outer_library]": [ + { + "checksum": "7d32049705e9db6cc34c43130c67a0bf", + "size": 187, + "uri": "https://storage.yandex-team.ru/get-devtools/1871182/78c93a3a7026d654a695a4da702ab87241ec4ff7/resource.tar.gz#test_sql_format.test_view-file_outer_library_/formatted.sql" + } + ], "test_sql_format.test[view-init_view_after_eval]": [ { "checksum": "2b80cc480f3257c380fa17a433bf7d55", @@ -30451,23 +30500,30 @@ ], "test_sql_format.test[view-secure]": [ { - "checksum": "f7a4451bd07ee41c06745e9766e5d99f", - "size": 112, - "uri": "https://storage.yandex-team.ru/get-devtools/1871002/192508ba8ad4749d5bbae0501a749b4bb1f0d76d/resource.tar.gz#test_sql_format.test_view-secure_/formatted.sql" + "checksum": "01b3b4504162a4838b20a430c8694111", + "size": 146, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql_format.test_view-secure_/formatted.sql" } ], "test_sql_format.test[view-secure_eval]": [ { - "checksum": "0a3275b6939091199795880d719b0851", - "size": 117, - "uri": "https://storage.yandex-team.ru/get-devtools/1942415/7da5301d8953b4fe62e551695771727982e93185/resource.tar.gz#test_sql_format.test_view-secure_eval_/formatted.sql" + "checksum": "f62fc74e339177a66f07678a3e312a9c", + "size": 151, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql_format.test_view-secure_eval_/formatted.sql" } ], "test_sql_format.test[view-secure_eval_dyn]": [ { - "checksum": "7481da5b8448c77add6da438a574a8b6", - "size": 125, - "uri": "https://storage.yandex-team.ru/get-devtools/1942415/7da5301d8953b4fe62e551695771727982e93185/resource.tar.gz#test_sql_format.test_view-secure_eval_dyn_/formatted.sql" + "checksum": "419c3de8cea51c291a756e32cb23af57", + "size": 159, + "uri": "https://storage.yandex-team.ru/get-devtools/1784826/9b2d34dc2cba491917a5449ca63c32b975fbc386/resource.tar.gz#test_sql_format.test_view-secure_eval_dyn_/formatted.sql" + } + ], + "test_sql_format.test[view-system_udf]": [ + { + "checksum": "ddace5499ed2df629996864120d65773", + "size": 150, + "uri": "https://storage.yandex-team.ru/get-devtools/1931696/42a6f80fb9a2cfd80b7c1295017e9eb6eae3c0b6/resource.tar.gz#test_sql_format.test_view-system_udf_/formatted.sql" } ], "test_sql_format.test[view-trivial_view]": [ diff --git a/ydb/library/yql/tests/sql/suites/view/file_inner_library.cfg b/ydb/library/yql/tests/sql/suites/view/file_inner_library.cfg new file mode 100644 index 0000000000..51e0aafced --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_inner_library.cfg @@ -0,0 +1,2 @@ +in Input view_file_inner_library.txt +http_file mylib.sql mylib.sql.txt diff --git a/ydb/library/yql/tests/sql/suites/view/file_inner_library.sql b/ydb/library/yql/tests/sql/suites/view/file_inner_library.sql new file mode 100644 index 0000000000..f81464e158 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_inner_library.sql @@ -0,0 +1,5 @@ +/* postgres can not */ +/* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; +USE plato; +SELECT k, s, v FROM Input VIEW file_inner_library; diff --git a/ydb/library/yql/tests/sql/suites/view/file_inner_udf.cfg b/ydb/library/yql/tests/sql/suites/view/file_inner_udf.cfg new file mode 100644 index 0000000000..f8e0bedf5a --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_inner_udf.cfg @@ -0,0 +1,4 @@ +in Input view_file_inner_udf.txt +http_file myfile ydb/library/yql/udfs/test/test_import/libtest_import_udf.so +os linux +providers yt diff --git a/ydb/library/yql/tests/sql/suites/view/file_inner_udf.sql b/ydb/library/yql/tests/sql/suites/view/file_inner_udf.sql new file mode 100644 index 0000000000..9b1a90e856 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_inner_udf.sql @@ -0,0 +1,8 @@ +/* postgres can not */ +/* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; +USE plato; +SELECT k, s, v FROM Input VIEW file_inner_udf1 +UNION ALL +SELECT k, s, v FROM Input VIEW file_inner_udf2; + diff --git a/ydb/library/yql/tests/sql/suites/view/file_outer_library.cfg b/ydb/library/yql/tests/sql/suites/view/file_outer_library.cfg new file mode 100644 index 0000000000..cf7d8cce56 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_outer_library.cfg @@ -0,0 +1,4 @@ +in Input view_file_outer_library.txt +http_file mylib.sql mylib.sql.txt +xfail + diff --git a/ydb/library/yql/tests/sql/suites/view/file_outer_library.sql b/ydb/library/yql/tests/sql/suites/view/file_outer_library.sql new file mode 100644 index 0000000000..b7e7ee2ad4 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/file_outer_library.sql @@ -0,0 +1,6 @@ +/* postgres can not */ +/* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; +USE plato; +PRAGMA library('mylib.sql'); +SELECT k, s, v FROM Input VIEW file_outer_library; diff --git a/ydb/library/yql/tests/sql/suites/view/mylib.sql.txt b/ydb/library/yql/tests/sql/suites/view/mylib.sql.txt new file mode 100644 index 0000000000..cd551d25c7 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/mylib.sql.txt @@ -0,0 +1,2 @@ +$x = 1; +export $x; diff --git a/ydb/library/yql/tests/sql/suites/view/secure.sql b/ydb/library/yql/tests/sql/suites/view/secure.sql index a93867374c..be73998ca2 100644 --- a/ydb/library/yql/tests/sql/suites/view/secure.sql +++ b/ydb/library/yql/tests/sql/suites/view/secure.sql @@ -1,4 +1,5 @@ /* postgres can not */ /* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; USE plato; SELECT k, s, v FROM Input VIEW secure; diff --git a/ydb/library/yql/tests/sql/suites/view/secure_eval.sql b/ydb/library/yql/tests/sql/suites/view/secure_eval.sql index 704df570e8..32f07f32df 100644 --- a/ydb/library/yql/tests/sql/suites/view/secure_eval.sql +++ b/ydb/library/yql/tests/sql/suites/view/secure_eval.sql @@ -1,4 +1,5 @@ /* postgres can not */ /* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; USE plato; SELECT k, s, v FROM Input VIEW secure_eval; diff --git a/ydb/library/yql/tests/sql/suites/view/secure_eval_dyn.sql b/ydb/library/yql/tests/sql/suites/view/secure_eval_dyn.sql index 7201183259..44f17b18da 100644 --- a/ydb/library/yql/tests/sql/suites/view/secure_eval_dyn.sql +++ b/ydb/library/yql/tests/sql/suites/view/secure_eval_dyn.sql @@ -1,4 +1,5 @@ /* postgres can not */ /* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; USE plato; SELECT k, s, v FROM Input VIEW secure_eval_dynamic; diff --git a/ydb/library/yql/tests/sql/suites/view/system_udf.cfg b/ydb/library/yql/tests/sql/suites/view/system_udf.cfg new file mode 100644 index 0000000000..3760566137 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/system_udf.cfg @@ -0,0 +1,2 @@ +in Input view_system_udf.txt +udf string_udf diff --git a/ydb/library/yql/tests/sql/suites/view/system_udf.sql b/ydb/library/yql/tests/sql/suites/view/system_udf.sql new file mode 100644 index 0000000000..e5599aa9c2 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/system_udf.sql @@ -0,0 +1,5 @@ +/* postgres can not */ +/* syntax version 1 */ +pragma yt.ViewIsolation = 'true'; +USE plato; +SELECT k, s, v FROM Input VIEW system_udf; diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt b/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt new file mode 100644 index 0000000000..df76c64125 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="1";"value"="a7c"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="3";"value"="q"}; +{"key"="150";"subkey"="4";"value"="q8z"}; diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt.attr b/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt.attr new file mode 100644 index 0000000000..340976f926 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_inner_library.txt.attr @@ -0,0 +1,32 @@ +{ + "_yql_view_file_inner_library"="PRAGMA library('mylib.sql','http_test://mylib.sql');IMPORT mylib SYMBOLS $x;select $x as k, 2 as s, 3 as v"; + "_yql_syntax_version_file_inner_library"=1; + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt b/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt new file mode 100644 index 0000000000..df76c64125 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="1";"value"="a7c"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="3";"value"="q"}; +{"key"="150";"subkey"="4";"value"="q8z"}; diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt.attr b/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt.attr new file mode 100644 index 0000000000..224776286f --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_inner_udf.txt.attr @@ -0,0 +1,34 @@ +{ + "_yql_view_file_inner_udf1"="PRAGMA file('myfile','http_test://myfile');PRAGMA udf('myfile');select TestImportUdf::Concat(key,'!') as k, 2 as s, 3 as v from self"; + "_yql_syntax_version_file_inner_udf1"=1; + "_yql_view_file_inner_udf2"="PRAGMA file('myfile','http_test://myfile');PRAGMA udf('myfile');select TestImportUdf::Concat(key,'?') as k, 2 as s, 3 as v from self"; + "_yql_syntax_version_file_inner_udf2"=1; + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt b/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt new file mode 100644 index 0000000000..df76c64125 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="1";"value"="a7c"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="3";"value"="q"}; +{"key"="150";"subkey"="4";"value"="q8z"}; diff --git a/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt.attr b/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt.attr new file mode 100644 index 0000000000..1237df5e5c --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_file_outer_library.txt.attr @@ -0,0 +1,32 @@ +{ + "_yql_view_file_outer_library"="IMPORT mylib SYMBOLS $x;select $x as k, 2 as s, 3 as v"; + "_yql_syntax_version_file_outer_library"=1; + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} diff --git a/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt b/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt new file mode 100644 index 0000000000..df76c64125 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="1";"value"="a7c"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="3";"value"="q"}; +{"key"="150";"subkey"="4";"value"="q8z"}; diff --git a/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt.attr b/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt.attr new file mode 100644 index 0000000000..fb7a941331 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/view/view_system_udf.txt.attr @@ -0,0 +1,32 @@ +{ + "_yql_view_system_udf"="SELECT String::Strip(key) as k, subkey as s, value as v FROM self"; + "_yql_syntax_version_system_udf"=1; + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} diff --git a/ydb/library/yql/tests/sql/yt_file.make b/ydb/library/yql/tests/sql/yt_file.make index f0f8493fe0..2fa88a44db 100644 --- a/ydb/library/yql/tests/sql/yt_file.make +++ b/ydb/library/yql/tests/sql/yt_file.make @@ -22,6 +22,7 @@ DEPENDS( ydb/library/yql/tools/astdiff ydb/library/yql/tools/yqlrun ydb/library/yql/tests/common/test_framework/udfs_deps + ydb/library/yql/udfs/test/test_import ) DATA( arcadia/ydb/library/yql/tests/sql # python files diff --git a/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json index 1e144de1e2..034beb6245 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/result.json @@ -5294,6 +5294,13 @@ "uri": "file://test.test_view-file_outer--Results_/extracted" } ], + "test.test[view-file_outer_library--Debug]": [], + "test.test[view-file_outer_library--Plan]": [], + "test.test[view-file_outer_library--Results]": [ + { + "uri": "file://test.test_view-file_outer_library--Results_/extracted" + } + ], "test.test[weak_field-yql-7888_mapfieldsubset--Debug]": [ { "checksum": "f9fd1a3080a43c21f74630204b263469", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted new file mode 100644 index 0000000000..1bd5f23f60 --- /dev/null +++ b/ydb/library/yql/tests/sql/yt_native_file/part2/canondata/test.test_view-file_outer_library--Results_/extracted @@ -0,0 +1,8 @@ +<tmp_path>/program.sql:<main>: Error: Table metadata loading + + <tmp_path>/program.sql:<main>: Error: Table Input + + <tmp_path>/program.sql:<main>:1:22: Error: Module 'mylib_module0''view_bb686f68-4245bd5f-2318fa8e-22eb9250/mylib' does not exist + /* postgres can not */ + ^ + <tmp_path>/program.sql:<main>: Error: Can't load sql view "IMPORT mylib SYMBOLS $x;select $x as k, 2 as s, 3 as v", table: plato.Input, view: file_outer_library
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json index a9c96c3218..4feb71c2a9 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json @@ -4636,6 +4636,27 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1031349/4e05335b70e88fbe50dc5b25964d793828845cf6/resource.tar.gz#test.test_union_all-union_all_subexpr-default.txt-Results_/results.txt" } ], + "test.test[view-system_udf--Debug]": [ + { + "checksum": "9a858b92a0ed057a1bf33361fb257633", + "size": 2035, + "uri": "https://storage.yandex-team.ru/get-devtools/1689644/1f27d82d0e1c3df2347579d3dfe480b03afc06b5/resource.tar.gz#test.test_view-system_udf--Debug_/opt.yql" + } + ], + "test.test[view-system_udf--Plan]": [ + { + "checksum": "e22c95d1936a82df6ec053f6263b3180", + "size": 3748, + "uri": "https://storage.yandex-team.ru/get-devtools/1689644/1f27d82d0e1c3df2347579d3dfe480b03afc06b5/resource.tar.gz#test.test_view-system_udf--Plan_/plan.txt" + } + ], + "test.test[view-system_udf--Results]": [ + { + "checksum": "a56683c0c85e277efa533fee73216a18", + "size": 1643, + "uri": "https://storage.yandex-team.ru/get-devtools/1689644/1f27d82d0e1c3df2347579d3dfe480b03afc06b5/resource.tar.gz#test.test_view-system_udf--Results_/results.txt" + } + ], "test.test[weak_field-weak_field--Debug]": [ { "checksum": "96f16498e2cace9f7cfeec4a948960fe", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json index 30f54615af..96bd019e20 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json @@ -5007,6 +5007,27 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1937150/b466c661905bdae484e68f0651bb2615aefc377c/resource.tar.gz#test.test_union_all-union_all_with_top_level_limits_ansi-default.txt-Results_/results.txt" } ], + "test.test[view-file_inner_udf--Debug]": [ + { + "checksum": "7a233b8d9bef9c818a8e0d656e59638a", + "size": 3593, + "uri": "https://storage.yandex-team.ru/get-devtools/1942100/6297aecbc0694806217b8a52e6fb0cd700717ad7/resource.tar.gz#test.test_view-file_inner_udf--Debug_/opt.yql" + } + ], + "test.test[view-file_inner_udf--Plan]": [ + { + "checksum": "ee9cde74803031c782857be7ba3085d1", + "size": 5216, + "uri": "https://storage.yandex-team.ru/get-devtools/1937424/b85ac811084823d85a64484cfff23bd15899ef59/resource.tar.gz#test.test_view-file_inner_udf--Plan_/plan.txt" + } + ], + "test.test[view-file_inner_udf--Results]": [ + { + "checksum": "94513895bbfce10e96014c7759a9b1d0", + "size": 2175, + "uri": "https://storage.yandex-team.ru/get-devtools/1937424/b85ac811084823d85a64484cfff23bd15899ef59/resource.tar.gz#test.test_view-file_inner_udf--Results_/results.txt" + } + ], "test.test[view-trivial_view--Debug]": [ { "checksum": "83e025e2244eb4fed6beb3a7b8d33ad2", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json index dca9e570a7..009361b1bf 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json @@ -4893,6 +4893,27 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1942415/213c597958ef31f8fd947ef18d05ecd9e11ff9e0/resource.tar.gz#test.test_view-all_from_view--Results_/results.txt" } ], + "test.test[view-file_inner_library--Debug]": [ + { + "checksum": "26b3469f8cbc6b7b1c54807777c91bc0", + "size": 402, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/c26ebfef92da5b72edbe456e4ce10e4d77cef322/resource.tar.gz#test.test_view-file_inner_library--Debug_/opt.yql" + } + ], + "test.test[view-file_inner_library--Plan]": [ + { + "checksum": "212be881133a20b5b73ef1250dbeda51", + "size": 960, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/c26ebfef92da5b72edbe456e4ce10e4d77cef322/resource.tar.gz#test.test_view-file_inner_library--Plan_/plan.txt" + } + ], + "test.test[view-file_inner_library--Results]": [ + { + "checksum": "92c66901bc19a741c6a514fff67c7f74", + "size": 1233, + "uri": "https://storage.yandex-team.ru/get-devtools/937458/c26ebfef92da5b72edbe456e4ce10e4d77cef322/resource.tar.gz#test.test_view-file_inner_library--Results_/results.txt" + } + ], "test.test[view-secure_eval_dyn--Debug]": [], "test.test[view-secure_eval_dyn--Plan]": [], "test.test[view-secure_eval_dyn--Results]": [ |