diff options
author | udovichenko-r <udovichenko-r@yandex-team.com> | 2024-11-12 22:07:02 +0300 |
---|---|---|
committer | udovichenko-r <udovichenko-r@yandex-team.com> | 2024-11-12 22:21:07 +0300 |
commit | fa5655229271d7a09cce8033d1097f1b03daf94e (patch) | |
tree | 5922c2db17789e411b6cc46069c66188e834f28b | |
parent | 77c13da33ae29c033359e516ac2eb55a6c3d5e9e (diff) | |
download | ydb-fa5655229271d7a09cce8033d1097f1b03daf94e.tar.gz |
Apply GH commits
Apply GH: Extract prefix and entries in backup-related sql (#10807)
Apply GH: Fix syntax for Column Family (#10781)
Apply GH: Case-insensitive mode for searching modules and functions (#10842)
Apply GH: Fixed i/o for pg_proc (#10914)
Apply GH: An option to render SQL transalation with Seq! (#11015)
commit_hash:d2d2fcdef2bbd0434236aef325aa071c7e39c526
31 files changed, 881 insertions, 163 deletions
diff --git a/yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp b/yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp index 2980c808d0..15cc73400a 100644 --- a/yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp +++ b/yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp @@ -109,6 +109,7 @@ private: TString SaveValue(const TFunction* f) const { auto node = NYT::TNode() + ("NormalizedName", f->NormalizedName) ("CallableType", TypeToYsonNode(f->CallableType)); if (f->NormalizedUserType && f->NormalizedUserType->GetKind() != ETypeAnnotationKind::Void) { node("NormalizedUserType", TypeToYsonNode(f->NormalizedUserType)); @@ -131,6 +132,12 @@ private: void LoadValue(TFunction* f, const TString& value, TExprContext& ctx) const { auto node = NYT::NodeFromYsonString(value); + if (node.HasKey("NormalizedName")) { + f->NormalizedName = node["NormalizedName"].AsString(); + } else { + f->NormalizedName = f->Name; + } + f->CallableType = ParseTypeFromYson(node["CallableType"], ctx); if (node.HasKey("NormalizedUserType")) { f->NormalizedUserType = ParseTypeFromYson(node["NormalizedUserType"], ctx); diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index d301430e29..ea068f3f4a 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -2115,7 +2115,7 @@ namespace NTypeAnnImpl { } } - const auto commonItemType = CommonTypeForChildren(*input, ctx.Expr); + const auto commonItemType = CommonTypeForChildren(*input, ctx.Expr); if (!commonItemType) { return IGraphTransformer::TStatus::Error; } @@ -7590,11 +7590,16 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } + cached.NormalizedName = description.NormalizedName; cached.FunctionType = description.CallableType; cached.RunConfigType = description.RunConfigType ? description.RunConfigType : ctx.Expr.MakeType<TVoidExprType>(); cached.NormalizedUserType = description.NormalizedUserType ? description.NormalizedUserType : ctx.Expr.MakeType<TVoidExprType>(); cached.SupportsBlocks = description.SupportsBlocks; cached.IsStrict = description.IsStrict; + + if (name != cached.NormalizedName) { + ctx.Types.UdfTypeCache[std::make_tuple(cached.NormalizedName, TString(typeConfig), userType)] = cached; + } } TStringBuf typeConfig = ""; @@ -7623,7 +7628,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> TStringBuf fileAlias = udfInfo ? udfInfo->FileAlias : ""_sb; auto ret = ctx.Expr.Builder(input->Pos()) .Callable("Udf") - .Add(0, input->HeadPtr()) + .Atom(0, cached.NormalizedName) .Add(1, runConfigValue) .Add(2, ExpandType(input->Pos(), *cached.NormalizedUserType, ctx.Expr)) .Atom(3, typeConfig) diff --git a/yql/essentials/core/ut/yql_udf_index_ut.cpp b/yql/essentials/core/ut/yql_udf_index_ut.cpp index 5ff58e1267..3ac395ac18 100644 --- a/yql/essentials/core/ut/yql_udf_index_ut.cpp +++ b/yql/essentials/core/ut/yql_udf_index_ut.cpp @@ -43,7 +43,7 @@ void EnsureLinksEqual(const TDownloadLink& link1, const TDownloadLink& link2) { void EnsureContainsFunction(TUdfIndex::TPtr index, TString module, const TFunctionInfo& f) { TFunctionInfo existingFunc; - UNIT_ASSERT(index->FindFunction(module, f.Name, existingFunc)); + UNIT_ASSERT(index->FindFunction(module, f.Name, existingFunc) == TUdfIndex::EStatus::Found); EnsureFunctionsEqual(f, existingFunc); } } @@ -52,15 +52,15 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { Y_UNIT_TEST(Empty) { auto index1 = MakeIntrusive<TUdfIndex>(); - UNIT_ASSERT(!index1->ContainsModule("M1")); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M1"), TUdfIndex::EStatus::NotFound); UNIT_ASSERT(index1->FindResourceByModule("M1") == nullptr); TFunctionInfo f1; - UNIT_ASSERT(!index1->FindFunction("M1", "M1.F1", f1)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M1", "M1.F1", f1), TUdfIndex::EStatus::NotFound); auto index2 = index1->Clone(); - UNIT_ASSERT(!index2->ContainsModule("M1")); + UNIT_ASSERT_EQUAL(index2->ContainsModule("M1"), TUdfIndex::EStatus::NotFound); UNIT_ASSERT(index2->FindResourceByModule("M1") == nullptr); - UNIT_ASSERT(!index2->FindFunction("M1", "M1.F1", f1)); + UNIT_ASSERT_EQUAL(index2->FindFunction("M1", "M1.F1", f1), TUdfIndex::EStatus::NotFound); } Y_UNIT_TEST(SingleModuleAndFunction) { @@ -72,8 +72,8 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { b.AddFunction(func1); index1->RegisterResource(b.Build(), TUdfIndex::EOverrideMode::RaiseError); - UNIT_ASSERT(index1->ContainsModule("M1")); - UNIT_ASSERT(!index1->ContainsModule("M2")); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M1"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M2"), TUdfIndex::EStatus::NotFound); UNIT_ASSERT(index1->FindResourceByModule("M2") == nullptr); auto resource1 = index1->FindResourceByModule("M1"); @@ -81,19 +81,19 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { EnsureLinksEqual(resource1->Link, link1); TFunctionInfo f1; - UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f1)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M2", "M2.F1", f1), TUdfIndex::EStatus::NotFound); - UNIT_ASSERT(index1->FindFunction("M1", "M1.F1", f1)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M1", "M1.F1", f1), TUdfIndex::EStatus::Found); EnsureFunctionsEqual(f1, func1); // ensure both indexes contain the same info auto index2 = index1->Clone(); - UNIT_ASSERT(index1->ContainsModule("M1")); - UNIT_ASSERT(index2->ContainsModule("M1")); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M1"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index2->ContainsModule("M1"), TUdfIndex::EStatus::Found); TFunctionInfo f2; - UNIT_ASSERT(index2->FindFunction("M1", "M1.F1", f2)); + UNIT_ASSERT_EQUAL(index2->FindFunction("M1", "M1.F1", f2), TUdfIndex::EStatus::Found); EnsureFunctionsEqual(f1, f2); auto resource2 = index2->FindResourceByModule("M1"); @@ -140,11 +140,11 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { EnsureLinksEqual(r22->Link, link2); // check modules - UNIT_ASSERT(index1->ContainsModule("M1")); - UNIT_ASSERT(index1->ContainsModule("M2")); - UNIT_ASSERT(index1->ContainsModule("M3")); - UNIT_ASSERT(index1->ContainsModule("M4")); - UNIT_ASSERT(!index1->ContainsModule("M5")); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M1"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M2"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M3"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M4"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index1->ContainsModule("M5"), TUdfIndex::EStatus::NotFound); EnsureContainsFunction(index1, "M1", func11); EnsureContainsFunction(index1, "M1", func12); @@ -157,8 +157,8 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { TFunctionInfo f; // known func, but non-existent module - UNIT_ASSERT(!index1->FindFunction("M5", "M1.F1", f)); - UNIT_ASSERT(!index1->FindFunction("M2", "M3.F1", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M5", "M1.F1", f), TUdfIndex::EStatus::NotFound); + UNIT_ASSERT_EQUAL(index1->FindFunction("M2", "M3.F1", f), TUdfIndex::EStatus::NotFound); } Y_UNIT_TEST(ConflictRaiseError) { @@ -199,7 +199,7 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { EnsureContainsFunction(index1, "M2", func13); TFunctionInfo f; - UNIT_ASSERT(!index1->FindFunction("M3", "M3.F1", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M3", "M3.F1", f), TUdfIndex::EStatus::NotFound); } Y_UNIT_TEST(ConflictPreserveExisting) { @@ -240,7 +240,7 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { EnsureContainsFunction(index1, "M2", func13); TFunctionInfo f; - UNIT_ASSERT(!index1->FindFunction("M3", "M3.F1", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M3", "M3.F1", f), TUdfIndex::EStatus::NotFound); } Y_UNIT_TEST(ConflictReplace1WithNew) { @@ -299,9 +299,9 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { // not here anymore TFunctionInfo f; - UNIT_ASSERT(!index1->FindFunction("M1", "M1.F1", f)); - UNIT_ASSERT(!index1->FindFunction("M1", "M1.F2", f)); - UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M1", "M1.F1", f), TUdfIndex::EStatus::NotFound); + UNIT_ASSERT_EQUAL(index1->FindFunction("M1", "M1.F2", f), TUdfIndex::EStatus::NotFound); + UNIT_ASSERT_EQUAL(index1->FindFunction("M2", "M2.F1", f), TUdfIndex::EStatus::NotFound); } Y_UNIT_TEST(ConflictReplace2WithNew) { @@ -359,10 +359,63 @@ Y_UNIT_TEST_SUITE(TUdfIndexTests) { // not here anymore TFunctionInfo f; - UNIT_ASSERT(!index1->FindFunction("M1", "M1.F2", f)); - UNIT_ASSERT(!index1->FindFunction("M2", "M2.F1", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M1", "M1.F2", f), TUdfIndex::EStatus::NotFound); + UNIT_ASSERT_EQUAL(index1->FindFunction("M2", "M2.F1", f), TUdfIndex::EStatus::NotFound); - UNIT_ASSERT(!index1->FindFunction("M3", "M3.F3", f)); - UNIT_ASSERT(!index1->FindFunction("M4", "M4.F4", f)); + UNIT_ASSERT_EQUAL(index1->FindFunction("M3", "M3.F3", f), TUdfIndex::EStatus::NotFound); + UNIT_ASSERT_EQUAL(index1->FindFunction("M4", "M4.F4", f), TUdfIndex::EStatus::NotFound); + } + + Y_UNIT_TEST(SetInsensitiveSearch) { + auto index1 = MakeIntrusive<TUdfIndex>(); + index1->SetCaseSentiveSearch(false); + auto func1 = BuildFunctionInfo("M1.FA", 1); + auto func2 = BuildFunctionInfo("M1.fa", 1); + auto func3 = BuildFunctionInfo("M1.g", 1); + auto func4 = BuildFunctionInfo("mx.h", 1); + auto func5 = BuildFunctionInfo("MX.g", 1); + auto link1 = TDownloadLink::File("file1"); + + TResourceBuilder b(link1); + b.AddFunction(func1); + b.AddFunction(func2); + b.AddFunction(func3); + b.AddFunction(func4); + b.AddFunction(func5); + + index1->RegisterResource(b.Build(), TUdfIndex::EOverrideMode::RaiseError); + + auto checkIndex = [&](auto index) { + UNIT_ASSERT_EQUAL(index->ContainsModule("M1"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index->ContainsModule("m1"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index->ContainsModule("mx"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index->ContainsModule("MX"), TUdfIndex::EStatus::Found); + UNIT_ASSERT_EQUAL(index->ContainsModule("mX"), TUdfIndex::EStatus::Ambigious); + UNIT_ASSERT_EQUAL(index->ContainsModule("M3"), TUdfIndex::EStatus::NotFound); + + UNIT_ASSERT(index->FindResourceByModule("M3") == nullptr); + auto resource1 = index->FindResourceByModule("m1"); + UNIT_ASSERT(resource1 != nullptr); + EnsureLinksEqual(resource1->Link, link1); + + TFunctionInfo f; + UNIT_ASSERT_EQUAL(index->FindFunction("m1", "M1.FA", f), TUdfIndex::EStatus::Found); + EnsureFunctionsEqual(f, func1); + UNIT_ASSERT_EQUAL(index->FindFunction("m1", "m1.Fa", f), TUdfIndex::EStatus::Ambigious); + UNIT_ASSERT_EQUAL(index->FindFunction("m1", "M1.fa", f), TUdfIndex::EStatus::Found); + EnsureFunctionsEqual(f, func2); + UNIT_ASSERT_EQUAL(index->FindFunction("m1", "m1.g", f), TUdfIndex::EStatus::Found); + EnsureFunctionsEqual(f, func3); + UNIT_ASSERT_EQUAL(index->FindFunction("Mx", "mx.h", f), TUdfIndex::EStatus::Ambigious); + UNIT_ASSERT_EQUAL(index->FindFunction("mx", "mx.H", f), TUdfIndex::EStatus::Found); + EnsureFunctionsEqual(f, func4); + UNIT_ASSERT_EQUAL(index->FindFunction("MX", "mx.g", f), TUdfIndex::EStatus::Found); + EnsureFunctionsEqual(f, func5); + }; + + checkIndex(index1); + // ensure both indexes contain the same info + auto index2 = index1->Clone(); + checkIndex(index2); } } diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index 1473b9deb0..7275801c6c 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -273,6 +273,7 @@ enum class EBlockEngineMode { }; struct TUdfCachedInfo { + TString NormalizedName; const TTypeAnnotationNode* FunctionType = nullptr; const TTypeAnnotationNode* RunConfigType = nullptr; const TTypeAnnotationNode* NormalizedUserType = nullptr; diff --git a/yql/essentials/core/yql_udf_index.cpp b/yql/essentials/core/yql_udf_index.cpp index 5c936dda68..7f2053693e 100644 --- a/yql/essentials/core/yql_udf_index.cpp +++ b/yql/essentials/core/yql_udf_index.cpp @@ -80,40 +80,137 @@ void AddResolveResultToRegistry(const TResolveResult& resolveResult, const TMap< TUdfIndex::TUdfIndex() { } -TUdfIndex::TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources) +void TUdfIndex::SetCaseSentiveSearch(bool caseSensitive) { + CaseSensitive_ = caseSensitive; +} + +TUdfIndex::TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources, bool caseSensitive) : Resources_(resources) + , CaseSensitive_(caseSensitive) { - + for (const auto& x : Resources_) { + ICaseModules_[to_lower(x.first)].insert(x.first); + } } -bool TUdfIndex::ContainsModule(const TString& moduleName) const { +bool TUdfIndex::ContainsModuleStrict(const TString& moduleName) const { return Resources_.contains(moduleName); } +bool TUdfIndex::CanonizeModule(TString& moduleName) const { + if (Resources_.contains(moduleName)) { + return true; + } + + if (CaseSensitive_) { + return false; + } + + auto p = ICaseModules_.FindPtr(to_lower(moduleName)); + if (!p) { + return false; + } + + Y_ENSURE(p->size() > 0); + if (p->size() > 1) { + return false; + } + + moduleName = *p->begin(); + return true; +} + +TUdfIndex::EStatus TUdfIndex::ContainsModule(const TString& moduleName) const { + if (Resources_.contains(moduleName)) { + return EStatus::Found; + } + + if (CaseSensitive_) { + return EStatus::NotFound; + } + + auto p = ICaseModules_.FindPtr(to_lower(moduleName)); + if (!p) { + return EStatus::NotFound; + } + + Y_ENSURE(p->size() > 0); + return p->size() > 1 ? EStatus::Ambigious : EStatus::Found; +} + bool TUdfIndex::ContainsAnyModule(const TSet<TString>& modules) const { return AnyOf(modules, [this](auto& m) { - return this->ContainsModule(m); + return Resources_.contains(m); }); } -bool TUdfIndex::FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const { - auto r = FindResourceByModule(moduleName); +TUdfIndex::EStatus TUdfIndex::FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const { + auto r = Resources_.FindPtr(moduleName); if (!r) { - return false; + if (CaseSensitive_) { + return EStatus::NotFound; + } + + auto p = ICaseModules_.FindPtr(to_lower(moduleName)); + if (!p) { + return EStatus::NotFound; + } + + Y_ENSURE(p->size() > 0); + if (p->size() > 1) { + return EStatus::Ambigious; + } + + r = Resources_.FindPtr(*p->begin()); + Y_ENSURE(r); } - auto f = r->Functions.FindPtr(functionName); + auto f = (*r)->Functions.FindPtr(functionName); if (!f) { - return false; + if (CaseSensitive_) { + return EStatus::NotFound; + } + + auto p = (*r)->ICaseFuncNames.FindPtr(to_lower(functionName)); + if (!p) { + return EStatus::NotFound; + } + + Y_ENSURE(p->size() > 0); + if (p->size() > 1) { + return EStatus::Ambigious; + } + + f = (*r)->Functions.FindPtr(*p->begin()); + Y_ENSURE(f); } function = *f; - return true; + return EStatus::Found; } TResourceInfo::TPtr TUdfIndex::FindResourceByModule(const TString& moduleName) const { auto p = Resources_.FindPtr(moduleName); - return p ? *p : nullptr; + if (!p) { + if (CaseSensitive_) { + return nullptr; + } + + auto n = ICaseModules_.FindPtr(to_lower(moduleName)); + if (!n) { + return nullptr; + } + + Y_ENSURE(n->size() > 0); + if (n->size() > 1) { + return nullptr; + } + + p = Resources_.FindPtr(*n->begin()); + Y_ENSURE(p); + } + + return *p; } TSet<TResourceInfo::TPtr> TUdfIndex::FindResourcesByModules(const TSet<TString>& modules) const { @@ -130,6 +227,11 @@ TSet<TResourceInfo::TPtr> TUdfIndex::FindResourcesByModules(const TSet<TString>& void TUdfIndex::UnregisterResource(TResourceInfo::TPtr resource) { for (auto& m : resource->Modules) { Resources_.erase(m); + auto& names = ICaseModules_[to_lower(m)]; + names.erase(m); + if (names.empty()) { + ICaseModules_.erase(to_lower(m)); + } } // resource pointer should be alive here to avoid problems with erase } @@ -170,11 +272,12 @@ void TUdfIndex::RegisterResource(const TResourceInfo::TPtr& resource, EOverrideM for (auto& m : resource->Modules) { Resources_.emplace(m, resource); + ICaseModules_[to_lower(m)].insert(m); } } TIntrusivePtr<TUdfIndex> TUdfIndex::Clone() const { - return new TUdfIndex(Resources_); + return new TUdfIndex(Resources_, CaseSensitive_); } void TUdfIndex::RegisterResources(const TVector<TResourceInfo::TPtr>& resources, EOverrideMode mode) { diff --git a/yql/essentials/core/yql_udf_index.h b/yql/essentials/core/yql_udf_index.h index 41896d191d..1ac882bd81 100644 --- a/yql/essentials/core/yql_udf_index.h +++ b/yql/essentials/core/yql_udf_index.h @@ -72,10 +72,12 @@ struct TResourceInfo : public TThrRefBase { TDownloadLink Link; TSet<TString> Modules; TMap<TString, TFunctionInfo> Functions; + TMap<TString, TSet<TString>> ICaseFuncNames; void SetFunctions(const TVector<TFunctionInfo>& functions) { for (auto& f : functions) { Functions.emplace(f.Name, f); + ICaseFuncNames[to_lower(f.Name)].insert(f.Name); } } }; @@ -96,12 +98,21 @@ public: RaiseError }; + enum class EStatus { + Found, + NotFound, + Ambigious + }; + public: TUdfIndex(); - bool ContainsModule(const TString& moduleName) const; - bool FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const; + void SetCaseSentiveSearch(bool caseSensitive); + bool CanonizeModule(TString& moduleName) const; + EStatus ContainsModule(const TString& moduleName) const; + EStatus FindFunction(const TString& moduleName, const TString& functionName, TFunctionInfo& function) const; TResourceInfo::TPtr FindResourceByModule(const TString& moduleName) const; + bool ContainsModuleStrict(const TString& moduleName) const; /* New resource can contain already registered module. In this case 'mode' will be used to resolve conflicts. @@ -114,7 +125,7 @@ public: TIntrusivePtr<TUdfIndex> Clone() const; private: - explicit TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources); + explicit TUdfIndex(const TMap<TString, TResourceInfo::TPtr>& resources, bool caseSensitive); bool ContainsAnyModule(const TSet<TString>& modules) const; TSet<TResourceInfo::TPtr> FindResourcesByModules(const TSet<TString>& modules) const; @@ -123,6 +134,8 @@ private: private: // module => Resource TMap<TString, TResourceInfo::TPtr> Resources_; + bool CaseSensitive_ = true; + TMap<TString, TSet<TString>> ICaseModules_; }; void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry); diff --git a/yql/essentials/core/yql_udf_resolver.h b/yql/essentials/core/yql_udf_resolver.h index 5dfb6744ed..9b6bf4f8ee 100644 --- a/yql/essentials/core/yql_udf_resolver.h +++ b/yql/essentials/core/yql_udf_resolver.h @@ -42,6 +42,7 @@ public: THashMap<TString, TString> SecureParams; // output + TString NormalizedName; const TTypeAnnotationNode* NormalizedUserType = nullptr; const TTypeAnnotationNode* RunConfigType = nullptr; const TTypeAnnotationNode* CallableType = nullptr; diff --git a/yql/essentials/parser/pg_catalog/catalog.cpp b/yql/essentials/parser/pg_catalog/catalog.cpp index 05e6a48875..c631fca46b 100644 --- a/yql/essentials/parser/pg_catalog/catalog.cpp +++ b/yql/essentials/parser/pg_catalog/catalog.cpp @@ -1990,7 +1990,7 @@ struct TCatalog : public IExtensionSqlBuilder { State->Types[newDesc.TypeId] = newDesc; State->TypeByName[newDesc.Name] = newDesc.TypeId; TTypeDesc newArrayDesc = newDesc; - newArrayDesc.TypeId += 1; + newArrayDesc.TypeId += 1; newArrayDesc.Name = "_" + newArrayDesc.Name; newArrayDesc.ElementTypeId = newDesc.TypeId; newArrayDesc.ArrayTypeId = newArrayDesc.TypeId; @@ -2241,7 +2241,7 @@ struct TCatalog : public IExtensionSqlBuilder { static TCatalog& MutableInstance() { return *Singleton<TCatalog>(); - } + } struct TState { TExtensionsByName ExtensionsByName, ExtensionsByInstallName; @@ -2525,11 +2525,11 @@ bool CanUseCoercionType(ECoercionCode requiredCoercionLevel, ECoercionCode actua enum class ECoercionSearchResult { - None, - Func, - BinaryCompatible, - ArrayCoerce, - IOCoerce, + None, + Func, + BinaryCompatible, + ArrayCoerce, + IOCoerce, }; ECoercionSearchResult FindCoercionPath(ui32 fromTypeId, ui32 toTypeId, ECoercionCode coercionType, const TCatalog& catalog) { @@ -3600,7 +3600,7 @@ const TVector<TMaybe<TString>>* ReadTable( const auto& catalog = TCatalog::Instance(); auto dataPtr = catalog.State->StaticTablesData.FindPtr(tableKey); if (!dataPtr) { - throw yexception() << "Missing data for table " + throw yexception() << "Missing data for table " << tableKey.Schema << "." << tableKey.Name; } @@ -3654,7 +3654,7 @@ void LoadSystemFunctions(ISystemFunctionsParser& parser) { if (catalog.State->SystemFunctionInit) { return; } - + TString data; Y_ENSURE(NResource::FindExact("system_functions.sql", &data)); TVector<TProcDesc> procs; @@ -3853,6 +3853,9 @@ TString ExportExtensions(const TMaybe<TSet<ui32>>& filter) { protoProc->SetIsStrict(desc.IsStrict); protoProc->SetLang(desc.Lang); + protoProc->SetResultType(desc.ResultType); + protoProc->SetReturnSet(desc.ReturnSet); + protoProc->SetKind((ui32)desc.Kind); } TVector<TTableInfoKey> extTables; @@ -4122,6 +4125,9 @@ void ImportExtensions(const TString& exported, bool typesOnly, IExtensionLoader* desc.Src = protoProc.GetSrc(); desc.IsStrict = protoProc.GetIsStrict(); desc.Lang = protoProc.GetLang(); + desc.ResultType = protoProc.GetResultType(); + desc.ReturnSet = protoProc.GetReturnSet(); + desc.Kind = (EProcKind)protoProc.GetKind(); for (const auto t : protoProc.GetArgType()) { desc.ArgTypes.push_back(t); } diff --git a/yql/essentials/parser/pg_catalog/proto/pg_catalog.proto b/yql/essentials/parser/pg_catalog/proto/pg_catalog.proto index 396f958a1d..b36113dff6 100644 --- a/yql/essentials/parser/pg_catalog/proto/pg_catalog.proto +++ b/yql/essentials/parser/pg_catalog/proto/pg_catalog.proto @@ -41,6 +41,9 @@ message TPgProc { optional string VariadicArgName = 13; optional bool IsStrict = 14; optional uint32 Lang = 15; + optional uint32 ResultType = 16; + optional bool ReturnSet = 17; + optional uint32 Kind = 18; } message TPgTable { diff --git a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index 02637d16d8..f02a666e5c 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -346,6 +346,7 @@ private: ctx.AddError(TIssue(udf->Pos, udfRes.GetError())); hasErrors = true; } else { + udf->NormalizedName = udf->Name; udf->CallableType = ParseTypeFromYson(TStringBuf{udfRes.GetCallableType()}, ctx, udf->Pos); if (!udf->CallableType) { hasErrors = true; diff --git a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index 73aab72bde..e8cad5736f 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -200,6 +200,7 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, continue; } + udf.NormalizedName = udf.Name; udf.CallableType = ConvertMiniKQLType(udf.Pos, funcInfo.FunctionType, ctx); YQL_ENSURE(udf.CallableType); if (funcInfo.RunConfigType) { diff --git a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp index 2ce67fa0f6..097135ae7d 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp @@ -69,7 +69,7 @@ public: TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override { with_lock(Lock_) { TString moduleNameStr(moduleName); - if (!UdfIndex_->ContainsModule(moduleNameStr)) { + if (!UdfIndex_->ContainsModuleStrict(moduleNameStr)) { return Nothing(); } @@ -115,7 +115,7 @@ public: bool ContainsModule(const TStringBuf& moduleName) const override { TString moduleNameStr = TString(moduleName); - if (UdfIndex_->ContainsModule(moduleNameStr)) { + if (UdfIndex_->ContainsModuleStrict(moduleNameStr)) { return true; } @@ -142,17 +142,29 @@ private: */ TString moduleNameStr = TString(moduleName); - if (!UdfIndex_->ContainsModule(moduleNameStr)) { + auto moduleStatus = UdfIndex_->ContainsModule(moduleNameStr); + if (moduleStatus == TUdfIndex::EStatus::NotFound) { fallbackFunction = &function; return true; } + if (moduleStatus == TUdfIndex::EStatus::Ambigious) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Ambigious module name: " << moduleName)); + return false; + } + TFunctionInfo info; - if (!UdfIndex_->FindFunction(moduleNameStr, function.Name, info)) { + auto functionStatus = UdfIndex_->FindFunction(moduleNameStr, function.Name, info); + if (functionStatus == TUdfIndex::EStatus::NotFound) { ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Function not found: " << function.Name)); return false; } + if (functionStatus == TUdfIndex::EStatus::Ambigious) { + ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Ambigious function: " << function.Name)); + return false; + } + TResourceFile::TPtr file = DownloadFileWithModule(moduleName, function.Pos, ctx); if (!file) { return false; @@ -161,6 +173,7 @@ private: additionalImport = &file->Import_; if (info.IsTypeAwareness) { + function.Name = info.Name; fallbackFunction = &function; return true; } @@ -170,6 +183,7 @@ private: return false; } + function.NormalizedName = info.Name; function.CallableType = ParseTypeFromYson(TStringBuf{info.CallableType}, ctx, function.Pos); if (!function.CallableType) { ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build callable type from YSON for function " << function.Name)); @@ -205,26 +219,29 @@ private: TResourceFile::TPtr DownloadFileWithModule(const TStringBuf& module) const { TString moduleName(module); - const auto it = DownloadedFiles_.find(module); - if (it != DownloadedFiles_.end()) { - return it->second; - } - auto resource = UdfIndex_->FindResourceByModule(moduleName); if (!resource) { ythrow yexception() << "No resource has been found for registered module " << moduleName; } + auto canonizedModuleName = moduleName; + Y_ENSURE(UdfIndex_->CanonizeModule(canonizedModuleName)); + + const auto it = DownloadedFiles_.find(canonizedModuleName); + if (it != DownloadedFiles_.end()) { + return it->second; + } + // token is empty for urls for now // assumption: file path is frozen already, no need to put into file storage const TDownloadLink& downloadLink = resource->Link; TFileLinkPtr link = downloadLink.IsUrl ? FileStorage_->PutUrl(downloadLink.Path, {}) : CreateFakeFileLink(downloadLink.Path, downloadLink.Md5); - TResourceFile::TPtr file = TResourceFile::Create(moduleName, resource->Modules, link); + TResourceFile::TPtr file = TResourceFile::Create(canonizedModuleName, resource->Modules, link); for (auto& d : resource->Modules) { auto p = DownloadedFiles_.emplace(d, file); if (!p.second) { // should not happen because UdfIndex handles conflicts - ythrow yexception() << "file already downloaded for module " << moduleName << ", conflicting path " << downloadLink.Path << ", existing local file " << p.first->second->Link_->GetPath(); + ythrow yexception() << "file already downloaded for module " << canonizedModuleName << ", conflicting path " << downloadLink.Path << ", existing local file " << p.first->second->Link_->GetPath(); } } diff --git a/yql/essentials/providers/config/yql_config_provider.cpp b/yql/essentials/providers/config/yql_config_provider.cpp index 5715eaabfb..5faf09217c 100644 --- a/yql/essentials/providers/config/yql_config_provider.cpp +++ b/yql/essentials/providers/config/yql_config_provider.cpp @@ -749,7 +749,19 @@ namespace { return false; } } - else if (name == "DqEngine") { + else if (name == "UdfIgnoreCase" || name == "UdfStrictCase") { + if (args.size() != 0) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size())); + return false; + } + + if (!Types.UdfIndex) { + ctx.AddError(TIssue(pos, "UdfIndex is not available")); + return false; + } + + Types.UdfIndex->SetCaseSentiveSearch(name == "UdfStrictCase"); + } else if (name == "DqEngine") { if (args.size() != 1) { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size())); return false; diff --git a/yql/essentials/sql/v1/context.cpp b/yql/essentials/sql/v1/context.cpp index 4637a3be9e..ab880f4315 100644 --- a/yql/essentials/sql/v1/context.cpp +++ b/yql/essentials/sql/v1/context.cpp @@ -65,6 +65,7 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { {"ValidateUnusedExprs", &TContext::ValidateUnusedExprs}, {"AnsiImplicitCrossJoin", &TContext::AnsiImplicitCrossJoin}, {"DistinctOverWindow", &TContext::DistinctOverWindow}, + {"SeqMode", &TContext::SeqMode}, }; typedef TMaybe<bool> TContext::*TPragmaMaybeField; diff --git a/yql/essentials/sql/v1/context.h b/yql/essentials/sql/v1/context.h index 4aa766e34a..61b2e4def3 100644 --- a/yql/essentials/sql/v1/context.h +++ b/yql/essentials/sql/v1/context.h @@ -325,6 +325,7 @@ namespace NSQLTranslationV1 { bool ValidateUnusedExprs = false; bool AnsiImplicitCrossJoin = false; // select * from A,B bool DistinctOverWindow = false; + bool SeqMode = false; }; class TColumnRefScope { diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h index d9eb154031..5805f92042 100644 --- a/yql/essentials/sql/v1/node.h +++ b/yql/essentials/sql/v1/node.h @@ -1521,7 +1521,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings); TNodePtr BuildCommitClusters(TPosition pos); TNodePtr BuildRollbackClusters(TPosition pos); - TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped); + TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq); TNodePtr BuildPragma(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault); TNodePtr BuildSqlLambda(TPosition pos, TVector<TString>&& args, TVector<TNodePtr>&& exprSeq); TNodePtr BuildWorldIfNode(TPosition pos, TNodePtr predicate, TNodePtr thenNode, TNodePtr elseNode, bool isEvaluate); @@ -1534,23 +1534,39 @@ namespace NSQLTranslationV1 { TNodePtr BuildDropTopic(TPosition pos, const TTopicRef& topic, const TDropTopicParameters& params, TScopedStatePtr scoped); - TNodePtr BuildCreateBackupCollection(TPosition pos, const TString& id, + TNodePtr BuildCreateBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TCreateBackupCollectionParameters& params, const TObjectOperatorContext& context); - TNodePtr BuildAlterBackupCollection(TPosition pos, const TString& id, + TNodePtr BuildAlterBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TAlterBackupCollectionParameters& params, const TObjectOperatorContext& context); - TNodePtr BuildDropBackupCollection(TPosition pos, const TString& id, + TNodePtr BuildDropBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TDropBackupCollectionParameters& params, const TObjectOperatorContext& context); - TNodePtr BuildBackup(TPosition pos, const TString& id, + TNodePtr BuildBackup( + TPosition pos, + const TString& prefix, + const TString& id, const TBackupParameters& params, const TObjectOperatorContext& context); - TNodePtr BuildRestore(TPosition pos, const TString& id, + TNodePtr BuildRestore( + TPosition pos, + const TString& prefix, + const TString& id, const TRestoreParameters& params, const TObjectOperatorContext& context); + template<class TContainer> TMaybe<TString> FindMistypeIn(const TContainer& container, const TString& name) { for (auto& item: container) { diff --git a/yql/essentials/sql/v1/query.cpp b/yql/essentials/sql/v1/query.cpp index 5548587090..56dd8bd63d 100644 --- a/yql/essentials/sql/v1/query.cpp +++ b/yql/essentials/sql/v1/query.cpp @@ -2693,15 +2693,44 @@ TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings class TYqlProgramNode: public TAstListNode { public: - TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped) + TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq) : TAstListNode(pos) , Blocks(blocks) , TopLevel(topLevel) , Scoped(scoped) + , UseSeq(useSeq) {} bool DoInit(TContext& ctx, ISource* src) override { bool hasError = false; + INode::TPtr currentWorldsHolder; + INode::TPtr seqNode; + if (UseSeq) { + currentWorldsHolder = new TAstListNodeImpl(GetPos()); + seqNode = new TAstListNodeImpl(GetPos()); + seqNode->Add("Seq!","world"); + } + + INode* currentWorlds = UseSeq ? currentWorldsHolder.Get() : this; + auto flushCurrentWorlds = [&](bool changeSeq, bool finish) { + currentWorldsHolder->Add(Y("return","world")); + auto lambda = BuildLambda(GetPos(), Y("world"), Y("block", Q(currentWorldsHolder))); + seqNode->Add(lambda); + + if (finish) { + Add(Y("let", "world", seqNode)); + } else { + currentWorldsHolder = new TAstListNodeImpl(GetPos()); + currentWorlds = currentWorldsHolder.Get(); + } + + if (changeSeq) { + Add(Y("let", "world", seqNode)); + seqNode = new TAstListNodeImpl(GetPos()); + seqNode->Add("Seq!","world"); + } + }; + if (TopLevel) { for (auto& var: ctx.Variables) { if (!var.second.second->Init(ctx, src)) { @@ -2798,33 +2827,33 @@ public: auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName))); for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); } if (ctx.ResultSizeLimit > 0) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink, BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit))))); } if (!ctx.PragmaPullUpFlatMapOverJoin) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin")))); } if (ctx.FilterPushdownOverJoinOptionalSide) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "FilterPushdownOverJoinOptionalSide")))); } if (!ctx.RotateJoinTree) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "RotateJoinTree"), BuildQuotedAtom(Pos, "false")))); } if (ctx.DiscoveryMode) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "DiscoveryMode")))); } @@ -2835,12 +2864,12 @@ public: } else if (ctx.DqEngineForce) { mode = "force"; } - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode)))); } if (ctx.CostBasedOptimizer) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "CostBasedOptimizer"), BuildQuotedAtom(Pos, ctx.CostBasedOptimizer)))); } @@ -2850,43 +2879,43 @@ public: pragmaName = "JsonQueryReturnsJsonDocument"; } - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName)))); + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName)))); } if (ctx.OrderedColumns) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "OrderedColumns")))); } if (ctx.PqReadByRtmrCluster) { auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all")); - Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll, BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); auto rtmrSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, "$all")); - Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll, BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)))); if (ctx.PqReadByRtmrCluster != "dq") { // set any dynamic settings for particular RTMR cluster for CommitAll! auto rtmrSource = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster)); - Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource, BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "Dummy_"), BuildQuotedAtom(Pos, "1")))); } } if (ctx.YsonCastToString.Defined()) { const TString pragmaName = *ctx.YsonCastToString ? "YsonCastToString" : "DisableYsonCastToString"; - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName)))); + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName)))); } if (ctx.UseBlocks) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks")))); + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks")))); } if (ctx.BlockEngineEnable) { TString mode = ctx.BlockEngineForce ? "force" : "auto"; - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "BlockEngine"), BuildQuotedAtom(Pos, mode)))); } } @@ -2914,22 +2943,41 @@ public: Add(Y("let", data.first, node)); } + if (UseSeq) { + flushCurrentWorlds(false, false); + } + for (auto& block: Blocks) { const auto subqueryAliasPtr = block->SubqueryAlias(); if (subqueryAliasPtr) { if (block->UsedSubquery()) { + if (UseSeq) { + flushCurrentWorlds(true, false); + } + const auto& ref = block->GetLabel(); YQL_ENSURE(!ref.empty()); Add(block); - Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0")))); + currentWorlds->Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0")))); Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1")))); } } else { const auto& ref = block->GetLabel(); - Add(Y("let", ref ? ref : "world", block)); + if (ref) { + Add(Y("let", ref, block)); + } else { + currentWorlds->Add(Y("let", "world", block)); + if (UseSeq) { + flushCurrentWorlds(false, false); + } + } } } + if (UseSeq) { + flushCurrentWorlds(false, true); + } + if (TopLevel) { if (ctx.UniversalAliases) { decltype(Nodes) preparedNodes; @@ -2974,10 +3022,11 @@ private: TVector<TNodePtr> Blocks; const bool TopLevel; TScopedStatePtr Scoped; + const bool UseSeq; }; -TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped) { - return new TYqlProgramNode(pos, blocks, topLevel, scoped); +TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq) { + return new TYqlProgramNode(pos, blocks, topLevel, scoped, useSeq); } class TPragmaNode final: public INode { @@ -3285,16 +3334,18 @@ class TBaseBackupCollectionNode public: TBaseBackupCollectionNode( TPosition pos, + const TString& prefix, const TString& objectId, const TObjectOperatorContext& context) : TBase(pos) , TObjectOperatorContext(context) + , Prefix(prefix) , Id(objectId) {} bool DoInit(TContext& ctx, ISource* src) final { auto keys = Y("Key"); - keys = L(keys, Q(Y(Q("backupCollection"), Y("String", BuildQuotedAtom(Pos, Id))))); + keys = L(keys, Q(Y(Q("backupCollection"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix))))); auto options = this->FillOptions(ctx, Y()); Add("block", Q(Y( @@ -3309,6 +3360,7 @@ public: virtual INode::TPtr FillOptions(TContext& ctx, INode::TPtr options) const = 0; protected: + TString Prefix; TString Id; }; @@ -3319,10 +3371,11 @@ class TCreateBackupCollectionNode public: TCreateBackupCollectionNode( TPosition pos, + const TString& prefix, const TString& objectId, const TCreateBackupCollectionParameters& params, const TObjectOperatorContext& context) - : TBase(pos, objectId, context) + : TBase(pos, prefix, objectId, context) , Params(params) {} @@ -3331,7 +3384,7 @@ public: auto settings = Y(); for (auto& [key, value] : Params.Settings) { - settings->Add(Q(Y(BuildQuotedAtom(Pos, key), value.Build()))); + settings->Add(Q(Y(BuildQuotedAtom(Pos, key), Y("String", value.Build())))); } options->Add(Q(Y(Q("settings"), Q(settings)))); @@ -3349,7 +3402,7 @@ public: } TPtr DoClone() const final { - return new TCreateBackupCollectionNode(GetPos(), Id, Params, *this); + return new TCreateBackupCollectionNode(GetPos(), Prefix, Id, Params, *this); } private: @@ -3363,10 +3416,11 @@ class TAlterBackupCollectionNode public: TAlterBackupCollectionNode( TPosition pos, + const TString& prefix, const TString& objectId, const TAlterBackupCollectionParameters& params, const TObjectOperatorContext& context) - : TBase(pos, objectId, context) + : TBase(pos, prefix, objectId, context) , Params(params) {} @@ -3375,7 +3429,7 @@ public: auto settings = Y(); for (auto& [key, value] : Params.Settings) { - settings->Add(Q(Y(BuildQuotedAtom(Pos, key), value.Build()))); + settings->Add(Q(Y(BuildQuotedAtom(Pos, key), Y("String", value.Build())))); } options->Add(Q(Y(Q("settings"), Q(settings)))); @@ -3403,7 +3457,7 @@ public: } TPtr DoClone() const final { - return new TAlterBackupCollectionNode(GetPos(), Id, Params, *this); + return new TAlterBackupCollectionNode(GetPos(), Prefix, Id, Params, *this); } private: @@ -3417,10 +3471,11 @@ class TDropBackupCollectionNode public: TDropBackupCollectionNode( TPosition pos, + const TString& prefix, const TString& objectId, const TDropBackupCollectionParameters&, const TObjectOperatorContext& context) - : TBase(pos, objectId, context) + : TBase(pos, prefix, objectId, context) {} virtual INode::TPtr FillOptions(TContext&, INode::TPtr options) const final { @@ -3431,29 +3486,38 @@ public: TPtr DoClone() const final { TDropBackupCollectionParameters params; - return new TDropBackupCollectionNode(GetPos(), Id, params, *this); + return new TDropBackupCollectionNode(GetPos(), Prefix, Id, params, *this); } }; -TNodePtr BuildCreateBackupCollection(TPosition pos, const TString& id, +TNodePtr BuildCreateBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TCreateBackupCollectionParameters& params, const TObjectOperatorContext& context) { - return new TCreateBackupCollectionNode(pos, id, params, context); + return new TCreateBackupCollectionNode(pos, prefix, id, params, context); } -TNodePtr BuildAlterBackupCollection(TPosition pos, const TString& id, +TNodePtr BuildAlterBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TAlterBackupCollectionParameters& params, const TObjectOperatorContext& context) { - return new TAlterBackupCollectionNode(pos, id, params, context); + return new TAlterBackupCollectionNode(pos, prefix, id, params, context); } -TNodePtr BuildDropBackupCollection(TPosition pos, const TString& id, +TNodePtr BuildDropBackupCollection( + TPosition pos, + const TString& prefix, + const TString& id, const TDropBackupCollectionParameters& params, const TObjectOperatorContext& context) { - return new TDropBackupCollectionNode(pos, id, params, context); + return new TDropBackupCollectionNode(pos, prefix, id, params, context); } class TBackupNode final @@ -3464,11 +3528,13 @@ class TBackupNode final public: TBackupNode( TPosition pos, + const TString& prefix, const TString& id, const TBackupParameters& params, const TObjectOperatorContext& context) : TBase(pos) , TObjectOperatorContext(context) + , Prefix(prefix) , Id(id) , Params(params) { @@ -3477,13 +3543,14 @@ public: bool DoInit(TContext& ctx, ISource* src) override { auto keys = Y("Key"); - keys = L(keys, Q(Y(Q("backup"), Y("String", BuildQuotedAtom(Pos, Id))))); + keys = L(keys, Q(Y(Q("backup"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix))))); auto opts = Y(); - opts->Add(Q(Y(Q("mode"), Q("backup")))); if (Params.Incremental) { - opts->Add(Q(Y(Q("incremental")))); + opts->Add(Q(Y(Q("mode"), Q("backupIncremental")))); + } else { + opts->Add(Q(Y(Q("mode"), Q("backup")))); } Add("block", Q(Y( @@ -3496,18 +3563,22 @@ public: } TPtr DoClone() const final { - return new TBackupNode(GetPos(), Id, Params, *this); + return new TBackupNode(GetPos(), Prefix, Id, Params, *this); } private: + TString Prefix; TString Id; TBackupParameters Params; }; -TNodePtr BuildBackup(TPosition pos, const TString& id, +TNodePtr BuildBackup( + TPosition pos, + const TString& prefix, + const TString& id, const TBackupParameters& params, const TObjectOperatorContext& context) { - return new TBackupNode(pos, id, params, context); + return new TBackupNode(pos, prefix, id, params, context); } class TRestoreNode final @@ -3518,11 +3589,13 @@ class TRestoreNode final public: TRestoreNode( TPosition pos, + const TString& prefix, const TString& id, const TRestoreParameters& params, const TObjectOperatorContext& context) : TBase(pos) , TObjectOperatorContext(context) + , Prefix(prefix) , Id(id) , Params(params) { @@ -3531,7 +3604,7 @@ public: bool DoInit(TContext& ctx, ISource* src) override { auto keys = Y("Key"); - keys = L(keys, Q(Y(Q("restore"), Y("String", BuildQuotedAtom(Pos, Id))))); + keys = L(keys, Q(Y(Q("restore"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix))))); auto opts = Y(); opts->Add(Q(Y(Q("mode"), Q("restore")))); @@ -3550,18 +3623,22 @@ public: } TPtr DoClone() const final { - return new TRestoreNode(GetPos(), Id, Params, *this); + return new TRestoreNode(GetPos(), Prefix, Id, Params, *this); } private: + TString Prefix; TString Id; TRestoreParameters Params; }; -TNodePtr BuildRestore(TPosition pos, const TString& id, +TNodePtr BuildRestore( + TPosition pos, + const TString& prefix, + const TString& id, const TRestoreParameters& params, const TObjectOperatorContext& context) { - return new TRestoreNode(pos, id, params, context); + return new TRestoreNode(pos, prefix, id, params, context); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql_query.cpp b/yql/essentials/sql/v1/sql_query.cpp index f3755bf6f0..5f2f31d776 100644 --- a/yql/essentials/sql/v1/sql_query.cpp +++ b/yql/essentials/sql/v1/sql_query.cpp @@ -1420,7 +1420,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& const TString& objectId = Id(node.GetRule_backup_collection2().GetRule_object_ref3().GetRule_id_or_at2(), *this).second; AddStatementToBlocks(blocks, BuildCreateBackupCollection(Ctx.Pos(), - BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), + TString(Ctx.GetPrefixPath(context.ServiceId, context.Cluster)), + objectId, TCreateBackupCollectionParameters { .Settings = std::move(kv), .Database = database, @@ -1482,7 +1483,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& const TString& objectId = Id(node.GetRule_backup_collection2().GetRule_object_ref3().GetRule_id_or_at2(), *this).second; AddStatementToBlocks(blocks, BuildAlterBackupCollection(Ctx.Pos(), - BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), + TString(Ctx.GetPrefixPath(context.ServiceId, context.Cluster)), + objectId, TAlterBackupCollectionParameters { .Settings = std::move(kv), .SettingsToReset = std::move(toReset), @@ -1510,7 +1512,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& const TString& objectId = Id(node.GetRule_backup_collection2().GetRule_object_ref3().GetRule_id_or_at2(), *this).second; AddStatementToBlocks(blocks, BuildDropBackupCollection(Ctx.Pos(), - BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), + TString(Ctx.GetPrefixPath(context.ServiceId, context.Cluster)), + objectId, TDropBackupCollectionParameters { .MissingOk = false, }, @@ -1630,7 +1633,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& AddStatementToBlocks(blocks, BuildBackup( Ctx.Pos(), - BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), + TString(Ctx.GetPrefixPath(context.ServiceId, context.Cluster)), + objectId, TBackupParameters{ .Incremental = incremental, }, @@ -1662,7 +1666,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& AddStatementToBlocks(blocks, BuildRestore( Ctx.Pos(), - BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), + TString(Ctx.GetPrefixPath(context.ServiceId, context.Cluster)), + objectId, TRestoreParameters{ .At = at, }, @@ -2022,22 +2027,29 @@ bool TSqlQuery::AlterTableAlterFamily(const TRule_alter_table_alter_column_famil << "' in one alter"; return false; } - const TString stringValue(Ctx.Token(value.GetAlt_family_setting_value1().GetToken1())); - entry->Data = BuildLiteralSmartString(Ctx, stringValue); + if (!StoreString(value, entry->Data, Ctx)) { + Ctx.Error() << to_upper(settingName.Name) << " value should be a string literal"; + return false; + } } else if (to_lower(settingName.Name) == "compression") { if (entry->Compression) { Ctx.Error() << "Redefinition of 'compression' setting for column family '" << name.Name << "' in one alter"; return false; } - const TString stringValue(Ctx.Token(value.GetAlt_family_setting_value1().GetToken1())); - entry->Compression = BuildLiteralSmartString(Ctx, stringValue); + if (!StoreString(value, entry->Compression, Ctx)) { + Ctx.Error() << to_upper(settingName.Name) << " value should be a string literal"; + return false; + } } else if (to_lower(settingName.Name) == "compression_level") { if (entry->CompressionLevel) { Ctx.Error() << "Redefinition of 'compression_level' setting for column family '" << name.Name << "' in one alter"; return false; } - entry->CompressionLevel = LiteralNumber(Ctx, value.GetAlt_family_setting_value2().GetRule_integer1()); + if (!StoreInt(value, entry->CompressionLevel, Ctx)) { + Ctx.Error() << to_upper(settingName.Name) << " value should be an integer"; + return false; + } } else { Ctx.Error() << "Unknown table setting: " << settingName.Name; return false; @@ -2954,6 +2966,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disabledistinctoverwindow") { Ctx.DistinctOverWindow = false; Ctx.IncrementMonCounter("sql_pragma", "DisableDistinctOverWindow"); + } else if (normalizedPragma == "seqmode") { + Ctx.SeqMode = true; + Ctx.IncrementMonCounter("sql_pragma", "SeqMode"); + } else if (normalizedPragma == "disableseqmode") { + Ctx.SeqMode = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableSeqMode"); } else { Error() << "Unknown pragma: " << pragma; Ctx.IncrementMonCounter("sql_errors", "UnknownPragma"); @@ -3302,7 +3320,7 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); } - auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); + auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped, Ctx.SeqMode); WarnUnusedNodes(); return result; } @@ -3372,7 +3390,7 @@ TNodePtr TSqlQuery::Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_co AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); } - auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); + auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped, Ctx.SeqMode); return result; } namespace { diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp index 4bcfb7de84..b298eb8abb 100644 --- a/yql/essentials/sql/v1/sql_translation.cpp +++ b/yql/essentials/sql/v1/sql_translation.cpp @@ -1567,17 +1567,59 @@ TNodePtr TSqlTranslation::SerialTypeNode(const TRule_type_name_or_bind& node) { return nullptr; } +bool StoreString(const TRule_family_setting_value& from, TNodePtr& to, TContext& ctx) { + switch (from.Alt_case()) { + case TRule_family_setting_value::kAltFamilySettingValue1: { + // STRING_VALUE + const TString stringValue(ctx.Token(from.GetAlt_family_setting_value1().GetToken1())); + TNodePtr literal = BuildLiteralSmartString(ctx, stringValue); + if (!literal) { + return false; + } + to = literal; + break; + } + default: + return false; + } + return true; +} + +bool StoreInt(const TRule_family_setting_value& from, TNodePtr& to, TContext& ctx) { + switch (from.Alt_case()) { + case TRule_family_setting_value::kAltFamilySettingValue2: { + // integer + TNodePtr literal = LiteralNumber(ctx, from.GetAlt_family_setting_value2().GetRule_integer1()); + if (!literal) { + return false; + } + to = literal; + break; + } + default: + return false; + } + return true; +} + bool TSqlTranslation::FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family) { TIdentifier id = IdEx(settingNode.GetRule_an_id1(), *this); const TRule_family_setting_value& value = settingNode.GetRule_family_setting_value3(); if (to_lower(id.Name) == "data") { - const TString stringValue(Ctx.Token(value.GetAlt_family_setting_value1().GetToken1())); - family.Data = BuildLiteralSmartString(Ctx, stringValue); + if (!StoreString(value, family.Data, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; + return false; + } } else if (to_lower(id.Name) == "compression") { - const TString stringValue(Ctx.Token(value.GetAlt_family_setting_value1().GetToken1())); - family.Compression = BuildLiteralSmartString(Ctx, stringValue); + if (!StoreString(value, family.Compression, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; + return false; + } } else if (to_lower(id.Name) == "compression_level") { - family.CompressionLevel = LiteralNumber(Ctx, value.GetAlt_family_setting_value2().GetRule_integer1()); + if (!StoreInt(value, family.CompressionLevel, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be an integer"; + return false; + } } else { Ctx.Error() << "Unknown table setting: " << id.Name; return false; @@ -4466,7 +4508,7 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda TBlocks innerBlocks; const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, body); - auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; + auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped, Ctx.SeqMode) : nullptr; WarnUnusedNodes(); Ctx.ScopeLevel--; Ctx.Scoped = saveScoped; @@ -4579,7 +4621,7 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ return false; } - auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; + auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped, Ctx.SeqMode) : nullptr; WarnUnusedNodes(); Ctx.Scoped = saveScoped; Ctx.ScopeLevel--; diff --git a/yql/essentials/sql/v1/sql_translation.h b/yql/essentials/sql/v1/sql_translation.h index 683647f16b..325640d74a 100644 --- a/yql/essentials/sql/v1/sql_translation.h +++ b/yql/essentials/sql/v1/sql_translation.h @@ -287,6 +287,9 @@ protected: TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node); +bool StoreString(const TRule_family_setting_value& from, TNodePtr& to, TContext& ctx); +bool StoreInt(const TRule_family_setting_value& from, TNodePtr& to, TContext& ctx); + template<typename TChar> struct TPatternComponent { TBasicString<TChar> Prefix; diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index a4251890a7..65ec39af2c 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -7081,7 +7081,9 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '()))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); } }; @@ -7105,9 +7107,10 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'database)))))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('type 'database)")); } }; @@ -7133,10 +7136,10 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'table) '('path '"someTable")) '('('type 'table) '('path '"prefix/anotherTable")))))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('('('type 'table) '('path '"someTable")))#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('('('type 'table) '('path '"prefix/anotherTable")))#")); } }; @@ -7192,7 +7195,8 @@ Y_UNIT_TEST_SUITE(BackupCollection) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('settings '('('"storage" '"remote") '('"tag1" '"123"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"remote")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag1" (String '"123"))))#"); UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetSettings '('"tag2" '"tag3")))#"); } }; @@ -7393,7 +7397,7 @@ Y_UNIT_TEST_SUITE(Backup) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'incremental")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'Incremental")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); } }; @@ -7414,8 +7418,7 @@ Y_UNIT_TEST_SUITE(Backup) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'incremental")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backupIncremental")); } }; @@ -7470,7 +7473,7 @@ Y_UNIT_TEST_SUITE(Restore) { } Y_UNIT_TEST_SUITE(ColumnFamily) { - Y_UNIT_TEST(CompressionLevel) { + Y_UNIT_TEST(CompressionLevelCorrectUsage) { NYql::TAstParseResult res = SqlToYql(R"( use plato; CREATE TABLE tableName ( Key Uint32 FAMILY default, @@ -7503,4 +7506,89 @@ Y_UNIT_TEST_SUITE(ColumnFamily) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); UNIT_ASSERT_VALUES_EQUAL(2, elementStat["compression_level"]); } + + Y_UNIT_TEST(FieldDataIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = 1, + COMPRESSION = "lz4", + COMPRESSION_LEVEL = 5 + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "DATA value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = 2, + COMPRESSION_LEVEL = 5 + ), + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionLevelIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4", + COMPRESSION_LEVEL = "5" + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } + + Y_UNIT_TEST(AlterCompressionCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION "lz4"; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionFieldIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION lz4; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "Unexpected token 'lz4' : cannot match to any predicted input"); + } + + Y_UNIT_TEST(AlterCompressionLevelCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL 5; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionLevelFieldIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL "5"; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } } diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 29a306cddf..e2a05cc229 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -7078,7 +7078,9 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '()))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); } }; @@ -7102,9 +7104,10 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'database)))))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('type 'database)")); } }; @@ -7130,10 +7133,10 @@ Y_UNIT_TEST_SUITE(BackupCollection) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"storage" '"local") '('"tag" '"test"))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"local")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag" (String '"test"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('entries '('('('type 'table) '('path '"someTable")) '('('type 'table) '('path '"prefix/anotherTable")))))#"); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'create")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('('('type 'table) '('path '"someTable")))#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('('('type 'table) '('path '"prefix/anotherTable")))#")); } }; @@ -7189,7 +7192,8 @@ Y_UNIT_TEST_SUITE(BackupCollection) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); UNIT_ASSERT_STRING_CONTAINS(line, R"#(('mode 'alter))#"); - UNIT_ASSERT_STRING_CONTAINS(line, R"#('('settings '('('"storage" '"remote") '('"tag1" '"123"))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"storage" (String '"remote")))#"); + UNIT_ASSERT_STRING_CONTAINS(line, R"#(('"tag1" (String '"123"))))#"); UNIT_ASSERT_STRING_CONTAINS(line, R"#('('resetSettings '('"tag2" '"tag3")))#"); } }; @@ -7365,7 +7369,7 @@ Y_UNIT_TEST_SUITE(Backup) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'incremental")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'Incremental")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); } }; @@ -7386,8 +7390,7 @@ Y_UNIT_TEST_SUITE(Backup) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"#('"TestCollection")#")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'incremental")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backup")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'backupIncremental")); } }; @@ -7442,7 +7445,7 @@ Y_UNIT_TEST_SUITE(Restore) { } Y_UNIT_TEST_SUITE(ColumnFamily) { - Y_UNIT_TEST(CompressionLevel) { + Y_UNIT_TEST(CompressionLevelCorrectUsage) { NYql::TAstParseResult res = SqlToYql(R"( use plato; CREATE TABLE tableName ( Key Uint32 FAMILY default, @@ -7475,4 +7478,89 @@ Y_UNIT_TEST_SUITE(ColumnFamily) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); UNIT_ASSERT_VALUES_EQUAL(2, elementStat["compression_level"]); } + + Y_UNIT_TEST(FieldDataIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = 1, + COMPRESSION = "lz4", + COMPRESSION_LEVEL = 5 + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "DATA value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = 2, + COMPRESSION_LEVEL = 5 + ), + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION value should be a string literal"); + } + + Y_UNIT_TEST(FieldCompressionLevelIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + CREATE TABLE tableName ( + Key Uint32 FAMILY default, + PRIMARY KEY (Key), + FAMILY default ( + DATA = "test", + COMPRESSION = "lz4", + COMPRESSION_LEVEL = "5" + ) + ); + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } + + Y_UNIT_TEST(AlterCompressionCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION "lz4"; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionFieldIsNotString) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION lz4; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "mismatched input 'lz4' expecting {STRING_VALUE, DIGITS, INTEGER_VALUE}"); + } + + Y_UNIT_TEST(AlterCompressionLevelCorrectUsage) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL 5; + )"); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AlterCompressionLevelFieldIsNotInteger) { + NYql::TAstParseResult res = SqlToYql(R"( use plato; + ALTER TABLE tableName ALTER FAMILY default SET COMPRESSION_LEVEL "5"; + )"); + UNIT_ASSERT(!res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 1); + UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "COMPRESSION_LEVEL value should be an integer"); + } } diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp index f21e837f66..8d2b8ec6cd 100644 --- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp +++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp @@ -31,6 +31,7 @@ #include <sys/resource.h> #include <sys/syscall.h> #include <sys/socket.h> +#include <sys/stat.h> #ifndef GRND_RANDOM #include <sys/random.h> #endif @@ -54,6 +55,43 @@ #endif #endif + +#if !defined(SYS_newfstatat) +#if defined(__x86_64__) + #define SYS_newfstatat 262 +#elif defined(__i386__) + #error Unsupported syscall +#elif defined(__aarch64__) + #define SYS_newfstatat 79 +#elif defined(__arm__) + #error Unsupported syscall +#elif defined(__powerpc__) + #define SYS_newfstatat 291 +#else +#error Unsupported platform +#endif +#endif + +#if !defined(SYS_clone3) + #define SYS_clone3 435 +#endif + +#if !defined(SYS_rseq) +#if defined(__x86_64__) + #define SYS_rseq 334 +#elif defined(__i386__) + #define SYS_rseq 386 +#elif defined(__aarch64__) + #define SYS_rseq 293 +#elif defined(__arm__) + #define SYS_rseq 398 +#elif defined(__powerpc__) + #define SYS_rseq 387 +#else +#error Unsupported platform +#endif +#endif + #endif using namespace NKikimr; @@ -308,6 +346,7 @@ int main(int argc, char **argv) { Allow(clock_gettime), Allow(clock_nanosleep), Allow(clone), + Allow(clone3), Allow(close), #ifndef _arm64_ Allow(creat), @@ -378,6 +417,7 @@ int main(int argc, char **argv) { Allow(munlockall), Allow(munmap), Allow(nanosleep), + Allow(newfstatat), #ifndef _arm64_ Allow(open), #endif @@ -399,6 +439,7 @@ int main(int argc, char **argv) { #ifndef _arm64_ Allow(rmdir), #endif + Allow(rseq), Allow(rt_sigaction), Allow(rt_sigpending), Allow(rt_sigprocmask), diff --git a/yql/essentials/udfs/common/datetime2/test/canondata/result.json b/yql/essentials/udfs/common/datetime2/test/canondata/result.json index 6e475365ea..0ab7651bdc 100644 --- a/yql/essentials/udfs/common/datetime2/test/canondata/result.json +++ b/yql/essentials/udfs/common/datetime2/test/canondata/result.json @@ -54,6 +54,11 @@ "uri": "file://test.test_Get_/results.txt" } ], + "test.test[IgnoreCaseFuncs]": [ + { + "uri": "file://test.test_IgnoreCaseFuncs_/results.txt" + } + ], "test.test[ImplicitSplit]": [ { "uri": "file://test.test_ImplicitSplit_/results.txt" diff --git a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt new file mode 100644 index 0000000000..2da9f03e5c --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt @@ -0,0 +1,36 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "Uint16" + ] + ]; + [ + "column1"; + [ + "DataType"; + "Uint8" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "2001"; + "1" + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.cfg b/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.cfg new file mode 100644 index 0000000000..9a686c75c4 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.cfg @@ -0,0 +1 @@ +scan_udfs
\ No newline at end of file diff --git a/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.sql b/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.sql new file mode 100644 index 0000000000..4378b0a6a6 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test/cases/IgnoreCaseFuncs.sql @@ -0,0 +1,3 @@ +pragma config.flags("UdfIgnoreCase"); +select + DATETIME::GETYEAR(Date('2001-01-01')),datetime::getmonth(Date('2001-01-01')) diff --git a/yql/essentials/udfs/common/yson2/test/canondata/result.json b/yql/essentials/udfs/common/yson2/test/canondata/result.json index e8db385e45..962904ec60 100644 --- a/yql/essentials/udfs/common/yson2/test/canondata/result.json +++ b/yql/essentials/udfs/common/yson2/test/canondata/result.json @@ -94,6 +94,11 @@ "uri": "file://test.test_GoodForYsonBadForJson_/results.txt" } ], + "test.test[IgnoreCaseFuncs]": [ + { + "uri": "file://test.test_IgnoreCaseFuncs_/results.txt" + } + ], "test.test[ImplicitFromRes]": [ { "uri": "file://test.test_ImplicitFromRes_/results.txt" @@ -109,14 +114,14 @@ "uri": "file://test.test_JsonSerializeSkipMapEntity_/results.txt" } ], - "test.test[JsonWithUtf8]": [ + "test.test[JsonWithNanAsString]": [ { - "uri": "file://test.test_JsonWithUtf8_/results.txt" + "uri": "file://test.test_JsonWithNanAsString_/results.txt" } ], - "test.test[JsonWithNanAsString]": [ + "test.test[JsonWithUtf8]": [ { - "uri": "file://test.test_JsonWithNanAsString_/results.txt" + "uri": "file://test.test_JsonWithUtf8_/results.txt" } ], "test.test[Lists]": [ diff --git a/yql/essentials/udfs/common/yson2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt b/yql/essentials/udfs/common/yson2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt new file mode 100644 index 0000000000..b6d6e7d1db --- /dev/null +++ b/yql/essentials/udfs/common/yson2/test/canondata/test.test_IgnoreCaseFuncs_/results.txt @@ -0,0 +1,65 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Yson" + ] + ] + ]; + [ + "column1"; + [ + "DataType"; + "Yson" + ] + ]; + [ + "column2"; + [ + "DataType"; + "Yson" + ] + ]; + [ + "column3"; + [ + "OptionalType"; + [ + "DataType"; + "Uint64" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + [] + ]; + {}; + { + "$type" = "int64"; + "$value" = "1" + }; + [ + "3" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.cfg b/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.cfg new file mode 100644 index 0000000000..135ef7371d --- /dev/null +++ b/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.cfg @@ -0,0 +1,2 @@ +scan_udfs + diff --git a/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.sql b/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.sql new file mode 100644 index 0000000000..d57d650984 --- /dev/null +++ b/yql/essentials/udfs/common/yson2/test/cases/IgnoreCaseFuncs.sql @@ -0,0 +1,3 @@ +pragma config.flags("UdfIgnoreCase"); +select + YSON::PARSE('[]'),yson::parse('{}'y),yson::from(1),yson::getlength('[1;2;3]'y); |