diff options
| author | aneporada <aneporada@yandex-team.ru> | 2022-02-10 16:47:56 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:56 +0300 |
| commit | dbca3ecc91dd33d6527ec66a8b06dd6fcb2a3a3e (patch) | |
| tree | c0748b5dcbade83af788c0abfa89c0383d6b779c | |
| parent | 9620067c7224b19b8e9ad6222468ecd222caae13 (diff) | |
| download | ydb-dbca3ecc91dd33d6527ec66a8b06dd6fcb2a3a3e.tar.gz | |
Restoring authorship annotation for <aneporada@yandex-team.ru>. Commit 2 of 2.
319 files changed, 31417 insertions, 31417 deletions
diff --git a/contrib/libs/antlr3_cpp_runtime/include/antlr3baserecognizer.inl b/contrib/libs/antlr3_cpp_runtime/include/antlr3baserecognizer.inl index be61b8ee59..b0c3fe8d51 100644 --- a/contrib/libs/antlr3_cpp_runtime/include/antlr3baserecognizer.inl +++ b/contrib/libs/antlr3_cpp_runtime/include/antlr3baserecognizer.inl @@ -651,7 +651,7 @@ ANTLR_MARKER BaseRecognizer<ImplTraits, StreamType>::getRuleMemoization( ANTLR_I /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST. */ typedef IntTrie<ImplTraits, ANTLR_MARKER> RuleListType; - typedef TrieEntry<ImplTraits, std::shared_ptr<RuleListType>> EntryType; + typedef TrieEntry<ImplTraits, std::shared_ptr<RuleListType>> EntryType; typedef TrieEntry<ImplTraits, ANTLR_MARKER> SubEntryType; ANTLR_MARKER stopIndex; EntryType* entry; @@ -670,14 +670,14 @@ ANTLR_MARKER BaseRecognizer<ImplTraits, StreamType>::getRuleMemoization( ANTLR_I * no idea of the size, but you should remember that this will cause the leftmost * bit match algorithm to run to 63 bits, which will be the whole time spent in the trie ;-) */ - m_state->get_ruleMemo()->add( ruleIndex, std::make_shared<RuleListType>(63) ); + m_state->get_ruleMemo()->add( ruleIndex, std::make_shared<RuleListType>(63) ); /* We cannot have a stopIndex in a trie we have just created of course */ return MEMO_RULE_UNKNOWN; } - std::shared_ptr<RuleListType> ruleList = entry->get_data(); + std::shared_ptr<RuleListType> ruleList = entry->get_data(); /* See if there is a stop index associated with the supplied start index. */ @@ -732,7 +732,7 @@ void BaseRecognizer<ImplTraits, StreamType>::memoize(ANTLR_MARKER ruleIndex, ANT /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST. */ typedef IntTrie<ImplTraits, ANTLR_MARKER> RuleListType; - typedef TrieEntry<ImplTraits, std::shared_ptr<RuleListType>> EntryType; + typedef TrieEntry<ImplTraits, std::shared_ptr<RuleListType>> EntryType; EntryType* entry; ANTLR_MARKER stopIndex; SuperType* super = static_cast<SuperType*>(this); @@ -744,7 +744,7 @@ void BaseRecognizer<ImplTraits, StreamType>::memoize(ANTLR_MARKER ruleIndex, ANT if (entry != NULL) { - std::shared_ptr<RuleListType> ruleList = entry->get_data(); + std::shared_ptr<RuleListType> ruleList = entry->get_data(); /* If we don't already have this entry, append it. The memoize trie does not * accept duplicates so it won't add it if already there and we just ignore the diff --git a/contrib/libs/antlr3_cpp_runtime/include/antlr3exception.inl b/contrib/libs/antlr3_cpp_runtime/include/antlr3exception.inl index f0d82d817d..27f00bdda8 100644 --- a/contrib/libs/antlr3_cpp_runtime/include/antlr3exception.inl +++ b/contrib/libs/antlr3_cpp_runtime/include/antlr3exception.inl @@ -307,7 +307,7 @@ void ANTLR_Exception<ImplTraits, Ex, StreamType>::displayRecognitionError( ANTLR // possible tokens at this point, but we did not see any // member of that set. // - str_stream << " : unexpected input :"; + str_stream << " : unexpected input :"; // What tokens could we have accepted at this point in the // parse? @@ -327,7 +327,7 @@ void ANTLR_Exception<ImplTraits, Ex, StreamType>::displayRecognitionError( ANTLR // here, but you should do whatever makes sense for you of course. // No token number 0, so look for bit 1 and on. // - str_stream << " expected one of : "; + str_stream << " expected one of : "; for (bit = 1; bit < numbits && count < 8 && count < size; bit++) { // TODO: This doesn;t look right - should be asking if the bit is set!! @@ -343,7 +343,7 @@ void ANTLR_Exception<ImplTraits, Ex, StreamType>::displayRecognitionError( ANTLR } else { - str_stream << " nothing is expected here\n"; + str_stream << " nothing is expected here\n"; } } break; diff --git a/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.hpp b/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.hpp index da64f7a9a4..ef0855ea08 100644 --- a/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.hpp +++ b/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.hpp @@ -73,7 +73,7 @@ public: typedef typename AllocPolicyType::template VectorType<RewriteStreamType> RewriteStreamsType; typedef IntTrie<ImplTraits, ANTLR_MARKER> RuleListType; - typedef IntTrie<ImplTraits, std::shared_ptr<RuleListType>> RuleMemoType; + typedef IntTrie<ImplTraits, std::shared_ptr<RuleListType>> RuleMemoType; private: /** Points to the first in a possible chain of exceptions that the diff --git a/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.inl b/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.inl index a18902271f..27732cb34f 100644 --- a/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.inl +++ b/contrib/libs/antlr3_cpp_runtime/include/antlr3recognizersharedstate.inl @@ -8,7 +8,7 @@ RecognizerSharedState<ImplTraits, StreamType>::RecognizerSharedState() m_error = false; m_errorRecovery = false; m_failed = false; - m_token_present = false; + m_token_present = false; m_lastErrorIndex = 0; m_errorCount = 0; m_backtracking = false; diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index ec9de2766c..76f38bf35c 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -267,7 +267,7 @@ public: { auto name = input->Content(); TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << name); + return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << name); }); if (input->IsCallable("SelectRow")) { @@ -1283,9 +1283,9 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr return compiler; } -TRuntimeNode CompileNode(const TExprNode& node, TExprContext& exprCtx, TContext::TPtr ctx, const IMkqlCallableCompiler* compiler) { +TRuntimeNode CompileNode(const TExprNode& node, TExprContext& exprCtx, TContext::TPtr ctx, const IMkqlCallableCompiler* compiler) { const auto guard = ctx->TypeEnv->BindAllocator(); - TMkqlBuildContext mkqlCtx(*compiler, *ctx->PgmBuilder, exprCtx); + TMkqlBuildContext mkqlCtx(*compiler, *ctx->PgmBuilder, exprCtx); return MkqlBuildExpr(node, mkqlCtx); } @@ -1369,14 +1369,14 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, return; } - TRuntimeNode convertedNode = CompileNode(*expr->Root, expr->Context, ctx, compiler.Get()); + TRuntimeNode convertedNode = CompileNode(*expr->Root, expr->Context, ctx, compiler.Get()); TConvertResult convRes = ctx->Finish(convertedNode); promise.SetValue(convRes); } catch (const TNodeException& ex) { // TODO: pass backtrace TConvertResult convRes; - convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); + convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); promise.SetValue(convRes); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. @@ -1395,14 +1395,14 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, return promise.GetFuture(); } - TRuntimeNode convertedNode = CompileNode(*expr->Root, expr->Context, ctx, compiler.Get()); + TRuntimeNode convertedNode = CompileNode(*expr->Root, expr->Context, ctx, compiler.Get()); TConvertResult convRes = ctx->Finish(convertedNode); promise.SetValue(convRes); } } catch (const TNodeException& ex) { // TODO: pass backtrace TConvertResult convRes; - convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); + convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); promise.SetValue(convRes); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace @@ -1474,7 +1474,7 @@ public: } } catch (const TNodeException& ex) { // TODO: pass backtrace - TMiniKQLCompileResult res(NYql::TIssue(Expr->Context.GetPosition(ex.Pos()), ex.what())); + TMiniKQLCompileResult res(NYql::TIssue(Expr->Context.GetPosition(ex.Pos()), ex.what())); return SendResponseAndDie(res, {}, ctx); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace @@ -1549,7 +1549,7 @@ private: } catch (const TNodeException& ex) { // TODO: pass backtrace - TMiniKQLCompileResult result(NYql::TIssue(Expr->Context.GetPosition(ex.Pos()), ex.what())); + TMiniKQLCompileResult result(NYql::TIssue(Expr->Context.GetPosition(ex.Pos()), ex.what())); return SendResponseAndDie(result, std::move(compileResolveCookies), ctx); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. @@ -1587,7 +1587,7 @@ private: } TString CompileProgram() { - TRuntimeNode convertedNode = CompileNode(*Expr->Root, Expr->Context, CompileCtx, Compiler.Get()); + TRuntimeNode convertedNode = CompileNode(*Expr->Root, Expr->Context, CompileCtx, Compiler.Get()); TConvertResult convRes = CompileCtx->Finish(convertedNode); return NMiniKQL::SerializeRuntimeNode(convRes.Node, CompileCtx->PgmBuilder->GetTypeEnvironment()); } diff --git a/ydb/core/kqp/common/kqp_transform.cpp b/ydb/core/kqp/common/kqp_transform.cpp index 77575169e2..3653c78d67 100644 --- a/ydb/core/kqp/common/kqp_transform.cpp +++ b/ydb/core/kqp/common/kqp_transform.cpp @@ -14,7 +14,7 @@ IGraphTransformer::TStatus TLogExprTransformer::operator()(const TExprNode::TPtr Y_UNUSED(ctx); output = input; - LogExpr(*input, ctx, Description, Component, Level); + LogExpr(*input, ctx, Description, Component, Level); return IGraphTransformer::TStatus::Ok; } @@ -24,7 +24,7 @@ TAutoPtr<IGraphTransformer> TLogExprTransformer::Sync(const TString& description return CreateFunctorTransformer(TLogExprTransformer(description, component, level)); } -void TLogExprTransformer::LogExpr(const TExprNode& input, TExprContext& ctx, const TString& description, NLog::EComponent component, +void TLogExprTransformer::LogExpr(const TExprNode& input, TExprContext& ctx, const TString& description, NLog::EComponent component, NLog::ELevel level) { YQL_CVLOG(level, component) << description << ":\n" << KqpExprToPrettyString(input, ctx); diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h index 4174c41841..457c424801 100644 --- a/ydb/core/kqp/common/kqp_transform.h +++ b/ydb/core/kqp/common/kqp_transform.h @@ -274,7 +274,7 @@ public: NYql::NLog::EComponent component = NYql::NLog::EComponent::ProviderKqp, NYql::NLog::ELevel level = NYql::NLog::ELevel::INFO); - static void LogExpr(const NYql::TExprNode& input, NYql::TExprContext& ctx, const TString& description, + static void LogExpr(const NYql::TExprNode& input, NYql::TExprContext& ctx, const TString& description, NYql::NLog::EComponent component = NYql::NLog::EComponent::ProviderKqp, NYql::NLog::ELevel level = NYql::NLog::ELevel::INFO); diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 68c381ee03..44af6d1168 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -91,9 +91,9 @@ struct TExecuteContext : TThrRefBase { } }; -void FillAstAndPlan(IKqpHost::TQueryResult& queryResult, TExprNode* queryRoot, TExprContext& ctx, IPlanBuilder& planBuilder) { +void FillAstAndPlan(IKqpHost::TQueryResult& queryResult, TExprNode* queryRoot, TExprContext& ctx, IPlanBuilder& planBuilder) { TStringStream astStream; - auto ast = ConvertToAst(*queryRoot, ctx, TExprAnnotationFlags::None, true); + auto ast = ConvertToAst(*queryRoot, ctx, TExprAnnotationFlags::None, true); ast.Root->PrettyPrintTo(astStream, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine); queryResult.QueryAst = astStream.Str(); @@ -224,7 +224,7 @@ public: } } - FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); + FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); queryResult.SqlVersion = SqlVersion; queryResult.QueryPlan = SerializeScriptPlan(queryPlans); } @@ -549,7 +549,7 @@ public: auto& paramDesc = *queryCtx->PreparingQuery->AddParameters(); paramDesc.SetName(TString(name)); if (!ExportTypeToKikimrProto(*expectedType, *paramDesc.MutableType(), ctx)) { - ctx.AddError(TIssue(ctx.GetPosition(parameter.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(parameter.Pos()), TStringBuilder() << "Failed to export parameter type: " << name)); return nullptr; } @@ -557,7 +557,7 @@ public: return ret; } - auto parameterValue = ValidateParameter(TString(name), *expectedType, ctx.GetPosition(parameter.Pos()), + auto parameterValue = ValidateParameter(TString(name), *expectedType, ctx.GetPosition(parameter.Pos()), queryCtx->Parameters, ctx); if (!parameterValue) { return nullptr; @@ -571,8 +571,8 @@ public: TExprNode::TPtr valueExpr; { - TIssueScopeGuard issueScope(ctx.IssueManager, [parameter, name, &ctx]() { - return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(parameter.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, + TIssueScopeGuard issueScope(ctx.IssueManager, [parameter, name, &ctx]() { + return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(parameter.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder() << "Failed to parse parameter value: " << name)); }); @@ -613,8 +613,8 @@ public: output = input; for (const auto& paramDesc : QueryCtx->PreparedQuery->GetParameters()) { - TIssueScopeGuard issueScope(ctx.IssueManager, [input, ¶mDesc, &ctx]() { - return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(input->Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder() + TIssueScopeGuard issueScope(ctx.IssueManager, [input, ¶mDesc, &ctx]() { + return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(input->Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder() << "Failed to parse parameter type: " << paramDesc.GetName())); }); @@ -834,7 +834,7 @@ public: YQL_ENSURE(TMaybeNode<TKiDataQuery>(query)); TKiDataQuery dataQuery(query); - auto queryAstStr = SerializeExpr(ctx, *query); + auto queryAstStr = SerializeExpr(ctx, *query); bool useScanQuery = ShouldUseScanQuery(dataQuery, settings); @@ -1062,14 +1062,14 @@ public: TypesCtx->AddDataSink(ResultProviderName, resultProvider); TypesCtx->AvailablePureResultDataSources = TVector<TString>(1, TString(KikimrProviderName)); - // Config provider - const TGatewaysConfig* gatewaysConfig = nullptr; // TODO: can we get real gatewaysConfig here? - auto allowSettings = [](TStringBuf settingName) { - return settingName == "OrderedColumns" || settingName == "DisableOrderedColumns"; - }; - auto configProvider = CreateConfigProvider(*TypesCtx, gatewaysConfig, allowSettings); - TypesCtx->AddDataSource(ConfigProviderName, configProvider); - + // Config provider + const TGatewaysConfig* gatewaysConfig = nullptr; // TODO: can we get real gatewaysConfig here? + auto allowSettings = [](TStringBuf settingName) { + return settingName == "OrderedColumns" || settingName == "DisableOrderedColumns"; + }; + auto configProvider = CreateConfigProvider(*TypesCtx, gatewaysConfig, allowSettings); + TypesCtx->AddDataSource(ConfigProviderName, configProvider); + YQL_ENSURE(TypesCtx->Initialize(*ExprCtx)); YqlTransformer = TTransformationPipeline(TypesCtx) @@ -1427,8 +1427,8 @@ private: } settings.InferSyntaxVersion = true; - settings.V0ForceDisable = false; - settings.WarnOnV0 = false; + settings.V0ForceDisable = false; + settings.WarnOnV0 = false; settings.DefaultCluster = Cluster; settings.ClusterMapping = ClustersMap; auto tablePathPrefix = SessionCtx->Config()._KqpTablePathPrefix.Get().GetRef(); diff --git a/ydb/core/kqp/host/kqp_host_impl.h b/ydb/core/kqp/host/kqp_host_impl.h index 0d122cd716..fb1fb139cd 100644 --- a/ydb/core/kqp/host/kqp_host_impl.h +++ b/ydb/core/kqp/host/kqp_host_impl.h @@ -85,7 +85,7 @@ protected: virtual void FillResult(TResult& result) const = 0; NYql::TExprNode::TPtr GetExprRoot() const { return ExprRoot; } - NYql::TExprContext& GetExprContext() const { return ExprCtx; } + NYql::TExprContext& GetExprContext() const { return ExprCtx; } NYql::IGraphTransformer& GetTransformer() const { return Transformer; } private: diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index ae1ea0ce83..42b4eca7c6 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -525,7 +525,7 @@ private: auto optimizedQuery = query; auto status = InstantTransform(*PhysicalOptimizeTransformer, optimizedQuery, ctx); if (status != IGraphTransformer::TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to optimize query.")); + ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to optimize query.")); return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); } @@ -536,7 +536,7 @@ private: auto builtQuery = optimizedQuery; status = InstantTransform(*PhysicalBuildQueryTransformer, builtQuery, ctx); if (status != IGraphTransformer::TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to build physical query.")); + ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to build physical query.")); return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); } diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index cef8b075aa..1eeb95daee 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -58,7 +58,7 @@ bool HasIndexesToWrite(const TKikimrTableDescription& tableData) { TExprBase BuildReadTable(const TKiReadTable& read, const TKikimrTableDescription& tableData, bool withSystemColumns, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { - bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); + bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); TExprNode::TPtr readTable; const auto& columns = read.GetSelectColumns(ctx, tableData, withSystemColumns); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp index 7f1c614009..a4a4a0b3e0 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp @@ -70,16 +70,16 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); THashSet<TString> possibleKeys; - TPredicateExtractorSettings settings; + TPredicateExtractorSettings settings; settings.MergeAdjacentPointRanges = true; - auto extractor = MakePredicateRangeExtractor(settings); - YQL_ENSURE(tableDesc.SchemeNode); + auto extractor = MakePredicateRangeExtractor(settings); + YQL_ENSURE(tableDesc.SchemeNode); - bool prepareSuccess = extractor->Prepare(flatmap.Lambda().Ptr(), *tableDesc.SchemeNode, possibleKeys, ctx, typesCtx); + bool prepareSuccess = extractor->Prepare(flatmap.Lambda().Ptr(), *tableDesc.SchemeNode, possibleKeys, ctx, typesCtx); YQL_ENSURE(prepareSuccess); - auto buildResult = extractor->BuildComputeNode(tableDesc.Metadata->KeyColumnNames, ctx); - TExprNode::TPtr ranges = buildResult.ComputeNode; + auto buildResult = extractor->BuildComputeNode(tableDesc.Metadata->KeyColumnNames, ctx); + TExprNode::TPtr ranges = buildResult.ComputeNode; if (!ranges) { return node; diff --git a/ydb/core/kqp/prepare/kqp_prepare_impl.h b/ydb/core/kqp/prepare/kqp_prepare_impl.h index aaf4bb7ef1..1d6e4c5064 100644 --- a/ydb/core/kqp/prepare/kqp_prepare_impl.h +++ b/ydb/core/kqp/prepare/kqp_prepare_impl.h @@ -24,7 +24,7 @@ enum class ETableWriteType { bool HasEffects(const NYql::NNodes::TKiProgram& program); bool HasResults(const NYql::NNodes::TKiProgram& program); -NYql::NNodes::TCoList GetEmptyEffectsList(NYql::TPositionHandle pos, NYql::TExprContext& ctx); +NYql::NNodes::TCoList GetEmptyEffectsList(NYql::TPositionHandle pos, NYql::TExprContext& ctx); TMkqlExecuteResult ExecuteMkql(NYql::NNodes::TKiProgram program, TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, NYql::TExprContext& ctx, TIntrusivePtr<TKqpTransactionState> txState, diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp index f67a9f7190..bc2d4e94ce 100644 --- a/ydb/core/kqp/prepare/kqp_query_exec.cpp +++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp @@ -112,7 +112,7 @@ bool ProcessEffect(TExprBase& effect, const THashMap<TString, NKikimrKqp::TParam auto predicateValue = GetPredicateValue(revert, transformCtx, bindingsMap); if (predicateValue) { - ctx.AddWarning(YqlIssue(ctx.GetPosition(revert.Pos()), TIssuesIds::KIKIMR_OPERATION_REVERTED, TStringBuilder() + ctx.AddWarning(YqlIssue(ctx.GetPosition(revert.Pos()), TIssuesIds::KIKIMR_OPERATION_REVERTED, TStringBuilder() << "Operation reverted due to constraint violation: " << revert.Constraint().Value())); return GetEmptyEffectsList(revert.Pos(), ctx).Ptr(); @@ -126,7 +126,7 @@ bool ProcessEffect(TExprBase& effect, const THashMap<TString, NKikimrKqp::TParam auto predicateValue = GetPredicateValue(abort, transformCtx, bindingsMap); if (predicateValue) { - ctx.AddError(YqlIssue(ctx.GetPosition(abort.Pos()), TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(abort.Pos()), TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, TStringBuilder() << "Operation aborted due to constraint violation: " << abort.Constraint().Value())); return TExprNode::TPtr(); @@ -247,7 +247,7 @@ public: TVector<TExprBase> results; for (auto& root : analyzeResults.ExecutionRoots) { if (root.Scope) { - ctx.AddError(TIssue(ctx.GetPosition(root.Node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(root.Node.Pos()), TStringBuilder() << "Unexpected nested execution roots in rewritten program.")); return TStatus::Error; } @@ -488,10 +488,10 @@ TMkqlExecuteResult ExecuteMkql(TKiProgram program, TIntrusivePtr<IKqpGateway> ga auto mkqlProgram = TranslateToMkql(program, ctx, TString(ReadTargetParamName)); if (!mkqlProgram) { return TMkqlExecuteResult(MakeFuture(ResultFromError<IKqpGateway::TMkqlResult>( - "Mkql translation failed.", ctx.GetPosition(program.Pos())))); + "Mkql translation failed.", ctx.GetPosition(program.Pos())))); } - auto mkqlProgramText = NCommon::SerializeExpr(ctx, mkqlProgram.Cast().Ref()); + auto mkqlProgramText = NCommon::SerializeExpr(ctx, mkqlProgram.Cast().Ref()); TFuture<IKqpGateway::TMkqlResult> future; auto paramBindings = CollectParams(mkqlProgram.Cast()); @@ -537,7 +537,7 @@ bool AddDeferredEffect(NNodes::TExprBase effect, const TVector<NKikimrKqp::TPara { if (transformCtx.QueryCtx->PrepareOnly) { auto& newEffect = *transformCtx.GetPreparingKql().AddEffects(); - newEffect.SetNodeAst(NCommon::SerializeExpr(ctx, effect.Ref())); + newEffect.SetNodeAst(NCommon::SerializeExpr(ctx, effect.Ref())); for (auto& binding : bindings) { newEffect.AddBindings()->CopyFrom(binding); } diff --git a/ydb/core/kqp/prepare/kqp_query_finalize.cpp b/ydb/core/kqp/prepare/kqp_query_finalize.cpp index 72334ef5ca..5e442780f7 100644 --- a/ydb/core/kqp/prepare/kqp_query_finalize.cpp +++ b/ydb/core/kqp/prepare/kqp_query_finalize.cpp @@ -41,7 +41,7 @@ TExprBase GetDeferredEffectsList(const TDeferredEffects& effects, TPositionHandl .Done(); } -TExprBase GetEraseLocksEffects(const TString& cluster, TPositionHandle pos, TCoParameter locksList, TExprContext& ctx) { +TExprBase GetEraseLocksEffects(const TString& cluster, TPositionHandle pos, TCoParameter locksList, TExprContext& ctx) { return Build<TKiMapParameter>(ctx, pos) .Input(locksList) .Lambda() @@ -147,7 +147,7 @@ public: } if (TxState->Tx().IsInvalidated()) { - ctx.AddError(YqlIssue(ctx.GetPosition(input->Pos()), TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(input->Pos()), TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder() << "Failed to commit transaction due to previous errors.")); return RollbackOnError(ctx); } @@ -293,10 +293,10 @@ private: << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size() << ", locks count: " << TxState->Tx().Locks.Size(); - auto program = Build<TKiProgram>(ctx, TPositionHandle()) + auto program = Build<TKiProgram>(ctx, TPositionHandle()) .Results() .Build() - .Effects(GetRollbackEffects(TPositionHandle(), ctx)) + .Effects(GetRollbackEffects(TPositionHandle(), ctx)) .Done(); if (!HasEffects(program)) { @@ -325,8 +325,8 @@ private: } bool CheckCommitEffects(TExprBase effects, TExprContext& ctx) const { - TIssueScopeGuard issueScope(ctx.IssueManager, [effects, &ctx]() { - return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(effects.Pos()), TIssuesIds::DEFAULT_ERROR, + TIssueScopeGuard issueScope(ctx.IssueManager, [effects, &ctx]() { + return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(effects.Pos()), TIssuesIds::DEFAULT_ERROR, "Failed to commit transaction")); }); @@ -356,14 +356,14 @@ private: }); if (blackistedNode) { - ctx.AddError(TIssue(ctx.GetPosition(blackistedNode.Cast().Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(blackistedNode.Cast().Pos()), TStringBuilder() << "Callable not expected in commit tx: " << blackistedNode.Cast<TCallable>().CallableName())); return false; } ui32 maxReadsCount = TransformCtx->Config->_CommitReadsLimit.Get().GetRef(); if (readsCount > maxReadsCount) { - ctx.AddError(TIssue(ctx.GetPosition(effects.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(effects.Pos()), TStringBuilder() << "Reads limit exceeded in commit tx: " << readsCount << " > " << maxReadsCount)); return false; } @@ -409,7 +409,7 @@ private: TxState->Tx().Finish(); } - TExprBase GetRollbackEffects(TPositionHandle pos, TExprContext& ctx) { + TExprBase GetRollbackEffects(TPositionHandle pos, TExprContext& ctx) { if (!TxState->Tx().Locks.HasLocks()) { return GetEmptyEffectsList(pos, ctx); } @@ -426,7 +426,7 @@ private: return GetEraseLocksEffects(Cluster, pos, locksParamNode, ctx); } - TExprBase GetCommitEffects(TPositionHandle pos, TExprContext& ctx, bool& hasDataEffects) { + TExprBase GetCommitEffects(TPositionHandle pos, TExprContext& ctx, bool& hasDataEffects) { hasDataEffects = !TxState->Tx().DeferredEffects.Empty(); Y_VERIFY_DEBUG(!hasDataEffects || !TxState->Tx().Locks.Broken()); @@ -603,7 +603,7 @@ private: } // namespace -TCoList GetEmptyEffectsList(const TPositionHandle pos, TExprContext& ctx) { +TCoList GetEmptyEffectsList(const TPositionHandle pos, TExprContext& ctx) { return Build<TCoList>(ctx, pos) .ListType<TCoListType>() .ItemType<TCoVoidType>() diff --git a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp index 62998368d9..cb24a1b8a1 100644 --- a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp +++ b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp @@ -321,7 +321,7 @@ TExprNode::TPtr UnnestExecutionRoots(TExprBase node, TExprContext& ctx, const TK return SplitMap(node, ctx, *execRoots, analyzeResults); } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Can't rewrite callable for KQP execution: " << node.Ptr()->Content())); YQL_ENSURE(false); return nullptr; diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index 6e74561c5b..ee51c7349c 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -46,7 +46,7 @@ bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tab } } - ctx.AddError(TIssue(ctx.GetPosition(tuple.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(tuple.Pos()), TStringBuilder() << "Table key type mismatch" << ", column: " << meta->KeyColumnNames[i] << ", table: " << meta->Name @@ -1162,7 +1162,7 @@ TAutoPtr<IGraphTransformer> CreateKqpCheckKiProgramTransformer() { typeOk = listType->GetItemType()->GetKind() == ETypeAnnotationKind::Void; } if (!typeOk) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Invalid program effects type: " << FormatType(effectsType))); return TStatus::Error; } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 822bf8c9cf..035d6b7701 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -132,7 +132,7 @@ private: NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( TExprList(node.Ref().ChildPtr(4)), ctx); if (!settings.Mode) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Mode option is required for Kikimr table writes.")); return TStatus::Error; } @@ -152,29 +152,29 @@ private: SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "insert_ignore") { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "INSERT OR IGNORE is not yet supported for Kikimr.")); return TStatus::Error; } else if (mode == "update") { if (!settings.Filter) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Filter option is required for table update.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Filter option is required for table update.")); return TStatus::Error; } if (!settings.Update) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Update option is required for table update.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Update option is required for table update.")); return TStatus::Error; } SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "delete") { if (!settings.Filter) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Filter option is required for table delete.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Filter option is required for table delete.")); return TStatus::Error; } SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unsupported Kikimr table write mode: " << settings.Mode.Cast().Value())); return TStatus::Error; } @@ -184,7 +184,7 @@ private: NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings( TExprList(node.Ref().ChildPtr(4)), ctx); if (!settings.Mode) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Mode option is required for Kikimr scheme writes.")); return TStatus::Error; } @@ -192,7 +192,7 @@ private: auto mode = settings.Mode.Cast(); if (mode == "create") { if (!settings.Columns) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "No columns provided for create mode.")); return TStatus::Error; } @@ -201,7 +201,7 @@ private: return TStatus::Ok; } else if (mode == "alter") { if (!settings.AlterActions) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "No actions provided for alter mode.")); return TStatus::Error; } @@ -210,7 +210,7 @@ private: return TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unsupported Kikimr scheme write mode: " << settings.Mode.Cast().Value())); return TStatus::Error; } @@ -222,7 +222,7 @@ private: return TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid table key type.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid table key type.")); return TStatus::Error; } @@ -338,7 +338,7 @@ public: return {}; } - TPositionHandle pos; + TPositionHandle pos; TVector<TExprBase> locators; auto& grpcData = config->GetGrpc(); @@ -390,7 +390,7 @@ public: if (node.IsCallable(TCoDataSink::CallableName())) { if (node.Child(0)->Content() == KikimrProviderName) { if (node.Child(1)->Content().empty()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name")); + ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name")); return false; } @@ -399,7 +399,7 @@ public: } } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Kikimr DataSink parameters.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Kikimr DataSink parameters.")); return false; } @@ -539,7 +539,7 @@ public: YQL_ENSURE(!settings.Columns.Cast().Empty()); if (!settings.PrimaryKey) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Primary key is required for ydb tables.")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Primary key is required for ydb tables.")); return nullptr; } @@ -647,7 +647,7 @@ public: } } - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Failed to rewrite IO.")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Failed to rewrite IO.")); return nullptr; } @@ -774,7 +774,7 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt return HandleKql(callable, ctx); } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Unsupported function: " + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Unsupported function: " << callable.CallableName())); return TStatus::Error; } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index b28cb8dcf8..65ef7eb932 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -124,7 +124,7 @@ public: case EKikimrQueryType::Unspecified: { if (value.Metadata) { if (!value.Metadata->Indexes.empty()) { - result->AddIssue(TIssue({}, TStringBuilder() + result->AddIssue(TIssue({}, TStringBuilder() << "Using index tables unsupported for legacy or unspecified request type")); result->SetStatus(TIssuesIds::KIKIMR_INDEX_METADATA_LOAD_FAILED); return; @@ -181,8 +181,8 @@ public: return TStatus::Error; } } else { - TIssueScopeGuard issueScope(ctx.IssueManager, [input, &table, &ctx]() { - return MakeIntrusive<TIssue>(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + TIssueScopeGuard issueScope(ctx.IssueManager, [input, &table, &ctx]() { + return MakeIntrusive<TIssue>(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Failed to load metadata for table: " << NCommon::FullTableName(table.first, table.second))); }); @@ -217,7 +217,7 @@ protected: "scanquery" }; - bool HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, const TMaybe<TString>& value, + bool HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, const TMaybe<TString>& value, TExprContext& ctx) final { YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified); @@ -251,11 +251,11 @@ protected: return true; } - bool HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx) final { + bool HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx) final { YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified); if (SessionCtx->Query().Type != EKikimrQueryType::YqlInternal) { - ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_PRAGMA_NOT_SUPPORTED, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_PRAGMA_NOT_SUPPORTED, TStringBuilder() << "Pragma auth not supported inside Kikimr query.")); return false; } @@ -302,7 +302,7 @@ public: TString defaultToken; if (auto credential = Types.FindCredential(TString("default_") + KikimrProviderName)) { if (credential->Category != KikimrProviderName) { - ctx.AddError(TIssue({}, TStringBuilder() + ctx.AddError(TIssue({}, TStringBuilder() << "Mismatch default credential category, expected: " << KikimrProviderName << ", but found: " << credential->Category)); return false; @@ -322,7 +322,7 @@ public: if (auto credential = Types.FindCredential(TString("default_") + cluster)) { if (credential->Category != KikimrProviderName) { - ctx.AddError(TIssue({}, TStringBuilder() + ctx.AddError(TIssue({}, TStringBuilder() << "Mismatch credential category, for cluster " << cluster << " expected: " << KikimrProviderName << ", but found: " << credential->Category)); @@ -365,7 +365,7 @@ public: if (node.IsCallable(TCoDataSource::CallableName())) { if (node.Child(0)->Content() == KikimrProviderName) { if (node.Child(1)->Content().empty()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name")); + ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name")); return false; } @@ -374,7 +374,7 @@ public: } } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Kikimr DataSource parameters.")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Kikimr DataSource parameters.")); return false; } @@ -616,7 +616,7 @@ IGraphTransformer::TStatus TKiSourceVisitorTransformer::DoTransform(TExprNode::T return HandleConfigure(TExprBase(input), ctx); } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSource) Unsupported function: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSource) Unsupported function: " << input->Content())); return TStatus::Error; } diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 5f4106c836..ad360fcaeb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -237,7 +237,7 @@ public: Y_UNUSED(output); Y_UNUSED(ctx); - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(TExprNode::TPtr(resultNode)); return IGraphTransformer::TStatus::Ok; }))); @@ -273,7 +273,7 @@ public: SessionCtx->Config().Dispatch(clusterName, name, value, NCommon::TSettingDispatcher::EStage::RUNTIME); } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } @@ -298,7 +298,7 @@ public: writer.OnEndList(); - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); return SyncOk(); } @@ -309,12 +309,12 @@ public: return SyncStatus(requireStatus); } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSource) Failed to execute node: " << input->Content())); return SyncError(); } @@ -366,7 +366,7 @@ private: } if (NCommon::HasResOrPullOption(res.Ref(), "ref")) { - ctx.AddError(TIssue(ctx.GetPosition(res.Pos()), TStringBuilder() << "refselect isn't supported for Kikimr provider.")); + ctx.AddError(TIssue(ctx.GetPosition(res.Pos()), TStringBuilder() << "refselect isn't supported for Kikimr provider.")); return SyncError(); } @@ -374,7 +374,7 @@ private: auto* runResult = SessionCtx->Query().Results.FindPtr(exec.Ref().UniqueId()); if (!runResult) { - ctx.AddError(TIssue(ctx.GetPosition(exec.Pos()), TStringBuilder() << "KiExecute run result not found.")); + ctx.AddError(TIssue(ctx.GetPosition(exec.Pos()), TStringBuilder() << "KiExecute run result not found.")); return SyncError(); } @@ -430,7 +430,7 @@ public: } } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } @@ -445,7 +445,7 @@ public: } if (TMaybeNode<TCoNth>(input)) { - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } @@ -503,7 +503,7 @@ public: return resultNode; }); - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } @@ -967,7 +967,7 @@ public: }, "Executing DROP GROUP"); } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Failed to execute node: " << input->Content())); return SyncError(); } @@ -982,7 +982,7 @@ private: TExprContext& ctx, const TString& cluster, TMaybe<TString> mode, TMaybe<bool> useNewEngine, const TExecuteRunFunc& runFunc, const TExecuteFinalizeFunc& finalizeFunc) { - if (node.Ref().GetState() == TExprNode::EState::ExecutionComplete) { + if (node.Ref().GetState() == TExprNode::EState::ExecutionComplete) { return SyncOk(); } @@ -1038,7 +1038,7 @@ private: auto result = asyncResult->GetResult(); if (!result.Success()) { - node.Ptr()->SetState(TExprNode::EState::Error); + node.Ptr()->SetState(TExprNode::EState::Error); return std::make_pair(IGraphTransformer::TStatus::Error, TAsyncTransformCallbackFuture()); } @@ -1048,7 +1048,7 @@ private: SessionCtx->Query().ExecutionOrder.push_back(resultId); - node.Ptr()->SetState(TExprNode::EState::ExecutionComplete); + node.Ptr()->SetState(TExprNode::EState::ExecutionComplete); node.Ptr()->SetResult(ctx.NewAtom(node.Pos(), ToString(resultId))); if (finalizeFunc) { @@ -1067,7 +1067,7 @@ private: Y_UNUSED(ctx); output = input; - input->SetState(TExprNode::EState::ExecutionRequired); + input->SetState(TExprNode::EState::ExecutionRequired); return IGraphTransformer::TStatus::Repeat; }); })); @@ -1115,16 +1115,16 @@ private: return SessionCtx->Tx().ApplyTableOperations(tableOps, tableInfo, isolationLevel, strictDml, queryType, ctx); } - bool ApplyDdlOperation(const TString& cluster, TPositionHandle pos, const TString& table, + bool ApplyDdlOperation(const TString& cluster, TPositionHandle pos, const TString& table, TYdbOperation op, TExprContext& ctx) { YQL_ENSURE(op & KikimrSchemeOps()); - auto position = ctx.GetPosition(pos); - + auto position = ctx.GetPosition(pos); + NKqpProto::TKqpTableOp protoOp; - protoOp.MutablePosition()->SetRow(position.Row); - protoOp.MutablePosition()->SetColumn(position.Column); + protoOp.MutablePosition()->SetRow(position.Row); + protoOp.MutablePosition()->SetColumn(position.Column); protoOp.SetTable(table); protoOp.SetOperation((ui32)op); diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h index 7d01ab1123..e91fd7635a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h @@ -74,7 +74,7 @@ template<typename TParent> class TNodeBuilder<TParent, TKiColumnRangeTuple> : public NGenerated::TKiColumnRangeTupleBuilder<TParent> { public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, typename NGenerated::TKiColumnRangeTupleBuilder<TParent>::BuildFuncType buildFunc, typename NGenerated::TKiColumnRangeTupleBuilder<TParent>::GetArgFuncType getArgFunc) : NGenerated::TKiColumnRangeTupleBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} diff --git a/ydb/core/kqp/provider/yql_kikimr_kql.cpp b/ydb/core/kqp/provider/yql_kikimr_kql.cpp index c024a0a8a1..95f5915a57 100644 --- a/ydb/core/kqp/provider/yql_kikimr_kql.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_kql.cpp @@ -59,7 +59,7 @@ TCoNameValueTupleList CreateSecondaryIndexKeyTuples(TCoArgument itemArg, const T TVector<std::pair<TExprNode::TPtr, const TIndexDescription*>> BuildSecondaryIndexVector( const TKikimrTableDescription& table, - TPositionHandle pos, + TPositionHandle pos, TExprContext& ctx, const THashSet<TStringBuf>* filter = nullptr) { @@ -517,7 +517,7 @@ TExprNode::TPtr KiDeleteOnTableToKql(const TKiWriteTable& node, TExprContext& ct TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikimrTablesData& tablesData, bool withSystemColumns) { const auto& read = right.Input().Cast<TKiReadTable>(); - bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); + bool unwrapValues = HasSetting(read.Settings().Ref(), "unwrap_values"); TKikimrKey key(ctx); YQL_ENSURE(key.Extract(read.TableKey().Ref())); @@ -554,7 +554,7 @@ TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikim if (secondaryIndex) { const auto& indexTableName = secondaryIndex.GetRef(); const auto& keyTableDesc = tablesData.ExistingTable(TString(cluster), TString(indexTableName)); - TKikimrKeyRange range(ctx, keyTableDesc); + TKikimrKeyRange range(ctx, keyTableDesc); const auto& selectRange = Build<TKiSelectIndexRange>(ctx, read.Pos()) .Cluster(cluster) @@ -574,7 +574,7 @@ TExprNode::TPtr KiReadTableToKql(TCoRight right, TExprContext& ctx, const TKikim return TExprBase(selectRange).Ptr(); } } else { - TKikimrKeyRange range(ctx, tableDesc); + TKikimrKeyRange range(ctx, tableDesc); const auto& selectRange = Build<TKiSelectRange>(ctx, read.Pos()) .Cluster(cluster) @@ -820,7 +820,7 @@ TExprNode::TPtr KiUpdateTableToKql(TKiUpdateTable update, TExprContext& ctx, auto schemaVersion = TStringBuilder() << tableDesc.Metadata->SchemaVersion; - TKikimrKeyRange range(ctx, tableDesc); + TKikimrKeyRange range(ctx, tableDesc); const auto& selectRange = Build<TKiSelectRange>(ctx, update.Pos()) .Cluster(cluster) .Table(versionedTable) @@ -996,7 +996,7 @@ TExprNode::TPtr KiDeleteTableToKql(TKiDeleteTable del, TExprContext& ctx, const auto versionedTable = BuildVersionedTable(*tableDesc.Metadata, del.Pos(), ctx); YQL_ENSURE(versionedTable.Path() == table); - TKikimrKeyRange range(ctx, tableDesc); + TKikimrKeyRange range(ctx, tableDesc); const auto& selectRange = Build<TKiSelectRange>(ctx, del.Pos()) .Cluster(cluster) .Table(versionedTable) diff --git a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp index 4a101b7913..4cd98892e4 100644 --- a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp @@ -75,7 +75,7 @@ TExprNode::TPtr MkqlRewriteCallables(TCallable callable, TExprContext& ctx, cons .Seal() .Build(); } else { - ctx.AddError(TIssue(ctx.GetPosition(selectRange.Pos()), TStringBuilder() << "Got unsupported callable: " + ctx.AddError(TIssue(ctx.GetPosition(selectRange.Pos()), TStringBuilder() << "Got unsupported callable: " << selectRange.CallableName())); return nullptr; } diff --git a/ydb/core/kqp/provider/yql_kikimr_opt.cpp b/ydb/core/kqp/provider/yql_kikimr_opt.cpp index 182371a740..cdbb6e600d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt.cpp @@ -155,7 +155,7 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx, auto selectRange = read.Cast<TKiSelectRange>(); - if (HasSetting(selectRange.Settings().Ref(), "Reverse")) { + if (HasSetting(selectRange.Settings().Ref(), "Reverse")) { // N.B. when SelectRange has a Reverse option we cannot optimize // sort without complex analysis of how it interacts with sorting return node.Ptr(); @@ -292,7 +292,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont auto take = node.Cast<TCoTake>(); TCoInputBase top = take; - if (!optCtx.IsSingleUsage(take.Input().Ref())) { + if (!optCtx.IsSingleUsage(take.Input().Ref())) { return node.Ptr(); } @@ -302,7 +302,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont auto skip = take.Input().Maybe<TCoSkip>(); if (skip) { - if (!optCtx.IsSingleUsage(skip.Cast().Input().Ref())) { + if (!optCtx.IsSingleUsage(skip.Cast().Input().Ref())) { return node.Ptr(); } @@ -315,7 +315,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont auto sort = top.Input().Maybe<TCoSort>(); if (sort) { - if (!optCtx.IsSingleUsage(sort.Cast().Input().Ref())) { + if (!optCtx.IsSingleUsage(sort.Cast().Input().Ref())) { return node.Ptr(); } @@ -347,7 +347,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont auto read = input; if (auto maybeFlatmap = input.Maybe<TCoFlatMap>()) { auto flatmap = maybeFlatmap.Cast(); - if (!optCtx.IsSingleUsage(flatmap.Input().Ref())) { + if (!optCtx.IsSingleUsage(flatmap.Input().Ref())) { return node.Ptr(); } @@ -648,7 +648,7 @@ TExprNode::TPtr KiApplyExtractMembersToSelectRange(TExprBase node, TExprContext& .IndexName(range.IndexName()) .Done().Ptr(); } else { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unexpected callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unexpected callable")); return nullptr; } } diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index e7a4d5a067..0ba0b34952 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -492,7 +492,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T txExplore.GetTableOperations(hasScheme, hasData); if (hasData && hasScheme) { - ctx.AddError(YqlIssue(ctx.GetPosition(commit.Pos()), TIssuesIds::KIKIMR_MIXED_SCHEME_DATA_TX)); + ctx.AddError(YqlIssue(ctx.GetPosition(commit.Pos()), TIssuesIds::KIKIMR_MIXED_SCHEME_DATA_TX)); return nullptr; } diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp index 07e34a7cf1..662082c543 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp @@ -71,7 +71,7 @@ TVector<TExprBase> ConvertToAtoms(const TSet<TString>& columns, TExprContext& ct TMaybeNode<TExprBase> EquiJoinGetIdxLookupValue(const TStringBuf& leftDataName, const TStringBuf& rightDataName, - TExprBase leftRow, const TString& leftMemberName, TPositionHandle pos, TExprContext& ctx) + TExprBase leftRow, const TString& leftMemberName, TPositionHandle pos, TExprContext& ctx) { auto leftMember = Build<TCoMember>(ctx, pos) .Struct(leftRow) @@ -134,8 +134,8 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c return false; } - auto linkSettings = GetEquiJoinLinkSettings(joinTuple.Options().Ref()); - if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { + auto linkSettings = GetEquiJoinLinkSettings(joinTuple.Options().Ref()); + if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { return false; } @@ -190,7 +190,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value(); const TKikimrTableDescription& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTable); - auto rightKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, lookupTableDesc, selectRange.Range()); + auto rightKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, lookupTableDesc, selectRange.Range()); if (!rightKeyRange) { // Don't rewrite join with arbitrary range return false; @@ -315,7 +315,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c ? BuildKeyColumnsList(lookupTableDesc, selectRange.Pos(), ctx) : lookupColumns; - auto lookup = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, TKeyRange(ctx, keyColumnRanges, {}), + auto lookup = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, TKeyRange(ctx, keyColumnRanges, {}), selectedColumns, false /* allowNulls */, ctx); // Skip null keys in lookup part as for equijoin semantics null != null, @@ -669,7 +669,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c return false; } -TExprBase GetEquiJoinLabelsNode(const TVector<TString>& labels, TPositionHandle pos, TExprContext& ctx) { +TExprBase GetEquiJoinLabelsNode(const TVector<TString>& labels, TPositionHandle pos, TExprContext& ctx) { TVector<TExprBase> labelAtoms; for (auto& label : labels) { auto atom = Build<TCoAtom>(ctx, pos) @@ -903,7 +903,7 @@ TExprNode::TPtr KiRewriteEquiJoin(TExprBase node, const TKikimrTablesData& table joinInputs.push_back(input); auto itemType = input.List().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - joinLabels.Add(ctx, *input.Scope().Ptr(), itemType->Cast<TStructExprType>()); + joinLabels.Add(ctx, *input.Scope().Ptr(), itemType->Cast<TStructExprType>()); } TMaybeNode<TExprBase> rewrittenJoin; diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp index 973c07b8ef..cce81efc42 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp @@ -345,7 +345,7 @@ TExprNode::TPtr KiApplyLimitToSelectRange(TExprBase node, TExprContext& ctx) { auto select = input.Cast<TKiSelectRangeBase>(); - if (HasSetting(select.Settings().Ref(), "ItemsLimit")) { + if (HasSetting(select.Settings().Ref(), "ItemsLimit")) { return node.Ptr(); } @@ -642,7 +642,7 @@ TExprNode::TPtr KiApplyExtractMembersToSelectRow(TExprBase node, TExprContext& c .Done().Ptr(); } -TKikimrKeyRange::TKikimrKeyRange(TExprContext& ctx, const TKikimrTableDescription& table) +TKikimrKeyRange::TKikimrKeyRange(TExprContext& ctx, const TKikimrTableDescription& table) : Table(table) , KeyRange(ctx, Table.Metadata->KeyColumnNames.size(), TMaybeNode<TExprBase>()) {} @@ -680,7 +680,7 @@ bool TKikimrKeyRange::IsFull(TExprList list) { return true; } -TMaybe<TKeyRange> TKikimrKeyRange::GetPointKeyRange(TExprContext& ctx, const TKikimrTableDescription& table, TExprList range) { +TMaybe<TKeyRange> TKikimrKeyRange::GetPointKeyRange(TExprContext& ctx, const TKikimrTableDescription& table, TExprList range) { size_t keyColumnsCount = table.Metadata->KeyColumnNames.size(); TVector<TMaybeNode<TExprBase>> fromValues(keyColumnsCount); TVector<TMaybeNode<TExprBase>> toValues(keyColumnsCount); @@ -734,7 +734,7 @@ TMaybe<TKeyRange> TKikimrKeyRange::GetPointKeyRange(TExprContext& ctx, const TKi } } - return TKeyRange(ctx, columnRanges, {}); + return TKeyRange(ctx, columnRanges, {}); } TExprList TKikimrKeyRange::ToRangeExpr(TExprBase owner, TExprContext& ctx) { @@ -1034,8 +1034,8 @@ TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesDat return CanRewriteSqlInToEquiJoin(sqlIn.Lookup().Ref().GetTypeAnn(), sqlIn.Collection().Ref().GetTypeAnn()); }; - const bool prefixOnly = true; - if (auto ret = TryConvertSqlInPredicatesToJoins(flatMap, shouldConvertSqlInToJoin, ctx, prefixOnly)) { + const bool prefixOnly = true; + if (auto ret = TryConvertSqlInPredicatesToJoins(flatMap, shouldConvertSqlInToJoin, ctx, prefixOnly)) { YQL_CLOG(INFO, ProviderKikimr) << "KiSqlInToEquiJoin"; return ret; } diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index d513a90e37..635d164827 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -118,14 +118,14 @@ struct TKikimrData { } // namespace const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TString& cluster, - const TString& table, TPositionHandle pos, TExprContext& ctx) const + const TString& table, TPositionHandle pos, TExprContext& ctx) const { auto desc = Tables.FindPtr(std::make_pair(cluster, table)); if (desc && desc->DoesExist()) { return desc; } - ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() << "Cannot find table '" << NCommon::FullTableName(cluster, table) << "' because it does not exist or you do not have access permissions." << " Please check correctness of table path and user permissions.")); @@ -316,12 +316,12 @@ void TKikimrTableDescription::ToYson(NYson::TYsonWriter& writer) const { bool TKikimrKey::Extract(const TExprNode& key) { if (key.IsCallable("MrTableConcat")) { - Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "CONCAT is not supported on Kikimr clusters.")); + Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "CONCAT is not supported on Kikimr clusters.")); return false; } if (!key.IsCallable("Key")) { - Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "Expected key")); + Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "Expected key")); return false; } @@ -331,12 +331,12 @@ bool TKikimrKey::Extract(const TExprNode& key) { const TExprNode* nameNode = key.Child(0)->Child(1); if (nameNode->IsCallable("MrTableRange") || nameNode->IsCallable("MrTableRangeStrict")) { - Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "RANGE is not supported on Kikimr clusters.")); + Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "RANGE is not supported on Kikimr clusters.")); return false; } if (!nameNode->IsCallable("String")) { - Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "Expected String as table key.")); + Ctx.AddError(TIssue(Ctx.GetPosition(key.Pos()), "Expected String as table key.")); return false; } @@ -361,16 +361,16 @@ bool TKikimrKey::Extract(const TExprNode& key) { if (tag->Content() == TStringBuf("view")) { const TExprNode* viewNode = key.Child(i)->Child(1); if (!viewNode->IsCallable("String")) { - Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Pos()), "Expected String")); + Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Pos()), "Expected String")); return false; } if (viewNode->ChildrenSize() != 1 || !EnsureAtom(*viewNode->Child(0), Ctx)) { - Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Dynamic views names are not supported")); + Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Dynamic views names are not supported")); return false; } if (viewNode->Child(0)->Content().empty()) { - Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Secondary index name must not be empty")); + Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Secondary index name must not be empty")); return false; } View = viewNode->Child(0)->Content(); @@ -489,11 +489,11 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TCoNameValueTupleLi for (const auto& op : operations) { auto table = TString(op.Name()); auto tableOp = FromString<TYdbOperation>(TString(op.Value().Cast<TCoAtom>())); - auto pos = ctx.GetPosition(op.Pos()); + auto pos = ctx.GetPosition(op.Pos()); NKqpProto::TKqpTableOp protoOp; - protoOp.MutablePosition()->SetRow(pos.Row); - protoOp.MutablePosition()->SetColumn(pos.Column); + protoOp.MutablePosition()->SetRow(pos.Row); + protoOp.MutablePosition()->SetColumn(pos.Column); protoOp.SetTable(table); protoOp.SetOperation((ui32)tableOp); @@ -508,11 +508,11 @@ TVector<NKqpProto::TKqpTableOp> TableOperationsToProto(const TKiOperationList& o for (const auto& op : operations) { auto table = TString(op.Table()); auto tableOp = FromString<TYdbOperation>(TString(op.Operation())); - auto pos = ctx.GetPosition(op.Pos()); + auto pos = ctx.GetPosition(op.Pos()); NKqpProto::TKqpTableOp protoOp; - protoOp.MutablePosition()->SetRow(pos.Row); - protoOp.MutablePosition()->SetColumn(pos.Column); + protoOp.MutablePosition()->SetRow(pos.Row); + protoOp.MutablePosition()->SetColumn(pos.Column); protoOp.SetTable(table); protoOp.SetOperation((ui32)tableOp); @@ -539,7 +539,7 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto { if (IsClosed()) { TString message = TStringBuilder() << "Cannot perform operations on closed transaction."; - ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_BAD_OPERATION, message)); + ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_BAD_OPERATION, message)); return false; } @@ -766,7 +766,7 @@ TKiExecDataQuerySettings TKiExecDataQuerySettings::Parse(TKiExecDataQuery exec) return settings; } -TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const { +TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const { TVector<TCoNameValueTuple> settings(Other); if (Mode) { diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index e6a23f5733..6903e6f193 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -215,7 +215,7 @@ public: TKikimrTableDescription& GetTable(const TString& cluster, const TString& table); const TKikimrTableDescription* EnsureTableExists(const TString& cluster, const TString& table, - TPositionHandle pos, TExprContext& ctx) const; + TPositionHandle pos, TExprContext& ctx) const; const TKikimrTableDescription& ExistingTable(const TStringBuf& cluster, const TStringBuf& table) const; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index d781cb0d53..e1aa3cda2b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -102,18 +102,18 @@ struct TKiExecDataQuerySettings { TMaybe<bool> UseNewEngine; TVector<NNodes::TCoNameValueTuple> Other; - NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const; + NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const; static TKiExecDataQuerySettings Parse(NNodes::TKiExecDataQuery exec); }; class TKikimrKeyRange { public: - TKikimrKeyRange(TExprContext& ctx, const TKikimrTableDescription& table); + TKikimrKeyRange(TExprContext& ctx, const TKikimrTableDescription& table); TKikimrKeyRange(const TKikimrTableDescription& table, const NCommon::TKeyRange& keyRange); static bool IsFull(NNodes::TExprList list); - static TMaybe<NCommon::TKeyRange> GetPointKeyRange(TExprContext& ctx, const TKikimrTableDescription& table, NNodes::TExprList range); + static TMaybe<NCommon::TKeyRange> GetPointKeyRange(TExprContext& ctx, const TKikimrTableDescription& table, NNodes::TExprList range); static NNodes::TExprBase BuildReadRangeExpr(const TKikimrTableDescription& tableDesc, const NCommon::TKeyRange& keyRange, NNodes::TCoAtomList select, bool allowNulls, TExprContext& ctx); diff --git a/ydb/core/kqp/provider/yql_kikimr_results.cpp b/ydb/core/kqp/provider/yql_kikimr_results.cpp index 4997ca3d64..8ec93f416d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_results.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_results.cpp @@ -229,7 +229,7 @@ void WriteValueToYson(const TStringStream& stream, NCommon::TYsonResultWriter& w } TExprNode::TPtr MakeAtomForDataType(EDataSlot slot, const NKikimrMiniKQL::TValue& value, - TPositionHandle pos, TExprContext& ctx) + TPositionHandle pos, TExprContext& ctx) { if (slot == EDataSlot::Bool) { return ctx.NewAtom(pos, ToString(value.GetBool())); @@ -625,9 +625,9 @@ bool ExportTypeToKikimrProto(const TTypeAnnotationNode& type, NKikimrMiniKQL::TT } TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, - TPositionHandle pos, TExprContext& ctx) + TPositionHandle pos, TExprContext& ctx) { - auto position = ctx.GetPosition(pos); + auto position = ctx.GetPosition(pos); switch (type.GetKind()) { case NKikimrMiniKQL::ETypeKind::Void: { return ctx.NewCallable(pos, "Void", {}); @@ -644,7 +644,7 @@ TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const N auto valueAtom = MakeAtomForDataType(dataTypeNode->GetSlot(), value, pos, ctx); if (!valueAtom) { - ctx.AddError(TIssue(position, TStringBuilder() << "Unsupported data type: " + ctx.AddError(TIssue(position, TStringBuilder() << "Unsupported data type: " << dataTypeNode->GetName())); return nullptr; } @@ -673,7 +673,7 @@ TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const N case NKikimrMiniKQL::ETypeKind::Tuple: { const auto& tupleType = type.GetTuple(); if (tupleType.ElementSize() != value.TupleSize()) { - ctx.AddError(TIssue(position, TStringBuilder() << "Bad tuple value, size mismatch")); + ctx.AddError(TIssue(position, TStringBuilder() << "Bad tuple value, size mismatch")); return nullptr; } @@ -717,7 +717,7 @@ TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const N case NKikimrMiniKQL::ETypeKind::Struct: { const auto& structType = type.GetStruct(); if (structType.MemberSize() != value.StructSize()) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Bad struct value, size mismatch")); + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Bad struct value, size mismatch")); return nullptr; } @@ -768,7 +768,7 @@ TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const N } default: { - ctx.AddError(TIssue(position, TStringBuilder() << "Unexpected type for protobuf value: " << type)); + ctx.AddError(TIssue(position, TStringBuilder() << "Unexpected type for protobuf value: " << type)); return nullptr; } } diff --git a/ydb/core/kqp/provider/yql_kikimr_results.h b/ydb/core/kqp/provider/yql_kikimr_results.h index 531bdb8ad0..c3460f8a07 100644 --- a/ydb/core/kqp/provider/yql_kikimr_results.h +++ b/ydb/core/kqp/provider/yql_kikimr_results.h @@ -23,7 +23,7 @@ bool IsRawKikimrResult(const NKikimrMiniKQL::TResult& result); const TTypeAnnotationNode* ParseTypeFromKikimrProto(const NKikimrMiniKQL::TType& type, TExprContext& ctx); bool ExportTypeToKikimrProto(const TTypeAnnotationNode& type, NKikimrMiniKQL::TType& protoType, TExprContext& ctx); TExprNode::TPtr ParseKikimrProtoValue(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value, - TPositionHandle pos, TExprContext& ctx); + TPositionHandle pos, TExprContext& ctx); bool CheckKqpResultType(const NKikimrMiniKQL::TResult& kqpResult, const TTypeAnnotationNode& expectedType, TExprContext& ctx); diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index bbc94d2650..692165204a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -68,8 +68,8 @@ IGraphTransformer::TStatus ConvertTableRowType(TExprNode::TPtr& input, const TKi YQL_ENSURE(actualType->GetKind() == ETypeAnnotationKind::Struct); auto rowType = actualType->Cast<TStructExprType>(); - auto pos = ctx.GetPosition(input->Pos()); - auto expectedType = GetExpectedRowType(tableDesc, *rowType, pos, ctx); + auto pos = ctx.GetPosition(input->Pos()); + auto expectedType = GetExpectedRowType(tableDesc, *rowType, pos, ctx); if (!expectedType) { return IGraphTransformer::TStatus::Error; } @@ -88,7 +88,7 @@ IGraphTransformer::TStatus ConvertTableRowType(TExprNode::TPtr& input, const TKi auto convertStatus = TryConvertTo(input, *expectedType, ctx); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.AddError(TIssue(pos, TStringBuilder() + ctx.AddError(TIssue(pos, TStringBuilder() << "Row type mismatch for table: " << FullTableName(tableDesc.Metadata->Cluster, tableDesc.Metadata->Name))); return IGraphTransformer::TStatus::Error; @@ -127,7 +127,7 @@ private: return TStatus::Error; } if (tableDesc->Metadata->GetIndexMetadata(view.GetRef()).first == nullptr) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() << "Required global index not found, index name: " << view.GetRef())); return TStatus::Error; } @@ -201,7 +201,7 @@ private: } TStatus HandleRead(TExprBase node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Failed to annotate Read!, IO rewrite should handle this")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Failed to annotate Read!, IO rewrite should handle this")); return TStatus::Error; } @@ -302,7 +302,7 @@ private: return TStatus::Error; } - auto pos = ctx.GetPosition(node.Pos()); + auto pos = ctx.GetPosition(node.Pos()); if (auto maybeTuple = node.Input().Maybe<TExprList>()) { auto tuple = maybeTuple.Cast(); @@ -310,12 +310,12 @@ private: for (const auto& value : tuple) { auto valueType = value.Ref().GetTypeAnn(); if (valueType->GetKind() != ETypeAnnotationKind::Struct) { - ctx.AddError(TIssue(pos, TStringBuilder() + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected structs as input, but got: " << *valueType)); return TStatus::Error; } - auto expectedType = GetExpectedRowType(*table, *valueType->Cast<TStructExprType>(), pos, ctx); + auto expectedType = GetExpectedRowType(*table, *valueType->Cast<TStructExprType>(), pos, ctx); if (!expectedType) { return TStatus::Error; } @@ -356,7 +356,7 @@ private: } if (!rowType) { - ctx.AddError(TIssue(pos, TStringBuilder() + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected list or stream of structs as input, but got: " << *inputType)); return TStatus::Error; } @@ -480,7 +480,7 @@ private: const auto& name = item->GetName(); if (table->GetKeyColumnIndex(TString(name))) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Cannot update primary key column: " << name)); return IGraphTransformer::TStatus::Error; } @@ -629,7 +629,7 @@ private: for (const auto& indexCol : index.Columns()) { if (!meta->Columns.contains(TString(indexCol.Value()))) { - ctx.AddError(TIssue(ctx.GetPosition(indexCol.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(indexCol.Pos()), TStringBuilder() << "Index column: " << indexCol.Value() << " was not found in the index table")); return IGraphTransformer::TStatus::Error; } @@ -822,7 +822,7 @@ private: auto& tableDesc = SessionCtx->Tables().GetTable(cluster, table); if (tableDesc.DoesExist() && !tableDesc.Metadata->IsSameTable(*meta)) { - ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() << "Table name conflict: " << NCommon::FullTableName(cluster, table) << " is used to reference multiple tables.")); return TStatus::Error; @@ -1129,7 +1129,7 @@ private: } virtual TStatus HandleWrite(TExprBase node, TExprContext& ctx) override { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Failed to annotate Write!, IO rewrite should handle this")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Failed to annotate Write!, IO rewrite should handle this")); return TStatus::Error; } @@ -1141,7 +1141,7 @@ private: auto mode = settings.Mode.Cast().Value(); if (!KikimrCommitModes().contains(mode)) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unsupported Kikimr commit mode: " << mode)); return TStatus::Error; } @@ -1164,7 +1164,7 @@ private: default: if (!isFlushCommit) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_BAD_OPERATION, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_BAD_OPERATION, TStringBuilder() << "COMMIT not supported inside Kikimr query")); return TStatus::Error; @@ -1183,7 +1183,7 @@ private: } if (!KikimrSupportedEffects().contains(effect.CallableName())) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unsupported Kikimr data query effect: " << effect.CallableName())); return TStatus::Error; } @@ -1212,7 +1212,7 @@ private: for (const auto& column : result.Columns()) { if (!structType->FindItem(column)) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Invalid column in result: " << column.Value())); return TStatus::Error; } @@ -1344,7 +1344,7 @@ private: auto retKind = lambda->GetTypeAnn()->GetKind(); if (retKind != ETypeAnnotationKind::List) { - ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected list as labmda return type, but got: " << *lambda->GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected list as labmda return type, but got: " << *lambda->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -1376,7 +1376,7 @@ private: YQL_ENSURE(predicateType); if (predicateType->GetSlot() != EDataSlot::Bool) { - ctx.AddError(TIssue(ctx.GetPosition(condEffect.Pos()), "Expected bool as predicate type")); + ctx.AddError(TIssue(ctx.GetPosition(condEffect.Pos()), "Expected bool as predicate type")); return IGraphTransformer::TStatus::Error; } @@ -1389,13 +1389,13 @@ private: return TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unknown Kql callable in type annotation: " << node.CallableName())); return TStatus::Error; } - bool EnsureModifyPermissions(const TString& cluster, const TString& table, TPositionHandle pos, TExprContext& ctx) { + bool EnsureModifyPermissions(const TString& cluster, const TString& table, TPositionHandle pos, TExprContext& ctx) { bool restrictPermissions = SessionCtx->Config()._RestrictModifyPermissions.Get(cluster).GetRef(); if (!restrictPermissions) { return true; @@ -1406,7 +1406,7 @@ private: auto tablePath = Gateway->CanonizePath(table); if (!tablePath.StartsWith(tmpDir) && !tablePath.StartsWith(homeDir)) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "User " << SessionCtx->GetUserName() << " doesn't have permissions to modify table: " << table)); return false; } diff --git a/ydb/library/yql/ast/serialize/yql_expr_serialize.cpp b/ydb/library/yql/ast/serialize/yql_expr_serialize.cpp index 89ebd5ff85..7f7bb063ac 100644 --- a/ydb/library/yql/ast/serialize/yql_expr_serialize.cpp +++ b/ydb/library/yql/ast/serialize/yql_expr_serialize.cpp @@ -27,9 +27,9 @@ using namespace NKikimr; class TWriter { public: - TWriter(TExprContext& ctx, ui16 components) - : Ctx(ctx) - , Components_(components) + TWriter(TExprContext& ctx, ui16 components) + : Ctx(ctx) + , Components_(components) { } @@ -117,7 +117,7 @@ public: if (Components_ & TSerializedExprGraphComponents::Positions) { // will write position - if (Ctx.GetPosition(node.Pos()) == LastPosition_) { + if (Ctx.GetPosition(node.Pos()) == LastPosition_) { command |= SAME_POSITION; } } @@ -139,7 +139,7 @@ public: Write(command); if ((Components_ & TSerializedExprGraphComponents::Positions) && !(command & SAME_POSITION)) { - const auto& pos = Ctx.GetPosition(node.Pos()); + const auto& pos = Ctx.GetPosition(node.Pos()); ui32 fileNum = 0; if (pos.File) { auto fileIt = Files_.find(pos.File); @@ -181,7 +181,7 @@ private: } if (Components_ & TSerializedExprGraphComponents::Positions) { - const auto& pos = Ctx.GetPosition(node.Pos()); + const auto& pos = Ctx.GetPosition(node.Pos()); const auto& file = pos.File; ui32 fileNum = 0; if (file) { @@ -216,7 +216,7 @@ private: } private: - TExprContext& Ctx; + TExprContext& Ctx; const ui16 Components_; THashMap<TStringBuf, ui32> Files_; THashMap<std::tuple<ui32, ui32, ui32>, ui32> Positions_; @@ -478,18 +478,18 @@ private: } -TString SerializeGraph(const TExprNode& node, TExprContext& ctx, ui16 components) { - TWriter writer(ctx, components); +TString SerializeGraph(const TExprNode& node, TExprContext& ctx, ui16 components) { + TWriter writer(ctx, components); writer.Prepare(node); writer.Init(); writer.Save(node); return writer.Out(); } -TExprNode::TPtr DeserializeGraph(TPositionHandle pos, TStringBuf buffer, TExprContext& ctx) { - return DeserializeGraph(ctx.GetPosition(pos), buffer, ctx); -} - +TExprNode::TPtr DeserializeGraph(TPositionHandle pos, TStringBuf buffer, TExprContext& ctx) { + return DeserializeGraph(ctx.GetPosition(pos), buffer, ctx); +} + TExprNode::TPtr DeserializeGraph(TPosition pos, TStringBuf buffer, TExprContext& ctx) { TReader reader(pos, buffer, ctx); return reader.Load(); diff --git a/ydb/library/yql/ast/serialize/yql_expr_serialize.h b/ydb/library/yql/ast/serialize/yql_expr_serialize.h index b60e01562b..e82f004cf9 100644 --- a/ydb/library/yql/ast/serialize/yql_expr_serialize.h +++ b/ydb/library/yql/ast/serialize/yql_expr_serialize.h @@ -11,8 +11,8 @@ struct TSerializedExprGraphComponents { }; }; -TString SerializeGraph(const TExprNode& node, TExprContext& ctx, ui16 components = TSerializedExprGraphComponents::Graph); -TExprNode::TPtr DeserializeGraph(TPositionHandle pos, TStringBuf buffer, TExprContext& ctx); +TString SerializeGraph(const TExprNode& node, TExprContext& ctx, ui16 components = TSerializedExprGraphComponents::Graph); +TExprNode::TPtr DeserializeGraph(TPositionHandle pos, TStringBuf buffer, TExprContext& ctx); TExprNode::TPtr DeserializeGraph(TPosition pos, TStringBuf buffer, TExprContext& ctx); } // namespace NYql diff --git a/ydb/library/yql/ast/ut/ya.make b/ydb/library/yql/ast/ut/ya.make index 5576e3ca4c..0a9b5944f2 100644 --- a/ydb/library/yql/ast/ut/ya.make +++ b/ydb/library/yql/ast/ut/ya.make @@ -6,7 +6,7 @@ FORK_SUBTESTS() SRCS( yql_ast_ut.cpp - yql_expr_check_args_ut.cpp + yql_expr_check_args_ut.cpp yql_expr_builder_ut.cpp yql_expr_ut.cpp yql_type_string_ut.cpp diff --git a/ydb/library/yql/ast/ya.make b/ydb/library/yql/ast/ya.make index 3b69da2677..2d55860c89 100644 --- a/ydb/library/yql/ast/ya.make +++ b/ydb/library/yql/ast/ya.make @@ -37,7 +37,7 @@ PEERDIR( library/cpp/deprecated/enum_codegen library/cpp/enumbitset library/cpp/string_utils/levenshtein_diff - library/cpp/yson + library/cpp/yson ydb/library/yql/public/udf ydb/library/yql/utils ydb/library/yql/core/issue diff --git a/ydb/library/yql/ast/yql_ast.cpp b/ydb/library/yql/ast/yql_ast.cpp index cbf361f5b9..16e521d901 100644 --- a/ydb/library/yql/ast/yql_ast.cpp +++ b/ydb/library/yql/ast/yql_ast.cpp @@ -35,7 +35,7 @@ namespace { } } - return str.empty(); + return str.empty(); } /////////////////////////////////////////////////////////////////////////// @@ -56,7 +56,7 @@ namespace { } inline char Peek() const { - Y_VERIFY(!AtEnd()); + Y_VERIFY(!AtEnd()); return Str_[Offset_]; } @@ -65,7 +65,7 @@ namespace { } inline char Next() { - Y_VERIFY(!AtEnd()); + Y_VERIFY(!AtEnd()); char ch = Str_[Offset_]; if (ch == '\n') { ++Position_.Row; @@ -86,7 +86,7 @@ namespace { } inline TStringBuf GetToken(ui32 begin, ui32 end) { - Y_VERIFY(end >= begin); + Y_VERIFY(end >= begin); return Str_.SubString(begin, end - begin); } @@ -326,7 +326,7 @@ namespace { Ctx_.Next(); } - if (unescapeResult != EUnescapeResult::OK) { + if (unescapeResult != EUnescapeResult::OK) { AddError(TString(UnescapeResultToString(unescapeResult))); return nullptr; } @@ -358,7 +358,7 @@ namespace { Ctx_.Next(); } - if (unescapeResult != EUnescapeResult::OK) { + if (unescapeResult != EUnescapeResult::OK) { AddError(TString(UnescapeResultToString(unescapeResult))); return nullptr; } @@ -477,8 +477,8 @@ namespace { EscapeBinaryAtom(node.GetContent(), '"', &out); } else if (node.GetFlags() & TNodeFlags::MultilineContent) { MultilineAtomPrint(out, node.GetContent()); - } else if (node.GetContent().empty()) { - EscapeArbitraryAtom(node.GetContent(), '"', &out); + } else if (node.GetContent().empty()) { + EscapeArbitraryAtom(node.GetContent(), '"', &out); } else { out << node.GetContent(); } @@ -527,8 +527,8 @@ namespace { } else if (node.GetFlags() & TNodeFlags::MultilineContent) { MultilineAtomPrint(out, node.GetContent()); } else { - if (((flags & TAstPrintFlags::AdaptArbitraryContent) && NeedEscaping(node.GetContent())) || - node.GetContent().empty()) + if (((flags & TAstPrintFlags::AdaptArbitraryContent) && NeedEscaping(node.GetContent())) || + node.GetContent().empty()) { EscapeArbitraryAtom(node.GetContent(), '"', &out); } else { diff --git a/ydb/library/yql/ast/yql_ast.h b/ydb/library/yql/ast/yql_ast.h index 231a2479db..887b53dbc5 100644 --- a/ydb/library/yql/ast/yql_ast.h +++ b/ydb/library/yql/ast/yql_ast.h @@ -63,10 +63,10 @@ struct TAstNode { return Type == List; } - inline bool IsListOfSize(ui32 len) const { - return Type == List && ListCount == len; - } - + inline bool IsListOfSize(ui32 len) const { + return Type == List && ListCount == len; + } + inline TPosition GetPosition() const { return Position; } @@ -76,41 +76,41 @@ struct TAstNode { } inline TStringBuf GetContent() const { - Y_VERIFY(IsAtom()); + Y_VERIFY(IsAtom()); return TStringBuf(Data.A.Content, Data.A.Size); } inline void SetContent(TStringBuf newContent, TMemoryPool& pool) { - Y_VERIFY(IsAtom()); + Y_VERIFY(IsAtom()); auto poolContent = pool.AppendString(newContent); Data.A.Content = poolContent.data(); Data.A.Size = poolContent.size(); } inline void SetLiteralContent(TStringBuf newContent) { - Y_VERIFY(IsAtom()); + Y_VERIFY(IsAtom()); Data.A.Content = newContent.data(); Data.A.Size = newContent.size(); } inline ui32 GetFlags() const { - Y_VERIFY(IsAtom()); + Y_VERIFY(IsAtom()); return Data.A.Flags; } inline void SetFlags(ui32 flags) { - Y_VERIFY(IsAtom()); + Y_VERIFY(IsAtom()); Data.A.Flags = flags; } inline ui32 GetChildrenCount() const { - Y_VERIFY(IsList()); + Y_VERIFY(IsList()); return ListCount; } inline const TAstNode* GetChild(ui32 index) const { - Y_VERIFY(IsList()); - Y_VERIFY(index < ListCount); + Y_VERIFY(IsList()); + Y_VERIFY(index < ListCount); if (ListCount <= SmallListCount) { return Data.S.Children[index]; } else { @@ -119,8 +119,8 @@ struct TAstNode { } inline TAstNode* GetChild(ui32 index) { - Y_VERIFY(IsList()); - Y_VERIFY(index < ListCount); + Y_VERIFY(IsList()); + Y_VERIFY(index < ListCount); if (ListCount <= SmallListCount) { return Data.S.Children[index]; } else { @@ -153,7 +153,7 @@ struct TAstNode { } for (ui32 index = 0; index < childrenCount; ++index) { - Y_VERIFY(poolChildren[index]); + Y_VERIFY(poolChildren[index]); } } diff --git a/ydb/library/yql/ast/yql_ast_escaping.cpp b/ydb/library/yql/ast/yql_ast_escaping.cpp index 707bba649c..56aee8f596 100644 --- a/ydb/library/yql/ast/yql_ast_escaping.cpp +++ b/ydb/library/yql/ast/yql_ast_escaping.cpp @@ -111,27 +111,27 @@ static bool IsValidUtf8Rune(wchar32 value) { return value <= 0x10ffff && (value < 0xd800 || value > 0xdfff); } -TStringBuf UnescapeResultToString(EUnescapeResult result) -{ - switch (result) { - case EUnescapeResult::OK: - return "OK"; - case EUnescapeResult::INVALID_ESCAPE_SEQUENCE: - return "Expected escape sequence"; - case EUnescapeResult::INVALID_BINARY: - return "Invalid binary value"; - case EUnescapeResult::INVALID_OCTAL: - return "Invalid octal value"; - case EUnescapeResult::INVALID_HEXADECIMAL: - return "Invalid hexadecimal value"; - case EUnescapeResult::INVALID_UNICODE: - return "Invalid unicode value"; - case EUnescapeResult::INVALID_END: - return "Unexpected end of atom"; - } - return "Unknown unescape error"; -} - +TStringBuf UnescapeResultToString(EUnescapeResult result) +{ + switch (result) { + case EUnescapeResult::OK: + return "OK"; + case EUnescapeResult::INVALID_ESCAPE_SEQUENCE: + return "Expected escape sequence"; + case EUnescapeResult::INVALID_BINARY: + return "Invalid binary value"; + case EUnescapeResult::INVALID_OCTAL: + return "Invalid octal value"; + case EUnescapeResult::INVALID_HEXADECIMAL: + return "Invalid hexadecimal value"; + case EUnescapeResult::INVALID_UNICODE: + return "Invalid unicode value"; + case EUnescapeResult::INVALID_END: + return "Unexpected end of atom"; + } + return "Unknown unescape error"; +} + void EscapeArbitraryAtom(TStringBuf atom, char quoteChar, IOutputStream* out) { out->Write(quoteChar); diff --git a/ydb/library/yql/ast/yql_ast_escaping.h b/ydb/library/yql/ast/yql_ast_escaping.h index b031561f8b..744ab98516 100644 --- a/ydb/library/yql/ast/yql_ast_escaping.h +++ b/ydb/library/yql/ast/yql_ast_escaping.h @@ -2,7 +2,7 @@ #include <util/generic/fwd.h> #include <util/system/types.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> class IOutputStream; @@ -20,8 +20,8 @@ enum class EUnescapeResult INVALID_END, }; -TStringBuf UnescapeResultToString(EUnescapeResult result); - +TStringBuf UnescapeResultToString(EUnescapeResult result); + void EscapeArbitraryAtom(TStringBuf atom, char quoteChar, IOutputStream* out); EUnescapeResult UnescapeArbitraryAtom( diff --git a/ydb/library/yql/ast/yql_ast_ut.cpp b/ydb/library/yql/ast/yql_ast_ut.cpp index f74da249d4..23e140653d 100644 --- a/ydb/library/yql/ast/yql_ast_ut.cpp +++ b/ydb/library/yql/ast/yql_ast_ut.cpp @@ -192,8 +192,8 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { TestBadArbitraryAtom("(a\")", "Unexpected \""); TestBadArbitraryAtom("(\"++++\"11111)", "Unexpected end of \""); TestBadArbitraryAtom("(\"\\", "Expected escape sequence"); - TestBadArbitraryAtom("(\"\\\")", "Unexpected end of atom"); - TestBadArbitraryAtom("(\"abc)", "Unexpected end of atom"); + TestBadArbitraryAtom("(\"\\\")", "Unexpected end of atom"); + TestBadArbitraryAtom("(\"abc)", "Unexpected end of atom"); TestBadArbitraryAtom("(\"\\018\")", "Invalid octal value"); TestBadArbitraryAtom("(\"\\01\")", "Invalid octal value"); @@ -227,7 +227,7 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { TestBadArbitraryAtom("(x\"ag\")", "Invalid binary value"); TestBadArbitraryAtom("(x\"abc\")", "Invalid binary value"); TestBadArbitraryAtom("(x\"abcd)", "Invalid binary value"); - TestBadArbitraryAtom("(x\"abcd", "Unexpected end of atom"); + TestBadArbitraryAtom("(x\"abcd", "Unexpected end of atom"); } void ParseAndPrint(const TString& program, const TString& expected) { @@ -252,8 +252,8 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { "(\"тестовая строка\")", "(\"\\u0442\\u0435\\u0441\\u0442\\u043E\\u0432\\u0430" "\\u044F \\u0441\\u0442\\u0440\\u043E\\u043A\\u0430\")"); - - ParseAndPrint("(\"\")", "(\"\")"); + + ParseAndPrint("(\"\")", "(\"\")"); } Y_UNIT_TEST(BinaryAtom) { @@ -283,7 +283,7 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { ParseAndAdaptPrint("(\"braces(in)test\")", "(\"braces(in)test\")"); ParseAndAdaptPrint("(\"escaped\\u1234sequence\")", "(\"escaped\\u1234sequence\")"); ParseAndAdaptPrint("(\"escaped\\x41sequence\")", "(escapedAsequence)"); - ParseAndAdaptPrint("(\"\")", "(\"\")"); + ParseAndAdaptPrint("(\"\")", "(\"\")"); } void ParseError(const TString& program) { @@ -380,24 +380,24 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { Y_UNIT_TEST(UnicodePrettyPrint) { ParseAndAdaptPrint("(\"абв αβγ ﬡ\")", "(\"\\u0430\\u0431\\u0432 \\u03B1\\u03B2\\u03B3 \\uFB21\")"); } - - Y_UNIT_TEST(SerializeQuotedEmptyAtom) { - TMemoryPool pool(1024); - TPosition pos(1, 1); - TAstNode* empty = TAstNode::Quote(pos, pool, TAstNode::NewAtom(pos, "", pool)); - TString expected = "'\"\""; - - UNIT_ASSERT_STRINGS_EQUAL(empty->ToString(), expected); - - TString pretty = empty->ToString(TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine | - TAstPrintFlags::AdaptArbitraryContent); - RemoveAll(pretty, '\n'); - UNIT_ASSERT_EQUAL(pretty, expected); - - pretty = empty->ToString(TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine); - RemoveAll(pretty, '\n'); - UNIT_ASSERT_EQUAL(pretty, expected); - } + + Y_UNIT_TEST(SerializeQuotedEmptyAtom) { + TMemoryPool pool(1024); + TPosition pos(1, 1); + TAstNode* empty = TAstNode::Quote(pos, pool, TAstNode::NewAtom(pos, "", pool)); + TString expected = "'\"\""; + + UNIT_ASSERT_STRINGS_EQUAL(empty->ToString(), expected); + + TString pretty = empty->ToString(TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine | + TAstPrintFlags::AdaptArbitraryContent); + RemoveAll(pretty, '\n'); + UNIT_ASSERT_EQUAL(pretty, expected); + + pretty = empty->ToString(TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine); + RemoveAll(pretty, '\n'); + UNIT_ASSERT_EQUAL(pretty, expected); + } } } // namespace NYql diff --git a/ydb/library/yql/ast/yql_constraint.cpp b/ydb/library/yql/ast/yql_constraint.cpp index 5f11b8c3a2..74adceb153 100644 --- a/ydb/library/yql/ast/yql_constraint.cpp +++ b/ydb/library/yql/ast/yql_constraint.cpp @@ -66,10 +66,10 @@ void TConstraintSet::ToJson(NJson::TJsonWriter& writer) const { for (const auto& node : Constraints_) { writer.WriteKey(node->GetName()); node->ToJson(writer); - } + } writer.CloseMap(); -} - +} + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TSetType& columns) @@ -165,10 +165,10 @@ void TColumnSetConstraintNodeBase::ToJson(NJson::TJsonWriter& out) const { out.OpenArray(); for (const auto& column : Columns_) { out.Write(column); - } + } out.CloseArray(); -} - +} + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// TSortedConstraintNode::TSortedConstraintNode(TExprContext& ctx, TContainerType&& content) @@ -245,10 +245,10 @@ void TSortedConstraintNode::ToJson(NJson::TJsonWriter& out) const { out.Write(JoinSeq(';', Content_[i].first)); out.Write(Content_[i].second); out.CloseArray(); - } + } out.CloseArray(); -} - +} + bool TSortedConstraintNode::IsPrefixOf(const TSortedConstraintNode& node) const { return node.Includes(*this); } @@ -603,10 +603,10 @@ void TPassthroughConstraintNode::ToJson(NJson::TJsonWriter& out) const { for (const auto& [resultColumn, originalColumn] : part.second) { out.Write(JoinSeq(';', resultColumn), originalColumn); } - } + } out.CloseMap(); -} - +} + const TPassthroughConstraintNode* TPassthroughConstraintNode::ExtractField(TExprContext& ctx, const std::string_view& field) const { TMapType passtrought; for (const auto& part : Mapping_) { @@ -722,8 +722,8 @@ bool TEmptyConstraintNode::Equals(const TConstraintNode& node) const { void TEmptyConstraintNode::ToJson(NJson::TJsonWriter& out) const { out.Write(true); -} - +} + const TEmptyConstraintNode* TEmptyConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& /*ctx*/) { if (constraints.empty()) { return nullptr; @@ -838,10 +838,10 @@ void TVarIndexConstraintNode::ToJson(NJson::TJsonWriter& out) const { out.Write(resultIndex); out.Write(originalIndex); out.CloseArray(); - } + } out.CloseArray(); -} - +} + const TVarIndexConstraintNode* TVarIndexConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) { if (constraints.empty()) { return nullptr; @@ -1003,10 +1003,10 @@ void TMultiConstraintNode::ToJson(NJson::TJsonWriter& out) const { for (const auto& [index, constraintSet] : Items_) { out.WriteKey(ToString(index)); constraintSet.ToJson(out); - } + } out.CloseMap(); -} - +} + const TMultiConstraintNode* TMultiConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) { if (constraints.empty()) { return nullptr; diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index d379921c2f..82f6c5812a 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -8,7 +8,7 @@ #include <util/generic/hash.h> #include <util/generic/size_literals.h> #include <util/string/cast.h> -#include <util/string/join.h> +#include <util/string/join.h> #include <util/digest/fnv.h> #include <util/digest/murmur.h> @@ -901,7 +901,7 @@ namespace { const auto versionNode = node.GetChild(2); if (versionNode->IsAtom() || versionNode->GetChildrenCount() != 2 || !versionNode->GetChild(0)->IsAtom() || !versionNode->GetChild(1)->IsAtom() || versionNode->GetChild(0)->GetContent() != TStringBuf("quote")) { - ctx.AddError(*versionNode, "Expected quoted atom for package version"); + ctx.AddError(*versionNode, "Expected quoted atom for package version"); return false; } @@ -955,7 +955,7 @@ namespace { return nullptr; } - const auto& exports = exportsPtr->Symbols(); + const auto& exports = exportsPtr->Symbols(); const auto ex = exports.find(aliasValue); if (exports.cend() == ex) { @@ -963,7 +963,7 @@ namespace { return nullptr; } - return ctx.Expr.DeepCopy(*ex->second, exportsPtr->ExprCtx(), ctx.DeepClones, true, false); + return ctx.Expr.DeepCopy(*ex->second, exportsPtr->ExprCtx(), ctx.DeepClones, true, false); } else { const auto stub = ctx.Expr.NewAtom(node.GetPosition(), "stub"); ctx.Frames.back().Bindings[name->GetContent()] = {stub}; @@ -1010,7 +1010,7 @@ namespace { } const auto alias = node.GetChild(2); - if (!alias->IsListOfSize(2) || !alias->GetChild(0)->IsAtom() || !alias->GetChild(1)->IsAtom() || + if (!alias->IsListOfSize(2) || !alias->GetChild(0)->IsAtom() || !alias->GetChild(1)->IsAtom() || alias->GetChild(0)->GetContent() != TStringBuf("quote")) { ctx.AddError(node, "Expected quoted pair"); return false; @@ -1393,11 +1393,11 @@ namespace { }; struct TVisitNodeContext { - explicit TVisitNodeContext(TExprContext& expr) - : Expr(expr) - {} - - TExprContext& Expr; + explicit TVisitNodeContext(TExprContext& expr) + : Expr(expr) + {} + + TExprContext& Expr; size_t Order = 0ULL; bool RefAtoms = false; std::unique_ptr<TMemoryPool> Pool; @@ -1517,7 +1517,7 @@ namespace { TAstNode* res = nullptr; const auto& name = ctx.FindBinding(&node); if (!name.empty() && name != topLevelName && useBindings) { - res = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); + res = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); } else { switch (node.Type()) { case TExprNode::Atom: @@ -1525,11 +1525,11 @@ namespace { auto quote = AnnotateAstNode(&TAstNode::QuoteAtom, nullptr, annotationFlags, pool, ctx.RefAtoms); auto content = AnnotateAstNode( ctx.RefAtoms ? - TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()) : - TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()), + TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()) : + TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()), &node, annotationFlags, pool, ctx.RefAtoms); - res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, content); + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, content); break; } @@ -1542,9 +1542,9 @@ namespace { auto quote = AnnotateAstNode(&TAstNode::QuoteAtom, nullptr, annotationFlags, pool, ctx.RefAtoms); auto list = AnnotateAstNode(TAstNode::NewList( - ctx.Expr.GetPosition(node.Pos()), values.data(), values.size(), pool), &node, annotationFlags, pool, ctx.RefAtoms); + ctx.Expr.GetPosition(node.Pos()), values.data(), values.size(), pool), &node, annotationFlags, pool, ctx.RefAtoms); - res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, list); + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, list); break; } @@ -1555,7 +1555,7 @@ namespace { const auto& typeNode = *node.Child(1); Y_UNUSED(typeNode); - res = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), nameNode.Content(), pool); + res = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), nameNode.Content(), pool); auto it = ctx.Parameters.find(nameNode.Content()); if (it != ctx.Parameters.end()) { @@ -1564,14 +1564,14 @@ namespace { auto declareAtom = TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), TStringBuf("declare"), pool); auto nameAtom = ctx.RefAtoms - ? TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(nameNode.Pos()), nameNode.Content(), pool) - : TAstNode::NewAtom(ctx.Expr.GetPosition(nameNode.Pos()), nameNode.Content(), pool); + ? TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(nameNode.Pos()), nameNode.Content(), pool) + : TAstNode::NewAtom(ctx.Expr.GetPosition(nameNode.Pos()), nameNode.Content(), pool); TSmallVec<TAstNode*> children; children.push_back(AnnotateAstNode(declareAtom, nullptr, annotationFlags, pool, ctx.RefAtoms)); children.push_back(AnnotateAstNode(nameAtom, nullptr, annotationFlags, pool, ctx.RefAtoms)); children.push_back(BuildValueNode(typeNode, ctx, topLevelName, annotationFlags, pool, false)); - auto declareNode = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), children.data(), children.size(), pool); + auto declareNode = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), children.data(), children.size(), pool); declareNode = AnnotateAstNode(declareNode, &node, annotationFlags, pool, ctx.RefAtoms); ctx.Parameters.insert(std::make_pair(nameNode.Content(), @@ -1582,14 +1582,14 @@ namespace { TSmallVec<TAstNode*> children; children.push_back(AnnotateAstNode( ctx.RefAtoms ? - TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool) : - TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool), + TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool) : + TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool), nullptr, annotationFlags, pool, ctx.RefAtoms)); for (const auto& child : node.Children()) { children.push_back(BuildValueNode(*child, ctx, topLevelName, annotationFlags, pool, useBindings)); } - res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), children.data(), children.size(), pool); + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), children.data(), children.size(), pool); break; } @@ -1602,7 +1602,7 @@ namespace { TSmallVec<TAstNode*> argsChildren; for (const auto& arg : args.Children()) { const auto& name = ctx.FindBinding(arg.Get()); - const auto atom = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); + const auto atom = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); argsChildren.emplace_back(AnnotateAstNode(atom, arg.Get(), annotationFlags, pool, ctx.RefAtoms)); } @@ -1671,10 +1671,10 @@ namespace { } const auto letAtom = TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node->Pos()), TStringBuf("let"), pool); - const auto nameAtom = TAstNode::NewAtom(ctx.Expr.GetPosition(node->Pos()), name, pool); + const auto nameAtom = TAstNode::NewAtom(ctx.Expr.GetPosition(node->Pos()), name, pool); const auto valueNode = BuildValueNode(*node, ctx, name, annotationFlags, pool, true); - const auto letNode = TAstNode::NewList(ctx.Expr.GetPosition(node->Pos()), pool, + const auto letNode = TAstNode::NewList(ctx.Expr.GetPosition(node->Pos()), pool, AnnotateAstNode(letAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), AnnotateAstNode(nameAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), valueNode); @@ -1702,7 +1702,7 @@ namespace { children.insert(children.begin(), parameterNodes.begin(), parameterNodes.end()); } - const auto res = TAstNode::NewList(ctx.Expr.GetPosition(position), children.data(), children.size(), pool); + const auto res = TAstNode::NewList(ctx.Expr.GetPosition(position), children.data(), children.size(), pool); return AnnotateAstNode(res, nullptr, annotationFlags, pool, ctx.RefAtoms); } @@ -1742,7 +1742,7 @@ namespace { typedef std::unordered_set<TPairOfNodePotinters, THash<TPairOfNodePotinters>> TNodesPairSet; typedef TNodeMap<std::pair<ui32, ui32>> TArgumentsMap; - bool CompareExpressions(const TExprNode*& one, const TExprNode*& two, TArgumentsMap& argumentsMap, ui32 level, TNodesPairSet& visited) { + bool CompareExpressions(const TExprNode*& one, const TExprNode*& two, TArgumentsMap& argumentsMap, ui32 level, TNodesPairSet& visited) { const auto ins = visited.emplace(one, two); if (!ins.second) { return true; @@ -1786,7 +1786,7 @@ namespace { const auto& l = one->Children(); const auto& r = two->Children(); for (ui32 i = 0U; i < childs; ++i) { - if (!CompareExpressions(one = l[i].Get(), two = r[i].Get(), argumentsMap, level, visited)) { + if (!CompareExpressions(one = l[i].Get(), two = r[i].Get(), argumentsMap, level, visited)) { return false; } } @@ -1795,101 +1795,101 @@ namespace { return true; } - using TNodeSetPtr = std::shared_ptr<TNodeSet>; - - TNodeSetPtr ExcludeFromUnresolved(const TExprNode& args, const TNodeSetPtr& unresolved) { - if (!unresolved || unresolved->empty() || args.ChildrenSize() == 0) { - return unresolved; - } - - size_t excluded = 0; - auto newUnresolved = std::make_shared<TNodeSet>(*unresolved); - for (auto& toExclude : args.Children()) { - excluded += newUnresolved->erase(toExclude.Get()); - } - - return excluded ? newUnresolved : unresolved; - } - - TNodeSetPtr MergeUnresolvedArgs(const TNodeSetPtr& one, const TNodeSetPtr& two) { - if (!one || one->empty()) { - return two; - } - - if (!two || two->empty()) { - return one; - } - - const TNodeSetPtr& bigger = (one->size() > two->size()) ? one : two; - const TNodeSetPtr& smaller = (one->size() > two->size()) ? two : one; - - TNodeSetPtr result = std::make_shared<TNodeSet>(*bigger); - - bool inserted = false; - for (auto& item : *smaller) { - if (result->insert(item).second) { - inserted = true; - } - } - - return inserted ? result : bigger; - } - - TNodeSetPtr CollectUnresolvedArgs(const TExprNode& root, TNodeMap<TNodeSetPtr>& unresolvedArgs, TNodeSet& allArgs) { - auto it = unresolvedArgs.find(&root); - if (it != unresolvedArgs.end()) { - return it->second; - } - - TNodeSetPtr result; - switch (root.Type()) { - case TExprNode::Argument: - result = std::make_shared<TNodeSet>(TNodeSet{&root}); - break; - case TExprNode::Lambda: - { + using TNodeSetPtr = std::shared_ptr<TNodeSet>; + + TNodeSetPtr ExcludeFromUnresolved(const TExprNode& args, const TNodeSetPtr& unresolved) { + if (!unresolved || unresolved->empty() || args.ChildrenSize() == 0) { + return unresolved; + } + + size_t excluded = 0; + auto newUnresolved = std::make_shared<TNodeSet>(*unresolved); + for (auto& toExclude : args.Children()) { + excluded += newUnresolved->erase(toExclude.Get()); + } + + return excluded ? newUnresolved : unresolved; + } + + TNodeSetPtr MergeUnresolvedArgs(const TNodeSetPtr& one, const TNodeSetPtr& two) { + if (!one || one->empty()) { + return two; + } + + if (!two || two->empty()) { + return one; + } + + const TNodeSetPtr& bigger = (one->size() > two->size()) ? one : two; + const TNodeSetPtr& smaller = (one->size() > two->size()) ? two : one; + + TNodeSetPtr result = std::make_shared<TNodeSet>(*bigger); + + bool inserted = false; + for (auto& item : *smaller) { + if (result->insert(item).second) { + inserted = true; + } + } + + return inserted ? result : bigger; + } + + TNodeSetPtr CollectUnresolvedArgs(const TExprNode& root, TNodeMap<TNodeSetPtr>& unresolvedArgs, TNodeSet& allArgs) { + auto it = unresolvedArgs.find(&root); + if (it != unresolvedArgs.end()) { + return it->second; + } + + TNodeSetPtr result; + switch (root.Type()) { + case TExprNode::Argument: + result = std::make_shared<TNodeSet>(TNodeSet{&root}); + break; + case TExprNode::Lambda: + { if (!root.ChildrenSize()) { - ythrow yexception() << "lambda #" << root.UniqueId() << " has " << root.ChildrenSize() << " children"; + ythrow yexception() << "lambda #" << root.UniqueId() << " has " << root.ChildrenSize() << " children"; } - + const auto& arguments = root.Head(); - if (arguments.Type() != TExprNode::Arguments) { - ythrow yexception() << "unexpected type of arguments node in lambda #" << root.UniqueId(); - } - - arguments.ForEachChild([&](const TExprNode& arg) { - if (arg.Type() != TExprNode::Argument) { - ythrow yexception() << "expecting argument, #[" << arg.UniqueId() << "]"; - } - if (!allArgs.insert(&arg).second) { - ythrow yexception() << "argument is duplicated, #[" << arg.UniqueId() << "]"; - } - }); - + if (arguments.Type() != TExprNode::Arguments) { + ythrow yexception() << "unexpected type of arguments node in lambda #" << root.UniqueId(); + } + + arguments.ForEachChild([&](const TExprNode& arg) { + if (arg.Type() != TExprNode::Argument) { + ythrow yexception() << "expecting argument, #[" << arg.UniqueId() << "]"; + } + if (!allArgs.insert(&arg).second) { + ythrow yexception() << "argument is duplicated, #[" << arg.UniqueId() << "]"; + } + }); + for (ui32 i = 1U; i < root.ChildrenSize(); ++i) { const auto bodyUnresolvedArgs = CollectUnresolvedArgs(*root.Child(i), unresolvedArgs, allArgs); result = ExcludeFromUnresolved(arguments, bodyUnresolvedArgs); } - break; - } - case TExprNode::Callable: - case TExprNode::List: - { - root.ForEachChild([&](const TExprNode& child) { - result = MergeUnresolvedArgs(result, CollectUnresolvedArgs(child, unresolvedArgs, allArgs)); - }); - break; - } - case TExprNode::Atom: - case TExprNode::World: - break; - case TExprNode::Arguments: - ythrow yexception() << "unexpected free arguments node #[" << root.UniqueId() << "]"; - break; - } - - unresolvedArgs[&root] = result; - return result; + break; + } + case TExprNode::Callable: + case TExprNode::List: + { + root.ForEachChild([&](const TExprNode& child) { + result = MergeUnresolvedArgs(result, CollectUnresolvedArgs(child, unresolvedArgs, allArgs)); + }); + break; + } + case TExprNode::Atom: + case TExprNode::World: + break; + case TExprNode::Arguments: + ythrow yexception() << "unexpected free arguments node #[" << root.UniqueId() << "]"; + break; + } + + unresolvedArgs[&root] = result; + return result; } typedef TNodeMap<long> TRefCountsMap; @@ -2014,7 +2014,7 @@ const TTypeAnnotationNode* CompileTypeAnnotation(const TAstNode& node, TExprCont } template<class Set> -bool IsDependedImpl(const TExprNode& node, const Set& dependences, TNodeSet& visited) { +bool IsDependedImpl(const TExprNode& node, const Set& dependences, TNodeSet& visited) { if (!visited.emplace(&node).second) return false; @@ -2022,7 +2022,7 @@ bool IsDependedImpl(const TExprNode& node, const Set& dependences, TNodeSet& vis return true; for (const auto& child : node.Children()) { - if (IsDependedImpl(*child, dependences, visited)) + if (IsDependedImpl(*child, dependences, visited)) return true; } @@ -2065,10 +2065,10 @@ EChangeState DoGetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, break; } - if (!repl.first->GetBloom()) - const_cast<TExprNode*>(repl.first)->SetBloom(CalcBloom(repl.first->UniqueId())); + if (!repl.first->GetBloom()) + const_cast<TExprNode*>(repl.first)->SetBloom(CalcBloom(repl.first->UniqueId())); - if (InBloom(start->GetBloom(), repl.first->GetBloom())) { + if (InBloom(start->GetBloom(), repl.first->GetBloom())) { maybe = true; break; } @@ -2080,12 +2080,12 @@ EChangeState DoGetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, } } - start->SetBloom(1ULL); + start->SetBloom(1ULL); ui32 combinedState = EChangeState::Unchanged; bool incompleteBloom = false; start->ForEachChild([&](TExprNode& child) { combinedState |= GetChanges(&child, replaces, localReplaces, changes, updatedLambdas); - start->SetBloom(start->GetBloom() | child.GetBloom()); + start->SetBloom(start->GetBloom() | child.GetBloom()); incompleteBloom = incompleteBloom || (child.Type() != TExprNode::Arguments && !child.GetBloom()); }); if (incompleteBloom) { @@ -2101,8 +2101,8 @@ EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, c return EChangeState::Unchanged; } - if (!start->GetBloom() && TExprNode::Argument == start->Type()) { - start->SetBloom(CalcBloom(start->UniqueId())); + if (!start->GetBloom() && TExprNode::Argument == start->Type()) { + start->SetBloom(CalcBloom(start->UniqueId())); } auto& state = changes[start]; @@ -2300,7 +2300,7 @@ TExprNode::TPtr TExprContext::ReplaceNode(TExprNode::TPtr&& start, const TExprNo const auto arg = args.Child(index); if (arg == &src) { if (argIndex) { - ythrow yexception() << "argument is duplicated, #[" << arg->UniqueId() << "]"; + ythrow yexception() << "argument is duplicated, #[" << arg->UniqueId() << "]"; } argIndex = index; @@ -2314,7 +2314,7 @@ TExprNode::TPtr TExprContext::ReplaceNode(TExprNode::TPtr&& start, const TExprNo for (ui32 i = 0U; i < args.ChildrenSize(); ++i) { const auto arg = args.Child(i); - auto newArg = (i == *argIndex) ? dst : ShallowCopy(*arg); + auto newArg = (i == *argIndex) ? dst : ShallowCopy(*arg); YQL_ENSURE(replaces.emplace(arg, newArg).second); newArgNodes.emplace_back(std::move(newArg)); } @@ -2348,25 +2348,25 @@ template TExprNode::TListType TExprContext::ReplaceNodes<true>(TExprNode::TListT template TExprNode::TListType TExprContext::ReplaceNodes<false>(TExprNode::TListType&& starts, const TNodeOnNodeOwnedMap& replaces); bool IsDepended(const TExprNode& node, const TNodeSet& dependences) { - TNodeSet visited; - return !dependences.empty() && IsDependedImpl(node, dependences, visited); + TNodeSet visited; + return !dependences.empty() && IsDependedImpl(node, dependences, visited); } void CheckArguments(const TExprNode& root) { try { - TNodeMap<TNodeSetPtr> unresolvedArgsMap; - TNodeSet allArgs; - auto rootUnresolved = CollectUnresolvedArgs(root, unresolvedArgsMap, allArgs); - if (rootUnresolved && !rootUnresolved->empty()) { - TVector<ui64> ids; - for (auto& i : *rootUnresolved) { - ids.push_back(i->UniqueId()); - } - ythrow yexception() << "detected unresolved arguments at top level: #[" << JoinSeq(", ", ids) << "]"; - } + TNodeMap<TNodeSetPtr> unresolvedArgsMap; + TNodeSet allArgs; + auto rootUnresolved = CollectUnresolvedArgs(root, unresolvedArgsMap, allArgs); + if (rootUnresolved && !rootUnresolved->empty()) { + TVector<ui64> ids; + for (auto& i : *rootUnresolved) { + ids.push_back(i->UniqueId()); + } + ythrow yexception() << "detected unresolved arguments at top level: #[" << JoinSeq(", ", ids) << "]"; + } } catch (yexception& e) { - e << "\n" << root.Dump(); - throw; + e << "\n" << root.Dump(); + throw; } } @@ -2374,7 +2374,7 @@ TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& exprContext, c #ifdef _DEBUG CheckArguments(root); #endif - TVisitNodeContext ctx(exprContext); + TVisitNodeContext ctx(exprContext); ctx.RefAtoms = settings.RefAtoms; ctx.Pool = std::make_unique<TMemoryPool>(4096); ctx.Frames.push_back(TFrameContext()); @@ -2427,10 +2427,10 @@ TString TExprNode::Dump() const { return out.Str(); } -TPosition TExprNode::Pos(const TExprContext& ctx) const { - return ctx.GetPosition(Pos()); -} - +TPosition TExprNode::Pos(const TExprContext& ctx) const { + return ctx.GetPosition(Pos()); +} + TExprNode::TPtr TExprContext::RenameNode(const TExprNode& node, const TStringBuf& name) { const auto newNode = node.ChangeContent(AllocateNextUniqueId(), AppendString(name)); ExprNodes.emplace_back(newNode.Get()); @@ -2460,7 +2460,7 @@ TExprNode::TPtr TExprContext::ExactChangeChildren(const TExprNode& node, TExprNo const auto newNode = node.ChangeChildren(AllocateNextUniqueId(), std::move(children)); newNode->SetTypeAnn(node.GetTypeAnn()); newNode->CopyConstraints(node); - newNode->SetState(node.GetState()); + newNode->SetState(node.GetState()); newNode->Result = node.Result; ExprNodes.emplace_back(newNode.Get()); return newNode; @@ -2471,7 +2471,7 @@ TExprNode::TPtr TExprContext::ExactShallowCopy(const TExprNode& node) { const auto newNode = node.Clone(AllocateNextUniqueId()); newNode->SetTypeAnn(node.GetTypeAnn()); newNode->CopyConstraints(node); - newNode->SetState(node.GetState()); + newNode->SetState(node.GetState()); newNode->Result = node.Result; ExprNodes.emplace_back(newNode.Get()); return newNode; @@ -2622,11 +2622,11 @@ TNodeException::TNodeException(const TExprNode& node) } TNodeException::TNodeException(const TExprNode* node) - : Pos_(node ? node->Pos() : TPositionHandle()) + : Pos_(node ? node->Pos() : TPositionHandle()) { } -TNodeException::TNodeException(const TPositionHandle& pos) +TNodeException::TNodeException(const TPositionHandle& pos) : Pos_(pos) { } @@ -2655,8 +2655,8 @@ bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprCo bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx) { return ValidateName(ctx.GetPosition(position), name, descr, ctx); -} - +} + bool TDataExprParamsType::Validate(TPosition position, TExprContext& ctx) const { if (GetSlot() != EDataSlot::Decimal) { ctx.AddError(TIssue(position, TStringBuilder() << "Only Decimal may contain parameters, but got: " << GetName())); @@ -2680,24 +2680,24 @@ bool TDataExprParamsType::Validate(TPosition position, TExprContext& ctx) const return true; } -bool TDataExprParamsType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TDataExprParamsType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TItemExprType::Validate(TPosition position, TExprContext& ctx) const { return ValidateName(position, Name, "member", ctx); } -bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TMultiExprType::Validate(TPosition position, TExprContext& ctx) const { if (Items.size() > Max<ui16>()) { ctx.AddError(TIssue(position, TStringBuilder() << "Too many elements: " << Items.size())); return false; } - + return true; } @@ -2714,10 +2714,10 @@ bool TTupleExprType::Validate(TPosition position, TExprContext& ctx) const { return true; } -bool TTupleExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TTupleExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TStructExprType::Validate(TPosition position, TExprContext& ctx) const { if (Items.size() > Max<ui16>()) { ctx.AddError(TIssue(position, TStringBuilder() << "Too many struct members: " << Items.size())); @@ -2741,10 +2741,10 @@ bool TStructExprType::Validate(TPosition position, TExprContext& ctx) const { return true; } -bool TStructExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TStructExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TVariantExprType::Validate(TPosition position, TExprContext& ctx) const { if (UnderlyingType->GetKind() == ETypeAnnotationKind::Tuple) { if (!UnderlyingType->Cast<TTupleExprType>()->GetSize()) { @@ -2766,10 +2766,10 @@ bool TVariantExprType::Validate(TPosition position, TExprContext& ctx) const { return true; } -bool TVariantExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TVariantExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + ui32 TVariantExprType::MakeFlags(const TTypeAnnotationNode* underlyingType) { switch (underlyingType->GetKind()) { case ETypeAnnotationKind::Tuple: { @@ -2804,10 +2804,10 @@ bool TDictExprType::Validate(TPosition position, TExprContext& ctx) const { return true; } -bool TDictExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TDictExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { if (OptionalArgumentsCount > Arguments.size()) { ctx.AddError(TIssue(position, TStringBuilder() << "Too many optional arguments: " << OptionalArgumentsCount @@ -2849,10 +2849,10 @@ bool TCallableExprType::Validate(TPosition position, TExprContext& ctx) const { return true; } -bool TCallableExprType::Validate(TPositionHandle position, TExprContext& ctx) const { - return Validate(ctx.GetPosition(position), ctx); -} - +bool TCallableExprType::Validate(TPositionHandle position, TExprContext& ctx) const { + return Validate(ctx.GetPosition(position), ctx); +} + bool TTaggedExprType::Validate(TPosition position, TExprContext& ctx) const { return ValidateName(position, Tag, "tag", ctx); } @@ -2865,41 +2865,41 @@ TExprContext::TExprContext(ui64 nextUniqueId) : StringPool(4096) , NextUniqueId(nextUniqueId) , Frozen(false) - , PositionSet( - 16, - [this](TPositionHandle p) { return GetHash(p); }, - [this](TPositionHandle a, TPositionHandle b) { return IsEqual(a, b); } - ) + , PositionSet( + 16, + [this](TPositionHandle p) { return GetHash(p); }, + [this](TPositionHandle a, TPositionHandle b) { return IsEqual(a, b); } + ) { - auto handle = AppendPosition(TPosition()); - YQL_ENSURE(handle.Handle == 0); + auto handle = AppendPosition(TPosition()); + YQL_ENSURE(handle.Handle == 0); IssueManager.SetWarningToErrorTreatMessage( "Treat warning as error mode enabled. " "To disable it use \"pragma warning(\"default\", <code>);\""); IssueManager.SetIssueCountLimit(100); } -TPositionHandle TExprContext::AppendPosition(const TPosition& pos) { - YQL_ENSURE(Positions.size() <= Max<ui32>(), "Too many positions"); - - TPositionHandle handle; - handle.Handle = (ui32)Positions.size(); - Positions.push_back(pos); - - auto inserted = PositionSet.insert(handle); - if (inserted.second) { - return handle; - } - - Positions.pop_back(); - return *inserted.first; -} - -TPosition TExprContext::GetPosition(TPositionHandle handle) const { - YQL_ENSURE(handle.Handle < Positions.size(), "Unknown PositionHandle"); - return Positions[handle.Handle]; -} - +TPositionHandle TExprContext::AppendPosition(const TPosition& pos) { + YQL_ENSURE(Positions.size() <= Max<ui32>(), "Too many positions"); + + TPositionHandle handle; + handle.Handle = (ui32)Positions.size(); + Positions.push_back(pos); + + auto inserted = PositionSet.insert(handle); + if (inserted.second) { + return handle; + } + + Positions.pop_back(); + return *inserted.first; +} + +TPosition TExprContext::GetPosition(TPositionHandle handle) const { + YQL_ENSURE(handle.Handle < Positions.size(), "Unknown PositionHandle"); + return Positions[handle.Handle]; +} + TExprContext::~TExprContext() { UnFreeze(); } @@ -2930,21 +2930,21 @@ void TExprContext::Reset() { RepeatTransformCounter = 0; } -bool TExprContext::IsEqual(TPositionHandle a, TPositionHandle b) const { - YQL_ENSURE(a.Handle < Positions.size()); - YQL_ENSURE(b.Handle < Positions.size()); - return Positions[a.Handle] == Positions[b.Handle]; -} - -size_t TExprContext::GetHash(TPositionHandle p) const { - YQL_ENSURE(p.Handle < Positions.size()); - - const TPosition& pos = Positions[p.Handle]; +bool TExprContext::IsEqual(TPositionHandle a, TPositionHandle b) const { + YQL_ENSURE(a.Handle < Positions.size()); + YQL_ENSURE(b.Handle < Positions.size()); + return Positions[a.Handle] == Positions[b.Handle]; +} + +size_t TExprContext::GetHash(TPositionHandle p) const { + YQL_ENSURE(p.Handle < Positions.size()); + + const TPosition& pos = Positions[p.Handle]; size_t h = ComputeHash(pos.File); - h = CombineHashes(h, NumericHash(pos.Row)); - return CombineHashes(h, NumericHash(pos.Column)); -} - + h = CombineHashes(h, NumericHash(pos.Row)); + return CombineHashes(h, NumericHash(pos.Column)); +} + template<class T, typename... Args> const T* MakeSinglethonType(TExprContext& ctx, Args&&... args) { auto& singleton = std::get<const T*>(ctx.SingletonTypeCache); @@ -3161,20 +3161,20 @@ const TFlowExprType* TMakeTypeImpl<TFlowExprType>::Make(TExprContext& ctx, const } bool CompareExprTrees(const TExprNode*& one, const TExprNode*& two) { - TArgumentsMap map; - ui32 level = 0; - TNodesPairSet visited; - return CompareExpressions(one, two, map, level, visited); + TArgumentsMap map; + ui32 level = 0; + TNodesPairSet visited; + return CompareExpressions(one, two, map, level, visited); } bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNodeMap<ui32>& argsMap) { TArgumentsMap map; - ui32 level = 0; - TNodesPairSet visited; + ui32 level = 0; + TNodesPairSet visited; map.reserve(argsMap.size()); std::for_each(argsMap.cbegin(), argsMap.cend(), [&](const TNodeMap<ui32>::value_type& v){ map.emplace(v.first, std::make_pair(0U, v.second)); }); auto l = &one, r = &two; - return CompareExpressions(l, r, map, level, visited); + return CompareExpressions(l, r, map, level, visited); } void GatherParents(const TExprNode& node, TParentsMap& parentsMap, bool withLeaves) { diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index eee0c63682..d6eb544f9a 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -6,7 +6,7 @@ #include "yql_expr_builder.h" #include "yql_gc_nodes.h" #include "yql_constraint.h" -#include "yql_pos_handle.h" +#include "yql_pos_handle.h" #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/library/yql/public/issue/yql_issue_manager.h> @@ -36,9 +36,9 @@ //#define YQL_CHECK_NODES_CONSISTENCY #ifdef YQL_CHECK_NODES_CONSISTENCY #define ENSURE_NOT_DELETED \ - YQL_ENSURE(!Dead(), "Access to dead node # " << UniqueId_ << ": " << Type_ << " '" << ContentUnchecked() << "'"); + YQL_ENSURE(!Dead(), "Access to dead node # " << UniqueId_ << ": " << Type_ << " '" << ContentUnchecked() << "'"); #define ENSURE_NOT_FROZEN \ - YQL_ENSURE(!Frozen(), "Change in frozen node # " << UniqueId_ << ": " << Type_ << " '" << ContentUnchecked() << "'"); + YQL_ENSURE(!Frozen(), "Change in frozen node # " << UniqueId_ << ": " << Type_ << " '" << ContentUnchecked() << "'"); #define ENSURE_NOT_FROZEN_CTX \ YQL_ENSURE(!Frozen, "Change in frozen expr context."); #else @@ -209,15 +209,15 @@ public: return (GetFlags() & TypeHasOptional) != 0; } - bool HasOptionalOrNull() const { - return (GetFlags() & (TypeHasOptional | TypeHasNull)) != 0; - } - - bool IsOptionalOrNull() const { - auto kind = GetKind(); - return kind == ETypeAnnotationKind::Optional || kind == ETypeAnnotationKind::Null; - } - + bool HasOptionalOrNull() const { + return (GetFlags() & (TypeHasOptional | TypeHasNull)) != 0; + } + + bool IsOptionalOrNull() const { + auto kind = GetKind(); + return kind == ETypeAnnotationKind::Optional || kind == ETypeAnnotationKind::Null; + } + bool IsSingleton() const { return (GetFlags() & TypeHasManyValues) == 0; } @@ -335,7 +335,7 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; + bool Validate(TPositionHandle position, TExprContext& ctx) const; private: TTypeAnnotationNode::TListType Items; @@ -390,9 +390,9 @@ private: TTypeAnnotationNode::TListType Items; }; -struct TExprContext; - - +struct TExprContext; + + bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx); bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx); @@ -415,7 +415,7 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; + bool Validate(TPositionHandle position, TExprContext& ctx) const; const TStringBuf& GetName() const { return Name; @@ -470,7 +470,7 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; + bool Validate(TPositionHandle position, TExprContext& ctx) const; size_t GetSize() const { return Items.size(); @@ -689,8 +689,8 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; - + bool Validate(TPositionHandle position, TExprContext& ctx) const; + private: const TStringBuf One, Two; }; @@ -776,7 +776,7 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; + bool Validate(TPositionHandle position, TExprContext& ctx) const; static ui32 MakeFlags(const TTypeAnnotationNode* underlyingType); @@ -828,7 +828,7 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; + bool Validate(TPositionHandle position, TExprContext& ctx) const; const TTypeAnnotationNode* GetKeyType() const { return KeyType; @@ -987,8 +987,8 @@ public: } bool Validate(TPosition position, TExprContext& ctx) const; - bool Validate(TPositionHandle position, TExprContext& ctx) const; - + bool Validate(TPositionHandle position, TExprContext& ctx) const; + TMaybe<ui32> ArgumentIndexByName(const TStringBuf& name) const { auto it = IndexByName.find(name); if (it == IndexByName.end()) { @@ -1314,7 +1314,7 @@ private: Dead = 0x01, Frozen = 0x02, }; - static constexpr ui32 FlagsMask = 0x03; // all flags should fit here + static constexpr ui32 FlagsMask = 0x03; // all flags should fit here }; public: @@ -1355,10 +1355,10 @@ public: xx(ExecutionInProgress, 8) \ xx(ExecutionPending, 9) \ xx(ExecutionComplete, 10) \ - xx(Error, 11) \ - xx(Last, 12) + xx(Error, 11) \ + xx(Last, 12) - enum class EState : ui8 { + enum class EState : ui8 { YQL_EXPR_NODE_STATE_MAP(ENUM_VALUE_GEN) }; @@ -1492,7 +1492,7 @@ public: void Ref() { ENSURE_NOT_DELETED ENSURE_NOT_FROZEN - Y_ENSURE(RefCount_ < Max<ui32>()); + Y_ENSURE(RefCount_ < Max<ui32>()); ++RefCount_; } @@ -1514,13 +1514,13 @@ public: return ExprFlags_ & TExprFlags::Dead; } - TPositionHandle Pos() const { + TPositionHandle Pos() const { ENSURE_NOT_DELETED return Position_; } - TPosition Pos(const TExprContext& ctx) const; - + TPosition Pos(const TExprContext& ctx) const; + EType Type() const { ENSURE_NOT_DELETED return (EType)Type_; @@ -1621,9 +1621,9 @@ public: visitor(*child); } - TStringBuf Content() const { + TStringBuf Content() const { ENSURE_NOT_DELETED - return ContentUnchecked(); + return ContentUnchecked(); } ui32 Flags() const { @@ -1697,15 +1697,15 @@ public: State = EState::ConstrComplete; } - static TPtr NewAtom(ui64 uniqueId, TPositionHandle pos, const TStringBuf& content, ui32 flags) { + static TPtr NewAtom(ui64 uniqueId, TPositionHandle pos, const TStringBuf& content, ui32 flags) { return Make(pos, Atom, {}, content, flags, uniqueId); } - static TPtr NewArgument(ui64 uniqueId, TPositionHandle pos, const TStringBuf& name) { + static TPtr NewArgument(ui64 uniqueId, TPositionHandle pos, const TStringBuf& name) { return Make(pos, Argument, {}, name, 0, uniqueId); } - static TPtr NewArguments(ui64 uniqueId, TPositionHandle pos, TListType&& argNodes) { + static TPtr NewArguments(ui64 uniqueId, TPositionHandle pos, TListType&& argNodes) { return Make(pos, Arguments, std::move(argNodes), ZeroString, 0, uniqueId); } @@ -1720,7 +1720,7 @@ public: return NewLambda(uniqueId, pos, std::move(lambda)); } - static TPtr NewLambda(ui64 uniqueId, TPositionHandle pos, TPtr&& args, TPtr&& body) { + static TPtr NewLambda(ui64 uniqueId, TPositionHandle pos, TPtr&& args, TPtr&& body) { TListType children(body ? 2 : 1); children.front() = std::move(args); if (body) { @@ -1730,24 +1730,24 @@ public: return NewLambda(uniqueId, pos, std::move(children)); } - static TPtr NewWorld(ui64 uniqueId, TPositionHandle pos) { + static TPtr NewWorld(ui64 uniqueId, TPositionHandle pos) { return Make(pos, World, {}, {}, 0, uniqueId); } - static TPtr NewList(ui64 uniqueId, TPositionHandle pos, TListType&& children) { + static TPtr NewList(ui64 uniqueId, TPositionHandle pos, TListType&& children) { return Make(pos, List, std::move(children), ZeroString, 0, uniqueId); } - static TPtr NewCallable(ui64 uniqueId, TPositionHandle pos, const TStringBuf& name, TListType&& children) { + static TPtr NewCallable(ui64 uniqueId, TPositionHandle pos, const TStringBuf& name, TListType&& children) { return Make(pos, Callable, std::move(children), name, 0, uniqueId); } TPtr Clone(ui64 newUniqueId) const { ENSURE_NOT_DELETED - return Make(Position_, (EType)Type_, TListType(Children_), Content(), Flags_, newUniqueId); + return Make(Position_, (EType)Type_, TListType(Children_), Content(), Flags_, newUniqueId); } - static TPtr NewNode(TPositionHandle position, EType type, TListType&& children, const TStringBuf& content, ui32 flags, ui64 uniqueId) { + static TPtr NewNode(TPositionHandle position, EType type, TListType&& children, const TStringBuf& content, ui32 flags, ui64 uniqueId) { return Make(position, type, std::move(children), content, flags, uniqueId); } @@ -1758,7 +1758,7 @@ public: TPtr ChangeChildren(ui64 newUniqueId, TListType&& children) const { ENSURE_NOT_DELETED - return Make(Position_, (EType)Type_, std::move(children), Content(), Flags_, newUniqueId); + return Make(Position_, (EType)Type_, std::move(children), Content(), Flags_, newUniqueId); } TPtr ChangeChild(ui64 newUniqueId, ui32 index, TPtr&& child) const { @@ -1766,7 +1766,7 @@ public: Y_ENSURE(index < Children_.size(), "index out of range"); TListType newChildren(Children_); newChildren[index] = std::move(child); - return Make(Position_, (EType)Type_, std::move(newChildren), Content(), Flags_, newUniqueId); + return Make(Position_, (EType)Type_, std::move(newChildren), Content(), Flags_, newUniqueId); } void SetTypeAnn(const TTypeAnnotationNode* typeAnn) { @@ -1778,34 +1778,34 @@ public: return TypeAnnotation_; } - EState GetState() const { - return State; - } - - void SetState(EState state) { - State = state; - } - - ui32 GetArgIndex() const { - YQL_ENSURE(Type() == EType::Argument); - return ArgIndex; - } - - void SetArgIndex(ui32 argIndex) { - YQL_ENSURE(Type() == EType::Argument); - YQL_ENSURE(argIndex <= Max<ui16>()); - ArgIndex = (ui16)argIndex; - } - - ui64 GetHash() const { + EState GetState() const { + return State; + } + + void SetState(EState state) { + State = state; + } + + ui32 GetArgIndex() const { + YQL_ENSURE(Type() == EType::Argument); + return ArgIndex; + } + + void SetArgIndex(ui32 argIndex) { + YQL_ENSURE(Type() == EType::Argument); + YQL_ENSURE(argIndex <= Max<ui16>()); + ArgIndex = (ui16)argIndex; + } + + ui64 GetHash() const { Y_VERIFY_DEBUG(HashAbove == HashBelow); return HashAbove; - } - - void SetHash(ui64 hash) { + } + + void SetHash(ui64 hash) { HashAbove = HashBelow = hash; - } - + } + ui64 GetHashAbove() const { return HashAbove; } @@ -1822,42 +1822,42 @@ public: HashBelow = hash; } - ui64 GetBloom() const { - return Bloom; - } - - void SetBloom(ui64 bloom) { - Bloom = bloom; - } - + ui64 GetBloom() const { + return Bloom; + } + + void SetBloom(ui64 bloom) { + Bloom = bloom; + } + // return pair of outer and inner lambda. std::optional<std::pair<const TExprNode*, const TExprNode*>> GetDependencyScope() const { if (HasLambdaScope) { return std::make_pair(OuterLambda, InnerLambda); - } + } return std::nullopt; - } - + } + void SetDependencyScope(const TExprNode* outerLambda, const TExprNode* innerLambda) { Y_VERIFY_DEBUG(outerLambda == innerLambda || outerLambda->GetLambdaLevel() < innerLambda->GetLambdaLevel(), "Wrong scope of closures."); HasLambdaScope = 1; - OuterLambda = outerLambda; + OuterLambda = outerLambda; InnerLambda = innerLambda; - } - + } + ui16 GetLambdaLevel() const { return LambdaLevel; } void SetLambdaLevel(ui16 lambdaLevel) { LambdaLevel = lambdaLevel; } - - bool IsUsedInDependsOn() const { - YQL_ENSURE(Type() == EType::Argument); - return UsedInDependsOn; - } - - void SetUsedInDependsOn() { - YQL_ENSURE(Type() == EType::Argument); - UsedInDependsOn = 1; - } - + + bool IsUsedInDependsOn() const { + YQL_ENSURE(Type() == EType::Argument); + return UsedInDependsOn; + } + + void SetUsedInDependsOn() { + YQL_ENSURE(Type() == EType::Argument); + UsedInDependsOn = 1; + } + void SetUnorderedChildren() { YQL_ENSURE(Type() == EType::List || Type() == EType::Callable); UnordChildren = 1; @@ -1870,35 +1870,35 @@ public: ~TExprNode() { Y_VERIFY(Dead(), "Node (id: %lu, type: %s, content: '%s') not dead on destruction.", - UniqueId_, ToString(Type_).data(), TString(ContentUnchecked()).data()); + UniqueId_, ToString(Type_).data(), TString(ContentUnchecked()).data()); Y_VERIFY(!UseCount(), "Node (id: %lu, type: %s, content: '%s') has non-zero use count on destruction.", - UniqueId_, ToString(Type_).data(), TString(ContentUnchecked()).data()); + UniqueId_, ToString(Type_).data(), TString(ContentUnchecked()).data()); } - + private: - static TPtr Make(TPositionHandle position, EType type, TListType&& children, const TStringBuf& content, ui32 flags, ui64 uniqueId) { - Y_ENSURE(flags <= TNodeFlags::FlagsMask); + static TPtr Make(TPositionHandle position, EType type, TListType&& children, const TStringBuf& content, ui32 flags, ui64 uniqueId) { + Y_ENSURE(flags <= TNodeFlags::FlagsMask); Y_ENSURE(children.size() <= Max<ui32>()); - Y_ENSURE(content.size() <= Max<ui32>()); - for (size_t i = 0; i < children.size(); ++i) { - Y_ENSURE(children[i], "Unable to create node " << content << ": " << i << "th child is null"); - } - return TPtr(new TExprNode(position, type, std::move(children), content.data(), ui32(content.size()), flags, uniqueId)); - } - - TExprNode(TPositionHandle position, EType type, TListType&& children, - const char* content, ui32 contentSize, ui32 flags, ui64 uniqueId) - : Children_(std::move(children)) - , Content_(content) - , UniqueId_(uniqueId) - , Position_(position) - , ContentSize(contentSize) - , Type_(type) + Y_ENSURE(content.size() <= Max<ui32>()); + for (size_t i = 0; i < children.size(); ++i) { + Y_ENSURE(children[i], "Unable to create node " << content << ": " << i << "th child is null"); + } + return TPtr(new TExprNode(position, type, std::move(children), content.data(), ui32(content.size()), flags, uniqueId)); + } + + TExprNode(TPositionHandle position, EType type, TListType&& children, + const char* content, ui32 contentSize, ui32 flags, ui64 uniqueId) + : Children_(std::move(children)) + , Content_(content) + , UniqueId_(uniqueId) + , Position_(position) + , ContentSize(contentSize) + , Type_(type) , Flags_(flags) , ExprFlags_(TExprFlags::Default) - , State(EState::Initial) + , State(EState::Initial) , HasLambdaScope(0) - , UsedInDependsOn(0) + , UsedInDependsOn(0) , UnordChildren(0) , ShallBeDisclosed(0) {} @@ -1923,51 +1923,51 @@ private: void MarkDead() { ExprFlags_ |= TExprFlags::Dead; } - - TStringBuf ContentUnchecked() const { - return TStringBuf(Content_, ContentSize); - } - - TListType Children_; + + TStringBuf ContentUnchecked() const { + return TStringBuf(Content_, ContentSize); + } + + TListType Children_; TConstraintSet Constraints_; - - const char* Content_ = nullptr; - - const TExprNode* OuterLambda = nullptr; + + const char* Content_ = nullptr; + + const TExprNode* OuterLambda = nullptr; const TExprNode* InnerLambda = nullptr; - - TPtr Result; - + + TPtr Result; + ui64 HashAbove = 0ULL; ui64 HashBelow = 0ULL; - ui64 Bloom = 0ULL; - - const ui64 UniqueId_; - const TTypeAnnotationNode* TypeAnnotation_ = nullptr; - - const TPositionHandle Position_; - ui32 RefCount_ = 0U; - const ui32 ContentSize; - - ui16 ArgIndex = ui16(-1); - ui16 LambdaLevel = 0; // filled together with OuterLambda - ui16 IntermediateHashesCount = 0; - - static_assert(TypeMask <= 7, "EType wont fit in 3 bits, increase Type_ bitfield size"); - static_assert(TNodeFlags::FlagsMask <= 7, "TNodeFlags wont fit in 3 bits, increase Flags_ bitfield size"); - static_assert(TExprFlags::FlagsMask <= 3, "TExprFlags wont fit in 2 bits, increase ExprFlags_ bitfield size"); - static_assert(int(EState::Last) <= 16, "EState wont fit in 4 bits, increase State bitfield size"); - struct { - ui8 Type_ : 3; - ui8 Flags_ : 3; - ui8 ExprFlags_ : 2; - - EState State : 4; + ui64 Bloom = 0ULL; + + const ui64 UniqueId_; + const TTypeAnnotationNode* TypeAnnotation_ = nullptr; + + const TPositionHandle Position_; + ui32 RefCount_ = 0U; + const ui32 ContentSize; + + ui16 ArgIndex = ui16(-1); + ui16 LambdaLevel = 0; // filled together with OuterLambda + ui16 IntermediateHashesCount = 0; + + static_assert(TypeMask <= 7, "EType wont fit in 3 bits, increase Type_ bitfield size"); + static_assert(TNodeFlags::FlagsMask <= 7, "TNodeFlags wont fit in 3 bits, increase Flags_ bitfield size"); + static_assert(TExprFlags::FlagsMask <= 3, "TExprFlags wont fit in 2 bits, increase ExprFlags_ bitfield size"); + static_assert(int(EState::Last) <= 16, "EState wont fit in 4 bits, increase State bitfield size"); + struct { + ui8 Type_ : 3; + ui8 Flags_ : 3; + ui8 ExprFlags_ : 2; + + EState State : 4; ui8 HasLambdaScope : 1; - ui8 UsedInDependsOn : 1; + ui8 UsedInDependsOn : 1; ui8 UnordChildren : 1; ui8 ShallBeDisclosed: 1; - }; + }; }; class TExportTable { @@ -2253,8 +2253,8 @@ struct TExprContext : private TNonCopyable { ui64 NodeAllocationCounter = 0; ui64 NodesAllocationLimit = 3000000; ui64 StringsAllocationLimit = 100000000; - ui64 RepeatTransformLimit = 1000000; - ui64 RepeatTransformCounter = 0; + ui64 RepeatTransformLimit = 1000000; + ui64 RepeatTransformCounter = 0; TGcNodeConfig GcConfig; @@ -2289,10 +2289,10 @@ struct TExprContext : private TNonCopyable { return newBuf; } - TPositionHandle AppendPosition(const TPosition& pos); - TPosition GetPosition(TPositionHandle handle) const; - - TExprNodeBuilder Builder(TPositionHandle pos) { + TPositionHandle AppendPosition(const TPosition& pos); + TPosition GetPosition(TPositionHandle handle) const; + + TExprNodeBuilder Builder(TPositionHandle pos) { return TExprNodeBuilder(pos, *this); } @@ -2328,21 +2328,21 @@ struct TExprContext : private TNonCopyable { template<bool KeepTypeAnns = false> TExprNode::TListType ReplaceNodes(TExprNode::TListType&& start, const TNodeOnNodeOwnedMap& replaces); - TExprNode::TPtr NewAtom(TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent) { + TExprNode::TPtr NewAtom(TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent) { ++NodeAllocationCounter; const auto node = TExprNode::NewAtom(AllocateNextUniqueId(), pos, AppendString(content), flags); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewArgument(TPositionHandle pos, const TStringBuf& name) { + TExprNode::TPtr NewArgument(TPositionHandle pos, const TStringBuf& name) { ++NodeAllocationCounter; const auto node = TExprNode::NewArgument(AllocateNextUniqueId(), pos, AppendString(name)); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewArguments(TPositionHandle pos, TExprNode::TListType&& argNodes) { + TExprNode::TPtr NewArguments(TPositionHandle pos, TExprNode::TListType&& argNodes) { ++NodeAllocationCounter; const auto node = TExprNode::NewArguments(AllocateNextUniqueId(), pos, std::move(argNodes)); ExprNodes.emplace_back(node.Get()); @@ -2363,46 +2363,46 @@ struct TExprContext : private TNonCopyable { return node; } - TExprNode::TPtr NewLambda(TPositionHandle pos, TExprNode::TPtr&& args, TExprNode::TPtr&& body) { + TExprNode::TPtr NewLambda(TPositionHandle pos, TExprNode::TPtr&& args, TExprNode::TPtr&& body) { ++NodeAllocationCounter; const auto node = TExprNode::NewLambda(AllocateNextUniqueId(), pos, std::move(args), std::move(body)); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewWorld(TPositionHandle pos) { + TExprNode::TPtr NewWorld(TPositionHandle pos) { ++NodeAllocationCounter; const auto node = TExprNode::NewWorld(AllocateNextUniqueId(), pos); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewList(TPositionHandle pos, TExprNode::TListType&& children) { + TExprNode::TPtr NewList(TPositionHandle pos, TExprNode::TListType&& children) { ++NodeAllocationCounter; const auto node = TExprNode::NewList(AllocateNextUniqueId(), pos, std::move(children)); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewCallable(TPositionHandle pos, const TStringBuf& name, TExprNode::TListType&& children) { + TExprNode::TPtr NewCallable(TPositionHandle pos, const TStringBuf& name, TExprNode::TListType&& children) { ++NodeAllocationCounter; const auto node = TExprNode::NewCallable(AllocateNextUniqueId(), pos, AppendString(name), std::move(children)); ExprNodes.emplace_back(node.Get()); return node; } - TExprNode::TPtr NewAtom(TPosition pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent) { - return NewAtom(AppendPosition(pos), content, flags); - } - - TExprNode::TPtr NewArgument(TPosition pos, const TStringBuf& name) { - return NewArgument(AppendPosition(pos), name); - } - - TExprNode::TPtr NewArguments(TPosition pos, TExprNode::TListType&& argNodes) { - return NewArguments(AppendPosition(pos), std::move(argNodes)); - } - + TExprNode::TPtr NewAtom(TPosition pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent) { + return NewAtom(AppendPosition(pos), content, flags); + } + + TExprNode::TPtr NewArgument(TPosition pos, const TStringBuf& name) { + return NewArgument(AppendPosition(pos), name); + } + + TExprNode::TPtr NewArguments(TPosition pos, TExprNode::TListType&& argNodes) { + return NewArguments(AppendPosition(pos), std::move(argNodes)); + } + TExprNode::TPtr NewLambda(TPosition pos, TExprNode::TListType&& lambda) { return NewLambda(AppendPosition(pos), std::move(lambda)); } @@ -2411,22 +2411,22 @@ struct TExprContext : private TNonCopyable { return NewLambda(AppendPosition(pos), std::move(args), std::move(body)); } - TExprNode::TPtr NewLambda(TPosition pos, TExprNode::TPtr&& args, TExprNode::TPtr&& body) { - return NewLambda(AppendPosition(pos), std::move(args), std::move(body)); - } - - TExprNode::TPtr NewWorld(TPosition pos) { - return NewWorld(AppendPosition(pos)); - } - - TExprNode::TPtr NewList(TPosition pos, TExprNode::TListType&& children) { - return NewList(AppendPosition(pos), std::move(children)); - } - - TExprNode::TPtr NewCallable(TPosition pos, const TStringBuf& name, TExprNode::TListType&& children) { - return NewCallable(AppendPosition(pos), name, std::move(children)); - } - + TExprNode::TPtr NewLambda(TPosition pos, TExprNode::TPtr&& args, TExprNode::TPtr&& body) { + return NewLambda(AppendPosition(pos), std::move(args), std::move(body)); + } + + TExprNode::TPtr NewWorld(TPosition pos) { + return NewWorld(AppendPosition(pos)); + } + + TExprNode::TPtr NewList(TPosition pos, TExprNode::TListType&& children) { + return NewList(AppendPosition(pos), std::move(children)); + } + + TExprNode::TPtr NewCallable(TPosition pos, const TStringBuf& name, TExprNode::TListType&& children) { + return NewCallable(AppendPosition(pos), name, std::move(children)); + } + TExprNode::TPtr WrapByCallableIf(bool condition, const TStringBuf& callable, TExprNode::TPtr&& node); template <typename T, typename... Args> @@ -2449,21 +2449,21 @@ struct TExprContext : private TNonCopyable { void UnFreeze(); void Reset(); - + template <class TConstraint> bool IsConstraintEnabled() const { return DisabledConstraints.find(TConstraint::Name()) == DisabledConstraints.end(); } -private: - using TPositionHandleEqualPred = std::function<bool(TPositionHandle, TPositionHandle)>; - using TPositionHandleHasher = std::function<size_t(TPositionHandle)>; - - bool IsEqual(TPositionHandle a, TPositionHandle b) const; - size_t GetHash(TPositionHandle p) const; - - std::unordered_set<TPositionHandle, TPositionHandleHasher, TPositionHandleEqualPred> PositionSet; - std::deque<TPosition> Positions; +private: + using TPositionHandleEqualPred = std::function<bool(TPositionHandle, TPositionHandle)>; + using TPositionHandleHasher = std::function<size_t(TPositionHandle)>; + + bool IsEqual(TPositionHandle a, TPositionHandle b) const; + size_t GetHash(TPositionHandle p) const; + + std::unordered_set<TPositionHandle, TPositionHandleHasher, TPositionHandleEqualPred> PositionSet; + std::deque<TPosition> Positions; }; template <typename T, typename... Args> @@ -2513,14 +2513,14 @@ public: TNodeException(); explicit TNodeException(const TExprNode& node); explicit TNodeException(const TExprNode* node); - explicit TNodeException(const TPositionHandle& pos); + explicit TNodeException(const TPositionHandle& pos); - inline const TPositionHandle& Pos() const { + inline const TPositionHandle& Pos() const { return Pos_; } private: - const TPositionHandle Pos_; + const TPositionHandle Pos_; }; bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx, @@ -2530,7 +2530,7 @@ bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx IModuleResolver* resolver, ui32 annotationFlags, ui16 syntaxVersion = 0); struct TLibraryCohesion { - TExportTable Exports; + TExportTable Exports; TNodeMap<std::pair<TString, TString>> Imports; }; diff --git a/ydb/library/yql/ast/yql_expr_builder.cpp b/ydb/library/yql/ast/yql_expr_builder.cpp index 2acaab11a6..4b22308b99 100644 --- a/ydb/library/yql/ast/yql_expr_builder.cpp +++ b/ydb/library/yql/ast/yql_expr_builder.cpp @@ -3,7 +3,7 @@ namespace NYql { -TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx) +TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx) : Ctx(ctx) , Parent(nullptr) , ParentReplacer(nullptr) @@ -12,7 +12,7 @@ TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx) , CurrentNode(nullptr) {} -TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx, ExtArgsFuncType extArgsFunc) +TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx, ExtArgsFuncType extArgsFunc) : Ctx(ctx) , Parent(nullptr) , ParentReplacer(nullptr) @@ -22,7 +22,7 @@ TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx, ExtAr , ExtArgsFunc(extArgsFunc) {} -TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent, const TExprNode::TPtr& container) +TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent, const TExprNode::TPtr& container) : Ctx(parent->Ctx) , Parent(parent) , ParentReplacer(nullptr) @@ -35,7 +35,7 @@ TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent } } -TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeReplaceBuilder* parentReplacer) +TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeReplaceBuilder* parentReplacer) : Ctx(parentReplacer->Owner->Ctx) , Parent(nullptr) , ParentReplacer(parentReplacer) @@ -77,7 +77,7 @@ TExprNodeReplaceBuilder& TExprNodeBuilder::Done() { return *ParentReplacer; } -TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, TPositionHandle pos, const TStringBuf& content, ui32 flags) { +TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, TPositionHandle pos, const TStringBuf& content, ui32 flags) { Y_ENSURE(Container && !Container->IsLambda(), "Container expected"); Y_ENSURE(Container->ChildrenSize() == index, "Container position mismatch, expected: " << Container->ChildrenSize() << @@ -87,7 +87,7 @@ TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, TPositionHandle pos, const return *this; } -TExprNodeBuilder& TExprNodeBuilder::Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags) { +TExprNodeBuilder& TExprNodeBuilder::Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags) { Y_ENSURE(!Container || Container->IsLambda(), "No container expected"); Y_ENSURE(!CurrentNode, "Node is already build"); CurrentNode = Ctx.NewAtom(pos, content, flags); @@ -102,7 +102,7 @@ TExprNodeBuilder& TExprNodeBuilder::Atom(const TStringBuf& content, ui32 flags) return Atom(Pos, content, flags); } -TExprNodeBuilder TExprNodeBuilder::List(ui32 index, TPositionHandle pos) { +TExprNodeBuilder TExprNodeBuilder::List(ui32 index, TPositionHandle pos) { Y_ENSURE(Container, "Container expected"); Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << @@ -112,7 +112,7 @@ TExprNodeBuilder TExprNodeBuilder::List(ui32 index, TPositionHandle pos) { return TExprNodeBuilder(pos, this, child); } -TExprNodeBuilder TExprNodeBuilder::List(TPositionHandle pos) { +TExprNodeBuilder TExprNodeBuilder::List(TPositionHandle pos) { Y_ENSURE(!Container || Container->Type() == TExprNode::Lambda, "No container expected"); Y_ENSURE(!CurrentNode, "Node is already build"); CurrentNode = Ctx.NewList(pos, {}); @@ -168,7 +168,7 @@ TExprNodeBuilder& TExprNodeBuilder::Set(const TExprNode::TPtr& body) { return *this; } -TExprNodeBuilder TExprNodeBuilder::Callable(ui32 index, TPositionHandle pos, const TStringBuf& content) { +TExprNodeBuilder TExprNodeBuilder::Callable(ui32 index, TPositionHandle pos, const TStringBuf& content) { Y_ENSURE(Container, "Container expected"); Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << @@ -178,7 +178,7 @@ TExprNodeBuilder TExprNodeBuilder::Callable(ui32 index, TPositionHandle pos, con return TExprNodeBuilder(pos, this, child); } -TExprNodeBuilder TExprNodeBuilder::Callable(TPositionHandle pos, const TStringBuf& content) { +TExprNodeBuilder TExprNodeBuilder::Callable(TPositionHandle pos, const TStringBuf& content) { Y_ENSURE(!Container || Container->Type() == TExprNode::Lambda, "No container expected"); Y_ENSURE(!CurrentNode, "Node is already build"); CurrentNode = Ctx.NewCallable(pos, content, {}); @@ -193,7 +193,7 @@ TExprNodeBuilder TExprNodeBuilder::Callable(const TStringBuf& content) { return Callable(Pos, content); } -TExprNodeBuilder& TExprNodeBuilder::World(ui32 index, TPositionHandle pos) { +TExprNodeBuilder& TExprNodeBuilder::World(ui32 index, TPositionHandle pos) { Y_ENSURE(Container, "Container expected"); Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << @@ -203,7 +203,7 @@ TExprNodeBuilder& TExprNodeBuilder::World(ui32 index, TPositionHandle pos) { return *this; } -TExprNodeBuilder& TExprNodeBuilder::World(TPositionHandle pos) { +TExprNodeBuilder& TExprNodeBuilder::World(TPositionHandle pos) { Y_ENSURE(!Container, "No container expected"); Y_ENSURE(!CurrentNode, "Node is already build"); CurrentNode = Ctx.NewWorld(pos); @@ -218,7 +218,7 @@ TExprNodeBuilder& TExprNodeBuilder::World() { return World(Pos); } -TExprNodeBuilder TExprNodeBuilder::Lambda(ui32 index, TPositionHandle pos) { +TExprNodeBuilder TExprNodeBuilder::Lambda(ui32 index, TPositionHandle pos) { Y_ENSURE(Container, "Container expected"); Y_ENSURE(Container->ChildrenSize() == index + (Container->IsLambda() ? 1U : 0U), "Container position mismatch, expected: " << Container->ChildrenSize() << @@ -228,7 +228,7 @@ TExprNodeBuilder TExprNodeBuilder::Lambda(ui32 index, TPositionHandle pos) { return TExprNodeBuilder(pos, this, child); } -TExprNodeBuilder TExprNodeBuilder::Lambda(TPositionHandle pos) { +TExprNodeBuilder TExprNodeBuilder::Lambda(TPositionHandle pos) { Y_ENSURE(!Container || Container->Type() == TExprNode::Lambda, "No container expected"); Y_ENSURE(!CurrentNode, "Node is already build"); CurrentNode = Ctx.NewLambda(pos, Ctx.NewArguments(pos, {}), nullptr); @@ -243,7 +243,7 @@ TExprNodeBuilder TExprNodeBuilder::Lambda() { return Lambda(Pos); } -TExprNodeBuilder& TExprNodeBuilder::Param(TPositionHandle pos, const TStringBuf& name) { +TExprNodeBuilder& TExprNodeBuilder::Param(TPositionHandle pos, const TStringBuf& name) { Y_ENSURE(!name.empty(), "Empty parameter name is not allowed"); Y_ENSURE(Container, "Container expected"); Y_ENSURE(Container->Type() == TExprNode::Lambda, "Container must be a lambda"); diff --git a/ydb/library/yql/ast/yql_expr_builder.h b/ydb/library/yql/ast/yql_expr_builder.h index 8bc8340371..18c63f2eea 100644 --- a/ydb/library/yql/ast/yql_expr_builder.h +++ b/ydb/library/yql/ast/yql_expr_builder.h @@ -2,7 +2,7 @@ #include "yql_ast.h" #include "yql_errors.h" -#include "yql_pos_handle.h" +#include "yql_pos_handle.h" #include <functional> @@ -20,21 +20,21 @@ friend class TExprNodeReplaceBuilder; public: typedef std::function<TExprNodePtr(const TStringBuf&)> ExtArgsFuncType; public: - TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx); - TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx, ExtArgsFuncType extArgsFunc); + TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx); + TExprNodeBuilder(TPositionHandle pos, TExprContext& ctx, ExtArgsFuncType extArgsFunc); TExprNodePtr Build(); TExprNodeBuilder& Seal(); TExprNodeReplaceBuilder& Done(); // Indexed version of methods must be used inside of Callable or List, otherwise // non-indexed version must be used (at root or as lambda body) - TExprNodeBuilder& Atom(ui32 index, TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); - TExprNodeBuilder& Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); + TExprNodeBuilder& Atom(ui32 index, TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); + TExprNodeBuilder& Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); TExprNodeBuilder& Atom(ui32 index, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); TExprNodeBuilder& Atom(const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent); - TExprNodeBuilder List(ui32 index, TPositionHandle pos); - TExprNodeBuilder List(TPositionHandle pos); + TExprNodeBuilder List(ui32 index, TPositionHandle pos); + TExprNodeBuilder List(TPositionHandle pos); TExprNodeBuilder List(ui32 index); TExprNodeBuilder List(); @@ -45,22 +45,22 @@ public: TExprNodeBuilder& Set(TExprNodePtr&& body); TExprNodeBuilder& Set(const TExprNodePtr& body); - TExprNodeBuilder Callable(ui32 index, TPositionHandle pos, const TStringBuf& content); - TExprNodeBuilder Callable(TPositionHandle pos, const TStringBuf& content); + TExprNodeBuilder Callable(ui32 index, TPositionHandle pos, const TStringBuf& content); + TExprNodeBuilder Callable(TPositionHandle pos, const TStringBuf& content); TExprNodeBuilder Callable(ui32 index, const TStringBuf& content); TExprNodeBuilder Callable(const TStringBuf& content); - TExprNodeBuilder& World(ui32 index, TPositionHandle pos); - TExprNodeBuilder& World(TPositionHandle pos); + TExprNodeBuilder& World(ui32 index, TPositionHandle pos); + TExprNodeBuilder& World(TPositionHandle pos); TExprNodeBuilder& World(ui32 index); TExprNodeBuilder& World(); - TExprNodeBuilder Lambda(ui32 index, TPositionHandle pos); - TExprNodeBuilder Lambda(TPositionHandle pos); + TExprNodeBuilder Lambda(ui32 index, TPositionHandle pos); + TExprNodeBuilder Lambda(TPositionHandle pos); TExprNodeBuilder Lambda(ui32 index); TExprNodeBuilder Lambda(); - TExprNodeBuilder& Param(TPositionHandle pos, const TStringBuf& name); + TExprNodeBuilder& Param(TPositionHandle pos, const TStringBuf& name); TExprNodeBuilder& Param(const TStringBuf& name); TExprNodeBuilder& Params(const TStringBuf& name, ui32 width); @@ -88,8 +88,8 @@ public: } private: - TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent, const TExprNodePtr& container); - TExprNodeBuilder(TPositionHandle pos, TExprNodeReplaceBuilder* parentReplacer); + TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent, const TExprNodePtr& container); + TExprNodeBuilder(TPositionHandle pos, TExprNodeReplaceBuilder* parentReplacer); TExprNodePtr FindArgument(const TStringBuf& name); private: @@ -97,7 +97,7 @@ private: TExprNodeBuilder* Parent; TExprNodeReplaceBuilder* ParentReplacer; TExprNodePtr Container; - TPositionHandle Pos; + TPositionHandle Pos; TExprNodePtr CurrentNode; ExtArgsFuncType ExtArgsFunc; }; diff --git a/ydb/library/yql/ast/yql_expr_builder_ut.cpp b/ydb/library/yql/ast/yql_expr_builder_ut.cpp index 30fb96ddbc..eb0f2a0bf1 100644 --- a/ydb/library/yql/ast/yql_expr_builder_ut.cpp +++ b/ydb/library/yql/ast/yql_expr_builder_ut.cpp @@ -6,20 +6,20 @@ namespace NYql { Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestEmpty) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Build(), yexception); } Y_UNIT_TEST(TestRootAtom) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()).Atom("ABC").Build(); + auto res = ctx.Builder(TPositionHandle()).Atom("ABC").Build(); UNIT_ASSERT_VALUES_EQUAL(res->Type(), TExprNode::Atom); UNIT_ASSERT_VALUES_EQUAL(res->Content(), "ABC"); } Y_UNIT_TEST(TestRootAtomTwice) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Atom("ABC") .Atom("ABC") .Build(), yexception); @@ -27,14 +27,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestRootEmptyList) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()).List().Seal().Build(); + auto res = ctx.Builder(TPositionHandle()).List().Seal().Build(); UNIT_ASSERT_VALUES_EQUAL(res->Type(), TExprNode::List); UNIT_ASSERT_VALUES_EQUAL(res->ChildrenSize(), 0); } Y_UNIT_TEST(TestRootEmptyListTwice) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List() .List() .Build(), yexception); @@ -42,7 +42,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestListWithAtoms) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .List() .Atom(0, "ABC") .Atom(1, "XYZ") @@ -59,7 +59,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestMismatchChildIndex) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List() .Atom(1, "") .Build(), yexception); @@ -67,9 +67,9 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestListWithAdd) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .List() - .Add(0, ctx.Builder(TPositionHandle()).Atom("ABC").Build()) + .Add(0, ctx.Builder(TPositionHandle()).Atom("ABC").Build()) .Atom(1, "XYZ") .Seal() .Build(); @@ -84,7 +84,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestNestedListWithAtoms) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .List() .List(0) .Atom(0, "ABC") @@ -108,20 +108,20 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongLevelBuild) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List() .Build(), yexception); } Y_UNIT_TEST(TestWrongLevelSeal) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Seal(), yexception); } Y_UNIT_TEST(TestCallableWithAtoms) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Callable("Func") .Atom(0, "ABC") .Atom(1, "XYZ") @@ -139,7 +139,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestNestedCallableWithAtoms) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Callable("Func1") .Callable(0, "Func2") .Atom(0, "ABC") @@ -165,13 +165,13 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestRootWorld) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()).World().Build(); + auto res = ctx.Builder(TPositionHandle()).World().Build(); UNIT_ASSERT_VALUES_EQUAL(res->Type(), TExprNode::World); } Y_UNIT_TEST(TestCallableWithWorld) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Callable("Func") .World(0) .Seal() @@ -185,14 +185,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestIncompleteRootLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Build(), yexception); } Y_UNIT_TEST(TestIncompleteInnerLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List() .Lambda() .Seal() @@ -201,7 +201,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestRootLambda) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()).Lambda().Atom("ABC").Seal().Build(); + auto res = ctx.Builder(TPositionHandle()).Lambda().Atom("ABC").Seal().Build(); UNIT_ASSERT_VALUES_EQUAL(res->Type(), TExprNode::Lambda); UNIT_ASSERT_VALUES_EQUAL(res->ChildrenSize(), 2); UNIT_ASSERT_VALUES_EQUAL(res->Head().Type(), TExprNode::Arguments); @@ -212,9 +212,9 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestRootLambdaWithBodyAsSet) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() - .Set(ctx.Builder(TPositionHandle()).Atom("ABC").Build()) + .Set(ctx.Builder(TPositionHandle()).Atom("ABC").Build()) .Seal() .Build(); @@ -228,7 +228,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestInnerLambdaWithParam) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .List() .Lambda(0) .Param("x") @@ -251,7 +251,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestDuplicateLambdaParamNames) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Param("x") @@ -262,14 +262,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestParamAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Param("aaa") .Build(), yexception); } Y_UNIT_TEST(TestParamInList) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List() .Param("aaa") .Seal() @@ -278,7 +278,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestParamInCallable) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Callable("Func") .Param("aaa") .Seal() @@ -287,7 +287,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestParamAfterLambdaBody) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Param("aaa") .Atom("ABC") @@ -298,14 +298,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestIndexedAtomAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Atom(0, "ABC") .Build(), yexception); } Y_UNIT_TEST(TestIndexedListAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .List(0) .Seal() .Build(), yexception); @@ -313,14 +313,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestIndexedWorldAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .World(0) .Build(), yexception); } Y_UNIT_TEST(TestIndexedCallableAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Callable(0, "Func") .Seal() .Build(), yexception); @@ -328,7 +328,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestIndexedLambdaAtRoot) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda(0) .Atom("ABC") .Seal() @@ -337,7 +337,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongIndexAtomAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Atom(1, "ABC") .Seal() @@ -346,7 +346,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongIndexListAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .List(1) .Seal() @@ -356,7 +356,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongIndexWorldAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .World(1) .Seal() @@ -365,7 +365,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongIndexCallableAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Callable(1, "Func") .Seal() @@ -375,7 +375,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongIndexLambdaAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Lambda(1) .Atom("ABC") @@ -386,7 +386,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestAddAtLambda) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()) .Lambda() .Add(1, ctx.Builder(TPositionHandle()).Atom("ABC").Build()) .Seal() @@ -395,7 +395,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestLambdaWithArgAsBody) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Param("y") @@ -416,7 +416,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestIndexedArgAsLambdaBody) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() .Param("x") .Arg(1, "x") .Seal() @@ -425,7 +425,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestWrongArgName) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() .Param("x") .Arg("y") .Seal() @@ -434,7 +434,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestLambdaWithArgInCallables) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Param("y") @@ -462,7 +462,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestNestedScopeInLambda) { TExprContext ctx; - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Param("y") @@ -506,7 +506,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestNonIndexedArg) { TExprContext ctx; - UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() + UNIT_ASSERT_EXCEPTION(ctx.Builder(TPositionHandle()).Lambda() .Param("x") .Callable("f") .Arg("x") @@ -517,14 +517,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestApplyLambdaArgAsRoot) { TExprContext ctx; - auto lambda = ctx.Builder(TPositionHandle()) + auto lambda = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Arg("x") .Seal() .Build(); - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("y") .Apply(lambda).With(0, "y").Seal() @@ -542,14 +542,14 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestApplyLambdaArgInContainer) { TExprContext ctx; - auto lambda = ctx.Builder(TPositionHandle()) + auto lambda = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Arg("x") .Seal() .Build(); - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("y") .List() @@ -571,7 +571,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestApplyPartialLambdaArgAsRoot) { TExprContext ctx; - auto lambda = ctx.Builder(TPositionHandle()) + auto lambda = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Callable("Func1") @@ -583,7 +583,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { .Seal() .Build(); - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("y") .ApplyPartial(lambda->HeadPtr(), lambda->Child(1)->HeadPtr()).With(0, "y").Seal() @@ -601,7 +601,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestApplyPartialLambdaArgInContainer) { TExprContext ctx; - auto lambda = ctx.Builder(TPositionHandle()) + auto lambda = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Callable("Func1") @@ -613,7 +613,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { .Seal() .Build(); - auto res = ctx.Builder(TPositionHandle()) + auto res = ctx.Builder(TPositionHandle()) .Lambda() .Param("y") .Callable("Func3") @@ -633,7 +633,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { Y_UNIT_TEST(TestApplyOuterArg) { TExprContext ctx; - auto ast = ctx.Builder(TPositionHandle()) + auto ast = ctx.Builder(TPositionHandle()) .Lambda() .Param("x") .Callable("Func1") @@ -648,7 +648,7 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { .Seal() .Build(); - auto res1 = ctx.Builder(TPositionHandle()) + auto res1 = ctx.Builder(TPositionHandle()) .Lambda() .Param("y") .Callable("Func3") @@ -666,11 +666,11 @@ Y_UNIT_TEST_SUITE(TExprBuilder) { UNIT_ASSERT_VALUES_EQUAL(res1->Head().Head().Content(), "y"); UNIT_ASSERT_EQUAL(res1->Child(1)->Head().Child(1), res1->Head().Child(0)); - auto atom = ctx.Builder(TPositionHandle()) + auto atom = ctx.Builder(TPositionHandle()) .Atom("const") .Build(); - auto res2 = ctx.Builder(TPositionHandle()) + auto res2 = ctx.Builder(TPositionHandle()) .Lambda() .Callable("Func3") .ApplyPartial(0, nullptr, ast->Child(1)->Child(1)->ChildPtr(1)) diff --git a/ydb/library/yql/ast/yql_expr_check_args_ut.cpp b/ydb/library/yql/ast/yql_expr_check_args_ut.cpp index 1a84799a5f..9f1a8af420 100644 --- a/ydb/library/yql/ast/yql_expr_check_args_ut.cpp +++ b/ydb/library/yql/ast/yql_expr_check_args_ut.cpp @@ -1,115 +1,115 @@ -#include "yql_expr.h" +#include "yql_expr.h" #include <library/cpp/testing/unittest/registar.h> - -namespace NYql { - -Y_UNIT_TEST_SUITE(TExprCheckArguments) { - Y_UNIT_TEST(TestDuplicateArgument) { - TExprContext ctx; - auto pos = TPositionHandle(); - auto arg0 = ctx.NewArgument(pos, "arg0"); - auto args = ctx.NewArguments(pos, { arg0 }); - auto body = ctx.Builder(pos) - .Callable("+") - .Add(0, arg0) - .Add(1, arg0) - .Seal() - .Build(); - - auto left = ctx.NewLambda(pos, std::move(args), std::move(body)); - - auto arg1 = ctx.NewArgument(pos, "arg0"); - args = ctx.NewArguments(pos, { arg0, arg1 }); - body = ctx.Builder(pos) - .Callable("+") - .Add(0, arg0) - .Add(1, arg1) - .Seal() - .Build(); - - auto right = ctx.NewLambda(pos, std::move(args), std::move(body)); - - auto root = ctx.Builder(pos) - .Callable("SomeTopLevelCallableWithTwoLambdas") - .Add(0, left) - .Add(1, right) - .Seal() - .Build(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "argument is duplicated, #[1]"); - } - - Y_UNIT_TEST(TestUnresolved) { - TExprContext ctx; - auto pos = TPositionHandle(); - - auto arg1 = ctx.NewArgument(pos, "arg1"); - auto arg0 = ctx.NewArgument(pos, "arg0"); - - auto innerLambdaBody = ctx.Builder(pos) - .Callable("+") - .Add(0, arg0) - .Add(1, arg1) - .Seal() - .Build(); - - auto innerLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { arg1 }), std::move(innerLambdaBody)); - - auto outerLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { arg0 }), TExprNode::TPtr(innerLambda)); - - auto root = ctx.Builder(pos) - .Callable("SomeTopLevelCallableWithTwoLambdasAndFreeArg") - .Add(0, outerLambda) - .Add(1, innerLambda) - .Seal() - .Build(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "detected unresolved arguments at top level: #[2]"); - - root = ctx.Builder(pos) - .Callable("SomeTopLevelCallableWithTwoLambdasAndFreeArg") - .Add(0, outerLambda) - .Add(1, innerLambda) - .Add(2, ctx.NewArgument(pos, "arg3")) - .Seal() - .Build(); - - UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "detected unresolved arguments at top level: #[2, 10]"); - } - - Y_UNIT_TEST(TestUnresolvedFreeArg) { - TExprContext ctx; - auto pos = TPositionHandle(); - auto arg = ctx.NewArgument(pos, "arg"); - UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*arg), yexception, "detected unresolved arguments at top level: #[1]"); - } - - Y_UNIT_TEST(TestOk) { - TExprContext ctx; - auto pos = TPositionHandle(); - - auto root = ctx.Builder(pos) - .Callable("TopLevelCallableWithTwoLambdas") - .Lambda(0) - .Param("one") - .Lambda() - .Param("two") - .Callable("+") - .Arg(0, "one") - .Arg(1, "two") - .Seal() - .Seal() - .Seal() - .Lambda(1) - .Param("three") - .Callable("Not") - .Arg(0, "three") - .Seal() - .Seal() - .Seal() - .Build(); - UNIT_ASSERT_NO_EXCEPTION(CheckArguments(*root)); - } -} - -} // namespace NYql + +namespace NYql { + +Y_UNIT_TEST_SUITE(TExprCheckArguments) { + Y_UNIT_TEST(TestDuplicateArgument) { + TExprContext ctx; + auto pos = TPositionHandle(); + auto arg0 = ctx.NewArgument(pos, "arg0"); + auto args = ctx.NewArguments(pos, { arg0 }); + auto body = ctx.Builder(pos) + .Callable("+") + .Add(0, arg0) + .Add(1, arg0) + .Seal() + .Build(); + + auto left = ctx.NewLambda(pos, std::move(args), std::move(body)); + + auto arg1 = ctx.NewArgument(pos, "arg0"); + args = ctx.NewArguments(pos, { arg0, arg1 }); + body = ctx.Builder(pos) + .Callable("+") + .Add(0, arg0) + .Add(1, arg1) + .Seal() + .Build(); + + auto right = ctx.NewLambda(pos, std::move(args), std::move(body)); + + auto root = ctx.Builder(pos) + .Callable("SomeTopLevelCallableWithTwoLambdas") + .Add(0, left) + .Add(1, right) + .Seal() + .Build(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "argument is duplicated, #[1]"); + } + + Y_UNIT_TEST(TestUnresolved) { + TExprContext ctx; + auto pos = TPositionHandle(); + + auto arg1 = ctx.NewArgument(pos, "arg1"); + auto arg0 = ctx.NewArgument(pos, "arg0"); + + auto innerLambdaBody = ctx.Builder(pos) + .Callable("+") + .Add(0, arg0) + .Add(1, arg1) + .Seal() + .Build(); + + auto innerLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { arg1 }), std::move(innerLambdaBody)); + + auto outerLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { arg0 }), TExprNode::TPtr(innerLambda)); + + auto root = ctx.Builder(pos) + .Callable("SomeTopLevelCallableWithTwoLambdasAndFreeArg") + .Add(0, outerLambda) + .Add(1, innerLambda) + .Seal() + .Build(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "detected unresolved arguments at top level: #[2]"); + + root = ctx.Builder(pos) + .Callable("SomeTopLevelCallableWithTwoLambdasAndFreeArg") + .Add(0, outerLambda) + .Add(1, innerLambda) + .Add(2, ctx.NewArgument(pos, "arg3")) + .Seal() + .Build(); + + UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*root), yexception, "detected unresolved arguments at top level: #[2, 10]"); + } + + Y_UNIT_TEST(TestUnresolvedFreeArg) { + TExprContext ctx; + auto pos = TPositionHandle(); + auto arg = ctx.NewArgument(pos, "arg"); + UNIT_ASSERT_EXCEPTION_CONTAINS(CheckArguments(*arg), yexception, "detected unresolved arguments at top level: #[1]"); + } + + Y_UNIT_TEST(TestOk) { + TExprContext ctx; + auto pos = TPositionHandle(); + + auto root = ctx.Builder(pos) + .Callable("TopLevelCallableWithTwoLambdas") + .Lambda(0) + .Param("one") + .Lambda() + .Param("two") + .Callable("+") + .Arg(0, "one") + .Arg(1, "two") + .Seal() + .Seal() + .Seal() + .Lambda(1) + .Param("three") + .Callable("Not") + .Arg(0, "three") + .Seal() + .Seal() + .Seal() + .Build(); + UNIT_ASSERT_NO_EXCEPTION(CheckArguments(*root)); + } +} + +} // namespace NYql diff --git a/ydb/library/yql/ast/yql_expr_ut.cpp b/ydb/library/yql/ast/yql_expr_ut.cpp index 811a6ba73b..06550f2507 100644 --- a/ydb/library/yql/ast/yql_expr_ut.cpp +++ b/ydb/library/yql/ast/yql_expr_ut.cpp @@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { exprCtx.IssueManager.GetIssues().PrintTo(Cout); UNIT_ASSERT(success); - UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), typeAnnotationIndex != Max<ui32>() ? TExprNode::EState::TypeComplete : TExprNode::EState::Initial); + UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), typeAnnotationIndex != Max<ui32>() ? TExprNode::EState::TypeComplete : TExprNode::EState::Initial); } static void CompileExprWithCheck(TAstNode& root, TLibraryCohesion& cohesion, TExprContext& exprCtx) { @@ -94,10 +94,10 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { TExprContext exprCtx; TLibraryCohesion cohesion; CompileExprWithCheck(*astRes.Root, cohesion, exprCtx); - auto& exports = cohesion.Exports.Symbols(exprCtx); - UNIT_ASSERT_VALUES_EQUAL(2U, exports.size()); - UNIT_ASSERT_VALUES_EQUAL("42", exports["ex"]->Content()); - UNIT_ASSERT_VALUES_EQUAL("Y", exports["X"]->Content()); + auto& exports = cohesion.Exports.Symbols(exprCtx); + UNIT_ASSERT_VALUES_EQUAL(2U, exports.size()); + UNIT_ASSERT_VALUES_EQUAL("42", exports["ex"]->Content()); + UNIT_ASSERT_VALUES_EQUAL("Y", exports["X"]->Content()); } Y_UNIT_TEST(TestEmptyLib) { @@ -110,7 +110,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { TExprContext exprCtx; TLibraryCohesion cohesion; CompileExprWithCheck(*astRes.Root, cohesion, exprCtx); - UNIT_ASSERT(cohesion.Exports.Symbols().empty()); + UNIT_ASSERT(cohesion.Exports.Symbols().empty()); UNIT_ASSERT(cohesion.Imports.empty()); } @@ -128,7 +128,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { UNIT_ASSERT_STRINGS_EQUAL(HexEncode(exprRoot->Content()), "0123456789ABCDEF"); UNIT_ASSERT(exprRoot->Flags() & TNodeFlags::ArbitraryContent); - auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); TAstNode* xValue = ast.Root->GetChild(0)->GetChild(1)->GetChild(1); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(TString(xValue->GetContent())), "0123456789ABCDEF"); UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::ArbitraryContent); @@ -148,7 +148,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { UNIT_ASSERT_STRINGS_EQUAL(HexEncode(exprRoot->Content()), "FEDCBA9876543210"); UNIT_ASSERT(exprRoot->Flags() & TNodeFlags::BinaryContent); - auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); TAstNode* xValue = ast.Root->GetChild(0)->GetChild(2)->GetChild(1); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(TString(xValue->GetContent())), "FEDCBA9876543210"); UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::BinaryContent); @@ -321,8 +321,8 @@ Y_UNIT_TEST_SUITE(TCompareExprTrees) { if (diffPositions) { UNIT_ASSERT(!CompareExprTrees(diffOne, diffTwo)); - UNIT_ASSERT_EQUAL(ctxOne.GetPosition(diffOne->Pos()), diffPositions->first); - UNIT_ASSERT_EQUAL(ctxTwo.GetPosition(diffTwo->Pos()), diffPositions->second); + UNIT_ASSERT_EQUAL(ctxOne.GetPosition(diffOne->Pos()), diffPositions->first); + UNIT_ASSERT_EQUAL(ctxTwo.GetPosition(diffTwo->Pos()), diffPositions->second); } else UNIT_ASSERT(CompareExprTrees(diffOne, diffTwo)); } @@ -896,7 +896,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) { UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)); UNIT_ASSERT(exprRoot); - const auto convRes = ConvertToAst(*exprRoot, exprCtx, 0, true); + const auto convRes = ConvertToAst(*exprRoot, exprCtx, 0, true); UNIT_ASSERT(convRes.IsOk()); TExprContext exprCtx2; diff --git a/ydb/library/yql/ast/yql_pos_handle.h b/ydb/library/yql/ast/yql_pos_handle.h index 1b8ea9304d..e00cf63a78 100644 --- a/ydb/library/yql/ast/yql_pos_handle.h +++ b/ydb/library/yql/ast/yql_pos_handle.h @@ -1,13 +1,13 @@ -#pragma once - -#include <util/system/types.h> - -namespace NYql { - -struct TPositionHandle { - friend struct TExprContext; -private: - ui32 Handle = 0; // 0 is guaranteed to represent default-constructed TPosition -}; - -} +#pragma once + +#include <util/system/types.h> + +namespace NYql { + +struct TPositionHandle { + friend struct TExprContext; +private: + ui32 Handle = 0; // 0 is guaranteed to represent default-constructed TPosition +}; + +} diff --git a/ydb/library/yql/core/common_opt/ya.make b/ydb/library/yql/core/common_opt/ya.make index 045c55181c..518c91c26a 100644 --- a/ydb/library/yql/core/common_opt/ya.make +++ b/ydb/library/yql/core/common_opt/ya.make @@ -12,7 +12,7 @@ SRCS( yql_co_finalizers.cpp yql_co_flow1.cpp yql_co_flow2.cpp - yql_co_last.cpp + yql_co_last.cpp yql_co_simple1.cpp yql_co_simple2.cpp yql_co_transformer.h diff --git a/ydb/library/yql/core/common_opt/yql_co.h b/ydb/library/yql/core/common_opt/yql_co.h index 3deac8dfb1..b8f2719037 100644 --- a/ydb/library/yql/core/common_opt/yql_co.h +++ b/ydb/library/yql/core/common_opt/yql_co.h @@ -17,20 +17,20 @@ struct TOptimizeContext { const auto it = ParentsMap->find(&node); YQL_ENSURE(it != ParentsMap->cend()); - auto& parents = it->second; - YQL_ENSURE(!parents.empty()); - if (parents.size() > 1) { + auto& parents = it->second; + YQL_ENSURE(!parents.empty()); + if (parents.size() > 1) { return nullptr; - } - - size_t usageCount = 0; + } + + size_t usageCount = 0; for (const auto& child : (*parents.cbegin())->ChildrenList()) { - if (child.Get() == &node && ++usageCount > 1) { + if (child.Get() == &node && ++usageCount > 1) { return nullptr; - } - } - - YQL_ENSURE(usageCount == 1); + } + } + + YQL_ENSURE(usageCount == 1); return *parents.cbegin(); } @@ -38,15 +38,15 @@ struct TOptimizeContext { return bool(GetParentIfSingle(node)); } - bool IsSingleUsage(const NNodes::TExprBase& node) const { - return IsSingleUsage(node.Ref()); + bool IsSingleUsage(const NNodes::TExprBase& node) const { + return IsSingleUsage(node.Ref()); + } + + bool HasParent(const TExprNode& node) const { + YQL_ENSURE(ParentsMap); + return ParentsMap->contains(&node); } - bool HasParent(const TExprNode& node) const { - YQL_ENSURE(ParentsMap); - return ParentsMap->contains(&node); - } - bool IsPersistentNode(const TExprNode& node) const { if (Types) { for (auto& source: Types->DataSources) { @@ -88,16 +88,16 @@ struct TCoCallableRules { FLOW_STEPS }; - // rules that don't make a flow fork - e.g. False || x -> x + // rules that don't make a flow fork - e.g. False || x -> x TCallableOptimizerMap SimpleCallables[SIMPLE_STEPS]; - // rules that make a flow fork - Join pushdown if Join has multiple usage + // rules that make a flow fork - Join pushdown if Join has multiple usage TCallableOptimizerMap FlowCallables[FLOW_STEPS]; TFinalizingOptimizerMap Finalizers; - // rules to be applied before execution - TCallableOptimizerMap FinalCallables; - + // rules to be applied before execution + TCallableOptimizerMap FinalCallables; + TCoCallableRules(); static const TCoCallableRules& Instance(); }; @@ -107,6 +107,6 @@ void RegisterCoSimpleCallables2(TCallableOptimizerMap& map); void RegisterCoFlowCallables1(TCallableOptimizerMap& map); void RegisterCoFlowCallables2(TCallableOptimizerMap& map); void RegisterCoFinalizers(TFinalizingOptimizerMap& map); -void RegisterCoFinalCallables(TCallableOptimizerMap& map); +void RegisterCoFinalCallables(TCallableOptimizerMap& map); } diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp index e7d2402859..215a1e9f69 100644 --- a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp @@ -513,18 +513,18 @@ TExprNode::TPtr ApplyExtractMembersToMapJoinCore(const TExprNode::TPtr& node, co } TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) { - YQL_ENSURE(node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})); - - auto input = node->ChildPtr(0); - + YQL_ENSURE(node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})); + + auto input = node->ChildPtr(0); + // window output = input fields + payload fields TSet<TStringBuf> outMembers; for (const auto& x : members->ChildrenList()) { outMembers.insert(x->Content()); } - auto inputStructType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - auto outputStructType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + auto inputStructType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + auto outputStructType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); TSet<TStringBuf> toDrop; for (const auto& out : outputStructType->GetItems()) { if (!outMembers.contains(out->GetName())) { @@ -532,106 +532,106 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, } } - TSet<TStringBuf> usedFields; + TSet<TStringBuf> usedFields; TSet<TStringBuf> payloadFields; - TExprNodeList newCalcs; - auto calcs = ExtractCalcsOverWindow(node, ctx); - bool dropped = false; - for (auto& calcNode : calcs) { - TCoCalcOverWindowTuple calc(calcNode); - - // all partition keys will be used - for (const auto& key : calc.Keys()) { - usedFields.insert(key.Value()); + TExprNodeList newCalcs; + auto calcs = ExtractCalcsOverWindow(node, ctx); + bool dropped = false; + for (auto& calcNode : calcs) { + TCoCalcOverWindowTuple calc(calcNode); + + // all partition keys will be used + for (const auto& key : calc.Keys()) { + usedFields.insert(key.Value()); } - auto processListType = [&](TExprBase typeNode) { - auto structType = typeNode.Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()-> - Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - for (const auto& item : structType->GetItems()) { - usedFields.insert(item->GetName()); - } - }; - - auto processSortTraits = [&](TExprBase node) { - if (auto maybeSortTraits = node.Maybe<TCoSortTraits>()) { - processListType(maybeSortTraits.Cast().ListType()); - } else { - YQL_ENSURE(node.Maybe<TCoVoid>()); - } - }; - - // all sort keys will be used - processSortTraits(calc.SortSpec()); - - // all session keys + session sort keys will be used - if (auto maybeSwt = calc.SessionSpec().Maybe<TCoSessionWindowTraits>()) { - processListType(maybeSwt.Cast().ListType()); - processSortTraits(maybeSwt.Cast().SortSpec()); - } else { - YQL_ENSURE(calc.SessionSpec().Maybe<TCoVoid>()); - } - - TExprNodeList newSessionColumns; - for (auto& sessionColumn : calc.SessionColumns().Ref().ChildrenList()) { - TStringBuf columnName = sessionColumn->Content(); - if (toDrop.contains(columnName)) { - dropped = true; - continue; - } - payloadFields.insert(columnName); - newSessionColumns.push_back(sessionColumn); - } - - TExprNodeList newFrames; - for (const auto& winOnRows : calc.Frames().Ref().ChildrenList()) { - YQL_ENSURE(winOnRows->IsCallable("WinOnRows")); - - TExprNodeList newFrameItems; - newFrameItems.push_back(winOnRows->ChildPtr(0)); - - for (ui32 i = 1; i < winOnRows->ChildrenSize(); ++i) { - auto field = winOnRows->Child(i)->Child(0)->Content(); - if (toDrop.contains(field)) { - dropped = true; - continue; - } - - payloadFields.insert(field); - newFrameItems.push_back(winOnRows->ChildPtr(i)); - auto payload = winOnRows->Child(i)->Child(1); - const TStructExprType* structType; - if (payload->IsCallable("WindowTraits")) { - structType = payload->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); - } - else if (payload->IsCallable({"Lead", "Lag", "Rank", "DenseRank"})) { - structType = payload->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>() - ->GetItemType()->Cast<TStructExprType>(); - } else { - continue; - } - - for (const auto& item : structType->GetItems()) { - usedFields.insert(item->GetName()); - } + auto processListType = [&](TExprBase typeNode) { + auto structType = typeNode.Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()-> + Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + for (const auto& item : structType->GetItems()) { + usedFields.insert(item->GetName()); } + }; + + auto processSortTraits = [&](TExprBase node) { + if (auto maybeSortTraits = node.Maybe<TCoSortTraits>()) { + processListType(maybeSortTraits.Cast().ListType()); + } else { + YQL_ENSURE(node.Maybe<TCoVoid>()); + } + }; + + // all sort keys will be used + processSortTraits(calc.SortSpec()); + + // all session keys + session sort keys will be used + if (auto maybeSwt = calc.SessionSpec().Maybe<TCoSessionWindowTraits>()) { + processListType(maybeSwt.Cast().ListType()); + processSortTraits(maybeSwt.Cast().SortSpec()); + } else { + YQL_ENSURE(calc.SessionSpec().Maybe<TCoVoid>()); + } - if (newFrameItems.size() == 1) { + TExprNodeList newSessionColumns; + for (auto& sessionColumn : calc.SessionColumns().Ref().ChildrenList()) { + TStringBuf columnName = sessionColumn->Content(); + if (toDrop.contains(columnName)) { + dropped = true; continue; } + payloadFields.insert(columnName); + newSessionColumns.push_back(sessionColumn); + } + + TExprNodeList newFrames; + for (const auto& winOnRows : calc.Frames().Ref().ChildrenList()) { + YQL_ENSURE(winOnRows->IsCallable("WinOnRows")); + + TExprNodeList newFrameItems; + newFrameItems.push_back(winOnRows->ChildPtr(0)); + + for (ui32 i = 1; i < winOnRows->ChildrenSize(); ++i) { + auto field = winOnRows->Child(i)->Child(0)->Content(); + if (toDrop.contains(field)) { + dropped = true; + continue; + } - newFrames.push_back(ctx.ChangeChildren(*winOnRows, std::move(newFrameItems))); + payloadFields.insert(field); + newFrameItems.push_back(winOnRows->ChildPtr(i)); + auto payload = winOnRows->Child(i)->Child(1); + const TStructExprType* structType; + if (payload->IsCallable("WindowTraits")) { + structType = payload->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); + } + else if (payload->IsCallable({"Lead", "Lag", "Rank", "DenseRank"})) { + structType = payload->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>() + ->GetItemType()->Cast<TStructExprType>(); + } else { + continue; + } + + for (const auto& item : structType->GetItems()) { + usedFields.insert(item->GetName()); + } + } + + if (newFrameItems.size() == 1) { + continue; + } + + newFrames.push_back(ctx.ChangeChildren(*winOnRows, std::move(newFrameItems))); } - newCalcs.emplace_back( - Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) - .Keys(calc.Keys()) - .SortSpec(calc.SortSpec()) - .Frames(ctx.NewList(calc.Frames().Pos(), std::move(newFrames))) - .SessionSpec(calc.SessionSpec()) - .SessionColumns(ctx.NewList(calc.SessionColumns().Pos(), std::move(newSessionColumns))) - .Done().Ptr() - ); + newCalcs.emplace_back( + Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) + .Keys(calc.Keys()) + .SortSpec(calc.SortSpec()) + .Frames(ctx.NewList(calc.Frames().Pos(), std::move(newFrames))) + .SessionSpec(calc.SessionSpec()) + .SessionColumns(ctx.NewList(calc.SessionColumns().Pos(), std::move(newSessionColumns))) + .Done().Ptr() + ); } // keep input fields @@ -641,29 +641,29 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, } } - if (usedFields.size() == inputStructType->GetSize() && !dropped) { + if (usedFields.size() == inputStructType->GetSize() && !dropped) { return {}; } TExprNode::TListType usedExprList; for (const auto& x : usedFields) { - usedExprList.push_back(ctx.NewAtom(node->Pos(), x)); + usedExprList.push_back(ctx.NewAtom(node->Pos(), x)); } - auto newInput = Build<TCoExtractMembers>(ctx, node->Pos()) - .Input(input) - .Members(ctx.NewList(node->Pos(), std::move(usedExprList))) + auto newInput = Build<TCoExtractMembers>(ctx, node->Pos()) + .Input(input) + .Members(ctx.NewList(node->Pos(), std::move(usedExprList))) .Done() .Ptr(); - auto calcOverWindow = Build<TCoCalcOverWindowGroup>(ctx, node->Pos()) - .Input(newInput) - .Calcs(ctx.NewList(node->Pos(), std::move(newCalcs))) - .Done().Ptr(); - + auto calcOverWindow = Build<TCoCalcOverWindowGroup>(ctx, node->Pos()) + .Input(newInput) + .Calcs(ctx.NewList(node->Pos(), std::move(newCalcs))) + .Done().Ptr(); + YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix; - return Build<TCoExtractMembers>(ctx, node->Pos()) - .Input(calcOverWindow) + return Build<TCoExtractMembers>(ctx, node->Pos()) + .Input(calcOverWindow) .Members(members) .Done() .Ptr(); @@ -676,19 +676,19 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons outMembers.insert(x->Content()); } - TMaybe<TStringBuf> sessionColumn; - const auto sessionSetting = GetSetting(aggr.Settings().Ref(), "session"); - if (sessionSetting) { - YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); - sessionColumn = sessionSetting->Child(1)->Child(0)->Content(); - } - + TMaybe<TStringBuf> sessionColumn; + const auto sessionSetting = GetSetting(aggr.Settings().Ref(), "session"); + if (sessionSetting) { + YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); + sessionColumn = sessionSetting->Child(1)->Child(0)->Content(); + } + TSet<TStringBuf> usedFields; - // all actual (non-session) keys will be used + // all actual (non-session) keys will be used for (const auto& key : aggr.Keys()) { - if (key.Value() != sessionColumn) { - usedFields.insert(key.Value()); - } + if (key.Value() != sessionColumn) { + usedFields.insert(key.Value()); + } } TExprNode::TListType newHandlers; @@ -742,19 +742,19 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons } } - if (sessionSetting) { - TCoSessionWindowTraits traits(sessionSetting->Child(1)->ChildPtr(1)); - - // TODO: same should be done for hopping - auto usedType = traits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()-> - GetItemType()->Cast<TStructExprType>(); - for (const auto& item : usedType->GetItems()) { - usedFields.insert(item->GetName()); - } - } - + if (sessionSetting) { + TCoSessionWindowTraits traits(sessionSetting->Child(1)->ChildPtr(1)); + + // TODO: same should be done for hopping + auto usedType = traits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()-> + GetItemType()->Cast<TStructExprType>(); + for (const auto& item : usedType->GetItems()) { + usedFields.insert(item->GetName()); + } + } + auto inputStructType = aggr.Input().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - YQL_ENSURE(usedFields.size() <= inputStructType->GetSize()); + YQL_ENSURE(usedFields.size() <= inputStructType->GetSize()); if (usedFields.size() == inputStructType->GetSize()) { return {}; } @@ -783,15 +783,15 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons .Ptr(); } -TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) { - TCoCollect collect(node); - YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix; - return Build<TCoCollect>(ctx, node->Pos()) - .Input<TCoExtractMembers>() - .Input(collect.Input()) - .Members(members) - .Build() - .Done().Ptr(); -} - +TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) { + TCoCollect collect(node); + YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix; + return Build<TCoCollect>(ctx, node->Pos()) + .Input<TCoExtractMembers>() + .Input(collect.Input()) + .Members(members) + .Build() + .Done().Ptr(); +} + } // NYql diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.h b/ydb/library/yql/core/common_opt/yql_co_extr_members.h index 33bcd77afa..8244636784 100644 --- a/ydb/library/yql/core/common_opt/yql_co_extr_members.h +++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.h @@ -23,7 +23,7 @@ TExprNode::TPtr ApplyExtractMembersToPartitionByKey(const TExprNode::TPtr& node, TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix); TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix); TExprNode::TPtr ApplyExtractMembersToChopper(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix); -TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix); +TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix); TExprNode::TPtr ApplyExtractMembersToMapJoinCore(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix); } // NYql diff --git a/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp b/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp index b64c553507..b77766bc6f 100644 --- a/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp @@ -169,10 +169,10 @@ void RegisterCoFinalizers(TFinalizingOptimizerMap& map) { ); }; - map[TCoCalcOverWindowGroup::CallableName()] = map[TCoCalcOverWindow::CallableName()] = - map[TCoCalcOverSessionWindow::CallableName()] = - [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) - { + map[TCoCalcOverWindowGroup::CallableName()] = map[TCoCalcOverWindow::CallableName()] = + map[TCoCalcOverSessionWindow::CallableName()] = + [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) + { SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx, [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) { return ApplyExtractMembersToCalcOverWindow(input, members, ctx, " with multi-usage"); @@ -195,14 +195,14 @@ void RegisterCoFinalizers(TFinalizingOptimizerMap& map) { } ); }; - - map[TCoCollect::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) { - SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx, - [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) { - return ApplyExtractMembersToCollect(input, members, ctx, " with multi-usage"); - } - ); - }; + + map[TCoCollect::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) { + SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx, + [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) { + return ApplyExtractMembersToCollect(input, members, ctx, " with multi-usage"); + } + ); + }; } } // NYql diff --git a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp index 3d6404d201..728838dc8e 100644 --- a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp @@ -517,7 +517,7 @@ TExprNode::TPtr FuseFlatMapOverByKey(const TExprNode& node, TExprContext& ctx) { ctx.Builder(node.Pos()) .Lambda() .Param("list") - .Callable(node.Content()) + .Callable(node.Content()) .Apply(0, node.Head().Tail()) .With(0, "list") .Seal() @@ -528,7 +528,7 @@ TExprNode::TPtr FuseFlatMapOverByKey(const TExprNode& node, TExprContext& ctx) { .Lambda() .Param("key") .Param("state") - .Callable(node.Content()) + .Callable(node.Content()) .Apply(0, node.Head().Tail()) .With(0, "key") .With(1, "state") @@ -1293,11 +1293,11 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct return node.Ptr(); } - if (GetSetting(settings.Ref(), "session")) { - // TODO: support - return node.Ptr(); - } - + if (GetSetting(settings.Ref(), "session")) { + // TODO: support + return node.Ptr(); + } + auto aggregatedColumn = aggregatedColumns.Item(0); const bool isDistinct = (aggregatedColumn.Ref().ChildrenSize() == 3); @@ -1549,7 +1549,7 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct } TExprNode::TPtr OptimizeReverse(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (optCtx.IsSingleUsage(node->Head()) && node->Head().IsCallable({"Sort", "AssumeSorted"})) { + if (optCtx.IsSingleUsage(node->Head()) && node->Head().IsCallable({"Sort", "AssumeSorted"})) { YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); auto asc = node->Head().ChildPtr(1); if (asc->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) { @@ -1588,7 +1588,7 @@ TExprNode::TPtr OptimizeReverse(const TExprNode::TPtr& node, TExprContext& ctx, } TExprNode::TPtr OptimizeLookup(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (optCtx.IsSingleUsage(node->Head()) && node->Head().IsCallable("ToIndexDict") && TMaybeNode<TCoIntegralCtor>(node->TailPtr())) { + if (optCtx.IsSingleUsage(node->Head()) && node->Head().IsCallable("ToIndexDict") && TMaybeNode<TCoIntegralCtor>(node->TailPtr())) { const auto& atom = node->Tail().Head(); if (atom.Content() == "0" && !(TNodeFlags::BinaryContent & atom.Flags())) { YQL_CLOG(DEBUG, Core) << node->Content() << " zero item over " << node->Head().Content(); @@ -1617,7 +1617,7 @@ constexpr std::initializer_list<std::string_view> FlowPriority = { }; TExprNode::TPtr OptimizeToFlow(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1706,7 +1706,7 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, if (node->Head().IsCallable({"PartitionByKey", "PartitionsByKeys"})) { return FuseFlatMapOverByKey<true>(*node, ctx); } - + if (node->Head().IsCallable("ForwardList")) { if (ETypeAnnotationKind::List == node->GetTypeAnn()->GetKind()) { YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); @@ -1771,11 +1771,11 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { map["FlatMap"] = std::bind(&OptimizeFlatMap<false>, _1, _2, _3); map["OrderedFlatMap"] = std::bind(&OptimizeFlatMap<true>, _1, _2, _3); - + map["Lookup"] = std::bind(&OptimizeLookup, _1, _2, _3); map["Skip"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { return node; } @@ -1800,7 +1800,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Take"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { return node; } /*TODO: Enable later. Providers is not ready right now. @@ -1838,7 +1838,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Length"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { return node; } @@ -1883,7 +1883,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["HasItems"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { return node; } @@ -1901,7 +1901,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Fold"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1929,7 +1929,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Fold1"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1957,7 +1957,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Condense"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1974,16 +1974,16 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { return PropagateMapToCondense(*node, ctx); } - if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { - YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); - return ctx.SwapWithHead(*node); - } - + if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { + YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); + return ctx.SwapWithHead(*node); + } + return node; }; map["Condense1"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2000,16 +2000,16 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { return PropagateMapToCondense1(*node, ctx); } - if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { - YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); - return ctx.SwapWithHead(*node); - } - + if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { + YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); + return ctx.SwapWithHead(*node); + } + return node; }; map["CombineByKey"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2048,7 +2048,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["Aggregate"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head()) && !optCtx.IsPersistentNode(node->Head())) { return node; } @@ -2065,7 +2065,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { map["Reverse"] = std::bind(&OptimizeReverse, _1, _2, _3); map["Visit"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2187,7 +2187,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2238,7 +2238,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { }; map["SkipNullMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2281,16 +2281,16 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { } } - if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { - YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); - return ctx.SwapWithHead(*node); - } - + if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { + YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); + return ctx.SwapWithHead(*node); + } + return node; }; map["Exists"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -2301,40 +2301,40 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) { return node; }; - + map["Sort"] = map["Top"] = map["TopSort"] = - map["TakeWhile"] = map["SkipWhile"] = map["TakeWhileInclusive"] = map["SkipWhileInclusive"] = - map["SkipNullElements"] = map["FilterNullMembers"] = map["FilterNullElements"] = map["ExtractMembers"] = + map["TakeWhile"] = map["SkipWhile"] = map["TakeWhileInclusive"] = map["SkipWhileInclusive"] = + map["SkipNullElements"] = map["FilterNullMembers"] = map["FilterNullElements"] = map["ExtractMembers"] = map["Chain1Map"] = map["Fold1Map"] = map["FoldMap"] = map["Map"] = map["OrderedMap"] = map["MultiMap"] = map["OrderedMultiMap"] = - [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) - { - if (!optCtx.IsSingleUsage(node->Head())) { - return node; - } - - if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { - YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); - return ctx.SwapWithHead(*node); - } - - return node; - }; - - - map["ForwardList"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (optCtx.HasParent(*node) && !optCtx.IsSingleUsage(*node)) { - YQL_ENSURE(optCtx.ParentsMap); - auto parentsIt = optCtx.ParentsMap->find(node.Get()); - YQL_ENSURE(parentsIt != optCtx.ParentsMap->cend()); - if (AnyOf(parentsIt->second, [](const TExprNode* parent) { return !parent->IsCallable({"Length", "HasItems"}); })) { - YQL_CLOG(DEBUG, Core) << "Collect list instead of " << node->Content(); - return ctx.RenameNode(*node, "Collect"); - } - } - - return node; - }; + [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) + { + if (!optCtx.IsSingleUsage(node->Head())) { + return node; + } + + if (node->Head().IsCallable({"ForwardList", "FromFlow"})) { + YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content(); + return ctx.SwapWithHead(*node); + } + + return node; + }; + + + map["ForwardList"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + if (optCtx.HasParent(*node) && !optCtx.IsSingleUsage(*node)) { + YQL_ENSURE(optCtx.ParentsMap); + auto parentsIt = optCtx.ParentsMap->find(node.Get()); + YQL_ENSURE(parentsIt != optCtx.ParentsMap->cend()); + if (AnyOf(parentsIt->second, [](const TExprNode* parent) { return !parent->IsCallable({"Length", "HasItems"}); })) { + YQL_CLOG(DEBUG, Core) << "Collect list instead of " << node->Content(); + return ctx.RenameNode(*node, "Collect"); + } + } + + return node; + }; } } diff --git a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp index a85ee29602..88f9140031 100644 --- a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp @@ -25,24 +25,24 @@ TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprCon return node.Ptr(); } - TMaybe<TStringBuf> sessionColumn; - const auto sessionSetting = GetSetting(node.Settings().Ref(), "session"); - if (sessionSetting) { - YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); - sessionColumn = sessionSetting->Child(1)->Child(0)->Content(); + TMaybe<TStringBuf> sessionColumn; + const auto sessionSetting = GetSetting(node.Settings().Ref(), "session"); + if (sessionSetting) { + YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); + sessionColumn = sessionSetting->Child(1)->Child(0)->Content(); } TSet<TStringBuf> usedFields; for (const auto& x : node.Keys()) { - if (x.Value() != sessionColumn) { - usedFields.insert(x.Value()); - } + if (x.Value() != sessionColumn) { + usedFields.insert(x.Value()); + } + } + + if (usedFields.size() == structType->GetSize()) { + return node.Ptr(); } - if (usedFields.size() == structType->GetSize()) { - return node.Ptr(); - } - for (const auto& x : node.Handlers()) { if (x.Ref().ChildrenSize() == 3) { // distinct field @@ -83,20 +83,20 @@ TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprCon } } - if (sessionSetting) { - TCoSessionWindowTraits traits(sessionSetting->Child(1)->ChildPtr(1)); - - auto usedType = traits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()-> - GetItemType()->Cast<TStructExprType>(); - for (const auto& item : usedType->GetItems()) { - usedFields.insert(item->GetName()); - } - - if (usedFields.size() == structType->GetSize()) { - return node.Ptr(); - } - } - + if (sessionSetting) { + TCoSessionWindowTraits traits(sessionSetting->Child(1)->ChildPtr(1)); + + auto usedType = traits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()-> + GetItemType()->Cast<TStructExprType>(); + for (const auto& item : usedType->GetItems()) { + usedFields.insert(item->GetName()); + } + + if (usedFields.size() == structType->GetSize()) { + return node.Ptr(); + } + } + TExprNode::TListType keepMembersList; for (const auto& x : usedFields) { keepMembersList.push_back(ctx.NewAtom(node.Pos(), x)); @@ -124,7 +124,7 @@ void GatherAndTerms(TExprNode::TPtr&& predicate, TExprNode::TListType& andTerms) } } -TExprNode::TPtr FuseAndTerms(TPositionHandle position, const TExprNode::TListType& andTerms, TExprNode::TPtr exclude, TExprContext& ctx) { +TExprNode::TPtr FuseAndTerms(TPositionHandle position, const TExprNode::TListType& andTerms, TExprNode::TPtr exclude, TExprContext& ctx) { TExprNode::TPtr prevAndNode = nullptr; TNodeSet added; for (const auto& otherAndTerm : andTerms) { @@ -489,7 +489,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct auto value = node.Lambda().Body().Ref().ChildPtr(1); TJoinLabels labels; for (ui32 i = 0; i < equiJoin.Ref().ChildrenSize() - 2; ++i) { - auto err = labels.Add(ctx, *equiJoin.Ref().Child(i)->Child(1), + auto err = labels.Add(ctx, *equiJoin.Ref().Child(i)->Child(1), equiJoin.Ref().Child(i)->Child(0)->GetTypeAnn()->Cast<TListExprType>() ->GetItemType()->Cast<TStructExprType>()); if (err) { @@ -654,7 +654,7 @@ TExprNode::TPtr FlatMapSubsetFields(const TCoFlatMapBase& node, TExprContext& ct .Ptr(); } -TExprNode::TPtr RenameJoinTable(TPositionHandle pos, TExprNode::TPtr table, +TExprNode::TPtr RenameJoinTable(TPositionHandle pos, TExprNode::TPtr table, const THashMap<TString, TString>& upstreamTablesRename, TExprContext& ctx) { if (auto renamed = upstreamTablesRename.FindPtr(table->Content())) { @@ -664,7 +664,7 @@ TExprNode::TPtr RenameJoinTable(TPositionHandle pos, TExprNode::TPtr table, return table; } -TExprNode::TPtr RenameEqualityTables(TPositionHandle pos, TExprNode::TPtr columns, +TExprNode::TPtr RenameEqualityTables(TPositionHandle pos, TExprNode::TPtr columns, const THashMap<TString, TString>& upstreamTablesRename, TExprContext& ctx) { TExprNode::TListType newChildren(columns->ChildrenList()); @@ -958,309 +958,309 @@ TExprNode::TPtr FuseEquiJoins(const TExprNode::TPtr& node, ui32 upstreamIndex, T return ret; } -bool HasOnlyCrossJoins(const TExprNode& joinTree) { - if (joinTree.IsAtom()) { - return true; - } - - YQL_ENSURE(joinTree.Child(0)->IsAtom()); - if (joinTree.Child(0)->Content() != "Cross") { - return false; - } - - return HasOnlyCrossJoins(*joinTree.Child(1)) && HasOnlyCrossJoins(*joinTree.Child(2)); +bool HasOnlyCrossJoins(const TExprNode& joinTree) { + if (joinTree.IsAtom()) { + return true; + } + + YQL_ENSURE(joinTree.Child(0)->IsAtom()); + if (joinTree.Child(0)->Content() != "Cross") { + return false; + } + + return HasOnlyCrossJoins(*joinTree.Child(1)) && HasOnlyCrossJoins(*joinTree.Child(2)); +} + +bool IsRenamingOrPassthroughFlatMap(const TCoFlatMapBase& flatMap, THashMap<TStringBuf, TStringBuf>& renames, + THashSet<TStringBuf>& outputMembers, bool& isIdentity) +{ + renames.clear(); + outputMembers.clear(); + isIdentity = false; + + auto body = flatMap.Lambda().Body(); + auto arg = flatMap.Lambda().Args().Arg(0); + + if (!IsJustOrSingleAsList(body.Ref())) { + return false; + } + + TExprBase outItem(body.Ref().ChildPtr(0)); + if (outItem.Raw() == arg.Raw()) { + isIdentity = true; + return true; + } + + if (auto maybeStruct = outItem.Maybe<TCoAsStruct>()) { + for (auto child : maybeStruct.Cast()) { + auto tuple = child.Cast<TCoNameValueTuple>(); + auto value = tuple.Value(); + YQL_ENSURE(outputMembers.insert(tuple.Name().Value()).second); + + if (auto maybeMember = value.Maybe<TCoMember>()) { + auto member = maybeMember.Cast(); + if (member.Struct().Raw() == arg.Raw()) { + TStringBuf oldName = member.Name().Value(); + TStringBuf newName = tuple.Name().Value(); + + YQL_ENSURE(renames.insert({newName, oldName}).second); + } + } + } + return true; + } + + return false; +} + +bool IsInputSuitableForPullingOverEquiJoin(const TCoEquiJoinInput& input, + const THashMap<TStringBuf, THashSet<TStringBuf>>& joinKeysByLabel, + THashMap<TStringBuf, TStringBuf>& renames, TOptimizeContext& optCtx) +{ + renames.clear(); + YQL_ENSURE(input.Scope().Ref().IsAtom()); + + auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(input.List().Ptr()); + if (!maybeFlatMap) { + return false; + } + + auto flatMap = maybeFlatMap.Cast(); + if (flatMap.Lambda().Args().Arg(0).Ref().IsUsedInDependsOn()) { + return false; + } + + if (!SilentGetSequenceItemType(flatMap.Input().Ref(), false)) { + return false; + } + + if (!optCtx.IsSingleUsage(input) || !optCtx.IsSingleUsage(flatMap)) { + return false; + } + + bool isIdentity = false; + THashSet<TStringBuf> outputMembers; + if (!IsRenamingOrPassthroughFlatMap(flatMap, renames, outputMembers, isIdentity)) { + return false; + } + + if (isIdentity) { + // all fields are passthrough + YQL_ENSURE(renames.empty()); + // do not bother pulling identity FlatMap + return false; + } + + if (IsTablePropsDependent(flatMap.Lambda().Body().Ref())) { + renames.clear(); + return false; + } + + auto keysIt = joinKeysByLabel.find(input.Scope().Ref().Content()); + const auto& joinKeys = (keysIt == joinKeysByLabel.end()) ? THashSet<TStringBuf>() : keysIt->second; + + size_t joinKeysFound = 0; + bool hasRename = false; + for (auto it = renames.begin(); it != renames.end();) { + auto inputName = it->second; + auto outputName = it->first; + if (inputName != outputName) { + hasRename = true; + } + YQL_ENSURE(outputMembers.erase(outputName) == 1); + if (joinKeys.contains(outputName)) { + joinKeysFound++; + if (inputName != outputName) { + it++; + continue; + } + } + renames.erase(it++); + } + + if (joinKeysFound != joinKeys.size()) { + // FlatMap is not renaming/passthrough for some join keys + renames.clear(); + return false; + } + + if (!hasRename && outputMembers.empty()) { + // FlatMap _only_ passes through some subset of input columns + // do not bother pulling such Flatmap - it will be optimized away later + renames.clear(); + return false; + } + + return true; } -bool IsRenamingOrPassthroughFlatMap(const TCoFlatMapBase& flatMap, THashMap<TStringBuf, TStringBuf>& renames, - THashSet<TStringBuf>& outputMembers, bool& isIdentity) -{ - renames.clear(); - outputMembers.clear(); - isIdentity = false; - - auto body = flatMap.Lambda().Body(); - auto arg = flatMap.Lambda().Args().Arg(0); - - if (!IsJustOrSingleAsList(body.Ref())) { - return false; - } - - TExprBase outItem(body.Ref().ChildPtr(0)); - if (outItem.Raw() == arg.Raw()) { - isIdentity = true; - return true; - } - - if (auto maybeStruct = outItem.Maybe<TCoAsStruct>()) { - for (auto child : maybeStruct.Cast()) { - auto tuple = child.Cast<TCoNameValueTuple>(); - auto value = tuple.Value(); - YQL_ENSURE(outputMembers.insert(tuple.Name().Value()).second); - - if (auto maybeMember = value.Maybe<TCoMember>()) { - auto member = maybeMember.Cast(); - if (member.Struct().Raw() == arg.Raw()) { - TStringBuf oldName = member.Name().Value(); - TStringBuf newName = tuple.Name().Value(); - - YQL_ENSURE(renames.insert({newName, oldName}).second); - } - } - } - return true; - } - - return false; -} - -bool IsInputSuitableForPullingOverEquiJoin(const TCoEquiJoinInput& input, - const THashMap<TStringBuf, THashSet<TStringBuf>>& joinKeysByLabel, - THashMap<TStringBuf, TStringBuf>& renames, TOptimizeContext& optCtx) -{ - renames.clear(); - YQL_ENSURE(input.Scope().Ref().IsAtom()); - - auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(input.List().Ptr()); - if (!maybeFlatMap) { - return false; - } - - auto flatMap = maybeFlatMap.Cast(); - if (flatMap.Lambda().Args().Arg(0).Ref().IsUsedInDependsOn()) { - return false; - } - - if (!SilentGetSequenceItemType(flatMap.Input().Ref(), false)) { - return false; - } - - if (!optCtx.IsSingleUsage(input) || !optCtx.IsSingleUsage(flatMap)) { - return false; - } - - bool isIdentity = false; - THashSet<TStringBuf> outputMembers; - if (!IsRenamingOrPassthroughFlatMap(flatMap, renames, outputMembers, isIdentity)) { - return false; - } - - if (isIdentity) { - // all fields are passthrough - YQL_ENSURE(renames.empty()); - // do not bother pulling identity FlatMap - return false; - } - - if (IsTablePropsDependent(flatMap.Lambda().Body().Ref())) { - renames.clear(); - return false; - } - - auto keysIt = joinKeysByLabel.find(input.Scope().Ref().Content()); - const auto& joinKeys = (keysIt == joinKeysByLabel.end()) ? THashSet<TStringBuf>() : keysIt->second; - - size_t joinKeysFound = 0; - bool hasRename = false; - for (auto it = renames.begin(); it != renames.end();) { - auto inputName = it->second; - auto outputName = it->first; - if (inputName != outputName) { - hasRename = true; - } - YQL_ENSURE(outputMembers.erase(outputName) == 1); - if (joinKeys.contains(outputName)) { - joinKeysFound++; - if (inputName != outputName) { - it++; - continue; - } - } - renames.erase(it++); - } - - if (joinKeysFound != joinKeys.size()) { - // FlatMap is not renaming/passthrough for some join keys - renames.clear(); - return false; - } - - if (!hasRename && outputMembers.empty()) { - // FlatMap _only_ passes through some subset of input columns - // do not bother pulling such Flatmap - it will be optimized away later - renames.clear(); - return false; - } - - return true; -} - -TExprNode::TPtr ApplyRenames(const TExprNode::TPtr& input, const TMap<TStringBuf, TVector<TStringBuf>>& renames, - const TStructExprType& noRenamesResultType, TStringBuf canaryBaseName, TExprContext& ctx) -{ - TExprNode::TListType asStructArgs; - for (auto& item : noRenamesResultType.GetItems()) { - auto memberName = item->GetName(); - - TStringBuf tableName; - TStringBuf columnName; - SplitTableName(memberName, tableName, columnName); - - if (columnName.find(canaryBaseName, 0) == 0) { - continue; - } - - auto it = renames.find(memberName); - TVector<TStringBuf> passAsIs = { memberName }; - const TVector<TStringBuf>& targets = (it == renames.end()) ? passAsIs : it->second; - if (targets.empty()) { - continue; - } - - auto member = ctx.Builder(input->Pos()) - .Callable("Member") - .Add(0, input) - .Atom(1, memberName) - .Seal() - .Build(); - - for (auto& to : targets) { - asStructArgs.push_back( - ctx.Builder(input->Pos()) - .List() - .Atom(0, to) - .Add(1, member) - .Seal() - .Build() - ); - } - } - - return ctx.NewCallable(input->Pos(), "AsStruct", std::move(asStructArgs)); -} - -TExprNode::TPtr ApplyRenamesToJoinKeys(const TExprNode::TPtr& joinKeys, - const THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& inputJoinKeyRenamesByLabel, TExprContext& ctx) -{ - YQL_ENSURE(joinKeys->ChildrenSize() % 2 == 0); - - TExprNode::TListType newKeys; - newKeys.reserve(joinKeys->ChildrenSize()); - - for (ui32 i = 0; i < joinKeys->ChildrenSize(); i += 2) { - auto table = joinKeys->ChildPtr(i); - auto column = joinKeys->ChildPtr(i + 1); - - YQL_ENSURE(table->IsAtom()); - YQL_ENSURE(column->IsAtom()); - - auto it = inputJoinKeyRenamesByLabel.find(table->Content()); - if (it != inputJoinKeyRenamesByLabel.end()) { - auto renameIt = it->second.find(column->Content()); - if (renameIt != it->second.end()) { - column = ctx.NewAtom(column->Pos(), renameIt->second); - } - } - - newKeys.push_back(table); - newKeys.push_back(column); - } - - return ctx.NewList(joinKeys->Pos(), std::move(newKeys)); -} - - -TExprNode::TPtr ApplyRenamesToJoinTree(const TExprNode::TPtr& joinTree, - const THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& inputJoinKeyRenamesByLabel, TExprContext& ctx) -{ - if (joinTree->IsAtom()) { - return joinTree; - } - - return ctx.Builder(joinTree->Pos()) - .List() - .Add(0, joinTree->ChildPtr(0)) - .Add(1, ApplyRenamesToJoinTree(joinTree->ChildPtr(1), inputJoinKeyRenamesByLabel, ctx)) - .Add(2, ApplyRenamesToJoinTree(joinTree->ChildPtr(2), inputJoinKeyRenamesByLabel, ctx)) - .Add(3, ApplyRenamesToJoinKeys(joinTree->ChildPtr(3), inputJoinKeyRenamesByLabel, ctx)) - .Add(4, ApplyRenamesToJoinKeys(joinTree->ChildPtr(4), inputJoinKeyRenamesByLabel, ctx)) - .Add(5, joinTree->ChildPtr(5)) - .Seal() - .Build(); -} - -const TTypeAnnotationNode* GetCanaryOutputType(const TStructExprType& outputType, TStringBuf fullCanaryName) { - auto maybeIndex = outputType.FindItem(fullCanaryName); - if (!maybeIndex) { - return nullptr; - } - return outputType.GetItems()[*maybeIndex]->GetItemType(); -} - -TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, const TExprNode::TPtr& inputArg, +TExprNode::TPtr ApplyRenames(const TExprNode::TPtr& input, const TMap<TStringBuf, TVector<TStringBuf>>& renames, + const TStructExprType& noRenamesResultType, TStringBuf canaryBaseName, TExprContext& ctx) +{ + TExprNode::TListType asStructArgs; + for (auto& item : noRenamesResultType.GetItems()) { + auto memberName = item->GetName(); + + TStringBuf tableName; + TStringBuf columnName; + SplitTableName(memberName, tableName, columnName); + + if (columnName.find(canaryBaseName, 0) == 0) { + continue; + } + + auto it = renames.find(memberName); + TVector<TStringBuf> passAsIs = { memberName }; + const TVector<TStringBuf>& targets = (it == renames.end()) ? passAsIs : it->second; + if (targets.empty()) { + continue; + } + + auto member = ctx.Builder(input->Pos()) + .Callable("Member") + .Add(0, input) + .Atom(1, memberName) + .Seal() + .Build(); + + for (auto& to : targets) { + asStructArgs.push_back( + ctx.Builder(input->Pos()) + .List() + .Atom(0, to) + .Add(1, member) + .Seal() + .Build() + ); + } + } + + return ctx.NewCallable(input->Pos(), "AsStruct", std::move(asStructArgs)); +} + +TExprNode::TPtr ApplyRenamesToJoinKeys(const TExprNode::TPtr& joinKeys, + const THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& inputJoinKeyRenamesByLabel, TExprContext& ctx) +{ + YQL_ENSURE(joinKeys->ChildrenSize() % 2 == 0); + + TExprNode::TListType newKeys; + newKeys.reserve(joinKeys->ChildrenSize()); + + for (ui32 i = 0; i < joinKeys->ChildrenSize(); i += 2) { + auto table = joinKeys->ChildPtr(i); + auto column = joinKeys->ChildPtr(i + 1); + + YQL_ENSURE(table->IsAtom()); + YQL_ENSURE(column->IsAtom()); + + auto it = inputJoinKeyRenamesByLabel.find(table->Content()); + if (it != inputJoinKeyRenamesByLabel.end()) { + auto renameIt = it->second.find(column->Content()); + if (renameIt != it->second.end()) { + column = ctx.NewAtom(column->Pos(), renameIt->second); + } + } + + newKeys.push_back(table); + newKeys.push_back(column); + } + + return ctx.NewList(joinKeys->Pos(), std::move(newKeys)); +} + + +TExprNode::TPtr ApplyRenamesToJoinTree(const TExprNode::TPtr& joinTree, + const THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>>& inputJoinKeyRenamesByLabel, TExprContext& ctx) +{ + if (joinTree->IsAtom()) { + return joinTree; + } + + return ctx.Builder(joinTree->Pos()) + .List() + .Add(0, joinTree->ChildPtr(0)) + .Add(1, ApplyRenamesToJoinTree(joinTree->ChildPtr(1), inputJoinKeyRenamesByLabel, ctx)) + .Add(2, ApplyRenamesToJoinTree(joinTree->ChildPtr(2), inputJoinKeyRenamesByLabel, ctx)) + .Add(3, ApplyRenamesToJoinKeys(joinTree->ChildPtr(3), inputJoinKeyRenamesByLabel, ctx)) + .Add(4, ApplyRenamesToJoinKeys(joinTree->ChildPtr(4), inputJoinKeyRenamesByLabel, ctx)) + .Add(5, joinTree->ChildPtr(5)) + .Seal() + .Build(); +} + +const TTypeAnnotationNode* GetCanaryOutputType(const TStructExprType& outputType, TStringBuf fullCanaryName) { + auto maybeIndex = outputType.FindItem(fullCanaryName); + if (!maybeIndex) { + return nullptr; + } + return outputType.GetItems()[*maybeIndex]->GetItemType(); +} + +TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, const TExprNode::TPtr& inputArg, const TString& canaryName, const TStructExprType& canaryResultTypeWithoutRenames, bool keepSys, TExprContext& ctx) -{ - YQL_ENSURE(input.Scope().Ref().IsAtom()); - TStringBuf label = input.Scope().Ref().Content(); - - auto flatMap = input.List().Cast<TCoFlatMapBase>(); - auto lambda = flatMap.Lambda(); - YQL_ENSURE(IsJustOrSingleAsList(lambda.Body().Ref())); +{ + YQL_ENSURE(input.Scope().Ref().IsAtom()); + TStringBuf label = input.Scope().Ref().Content(); + + auto flatMap = input.List().Cast<TCoFlatMapBase>(); + auto lambda = flatMap.Lambda(); + YQL_ENSURE(IsJustOrSingleAsList(lambda.Body().Ref())); auto strippedLambdaBody = lambda.Body().Ref().HeadPtr(); - - const TString labelPrefix = TString::Join(label, "."); - const TString fullCanaryName = FullColumnName(label, canaryName); - - const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(canaryResultTypeWithoutRenames, fullCanaryName); - if (!canaryOutType) { - // canary didn't survive join - return {}; - } - - auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); - auto flatMapOutputItem = GetSequenceItemType(flatMap, false); - - auto myStruct = ctx.Builder(input.Pos()) - .Callable("DivePrefixMembers") - .Add(0, inputArg) - .List(1) - .Atom(0, labelPrefix) - .Seal() - .Seal() - .Build(); - - if (canaryOutType->GetKind() == ETypeAnnotationKind::Data) { - YQL_ENSURE(canaryOutType->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool); - // our input passed as-is - return ctx.Builder(input.Pos()) - .List() - .Atom(0, labelPrefix) + + const TString labelPrefix = TString::Join(label, "."); + const TString fullCanaryName = FullColumnName(label, canaryName); + + const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(canaryResultTypeWithoutRenames, fullCanaryName); + if (!canaryOutType) { + // canary didn't survive join + return {}; + } + + auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); + auto flatMapOutputItem = GetSequenceItemType(flatMap, false); + + auto myStruct = ctx.Builder(input.Pos()) + .Callable("DivePrefixMembers") + .Add(0, inputArg) + .List(1) + .Atom(0, labelPrefix) + .Seal() + .Seal() + .Build(); + + if (canaryOutType->GetKind() == ETypeAnnotationKind::Data) { + YQL_ENSURE(canaryOutType->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool); + // our input passed as-is + return ctx.Builder(input.Pos()) + .List() + .Atom(0, labelPrefix) .ApplyPartial(1, lambda.Args().Ptr(), std::move(strippedLambdaBody)) .With(0, std::move(myStruct)) - .Seal() - .Seal() - .Build(); - } - - YQL_ENSURE(canaryOutType->GetKind() == ETypeAnnotationKind::Optional); - + .Seal() + .Seal() + .Build(); + } + + YQL_ENSURE(canaryOutType->GetKind() == ETypeAnnotationKind::Optional); + TExprNode::TListType membersForCheck; - auto flatMapInputItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); + auto flatMapInputItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); if (!keepSys) { EraseIf(flatMapInputItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); }); } - flatMapInputItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); - for (auto& item : flatMapInputItems) { + flatMapInputItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); + for (auto& item : flatMapInputItems) { if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) { membersForCheck.emplace_back(ctx.NewAtom(input.Pos(), item->GetName())); - } - } - + } + } + auto checkedMembersList = ctx.NewList(input.Pos(), std::move(membersForCheck)); - - return ctx.Builder(input.Pos()) - .List() - .Atom(0, labelPrefix) - .Callable(1, "IfPresent") + + return ctx.Builder(input.Pos()) + .List() + .Atom(0, labelPrefix) + .Callable(1, "IfPresent") .Callable(0, "FilterNullMembers") .Callable(0, "AssumeAllMembersNullableAtOnce") .Callable(0, "Just") @@ -1268,240 +1268,240 @@ TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, cons .Seal() .Seal() .Add(1, std::move(checkedMembersList)) - .Seal() - .Lambda(1) - .Param("canaryInput") - .Callable("FlattenMembers") - .List(0) - .Atom(0, "") - .Callable(1, "Just") + .Seal() + .Lambda(1) + .Param("canaryInput") + .Callable("FlattenMembers") + .List(0) + .Atom(0, "") + .Callable(1, "Just") .ApplyPartial(0, lambda.Args().Ptr(), std::move(strippedLambdaBody)) - .With(0) - .Callable("RemoveMember") - .Arg(0, "canaryInput") - .Atom(1, canaryName) - .Seal() - .Done() - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Callable(2, "FlattenMembers") - .List(0) - .Atom(0, "") - .Callable(1, "Nothing") - .Add(0, ExpandType(input.Pos(), *ctx.MakeType<TOptionalExprType>(flatMapOutputItem), ctx)) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.Types->PullUpFlatMapOverJoin) { - return node; - } - - YQL_ENSURE(node->ChildrenSize() >= 4); - auto inputsCount = ui32(node->ChildrenSize() - 2); - - auto joinTree = node->ChildPtr(inputsCount); - if (HasOnlyCrossJoins(*joinTree)) { - return node; - } - + .With(0) + .Callable("RemoveMember") + .Arg(0, "canaryInput") + .Atom(1, canaryName) + .Seal() + .Done() + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Callable(2, "FlattenMembers") + .List(0) + .Atom(0, "") + .Callable(1, "Nothing") + .Add(0, ExpandType(input.Pos(), *ctx.MakeType<TOptionalExprType>(flatMapOutputItem), ctx)) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + if (!optCtx.Types->PullUpFlatMapOverJoin) { + return node; + } + + YQL_ENSURE(node->ChildrenSize() >= 4); + auto inputsCount = ui32(node->ChildrenSize() - 2); + + auto joinTree = node->ChildPtr(inputsCount); + if (HasOnlyCrossJoins(*joinTree)) { + return node; + } + bool keepSys = false; - auto settings = node->ChildPtr(inputsCount + 1); - for (auto& child : settings->Children()) { - if (child->Child(0)->Content() == "flatten") { - return node; - } + auto settings = node->ChildPtr(inputsCount + 1); + for (auto& child : settings->Children()) { + if (child->Child(0)->Content() == "flatten") { + return node; + } if (child->Child(0)->Content() == "keep_sys") { keepSys = true; } - } - - static const TStringBuf canaryBaseName = "_yql_canary_"; - - THashMap<TStringBuf, THashSet<TStringBuf>> joinKeysByLabel = CollectEquiJoinKeyColumnsByLabel(*joinTree); - const auto renames = LoadJoinRenameMap(*settings); - - TVector<ui32> toPull; - TJoinLabels canaryLabels; - TJoinLabels actualLabels; - THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>> inputJoinKeyRenamesByLabel; - for (ui32 i = 0; i < inputsCount; ++i) { - TCoEquiJoinInput input(node->ChildPtr(i)); - - if (!input.Scope().Ref().IsAtom()) { - return node; - } - - const TTypeAnnotationNode* itemType = input.List().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - auto structType = itemType->Cast<TStructExprType>(); - for (auto& si : structType->GetItems()) { - if (si->GetName().find(canaryBaseName, 0) == 0) { - // EquiJoin already processed - return node; - } - } - - auto err = actualLabels.Add(ctx, *input.Scope().Ptr(), structType); - YQL_ENSURE(!err); - - auto label = input.Scope().Ref().Content(); - - - if (IsInputSuitableForPullingOverEquiJoin(input, joinKeysByLabel, inputJoinKeyRenamesByLabel[label], optCtx)) { - auto flatMap = input.List().Cast<TCoFlatMapBase>(); - - auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); - auto structItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); + } + + static const TStringBuf canaryBaseName = "_yql_canary_"; + + THashMap<TStringBuf, THashSet<TStringBuf>> joinKeysByLabel = CollectEquiJoinKeyColumnsByLabel(*joinTree); + const auto renames = LoadJoinRenameMap(*settings); + + TVector<ui32> toPull; + TJoinLabels canaryLabels; + TJoinLabels actualLabels; + THashMap<TStringBuf, THashMap<TStringBuf, TStringBuf>> inputJoinKeyRenamesByLabel; + for (ui32 i = 0; i < inputsCount; ++i) { + TCoEquiJoinInput input(node->ChildPtr(i)); + + if (!input.Scope().Ref().IsAtom()) { + return node; + } + + const TTypeAnnotationNode* itemType = input.List().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + auto structType = itemType->Cast<TStructExprType>(); + for (auto& si : structType->GetItems()) { + if (si->GetName().find(canaryBaseName, 0) == 0) { + // EquiJoin already processed + return node; + } + } + + auto err = actualLabels.Add(ctx, *input.Scope().Ptr(), structType); + YQL_ENSURE(!err); + + auto label = input.Scope().Ref().Content(); + + + if (IsInputSuitableForPullingOverEquiJoin(input, joinKeysByLabel, inputJoinKeyRenamesByLabel[label], optCtx)) { + auto flatMap = input.List().Cast<TCoFlatMapBase>(); + + auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false); + auto structItems = flatMapInputItem->Cast<TStructExprType>()->GetItems(); if (!keepSys) { EraseIf(structItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); }); } - - TString canaryName = TStringBuilder() << canaryBaseName << i; - structItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); - structType = ctx.MakeType<TStructExprType>(structItems); - - YQL_CLOG(DEBUG, Core) << "Will pull up EquiJoin input #" << i; - toPull.push_back(i); - } - - err = canaryLabels.Add(ctx, *input.Scope().Ptr(), structType); - YQL_ENSURE(!err); - } - - if (toPull.empty()) { - return node; - } - - const TStructExprType* canaryResultType = nullptr; - const TStructExprType* noRenamesResultType = nullptr; - const auto settingsWithoutRenames = RemoveSetting(*settings, "rename", ctx); - const auto joinTreeWithInputRenames = ApplyRenamesToJoinTree(joinTree, inputJoinKeyRenamesByLabel, ctx); - - - { - TJoinOptions options; - auto status = ValidateEquiJoinOptions(node->Pos(), *settingsWithoutRenames, options, ctx); - YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); - + + TString canaryName = TStringBuilder() << canaryBaseName << i; + structItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool))); + structType = ctx.MakeType<TStructExprType>(structItems); + + YQL_CLOG(DEBUG, Core) << "Will pull up EquiJoin input #" << i; + toPull.push_back(i); + } + + err = canaryLabels.Add(ctx, *input.Scope().Ptr(), structType); + YQL_ENSURE(!err); + } + + if (toPull.empty()) { + return node; + } + + const TStructExprType* canaryResultType = nullptr; + const TStructExprType* noRenamesResultType = nullptr; + const auto settingsWithoutRenames = RemoveSetting(*settings, "rename", ctx); + const auto joinTreeWithInputRenames = ApplyRenamesToJoinTree(joinTree, inputJoinKeyRenamesByLabel, ctx); + + + { + TJoinOptions options; + auto status = ValidateEquiJoinOptions(node->Pos(), *settingsWithoutRenames, options, ctx); + YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); + status = EquiJoinAnnotation(node->Pos(), canaryResultType, canaryLabels, - *joinTreeWithInputRenames, options, ctx); + *joinTreeWithInputRenames, options, ctx); YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); - status = EquiJoinAnnotation(node->Pos(), noRenamesResultType, actualLabels, - *joinTree, options, ctx); - YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); - } - - - - TExprNode::TListType newEquiJoinArgs; - newEquiJoinArgs.reserve(node->ChildrenSize()); - - TExprNode::TListType flattenMembersArgs; - - auto afterJoinArg = ctx.NewArgument(node->Pos(), "joinOut"); - - for (ui32 i = 0, j = 0; i < inputsCount; ++i) { - TCoEquiJoinInput input(node->ChildPtr(i)); - - TStringBuf label = input.Scope().Ref().Content(); - TString labelPrefix = TString::Join(label, "."); - - if (j < toPull.size() && i == toPull[j]) { - j++; - - - const TString canaryName = TStringBuilder() << canaryBaseName << i; - const TString fullCanaryName = FullColumnName(label, canaryName); - - TCoFlatMapBase flatMap = input.List().Cast<TCoFlatMapBase>(); - - const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(*canaryResultType, fullCanaryName); - if (canaryOutType && canaryOutType->GetKind() == ETypeAnnotationKind::Optional) { - // remove leading flatmap from input and launch canary - newEquiJoinArgs.push_back( - ctx.Builder(input.Pos()) - .List() - .Callable(0, flatMap.CallableName()) - .Add(0, flatMap.Input().Ptr()) - .Lambda(1) - .Param("item") - .Callable("Just") - .Callable(0, "AddMember") - .Arg(0, "item") - .Atom(1, canaryName) - .Callable(2, "Bool") - .Atom(0, "true") - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Add(1, input.Scope().Ptr()) - .Seal() - .Build() - ); - } else { - // just remove leading flatmap from input - newEquiJoinArgs.push_back( - ctx.Builder(input.Pos()) - .List() - .Add(0, flatMap.Input().Ptr()) - .Add(1, input.Scope().Ptr()) - .Seal() - .Build() - ); - } - + status = EquiJoinAnnotation(node->Pos(), noRenamesResultType, actualLabels, + *joinTree, options, ctx); + YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); + } + + + + TExprNode::TListType newEquiJoinArgs; + newEquiJoinArgs.reserve(node->ChildrenSize()); + + TExprNode::TListType flattenMembersArgs; + + auto afterJoinArg = ctx.NewArgument(node->Pos(), "joinOut"); + + for (ui32 i = 0, j = 0; i < inputsCount; ++i) { + TCoEquiJoinInput input(node->ChildPtr(i)); + + TStringBuf label = input.Scope().Ref().Content(); + TString labelPrefix = TString::Join(label, "."); + + if (j < toPull.size() && i == toPull[j]) { + j++; + + + const TString canaryName = TStringBuilder() << canaryBaseName << i; + const TString fullCanaryName = FullColumnName(label, canaryName); + + TCoFlatMapBase flatMap = input.List().Cast<TCoFlatMapBase>(); + + const TTypeAnnotationNode* canaryOutType = GetCanaryOutputType(*canaryResultType, fullCanaryName); + if (canaryOutType && canaryOutType->GetKind() == ETypeAnnotationKind::Optional) { + // remove leading flatmap from input and launch canary + newEquiJoinArgs.push_back( + ctx.Builder(input.Pos()) + .List() + .Callable(0, flatMap.CallableName()) + .Add(0, flatMap.Input().Ptr()) + .Lambda(1) + .Param("item") + .Callable("Just") + .Callable(0, "AddMember") + .Arg(0, "item") + .Atom(1, canaryName) + .Callable(2, "Bool") + .Atom(0, "true") + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Add(1, input.Scope().Ptr()) + .Seal() + .Build() + ); + } else { + // just remove leading flatmap from input + newEquiJoinArgs.push_back( + ctx.Builder(input.Pos()) + .List() + .Add(0, flatMap.Input().Ptr()) + .Add(1, input.Scope().Ptr()) + .Seal() + .Build() + ); + } + auto flattenMembersArg = BuildOutputFlattenMembersArg(input, afterJoinArg, canaryName, *canaryResultType, keepSys, ctx); - if (flattenMembersArg) { - flattenMembersArgs.push_back(flattenMembersArg); - } - } else { - flattenMembersArgs.push_back(ctx.Builder(input.Pos()) - .List() - .Atom(0, labelPrefix) - .Callable(1, "DivePrefixMembers") - .Add(0, afterJoinArg) - .List(1) - .Atom(0, labelPrefix) - .Seal() - .Seal() - .Seal() - .Build()); - newEquiJoinArgs.push_back(input.Ptr()); - } - } - - newEquiJoinArgs.push_back(joinTreeWithInputRenames); - newEquiJoinArgs.push_back(settingsWithoutRenames); - - auto newEquiJoin = ctx.NewCallable(node->Pos(), "EquiJoin", std::move(newEquiJoinArgs)); - - auto flattenMembers = flattenMembersArgs.empty() ? afterJoinArg : - ctx.NewCallable(node->Pos(), "FlattenMembers", std::move(flattenMembersArgs)); - - auto newLambdaBody = ctx.Builder(node->Pos()) - .Callable("Just") - .Add(0, ApplyRenames(flattenMembers, renames, *noRenamesResultType, canaryBaseName, ctx)) - .Seal() - .Build(); - - auto newLambda = ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { afterJoinArg }), std::move(newLambdaBody)); - - return ctx.NewCallable(node->Pos(), "FlatMap", { newEquiJoin, newLambda }); -} - + if (flattenMembersArg) { + flattenMembersArgs.push_back(flattenMembersArg); + } + } else { + flattenMembersArgs.push_back(ctx.Builder(input.Pos()) + .List() + .Atom(0, labelPrefix) + .Callable(1, "DivePrefixMembers") + .Add(0, afterJoinArg) + .List(1) + .Atom(0, labelPrefix) + .Seal() + .Seal() + .Seal() + .Build()); + newEquiJoinArgs.push_back(input.Ptr()); + } + } + + newEquiJoinArgs.push_back(joinTreeWithInputRenames); + newEquiJoinArgs.push_back(settingsWithoutRenames); + + auto newEquiJoin = ctx.NewCallable(node->Pos(), "EquiJoin", std::move(newEquiJoinArgs)); + + auto flattenMembers = flattenMembersArgs.empty() ? afterJoinArg : + ctx.NewCallable(node->Pos(), "FlattenMembers", std::move(flattenMembersArgs)); + + auto newLambdaBody = ctx.Builder(node->Pos()) + .Callable("Just") + .Add(0, ApplyRenames(flattenMembers, renames, *noRenamesResultType, canaryBaseName, ctx)) + .Seal() + .Build(); + + auto newLambda = ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { afterJoinArg }), std::move(newLambdaBody)); + + return ctx.NewCallable(node->Pos(), "FlatMap", { newEquiJoin, newLambda }); +} + TExprNode::TPtr OptimizeFromFlow(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1522,10 +1522,10 @@ TExprNode::TPtr OptimizeFromFlow(const TExprNode::TPtr& node, TExprContext& ctx, } return node; -} - +} + TExprNode::TPtr OptimizeCollect(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (!optCtx.IsSingleUsage(node->Head())) { + if (!optCtx.IsSingleUsage(node->Head())) { return node; } @@ -1548,7 +1548,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { map["FlatMap"] = map["OrderedFlatMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { TCoFlatMapBase self(node); - if (!optCtx.IsSingleUsage(self.Input().Ref())) { + if (!optCtx.IsSingleUsage(self.Input().Ref())) { return node; } @@ -1726,7 +1726,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { map["CombineByKey"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { TCoCombineByKey self(node); - if (!optCtx.IsSingleUsage(self.Input().Ref())) { + if (!optCtx.IsSingleUsage(self.Input().Ref())) { return node; } @@ -1800,8 +1800,8 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { ui32 inputsCount = node->ChildrenSize() - 2; for (ui32 i = 0; i < inputsCount; ++i) { if (node->Child(i)->Child(0)->IsCallable("EquiJoin") && - optCtx.IsSingleUsage(*node->Child(i)) && - optCtx.IsSingleUsage(*node->Child(i)->Child(0))) { + optCtx.IsSingleUsage(*node->Child(i)) && + optCtx.IsSingleUsage(*node->Child(i)->Child(0))) { auto ret = FuseEquiJoins(node, i, ctx); if (ret != node) { YQL_CLOG(DEBUG, Core) << "FuseEquiJoins"; @@ -1810,18 +1810,18 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { } } - auto ret = PullUpFlatMapOverEquiJoin(node, ctx, optCtx); - if (ret != node) { - YQL_CLOG(DEBUG, Core) << "PullUpFlatMapOverEquiJoin"; - return ret; - } - + auto ret = PullUpFlatMapOverEquiJoin(node, ctx, optCtx); + if (ret != node) { + YQL_CLOG(DEBUG, Core) << "PullUpFlatMapOverEquiJoin"; + return ret; + } + return node; }; map["ExtractMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { TCoExtractMembers self(node); - if (!optCtx.IsSingleUsage(self.Input())) { + if (!optCtx.IsSingleUsage(self.Input())) { return node; } @@ -1902,7 +1902,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { return node; } - if (self.Input().Maybe<TCoCalcOverWindowBase>() || self.Input().Maybe<TCoCalcOverWindowGroup>()) { + if (self.Input().Maybe<TCoCalcOverWindowBase>() || self.Input().Maybe<TCoCalcOverWindowGroup>()) { if (auto res = ApplyExtractMembersToCalcOverWindow(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) { return res; } @@ -1923,13 +1923,13 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { return node; } - if (self.Input().Maybe<TCoCollect>()) { - if (auto res = ApplyExtractMembersToCollect(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) { - return res; - } - return node; - } - + if (self.Input().Maybe<TCoCollect>()) { + if (auto res = ApplyExtractMembersToCollect(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) { + return res; + } + return node; + } + if (self.Input().Maybe<TCoMapJoinCore>()) { if (auto res = ApplyExtractMembersToMapJoinCore(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) { return res; @@ -2010,11 +2010,11 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { return ctx.Builder(node->Pos()) .Callable("WindowTraits") .Add(0, ExpandType(node->Pos(), *subsetType, ctx)) - .Add(1, ctx.DeepCopyLambda(*node->Child(1))) - .Add(2, ctx.DeepCopyLambda(*node->Child(2))) - .Add(3, ctx.DeepCopyLambda(*node->Child(3))) - .Add(4, ctx.DeepCopyLambda(*node->Child(4))) - .Add(5, node->Child(5)->IsLambda() ? ctx.DeepCopyLambda(*node->Child(5)) : node->ChildPtr(5)) + .Add(1, ctx.DeepCopyLambda(*node->Child(1))) + .Add(2, ctx.DeepCopyLambda(*node->Child(2))) + .Add(3, ctx.DeepCopyLambda(*node->Child(3))) + .Add(4, ctx.DeepCopyLambda(*node->Child(4))) + .Add(5, node->Child(5)->IsLambda() ? ctx.DeepCopyLambda(*node->Child(5)) : node->ChildPtr(5)) .Seal() .Build(); }; @@ -2093,20 +2093,20 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { return ctx.Builder(node->Pos()) .Callable("AggregationTraits") .Add(0, ExpandType(node->Pos(), *subsetType, ctx)) - .Add(1, ctx.DeepCopyLambda(*node->Child(1))) - .Add(2, ctx.DeepCopyLambda(*node->Child(2))) - .Add(3, ctx.DeepCopyLambda(*node->Child(3))) - .Add(4, ctx.DeepCopyLambda(*node->Child(4))) - .Add(5, ctx.DeepCopyLambda(*node->Child(5))) - .Add(6, ctx.DeepCopyLambda(*node->Child(6))) - .Add(7, node->Child(7)->IsLambda() ? ctx.DeepCopyLambda(*node->Child(7)) : node->ChildPtr(7)) + .Add(1, ctx.DeepCopyLambda(*node->Child(1))) + .Add(2, ctx.DeepCopyLambda(*node->Child(2))) + .Add(3, ctx.DeepCopyLambda(*node->Child(3))) + .Add(4, ctx.DeepCopyLambda(*node->Child(4))) + .Add(5, ctx.DeepCopyLambda(*node->Child(5))) + .Add(6, ctx.DeepCopyLambda(*node->Child(6))) + .Add(7, node->Child(7)->IsLambda() ? ctx.DeepCopyLambda(*node->Child(7)) : node->ChildPtr(7)) .Seal() .Build(); }; - map["SessionWindowTraits"] = map["SortTraits"] = map["Lag"] = map["Lead"] = map["RowNumber"] = map["Rank"] = map["DenseRank"] = - [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) - { + map["SessionWindowTraits"] = map["SortTraits"] = map["Lag"] = map["Lead"] = map["RowNumber"] = map["Rank"] = map["DenseRank"] = + [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) + { auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType() ->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); if (node->IsCallable("RowNumber")) { @@ -2123,31 +2123,31 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { .Build(); } - TSet<ui32> lambdaIndexes; + TSet<ui32> lambdaIndexes; TSet<TStringBuf> lambdaSubset; - if (node->IsCallable("SessionWindowTraits")) { - lambdaIndexes = { 2, 3, 4 }; - TCoSessionWindowTraits self(node); - if (auto maybeSort = self.SortSpec().Maybe<TCoSortTraits>()) { - const TTypeAnnotationNode* itemType = - maybeSort.Cast().ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); - if (itemType->GetKind() == ETypeAnnotationKind::Struct) { - for (auto& col : itemType->Cast<TStructExprType>()->GetItems()) { - lambdaSubset.insert(col->GetName()); - } - } - } - } else { - lambdaIndexes = { node->IsCallable("SortTraits") ? 2u : 1u }; - } - - for (ui32 idx : lambdaIndexes) { - auto lambda = node->Child(idx); - if (!HaveFieldsSubset(lambda->ChildPtr(1), *lambda->Child(0)->Child(0), lambdaSubset, *optCtx.ParentsMap)) { - return node; - } - } - + if (node->IsCallable("SessionWindowTraits")) { + lambdaIndexes = { 2, 3, 4 }; + TCoSessionWindowTraits self(node); + if (auto maybeSort = self.SortSpec().Maybe<TCoSortTraits>()) { + const TTypeAnnotationNode* itemType = + maybeSort.Cast().ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); + if (itemType->GetKind() == ETypeAnnotationKind::Struct) { + for (auto& col : itemType->Cast<TStructExprType>()->GetItems()) { + lambdaSubset.insert(col->GetName()); + } + } + } + } else { + lambdaIndexes = { node->IsCallable("SortTraits") ? 2u : 1u }; + } + + for (ui32 idx : lambdaIndexes) { + auto lambda = node->Child(idx); + if (!HaveFieldsSubset(lambda->ChildPtr(1), *lambda->Child(0)->Child(0), lambdaSubset, *optCtx.ParentsMap)) { + return node; + } + } + if (lambdaSubset.size() == structType->GetSize()) { return node; } @@ -2160,7 +2160,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { } auto subsetType = ctx.MakeType<TListExprType>(ctx.MakeType<TStructExprType>(subsetItems)); - YQL_CLOG(DEBUG, Core) << "FieldSubset for " << node->Content(); + YQL_CLOG(DEBUG, Core) << "FieldSubset for " << node->Content(); if (node->IsCallable("SortTraits")) { return ctx.Builder(node->Pos()) .Callable("SortTraits") @@ -2169,16 +2169,16 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { .Add(2, ctx.DeepCopyLambda(*node->ChildPtr(2))) .Seal() .Build(); - } else if (node->IsCallable("SessionWindowTraits")) { - return ctx.Builder(node->Pos()) - .Callable("SessionWindowTraits") - .Add(0, ExpandType(node->Pos(), *subsetType, ctx)) - .Add(1, node->ChildPtr(1)) - .Add(2, ctx.DeepCopyLambda(*node->ChildPtr(2))) - .Add(3, ctx.DeepCopyLambda(*node->ChildPtr(3))) - .Add(4, ctx.DeepCopyLambda(*node->ChildPtr(4))) - .Seal() - .Build(); + } else if (node->IsCallable("SessionWindowTraits")) { + return ctx.Builder(node->Pos()) + .Callable("SessionWindowTraits") + .Add(0, ExpandType(node->Pos(), *subsetType, ctx)) + .Add(1, node->ChildPtr(1)) + .Add(2, ctx.DeepCopyLambda(*node->ChildPtr(2))) + .Add(3, ctx.DeepCopyLambda(*node->ChildPtr(3))) + .Add(4, ctx.DeepCopyLambda(*node->ChildPtr(4))) + .Seal() + .Build(); } else { if (node->ChildrenSize() == 2) { return ctx.Builder(node->Pos()) @@ -2201,7 +2201,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { map["Aggregate"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { TCoAggregate self(node); - if (!optCtx.IsSingleUsage(self.Input()) && !optCtx.IsPersistentNode(self.Input())) { + if (!optCtx.IsSingleUsage(self.Input()) && !optCtx.IsPersistentNode(self.Input())) { return node; } @@ -2213,26 +2213,26 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { return node; }; - - map["CalcOverWindow"] = map["CalcOverSessionWindow"] = map["CalcOverWindowGroup"] = - [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) - { - if (!optCtx.IsSingleUsage(*node->Child(0))) { - return node; - } - - if (!node->Child(0)->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})) { - return node; - } - - TExprNodeList parentCalcs = ExtractCalcsOverWindow(node, ctx); - TExprNodeList calcs = ExtractCalcsOverWindow(node->ChildPtr(0), ctx); - calcs.insert(calcs.end(), parentCalcs.begin(), parentCalcs.end()); - - YQL_CLOG(DEBUG, Core) << "Fuse nested CalcOverWindow/CalcOverSessionWindow/CalcOverWindowGroup"; - - return RebuildCalcOverWindowGroup(node->Child(0)->Pos(), node->Child(0)->ChildPtr(0), calcs, ctx); - }; + + map["CalcOverWindow"] = map["CalcOverSessionWindow"] = map["CalcOverWindowGroup"] = + [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) + { + if (!optCtx.IsSingleUsage(*node->Child(0))) { + return node; + } + + if (!node->Child(0)->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})) { + return node; + } + + TExprNodeList parentCalcs = ExtractCalcsOverWindow(node, ctx); + TExprNodeList calcs = ExtractCalcsOverWindow(node->ChildPtr(0), ctx); + calcs.insert(calcs.end(), parentCalcs.begin(), parentCalcs.end()); + + YQL_CLOG(DEBUG, Core) << "Fuse nested CalcOverWindow/CalcOverSessionWindow/CalcOverWindowGroup"; + + return RebuildCalcOverWindowGroup(node->Child(0)->Pos(), node->Child(0)->ChildPtr(0), calcs, ctx); + }; map[TCoCondense::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { TCoCondense self(node); diff --git a/ydb/library/yql/core/common_opt/yql_co_last.cpp b/ydb/library/yql/core/common_opt/yql_co_last.cpp index b43009d099..6323d4bca7 100644 --- a/ydb/library/yql/core/common_opt/yql_co_last.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_last.cpp @@ -3,16 +3,16 @@ namespace NYql { -void RegisterCoFinalCallables(TCallableOptimizerMap& map) { - map["UnorderedSubquery"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - Y_UNUSED(optCtx); - if (node->Head().IsCallable("Sort")) { - if (!WarnUnroderedSubquery(*node, ctx)) { - return TExprNode::TPtr(); - } - } - return ctx.RenameNode(*node, "Unordered"); - }; +void RegisterCoFinalCallables(TCallableOptimizerMap& map) { + map["UnorderedSubquery"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + Y_UNUSED(optCtx); + if (node->Head().IsCallable("Sort")) { + if (!WarnUnroderedSubquery(*node, ctx)) { + return TExprNode::TPtr(); + } + } + return ctx.RenameNode(*node, "Unordered"); + }; +} + } - -} diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp index ce8a81bd2b..a32a6c3dab 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp @@ -13,7 +13,7 @@ #include <util/generic/map.h> #include <util/generic/bitmap.h> #include <util/string/cast.h> -#include <util/generic/xrange.h> +#include <util/generic/xrange.h> #include <algorithm> #include <iterator> @@ -229,7 +229,7 @@ TExprNode::TPtr ExpandFlattenEquiJoin(const TExprNode::TPtr& node, TExprContext& const auto& list = listPair.Head(); const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType(); auto structType = itemType->Cast<TStructExprType>(); - if (auto err = labels.Add(ctx, *listPair.Child(1), structType)) { + if (auto err = labels.Add(ctx, *listPair.Child(1), structType)) { ctx.AddError(*err); return nullptr; } @@ -379,7 +379,7 @@ TExprNode::TPtr RemoveDeadPayloadColumns(const TExprNode::TPtr& node, TExprConte TJoinLabels labels; for (ui32 i = 0; i < node->ChildrenSize() - 2; ++i) { - auto err = labels.Add(ctx, *node->Child(i)->Child(1), + auto err = labels.Add(ctx, *node->Child(i)->Child(1), node->Child(i)->Head().GetTypeAnn()->Cast<TListExprType>() ->GetItemType()->Cast<TStructExprType>()); if (err) { @@ -648,7 +648,7 @@ std::vector<TExprNode::TListType> GroupNodeChildrenByType(const TExprNode::TPtr& } template <bool Ordered> -TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { +TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); if (node->ChildrenSize() == 1) { return node->HeadPtr(); @@ -686,7 +686,7 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T member = ctx.NewCallable(pos, "Member", { arg, name }); if (TrySilentConvertTo(member, *myType, *resultType, ctx) == IGraphTransformer::TStatus::Error) { err = TIssue( - ctx.GetPosition(pos), + ctx.GetPosition(pos), TStringBuilder() << "Uncompatible member " << item->GetName() << " types: " << *myType << " and " << *resultType @@ -696,7 +696,7 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T } if (err) { - member = ctx.NewCallable(pos, "Error", { ExpandType(pos, *ctx.MakeType<TErrorExprType>(*err), ctx) }); + member = ctx.NewCallable(pos, "Error", { ExpandType(pos, *ctx.MakeType<TErrorExprType>(*err), ctx) }); } bodyItems.push_back(ctx.NewList(pos, { name, member })); @@ -733,8 +733,8 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T remappedList.push_back(remapped); } - auto res = ctx.NewCallable(node->Pos(), Ordered ? "Merge" : "Extend", std::move(remappedList)); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + auto res = ctx.NewCallable(node->Pos(), Ordered ? "Merge" : "Extend", std::move(remappedList)); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); } TExprNode::TPtr RemoveNothingFromCoalesce(const TExprNode& node, TExprContext& ctx) { @@ -830,12 +830,12 @@ TExprNode::TPtr PropagateCoalesceWithConstIntoLogicalOps(const TExprNode::TPtr& } if (node->Head().IsCallable({"And", "Or"})) { - YQL_CLOG(DEBUG, Core) << "PropagateCoalesceWithConst over " << node->Head().Content(); - auto children = node->Head().ChildrenList(); - for (auto& child : children) { - child = ctx.NewCallable(node->Pos(), node->Content(), {std::move(child), node->TailPtr()}); + YQL_CLOG(DEBUG, Core) << "PropagateCoalesceWithConst over " << node->Head().Content(); + auto children = node->Head().ChildrenList(); + for (auto& child : children) { + child = ctx.NewCallable(node->Pos(), node->Content(), {std::move(child), node->TailPtr()}); } - return ctx.NewCallable(node->Head().Pos(), node->Head().Content(), std::move(children)); + return ctx.NewCallable(node->Head().Pos(), node->Head().Content(), std::move(children)); } return node; @@ -1029,22 +1029,22 @@ TExprNode::TPtr OptimizeEquality(const TExprNode::TPtr& node, TExprContext& ctx) return MakeBoolNothing(node->Pos(), ctx); } - if (node->Head().IsCallable("Just")) { - TCoJust just(node->HeadPtr()); - if (IsDataType(just.Input().Ref())) { - YQL_CLOG(DEBUG, Core) << "Compare '" << node->Content() << "' over Just"; - auto ret = ctx.ChangeChild(*node, 0U, just.Input().Ptr()); - return ctx.WrapByCallableIf(IsDataType(node->Tail()), "Just", std::move(ret)); - } + if (node->Head().IsCallable("Just")) { + TCoJust just(node->HeadPtr()); + if (IsDataType(just.Input().Ref())) { + YQL_CLOG(DEBUG, Core) << "Compare '" << node->Content() << "' over Just"; + auto ret = ctx.ChangeChild(*node, 0U, just.Input().Ptr()); + return ctx.WrapByCallableIf(IsDataType(node->Tail()), "Just", std::move(ret)); + } } - if (node->Tail().IsCallable("Just")) { - TCoJust just(node->TailPtr()); - if (IsDataType(just.Input().Ref())) { - YQL_CLOG(DEBUG, Core) << "Compare '" << node->Content() << "' over Just"; - auto ret = ctx.ChangeChild(*node, 1U, just.Input().Ptr()); - return ctx.WrapByCallableIf(IsDataType(node->Head()), "Just", std::move(ret)); - } + if (node->Tail().IsCallable("Just")) { + TCoJust just(node->TailPtr()); + if (IsDataType(just.Input().Ref())) { + YQL_CLOG(DEBUG, Core) << "Compare '" << node->Content() << "' over Just"; + auto ret = ctx.ChangeChild(*node, 1U, just.Input().Ptr()); + return ctx.WrapByCallableIf(IsDataType(node->Head()), "Just", std::move(ret)); + } } if (IsBoolType(*node) || IsOptBoolType(*node)) { @@ -1322,7 +1322,7 @@ TExprNode::TPtr ConvertMapToFlatmap(TMapType map, TExprContext& ctx) { } template <typename TFilterType, typename TFlatMapType> -TExprNode::TPtr ConvertFilterToFlatmap(TFilterType filter, TExprContext& ctx, TOptimizeContext& optCtx) { +TExprNode::TPtr ConvertFilterToFlatmap(TFilterType filter, TExprContext& ctx, TOptimizeContext& optCtx) { const auto& list = filter.Input(); const auto& lambda = filter.Lambda(); @@ -1350,7 +1350,7 @@ TExprNode::TPtr ConvertFilterToFlatmap(TFilterType filter, TExprContext& ctx, TO .Build() .Build() .Done(); - return KeepColumnOrder(ret.Ptr(), filter.Ref(), ctx, *optCtx.Types); + return KeepColumnOrder(ret.Ptr(), filter.Ref(), ctx, *optCtx.Types); } TExprNode::TPtr ExtractPredicateFromFlatmapOverListIf(const TExprNode& node, TExprContext& ctx) { @@ -1384,78 +1384,78 @@ TExprNode::TPtr ExtractPredicateFromFlatmapOverFlatListIf(const TExprNode& node, }); } -TExprNode::TPtr FuseJustOrSingleAsListWithFlatmap(const TExprNode::TPtr& node, TExprContext& ctx) { - // input F L S O - // lambda L F L S L - // lambda S F L S S - // lambda O F L S O - // lambda F F F - F - TCoFlatMapBase self(node); - const auto inputItem = self.Input().Ref().HeadPtr(); - auto result = ctx.ReplaceNode(self.Lambda().Body().Ptr(), self.Lambda().Args().Arg(0).Ref(), inputItem); - if (self.Input().Maybe<TCoJust>()) { - // output type is the same as lambda return type - return result; - } - - const auto lambdaReturnKind = self.Lambda().Ref().GetTypeAnn()->GetKind(); - switch (lambdaReturnKind) { - case ETypeAnnotationKind::List: - case ETypeAnnotationKind::Flow: - // output type is the same as lambda return type - break; - case ETypeAnnotationKind::Optional: - result = ctx.NewCallable(result->Pos(), "ToList", { result }); - break; - default: - YQL_ENSURE(lambdaReturnKind == ETypeAnnotationKind::Stream); - // we can safely use ForwardList here since lambda can not yield - result = ctx.NewCallable(result->Pos(), "ForwardList", { result }); - } - - return result; +TExprNode::TPtr FuseJustOrSingleAsListWithFlatmap(const TExprNode::TPtr& node, TExprContext& ctx) { + // input F L S O + // lambda L F L S L + // lambda S F L S S + // lambda O F L S O + // lambda F F F - F + TCoFlatMapBase self(node); + const auto inputItem = self.Input().Ref().HeadPtr(); + auto result = ctx.ReplaceNode(self.Lambda().Body().Ptr(), self.Lambda().Args().Arg(0).Ref(), inputItem); + if (self.Input().Maybe<TCoJust>()) { + // output type is the same as lambda return type + return result; + } + + const auto lambdaReturnKind = self.Lambda().Ref().GetTypeAnn()->GetKind(); + switch (lambdaReturnKind) { + case ETypeAnnotationKind::List: + case ETypeAnnotationKind::Flow: + // output type is the same as lambda return type + break; + case ETypeAnnotationKind::Optional: + result = ctx.NewCallable(result->Pos(), "ToList", { result }); + break; + default: + YQL_ENSURE(lambdaReturnKind == ETypeAnnotationKind::Stream); + // we can safely use ForwardList here since lambda can not yield + result = ctx.NewCallable(result->Pos(), "ForwardList", { result }); + } + + return result; } -TExprNode::TPtr FuseToListWithFlatmap(const TExprNode::TPtr& node, TExprContext& ctx) { - TCoFlatMapBase self(node); - const auto inputItem = self.Input().Ref().HeadPtr(); - YQL_ENSURE(inputItem->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional); - - const auto lambdaReturnKind = self.Lambda().Ref().GetTypeAnn()->GetKind(); - auto result = ctx.ChangeChild(*node, 0U, TExprNode::TPtr(inputItem)); - if (lambdaReturnKind == ETypeAnnotationKind::Optional) { - result = ctx.NewCallable(result->Pos(), "ToList", { result }); - } else if (lambdaReturnKind == ETypeAnnotationKind::Stream) { - result = ctx.NewCallable(result->Pos(), "ForwardList", { result }); - } - return result; +TExprNode::TPtr FuseToListWithFlatmap(const TExprNode::TPtr& node, TExprContext& ctx) { + TCoFlatMapBase self(node); + const auto inputItem = self.Input().Ref().HeadPtr(); + YQL_ENSURE(inputItem->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional); + + const auto lambdaReturnKind = self.Lambda().Ref().GetTypeAnn()->GetKind(); + auto result = ctx.ChangeChild(*node, 0U, TExprNode::TPtr(inputItem)); + if (lambdaReturnKind == ETypeAnnotationKind::Optional) { + result = ctx.NewCallable(result->Pos(), "ToList", { result }); + } else if (lambdaReturnKind == ETypeAnnotationKind::Stream) { + result = ctx.NewCallable(result->Pos(), "ForwardList", { result }); + } + return result; } bool ShouldConvertSqlInToJoin(const TCoSqlIn& sqlIn, bool /* negated */) { bool tableSource = false; - + for (const auto& hint : sqlIn.Options()) { if (hint.Name().Value() == TStringBuf("isCompact")) { return false; - } + } if (hint.Name().Value() == TStringBuf("tableSource")) { tableSource = true; } - } - + } + return tableSource; -} - +} + bool CanConvertSqlInToJoin(const TCoSqlIn& sqlIn) { auto leftArg = sqlIn.Lookup(); auto leftColumnType = leftArg.Ref().GetTypeAnn(); - + auto rightArg = sqlIn.Collection(); auto rightArgType = rightArg.Ref().GetTypeAnn(); - + if (rightArgType->GetKind() == ETypeAnnotationKind::List) { auto rightListItemType = rightArgType->Cast<TListExprType>()->GetItemType(); - + auto isDataOrTupleOfData = [](const TTypeAnnotationNode* type) { if (IsDataOrOptionalOfData(type)) { return true; @@ -1493,241 +1493,241 @@ bool CanConvertSqlInToJoin(const TCoSqlIn& sqlIn) { } return false; -} - -struct TPredicateChainNode { - TExprNode::TPtr Pred; - - bool Negated = false; - bool ConvertibleToJoin = false; - - // extra predicates due to NOT IN + nulls - TExprNode::TPtr ExtraLeftPred; - TExprNode::TPtr ExtraRightPred; - - // SqlIn params - TPositionHandle SqlInPos; +} + +struct TPredicateChainNode { + TExprNode::TPtr Pred; + + bool Negated = false; + bool ConvertibleToJoin = false; + + // extra predicates due to NOT IN + nulls + TExprNode::TPtr ExtraLeftPred; + TExprNode::TPtr ExtraRightPred; + + // SqlIn params + TPositionHandle SqlInPos; TExprNode::TPtr Left; // used only if LeftArgColumns is empty - TExprNode::TPtr Right; - + TExprNode::TPtr Right; + TVector<TStringBuf> LeftArgColumns; // set if left side of IN is input column reference or tuple of columns references - TVector<TString> RightArgColumns; // always set -}; - -using TPredicateChain = TVector<TPredicateChainNode>; - -void SplitSqlInCollection(const TCoSqlIn& sqlIn, TExprNode::TPtr& collectionNoNulls, - TExprNode::TPtr& collectionNulls, TExprContext& ctx) -{ - auto collection = sqlIn.Collection().Ptr(); - const bool isTableSource = HasSetting(sqlIn.Options().Ref(), "tableSource"); - - auto collectionItemExtractorLambda = ctx.Builder(collection->Pos()) - .Lambda() - .Param("listItem") - .Arg("listItem") - .Seal() - .Build(); - - TExprNode::TPtr collectionAsList = collection; - auto collectionKind = collection->GetTypeAnn()->GetKind(); - if (collectionKind == ETypeAnnotationKind::Dict) { - collectionAsList = ctx.Builder(collection->Pos()) - .Callable("DictKeys") - .Add(0, collectionAsList) - .Seal() - .Build(); - } else { - YQL_ENSURE(collectionKind == ETypeAnnotationKind::List, - "Unexpected collection type: " << *collection->GetTypeAnn()); - if (isTableSource) { - auto listItemType = collection->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - YQL_ENSURE(listItemType->GetKind() == ETypeAnnotationKind::Struct); - - auto structType = listItemType->Cast<TStructExprType>(); - YQL_ENSURE(structType->GetSize() == 1); - TStringBuf memberName = structType->GetItems()[0]->GetName(); - - collectionItemExtractorLambda = ctx.Builder(collection->Pos()) - .Lambda() - .Param("listItem") - .Callable("Member") - .Arg(0, "listItem") - .Atom(1, memberName) - .Seal() - .Seal() - .Build(); - } - } - - auto buildFilter = [&](bool nulls) { - return ctx.Builder(collection->Pos()) - .Callable("OrderedFilter") - .Add(0, collectionAsList) - .Lambda(1) - .Param("listItem") - .Callable("If") - .Callable(0, "Exists") - .Apply(0, collectionItemExtractorLambda) - .With(0, "listItem") - .Seal() - .Seal() - .Add(1, MakeBool(collection->Pos(), !nulls, ctx)) - .Add(2, MakeBool(collection->Pos(), nulls, ctx)) - .Seal() - .Seal() - .Seal() - .Build(); - }; - - collectionNoNulls = buildFilter(false); - collectionNulls = buildFilter(true); -} - -TExprNode::TPtr BuildCollectionEmptyPred(TPositionHandle pos, const TExprNode::TPtr& collectionAsList, TExprContext& ctx) { - return ctx.Builder(pos) - .Callable("Not") - .Callable(0, "HasItems") - .Callable(0, "Take") - .Add(0, collectionAsList) - .Callable(1, "Uint64") - .Atom(0, "1", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildSqlInCollectionEmptyPred(const TCoSqlIn& sqlIn, TExprContext& ctx) { - auto collection = sqlIn.Collection().Ptr(); - const auto collectionType = sqlIn.Collection().Ref().GetTypeAnn(); - - TExprNode::TPtr collectionEmptyPred; - switch (collectionType->GetKind()) { - case ETypeAnnotationKind::Tuple: - collectionEmptyPred = MakeBool(sqlIn.Pos(), collectionType->Cast<TTupleExprType>()->GetSize() == 0, ctx); - break; - case ETypeAnnotationKind::Dict: - collection = ctx.Builder(sqlIn.Pos()) - .Callable("DictKeys") - .Add(0, collection) - .Seal() - .Build(); + TVector<TString> RightArgColumns; // always set +}; + +using TPredicateChain = TVector<TPredicateChainNode>; + +void SplitSqlInCollection(const TCoSqlIn& sqlIn, TExprNode::TPtr& collectionNoNulls, + TExprNode::TPtr& collectionNulls, TExprContext& ctx) +{ + auto collection = sqlIn.Collection().Ptr(); + const bool isTableSource = HasSetting(sqlIn.Options().Ref(), "tableSource"); + + auto collectionItemExtractorLambda = ctx.Builder(collection->Pos()) + .Lambda() + .Param("listItem") + .Arg("listItem") + .Seal() + .Build(); + + TExprNode::TPtr collectionAsList = collection; + auto collectionKind = collection->GetTypeAnn()->GetKind(); + if (collectionKind == ETypeAnnotationKind::Dict) { + collectionAsList = ctx.Builder(collection->Pos()) + .Callable("DictKeys") + .Add(0, collectionAsList) + .Seal() + .Build(); + } else { + YQL_ENSURE(collectionKind == ETypeAnnotationKind::List, + "Unexpected collection type: " << *collection->GetTypeAnn()); + if (isTableSource) { + auto listItemType = collection->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + YQL_ENSURE(listItemType->GetKind() == ETypeAnnotationKind::Struct); + + auto structType = listItemType->Cast<TStructExprType>(); + YQL_ENSURE(structType->GetSize() == 1); + TStringBuf memberName = structType->GetItems()[0]->GetName(); + + collectionItemExtractorLambda = ctx.Builder(collection->Pos()) + .Lambda() + .Param("listItem") + .Callable("Member") + .Arg(0, "listItem") + .Atom(1, memberName) + .Seal() + .Seal() + .Build(); + } + } + + auto buildFilter = [&](bool nulls) { + return ctx.Builder(collection->Pos()) + .Callable("OrderedFilter") + .Add(0, collectionAsList) + .Lambda(1) + .Param("listItem") + .Callable("If") + .Callable(0, "Exists") + .Apply(0, collectionItemExtractorLambda) + .With(0, "listItem") + .Seal() + .Seal() + .Add(1, MakeBool(collection->Pos(), !nulls, ctx)) + .Add(2, MakeBool(collection->Pos(), nulls, ctx)) + .Seal() + .Seal() + .Seal() + .Build(); + }; + + collectionNoNulls = buildFilter(false); + collectionNulls = buildFilter(true); +} + +TExprNode::TPtr BuildCollectionEmptyPred(TPositionHandle pos, const TExprNode::TPtr& collectionAsList, TExprContext& ctx) { + return ctx.Builder(pos) + .Callable("Not") + .Callable(0, "HasItems") + .Callable(0, "Take") + .Add(0, collectionAsList) + .Callable(1, "Uint64") + .Atom(0, "1", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildSqlInCollectionEmptyPred(const TCoSqlIn& sqlIn, TExprContext& ctx) { + auto collection = sqlIn.Collection().Ptr(); + const auto collectionType = sqlIn.Collection().Ref().GetTypeAnn(); + + TExprNode::TPtr collectionEmptyPred; + switch (collectionType->GetKind()) { + case ETypeAnnotationKind::Tuple: + collectionEmptyPred = MakeBool(sqlIn.Pos(), collectionType->Cast<TTupleExprType>()->GetSize() == 0, ctx); + break; + case ETypeAnnotationKind::Dict: + collection = ctx.Builder(sqlIn.Pos()) + .Callable("DictKeys") + .Add(0, collection) + .Seal() + .Build(); [[fallthrough]]; - case ETypeAnnotationKind::List: - collectionEmptyPred = BuildCollectionEmptyPred(sqlIn.Pos(), collection, ctx); - break; - default: - YQL_ENSURE(false, "Unexpected collection type: " << *collectionType); - } - return collectionEmptyPred; -} - + case ETypeAnnotationKind::List: + collectionEmptyPred = BuildCollectionEmptyPred(sqlIn.Pos(), collection, ctx); + break; + default: + YQL_ENSURE(false, "Unexpected collection type: " << *collectionType); + } + return collectionEmptyPred; +} + TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, const TExprNode::TPtr& topLambdaArg, std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TExprContext& ctx) { - TPredicateChainNode result; - + TPredicateChainNode result; + result.Pred = predicate; - + auto curr = predicate; - TExprNode::TPtr pred; - if (curr->IsCallable("Not")) { + TExprNode::TPtr pred; + if (curr->IsCallable("Not")) { curr = curr->HeadPtr(); - result.Negated = true; - } - - TExprNode::TPtr leftArg; - bool hasCoalesce = false; - if (curr->IsCallable("SqlIn")) { - leftArg = curr->ChildPtr(1); - } else if (curr->IsCallable("Coalesce") && + result.Negated = true; + } + + TExprNode::TPtr leftArg; + bool hasCoalesce = false; + if (curr->IsCallable("SqlIn")) { + leftArg = curr->ChildPtr(1); + } else if (curr->IsCallable("Coalesce") && curr->Head().IsCallable("SqlIn") && - curr->Child(1)->IsCallable("Bool")) { + curr->Child(1)->IsCallable("Bool")) { bool coalesceVal = FromString<bool>(curr->Child(1)->Head().Content()); - if (coalesceVal == result.Negated) { + if (coalesceVal == result.Negated) { curr = curr->HeadPtr(); - leftArg = curr->ChildPtr(1); - } - hasCoalesce = true; - } - + leftArg = curr->ChildPtr(1); + } + hasCoalesce = true; + } + if (!leftArg) { // not SqlIn - return result; - } - + return result; + } + TCoSqlIn sqlIn(curr); if (!shouldConvertSqlInToJoin(sqlIn, result.Negated) || !CanConvertSqlInToJoin(sqlIn)) { // not convertible to join return result; } - result.SqlInPos = sqlIn.Pos(); - result.ConvertibleToJoin = true; - result.Left = leftArg; - - if (result.Negated && HasSetting(sqlIn.Options().Ref(), "ansi")) { - const bool nullsProcessed = HasSetting(sqlIn.Options().Ref(), "nullsProcessed"); - const bool lookupIsOptional = sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; - const bool collectionItemsNullable = IsSqlInCollectionItemsNullable(sqlIn); - if (!nullsProcessed && (collectionItemsNullable || lookupIsOptional)) { - YQL_ENSURE(sqlIn.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional); - YQL_ENSURE(hasCoalesce); - - // need to add nullsProcessed setting - result.Pred = nullptr; - - auto rebuildMainPred = [&sqlIn, &ctx](const TExprNode::TPtr& collection) { - return ctx.Builder(sqlIn.Pos()) - .Callable("Not") - .Callable(0, "Coalesce") - .Callable(0, "SqlIn") - .Add(0, collection) - .Add(1, sqlIn.Lookup().Ptr()) - .Add(2, AddSetting(sqlIn.Options().Ref(), sqlIn.Options().Pos(), "nullsProcessed", nullptr, ctx)) - .Seal() - .Add(1, MakeBool(sqlIn.Pos(), true, ctx)) - .Seal() - .Seal() - .Build(); - }; - - if (collectionItemsNullable) { - TExprNode::TPtr collectionNoNulls; - TExprNode::TPtr collectionNulls; - SplitSqlInCollection(sqlIn, collectionNoNulls, collectionNulls, ctx); - - result.ExtraRightPred = BuildCollectionEmptyPred(sqlIn.Pos(), collectionNulls, ctx); - result.Pred = rebuildMainPred(collectionNoNulls); - } - - if (lookupIsOptional) { - result.ExtraLeftPred = ctx.Builder(sqlIn.Pos()) - .Callable("Or") - .Callable(0, "Exists") - .Add(0, sqlIn.Lookup().Ptr()) - .Seal() - .Add(1, BuildSqlInCollectionEmptyPred(sqlIn, ctx)) - .Seal() - .Build(); - } - - if (!result.Pred) { - result.Pred = rebuildMainPred(sqlIn.Collection().Ptr()); - } - - return result; - } - } - + result.SqlInPos = sqlIn.Pos(); + result.ConvertibleToJoin = true; + result.Left = leftArg; + + if (result.Negated && HasSetting(sqlIn.Options().Ref(), "ansi")) { + const bool nullsProcessed = HasSetting(sqlIn.Options().Ref(), "nullsProcessed"); + const bool lookupIsOptional = sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; + const bool collectionItemsNullable = IsSqlInCollectionItemsNullable(sqlIn); + if (!nullsProcessed && (collectionItemsNullable || lookupIsOptional)) { + YQL_ENSURE(sqlIn.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional); + YQL_ENSURE(hasCoalesce); + + // need to add nullsProcessed setting + result.Pred = nullptr; + + auto rebuildMainPred = [&sqlIn, &ctx](const TExprNode::TPtr& collection) { + return ctx.Builder(sqlIn.Pos()) + .Callable("Not") + .Callable(0, "Coalesce") + .Callable(0, "SqlIn") + .Add(0, collection) + .Add(1, sqlIn.Lookup().Ptr()) + .Add(2, AddSetting(sqlIn.Options().Ref(), sqlIn.Options().Pos(), "nullsProcessed", nullptr, ctx)) + .Seal() + .Add(1, MakeBool(sqlIn.Pos(), true, ctx)) + .Seal() + .Seal() + .Build(); + }; + + if (collectionItemsNullable) { + TExprNode::TPtr collectionNoNulls; + TExprNode::TPtr collectionNulls; + SplitSqlInCollection(sqlIn, collectionNoNulls, collectionNulls, ctx); + + result.ExtraRightPred = BuildCollectionEmptyPred(sqlIn.Pos(), collectionNulls, ctx); + result.Pred = rebuildMainPred(collectionNoNulls); + } + + if (lookupIsOptional) { + result.ExtraLeftPred = ctx.Builder(sqlIn.Pos()) + .Callable("Or") + .Callable(0, "Exists") + .Add(0, sqlIn.Lookup().Ptr()) + .Seal() + .Add(1, BuildSqlInCollectionEmptyPred(sqlIn, ctx)) + .Seal() + .Build(); + } + + if (!result.Pred) { + result.Pred = rebuildMainPred(sqlIn.Collection().Ptr()); + } + + return result; + } + } + auto isMemberOf = [](const TExprNode::TPtr& node, const TExprNode::TPtr& arg) { return node->IsCallable("Member") && node->HeadPtr() == arg; }; if (isMemberOf(leftArg, topLambdaArg)) { - // left side of IN is column reference + // left side of IN is column reference result.LeftArgColumns.emplace_back(leftArg->Child(1)->Content()); } else if (leftArg->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) { // if leftArg is tuple of members then replace tuple with its members @@ -1740,14 +1740,14 @@ TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, co break; } } - } - + } + auto rightArg = sqlIn.Collection().Ptr(); auto rightArgType = rightArg->GetTypeAnn(); - + if (rightArgType->GetKind() == ETypeAnnotationKind::List) { auto rightArgItemType = rightArgType->Cast<TListExprType>()->GetItemType(); - + if (rightArgItemType->GetKind() == ETypeAnnotationKind::Struct) { auto rightStructType = rightArgItemType->Cast<TStructExprType>(); YQL_ENSURE(rightStructType->GetSize() == 1); @@ -1884,102 +1884,102 @@ TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, co .Ptr(); result.RightArgColumns = { "key" }; - return result; -} - + return result; +} + TExprNode::TPtr SplitPredicateChain(TExprNode::TPtr&& node, const TExprNode::TPtr& topLambdaArg, std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TPredicateChain& prefix, TExprContext& ctx) { - if (!node->IsCallable("And")) { + if (!node->IsCallable("And")) { TPredicateChainNode curr = ParsePredicateChainNode(node, topLambdaArg, shouldConvertSqlInToJoin, ctx); - if (!prefix.empty() && prefix.back().ConvertibleToJoin != curr.ConvertibleToJoin) { - // stop splitting + if (!prefix.empty() && prefix.back().ConvertibleToJoin != curr.ConvertibleToJoin) { + // stop splitting return std::move(node); - } - + } + prefix.emplace_back(curr); - return {}; - } - + return {}; + } + auto children = node->ChildrenList(); for (auto& child : children) { - child = SplitPredicateChain(std::move(child), topLambdaArg, shouldConvertSqlInToJoin, prefix, ctx); - if (child) { + child = SplitPredicateChain(std::move(child), topLambdaArg, shouldConvertSqlInToJoin, prefix, ctx); + if (child) { break; } } if (children.front().Get() == &node->Head()) { return std::move(node); - } - + } + children.erase(std::remove_if(children.begin(), children.end(), std::logical_not<TExprNode::TPtr>()), children.end()); - + if (children.empty()) { return {}; - } + } return 1U == children.size() ? std::move(children.front()) : ctx.ChangeChildren(*node, std::move(children)); -} - -TExprNode::TPtr RebuildFlatmapOverPartOfPredicate(const TExprNode::TPtr& origFlatMap, const TExprNode::TPtr& input, - const TExprNode::TPtr& pred, bool isOuter, TExprContext& ctx) -{ +} + +TExprNode::TPtr RebuildFlatmapOverPartOfPredicate(const TExprNode::TPtr& origFlatMap, const TExprNode::TPtr& input, + const TExprNode::TPtr& pred, bool isOuter, TExprContext& ctx) +{ auto origLambdaArgs = origFlatMap->Child(1)->HeadPtr(); - TCoConditionalValueBase origConditional(origFlatMap->Child(1)->TailPtr()); - auto newLambdaBody = isOuter ? - ctx.ChangeChild(origConditional.Ref(), TCoConditionalValueBase::idx_Predicate, TExprNode::TPtr(pred)) : - ctx.NewCallable(origFlatMap->Pos(), "OptionalIf", {pred, origLambdaArgs->HeadPtr()}); - - bool isOrdered = origFlatMap->IsCallable({"OrderedFlatMap", "OrderedFlatMapToEquiJoin"}); - auto resultingName = isOrdered ? "OrderedFlatMap" : "FlatMap"; - - return ctx.Builder(origFlatMap->Pos()) - .Callable(resultingName) - .Add(0, input) - .Lambda(1) - .Param("item") - .ApplyPartial(origLambdaArgs, newLambdaBody) - .With(0, "item") - .Seal() - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, const TPredicateChain& chain, TExprContext& ctx) { - YQL_ENSURE(!chain.empty()); - + TCoConditionalValueBase origConditional(origFlatMap->Child(1)->TailPtr()); + auto newLambdaBody = isOuter ? + ctx.ChangeChild(origConditional.Ref(), TCoConditionalValueBase::idx_Predicate, TExprNode::TPtr(pred)) : + ctx.NewCallable(origFlatMap->Pos(), "OptionalIf", {pred, origLambdaArgs->HeadPtr()}); + + bool isOrdered = origFlatMap->IsCallable({"OrderedFlatMap", "OrderedFlatMapToEquiJoin"}); + auto resultingName = isOrdered ? "OrderedFlatMap" : "FlatMap"; + + return ctx.Builder(origFlatMap->Pos()) + .Callable(resultingName) + .Add(0, input) + .Lambda(1) + .Param("item") + .ApplyPartial(origLambdaArgs, newLambdaBody) + .With(0, "item") + .Seal() + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, const TPredicateChain& chain, TExprContext& ctx) { + YQL_ENSURE(!chain.empty()); + auto input = flatMapNode->HeadPtr(); - bool isOrdered = flatMapNode->IsCallable({"OrderedFlatMap", "OrderedFlatMapToEquiJoin"}); + bool isOrdered = flatMapNode->IsCallable({"OrderedFlatMap", "OrderedFlatMapToEquiJoin"}); auto origLambdaArgs = flatMapNode->Child(1)->HeadPtr(); - - // placeholder for input table - TExprNode::TListType equiJoinArgs(1); - equiJoinArgs.reserve(chain.size() + 3); - - TExprNode::TPtr joinChain; - TExprNode::TPtr addMemberChain; - TExprNode::TListType renames; - - static const TStringBuf inputTable = "_yql_injoin_input"; - auto inputTableAtom = ctx.NewAtom(input->Pos(), inputTable); - - for (size_t i = 0; i < chain.size(); ++i) { + + // placeholder for input table + TExprNode::TListType equiJoinArgs(1); + equiJoinArgs.reserve(chain.size() + 3); + + TExprNode::TPtr joinChain; + TExprNode::TPtr addMemberChain; + TExprNode::TListType renames; + + static const TStringBuf inputTable = "_yql_injoin_input"; + auto inputTableAtom = ctx.NewAtom(input->Pos(), inputTable); + + for (size_t i = 0; i < chain.size(); ++i) { const TString tableName = TStringBuilder() << "_yql_injoin_" << i; const TString columnName = TStringBuilder() << "_yql_injoin_column_" << i; const auto pos = chain[i].SqlInPos; - + auto equiJoinArg = ctx.Builder(pos) - .List() - .Add(0, chain[i].Right) - .Atom(1, tableName) - .Seal() - .Build(); - - equiJoinArgs.push_back(equiJoinArg); - + .List() + .Add(0, chain[i].Right) + .Atom(1, tableName) + .Seal() + .Build(); + + equiJoinArgs.push_back(equiJoinArg); + TExprNodeList leftKeys; if (chain[i].LeftArgColumns.empty()) { leftKeys.push_back(inputTableAtom); @@ -1998,88 +1998,88 @@ TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, c } joinChain = ctx.Builder(pos) - .List() - .Atom(0, chain[i].Negated ? "LeftOnly" : "LeftSemi") - .Add(1, joinChain ? joinChain : inputTableAtom) - .Atom(2, tableName) - .List(3) + .List() + .Atom(0, chain[i].Negated ? "LeftOnly" : "LeftSemi") + .Add(1, joinChain ? joinChain : inputTableAtom) + .Atom(2, tableName) + .List(3) .Add(std::move(leftKeys)) - .Seal() - .List(4) + .Seal() + .List(4) .Add(std::move(rightKeys)) - .Seal() - .List(5) - .Seal() - .Seal() - .Build(); - + .Seal() + .List(5) + .Seal() + .Seal() + .Build(); + if (chain[i].LeftArgColumns.empty()) { auto rename = ctx.Builder(pos) - .List() - .Atom(0, "rename") - .Atom(1, FullColumnName(inputTable, columnName)) - .Atom(2, "") - .Seal() - .Build(); - renames.push_back(rename); - - addMemberChain = ctx.Builder(chain[i].SqlInPos) - .Callable("AddMember") + .List() + .Atom(0, "rename") + .Atom(1, FullColumnName(inputTable, columnName)) + .Atom(2, "") + .Seal() + .Build(); + renames.push_back(rename); + + addMemberChain = ctx.Builder(chain[i].SqlInPos) + .Callable("AddMember") .Add(0, addMemberChain ? addMemberChain : origLambdaArgs->HeadPtr()) - .Atom(1, columnName) - .Add(2, chain[i].Left) - .Seal() - .Build(); - } - } - + .Atom(1, columnName) + .Add(2, chain[i].Left) + .Seal() + .Build(); + } + } + YQL_ENSURE(input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List); auto inputRowType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - YQL_ENSURE(inputRowType->GetKind() == ETypeAnnotationKind::Struct); - - for (const auto& i : inputRowType->Cast<TStructExprType>()->GetItems()) { - auto rename = ctx.Builder(input->Pos()) - .List() - .Atom(0, "rename") - .Atom(1, FullColumnName(inputTable, i->GetName())) - .Atom(2, i->GetName()) - .Seal() - .Build(); - renames.push_back(rename); - } + YQL_ENSURE(inputRowType->GetKind() == ETypeAnnotationKind::Struct); + + for (const auto& i : inputRowType->Cast<TStructExprType>()->GetItems()) { + auto rename = ctx.Builder(input->Pos()) + .List() + .Atom(0, "rename") + .Atom(1, FullColumnName(inputTable, i->GetName())) + .Atom(2, i->GetName()) + .Seal() + .Build(); + renames.push_back(rename); + } renames.push_back(ctx.Builder(input->Pos()) .List() .Atom(0, "keep_sys") .Seal() .Build()); - - equiJoinArgs.push_back(joinChain); - equiJoinArgs.push_back(ctx.NewList(input->Pos(), std::move(renames))); - - if (addMemberChain) { - input = ctx.Builder(input->Pos()) - .Callable(isOrdered ? "OrderedMap" : "Map") - .Add(0, input) - .Lambda(1) - .Param("item") - .ApplyPartial(origLambdaArgs, addMemberChain) - .With(0, "item") - .Seal() - .Seal() - .Seal() - .Build(); - } - - equiJoinArgs[0] = ctx.Builder(input->Pos()) - .List() - .Add(0, input) - .Add(1, inputTableAtom) - .Seal() - .Build(); - - return ctx.NewCallable(input->Pos(), "EquiJoin", std::move(equiJoinArgs)); -} - + + equiJoinArgs.push_back(joinChain); + equiJoinArgs.push_back(ctx.NewList(input->Pos(), std::move(renames))); + + if (addMemberChain) { + input = ctx.Builder(input->Pos()) + .Callable(isOrdered ? "OrderedMap" : "Map") + .Add(0, input) + .Lambda(1) + .Param("item") + .ApplyPartial(origLambdaArgs, addMemberChain) + .With(0, "item") + .Seal() + .Seal() + .Seal() + .Build(); + } + + equiJoinArgs[0] = ctx.Builder(input->Pos()) + .List() + .Add(0, input) + .Add(1, inputTableAtom) + .Seal() + .Build(); + + return ctx.NewCallable(input->Pos(), "EquiJoin", std::move(equiJoinArgs)); +} + TStringBuf GetEmptyCollectionName(ETypeAnnotationKind kind) { switch (kind) { case ETypeAnnotationKind::Flow: @@ -2124,12 +2124,12 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO if (IsJustOrSingleAsList(node->Head()) && !lambdaArg.IsUsedInDependsOn()) { YQL_CLOG(DEBUG, Core) << "Fuse " << node->Content() << " over " << node->Head().Content(); - return FuseJustOrSingleAsListWithFlatmap(node, ctx); + return FuseJustOrSingleAsListWithFlatmap(node, ctx); } if (node->Head().IsCallable("ToList")) { YQL_CLOG(DEBUG, Core) << "Fuse " << node->Content() << " over " << node->Head().Content(); - return FuseToListWithFlatmap(node, ctx); + return FuseToListWithFlatmap(node, ctx); } if (node->Head().IsCallable("FromFlow")) { @@ -2142,68 +2142,68 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO } } - if (lambdaBody.IsCallable("AsList") && lambdaBody.ChildrenSize() == 1 && - node->Head().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional) - { - YQL_CLOG(DEBUG, Core) << node->Content() << " with single arg AsList"; - auto newLambda = ctx.ChangeChild(self.Lambda().Ref(), 1U, ctx.RenameNode(lambdaBody, "Just")); - return ctx.ChangeChild(*node, 1U, ctx.DeepCopyLambda(*newLambda)); - } - - if (IsJustOrSingleAsList(lambdaBody)) { - const bool isIdentical = &lambdaBody.Head() == &lambdaArg; - const auto type = lambdaArg.GetTypeAnn(); - const bool sameType = IsSameAnnotation(*lambdaBody.Head().GetTypeAnn(), *type); - const bool toList = self.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List - && self.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; + if (lambdaBody.IsCallable("AsList") && lambdaBody.ChildrenSize() == 1 && + node->Head().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional) + { + YQL_CLOG(DEBUG, Core) << node->Content() << " with single arg AsList"; + auto newLambda = ctx.ChangeChild(self.Lambda().Ref(), 1U, ctx.RenameNode(lambdaBody, "Just")); + return ctx.ChangeChild(*node, 1U, ctx.DeepCopyLambda(*newLambda)); + } + + if (IsJustOrSingleAsList(lambdaBody)) { + const bool isIdentical = &lambdaBody.Head() == &lambdaArg; + const auto type = lambdaArg.GetTypeAnn(); + const bool sameType = IsSameAnnotation(*lambdaBody.Head().GetTypeAnn(), *type); + const bool toList = self.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List + && self.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; if (isIdentical || (sameType && type->IsSingleton())) { - YQL_CLOG(DEBUG, Core) << node->Content() << " with " << lambdaBody.Content(); - return ctx.WrapByCallableIf(toList, "ToList", self.Input().Ptr()); - } - - auto maybeStruct = TMaybeNode<TCoAsStruct>(lambdaBody.ChildPtr(0)); - if (maybeStruct && type->GetKind() == ETypeAnnotationKind::Struct ) { - bool replaceByExtractMembers = true; - TMap<TStringBuf, TPositionHandle> membersToExtract; - - for (auto child : maybeStruct.Cast()) { - auto tuple = child.Cast<TCoNameValueTuple>(); - auto value = tuple.Value(); - - if (auto maybeMember = value.Maybe<TCoMember>()) { - auto member = maybeMember.Cast(); - if (member.Struct().Raw() == &lambdaArg) { - TStringBuf inputName = member.Name().Value(); - TStringBuf outputName = tuple.Name().Value(); - if (inputName == outputName) { - membersToExtract[inputName] = member.Name().Pos(); - continue; - } - } - } - replaceByExtractMembers = false; - break; - } - - if (replaceByExtractMembers) { - TExprNodeList members; - members.reserve(membersToExtract.size()); - for (const auto& m : membersToExtract) { - members.push_back(ctx.NewAtom(m.second, m.first)); - } - - - auto extractMembers = ctx.Builder(node->Pos()) - .Callable("ExtractMembers") - .Add(0, self.Input().Ptr()) - .Add(1, ctx.NewList(node->Pos(), std::move(members))) - .Seal() - .Build(); - - YQL_CLOG(DEBUG, Core) << node->Content() << " to ExtractMembers"; - return ctx.WrapByCallableIf(toList, "ToList", std::move(extractMembers)); - } - } + YQL_CLOG(DEBUG, Core) << node->Content() << " with " << lambdaBody.Content(); + return ctx.WrapByCallableIf(toList, "ToList", self.Input().Ptr()); + } + + auto maybeStruct = TMaybeNode<TCoAsStruct>(lambdaBody.ChildPtr(0)); + if (maybeStruct && type->GetKind() == ETypeAnnotationKind::Struct ) { + bool replaceByExtractMembers = true; + TMap<TStringBuf, TPositionHandle> membersToExtract; + + for (auto child : maybeStruct.Cast()) { + auto tuple = child.Cast<TCoNameValueTuple>(); + auto value = tuple.Value(); + + if (auto maybeMember = value.Maybe<TCoMember>()) { + auto member = maybeMember.Cast(); + if (member.Struct().Raw() == &lambdaArg) { + TStringBuf inputName = member.Name().Value(); + TStringBuf outputName = tuple.Name().Value(); + if (inputName == outputName) { + membersToExtract[inputName] = member.Name().Pos(); + continue; + } + } + } + replaceByExtractMembers = false; + break; + } + + if (replaceByExtractMembers) { + TExprNodeList members; + members.reserve(membersToExtract.size()); + for (const auto& m : membersToExtract) { + members.push_back(ctx.NewAtom(m.second, m.first)); + } + + + auto extractMembers = ctx.Builder(node->Pos()) + .Callable("ExtractMembers") + .Add(0, self.Input().Ptr()) + .Add(1, ctx.NewList(node->Pos(), std::move(members))) + .Seal() + .Build(); + + YQL_CLOG(DEBUG, Core) << node->Content() << " to ExtractMembers"; + return ctx.WrapByCallableIf(toList, "ToList", std::move(extractMembers)); + } + } } if (CanRewriteToEmptyContainer(*node)) { @@ -2245,10 +2245,10 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO return ctx.ChangeChild(*node, 1U, std::move(newLambda)); } - if (auto expr = TryConvertSqlInPredicatesToJoins(self, ShouldConvertSqlInToJoin, ctx)) { + if (auto expr = TryConvertSqlInPredicatesToJoins(self, ShouldConvertSqlInToJoin, ctx)) { return expr; - } - + } + if (auto just = self.Lambda().Body().Maybe<TCoJust>()) { if (auto tuple = just.Cast().Input().Maybe<TExprList>()) { if (tuple.Cast().Size() > 0) { @@ -2278,7 +2278,7 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO break; } - if (inner->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() != tuple.Cast().Size()) { + if (inner->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() != tuple.Cast().Size()) { inner = nullptr; break; } @@ -2295,87 +2295,87 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO return node; } -TExprNode::TPtr HasNullOverTuple(const TExprNode::TPtr& node, TExprContext& ctx) { +TExprNode::TPtr HasNullOverTuple(const TExprNode::TPtr& node, TExprContext& ctx) { auto value = node->HeadPtr(); - - TExprNode::TListType predicates; - for (auto i : xrange(value->GetTypeAnn()->Cast<TTupleExprType>()->GetSize())) { - predicates.push_back(ctx.Builder(node->Pos()) - .Callable("HasNull") - .Callable(0, "Nth") - .Add(0, value) + + TExprNode::TListType predicates; + for (auto i : xrange(value->GetTypeAnn()->Cast<TTupleExprType>()->GetSize())) { + predicates.push_back(ctx.Builder(node->Pos()) + .Callable("HasNull") + .Callable(0, "Nth") + .Add(0, value) .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Seal() - .Build()); - } - - if (predicates.empty()) { + .Seal() + .Seal() + .Build()); + } + + if (predicates.empty()) { return MakeBool<false>(node->Pos(), ctx); - } - + } + return ctx.NewCallable(node->Pos(), "Or", std::move(predicates)); -} - -TExprNode::TPtr HasNullOverStruct(const TExprNode::TPtr& node, TExprContext& ctx) { +} + +TExprNode::TPtr HasNullOverStruct(const TExprNode::TPtr& node, TExprContext& ctx) { auto value = node->HeadPtr(); - - TExprNode::TListType predicates; - for (auto& item : value->GetTypeAnn()->Cast<TStructExprType>()->GetItems()) { - predicates.push_back(ctx.Builder(node->Pos()) - .Callable("HasNull") - .Callable(0, "Member") - .Add(0, value) - .Atom(1, item->GetName()) - .Seal() - .Seal() - .Build()); - } - - if (predicates.empty()) { + + TExprNode::TListType predicates; + for (auto& item : value->GetTypeAnn()->Cast<TStructExprType>()->GetItems()) { + predicates.push_back(ctx.Builder(node->Pos()) + .Callable("HasNull") + .Callable(0, "Member") + .Add(0, value) + .Atom(1, item->GetName()) + .Seal() + .Seal() + .Build()); + } + + if (predicates.empty()) { return MakeBool<false>(node->Pos(), ctx); - } - + } + return ctx.NewCallable(node->Pos(), "Or", std::move(predicates)); -} - -TExprNode::TPtr HasNullOverVariant(const TExprNode::TPtr& node, TExprContext& ctx) { +} + +TExprNode::TPtr HasNullOverVariant(const TExprNode::TPtr& node, TExprContext& ctx) { auto value = node->HeadPtr(); - - auto underlyingType = value->GetTypeAnn()->Cast<TVariantExprType>()->GetUnderlyingType(); - - const size_t size = underlyingType->GetKind() == ETypeAnnotationKind::Struct ? - underlyingType->Cast<TStructExprType>()->GetSize() : - underlyingType->Cast<TTupleExprType>()->GetSize(); - - return ctx.Builder(node->Pos()) - .Callable("Visit") - .Add(0, value) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (auto i : xrange(size)) { - TString index; - if (underlyingType->GetKind() == ETypeAnnotationKind::Struct) { - index = underlyingType->Cast<TStructExprType>()->GetItems()[i]->GetName(); - } else { - index = ToString(i); - } - - parent - .Atom(2 * i + 1, index) - .Lambda(2 * i + 2) - .Param("item") - .Callable("HasNull") - .Arg(0, "item") - .Seal() - .Seal(); - } - return parent; - }) - .Seal() - .Build(); - -} - + + auto underlyingType = value->GetTypeAnn()->Cast<TVariantExprType>()->GetUnderlyingType(); + + const size_t size = underlyingType->GetKind() == ETypeAnnotationKind::Struct ? + underlyingType->Cast<TStructExprType>()->GetSize() : + underlyingType->Cast<TTupleExprType>()->GetSize(); + + return ctx.Builder(node->Pos()) + .Callable("Visit") + .Add(0, value) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (auto i : xrange(size)) { + TString index; + if (underlyingType->GetKind() == ETypeAnnotationKind::Struct) { + index = underlyingType->Cast<TStructExprType>()->GetItems()[i]->GetName(); + } else { + index = ToString(i); + } + + parent + .Atom(2 * i + 1, index) + .Lambda(2 * i + 2) + .Param("item") + .Callable("HasNull") + .Arg(0, "item") + .Seal() + .Seal(); + } + return parent; + }) + .Seal() + .Build(); + +} + TExprNode::TPtr OptimizeToFlow(const TExprNode::TPtr& node, TExprContext& ctx) { if (node->Head().IsCallable("Nothing")) { YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); @@ -2682,324 +2682,324 @@ TExprNode::TPtr OptimizeReorder(const TExprNode::TPtr& node, TExprContext& ctx) return node; } -void FixSortness(const TExprNode& origNode, TExprNode::TPtr& node, TExprContext& ctx) { - if (auto sorted = origNode.GetConstraint<TSortedConstraintNode>()) { +void FixSortness(const TExprNode& origNode, TExprNode::TPtr& node, TExprContext& ctx) { + if (auto sorted = origNode.GetConstraint<TSortedConstraintNode>()) { const auto& content = sorted->GetContent(); - node = ctx.Builder(origNode.Pos()) - .Callable("Sort") - .Add(0, std::move(node)) - .List(1) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - size_t index = 0; + node = ctx.Builder(origNode.Pos()) + .Callable("Sort") + .Add(0, std::move(node)) + .List(1) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t index = 0; for (auto c : content) { - parent.Callable(index++, "Bool") + parent.Callable(index++, "Bool") .Atom(0, ToString(c.second), TNodeFlags::Default) - .Seal(); - } - return parent; - }) - .Seal() - .Lambda(2) - .Param("item") - .List() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - size_t index = 0; + .Seal(); + } + return parent; + }) + .Seal() + .Lambda(2) + .Param("item") + .List() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t index = 0; for (auto c : content) { - parent.Callable(index++, "Member") - .Arg(0, "item") + parent.Callable(index++, "Member") + .Arg(0, "item") .Atom(1, c.first.front()) - .Seal(); - } - return parent; - }) - .Seal() - .Seal() - .Seal() - .Build(); - } -} - -TExprNode::TPtr ConvertSqlInPredicatesPrefixToJoins(const TExprNode::TPtr& flatMap, const TPredicateChain& chain, - const TExprNode::TPtr& sqlInTail, TExprContext& ctx) -{ - YQL_ENSURE(!chain.empty()); - YQL_ENSURE(chain.front().ConvertibleToJoin); - - TExprNodeList extraLefts; - TExprNodeList extraRights; - for (auto& node : chain) { - if (node.ExtraLeftPred) { - extraLefts.push_back(node.ExtraLeftPred); - } - if (node.ExtraRightPred) { - extraRights.push_back(node.ExtraRightPred); - } - } - - if (!extraLefts.empty() || !extraRights.empty()) - { - TExprNodeList predicates; - predicates.reserve(extraLefts.size() + extraRights.size() + chain.size() + 1); - - predicates.insert(predicates.end(), extraLefts.begin(), extraLefts.end()); - predicates.insert(predicates.end(), extraRights.begin(), extraRights.end()); - - for (auto& node: chain) { - YQL_ENSURE(node.Pred); - predicates.push_back(node.Pred); - } - - if (sqlInTail) { - predicates.push_back(sqlInTail); - } - - YQL_CLOG(DEBUG, Core) << "FlatMapOverJoinableSqlInChain of size " << chain.size() << " with " - << extraLefts.size() << " extra left predicates and " - << extraRights.size() << " extra right predicates due to NOT IN"; - - auto combinedPred = ctx.NewCallable(predicates.front()->Pos(), "And", std::move(predicates)); - return RebuildFlatmapOverPartOfPredicate(flatMap, flatMap->HeadPtr(), combinedPred, true, ctx); - } - - YQL_CLOG(DEBUG, Core) << "FlatMapOverJoinableSqlInChain of size " << chain.size(); - - auto eq = BuildEquiJoinForSqlInChain(flatMap, chain, ctx); - FixSortness(*flatMap, eq, ctx); - - auto tail = sqlInTail ? sqlInTail : MakeBool<true>(flatMap->Pos(), ctx); - return RebuildFlatmapOverPartOfPredicate(flatMap, eq, tail, true, ctx); -} - -TExprNode::TPtr ConvertSqlInPredicatesToJoins(const TCoFlatMapToEquiJoinBase& flatMap, TExprContext& ctx) { - TCoLambda lambda = flatMap.Lambda(); - YQL_ENSURE(lambda.Body().Maybe<TCoOptionalIf>()); - - TPredicateChain chain; - auto lambdaArg = lambda.Ptr()->Head().HeadPtr(); - auto sqlInTail = SplitPredicateChain(lambda.Ptr()->Child(1)->HeadPtr(), lambdaArg, ShouldConvertSqlInToJoin, chain, ctx); - return ConvertSqlInPredicatesPrefixToJoins(flatMap.Ptr(), chain, sqlInTail, ctx); -} - -TExprNodeList DeduplicateAndSplitTupleCollectionByTypes(const TExprNode &collection, TExprContext &ctx) { - const auto& tupleItemsTypes = collection.GetTypeAnn()->Cast<TTupleExprType>()->GetItems(); - - TVector<TExprNodeList> collections; - THashMap<const TTypeAnnotationNode*, size_t> indexByType; - THashSet<const TExprNode*> uniqNodes; - - for (size_t i = 0; i < tupleItemsTypes.size(); ++i) { - auto item = collection.ChildPtr(i); - if (uniqNodes.contains(item.Get())) { - continue; - } - uniqNodes.insert(item.Get()); - - auto itemType = tupleItemsTypes[i]; - - size_t idx; - auto it = indexByType.find(itemType); - if (it == indexByType.end()) { - idx = collections.size(); - indexByType[itemType] = idx; - collections.emplace_back(); - } else { - idx = it->second; - } - - collections[idx].push_back(item); - } - - TExprNodeList result; - result.reserve(collections.size()); - - for (auto& c : collections) { - result.push_back(ctx.NewList(collection.Pos(), std::move(c))); - } - - return result; -} - -TExprNode::TPtr MergeCalcOverWindowFrames(const TExprNode::TPtr& frames, TExprContext& ctx) { - YQL_ENSURE(frames->IsList()); - - - TNodeMap<size_t> uniqueFrameIndexes; - struct TWinOnRowsContent { - TExprNodeList Args; - TPositionHandle Pos; - }; - - TVector<TWinOnRowsContent> winOnRows; - - for (auto& winOn: frames->Children()) { - YQL_ENSURE(winOn->IsCallable("WinOnRows")); - - if (winOn->ChildrenSize() == 1) { - // skip empty frames - continue; - } - auto args = winOn->ChildrenList(); - auto frameSpec = winOn->Child(0); - auto frameIt = uniqueFrameIndexes.find(frameSpec); - if (frameIt == uniqueFrameIndexes.end()) { - YQL_ENSURE(uniqueFrameIndexes.size() == winOnRows.size()); - uniqueFrameIndexes[frameSpec] = winOnRows.size(); - - TWinOnRowsContent content{std::move(args), winOn->Pos()}; - winOnRows.emplace_back(std::move(content)); - } else { - auto& combined = winOnRows[frameIt->second]; - combined.Args.insert(combined.Args.end(), args.begin() + 1, args.end()); - } - } - - if (uniqueFrameIndexes.size() != frames->ChildrenSize()) { - TExprNodeList winOnRowsNodes; - for (auto &item : winOnRows) { - winOnRowsNodes.emplace_back(ctx.NewCallable(item.Pos, "WinOnRows", std::move(item.Args))); - } - return ctx.NewList(frames->Pos(), std::move(winOnRowsNodes)); - } - - return frames; -} - -TExprNodeList DedupCalcOverWindowsOnSamePartitioning(const TExprNodeList& calcs, TExprContext& ctx) { - struct TDedupKey { - const TExprNode* Keys = nullptr; - const TExprNode* SortSpec = nullptr; - const TExprNode* SessionSpec = nullptr; - bool operator<(const TDedupKey& other) const { - return std::tie(Keys, SortSpec, SessionSpec) < std::tie(other.Keys, other.SortSpec, other.SessionSpec); - } - }; - - TMap<TDedupKey, size_t> uniqueIndexes; - TExprNodeList uniqueCalcs; - for (auto& calcNode : calcs) { - TCoCalcOverWindowTuple calc(calcNode); - if (calc.Frames().Size() == 0 && calc.SessionColumns().Size() == 0) { - continue; - } - TDedupKey key{calc.Keys().Raw(), calc.SortSpec().Raw(), calc.SessionSpec().Raw()}; - - auto it = uniqueIndexes.find(key); - if (it == uniqueIndexes.end()) { - YQL_ENSURE(uniqueIndexes.size() == uniqueCalcs.size()); - const size_t idx = uniqueCalcs.size(); - uniqueIndexes[key] = idx; - uniqueCalcs.emplace_back(calc.Ptr()); - } else { - const size_t idx = it->second; - TCoCalcOverWindowTuple existing(uniqueCalcs[idx]); - - auto existingFrames = existing.Frames().Ref().ChildrenList(); - auto existingSessionColumns = existing.SessionColumns().Ref().ChildrenList(); - - auto frames = calc.Frames().Ref().ChildrenList(); - auto sessionColumns = calc.SessionColumns().Ref().ChildrenList(); - - frames.insert(frames.end(), existingFrames.begin(), existingFrames.end()); - sessionColumns.insert(sessionColumns.end(), existingSessionColumns.begin(), existingSessionColumns.end()); - - uniqueCalcs[idx] = Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) - .Keys(calc.Keys()) - .SortSpec(calc.SortSpec()) - .Frames(ctx.NewList(calc.Frames().Pos(), std::move(frames))) - .SessionSpec(calc.SessionSpec()) - .SessionColumns(ctx.NewList(calc.SessionColumns().Pos(), std::move(sessionColumns))) - .Done().Ptr(); - } - } - return uniqueCalcs; -} - -TExprNode::TPtr BuildCalcOverWindowGroup(TCoCalcOverWindowGroup node, TExprNodeList&& calcs, TExprContext& ctx) { - if (calcs.size() == 0) { - return node.Input().Ptr(); - } - - if (calcs.size() == 1) { - TCoCalcOverWindowTuple calc(calcs[0]); - if (calc.SessionSpec().Maybe<TCoVoid>()) { - YQL_ENSURE(calc.SessionColumns().Size() == 0); - return Build<TCoCalcOverWindow>(ctx, node.Pos()) - .Input(node.Input()) - .Keys(calc.Keys()) - .SortSpec(calc.SortSpec()) - .Frames(calc.Frames()) - .Done().Ptr(); - } else { - return Build<TCoCalcOverSessionWindow>(ctx, node.Pos()) - .Input(node.Input()) - .Keys(calc.Keys()) - .SortSpec(calc.SortSpec()) - .Frames(calc.Frames()) - .SessionSpec(calc.SessionSpec()) - .SessionColumns(calc.SessionColumns()) - .Done().Ptr(); - } - } - - return Build<TCoCalcOverWindowGroup>(ctx, node.Pos()) - .Input(node.Input()) - .Calcs(ctx.NewList(node.Pos(), std::move(calcs))) - .Done().Ptr(); -} - -bool HasPayload(const TCoAggregate& node) { - return node.Handlers().Size() > 0 || HasSetting(node.Settings().Ref(), "hopping"); -} - -TExprNode::TPtr PullAssumeColumnOrderOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - TVector<ui32> withAssume; - for (ui32 i = 0; i < node->ChildrenSize() - 2; i++) { - if (node->Child(i)->Child(0)->IsCallable("AssumeColumnOrder")) { - withAssume.push_back(i); - } - } - - if (withAssume) { - YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); - auto inputs = node->ChildrenList(); - for (ui32 idx : withAssume) { - inputs[idx] = ctx.NewList(inputs[idx]->Pos(), { inputs[idx]->Child(0)->ChildPtr(0), inputs[idx]->ChildPtr(1)}); - } - auto result = ctx.ChangeChildren(*node, std::move(inputs)); - return KeepColumnOrder(result, *node, ctx, *optCtx.Types); - } - return node; -} - + .Seal(); + } + return parent; + }) + .Seal() + .Seal() + .Seal() + .Build(); + } +} + +TExprNode::TPtr ConvertSqlInPredicatesPrefixToJoins(const TExprNode::TPtr& flatMap, const TPredicateChain& chain, + const TExprNode::TPtr& sqlInTail, TExprContext& ctx) +{ + YQL_ENSURE(!chain.empty()); + YQL_ENSURE(chain.front().ConvertibleToJoin); + + TExprNodeList extraLefts; + TExprNodeList extraRights; + for (auto& node : chain) { + if (node.ExtraLeftPred) { + extraLefts.push_back(node.ExtraLeftPred); + } + if (node.ExtraRightPred) { + extraRights.push_back(node.ExtraRightPred); + } + } + + if (!extraLefts.empty() || !extraRights.empty()) + { + TExprNodeList predicates; + predicates.reserve(extraLefts.size() + extraRights.size() + chain.size() + 1); + + predicates.insert(predicates.end(), extraLefts.begin(), extraLefts.end()); + predicates.insert(predicates.end(), extraRights.begin(), extraRights.end()); + + for (auto& node: chain) { + YQL_ENSURE(node.Pred); + predicates.push_back(node.Pred); + } + + if (sqlInTail) { + predicates.push_back(sqlInTail); + } + + YQL_CLOG(DEBUG, Core) << "FlatMapOverJoinableSqlInChain of size " << chain.size() << " with " + << extraLefts.size() << " extra left predicates and " + << extraRights.size() << " extra right predicates due to NOT IN"; + + auto combinedPred = ctx.NewCallable(predicates.front()->Pos(), "And", std::move(predicates)); + return RebuildFlatmapOverPartOfPredicate(flatMap, flatMap->HeadPtr(), combinedPred, true, ctx); + } + + YQL_CLOG(DEBUG, Core) << "FlatMapOverJoinableSqlInChain of size " << chain.size(); + + auto eq = BuildEquiJoinForSqlInChain(flatMap, chain, ctx); + FixSortness(*flatMap, eq, ctx); + + auto tail = sqlInTail ? sqlInTail : MakeBool<true>(flatMap->Pos(), ctx); + return RebuildFlatmapOverPartOfPredicate(flatMap, eq, tail, true, ctx); +} + +TExprNode::TPtr ConvertSqlInPredicatesToJoins(const TCoFlatMapToEquiJoinBase& flatMap, TExprContext& ctx) { + TCoLambda lambda = flatMap.Lambda(); + YQL_ENSURE(lambda.Body().Maybe<TCoOptionalIf>()); + + TPredicateChain chain; + auto lambdaArg = lambda.Ptr()->Head().HeadPtr(); + auto sqlInTail = SplitPredicateChain(lambda.Ptr()->Child(1)->HeadPtr(), lambdaArg, ShouldConvertSqlInToJoin, chain, ctx); + return ConvertSqlInPredicatesPrefixToJoins(flatMap.Ptr(), chain, sqlInTail, ctx); +} + +TExprNodeList DeduplicateAndSplitTupleCollectionByTypes(const TExprNode &collection, TExprContext &ctx) { + const auto& tupleItemsTypes = collection.GetTypeAnn()->Cast<TTupleExprType>()->GetItems(); + + TVector<TExprNodeList> collections; + THashMap<const TTypeAnnotationNode*, size_t> indexByType; + THashSet<const TExprNode*> uniqNodes; + + for (size_t i = 0; i < tupleItemsTypes.size(); ++i) { + auto item = collection.ChildPtr(i); + if (uniqNodes.contains(item.Get())) { + continue; + } + uniqNodes.insert(item.Get()); + + auto itemType = tupleItemsTypes[i]; + + size_t idx; + auto it = indexByType.find(itemType); + if (it == indexByType.end()) { + idx = collections.size(); + indexByType[itemType] = idx; + collections.emplace_back(); + } else { + idx = it->second; + } + + collections[idx].push_back(item); + } + + TExprNodeList result; + result.reserve(collections.size()); + + for (auto& c : collections) { + result.push_back(ctx.NewList(collection.Pos(), std::move(c))); + } + + return result; +} + +TExprNode::TPtr MergeCalcOverWindowFrames(const TExprNode::TPtr& frames, TExprContext& ctx) { + YQL_ENSURE(frames->IsList()); + + + TNodeMap<size_t> uniqueFrameIndexes; + struct TWinOnRowsContent { + TExprNodeList Args; + TPositionHandle Pos; + }; + + TVector<TWinOnRowsContent> winOnRows; + + for (auto& winOn: frames->Children()) { + YQL_ENSURE(winOn->IsCallable("WinOnRows")); + + if (winOn->ChildrenSize() == 1) { + // skip empty frames + continue; + } + auto args = winOn->ChildrenList(); + auto frameSpec = winOn->Child(0); + auto frameIt = uniqueFrameIndexes.find(frameSpec); + if (frameIt == uniqueFrameIndexes.end()) { + YQL_ENSURE(uniqueFrameIndexes.size() == winOnRows.size()); + uniqueFrameIndexes[frameSpec] = winOnRows.size(); + + TWinOnRowsContent content{std::move(args), winOn->Pos()}; + winOnRows.emplace_back(std::move(content)); + } else { + auto& combined = winOnRows[frameIt->second]; + combined.Args.insert(combined.Args.end(), args.begin() + 1, args.end()); + } + } + + if (uniqueFrameIndexes.size() != frames->ChildrenSize()) { + TExprNodeList winOnRowsNodes; + for (auto &item : winOnRows) { + winOnRowsNodes.emplace_back(ctx.NewCallable(item.Pos, "WinOnRows", std::move(item.Args))); + } + return ctx.NewList(frames->Pos(), std::move(winOnRowsNodes)); + } + + return frames; +} + +TExprNodeList DedupCalcOverWindowsOnSamePartitioning(const TExprNodeList& calcs, TExprContext& ctx) { + struct TDedupKey { + const TExprNode* Keys = nullptr; + const TExprNode* SortSpec = nullptr; + const TExprNode* SessionSpec = nullptr; + bool operator<(const TDedupKey& other) const { + return std::tie(Keys, SortSpec, SessionSpec) < std::tie(other.Keys, other.SortSpec, other.SessionSpec); + } + }; + + TMap<TDedupKey, size_t> uniqueIndexes; + TExprNodeList uniqueCalcs; + for (auto& calcNode : calcs) { + TCoCalcOverWindowTuple calc(calcNode); + if (calc.Frames().Size() == 0 && calc.SessionColumns().Size() == 0) { + continue; + } + TDedupKey key{calc.Keys().Raw(), calc.SortSpec().Raw(), calc.SessionSpec().Raw()}; + + auto it = uniqueIndexes.find(key); + if (it == uniqueIndexes.end()) { + YQL_ENSURE(uniqueIndexes.size() == uniqueCalcs.size()); + const size_t idx = uniqueCalcs.size(); + uniqueIndexes[key] = idx; + uniqueCalcs.emplace_back(calc.Ptr()); + } else { + const size_t idx = it->second; + TCoCalcOverWindowTuple existing(uniqueCalcs[idx]); + + auto existingFrames = existing.Frames().Ref().ChildrenList(); + auto existingSessionColumns = existing.SessionColumns().Ref().ChildrenList(); + + auto frames = calc.Frames().Ref().ChildrenList(); + auto sessionColumns = calc.SessionColumns().Ref().ChildrenList(); + + frames.insert(frames.end(), existingFrames.begin(), existingFrames.end()); + sessionColumns.insert(sessionColumns.end(), existingSessionColumns.begin(), existingSessionColumns.end()); + + uniqueCalcs[idx] = Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) + .Keys(calc.Keys()) + .SortSpec(calc.SortSpec()) + .Frames(ctx.NewList(calc.Frames().Pos(), std::move(frames))) + .SessionSpec(calc.SessionSpec()) + .SessionColumns(ctx.NewList(calc.SessionColumns().Pos(), std::move(sessionColumns))) + .Done().Ptr(); + } + } + return uniqueCalcs; +} + +TExprNode::TPtr BuildCalcOverWindowGroup(TCoCalcOverWindowGroup node, TExprNodeList&& calcs, TExprContext& ctx) { + if (calcs.size() == 0) { + return node.Input().Ptr(); + } + + if (calcs.size() == 1) { + TCoCalcOverWindowTuple calc(calcs[0]); + if (calc.SessionSpec().Maybe<TCoVoid>()) { + YQL_ENSURE(calc.SessionColumns().Size() == 0); + return Build<TCoCalcOverWindow>(ctx, node.Pos()) + .Input(node.Input()) + .Keys(calc.Keys()) + .SortSpec(calc.SortSpec()) + .Frames(calc.Frames()) + .Done().Ptr(); + } else { + return Build<TCoCalcOverSessionWindow>(ctx, node.Pos()) + .Input(node.Input()) + .Keys(calc.Keys()) + .SortSpec(calc.SortSpec()) + .Frames(calc.Frames()) + .SessionSpec(calc.SessionSpec()) + .SessionColumns(calc.SessionColumns()) + .Done().Ptr(); + } + } + + return Build<TCoCalcOverWindowGroup>(ctx, node.Pos()) + .Input(node.Input()) + .Calcs(ctx.NewList(node.Pos(), std::move(calcs))) + .Done().Ptr(); +} + +bool HasPayload(const TCoAggregate& node) { + return node.Handlers().Size() > 0 || HasSetting(node.Settings().Ref(), "hopping"); +} + +TExprNode::TPtr PullAssumeColumnOrderOverEquiJoin(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + TVector<ui32> withAssume; + for (ui32 i = 0; i < node->ChildrenSize() - 2; i++) { + if (node->Child(i)->Child(0)->IsCallable("AssumeColumnOrder")) { + withAssume.push_back(i); + } + } + + if (withAssume) { + YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); + auto inputs = node->ChildrenList(); + for (ui32 idx : withAssume) { + inputs[idx] = ctx.NewList(inputs[idx]->Pos(), { inputs[idx]->Child(0)->ChildPtr(0), inputs[idx]->ChildPtr(1)}); + } + auto result = ctx.ChangeChildren(*node, std::move(inputs)); + return KeepColumnOrder(result, *node, ctx, *optCtx.Types); + } + return node; +} + } // namespace TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const TCoFlatMapBase& flatMap, TShouldConvertSqlInToJoinPredicate shouldConvertSqlInToJoin, TExprContext& ctx, bool prefixOnly) { - // FlatMap input should be List<Struct<...>> to be accepted as EquiJoin input - auto inputType = flatMap.Input().Ref().GetTypeAnn(); - if (inputType->GetKind() != ETypeAnnotationKind::List || - inputType->Cast<TListExprType>()->GetItemType()->GetKind() != ETypeAnnotationKind::Struct) - { - return {}; - } - + // FlatMap input should be List<Struct<...>> to be accepted as EquiJoin input + auto inputType = flatMap.Input().Ref().GetTypeAnn(); + if (inputType->GetKind() != ETypeAnnotationKind::List || + inputType->Cast<TListExprType>()->GetItemType()->GetKind() != ETypeAnnotationKind::Struct) + { + return {}; + } + TCoLambda lambda = flatMap.Lambda(); - if (!lambda.Body().Maybe<TCoConditionalValueBase>()) { + if (!lambda.Body().Maybe<TCoConditionalValueBase>()) { return {}; } - TCoConditionalValueBase conditional(lambda.Body().Ptr()); - TPredicateChain chain; + TCoConditionalValueBase conditional(lambda.Body().Ptr()); + TPredicateChain chain; auto lambdaArg = lambda.Ptr()->Head().HeadPtr(); - auto sqlInTail = SplitPredicateChain(conditional.Predicate().Ptr(), lambdaArg, shouldConvertSqlInToJoin, chain, ctx); + auto sqlInTail = SplitPredicateChain(conditional.Predicate().Ptr(), lambdaArg, shouldConvertSqlInToJoin, chain, ctx); if (!chain.empty()) { if (chain.front().ConvertibleToJoin) { - return ConvertSqlInPredicatesPrefixToJoins(flatMap.Ptr(), chain, sqlInTail, ctx); + return ConvertSqlInPredicatesPrefixToJoins(flatMap.Ptr(), chain, sqlInTail, ctx); } - if (sqlInTail && !prefixOnly) { + if (sqlInTail && !prefixOnly) { YQL_CLOG(DEBUG, Core) << "FlatMapOverNonJoinableSqlInChain of size " << chain.size(); TExprNode::TListType predicates; predicates.reserve(chain.size()); @@ -3008,10 +3008,10 @@ TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const TCoFlatMapBase& flatMap, } auto prefixPred = ctx.NewCallable(flatMap.Pos(), "And", std::move(predicates)); - auto innerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), flatMap.Input().Ptr(), prefixPred, false, ctx); - auto outerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), innerFlatMap, sqlInTail, true, ctx); - return ctx.RenameNode(*outerFlatMap, - outerFlatMap->Content() == "OrderedFlatMap" ? "OrderedFlatMapToEquiJoin" : "FlatMapToEquiJoin"); + auto innerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), flatMap.Input().Ptr(), prefixPred, false, ctx); + auto outerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), innerFlatMap, sqlInTail, true, ctx); + return ctx.RenameNode(*outerFlatMap, + outerFlatMap->Content() == "OrderedFlatMap" ? "OrderedFlatMapToEquiJoin" : "FlatMapToEquiJoin"); } } @@ -3436,12 +3436,12 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["FlatMapToEquiJoin"] = map["OrderedFlatMapToEquiJoin"] = - [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { - const auto self = TCoFlatMapToEquiJoinBase(node); - return ConvertSqlInPredicatesToJoins(self, ctx); - }; - + map["FlatMapToEquiJoin"] = map["OrderedFlatMapToEquiJoin"] = + [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { + const auto self = TCoFlatMapToEquiJoinBase(node); + return ConvertSqlInPredicatesToJoins(self, ctx); + }; + map["SkipNullMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { const auto skipNullMembers = TCoSkipNullMembers(node); if (!skipNullMembers.Members()) { @@ -3524,14 +3524,14 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["Filter"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + map["Filter"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { YQL_CLOG(DEBUG, Core) << "Canonize " << node->Content(); - return ConvertFilterToFlatmap<TCoFilter, TCoFlatMap>(TCoFilter(node), ctx, optCtx); + return ConvertFilterToFlatmap<TCoFilter, TCoFlatMap>(TCoFilter(node), ctx, optCtx); }; - map["OrderedFilter"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + map["OrderedFilter"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { YQL_CLOG(DEBUG, Core) << "Canonize " << node->Content(); - return ConvertFilterToFlatmap<TCoOrderedFilter, TCoOrderedFlatMap>(TCoOrderedFilter(node), ctx, optCtx); + return ConvertFilterToFlatmap<TCoOrderedFilter, TCoOrderedFlatMap>(TCoOrderedFilter(node), ctx, optCtx); }; map["Map"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { @@ -3586,129 +3586,129 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["Lookup"] = std::bind(&OptimizeContains<false, true>, _1, _2); map["Contains"] = std::bind(&OptimizeContains<false>, _1, _2); map["ListHas"] = std::bind(&OptimizeContains<true>, _1, _2); - map["SqlIn"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { - auto collection = node->HeadPtr(); - auto lookup = node->ChildPtr(1); - auto options = node->ChildPtr(2); - - auto collectionType = collection->GetTypeAnn(); - auto collectionKind = collectionType->GetKind(); - - if (collectionKind == ETypeAnnotationKind::Null) { - YQL_CLOG(DEBUG, Core) << "IN Null"; - return MakeBoolNothing(node->Pos(), ctx); - } - - if (collectionKind == ETypeAnnotationKind::Optional) { - YQL_CLOG(DEBUG, Core) << "IN Optional"; - - return ctx.Builder(node->Pos()) - .Callable("FlatMap") - .Add(0, collection) - .Lambda(1) - .Param("collection") - .Callable("MatchType") - .Callable(0, "SqlIn") - .Arg(0, "collection") - .Add(1, lookup) - .Add(2, options) - .Seal() - .Atom(1, "Optional", TNodeFlags::Default) - .Lambda(2) - .Param("input") - .Arg("input") - .Seal() - .Lambda(3) - .Param("input") - .Callable("Just") - .Arg(0, "input") - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - } - - const bool isAnsi = HasSetting(*options, "ansi"); - if (collectionKind == ETypeAnnotationKind::EmptyDict || - collectionKind == ETypeAnnotationKind::EmptyList || - ( - collectionKind == ETypeAnnotationKind::Tuple && - collectionType->Cast<TTupleExprType>()->GetSize() == 0 - )) - { - if (!isAnsi) { - // legacy IN: null in () should equals null - if (lookup->GetTypeAnn()->HasOptionalOrNull()) { - YQL_CLOG(DEBUG, Core) << "NULL IN legacy"; - return ctx.Builder(node->Pos()) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, lookup) - .Seal() - .Callable(1, "Null") - .Seal() - .Add(2, MakeBool(node->Pos(), false, ctx)) - .Seal() - .Build(); - } - auto lookupTypeNoOpt = RemoveAllOptionals(lookup->GetTypeAnn()); - if (lookupTypeNoOpt->GetKind() == ETypeAnnotationKind::Null) { - return MakeBoolNothing(node->Pos(), ctx); - } - } - - YQL_CLOG(DEBUG, Core) << "IN Empty collection"; - return (node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) ? - MakeOptionalBool(node->Pos(), false, ctx) : MakeBool(node->Pos(), false, ctx); - } - - if (collectionKind == ETypeAnnotationKind::Tuple) { - auto tupleType = collectionType->Cast<TTupleExprType>(); - YQL_ENSURE(tupleType->GetSize()); - auto firstItemType = tupleType->GetItems().front(); - - bool tupleElementsHaveSameType = - AllOf(tupleType->GetItems(), - [&](const TTypeAnnotationNode* item) { - return IsSameAnnotation(*firstItemType, *item); - }); - - if (!tupleElementsHaveSameType) { - YQL_CLOG(DEBUG, Core) << "IN heterogeneous tuple"; - auto collections = DeduplicateAndSplitTupleCollectionByTypes(*collection, ctx); - YQL_ENSURE(collections.size() > 1); - - TExprNodeList predicates; - predicates.reserve(collections.size()); - - for (auto& splittedCollection : collections) { - predicates.push_back(ctx.NewCallable(node->Pos(), "SqlIn", { splittedCollection, lookup, options})); - } - - return ctx.NewCallable(node->Pos(), "Or", std::move(predicates)); - } - } - - if (isAnsi) { - auto lookupTypeNoOpt = RemoveAllOptionals(lookup->GetTypeAnn()); - if (lookupTypeNoOpt->GetKind() == ETypeAnnotationKind::Null) { - YQL_CLOG(DEBUG, Core) << "NULL IN"; - return ctx.Builder(node->Pos()) - .Callable("If") - .Add(0, BuildSqlInCollectionEmptyPred(TCoSqlIn(node), ctx)) - .Add(1, MakeBool(node->Pos(), false, ctx)) - .Callable(2, "Null") - .Seal() - .Seal() - .Build(); - } - } - - return node; - }; - + map["SqlIn"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) { + auto collection = node->HeadPtr(); + auto lookup = node->ChildPtr(1); + auto options = node->ChildPtr(2); + + auto collectionType = collection->GetTypeAnn(); + auto collectionKind = collectionType->GetKind(); + + if (collectionKind == ETypeAnnotationKind::Null) { + YQL_CLOG(DEBUG, Core) << "IN Null"; + return MakeBoolNothing(node->Pos(), ctx); + } + + if (collectionKind == ETypeAnnotationKind::Optional) { + YQL_CLOG(DEBUG, Core) << "IN Optional"; + + return ctx.Builder(node->Pos()) + .Callable("FlatMap") + .Add(0, collection) + .Lambda(1) + .Param("collection") + .Callable("MatchType") + .Callable(0, "SqlIn") + .Arg(0, "collection") + .Add(1, lookup) + .Add(2, options) + .Seal() + .Atom(1, "Optional", TNodeFlags::Default) + .Lambda(2) + .Param("input") + .Arg("input") + .Seal() + .Lambda(3) + .Param("input") + .Callable("Just") + .Arg(0, "input") + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + } + + const bool isAnsi = HasSetting(*options, "ansi"); + if (collectionKind == ETypeAnnotationKind::EmptyDict || + collectionKind == ETypeAnnotationKind::EmptyList || + ( + collectionKind == ETypeAnnotationKind::Tuple && + collectionType->Cast<TTupleExprType>()->GetSize() == 0 + )) + { + if (!isAnsi) { + // legacy IN: null in () should equals null + if (lookup->GetTypeAnn()->HasOptionalOrNull()) { + YQL_CLOG(DEBUG, Core) << "NULL IN legacy"; + return ctx.Builder(node->Pos()) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, lookup) + .Seal() + .Callable(1, "Null") + .Seal() + .Add(2, MakeBool(node->Pos(), false, ctx)) + .Seal() + .Build(); + } + auto lookupTypeNoOpt = RemoveAllOptionals(lookup->GetTypeAnn()); + if (lookupTypeNoOpt->GetKind() == ETypeAnnotationKind::Null) { + return MakeBoolNothing(node->Pos(), ctx); + } + } + + YQL_CLOG(DEBUG, Core) << "IN Empty collection"; + return (node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) ? + MakeOptionalBool(node->Pos(), false, ctx) : MakeBool(node->Pos(), false, ctx); + } + + if (collectionKind == ETypeAnnotationKind::Tuple) { + auto tupleType = collectionType->Cast<TTupleExprType>(); + YQL_ENSURE(tupleType->GetSize()); + auto firstItemType = tupleType->GetItems().front(); + + bool tupleElementsHaveSameType = + AllOf(tupleType->GetItems(), + [&](const TTypeAnnotationNode* item) { + return IsSameAnnotation(*firstItemType, *item); + }); + + if (!tupleElementsHaveSameType) { + YQL_CLOG(DEBUG, Core) << "IN heterogeneous tuple"; + auto collections = DeduplicateAndSplitTupleCollectionByTypes(*collection, ctx); + YQL_ENSURE(collections.size() > 1); + + TExprNodeList predicates; + predicates.reserve(collections.size()); + + for (auto& splittedCollection : collections) { + predicates.push_back(ctx.NewCallable(node->Pos(), "SqlIn", { splittedCollection, lookup, options})); + } + + return ctx.NewCallable(node->Pos(), "Or", std::move(predicates)); + } + } + + if (isAnsi) { + auto lookupTypeNoOpt = RemoveAllOptionals(lookup->GetTypeAnn()); + if (lookupTypeNoOpt->GetKind() == ETypeAnnotationKind::Null) { + YQL_CLOG(DEBUG, Core) << "NULL IN"; + return ctx.Builder(node->Pos()) + .Callable("If") + .Add(0, BuildSqlInCollectionEmptyPred(TCoSqlIn(node), ctx)) + .Add(1, MakeBool(node->Pos(), false, ctx)) + .Callable(2, "Null") + .Seal() + .Seal() + .Build(); + } + } + + return node; + }; + map["DictItems"] = std::bind(&OptimizeDictItems, _1, _2); map["DictKeys"] = std::bind(&OptimizeDictItems, _1, _2); map["DictPayloads"] = std::bind(&OptimizeDictItems, _1, _2); @@ -3731,14 +3731,14 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["Take"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + map["Take"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { if (node->Tail().IsCallable("Uint64")) { const auto value = FromString<ui64>(node->Tail().Head().Content()); if (!value) { YQL_CLOG(DEBUG, Core) << node->Content() << " with " << node->Tail().Content() << " '" << node->Tail().Head().Content(); auto res = ctx.NewCallable(node->Tail().Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)}); res = KeepConstraints(res, *node, ctx); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); } } @@ -3747,10 +3747,10 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["TakeWhile"] = std::bind(&OptimizeWhile<true>, _1, _2); map["SkipWhile"] = std::bind(&OptimizeWhile<false>, _1, _2); - + map["TakeWhileInclusive"] = std::bind(&OptimizeWhile<true, true>, _1, _2); map["SkipWhileInclusive"] = std::bind(&OptimizeWhile<false, true>, _1, _2); - + map[TCoExtend::CallableName()] = map[TCoOrderedExtend::CallableName()] = map[TCoMerge::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { if (node->ChildrenSize() == 1) { YQL_CLOG(DEBUG, Core) << node->Content() << " over one child"; @@ -3847,46 +3847,46 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { { ctx.NewAtom(node->Pos(), ToString(nodeToCheck.ChildrenSize() - 1), TNodeFlags::Default) }); } - if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable( - {"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup", "Chain1Map", "FoldMap", "Fold1Map"})) - { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); - } - - if (nodeToCheck.IsCallable({"FlatMap", "OrderedFlatMap"}) - && nodeToCheck.Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List - && IsJustOrSingleAsList(nodeToCheck.Tail().Tail())) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); - } - - if (nodeToCheck.IsCallable("Take") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.Builder(node->Pos()) - .Callable("Min") - .Add(0, nodeToCheck.TailPtr()) - .Callable(1, "Length") - .Add(0, nodeToCheck.HeadPtr()) - .Seal() - .Seal() - .Build(); - } - - if (nodeToCheck.IsCallable("Skip") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - auto fullLen = ctx.NewCallable(node->Pos(), "Length", { nodeToCheck.HeadPtr() }); - return ctx.Builder(node->Pos()) - .Callable("-") - .Add(0, fullLen) - .Callable(1, "Min") - .Add(0, nodeToCheck.TailPtr()) - .Add(1, fullLen) - .Seal() - .Seal() - .Build(); - } - + if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable( + {"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup", "Chain1Map", "FoldMap", "Fold1Map"})) + { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); + } + + if (nodeToCheck.IsCallable({"FlatMap", "OrderedFlatMap"}) + && nodeToCheck.Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List + && IsJustOrSingleAsList(nodeToCheck.Tail().Tail())) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); + } + + if (nodeToCheck.IsCallable("Take") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.Builder(node->Pos()) + .Callable("Min") + .Add(0, nodeToCheck.TailPtr()) + .Callable(1, "Length") + .Add(0, nodeToCheck.HeadPtr()) + .Seal() + .Seal() + .Build(); + } + + if (nodeToCheck.IsCallable("Skip") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + auto fullLen = ctx.NewCallable(node->Pos(), "Length", { nodeToCheck.HeadPtr() }); + return ctx.Builder(node->Pos()) + .Callable("-") + .Add(0, fullLen) + .Callable(1, "Min") + .Add(0, nodeToCheck.TailPtr()) + .Add(1, fullLen) + .Seal() + .Seal() + .Build(); + } + return node; }; @@ -3907,50 +3907,50 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return MakeBool(node->Pos(), nodeToCheck.ChildrenSize() > 1U, ctx); } - if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable( - {"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup", "Chain1Map", "FoldMap", "Fold1Map"})) - { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); - } - - if (nodeToCheck.IsCallable({"FlatMap", "OrderedFlatMap"}) - && nodeToCheck.Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List - && IsJustOrSingleAsList(nodeToCheck.Tail().Tail())) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); - } - - if (nodeToCheck.IsCallable("Take") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.Builder(node->Pos()) - .Callable("If") - .Callable(0, "==") - .Callable(0, "Uint64") - .Atom(0, "0", TNodeFlags::Default) - .Seal() - .Add(1, nodeToCheck.TailPtr()) - .Seal() - .Add(1, MakeBool<false>(node->Pos(), ctx)) - .Callable(2, "HasItems") - .Add(0, nodeToCheck.HeadPtr()) - .Seal() - .Seal() - .Build(); - } - - if (nodeToCheck.IsCallable("Skip") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); - return ctx.Builder(node->Pos()) - .Callable(">") - .Callable(0, "Length") - .Add(0, nodeToCheck.HeadPtr()) - .Seal() - .Add(1, nodeToCheck.TailPtr()) - .Seal() - .Build(); - } - + if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable( + {"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup", "Chain1Map", "FoldMap", "Fold1Map"})) + { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); + } + + if (nodeToCheck.IsCallable({"FlatMap", "OrderedFlatMap"}) + && nodeToCheck.Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List + && IsJustOrSingleAsList(nodeToCheck.Tail().Tail())) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.ChangeChild(*node, 0U, nodeToCheck.HeadPtr()); + } + + if (nodeToCheck.IsCallable("Take") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.Builder(node->Pos()) + .Callable("If") + .Callable(0, "==") + .Callable(0, "Uint64") + .Atom(0, "0", TNodeFlags::Default) + .Seal() + .Add(1, nodeToCheck.TailPtr()) + .Seal() + .Add(1, MakeBool<false>(node->Pos(), ctx)) + .Callable(2, "HasItems") + .Add(0, nodeToCheck.HeadPtr()) + .Seal() + .Seal() + .Build(); + } + + if (nodeToCheck.IsCallable("Skip") && nodeToCheck.Head().IsCallable({"ForwardList", "Collect"})) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content(); + return ctx.Builder(node->Pos()) + .Callable(">") + .Callable(0, "Length") + .Add(0, nodeToCheck.HeadPtr()) + .Seal() + .Add(1, nodeToCheck.TailPtr()) + .Seal() + .Build(); + } + return node; }; @@ -3976,7 +3976,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { auto ret = ctx.ChangeChild(*node, 0U, node->Head().HeadPtr()); const auto structType = node->Head().Head().GetTypeAnn()->Cast<TStructExprType>(); const auto memberType = structType->GetItems()[*structType->FindItem(node->Tail().Content())]->GetItemType(); - return ctx.WrapByCallableIf(!memberType->IsOptionalOrNull(), "Just", std::move(ret)); + return ctx.WrapByCallableIf(!memberType->IsOptionalOrNull(), "Just", std::move(ret)); } if (node->Head().IsCallable("Nothing")) { @@ -3984,11 +3984,11 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return ctx.ChangeChild(node->Head(), 0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)); } - if (node->Head().IsCallable("ExtractMembers")) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return ctx.ChangeChild(*node, 0, node->Head().HeadPtr()); - } - + if (node->Head().IsCallable("ExtractMembers")) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return ctx.ChangeChild(*node, 0, node->Head().HeadPtr()); + } + return node; }; @@ -4133,13 +4133,13 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { if (3U == node->ChildrenSize() && node->Child(1)->IsCallable("Bool") && node->Child(2)->IsCallable("Bool")) { const auto thenValue = FromString<bool>(node->Child(1)->Head().Content()); const auto elseValue = FromString<bool>(node->Child(2)->Head().Content()); - + if (thenValue != elseValue) { YQL_CLOG(DEBUG, Core) << node->Content() << " with literals in branches"; return ctx.WrapByCallableIf(elseValue, "Not", node->HeadPtr()); - } - } - + } + } + return node; }; @@ -4318,7 +4318,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["IsNotDistinctFrom"] = std::bind(&OptimizeDistinctFrom<true>, _1, _2); map["IsDistinctFrom"] = std::bind(&OptimizeDistinctFrom<false>, _1, _2); - + map["StartsWith"] = std::bind(&OptimizeEquality<true>, _1, _2); map["EndsWith"] = std::bind(&OptimizeEquality<true>, _1, _2); @@ -4362,24 +4362,24 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { map["Extract"] = std::bind(&ExpandExtract<false>, _1, _2); map["OrderedExtract"] = std::bind(&ExpandExtract<true>, _1, _2); - map["UnionAll"] = std::bind(&ExpandUnionAll<false>, _1, _2, _3); - map["UnionMerge"] = std::bind(&ExpandUnionAll<true>, _1, _2, _3); + map["UnionAll"] = std::bind(&ExpandUnionAll<false>, _1, _2, _3); + map["UnionMerge"] = std::bind(&ExpandUnionAll<true>, _1, _2, _3); map["Aggregate"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { TCoAggregate self(node); - if (self.Keys().Size() == 0 && !HasPayload(self)) { + if (self.Keys().Size() == 0 && !HasPayload(self)) { YQL_CLOG(DEBUG, Core) << node->Content() << " with empty fields"; return ctx.NewCallable(node->Pos(), "AsList", {ctx.NewCallable(node->Pos(), "AsStruct", {})}); } - if (auto maybeAggregate = self.Input().Maybe<TCoAggregate>()) { - auto child = maybeAggregate.Cast(); - if (!HasPayload(self) && !HasPayload(child) && self.Keys().Size() == child.Keys().Size()) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Content() << " without payload with same keys"; - return self.Input().Ptr(); - } - } - + if (auto maybeAggregate = self.Input().Maybe<TCoAggregate>()) { + auto child = maybeAggregate.Cast(); + if (!HasPayload(self) && !HasPayload(child) && self.Keys().Size() == child.Keys().Size()) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Content() << " without payload with same keys"; + return self.Input().Ptr(); + } + } + return DropReorder(node, ctx); }; @@ -4467,12 +4467,12 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return ret; } - ret = PullAssumeColumnOrderOverEquiJoin(node, ctx, optCtx); - if (ret != node) { - YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over EquiJoin"; - return ret; - } - + ret = PullAssumeColumnOrderOverEquiJoin(node, ctx, optCtx); + if (ret != node) { + YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over EquiJoin"; + return ret; + } + return node; }; @@ -4999,7 +4999,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { .Seal() .Build(); }; - + map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { if (TCoJust::Match(&node->Head())) { YQL_CLOG(DEBUG, Core) << "Move " << node->Content() << " over " << node->Head().Content(); @@ -5024,16 +5024,16 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["Untag"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - Y_UNUSED(ctx); + map["Untag"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + Y_UNUSED(ctx); if (node->Head().IsCallable("AsTagged")) { YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); return node->Head().HeadPtr(); - } - - return node; - }; - + } + + return node; + }; + map["SqueezeToDict"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { if (const auto& inputToCheck = SkipCallables(node->Head(), SkippableCallables); IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) { YQL_CLOG(DEBUG, Core) << "Empty " << node->Content(); @@ -5057,33 +5057,33 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { } if (node->Head().IsCallable("AsList") && node->Child(2)->Child(1)->IsCallable("Void")) { - TMaybe<bool> isMany; - TMaybe<bool> isHashed; - TMaybe<ui64> itemsCount; - bool isCompact; - auto settingsError = ParseToDictSettings(*node, ctx, isMany, isHashed, itemsCount, isCompact); - YQL_ENSURE(!settingsError); - - if (!*isMany && *isHashed) { - YQL_CLOG(DEBUG, Core) << "ToDict without payload over list literal"; - return ctx.Builder(node->Pos()) - .Callable("DictFromKeys") + TMaybe<bool> isMany; + TMaybe<bool> isHashed; + TMaybe<ui64> itemsCount; + bool isCompact; + auto settingsError = ParseToDictSettings(*node, ctx, isMany, isHashed, itemsCount, isCompact); + YQL_ENSURE(!settingsError); + + if (!*isMany && *isHashed) { + YQL_CLOG(DEBUG, Core) << "ToDict without payload over list literal"; + return ctx.Builder(node->Pos()) + .Callable("DictFromKeys") .Add(0, ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TDictExprType>()->GetKeyType(), ctx)) - .List(1) - .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& { + .List(1) + .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& { for (ui32 i = 0; i < node->Head().ChildrenSize(); ++i) { - builder.Apply(i, node->ChildPtr(1)) + builder.Apply(i, node->ChildPtr(1)) .With(0, node->Head().ChildPtr(i)) - .Seal(); - } - return builder; - }) - .Seal() - .Seal() - .Build(); - } - } - + .Seal(); + } + return builder; + }) + .Seal() + .Seal() + .Build(); + } + } + if (node->Head().IsCallable("DictItems")) { auto inner = node->Head().ChildPtr(0); if (inner->IsCallable("ToDict")) { @@ -5106,82 +5106,82 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { } } - return node; - }; - - map["HasNull"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + return node; + }; + + map["HasNull"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { YQL_CLOG(DEBUG, Core) << node->Content(); - + auto value = node->HeadPtr(); - auto valueType = value->GetTypeAnn(); - - if (!valueType->HasOptionalOrNull()) { + auto valueType = value->GetTypeAnn(); + + if (!valueType->HasOptionalOrNull()) { return MakeBool<false>(node->Pos(), ctx); - } - - switch (valueType->GetKind()) { - case ETypeAnnotationKind::Null: + } + + switch (valueType->GetKind()) { + case ETypeAnnotationKind::Null: return MakeBool<true>(node->Pos(), ctx); - case ETypeAnnotationKind::Optional: - return ctx.Builder(node->Pos()) - .Callable("IfPresent") - .Add(0, value) - .Lambda(1) - .Param("item") - .Callable("HasNull") - .Arg(0, "item") - .Seal() - .Seal() + case ETypeAnnotationKind::Optional: + return ctx.Builder(node->Pos()) + .Callable("IfPresent") + .Add(0, value) + .Lambda(1) + .Param("item") + .Callable("HasNull") + .Arg(0, "item") + .Seal() + .Seal() .Add(2, MakeBool<true>(node->Pos(), ctx)) - .Seal() - .Build(); - case ETypeAnnotationKind::Tagged: - return ctx.Builder(node->Pos()) - .Callable("HasNull") - .Callable(0, "Untag") - .Add(0, value) - .Atom(1, valueType->Cast<TTaggedExprType>()->GetTag()) - .Seal() - .Seal() - .Build(); - case ETypeAnnotationKind::Dict: - return ctx.Builder(node->Pos()) - .Callable("HasNull") - .Callable(0, "DictItems") - .Add(0, value) - .Seal() - .Seal() - .Build(); - case ETypeAnnotationKind::List: - return ctx.Builder(node->Pos()) - .Callable("HasItems") - .Callable(0, "SkipWhile") - .Add(0, value) - .Lambda(1) - .Param("item") - .Callable("Not") - .Callable(0, "HasNull") - .Arg(0, "item") - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - case ETypeAnnotationKind::Tuple: - return HasNullOverTuple(node, ctx); - case ETypeAnnotationKind::Struct: - return HasNullOverStruct(node, ctx); - case ETypeAnnotationKind::Variant: - return HasNullOverVariant(node, ctx); - default: - YQL_ENSURE(false, "Value type " << *valueType << " is not supported!"); - } - - Y_UNREACHABLE(); - }; - - map["Unordered"] = map["UnorderedSubquery"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + .Seal() + .Build(); + case ETypeAnnotationKind::Tagged: + return ctx.Builder(node->Pos()) + .Callable("HasNull") + .Callable(0, "Untag") + .Add(0, value) + .Atom(1, valueType->Cast<TTaggedExprType>()->GetTag()) + .Seal() + .Seal() + .Build(); + case ETypeAnnotationKind::Dict: + return ctx.Builder(node->Pos()) + .Callable("HasNull") + .Callable(0, "DictItems") + .Add(0, value) + .Seal() + .Seal() + .Build(); + case ETypeAnnotationKind::List: + return ctx.Builder(node->Pos()) + .Callable("HasItems") + .Callable(0, "SkipWhile") + .Add(0, value) + .Lambda(1) + .Param("item") + .Callable("Not") + .Callable(0, "HasNull") + .Arg(0, "item") + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + case ETypeAnnotationKind::Tuple: + return HasNullOverTuple(node, ctx); + case ETypeAnnotationKind::Struct: + return HasNullOverStruct(node, ctx); + case ETypeAnnotationKind::Variant: + return HasNullOverVariant(node, ctx); + default: + YQL_ENSURE(false, "Value type " << *valueType << " is not supported!"); + } + + Y_UNREACHABLE(); + }; + + map["Unordered"] = map["UnorderedSubquery"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { if (node->Head().IsCallable("AsList")) { YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); return node->HeadPtr(); @@ -5912,223 +5912,223 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { .Build() .Done().Ptr(); }; - - map["CalcOverWindow"] = map["CalcOverSessionWindow"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - TCoCalcOverWindowBase self(node); - auto frames = self.Frames(); - size_t sessionColumnsSize = 0; - if (auto maybeSession = TMaybeNode<TCoCalcOverSessionWindow>(node)) { - sessionColumnsSize = maybeSession.Cast().SessionColumns().Size(); - } - if (frames.Size() == 0 && sessionColumnsSize == 0) { - YQL_CLOG(DEBUG, Core) << node->Content() << " without payload"; - return self.Input().Ptr(); - } - - auto mergedFrames = MergeCalcOverWindowFrames(frames.Ptr(), ctx); - if (mergedFrames == frames.Ptr()) { - return node; - } - - YQL_CLOG(DEBUG, Core) << node->Content() << " with duplicate or empty frames"; - return ctx.ChangeChild(*node, TCoCalcOverWindowBase::idx_Frames, std::move(mergedFrames)); - }; - - map["CalcOverWindowGroup"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - TCoCalcOverWindowGroup self(node); - - auto dedupCalcs = DedupCalcOverWindowsOnSamePartitioning(self.Calcs().Ref().ChildrenList(), ctx); - YQL_ENSURE(dedupCalcs.size() <= self.Calcs().Size()); - - TExprNodeList mergedCalcs; - bool merged = false; - for (auto& calcNode : dedupCalcs) { - TCoCalcOverWindowTuple calc(calcNode); - - auto origFrames = calc.Frames().Ptr(); - auto mergedFrames = MergeCalcOverWindowFrames(origFrames, ctx); - if (mergedFrames != origFrames) { - merged = true; - mergedCalcs.emplace_back( - Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) - .Keys(calc.Keys()) - .SortSpec(calc.SortSpec()) - .Frames(mergedFrames) - .SessionSpec(calc.SessionSpec()) - .SessionColumns(calc.SessionColumns()) - .Done().Ptr() - ); - } else { - mergedCalcs.emplace_back(std::move(calcNode)); - } - } - - if (merged || dedupCalcs.size() < self.Calcs().Size()) { - YQL_CLOG(DEBUG, Core) << "CalcOverWindowGroup with duplicate/empty frames and/or duplicate windows"; - return BuildCalcOverWindowGroup(self, std::move(mergedCalcs), ctx); - } - - if (mergedCalcs.size() <= 1) { - TStringBuf msg = mergedCalcs.empty() ? "CalcOverWindowGroup without windows" : "CalcOverWindowGroup with single window"; - YQL_CLOG(DEBUG, Core) << msg; - return BuildCalcOverWindowGroup(self, std::move(mergedCalcs), ctx); - } - - return node; - }; - - map["AssumeColumnOrder"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - auto input = node->HeadPtr(); - if (input->IsCallable("AssumeColumnOrder")) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << input->Content(); - return ctx.ChangeChild(*node, 0u, input->HeadPtr()); - } - - return node; - }; - - map["SqlProject"] = map["OrderedSqlProject"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - - TExprNodeList lambdas; - for (auto& item : node->Child(1)->Children()) { - YQL_ENSURE(item->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})); - YQL_ENSURE(item->Child(1)->IsAtom()); - YQL_ENSURE(item->Child(2)->IsLambda()); - if (item->IsCallable("SqlProjectStarItem")) { - lambdas.push_back(item->ChildPtr(2)); - } else { - auto targetName = item->Child(1)->Content(); - lambdas.push_back( - ctx.Builder(item->Pos()) - .Lambda() - .Param("row") - .Callable("AsStruct") - .List(0) - .Atom(0, targetName) - .Apply(1, item->ChildPtr(2)) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Seal() - .Build()); - } - } - - auto res = ctx.Builder(node->Pos()) - .Callable(node->IsCallable("SqlProject") ? "FlatMap" : "OrderedFlatMap") - .Add(0, node->ChildPtr(0)) - .Lambda(1) - .Param("row") - .Callable("AsList") - .Callable(0, "FlattenMembers") - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - size_t index = 0; - for (auto lambda: lambdas) { - parent - .List(index++) - .Atom(0, "") - .Apply(1, lambda) - .With(0, "row") - .Seal() - .Seal(); - } - return parent; - }) - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); - }; - - map["SqlFlattenByColumns"] = map["OrderedSqlFlattenByColumns"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - - auto row = ctx.NewArgument(node->Pos(), "row"); - - auto flattenByArgs = node->ChildrenList(); - flattenByArgs[0] = flattenByArgs[1]; - flattenByArgs[1] = row; - - auto body = ctx.NewCallable(node->Pos(), "FlattenByColumns", std::move(flattenByArgs)); - - auto res = ctx.Builder(node->Pos()) - .Callable(node->Content().StartsWith("Ordered") ? "OrderedFlatMap" : "FlatMap") - .Add(0, node->HeadPtr()) - .Add(1, ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { row }), std::move(body))) - .Seal() - .Build(); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); - }; - - map["SqlFlattenColumns"] = map["OrderedSqlFlattenColumns"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - - auto res = ctx.Builder(node->Pos()) - .Callable(node->Content().StartsWith("Ordered") ? "OrderedFlatMap" : "FlatMap") - .Add(0, node->HeadPtr()) - .Lambda(1) - .Param("row") - .Callable("Just") - .Callable(0, "FlattenStructs") - .Arg(0, "row") - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); - }; - - map["SqlAggregateAll"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - auto voidNode = ctx.NewCallable(node->Pos(), "Void", {}); - auto emptyTuple = ctx.NewList(node->Pos(), {}); - auto res = ctx.NewCallable(node->Pos(), "Aggregate", { node->HeadPtr(), voidNode, emptyTuple, emptyTuple}); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); - }; - - map["Mux"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - if (node->Head().IsList()) { - TExprNodeList children = node->Head().ChildrenList(); - bool found = false; - for (auto& child : children) { - if (child->IsCallable("AssumeColumnOrder")) { - found = true; - child = child->HeadPtr(); - } - } - - if (found) { - YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); - auto res = ctx.ChangeChild(*node, 0, ctx.NewList(node->Pos(), std::move(children))); - return KeepColumnOrder(res, *node, ctx, *optCtx.Types); - } - } - - return node; - }; - - map["UnionAllPositional"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - if (node->ChildrenSize() == 1) { - return node->HeadPtr(); - } - + + map["CalcOverWindow"] = map["CalcOverSessionWindow"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + TCoCalcOverWindowBase self(node); + auto frames = self.Frames(); + size_t sessionColumnsSize = 0; + if (auto maybeSession = TMaybeNode<TCoCalcOverSessionWindow>(node)) { + sessionColumnsSize = maybeSession.Cast().SessionColumns().Size(); + } + if (frames.Size() == 0 && sessionColumnsSize == 0) { + YQL_CLOG(DEBUG, Core) << node->Content() << " without payload"; + return self.Input().Ptr(); + } + + auto mergedFrames = MergeCalcOverWindowFrames(frames.Ptr(), ctx); + if (mergedFrames == frames.Ptr()) { + return node; + } + + YQL_CLOG(DEBUG, Core) << node->Content() << " with duplicate or empty frames"; + return ctx.ChangeChild(*node, TCoCalcOverWindowBase::idx_Frames, std::move(mergedFrames)); + }; + + map["CalcOverWindowGroup"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + TCoCalcOverWindowGroup self(node); + + auto dedupCalcs = DedupCalcOverWindowsOnSamePartitioning(self.Calcs().Ref().ChildrenList(), ctx); + YQL_ENSURE(dedupCalcs.size() <= self.Calcs().Size()); + + TExprNodeList mergedCalcs; + bool merged = false; + for (auto& calcNode : dedupCalcs) { + TCoCalcOverWindowTuple calc(calcNode); + + auto origFrames = calc.Frames().Ptr(); + auto mergedFrames = MergeCalcOverWindowFrames(origFrames, ctx); + if (mergedFrames != origFrames) { + merged = true; + mergedCalcs.emplace_back( + Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) + .Keys(calc.Keys()) + .SortSpec(calc.SortSpec()) + .Frames(mergedFrames) + .SessionSpec(calc.SessionSpec()) + .SessionColumns(calc.SessionColumns()) + .Done().Ptr() + ); + } else { + mergedCalcs.emplace_back(std::move(calcNode)); + } + } + + if (merged || dedupCalcs.size() < self.Calcs().Size()) { + YQL_CLOG(DEBUG, Core) << "CalcOverWindowGroup with duplicate/empty frames and/or duplicate windows"; + return BuildCalcOverWindowGroup(self, std::move(mergedCalcs), ctx); + } + + if (mergedCalcs.size() <= 1) { + TStringBuf msg = mergedCalcs.empty() ? "CalcOverWindowGroup without windows" : "CalcOverWindowGroup with single window"; + YQL_CLOG(DEBUG, Core) << msg; + return BuildCalcOverWindowGroup(self, std::move(mergedCalcs), ctx); + } + + return node; + }; + + map["AssumeColumnOrder"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + auto input = node->HeadPtr(); + if (input->IsCallable("AssumeColumnOrder")) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << input->Content(); + return ctx.ChangeChild(*node, 0u, input->HeadPtr()); + } + + return node; + }; + + map["SqlProject"] = map["OrderedSqlProject"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + + TExprNodeList lambdas; + for (auto& item : node->Child(1)->Children()) { + YQL_ENSURE(item->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})); + YQL_ENSURE(item->Child(1)->IsAtom()); + YQL_ENSURE(item->Child(2)->IsLambda()); + if (item->IsCallable("SqlProjectStarItem")) { + lambdas.push_back(item->ChildPtr(2)); + } else { + auto targetName = item->Child(1)->Content(); + lambdas.push_back( + ctx.Builder(item->Pos()) + .Lambda() + .Param("row") + .Callable("AsStruct") + .List(0) + .Atom(0, targetName) + .Apply(1, item->ChildPtr(2)) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Seal() + .Build()); + } + } + + auto res = ctx.Builder(node->Pos()) + .Callable(node->IsCallable("SqlProject") ? "FlatMap" : "OrderedFlatMap") + .Add(0, node->ChildPtr(0)) + .Lambda(1) + .Param("row") + .Callable("AsList") + .Callable(0, "FlattenMembers") + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t index = 0; + for (auto lambda: lambdas) { + parent + .List(index++) + .Atom(0, "") + .Apply(1, lambda) + .With(0, "row") + .Seal() + .Seal(); + } + return parent; + }) + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + }; + + map["SqlFlattenByColumns"] = map["OrderedSqlFlattenByColumns"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + + auto row = ctx.NewArgument(node->Pos(), "row"); + + auto flattenByArgs = node->ChildrenList(); + flattenByArgs[0] = flattenByArgs[1]; + flattenByArgs[1] = row; + + auto body = ctx.NewCallable(node->Pos(), "FlattenByColumns", std::move(flattenByArgs)); + + auto res = ctx.Builder(node->Pos()) + .Callable(node->Content().StartsWith("Ordered") ? "OrderedFlatMap" : "FlatMap") + .Add(0, node->HeadPtr()) + .Add(1, ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { row }), std::move(body))) + .Seal() + .Build(); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + }; + + map["SqlFlattenColumns"] = map["OrderedSqlFlattenColumns"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + + auto res = ctx.Builder(node->Pos()) + .Callable(node->Content().StartsWith("Ordered") ? "OrderedFlatMap" : "FlatMap") + .Add(0, node->HeadPtr()) + .Lambda(1) + .Param("row") + .Callable("Just") + .Callable(0, "FlattenStructs") + .Arg(0, "row") + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + }; + + map["SqlAggregateAll"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + auto voidNode = ctx.NewCallable(node->Pos(), "Void", {}); + auto emptyTuple = ctx.NewList(node->Pos(), {}); + auto res = ctx.NewCallable(node->Pos(), "Aggregate", { node->HeadPtr(), voidNode, emptyTuple, emptyTuple}); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + }; + + map["Mux"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + if (node->Head().IsList()) { + TExprNodeList children = node->Head().ChildrenList(); + bool found = false; + for (auto& child : children) { + if (child->IsCallable("AssumeColumnOrder")) { + found = true; + child = child->HeadPtr(); + } + } + + if (found) { + YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); + auto res = ctx.ChangeChild(*node, 0, ctx.NewList(node->Pos(), std::move(children))); + return KeepColumnOrder(res, *node, ctx, *optCtx.Types); + } + } + + return node; + }; + + map["UnionAllPositional"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + if (node->ChildrenSize() == 1) { + return node->HeadPtr(); + } + TVector<TColumnOrder> columnOrders; for (auto child : node->Children()) { - auto childColumnOrder = optCtx.Types->LookupColumnOrder(*child); - YQL_ENSURE(childColumnOrder); + auto childColumnOrder = optCtx.Types->LookupColumnOrder(*child); + YQL_ENSURE(childColumnOrder); columnOrders.push_back(*childColumnOrder); - } - + } + return ExpandPositionalUnionAll(*node, columnOrders, node->ChildrenList(), ctx, optCtx); - }; - + }; + map["MapJoinCore"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { if (const auto& inputToCheck = SkipCallables(node->Head(), SkippableCallables); IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) { YQL_CLOG(DEBUG, Core) << "Empty " << node->Content(); @@ -6201,42 +6201,42 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["RangeIntersect"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { - if (node->ChildrenSize() == 1) { - YQL_CLOG(DEBUG, Core) << "Single arg " << node->Content(); - return node->HeadPtr(); - } - return node; - }; - - map["RangeUnion"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { - if (node->ChildrenSize() == 1) { - if (node->Head().IsCallable("RangeUnion")) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return node->HeadPtr(); - } - - if (node->Head().IsCallable("RangeMultiply")) { - auto children = node->Head().ChildrenList(); - YQL_ENSURE(children.size() > 1); - if (AllOf(children.begin() + 1, children.end(), [](const auto& child) { return child->IsCallable("RangeUnion"); })) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return node->HeadPtr(); - } - } - } - return node; - }; - - map["RangeMultiply"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - if (node->ChildrenSize() == 2 && node->Tail().IsCallable("RangeMultiply")) { - auto minLimit = ctx.NewCallable(node->Pos(), "Min", { node->HeadPtr(), node->Tail().HeadPtr() }); - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Tail().Content(); - return ctx.ChangeChild(node->Tail(), 0, std::move(minLimit)); - } - return node; - }; - + map["RangeIntersect"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { + if (node->ChildrenSize() == 1) { + YQL_CLOG(DEBUG, Core) << "Single arg " << node->Content(); + return node->HeadPtr(); + } + return node; + }; + + map["RangeUnion"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { + if (node->ChildrenSize() == 1) { + if (node->Head().IsCallable("RangeUnion")) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return node->HeadPtr(); + } + + if (node->Head().IsCallable("RangeMultiply")) { + auto children = node->Head().ChildrenList(); + YQL_ENSURE(children.size() > 1); + if (AllOf(children.begin() + 1, children.end(), [](const auto& child) { return child->IsCallable("RangeUnion"); })) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return node->HeadPtr(); + } + } + } + return node; + }; + + map["RangeMultiply"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + if (node->ChildrenSize() == 2 && node->Tail().IsCallable("RangeMultiply")) { + auto minLimit = ctx.NewCallable(node->Pos(), "Min", { node->HeadPtr(), node->Tail().HeadPtr() }); + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Tail().Content(); + return ctx.ChangeChild(node->Tail(), 0, std::move(minLimit)); + } + return node; + }; + map["PgSelect"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) -> TExprNode::TPtr { auto setItems = GetSetting(node->Head(), "set_items"); auto order = optCtx.Types->LookupColumnOrder(*node); @@ -7177,60 +7177,60 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { .Build(); }; - map["SqlColumnOrType"] = map["SqlPlainColumnOrType"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - YQL_CLOG(DEBUG, Core) << "Decay of never inspected " << node->Content(); - return ctx.NewCallable(node->Pos(), "Error", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) }); - }; - - map["SqlColumnFromType"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { - YQL_CLOG(DEBUG, Core) << "Decay of " << node->Content(); - return ctx.NewCallable(node->Pos(), "Member", { node->HeadPtr(), node->ChildPtr(1) }); - }; - - // will be applied to any callable after all above - map[""] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - YQL_ENSURE(node->IsCallable()); - - if (AnyOf(node->ChildrenList(), [](const auto& child) { return child->IsCallable("AssumeColumnOrder"); })) { - auto type = node->GetTypeAnn(); - if (type->GetKind() == ETypeAnnotationKind::World) { - // stop on world - return node; - } - - // push over sequence-of-structs or tuple(world, sequence-of-structs) - if (type->GetKind() == ETypeAnnotationKind::Tuple) { - auto tupleType = type->Cast<TTupleExprType>(); - if (tupleType->GetSize() == 2 && tupleType->GetItems()[0]->GetKind() == ETypeAnnotationKind::World) { - type = tupleType->GetItems()[1]; - } - } - - if (type->GetKind() != ETypeAnnotationKind::Struct) { - type = GetItemType(*type); - } - - - auto newChildren = node->ChildrenList(); - for (auto& child : newChildren) { - if (child->IsCallable("AssumeColumnOrder")) { - child = child->HeadPtr(); - } - } - - auto result = ctx.ChangeChildren(*node, std::move(newChildren)); - if (type && type->GetKind() == ETypeAnnotationKind::Struct) { - YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); - result = KeepColumnOrder(result, *node, ctx, *optCtx.Types); - } else { - YQL_CLOG(DEBUG, Core) << "Drop AssumeColumnOrder as input of " << node->Content(); - } - - return result; - } - - return node; - }; + map["SqlColumnOrType"] = map["SqlPlainColumnOrType"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + YQL_CLOG(DEBUG, Core) << "Decay of never inspected " << node->Content(); + return ctx.NewCallable(node->Pos(), "Error", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) }); + }; + + map["SqlColumnFromType"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) { + YQL_CLOG(DEBUG, Core) << "Decay of " << node->Content(); + return ctx.NewCallable(node->Pos(), "Member", { node->HeadPtr(), node->ChildPtr(1) }); + }; + + // will be applied to any callable after all above + map[""] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + YQL_ENSURE(node->IsCallable()); + + if (AnyOf(node->ChildrenList(), [](const auto& child) { return child->IsCallable("AssumeColumnOrder"); })) { + auto type = node->GetTypeAnn(); + if (type->GetKind() == ETypeAnnotationKind::World) { + // stop on world + return node; + } + + // push over sequence-of-structs or tuple(world, sequence-of-structs) + if (type->GetKind() == ETypeAnnotationKind::Tuple) { + auto tupleType = type->Cast<TTupleExprType>(); + if (tupleType->GetSize() == 2 && tupleType->GetItems()[0]->GetKind() == ETypeAnnotationKind::World) { + type = tupleType->GetItems()[1]; + } + } + + if (type->GetKind() != ETypeAnnotationKind::Struct) { + type = GetItemType(*type); + } + + + auto newChildren = node->ChildrenList(); + for (auto& child : newChildren) { + if (child->IsCallable("AssumeColumnOrder")) { + child = child->HeadPtr(); + } + } + + auto result = ctx.ChangeChildren(*node, std::move(newChildren)); + if (type && type->GetKind() == ETypeAnnotationKind::Struct) { + YQL_CLOG(DEBUG, Core) << "Pull AssumeColumnOrder over " << node->Content(); + result = KeepColumnOrder(result, *node, ctx, *optCtx.Types); + } else { + YQL_CLOG(DEBUG, Core) << "Drop AssumeColumnOrder as input of " << node->Content(); + } + + return result; + } + + return node; + }; } } // namespace NYql diff --git a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp index 66e0ca085d..6d59989b6a 100644 --- a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp @@ -307,68 +307,68 @@ TExprNode::TPtr OptimizeNot(const TExprNode::TPtr& node, TExprContext& ctx) { {"AggrGreater", "AggrLessOrEqual"}, }; - auto& arg = node->Head(); - const auto it = InverseComparators.find(arg.Content()); + auto& arg = node->Head(); + const auto it = InverseComparators.find(arg.Content()); if (InverseComparators.cend() != it && ( - (arg.Content().front() != '<' && arg.Content().front() != '>') || - (HasTotalOrder(*arg.Head().GetTypeAnn()) && HasTotalOrder(*arg.Tail().GetTypeAnn())))) - { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << arg.Content(); - return ctx.RenameNode(arg, it->second); + (arg.Content().front() != '<' && arg.Content().front() != '>') || + (HasTotalOrder(*arg.Head().GetTypeAnn()) && HasTotalOrder(*arg.Tail().GetTypeAnn())))) + { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << arg.Content(); + return ctx.RenameNode(arg, it->second); } return node; } -TExprNode::TPtr OptimizeAnd(const TExprNode::TPtr& node, TExprContext& ctx) { - if (auto opt = OptimizeDups(node, ctx); opt != node) { - return opt; - } - - TExprNodeList children = node->ChildrenList(); - TNodeMap<size_t> exists; - TNodeSet toReplace; - for (size_t i = 0; i < children.size(); ++i) { - const auto& child = children[i]; - if (child->IsCallable("Exists")) { - auto pred = child->Child(0); - YQL_ENSURE(!exists.contains(pred)); - exists[pred] = i; - } else if (child->IsCallable("Unwrap")) { - auto pred = child->Child(0); - if (exists.contains(pred)) { - toReplace.insert(pred); - } - } - } - - if (toReplace.empty()) { - return node; - } - - TExprNodeList newChildren; - for (size_t i = 0; i < children.size(); ++i) { - const auto& child = children[i]; - if (child->IsCallable({"Exists", "Unwrap"})) { - auto pred = child->HeadPtr(); - auto it = exists.find(pred.Get()); - if (it != exists.end() && toReplace.contains(it->first)) { - if (i == it->second) { - newChildren.push_back(ctx.NewCallable(pred->Pos(), "Coalesce", { pred, MakeBool<false>(pred->Pos(), ctx)})); - continue; - } - if (i > it->second) { - continue; - } - } - } - newChildren.push_back(child); - } - - YQL_CLOG(DEBUG, Core) << "Exist(pred) AND Unwrap(pred) -> Coalesce(pred, false)"; - return ctx.ChangeChildren(*node, std::move(newChildren)); -} - +TExprNode::TPtr OptimizeAnd(const TExprNode::TPtr& node, TExprContext& ctx) { + if (auto opt = OptimizeDups(node, ctx); opt != node) { + return opt; + } + + TExprNodeList children = node->ChildrenList(); + TNodeMap<size_t> exists; + TNodeSet toReplace; + for (size_t i = 0; i < children.size(); ++i) { + const auto& child = children[i]; + if (child->IsCallable("Exists")) { + auto pred = child->Child(0); + YQL_ENSURE(!exists.contains(pred)); + exists[pred] = i; + } else if (child->IsCallable("Unwrap")) { + auto pred = child->Child(0); + if (exists.contains(pred)) { + toReplace.insert(pred); + } + } + } + + if (toReplace.empty()) { + return node; + } + + TExprNodeList newChildren; + for (size_t i = 0; i < children.size(); ++i) { + const auto& child = children[i]; + if (child->IsCallable({"Exists", "Unwrap"})) { + auto pred = child->HeadPtr(); + auto it = exists.find(pred.Get()); + if (it != exists.end() && toReplace.contains(it->first)) { + if (i == it->second) { + newChildren.push_back(ctx.NewCallable(pred->Pos(), "Coalesce", { pred, MakeBool<false>(pred->Pos(), ctx)})); + continue; + } + if (i > it->second) { + continue; + } + } + } + newChildren.push_back(child); + } + + YQL_CLOG(DEBUG, Core) << "Exist(pred) AND Unwrap(pred) -> Coalesce(pred, false)"; + return ctx.ChangeChildren(*node, std::move(newChildren)); +} + TExprNode::TPtr CheckIfWorldWithSame(const TExprNode::TPtr& node, TExprContext& ctx) { if (node->Child(3U) == node->Child(2U)) { YQL_CLOG(DEBUG, Core) << node->Content() << " with identical branches"; @@ -472,8 +472,8 @@ void RegisterCoSimpleCallables2(TCallableOptimizerMap& map) { map["Xor"] = std::bind(&OptimizeXor, _1, _2); map["Not"] = std::bind(&OptimizeNot, _1, _2); - map["And"] = std::bind(&OptimizeAnd, _1, _2); - map["Or"] = std::bind(OptimizeDups, _1, _2); + map["And"] = std::bind(&OptimizeAnd, _1, _2); + map["Or"] = std::bind(OptimizeDups, _1, _2); map["Min"] = map["Max"] = std::bind(&OptimizeDups, _1, _2); diff --git a/ydb/library/yql/core/common_opt/yql_co_transformer.cpp b/ydb/library/yql/core/common_opt/yql_co_transformer.cpp index fd60bca4e1..8a98716eac 100644 --- a/ydb/library/yql/core/common_opt/yql_co_transformer.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_transformer.cpp @@ -19,7 +19,7 @@ namespace { class TCommonOptTransformer final : public TSyncTransformerBase { public: - TCommonOptTransformer(TTypeAnnotationContext* typeCtx, bool final) + TCommonOptTransformer(TTypeAnnotationContext* typeCtx, bool final) : TypeCtx(typeCtx) , Final(final) {} @@ -37,29 +37,29 @@ private: private: TProcessedNodesSet SimpleProcessedNodes[TCoCallableRules::SIMPLE_STEPS]; TProcessedNodesSet FlowProcessedNodes[TCoCallableRules::FLOW_STEPS]; - TProcessedNodesSet FinalProcessedNodes; + TProcessedNodesSet FinalProcessedNodes; TTypeAnnotationContext* TypeCtx; - const bool Final; + const bool Final; }; } TAutoPtr<IGraphTransformer> CreateCommonOptTransformer(TTypeAnnotationContext* typeCtx) { - return new TCommonOptTransformer(typeCtx, false); + return new TCommonOptTransformer(typeCtx, false); +} + +TAutoPtr<IGraphTransformer> CreateCommonOptFinalTransformer(TTypeAnnotationContext* typeCtx) { + return new TCommonOptTransformer(typeCtx, true); } -TAutoPtr<IGraphTransformer> CreateCommonOptFinalTransformer(TTypeAnnotationContext* typeCtx) { - return new TCommonOptTransformer(typeCtx, true); -} - IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) { IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; output = std::move(input); - - if (Final) { - return DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FinalCallables, FinalProcessedNodes, true); - } - + + if (Final) { + return DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FinalCallables, FinalProcessedNodes, true); + } + for (ui32 i = 0; i < TCoCallableRules::SIMPLE_STEPS; ++i) { status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().SimpleCallables[i], SimpleProcessedNodes[i], true); if (status.Level != IGraphTransformer::TStatus::Ok) { @@ -110,24 +110,24 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform( optCtx.ParentsMap = &parentsMap; } - TCallableOptimizerExt defaultOpt; - auto defaultIt = callables.find(""); - if (defaultIt != callables.end()) { - defaultOpt = defaultIt->second; - } - - return OptimizeExpr(input, output, [&callables, &optCtx, defaultOpt](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + TCallableOptimizerExt defaultOpt; + auto defaultIt = callables.find(""); + if (defaultIt != callables.end()) { + defaultOpt = defaultIt->second; + } + + return OptimizeExpr(input, output, [&callables, &optCtx, defaultOpt](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { const auto rule = callables.find(node->Content()); - TExprNode::TPtr result = node; - if (rule != callables.cend()) { - result = (rule->second)(node, ctx, optCtx); - } - - if (defaultOpt && result == node) { - result = defaultOpt(node, ctx, optCtx); - } - - return result; + TExprNode::TPtr result = node; + if (rule != callables.cend()) { + result = (rule->second)(node, ctx, optCtx); + } + + if (defaultOpt && result == node) { + result = defaultOpt(node, ctx, optCtx); + } + + return result; }, ctx, settings); } @@ -177,7 +177,7 @@ TCoCallableRules::TCoCallableRules() { RegisterCoFlowCallables1(FlowCallables[FLOW_STEP_1]); RegisterCoFlowCallables2(FlowCallables[FLOW_STEP_2]); RegisterCoFinalizers(Finalizers); - RegisterCoFinalCallables(FinalCallables); + RegisterCoFinalCallables(FinalCallables); } } diff --git a/ydb/library/yql/core/common_opt/yql_co_transformer.h b/ydb/library/yql/core/common_opt/yql_co_transformer.h index 76b8d7d26f..a2e8449e1e 100644 --- a/ydb/library/yql/core/common_opt/yql_co_transformer.h +++ b/ydb/library/yql/core/common_opt/yql_co_transformer.h @@ -9,6 +9,6 @@ namespace NYql { TAutoPtr<IGraphTransformer> CreateCommonOptTransformer(TTypeAnnotationContext* typeCtx); -TAutoPtr<IGraphTransformer> CreateCommonOptFinalTransformer(TTypeAnnotationContext* typeCtx); +TAutoPtr<IGraphTransformer> CreateCommonOptFinalTransformer(TTypeAnnotationContext* typeCtx); } diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h index d783488706..01cc777118 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h @@ -53,7 +53,7 @@ template<typename TParent> class TNodeBuilder<TParent, TCoWorld> : public NGenerated::TCoWorldBuilder<TParent> { public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, typename NGenerated::TCoWorldBuilder<TParent>::BuildFuncType buildFunc, typename NGenerated::TCoWorldBuilder<TParent>::GetArgFuncType getArgFunc) : NGenerated::TCoWorldBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} @@ -68,7 +68,7 @@ template<typename TParent> class TNodeBuilder<TParent, TCoArgument> : public NGenerated::TCoArgumentBuilder<TParent> { public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, typename NGenerated::TCoArgumentBuilder<TParent>::BuildFuncType buildFunc, typename NGenerated::TCoArgumentBuilder<TParent>::GetArgFuncType getArgFunc) : NGenerated::TCoArgumentBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} @@ -92,7 +92,7 @@ template<typename TParent> class TNodeBuilder<TParent, TCoAtom> : public NGenerated::TCoAtomBuilder<TParent> { public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, typename NGenerated::TCoAtomBuilder<TParent>::BuildFuncType buildFunc, typename NGenerated::TCoAtomBuilder<TParent>::GetArgFuncType getArgFunc) : NGenerated::TCoAtomBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} @@ -131,10 +131,10 @@ template<typename TParent> class TNodeBuilder<TParent, TCoLambda> : public NGenerated::TCoLambdaBuilder<TParent> { public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, typename NGenerated::TCoLambdaBuilder<TParent>::BuildFuncType buildFunc, typename NGenerated::TCoLambdaBuilder<TParent>::GetArgFuncType getArgFunc, - std::shared_ptr<TMap<TStringBuf, TExprBase>> argsStore = std::make_shared<TMap<TStringBuf, TExprBase>>()) + std::shared_ptr<TMap<TStringBuf, TExprBase>> argsStore = std::make_shared<TMap<TStringBuf, TExprBase>>()) : NGenerated::TCoLambdaBuilder<TParent>(ctx, pos, buildFunc, [argsStore, getArgFunc] (const TStringBuf& argName) { if (auto res = argsStore->FindPtr(argName)) { return *res; @@ -209,7 +209,7 @@ public: } private: - std::shared_ptr<TMap<TStringBuf, TExprBase>> ArgsMap; + std::shared_ptr<TMap<TStringBuf, TExprBase>> ArgsMap; }; class TExprApplier : public TExprBase { @@ -250,7 +250,7 @@ public: typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType; typedef TExprApplier ResultType; - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) , BuildFunc(buildFunc) {} diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index b5a94e21e4..edfcaa1530 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -138,15 +138,15 @@ ] }, { - "Name": "TCoImplicitEquals", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "ImplicitEquals"}, - "Children": [ - {"Index": 0, "Name": "Value", "Type": "TExprBase"}, - {"Index": 1, "Name": "CommonType", "Type": "TExprBase"} - ] - }, - { + "Name": "TCoImplicitEquals", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "ImplicitEquals"}, + "Children": [ + {"Index": 0, "Name": "Value", "Type": "TExprBase"}, + {"Index": 1, "Name": "CommonType", "Type": "TExprBase"} + ] + }, + { "Name": "TCoNothing", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "Nothing"}, @@ -320,24 +320,24 @@ "Match": {"Type": "Callable", "Name": "OrderedFlatMap"} }, { - "Name": "TCoFlatMapToEquiJoinBase", - "Base": "TCoInputBase", - "Match": {"Type": "CallableBase"}, - "Children": [ - {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"} - ] - }, - { - "Name": "TCoFlatMapToEquiJoin", - "Base": "TCoFlatMapToEquiJoinBase", - "Match": {"Type": "Callable", "Name": "FlatMapToEquiJoin"} - }, - { - "Name": "TCoOrderedFlatMapToEquiJoin", - "Base": "TCoFlatMapToEquiJoinBase", - "Match": {"Type": "Callable", "Name": "OrderedFlatMapToEquiJoin"} - }, - { + "Name": "TCoFlatMapToEquiJoinBase", + "Base": "TCoInputBase", + "Match": {"Type": "CallableBase"}, + "Children": [ + {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"} + ] + }, + { + "Name": "TCoFlatMapToEquiJoin", + "Base": "TCoFlatMapToEquiJoinBase", + "Match": {"Type": "Callable", "Name": "FlatMapToEquiJoin"} + }, + { + "Name": "TCoOrderedFlatMapToEquiJoin", + "Base": "TCoFlatMapToEquiJoinBase", + "Match": {"Type": "Callable", "Name": "OrderedFlatMapToEquiJoin"} + }, + { "Name": "TCoCollect", "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "Collect"} @@ -353,21 +353,21 @@ "Match": {"Type": "Callable", "Name": "Last"} }, { - "Name": "TCoUnorderedBase", - "Base": "TCoInputBase", - "Match": {"Type": "CallableBase"} - }, - { + "Name": "TCoUnorderedBase", + "Base": "TCoInputBase", + "Match": {"Type": "CallableBase"} + }, + { "Name": "TCoUnordered", - "Base": "TCoUnorderedBase", + "Base": "TCoUnorderedBase", "Match": {"Type": "Callable", "Name": "Unordered"} }, { - "Name": "TCoUnorderedSubquery", - "Base": "TCoUnorderedBase", - "Match": {"Type": "Callable", "Name": "UnorderedSubquery"} - }, - { + "Name": "TCoUnorderedSubquery", + "Base": "TCoUnorderedBase", + "Match": {"Type": "Callable", "Name": "UnorderedSubquery"} + }, + { "Name": "TCoConditionalValueBase", "Base": "TCallable", "Match": {"Type": "CallableBase"}, @@ -538,16 +538,16 @@ "Match": {"Type": "Callable", "Name": "PartitionsByKeys"} }, { - "Name": "TCoSortTraits", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SortTraits"}, - "Children": [ - {"Index": 0, "Name": "ListType", "Type": "TExprBase"}, - {"Index": 1, "Name": "SortDirections", "Type": "TExprBase"}, - {"Index": 2, "Name": "SortKeySelectorLambda", "Type": "TCoLambda"} - ] - }, - { + "Name": "TCoSortTraits", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SortTraits"}, + "Children": [ + {"Index": 0, "Name": "ListType", "Type": "TExprBase"}, + {"Index": 1, "Name": "SortDirections", "Type": "TExprBase"}, + {"Index": 2, "Name": "SortKeySelectorLambda", "Type": "TCoLambda"} + ] + }, + { "Name": "TCoHoppingTraits", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "HoppingTraits"}, @@ -561,18 +561,18 @@ ] }, { - "Name": "TCoSessionWindowTraits", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SessionWindowTraits"}, - "Children": [ - {"Index": 0, "Name": "ListType", "Type": "TExprBase"}, - {"Index": 1, "Name": "SortSpec", "Type": "TExprBase"}, - {"Index": 2, "Name": "InitState", "Type": "TCoLambda"}, - {"Index": 3, "Name": "UpdateState", "Type": "TCoLambda"}, - {"Index": 4, "Name": "Calculate", "Type": "TCoLambda"} - ] - }, - { + "Name": "TCoSessionWindowTraits", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SessionWindowTraits"}, + "Children": [ + {"Index": 0, "Name": "ListType", "Type": "TExprBase"}, + {"Index": 1, "Name": "SortSpec", "Type": "TExprBase"}, + {"Index": 2, "Name": "InitState", "Type": "TCoLambda"}, + {"Index": 3, "Name": "UpdateState", "Type": "TCoLambda"}, + {"Index": 4, "Name": "Calculate", "Type": "TCoLambda"} + ] + }, + { "Name": "TCoAggregationTraits", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "AggregationTraits"}, @@ -1015,32 +1015,32 @@ ] }, { - "Name": "TCoLookupBase", + "Name": "TCoLookupBase", "Base": "TCallable", - "Match": {"Type": "CallableBase"}, - "Builder": {"Generate": "None"}, + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, "Children": [ - {"Index": 0, "Name": "Collection", "Type": "TExprBase"}, + {"Index": 0, "Name": "Collection", "Type": "TExprBase"}, {"Index": 1, "Name": "Lookup", "Type": "TExprBase"} ] }, { - "Name": "TCoContains", - "Base": "TCoLookupBase", + "Name": "TCoContains", + "Base": "TCoLookupBase", "Match": {"Type": "Callable", "Name": "Contains"} - }, - { - "Name": "TCoSqlIn", - "Base": "TCoLookupBase", - "Match": {"Type": "Callable", "Name": "SqlIn"}, - "Children": [ - {"Index": 2, "Name": "Options", "Type": "TCoNameValueTupleList"} - ] - }, - { + }, + { + "Name": "TCoSqlIn", + "Base": "TCoLookupBase", + "Match": {"Type": "Callable", "Name": "SqlIn"}, + "Children": [ + {"Index": 2, "Name": "Options", "Type": "TCoNameValueTupleList"} + ] + }, + { "Name": "TCoLookup", - "Base": "TCoLookupBase", - "Match": {"Type": "Callable", "Name": "Lookup"} + "Base": "TCoLookupBase", + "Match": {"Type": "Callable", "Name": "Lookup"} }, { "Name": "TCoMember", @@ -1289,10 +1289,10 @@ ] }, { - "Name": "TCoCalcOverWindowBase", + "Name": "TCoCalcOverWindowBase", "Base": "TCoInputBase", - "Match": {"Type": "CallableBase"}, - "Builder": {"Generate": "None"}, + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, "Children": [ {"Index": 1, "Name": "Keys", "Type": "TCoAtomList"}, {"Index": 2, "Name": "SortSpec", "Type": "TExprBase"}, @@ -1300,44 +1300,44 @@ ] }, { - "Name": "TCoCalcOverWindow", - "Base": "TCoCalcOverWindowBase", - "Match": {"Type": "Callable", "Name": "CalcOverWindow"} - }, - { - "Name": "TCoCalcOverSessionWindow", - "Base": "TCoCalcOverWindowBase", - "Match": {"Type": "Callable", "Name": "CalcOverSessionWindow"}, - "Children": [ - {"Index": 4, "Name": "SessionSpec", "Type": "TExprBase"}, - {"Index": 5, "Name": "SessionColumns", "Type": "TCoAtomList"} - ] - }, - { - "Name": "TCoCalcOverWindowTuple", - "Base": "TExprBase", - "Match": {"Type": "Tuple"}, - "Children": [ - {"Index": 0, "Name": "Keys", "Type": "TCoAtomList"}, - {"Index": 1, "Name": "SortSpec", "Type": "TExprBase"}, - {"Index": 2, "Name": "Frames", "Type": "TExprList"}, - {"Index": 3, "Name": "SessionSpec", "Type": "TExprBase"}, - {"Index": 4, "Name": "SessionColumns", "Type": "TCoAtomList"} - ] - }, - { - "Name": "TCoCalcOverWindowTupleList", - "ListBase": "TCoCalcOverWindowTuple" - }, - { - "Name": "TCoCalcOverWindowGroup", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "CalcOverWindowGroup"}, - "Children": [ - {"Index": 1, "Name": "Calcs", "Type": "TCoCalcOverWindowTupleList"} - ] - }, - { + "Name": "TCoCalcOverWindow", + "Base": "TCoCalcOverWindowBase", + "Match": {"Type": "Callable", "Name": "CalcOverWindow"} + }, + { + "Name": "TCoCalcOverSessionWindow", + "Base": "TCoCalcOverWindowBase", + "Match": {"Type": "Callable", "Name": "CalcOverSessionWindow"}, + "Children": [ + {"Index": 4, "Name": "SessionSpec", "Type": "TExprBase"}, + {"Index": 5, "Name": "SessionColumns", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TCoCalcOverWindowTuple", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Keys", "Type": "TCoAtomList"}, + {"Index": 1, "Name": "SortSpec", "Type": "TExprBase"}, + {"Index": 2, "Name": "Frames", "Type": "TExprList"}, + {"Index": 3, "Name": "SessionSpec", "Type": "TExprBase"}, + {"Index": 4, "Name": "SessionColumns", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TCoCalcOverWindowTupleList", + "ListBase": "TCoCalcOverWindowTuple" + }, + { + "Name": "TCoCalcOverWindowGroup", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "CalcOverWindowGroup"}, + "Children": [ + {"Index": 1, "Name": "Calcs", "Type": "TCoCalcOverWindowTupleList"} + ] + }, + { "Name": "TCoEquiJoinTuple", "Base": "TExprBase", "Match": {"Type": "Tuple"}, @@ -1511,22 +1511,22 @@ ] }, { - "Name": "TCoAssumeColumnOrder", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "AssumeColumnOrder"}, - "Children": [ - {"Index": 1, "Name": "ColumnOrder", "Type": "TCoAtomList"} - ] - }, - { - "Name": "TCoAssumeColumnOrderPartial", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "AssumeColumnOrderPartial"}, - "Children": [ - {"Index": 1, "Name": "ColumnOrder", "Type": "TCoAtomList"} - ] - }, - { + "Name": "TCoAssumeColumnOrder", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "AssumeColumnOrder"}, + "Children": [ + {"Index": 1, "Name": "ColumnOrder", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TCoAssumeColumnOrderPartial", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "AssumeColumnOrderPartial"}, + "Children": [ + {"Index": 1, "Name": "ColumnOrder", "Type": "TCoAtomList"} + ] + }, + { "Name": "TCoAssumeAllMembersNullableAtOnce", "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "AssumeAllMembersNullableAtOnce"} @@ -1954,16 +1954,16 @@ ] }, { - "Name": "TCoAsRange", - "VarArgBase": "TExprBase", - "Match": {"Type": "Callable", "Name": "AsRange"} + "Name": "TCoAsRange", + "VarArgBase": "TExprBase", + "Match": {"Type": "Callable", "Name": "AsRange"} }, { - "Name": "TCoRangeCreate", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "RangeCreate"}, + "Name": "TCoRangeCreate", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "RangeCreate"}, "Children": [ - {"Index": 0, "Name": "UserRange", "Type": "TExprBase"} + {"Index": 0, "Name": "UserRange", "Type": "TExprBase"} ] }, { diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h index e097dede5a..0974e8d2d3 100644 --- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h +++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h @@ -55,7 +55,7 @@ public: return TExprBase(Raw_); } - TPositionHandle Pos() const { + TPositionHandle Pos() const { return Raw_->Pos(); } @@ -127,10 +127,10 @@ private: template<typename TNode> class TChildIterator : public std::iterator<std::forward_iterator_tag, TNode> { public: - TChildIterator() - { - CurIt = EndIt = {}; - } + TChildIterator() + { + CurIt = EndIt = {}; + } TChildIterator(const TExprBase &node, size_t startIndex = 0) : CurIt(node.Ref().Children().begin() + startIndex) @@ -407,14 +407,14 @@ class TNodeBuilderBase { protected: typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType; - TNodeBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) + TNodeBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) : Ctx(ctx) , Pos(pos) , GetArgFunc(getArgFunc) {} protected: TExprContext& Ctx; - TPositionHandle Pos; + TPositionHandle Pos; GetArgFuncType GetArgFunc; }; @@ -423,7 +423,7 @@ class TListBuilderBase : public TNodeBuilderBase { protected: typedef std::function<TParent& (const TDerived&)> BuildFuncType; - TListBuilderBase(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) + TListBuilderBase(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) , BuildFunc(buildFunc) {} @@ -510,7 +510,7 @@ class TNodeBuilder<TParent, TVector<TExprBase>> : public TListBuilderBase<TParen public: typedef std::function<TParent& (const TVector<TExprBase>&)> BuildFuncType; - TNodeBuilder<TParent, TVector<TExprBase>>(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, + TNodeBuilder<TParent, TVector<TExprBase>>(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, TNodeBuilderBase::GetArgFuncType getArgFunc) : TListBuilderBase<TParent, TVector<TExprBase>, TExprBase>(ctx, pos, buildFunc, getArgFunc) {} @@ -522,7 +522,7 @@ public: template <typename TParent, typename TDerived> class TFreeArgCallableBuilderBase : public TNodeBuilderBase { protected: - TFreeArgCallableBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) + TFreeArgCallableBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) {} TVector<TExprBase> FreeArgsHolder; @@ -557,7 +557,7 @@ private: }; template<typename TNode> -TNodeBuilder<TBuildValueHolder<TNode>, TNode> Build(TExprContext& ctx, TPositionHandle pos) { +TNodeBuilder<TBuildValueHolder<TNode>, TNode> Build(TExprContext& ctx, TPositionHandle pos) { TBuildValueHolder<TNode> holder; TNodeBuilder<TBuildValueHolder<TNode>, TNode> builder(ctx, pos, [holder](const TNode& node) mutable -> TBuildValueHolder<TNode>& { diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj index 9de896333f..4b5733693f 100644 --- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj +++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj @@ -113,7 +113,7 @@ class {{ node.aux.stubBuilderName }} : public TBase { protected: typedef std::function<TParent& (const TDerived&)> BuildFuncType; - {{ node.aux.stubBuilderName }}(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, typename TBase::GetArgFuncType getArgFunc) + {{ node.aux.stubBuilderName }}(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, typename TBase::GetArgFuncType getArgFunc) : TBase(ctx, pos, getArgFunc) , BuildFunc(buildFunc) {} @@ -304,7 +304,7 @@ private: typedef typename NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::BuildFuncType BuildFuncType; typedef typename NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::GetArgFuncType GetArgFuncType; public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>(ctx, pos, buildFunc, getArgFunc) {} {% elif node.Builder.Kind == "List" %} template<typename TParent> @@ -314,7 +314,7 @@ private: typedef typename TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>::BuildFuncType BuildFuncType; typedef typename TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>::GetArgFuncType GetArgFuncType; public: - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>(ctx, pos, buildFunc, getArgFunc) {} {% else %} class UnknownBuilder { diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate.h b/ydb/library/yql/core/extract_predicate/extract_predicate.h index dce4b45020..0aa34dec77 100644 --- a/ydb/library/yql/core/extract_predicate/extract_predicate.h +++ b/ydb/library/yql/core/extract_predicate/extract_predicate.h @@ -1,35 +1,35 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/yql_type_annotation.h> - -namespace NYql { - -struct TPredicateExtractorSettings { - size_t MaxRanges = 10000; - bool MergeAdjacentPointRanges = true; - bool HaveNextValueCallable = false; -}; - -class IPredicateRangeExtractor { -public: - using TPtr = THolder<IPredicateRangeExtractor>; - - virtual bool Prepare(const TExprNode::TPtr& filterLambda, const TTypeAnnotationNode& rowType, - THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) = 0; - - struct TBuildResult { - TExprNode::TPtr ComputeNode; - TExprNode::TPtr PrunedLambda; - size_t UsedPrefixLen = 0; - TMaybe<size_t> ExpectedMaxRanges; - }; - - virtual TBuildResult BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx) const = 0; - - virtual ~IPredicateRangeExtractor() = default; -}; - -IPredicateRangeExtractor::TPtr MakePredicateRangeExtractor(const TPredicateExtractorSettings& settings = {}); - -} + +namespace NYql { + +struct TPredicateExtractorSettings { + size_t MaxRanges = 10000; + bool MergeAdjacentPointRanges = true; + bool HaveNextValueCallable = false; +}; + +class IPredicateRangeExtractor { +public: + using TPtr = THolder<IPredicateRangeExtractor>; + + virtual bool Prepare(const TExprNode::TPtr& filterLambda, const TTypeAnnotationNode& rowType, + THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) = 0; + + struct TBuildResult { + TExprNode::TPtr ComputeNode; + TExprNode::TPtr PrunedLambda; + size_t UsedPrefixLen = 0; + TMaybe<size_t> ExpectedMaxRanges; + }; + + virtual TBuildResult BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx) const = 0; + + virtual ~IPredicateRangeExtractor() = default; +}; + +IPredicateRangeExtractor::TPtr MakePredicateRangeExtractor(const TPredicateExtractorSettings& settings = {}); + +} diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.cpp b/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.cpp index d1272b4da9..67512884ec 100644 --- a/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.cpp +++ b/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.cpp @@ -1,89 +1,89 @@ -#include "extract_predicate_dbg.h" -#include "extract_predicate.h" - +#include "extract_predicate_dbg.h" +#include "extract_predicate.h" + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/core/yql_expr_type_annotation.h> - -namespace NYql { -namespace { - -TExprNode::TPtr ExpandRangeComputeFor(const TExprNode::TPtr& node, TExprContext& ctx, - const TIntrusivePtr<TTypeAnnotationContext>& typesCtx) -{ - if (!node->IsCallable("RangeComputeFor")) { - return node; - } - - YQL_CLOG(DEBUG, Core) << "Expanding " << node->Content(); - - TPositionHandle pos = node->Pos(); - TExprNode::TPtr result = ctx.NewCallable(pos, "Nothing", { ExpandType(pos, *node->GetTypeAnn(), ctx) }); - - TPredicateExtractorSettings settings; - settings.HaveNextValueCallable = true; - auto extractor = MakePredicateRangeExtractor(settings); - YQL_ENSURE(extractor); - - TVector<TString> indexKeys; - for (auto& key : node->Tail().ChildrenList()) { - YQL_ENSURE(key->IsAtom()); - indexKeys.push_back(ToString(key->Content())); - } - - THashSet<TString> possibleKeys; - if (!extractor->Prepare(node->ChildPtr(1), *node->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), possibleKeys, ctx, *typesCtx)) { - YQL_CLOG(DEBUG, Core) << "Prepare: ranges can not be built for predicate"; - return result; - } - - if (AllOf(indexKeys, [&possibleKeys](const TString& key) { return !possibleKeys.contains(key); })) { - YQL_CLOG(DEBUG, Core) << "No intersection between possible range keys and actual keys"; - return result; - } - - - auto buildResult = extractor->BuildComputeNode(indexKeys, ctx); - if (!buildResult.ComputeNode) { - YQL_CLOG(DEBUG, Core) << "BuildComputeNode: ranges can not be built for predicate"; - return result; - } - - TString prunedLambdaSerialized; - { - auto ast = ConvertToAst(*buildResult.PrunedLambda, ctx, TExprAnnotationFlags::None, true); - YQL_ENSURE(ast.Root); - prunedLambdaSerialized = "__yql_ast:" + ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); - } - - result = ctx.Builder(pos) - .Callable("Just") - .List(0) - .Add(0, buildResult.ComputeNode) - .Callable(1, "AsTagged") - .Callable(0, "String") - .Atom(0, prunedLambdaSerialized) - .Seal() - .Atom(1, "AST", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Build(); - - return result; -} - -} // namespace - -THolder<IGraphTransformer> MakeExpandRangeComputeForTransformer(const TIntrusivePtr<TTypeAnnotationContext>& types) { - return CreateFunctorTransformer( - [types](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - TOptimizeExprSettings settings(types.Get()); - settings.CustomInstantTypeTransformer = types->CustomInstantTypeTransformer.Get(); - using namespace std::placeholders; - return OptimizeExpr(input, output, std::bind(&ExpandRangeComputeFor, _1, _2, types), ctx, settings); - } - ); -} - -} // namespace NYql + +namespace NYql { +namespace { + +TExprNode::TPtr ExpandRangeComputeFor(const TExprNode::TPtr& node, TExprContext& ctx, + const TIntrusivePtr<TTypeAnnotationContext>& typesCtx) +{ + if (!node->IsCallable("RangeComputeFor")) { + return node; + } + + YQL_CLOG(DEBUG, Core) << "Expanding " << node->Content(); + + TPositionHandle pos = node->Pos(); + TExprNode::TPtr result = ctx.NewCallable(pos, "Nothing", { ExpandType(pos, *node->GetTypeAnn(), ctx) }); + + TPredicateExtractorSettings settings; + settings.HaveNextValueCallable = true; + auto extractor = MakePredicateRangeExtractor(settings); + YQL_ENSURE(extractor); + + TVector<TString> indexKeys; + for (auto& key : node->Tail().ChildrenList()) { + YQL_ENSURE(key->IsAtom()); + indexKeys.push_back(ToString(key->Content())); + } + + THashSet<TString> possibleKeys; + if (!extractor->Prepare(node->ChildPtr(1), *node->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), possibleKeys, ctx, *typesCtx)) { + YQL_CLOG(DEBUG, Core) << "Prepare: ranges can not be built for predicate"; + return result; + } + + if (AllOf(indexKeys, [&possibleKeys](const TString& key) { return !possibleKeys.contains(key); })) { + YQL_CLOG(DEBUG, Core) << "No intersection between possible range keys and actual keys"; + return result; + } + + + auto buildResult = extractor->BuildComputeNode(indexKeys, ctx); + if (!buildResult.ComputeNode) { + YQL_CLOG(DEBUG, Core) << "BuildComputeNode: ranges can not be built for predicate"; + return result; + } + + TString prunedLambdaSerialized; + { + auto ast = ConvertToAst(*buildResult.PrunedLambda, ctx, TExprAnnotationFlags::None, true); + YQL_ENSURE(ast.Root); + prunedLambdaSerialized = "__yql_ast:" + ast.Root->ToString(TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote); + } + + result = ctx.Builder(pos) + .Callable("Just") + .List(0) + .Add(0, buildResult.ComputeNode) + .Callable(1, "AsTagged") + .Callable(0, "String") + .Atom(0, prunedLambdaSerialized) + .Seal() + .Atom(1, "AST", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Build(); + + return result; +} + +} // namespace + +THolder<IGraphTransformer> MakeExpandRangeComputeForTransformer(const TIntrusivePtr<TTypeAnnotationContext>& types) { + return CreateFunctorTransformer( + [types](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + TOptimizeExprSettings settings(types.Get()); + settings.CustomInstantTypeTransformer = types->CustomInstantTypeTransformer.Get(); + using namespace std::placeholders; + return OptimizeExpr(input, output, std::bind(&ExpandRangeComputeFor, _1, _2, types), ctx, settings); + } + ); +} + +} // namespace NYql diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.h b/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.h index 6b4fef276f..d6bd6ffaa8 100644 --- a/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.h +++ b/ydb/library/yql/core/extract_predicate/extract_predicate_dbg.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_graph_transformer.h> #include <ydb/library/yql/core/yql_type_annotation.h> - -namespace NYql { - -// Expands callable ComputeRangeFor (use for debug purposes) -THolder<IGraphTransformer> MakeExpandRangeComputeForTransformer(const TIntrusivePtr<TTypeAnnotationContext>& types); - -} + +namespace NYql { + +// Expands callable ComputeRangeFor (use for debug purposes) +THolder<IGraphTransformer> MakeExpandRangeComputeForTransformer(const TIntrusivePtr<TTypeAnnotationContext>& types); + +} diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp index 6a5d528a45..e2c16395b7 100644 --- a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp +++ b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp @@ -1,1614 +1,1614 @@ -#include "extract_predicate_impl.h" - +#include "extract_predicate_impl.h" + #include <ydb/library/yql/core/yql_expr_type_annotation.h> #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/core/yql_expr_constraint.h> #include <ydb/library/yql/core/yql_expr_optimize.h> - + #include <ydb/library/yql/core/services/yql_transform_pipeline.h> #include <ydb/library/yql/core/services/yql_out_transformers.h> - -namespace NYql { -namespace NDetail { -namespace { - -using namespace NNodes; -using NUdf::TCastResultOptions; - -const TTypeAnnotationNode* GetBaseDataType(const TTypeAnnotationNode *type) { - type = RemoveAllOptionals(type); - return (type && type->GetKind() == ETypeAnnotationKind::Data) ? type : nullptr; -} - -bool IsDataOrMultiOptionalOfData(const TTypeAnnotationNode* type) { - return GetBaseDataType(type) != nullptr; -} - -TMaybe<size_t> GetSqlInCollectionSize(const TExprNode::TPtr& collection) { - TExprNode::TPtr curr = collection; - if (curr->IsCallable("Just")) { - curr = curr->HeadPtr(); - } - - if (curr->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple || curr->IsCallable({"AsList", "AsDict", "AsSet"})) { - return std::max<size_t>(curr->ChildrenSize(), 1); - } - - return {}; -} - -const TTypeAnnotationNode* GetSqlInCollectionItemType(const TTypeAnnotationNode* collectionType) { - collectionType = RemoveOptionalType(collectionType); - YQL_ENSURE(collectionType); - switch (collectionType->GetKind()) { - case ETypeAnnotationKind::Tuple: { - const auto& items = collectionType->Cast<TTupleExprType>()->GetItems(); - YQL_ENSURE(items.size()); - YQL_ENSURE(AllOf(items, [&items](const TTypeAnnotationNode* type) { return type == items.front(); } )); - return items.front(); - } - case ETypeAnnotationKind::List: { - const TTypeAnnotationNode* listItem = collectionType->Cast<TListExprType>()->GetItemType(); - if (listItem->GetKind() == ETypeAnnotationKind::Struct) { - // single column table source - const auto& items = listItem->Cast<TStructExprType>()->GetItems(); - YQL_ENSURE(items.size() == 1); - return items.front()->GetItemType(); - } - return listItem; - } - case ETypeAnnotationKind::Dict: { - return collectionType->Cast<TDictExprType>()->GetKeyType(); - } - default: - YQL_ENSURE(false, "Unexpected IN collection type: " << *collectionType); - } -} - -TExprNode::TPtr GetSqlInCollectionAsOptList(const TExprNode::TPtr& collection, TExprContext& ctx) { - auto collectionType = collection->GetTypeAnn(); - bool isOptionalCollection = false; - if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { - isOptionalCollection = true; - collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); - } - - TPositionHandle pos = collection->Pos(); - TExprNode::TPtr body; - TExprNode::TPtr collectionArg = ctx.NewArgument(pos, "input"); - switch (collectionType->GetKind()) { - case ETypeAnnotationKind::List: { - const TTypeAnnotationNode* listItem = collectionType->Cast<TListExprType>()->GetItemType(); - if (listItem->GetKind() == ETypeAnnotationKind::Struct) { - // single column table source - const auto& items = listItem->Cast<TStructExprType>()->GetItems(); - YQL_ENSURE(items.size() == 1); - body = ctx.Builder(pos) - .Callable("Map") - .Add(0, collectionArg) - .Lambda(1) - .Param("item") - .Callable("Member") - .Arg(0, "item") - .Atom(1, items.front()->GetName()) - .Seal() - .Seal() - .Seal() - .Build(); - } else { - body = collectionArg; - } - break; - } - case ETypeAnnotationKind::Dict: { - body = ctx.NewCallable(pos, "DictKeys", { collectionArg }); - break; - } - case ETypeAnnotationKind::Tuple: { - TExprNodeList items; - for (size_t i = 0; i < collectionType->Cast<TTupleExprType>()->GetSize(); ++i) { - items.push_back( - ctx.Builder(pos) - .Callable("Nth") - .Add(0, collectionArg) - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Build() - ); - } - body = ctx.NewCallable(pos, "AsList", std::move(items)); - break; - } - default: YQL_ENSURE(false, "Unexpected type for IN collection: " << *collectionType); - } - - TExprNode::TPtr lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { collectionArg }), std::move(body)); - auto optCollection = isOptionalCollection ? collection : ctx.NewCallable(pos, "Just", { collection }); - return ctx.NewCallable(pos, "Map", { optCollection, lambda }); -} - -bool IsRoundingSupported(const TExprNode& op, bool negated) { - if (op.IsCallable({"Exists", "StartsWith"}) || (op.IsCallable("SqlIn") && !negated)) { - return true; - } - - if (op.IsCallable("==")) { - YQL_ENSURE(!negated, "Not should be expanded earlier"); - return true; - } - - struct TKeyValueTypes { - const TTypeAnnotationNode* KeyBaseType = nullptr; - const TTypeAnnotationNode* ValueBaseType = nullptr; - - static TVector<TKeyValueTypes> Extract(const TTypeAnnotationNode* keyType, const TTypeAnnotationNode* valueType) { - TVector<TKeyValueTypes> result; - if (keyType->GetKind() == ETypeAnnotationKind::Tuple) { - const auto& keyTypes = keyType->Cast<TTupleExprType>()->GetItems(); - auto valueTypeStipped = RemoveAllOptionals(valueType); - YQL_ENSURE(valueTypeStipped->GetKind() == ETypeAnnotationKind::Tuple); - const auto& valueTypes = valueTypeStipped->Cast<TTupleExprType>()->GetItems(); - - YQL_ENSURE(keyTypes.size() == valueTypes.size()); - YQL_ENSURE(!keyTypes.empty()); - - for (size_t i = 0; i < keyTypes.size(); ++i) { - result.emplace_back(); - result.back().KeyBaseType = GetBaseDataType(keyTypes[i]); - result.back().ValueBaseType = GetBaseDataType(valueTypes[i]); - } - } else { - result.emplace_back(); - result.back().KeyBaseType = GetBaseDataType(keyType); - result.back().ValueBaseType = GetBaseDataType(valueType); - } - - return result; - } - }; - - TVector<TKeyValueTypes> types; - if (op.IsCallable("SqlIn")) { - YQL_ENSURE(negated); - TCoSqlIn in(&op); - const auto& key = in.Lookup().Ref(); - const auto& collection = in.Collection().Ref(); - - types = TKeyValueTypes::Extract(key.GetTypeAnn(), GetSqlInCollectionItemType(collection.GetTypeAnn())); - // temporarily disable NOT IN - return false; - } else { - YQL_ENSURE(op.IsCallable({"<", "<=", ">", ">=", "!="})); - YQL_ENSURE(!negated, "Not should be expanded earlier"); - - types = TKeyValueTypes::Extract(op.Head().GetTypeAnn(), op.Tail().GetTypeAnn()); - } - - for (auto& item : types) { - YQL_ENSURE(item.KeyBaseType); - YQL_ENSURE(item.ValueBaseType); - - EDataSlot valueSlot = item.ValueBaseType->Cast<TDataExprType>()->GetSlot(); - EDataSlot keySlot = item.KeyBaseType->Cast<TDataExprType>()->GetSlot(); - - auto maybeCastoptions = NUdf::GetCastResult(valueSlot, keySlot); - if (!maybeCastoptions) { - return IsSameAnnotation(*item.KeyBaseType, *item.ValueBaseType); - } - if (*maybeCastoptions & (NUdf::ECastOptions::MayLoseData | NUdf::ECastOptions::MayFail)) { - auto valueTypeInfo = NUdf::GetDataTypeInfo(valueSlot); - auto keyTypeInfo = NUdf::GetDataTypeInfo(keySlot); - auto supportedFeatures = NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | - NUdf::EDataTypeFeatures::StringType; - return (valueTypeInfo.Features & supportedFeatures) && - (keyTypeInfo.Features & supportedFeatures); - } - } - return true; -} - -bool IsSupportedMemberNode(const TExprNode& node, const TExprNode& row) { - return node.IsCallable("Member") && node.Child(0) == &row && IsDataOrMultiOptionalOfData(node.GetTypeAnn()); -} - -bool IsValidForRange(const TExprNode& node, const TExprNode* otherNode, const TExprNode& row) { - if (!IsSupportedMemberNode(node, row)) { - return false; - } - - if (otherNode && IsDepended(*otherNode, row)) { - return false; - } - - return true; -} - -const THashMap<TStringBuf, TStringBuf> SupportedBinOps = { - { "<", ">"}, - { "<=", ">="}, - { ">", "<"}, - { ">=", "<="}, - { "==", "=="}, - { "!=", "!="}, -}; - -bool IsValidForRange(TExprNode::TPtr& node, const TExprNode& row, const TPredicateExtractorSettings& settings, TExprContext& ctx) { - auto it = SupportedBinOps.find(node->Content()); - if (it != SupportedBinOps.end()) { - if (IsValidForRange(node->Head(), &node->Tail(), row)) { - return true; - } - - if (IsValidForRange(node->Tail(), &node->Head(), row)) { - node = ctx.NewCallable(node->Pos(), it->second, { node->TailPtr(), node->HeadPtr() }); - return true; - } - - return false; - } - - if (node->IsCallable("Exists")) { - return IsValidForRange(node->Head(), nullptr, row); - } - - if (node->IsCallable("StartsWith") && settings.HaveNextValueCallable) { - return IsValidForRange(node->Head(), node->Child(1), row); - } - - if (node->IsCallable("SqlIn")) { - TCoSqlIn sqlIn(node); - - const auto& collection = sqlIn.Collection().Ptr(); - const auto& lookup = sqlIn.Lookup().Ptr(); - - if (IsDepended(*collection, row)) { - return false; - } - - TExprNodeList members; - if (lookup->IsList()) { - auto collectionItemBaseType = RemoveAllOptionals(GetSqlInCollectionItemType(collection->GetTypeAnn())); - YQL_ENSURE(collectionItemBaseType); - YQL_ENSURE(collectionItemBaseType->GetKind() == ETypeAnnotationKind::Tuple); - if (collectionItemBaseType->Cast<TTupleExprType>()->GetSize() != lookup->ChildrenSize()) { - return false; - } - members = lookup->ChildrenList(); - } else { - members.push_back(lookup); - } - return AllOf(members, [&](const auto& child) { return IsSupportedMemberNode(*child, row); }); - } - - return false; -} - -TExprNode::TPtr GetOpFromRange(const TExprNode& node, bool& negated) { - YQL_ENSURE(node.IsCallable("Range")); - negated = false; - auto opNode = node.TailPtr(); - if (opNode->IsLambda()) { - opNode = opNode->TailPtr(); - } - if (opNode->IsCallable("Coalesce")) { - opNode = opNode->HeadPtr(); - } - if (opNode->IsCallable("Not")) { - negated = true; - opNode = opNode->HeadPtr(); - } - - return opNode; -} - -TExprNode::TPtr GetOpFromRange(const TExprNode& node) { - bool negated; - return GetOpFromRange(node, negated); -} - -TVector<TString> GetRawColumnsFromRange(const TExprNode& node) { - auto opNode = GetOpFromRange(node); - auto memberNode = opNode->IsCallable("SqlIn") ? opNode->ChildPtr(1) : opNode->ChildPtr(0); - - TExprNodeList members; - if (memberNode->IsList()) { - members = memberNode->ChildrenList(); - } else { - members.push_back(std::move(memberNode)); - } - - TVector<TString> result; - for (auto& member : members) { - YQL_ENSURE(member->IsCallable("Member")); - result.push_back(ToString(member->Tail().Content())); - } - YQL_ENSURE(!result.empty()); - return result; -} - -TVector<TString> GetColumnsFromRange(const TExprNode& node, const THashMap<TString, size_t>& indexKeysOrder) { - TVector<TString> result = GetRawColumnsFromRange(node); - - auto opNode = GetOpFromRange(node); - YQL_ENSURE(opNode->IsCallable("SqlIn") || result.size() == 1); - result.erase( - std::remove_if(result.begin(), result.end(), [&](const TString& col) { - return indexKeysOrder.find(col) == indexKeysOrder.end(); - }), - result.end() - ); - - std::sort(result.begin(), result.end(), [&](const TString& a, const TString& b) { - auto aIt = indexKeysOrder.find(a); - auto bIt = indexKeysOrder.find(b); - - YQL_ENSURE(aIt != indexKeysOrder.end() && bIt != indexKeysOrder.end()); - return aIt->second < bIt->second; - }); - - if (!result.empty()) { - size_t prevIdx = indexKeysOrder.find(result.front())->second; - TVector<TString> resultNoGap; - resultNoGap.push_back(result.front()); - for (size_t i = 1; i < result.size(); ++i) { - size_t currIdx = indexKeysOrder.find(result[i])->second; - YQL_ENSURE(currIdx >= prevIdx); - if (currIdx == prevIdx) { - continue; - } - if (currIdx != prevIdx + 1) { - break; - } - prevIdx = currIdx; - resultNoGap.push_back(result[i]); - } - result.swap(resultNoGap); - } - return result; -} - -bool IsPointRange(const TExprNode& node) { - bool negated; - auto op = GetOpFromRange(node, negated); - if (op->IsCallable("SqlIn")) { - return !negated; - } - if (op->IsCallable("Exists")) { - return negated; - } - if (op->IsCallable("StartsWith")) { - return false; - } - YQL_ENSURE(!negated); - return op->IsCallable("=="); -} - -bool IsCoalesceValidForRange(const TExprNode& node) { - return node.IsCallable("Coalesce") && node.ChildrenSize() == 2 && node.Tail().IsCallable("Bool"); -} - -bool IsCoalesceValidForRange(const TExprNode& node, bool negated) { - return IsCoalesceValidForRange(node) && FromString<bool>(node.Tail().Head().Content()) == negated; -} - -TExprNode::TPtr ExpandTupleBinOp(const TExprNode& node, TExprContext& ctx) { - YQL_ENSURE(node.Child(0)->IsList()); - YQL_ENSURE(node.Child(0)->ChildrenSize() > 0); - if (node.IsCallable({"<=", ">="})) { - return ctx.Builder(node.Pos()) - .Callable("Or") - .Add(0, ctx.RenameNode(node, "==")) - .Add(1, ctx.RenameNode(node, node.IsCallable("<=") ? "<" : ">")) - .Seal() - .Build(); - } - - if (node.IsCallable({"==", "!="})) { - TExprNodeList predicates; - for (ui32 i = 0; i < node.Child(0)->ChildrenSize(); ++i) { - auto child = node.Child(0)->ChildPtr(i); - predicates.push_back( - ctx.Builder(child->Pos()) - .Callable(node.Content()) - .Add(0, child) - .Callable(1, "Nth") - .Add(0, node.ChildPtr(1)) - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Seal() - .Build()); - } - return ctx.NewCallable(node.Pos(), node.IsCallable("==") ? "And" : "Or", std::move(predicates)); - } - - YQL_ENSURE(node.IsCallable({"<", ">"})); - TExprNodeList tupleItems = node.Child(0)->ChildrenList(); - TExprNode::TPtr firstPred = ctx.Builder(node.Pos()) - .Callable(node.Content()) - .Add(0, tupleItems.front()) - .Callable(1, "Nth") - .Add(0, node.ChildPtr(1)) - .Atom(1, "0", TNodeFlags::Default) - .Seal() - .Seal() - .Build(); - - if (tupleItems.size() == 1) { - return firstPred; - } - - TExprNodeList otherTupleItems(tupleItems.begin() + 1, tupleItems.end()); - TExprNodeList otherValues; - for (ui32 i = 1; i < tupleItems.size(); ++i) { - otherValues.push_back( - ctx.Builder(node.Child(1)->Pos()) - .Callable("Nth") - .Add(0, node.ChildPtr(1)) - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Build()); - } - - return ctx.Builder(node.Pos()) - .Callable("Or") - .Add(0, firstPred) - .Callable(1, "And") - .Add(0, ctx.RenameNode(*firstPred, "==")) - .Callable(1, node.Content()) - .Add(0, ctx.NewList(node.Child(0)->Pos(), std::move(otherTupleItems))) - .Add(1, ctx.NewList(node.Child(1)->Pos(), std::move(otherValues))) - .Seal() - .Seal() - .Seal() - .Build(); -} - -void DoBuildRanges(const TExprNode::TPtr& lambdaArg, const TExprNode::TPtr& currentNode, const TPredicateExtractorSettings& settings, - TExprNode::TPtr& range, TSet<TString>& keysInScope, TExprContext& ctx, bool negated) -{ - keysInScope.clear(); - const TPositionHandle pos = currentNode->Pos(); - if (currentNode->IsCallable({"Or", "And"})) { - YQL_ENSURE(currentNode->ChildrenSize() != 0); - TExprNodeList children; - const bool isUnion = currentNode->IsCallable("Or"); - TMaybe<TSet<TString>> commonKeysInScope; - for (size_t i = 0; i < currentNode->ChildrenSize(); ++i) { - const auto& child = currentNode->ChildPtr(i); - children.emplace_back(); - TSet<TString> childKeys; - DoBuildRanges(lambdaArg, child, settings, children.back(), childKeys, ctx, negated); - if (!children.back()->IsCallable("RangeConst")) { - if (!commonKeysInScope.Defined()) { - commonKeysInScope = childKeys; - } else if (isUnion) { - TSet<TString> intersected; - std::set_intersection(commonKeysInScope->begin(), commonKeysInScope->end(), - childKeys.begin(), childKeys.end(), std::inserter(intersected, intersected.end())); - - if (intersected.empty() || intersected.size() < std::min(childKeys.size(), commonKeysInScope->size())) { - *commonKeysInScope = std::move(intersected); - } else if (intersected.size() == commonKeysInScope->size()) { - *commonKeysInScope = std::move(childKeys); - } - - if (commonKeysInScope->empty()) { - break; - } - } else { - commonKeysInScope->insert(childKeys.begin(), childKeys.end()); - } - } - } - - if (!commonKeysInScope.Defined()) { - range = ctx.NewCallable(pos, "RangeConst", { currentNode }); - return; - } - - if (commonKeysInScope->empty()) { - range = ctx.NewCallable(pos, "RangeRest", { currentNode }); - return; - } - - keysInScope = std::move(*commonKeysInScope); - range = ctx.NewCallable(pos, currentNode->IsCallable("Or") ? "RangeOr" : "RangeAnd", std::move(children)); - return; - } - - if (IsCoalesceValidForRange(*currentNode, negated) || currentNode->IsCallable("Not")) { - DoBuildRanges(lambdaArg, currentNode->HeadPtr(), settings, range, keysInScope, ctx, currentNode->IsCallable("Not") ? !negated : negated); - YQL_ENSURE(range->IsCallable({"Range", "RangeRest", "RangeConst"})); // Coalesce/Not should be pushed down through Range{Unionn/Intersect} - TExprNode::TPtr child; - if (currentNode->IsCallable("Not")) { - child = ctx.NewCallable(pos, "Not", { range->HeadPtr() }); - } else { - child = ctx.NewCallable(pos, "Coalesce", { range->HeadPtr(), currentNode->TailPtr() }); - } - - range = ctx.NewCallable(pos, range->Content(), { child }); - return; - } - - if (!IsDepended(*currentNode, *lambdaArg)) { - range = ctx.NewCallable(pos, "RangeConst", { currentNode }); - } else { - TExprNode::TPtr currentNodeNormalized = currentNode; - bool isValid = IsValidForRange(currentNodeNormalized, *lambdaArg, settings, ctx); - // TODO: all roundings should be supported - isValid = isValid && IsRoundingSupported(*currentNodeNormalized, negated); - range = ctx.NewCallable(pos, isValid ? "Range" : "RangeRest", { currentNodeNormalized }); - if (isValid) { - keysInScope.clear(); - auto cols = GetRawColumnsFromRange(*range); - keysInScope.insert(cols.begin(), cols.end()); - } - } -} - -bool IsListOfMembers(const TExprNode& node) { - if (!node.IsList()) { - return false; - } - - return AllOf(node.ChildrenList(), [](const auto& child) { - return child->IsCallable("Member") && IsDataOrMultiOptionalOfData(child->GetTypeAnn()); - }); -} - -bool IsMemberBinOpNode(const TExprNode& node) { - YQL_ENSURE(node.ChildrenSize() == 2); - return node.Head().IsCallable("Member") && IsDataOrMultiOptionalOfData(node.Head().GetTypeAnn()) || - node.Tail().IsCallable("Member") && IsDataOrMultiOptionalOfData(node.Tail().GetTypeAnn()); -} - -bool IsMemberListBinOpNode(const TExprNode& node) { - YQL_ENSURE(node.ChildrenSize() == 2); - return IsListOfMembers(node.Head()) || IsListOfMembers(node.Tail()); -} - -TExprNode::TPtr OptimizeNodeForRangeExtraction(const TExprNode::TPtr& node, TExprContext& ctx) { - auto it = node->IsCallable() ? SupportedBinOps.find(node->Content()) : SupportedBinOps.end(); - if (it != SupportedBinOps.end()) { - TExprNode::TPtr toExpand; - if (IsListOfMembers(node->Head())) { - toExpand = node; - } else if (IsListOfMembers(node->Tail())) { - toExpand = ctx.NewCallable(node->Pos(), it->second, { node->TailPtr(), node->HeadPtr() }); - } - - if (toExpand) { - auto baseType = RemoveAllOptionals(toExpand->Tail().GetTypeAnn()); - YQL_ENSURE(baseType); - YQL_ENSURE(baseType->GetKind() == ETypeAnnotationKind::Tuple); - if (baseType->Cast<TTupleExprType>()->GetSize() != node->Head().ChildrenSize()) { - // comparison of tuples of different size - return node; - } - YQL_CLOG(DEBUG, Core) << node->Content() << " over tuple"; - return ExpandTupleBinOp(*toExpand, ctx); - } - - return node; - } - - if (node->IsCallable("Not")) { - if (IsCoalesceValidForRange(node->Head())) { - // The effect of this optimizer is the opposite of Coalesce optimizer in yql_co_simple1.cpp - auto firstArg = node->Head().HeadPtr(); - auto secondArg = node->Head().TailPtr(); - YQL_CLOG(DEBUG, Core) << "Push down " << node->Content() << " over Coalesce"; - return ctx.Builder(node->Pos()) - .Callable("Coalesce") - .Callable(0, "Not") - .Add(0, firstArg) - .Seal() - .Callable(1, "Not") - .Add(0, secondArg) - .Seal() - .Seal() - .Build(); - } - - static const THashMap<TStringBuf, TStringBuf> binOpsWithNegations = { - { "<", ">="}, - { "<=", ">"}, - { ">", "<="}, - { ">=", "<"}, - { "==", "!="}, - { "!=", "=="}, - }; - - auto it = node->Head().IsCallable() ? binOpsWithNegations.find(node->Head().Content()) : binOpsWithNegations.end(); - if (it != binOpsWithNegations.end() && (IsMemberBinOpNode(node->Head()) || IsMemberListBinOpNode(node->Head()))) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return ctx.RenameNode(node->Head(), it->second); - } - - if (node->Head().IsCallable({"And", "Or"})) { - TExprNodeList children; - for (auto& child : node->Head().ChildrenList()) { - children.push_back(ctx.NewCallable(child->Pos(), "Not", { child })); - } - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return ctx.NewCallable(node->Pos(), node->Head().IsCallable("Or") ? "And" : "Or", std::move(children)); - } - - if (node->Head().IsCallable("Not")) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); - return node->Head().HeadPtr(); - } - - if (node->Head().IsCallable("Bool")) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over literal Bool"; - bool value = FromString<bool>(node->Head().Head().Content()); - return MakeBool(node->Pos(), !value, ctx); - } - - return node; - } - - if (node->IsCallable("Nth")) { - if (node->Head().Type() == TExprNode::List) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over tuple literal"; - const auto index = FromString<ui32>(node->Tail().Content()); - return node->Head().ChildPtr(index); - } - } - - if (IsCoalesceValidForRange(*node) && node->Head().IsCallable({"And", "Or"})) { - YQL_CLOG(DEBUG, Core) << "PropagateCoalesceWithConst over " << node->Head().Content(); - auto children = node->Head().ChildrenList(); - for (auto& child : children) { - child = ctx.NewCallable(node->Pos(), node->Content(), {std::move(child), node->TailPtr()}); - } - return ctx.NewCallable(node->Head().Pos(), node->Head().Content(), std::move(children)); - } - - if (node->IsCallable({"And", "Or"})) { - if (AnyOf(node->ChildrenList(), [&node](const auto& child) { return child->IsCallable(node->Content()); })) { - YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Content(); - TExprNodeList newChildren; - for (auto& child : node->ChildrenList()) { - if (child->IsCallable(node->Content())) { - auto chs = child->ChildrenList(); - newChildren.insert(newChildren.end(), chs.begin(), chs.end()); - } else { - newChildren.emplace_back(std::move(child)); - } - } - return ctx.ChangeChildren(*node, std::move(newChildren)); - } - } - - return node; -} - -void DoOptimizeForRangeExtraction(const TExprNode::TPtr& input, TExprNode::TPtr& output, bool topLevel, TExprContext& ctx) { - output = OptimizeNodeForRangeExtraction(input, ctx); - if (output != input) { - return; - } - - auto children = input->ChildrenList(); - bool changed = false; - for (auto& child : children) { - if (!topLevel && !child->IsComputable()) { - continue; - } - TExprNode::TPtr newChild = child; - DoOptimizeForRangeExtraction(child, newChild, false, ctx); - if (newChild != child) { - changed = true; - child = std::move(newChild); - } - } - - if (changed) { - output = ctx.ChangeChildren(*input, std::move(children)); - } -} - -THolder<IGraphTransformer> CreateRangeExtractionOptimizer(TTypeAnnotationContext& types) { - TTransformationPipeline pipeline(&types); - - auto issueCode = TIssuesIds::CORE_EXEC; - pipeline.AddServiceTransformers(issueCode); - pipeline.AddTypeAnnotationTransformer(issueCode); + +namespace NYql { +namespace NDetail { +namespace { + +using namespace NNodes; +using NUdf::TCastResultOptions; + +const TTypeAnnotationNode* GetBaseDataType(const TTypeAnnotationNode *type) { + type = RemoveAllOptionals(type); + return (type && type->GetKind() == ETypeAnnotationKind::Data) ? type : nullptr; +} + +bool IsDataOrMultiOptionalOfData(const TTypeAnnotationNode* type) { + return GetBaseDataType(type) != nullptr; +} + +TMaybe<size_t> GetSqlInCollectionSize(const TExprNode::TPtr& collection) { + TExprNode::TPtr curr = collection; + if (curr->IsCallable("Just")) { + curr = curr->HeadPtr(); + } + + if (curr->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple || curr->IsCallable({"AsList", "AsDict", "AsSet"})) { + return std::max<size_t>(curr->ChildrenSize(), 1); + } + + return {}; +} + +const TTypeAnnotationNode* GetSqlInCollectionItemType(const TTypeAnnotationNode* collectionType) { + collectionType = RemoveOptionalType(collectionType); + YQL_ENSURE(collectionType); + switch (collectionType->GetKind()) { + case ETypeAnnotationKind::Tuple: { + const auto& items = collectionType->Cast<TTupleExprType>()->GetItems(); + YQL_ENSURE(items.size()); + YQL_ENSURE(AllOf(items, [&items](const TTypeAnnotationNode* type) { return type == items.front(); } )); + return items.front(); + } + case ETypeAnnotationKind::List: { + const TTypeAnnotationNode* listItem = collectionType->Cast<TListExprType>()->GetItemType(); + if (listItem->GetKind() == ETypeAnnotationKind::Struct) { + // single column table source + const auto& items = listItem->Cast<TStructExprType>()->GetItems(); + YQL_ENSURE(items.size() == 1); + return items.front()->GetItemType(); + } + return listItem; + } + case ETypeAnnotationKind::Dict: { + return collectionType->Cast<TDictExprType>()->GetKeyType(); + } + default: + YQL_ENSURE(false, "Unexpected IN collection type: " << *collectionType); + } +} + +TExprNode::TPtr GetSqlInCollectionAsOptList(const TExprNode::TPtr& collection, TExprContext& ctx) { + auto collectionType = collection->GetTypeAnn(); + bool isOptionalCollection = false; + if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { + isOptionalCollection = true; + collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); + } + + TPositionHandle pos = collection->Pos(); + TExprNode::TPtr body; + TExprNode::TPtr collectionArg = ctx.NewArgument(pos, "input"); + switch (collectionType->GetKind()) { + case ETypeAnnotationKind::List: { + const TTypeAnnotationNode* listItem = collectionType->Cast<TListExprType>()->GetItemType(); + if (listItem->GetKind() == ETypeAnnotationKind::Struct) { + // single column table source + const auto& items = listItem->Cast<TStructExprType>()->GetItems(); + YQL_ENSURE(items.size() == 1); + body = ctx.Builder(pos) + .Callable("Map") + .Add(0, collectionArg) + .Lambda(1) + .Param("item") + .Callable("Member") + .Arg(0, "item") + .Atom(1, items.front()->GetName()) + .Seal() + .Seal() + .Seal() + .Build(); + } else { + body = collectionArg; + } + break; + } + case ETypeAnnotationKind::Dict: { + body = ctx.NewCallable(pos, "DictKeys", { collectionArg }); + break; + } + case ETypeAnnotationKind::Tuple: { + TExprNodeList items; + for (size_t i = 0; i < collectionType->Cast<TTupleExprType>()->GetSize(); ++i) { + items.push_back( + ctx.Builder(pos) + .Callable("Nth") + .Add(0, collectionArg) + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Build() + ); + } + body = ctx.NewCallable(pos, "AsList", std::move(items)); + break; + } + default: YQL_ENSURE(false, "Unexpected type for IN collection: " << *collectionType); + } + + TExprNode::TPtr lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { collectionArg }), std::move(body)); + auto optCollection = isOptionalCollection ? collection : ctx.NewCallable(pos, "Just", { collection }); + return ctx.NewCallable(pos, "Map", { optCollection, lambda }); +} + +bool IsRoundingSupported(const TExprNode& op, bool negated) { + if (op.IsCallable({"Exists", "StartsWith"}) || (op.IsCallable("SqlIn") && !negated)) { + return true; + } + + if (op.IsCallable("==")) { + YQL_ENSURE(!negated, "Not should be expanded earlier"); + return true; + } + + struct TKeyValueTypes { + const TTypeAnnotationNode* KeyBaseType = nullptr; + const TTypeAnnotationNode* ValueBaseType = nullptr; + + static TVector<TKeyValueTypes> Extract(const TTypeAnnotationNode* keyType, const TTypeAnnotationNode* valueType) { + TVector<TKeyValueTypes> result; + if (keyType->GetKind() == ETypeAnnotationKind::Tuple) { + const auto& keyTypes = keyType->Cast<TTupleExprType>()->GetItems(); + auto valueTypeStipped = RemoveAllOptionals(valueType); + YQL_ENSURE(valueTypeStipped->GetKind() == ETypeAnnotationKind::Tuple); + const auto& valueTypes = valueTypeStipped->Cast<TTupleExprType>()->GetItems(); + + YQL_ENSURE(keyTypes.size() == valueTypes.size()); + YQL_ENSURE(!keyTypes.empty()); + + for (size_t i = 0; i < keyTypes.size(); ++i) { + result.emplace_back(); + result.back().KeyBaseType = GetBaseDataType(keyTypes[i]); + result.back().ValueBaseType = GetBaseDataType(valueTypes[i]); + } + } else { + result.emplace_back(); + result.back().KeyBaseType = GetBaseDataType(keyType); + result.back().ValueBaseType = GetBaseDataType(valueType); + } + + return result; + } + }; + + TVector<TKeyValueTypes> types; + if (op.IsCallable("SqlIn")) { + YQL_ENSURE(negated); + TCoSqlIn in(&op); + const auto& key = in.Lookup().Ref(); + const auto& collection = in.Collection().Ref(); + + types = TKeyValueTypes::Extract(key.GetTypeAnn(), GetSqlInCollectionItemType(collection.GetTypeAnn())); + // temporarily disable NOT IN + return false; + } else { + YQL_ENSURE(op.IsCallable({"<", "<=", ">", ">=", "!="})); + YQL_ENSURE(!negated, "Not should be expanded earlier"); + + types = TKeyValueTypes::Extract(op.Head().GetTypeAnn(), op.Tail().GetTypeAnn()); + } + + for (auto& item : types) { + YQL_ENSURE(item.KeyBaseType); + YQL_ENSURE(item.ValueBaseType); + + EDataSlot valueSlot = item.ValueBaseType->Cast<TDataExprType>()->GetSlot(); + EDataSlot keySlot = item.KeyBaseType->Cast<TDataExprType>()->GetSlot(); + + auto maybeCastoptions = NUdf::GetCastResult(valueSlot, keySlot); + if (!maybeCastoptions) { + return IsSameAnnotation(*item.KeyBaseType, *item.ValueBaseType); + } + if (*maybeCastoptions & (NUdf::ECastOptions::MayLoseData | NUdf::ECastOptions::MayFail)) { + auto valueTypeInfo = NUdf::GetDataTypeInfo(valueSlot); + auto keyTypeInfo = NUdf::GetDataTypeInfo(keySlot); + auto supportedFeatures = NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | + NUdf::EDataTypeFeatures::StringType; + return (valueTypeInfo.Features & supportedFeatures) && + (keyTypeInfo.Features & supportedFeatures); + } + } + return true; +} + +bool IsSupportedMemberNode(const TExprNode& node, const TExprNode& row) { + return node.IsCallable("Member") && node.Child(0) == &row && IsDataOrMultiOptionalOfData(node.GetTypeAnn()); +} + +bool IsValidForRange(const TExprNode& node, const TExprNode* otherNode, const TExprNode& row) { + if (!IsSupportedMemberNode(node, row)) { + return false; + } + + if (otherNode && IsDepended(*otherNode, row)) { + return false; + } + + return true; +} + +const THashMap<TStringBuf, TStringBuf> SupportedBinOps = { + { "<", ">"}, + { "<=", ">="}, + { ">", "<"}, + { ">=", "<="}, + { "==", "=="}, + { "!=", "!="}, +}; + +bool IsValidForRange(TExprNode::TPtr& node, const TExprNode& row, const TPredicateExtractorSettings& settings, TExprContext& ctx) { + auto it = SupportedBinOps.find(node->Content()); + if (it != SupportedBinOps.end()) { + if (IsValidForRange(node->Head(), &node->Tail(), row)) { + return true; + } + + if (IsValidForRange(node->Tail(), &node->Head(), row)) { + node = ctx.NewCallable(node->Pos(), it->second, { node->TailPtr(), node->HeadPtr() }); + return true; + } + + return false; + } + + if (node->IsCallable("Exists")) { + return IsValidForRange(node->Head(), nullptr, row); + } + + if (node->IsCallable("StartsWith") && settings.HaveNextValueCallable) { + return IsValidForRange(node->Head(), node->Child(1), row); + } + + if (node->IsCallable("SqlIn")) { + TCoSqlIn sqlIn(node); + + const auto& collection = sqlIn.Collection().Ptr(); + const auto& lookup = sqlIn.Lookup().Ptr(); + + if (IsDepended(*collection, row)) { + return false; + } + + TExprNodeList members; + if (lookup->IsList()) { + auto collectionItemBaseType = RemoveAllOptionals(GetSqlInCollectionItemType(collection->GetTypeAnn())); + YQL_ENSURE(collectionItemBaseType); + YQL_ENSURE(collectionItemBaseType->GetKind() == ETypeAnnotationKind::Tuple); + if (collectionItemBaseType->Cast<TTupleExprType>()->GetSize() != lookup->ChildrenSize()) { + return false; + } + members = lookup->ChildrenList(); + } else { + members.push_back(lookup); + } + return AllOf(members, [&](const auto& child) { return IsSupportedMemberNode(*child, row); }); + } + + return false; +} + +TExprNode::TPtr GetOpFromRange(const TExprNode& node, bool& negated) { + YQL_ENSURE(node.IsCallable("Range")); + negated = false; + auto opNode = node.TailPtr(); + if (opNode->IsLambda()) { + opNode = opNode->TailPtr(); + } + if (opNode->IsCallable("Coalesce")) { + opNode = opNode->HeadPtr(); + } + if (opNode->IsCallable("Not")) { + negated = true; + opNode = opNode->HeadPtr(); + } + + return opNode; +} + +TExprNode::TPtr GetOpFromRange(const TExprNode& node) { + bool negated; + return GetOpFromRange(node, negated); +} + +TVector<TString> GetRawColumnsFromRange(const TExprNode& node) { + auto opNode = GetOpFromRange(node); + auto memberNode = opNode->IsCallable("SqlIn") ? opNode->ChildPtr(1) : opNode->ChildPtr(0); + + TExprNodeList members; + if (memberNode->IsList()) { + members = memberNode->ChildrenList(); + } else { + members.push_back(std::move(memberNode)); + } + + TVector<TString> result; + for (auto& member : members) { + YQL_ENSURE(member->IsCallable("Member")); + result.push_back(ToString(member->Tail().Content())); + } + YQL_ENSURE(!result.empty()); + return result; +} + +TVector<TString> GetColumnsFromRange(const TExprNode& node, const THashMap<TString, size_t>& indexKeysOrder) { + TVector<TString> result = GetRawColumnsFromRange(node); + + auto opNode = GetOpFromRange(node); + YQL_ENSURE(opNode->IsCallable("SqlIn") || result.size() == 1); + result.erase( + std::remove_if(result.begin(), result.end(), [&](const TString& col) { + return indexKeysOrder.find(col) == indexKeysOrder.end(); + }), + result.end() + ); + + std::sort(result.begin(), result.end(), [&](const TString& a, const TString& b) { + auto aIt = indexKeysOrder.find(a); + auto bIt = indexKeysOrder.find(b); + + YQL_ENSURE(aIt != indexKeysOrder.end() && bIt != indexKeysOrder.end()); + return aIt->second < bIt->second; + }); + + if (!result.empty()) { + size_t prevIdx = indexKeysOrder.find(result.front())->second; + TVector<TString> resultNoGap; + resultNoGap.push_back(result.front()); + for (size_t i = 1; i < result.size(); ++i) { + size_t currIdx = indexKeysOrder.find(result[i])->second; + YQL_ENSURE(currIdx >= prevIdx); + if (currIdx == prevIdx) { + continue; + } + if (currIdx != prevIdx + 1) { + break; + } + prevIdx = currIdx; + resultNoGap.push_back(result[i]); + } + result.swap(resultNoGap); + } + return result; +} + +bool IsPointRange(const TExprNode& node) { + bool negated; + auto op = GetOpFromRange(node, negated); + if (op->IsCallable("SqlIn")) { + return !negated; + } + if (op->IsCallable("Exists")) { + return negated; + } + if (op->IsCallable("StartsWith")) { + return false; + } + YQL_ENSURE(!negated); + return op->IsCallable("=="); +} + +bool IsCoalesceValidForRange(const TExprNode& node) { + return node.IsCallable("Coalesce") && node.ChildrenSize() == 2 && node.Tail().IsCallable("Bool"); +} + +bool IsCoalesceValidForRange(const TExprNode& node, bool negated) { + return IsCoalesceValidForRange(node) && FromString<bool>(node.Tail().Head().Content()) == negated; +} + +TExprNode::TPtr ExpandTupleBinOp(const TExprNode& node, TExprContext& ctx) { + YQL_ENSURE(node.Child(0)->IsList()); + YQL_ENSURE(node.Child(0)->ChildrenSize() > 0); + if (node.IsCallable({"<=", ">="})) { + return ctx.Builder(node.Pos()) + .Callable("Or") + .Add(0, ctx.RenameNode(node, "==")) + .Add(1, ctx.RenameNode(node, node.IsCallable("<=") ? "<" : ">")) + .Seal() + .Build(); + } + + if (node.IsCallable({"==", "!="})) { + TExprNodeList predicates; + for (ui32 i = 0; i < node.Child(0)->ChildrenSize(); ++i) { + auto child = node.Child(0)->ChildPtr(i); + predicates.push_back( + ctx.Builder(child->Pos()) + .Callable(node.Content()) + .Add(0, child) + .Callable(1, "Nth") + .Add(0, node.ChildPtr(1)) + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Seal() + .Build()); + } + return ctx.NewCallable(node.Pos(), node.IsCallable("==") ? "And" : "Or", std::move(predicates)); + } + + YQL_ENSURE(node.IsCallable({"<", ">"})); + TExprNodeList tupleItems = node.Child(0)->ChildrenList(); + TExprNode::TPtr firstPred = ctx.Builder(node.Pos()) + .Callable(node.Content()) + .Add(0, tupleItems.front()) + .Callable(1, "Nth") + .Add(0, node.ChildPtr(1)) + .Atom(1, "0", TNodeFlags::Default) + .Seal() + .Seal() + .Build(); + + if (tupleItems.size() == 1) { + return firstPred; + } + + TExprNodeList otherTupleItems(tupleItems.begin() + 1, tupleItems.end()); + TExprNodeList otherValues; + for (ui32 i = 1; i < tupleItems.size(); ++i) { + otherValues.push_back( + ctx.Builder(node.Child(1)->Pos()) + .Callable("Nth") + .Add(0, node.ChildPtr(1)) + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Build()); + } + + return ctx.Builder(node.Pos()) + .Callable("Or") + .Add(0, firstPred) + .Callable(1, "And") + .Add(0, ctx.RenameNode(*firstPred, "==")) + .Callable(1, node.Content()) + .Add(0, ctx.NewList(node.Child(0)->Pos(), std::move(otherTupleItems))) + .Add(1, ctx.NewList(node.Child(1)->Pos(), std::move(otherValues))) + .Seal() + .Seal() + .Seal() + .Build(); +} + +void DoBuildRanges(const TExprNode::TPtr& lambdaArg, const TExprNode::TPtr& currentNode, const TPredicateExtractorSettings& settings, + TExprNode::TPtr& range, TSet<TString>& keysInScope, TExprContext& ctx, bool negated) +{ + keysInScope.clear(); + const TPositionHandle pos = currentNode->Pos(); + if (currentNode->IsCallable({"Or", "And"})) { + YQL_ENSURE(currentNode->ChildrenSize() != 0); + TExprNodeList children; + const bool isUnion = currentNode->IsCallable("Or"); + TMaybe<TSet<TString>> commonKeysInScope; + for (size_t i = 0; i < currentNode->ChildrenSize(); ++i) { + const auto& child = currentNode->ChildPtr(i); + children.emplace_back(); + TSet<TString> childKeys; + DoBuildRanges(lambdaArg, child, settings, children.back(), childKeys, ctx, negated); + if (!children.back()->IsCallable("RangeConst")) { + if (!commonKeysInScope.Defined()) { + commonKeysInScope = childKeys; + } else if (isUnion) { + TSet<TString> intersected; + std::set_intersection(commonKeysInScope->begin(), commonKeysInScope->end(), + childKeys.begin(), childKeys.end(), std::inserter(intersected, intersected.end())); + + if (intersected.empty() || intersected.size() < std::min(childKeys.size(), commonKeysInScope->size())) { + *commonKeysInScope = std::move(intersected); + } else if (intersected.size() == commonKeysInScope->size()) { + *commonKeysInScope = std::move(childKeys); + } + + if (commonKeysInScope->empty()) { + break; + } + } else { + commonKeysInScope->insert(childKeys.begin(), childKeys.end()); + } + } + } + + if (!commonKeysInScope.Defined()) { + range = ctx.NewCallable(pos, "RangeConst", { currentNode }); + return; + } + + if (commonKeysInScope->empty()) { + range = ctx.NewCallable(pos, "RangeRest", { currentNode }); + return; + } + + keysInScope = std::move(*commonKeysInScope); + range = ctx.NewCallable(pos, currentNode->IsCallable("Or") ? "RangeOr" : "RangeAnd", std::move(children)); + return; + } + + if (IsCoalesceValidForRange(*currentNode, negated) || currentNode->IsCallable("Not")) { + DoBuildRanges(lambdaArg, currentNode->HeadPtr(), settings, range, keysInScope, ctx, currentNode->IsCallable("Not") ? !negated : negated); + YQL_ENSURE(range->IsCallable({"Range", "RangeRest", "RangeConst"})); // Coalesce/Not should be pushed down through Range{Unionn/Intersect} + TExprNode::TPtr child; + if (currentNode->IsCallable("Not")) { + child = ctx.NewCallable(pos, "Not", { range->HeadPtr() }); + } else { + child = ctx.NewCallable(pos, "Coalesce", { range->HeadPtr(), currentNode->TailPtr() }); + } + + range = ctx.NewCallable(pos, range->Content(), { child }); + return; + } + + if (!IsDepended(*currentNode, *lambdaArg)) { + range = ctx.NewCallable(pos, "RangeConst", { currentNode }); + } else { + TExprNode::TPtr currentNodeNormalized = currentNode; + bool isValid = IsValidForRange(currentNodeNormalized, *lambdaArg, settings, ctx); + // TODO: all roundings should be supported + isValid = isValid && IsRoundingSupported(*currentNodeNormalized, negated); + range = ctx.NewCallable(pos, isValid ? "Range" : "RangeRest", { currentNodeNormalized }); + if (isValid) { + keysInScope.clear(); + auto cols = GetRawColumnsFromRange(*range); + keysInScope.insert(cols.begin(), cols.end()); + } + } +} + +bool IsListOfMembers(const TExprNode& node) { + if (!node.IsList()) { + return false; + } + + return AllOf(node.ChildrenList(), [](const auto& child) { + return child->IsCallable("Member") && IsDataOrMultiOptionalOfData(child->GetTypeAnn()); + }); +} + +bool IsMemberBinOpNode(const TExprNode& node) { + YQL_ENSURE(node.ChildrenSize() == 2); + return node.Head().IsCallable("Member") && IsDataOrMultiOptionalOfData(node.Head().GetTypeAnn()) || + node.Tail().IsCallable("Member") && IsDataOrMultiOptionalOfData(node.Tail().GetTypeAnn()); +} + +bool IsMemberListBinOpNode(const TExprNode& node) { + YQL_ENSURE(node.ChildrenSize() == 2); + return IsListOfMembers(node.Head()) || IsListOfMembers(node.Tail()); +} + +TExprNode::TPtr OptimizeNodeForRangeExtraction(const TExprNode::TPtr& node, TExprContext& ctx) { + auto it = node->IsCallable() ? SupportedBinOps.find(node->Content()) : SupportedBinOps.end(); + if (it != SupportedBinOps.end()) { + TExprNode::TPtr toExpand; + if (IsListOfMembers(node->Head())) { + toExpand = node; + } else if (IsListOfMembers(node->Tail())) { + toExpand = ctx.NewCallable(node->Pos(), it->second, { node->TailPtr(), node->HeadPtr() }); + } + + if (toExpand) { + auto baseType = RemoveAllOptionals(toExpand->Tail().GetTypeAnn()); + YQL_ENSURE(baseType); + YQL_ENSURE(baseType->GetKind() == ETypeAnnotationKind::Tuple); + if (baseType->Cast<TTupleExprType>()->GetSize() != node->Head().ChildrenSize()) { + // comparison of tuples of different size + return node; + } + YQL_CLOG(DEBUG, Core) << node->Content() << " over tuple"; + return ExpandTupleBinOp(*toExpand, ctx); + } + + return node; + } + + if (node->IsCallable("Not")) { + if (IsCoalesceValidForRange(node->Head())) { + // The effect of this optimizer is the opposite of Coalesce optimizer in yql_co_simple1.cpp + auto firstArg = node->Head().HeadPtr(); + auto secondArg = node->Head().TailPtr(); + YQL_CLOG(DEBUG, Core) << "Push down " << node->Content() << " over Coalesce"; + return ctx.Builder(node->Pos()) + .Callable("Coalesce") + .Callable(0, "Not") + .Add(0, firstArg) + .Seal() + .Callable(1, "Not") + .Add(0, secondArg) + .Seal() + .Seal() + .Build(); + } + + static const THashMap<TStringBuf, TStringBuf> binOpsWithNegations = { + { "<", ">="}, + { "<=", ">"}, + { ">", "<="}, + { ">=", "<"}, + { "==", "!="}, + { "!=", "=="}, + }; + + auto it = node->Head().IsCallable() ? binOpsWithNegations.find(node->Head().Content()) : binOpsWithNegations.end(); + if (it != binOpsWithNegations.end() && (IsMemberBinOpNode(node->Head()) || IsMemberListBinOpNode(node->Head()))) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return ctx.RenameNode(node->Head(), it->second); + } + + if (node->Head().IsCallable({"And", "Or"})) { + TExprNodeList children; + for (auto& child : node->Head().ChildrenList()) { + children.push_back(ctx.NewCallable(child->Pos(), "Not", { child })); + } + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return ctx.NewCallable(node->Pos(), node->Head().IsCallable("Or") ? "And" : "Or", std::move(children)); + } + + if (node->Head().IsCallable("Not")) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); + return node->Head().HeadPtr(); + } + + if (node->Head().IsCallable("Bool")) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over literal Bool"; + bool value = FromString<bool>(node->Head().Head().Content()); + return MakeBool(node->Pos(), !value, ctx); + } + + return node; + } + + if (node->IsCallable("Nth")) { + if (node->Head().Type() == TExprNode::List) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over tuple literal"; + const auto index = FromString<ui32>(node->Tail().Content()); + return node->Head().ChildPtr(index); + } + } + + if (IsCoalesceValidForRange(*node) && node->Head().IsCallable({"And", "Or"})) { + YQL_CLOG(DEBUG, Core) << "PropagateCoalesceWithConst over " << node->Head().Content(); + auto children = node->Head().ChildrenList(); + for (auto& child : children) { + child = ctx.NewCallable(node->Pos(), node->Content(), {std::move(child), node->TailPtr()}); + } + return ctx.NewCallable(node->Head().Pos(), node->Head().Content(), std::move(children)); + } + + if (node->IsCallable({"And", "Or"})) { + if (AnyOf(node->ChildrenList(), [&node](const auto& child) { return child->IsCallable(node->Content()); })) { + YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Content(); + TExprNodeList newChildren; + for (auto& child : node->ChildrenList()) { + if (child->IsCallable(node->Content())) { + auto chs = child->ChildrenList(); + newChildren.insert(newChildren.end(), chs.begin(), chs.end()); + } else { + newChildren.emplace_back(std::move(child)); + } + } + return ctx.ChangeChildren(*node, std::move(newChildren)); + } + } + + return node; +} + +void DoOptimizeForRangeExtraction(const TExprNode::TPtr& input, TExprNode::TPtr& output, bool topLevel, TExprContext& ctx) { + output = OptimizeNodeForRangeExtraction(input, ctx); + if (output != input) { + return; + } + + auto children = input->ChildrenList(); + bool changed = false; + for (auto& child : children) { + if (!topLevel && !child->IsComputable()) { + continue; + } + TExprNode::TPtr newChild = child; + DoOptimizeForRangeExtraction(child, newChild, false, ctx); + if (newChild != child) { + changed = true; + child = std::move(newChild); + } + } + + if (changed) { + output = ctx.ChangeChildren(*input, std::move(children)); + } +} + +THolder<IGraphTransformer> CreateRangeExtractionOptimizer(TTypeAnnotationContext& types) { + TTransformationPipeline pipeline(&types); + + auto issueCode = TIssuesIds::CORE_EXEC; + pipeline.AddServiceTransformers(issueCode); + pipeline.AddTypeAnnotationTransformer(issueCode); // pipeline.Add(TExprLogTransformer::Sync("ExtractPredicateOpt", NLog::EComponent::Core, NLog::ELevel::TRACE), // "ExtractPredicateOpt", issueCode, "ExtractPredicateOpt"); - pipeline.Add(CreateFunctorTransformer( - [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - DoOptimizeForRangeExtraction(input, output, true, ctx); - IGraphTransformer::TStatus result = IGraphTransformer::TStatus::Ok; - if (input != output) { - result = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); - } - return result; - } - ), "ExtractPredicate", issueCode); - - return pipeline.BuildWithNoArgChecks(false); -} - -TCoLambda OptimizeLambdaForRangeExtraction(const TExprNode::TPtr& filterLambdaNode, TExprContext& ctx, TTypeAnnotationContext& typesCtx) { - YQL_ENSURE(filterLambdaNode->IsLambda()); - TCoLambda filterLambda(filterLambdaNode); - YQL_ENSURE(filterLambda.Args().Size() == 1); - - const TTypeAnnotationNode* argType = filterLambda.Args().Arg(0).Ref().GetTypeAnn(); - TConstraintNode::TListType argConstraints = filterLambda.Args().Arg(0).Ref().GetAllConstraints(); - - YQL_ENSURE(argType && argType->GetKind() == ETypeAnnotationKind::Struct); - - auto transformer = CreateRangeExtractionOptimizer(typesCtx); - TExprNode::TPtr output = filterLambdaNode; - YQL_CLOG(DEBUG, Core) << "Begin optimizing lambda for range extraction"; - for (;;) { - const auto status = InstantTransform(*transformer, output, ctx, true); - if (status == IGraphTransformer::TStatus::Ok) { - break; - } - YQL_ENSURE(status != IGraphTransformer::TStatus::Error); - YQL_ENSURE(status.HasRestart); - YQL_ENSURE(UpdateLambdaAllArgumentsTypes(output, { argType }, ctx)); - YQL_ENSURE(UpdateLambdaConstraints(output, ctx, { argConstraints }) != IGraphTransformer::TStatus::Error); - } - YQL_CLOG(DEBUG, Core) << "Finish optimizing lambda for range extraction"; - - return TCoLambda(output); -} - -TExprNode::TPtr BuildSingleComputeRange(const TStructExprType& rowType, - const TExprNode& range, const THashMap<TString, size_t>& indexKeysOrder, - const TPredicateExtractorSettings& settings, const TString& lastKey, TExprContext& ctx) -{ - TVector<TString> keys = GetColumnsFromRange(range, indexKeysOrder); - YQL_ENSURE(!keys.empty()); - - auto idx = rowType.FindItem(keys.front()); - YQL_ENSURE(idx); - const TTypeAnnotationNode* firstKeyType = rowType.GetItems()[*idx]->GetItemType(); - - bool hasNot = false; - auto opNode = GetOpFromRange(range, hasNot); - TPositionHandle pos = opNode->Pos(); - if (opNode->IsCallable("Exists")) { - YQL_ENSURE(keys.size() == 1); - return ctx.Builder(pos) - .Callable("RangeFor") - .Atom(0, hasNot ? "NotExists" : "Exists", TNodeFlags::Default) - .Callable(1, "Void") - .Seal() - .Add(2, ExpandType(pos, *firstKeyType, ctx)) - .Seal() - .Build(); - } - - if (opNode->IsCallable("StartsWith")) { - YQL_ENSURE(keys.size() == 1); - return ctx.Builder(pos) - .Callable("RangeFor") - .Atom(0, hasNot ? "NotStartsWith" : "StartsWith", TNodeFlags::Default) - .Add(1, opNode->TailPtr()) - .Add(2, ExpandType(pos, *firstKeyType, ctx)) - .Seal() - .Build(); - } - - if (opNode->IsCallable("SqlIn")) { - TCoSqlIn sqlIn(opNode); - - const auto inputCollection = ctx.NewArgument(pos, "collection"); - const bool haveTuples = sqlIn.Lookup().Ref().IsList(); - - TTypeAnnotationNode::TListType keyTypes; - for (auto& key : keys) { - auto idx = rowType.FindItem(key); - YQL_ENSURE(idx); - keyTypes.push_back(rowType.GetItems()[*idx]->GetItemType()); - } - - const TTypeAnnotationNode* compositeKeyType = nullptr; - if (haveTuples) { - compositeKeyType = ctx.MakeType<TTupleExprType>(std::move(keyTypes)); - } else { - YQL_ENSURE(keyTypes.size() == 1); - compositeKeyType = keyTypes.front(); - } - - TExprNode::TPtr collection = inputCollection; - if (haveTuples) { - TVector<TString> rawKeys = GetRawColumnsFromRange(range); - YQL_ENSURE(rawKeys.size() >= keys.size()); - THashMap<TString, size_t> rawKeysPos; - for (size_t i = 0; i < rawKeys.size(); ++i) { - // in case of duplicate members in tuple prefer the first one - if (!rawKeysPos.contains(rawKeys[i])) { - rawKeysPos[rawKeys[i]] = i; - } - } - - // sourcePosition for given targetPosition - TVector<size_t> sources; - for (size_t target = 0; target < keys.size(); ++target) { - TString key = keys[target]; - auto it = rawKeysPos.find(key); - YQL_ENSURE(it != rawKeysPos.end()); - size_t source = it->second; - sources.push_back(source); - } - - // reorder and normalize collection - auto collectionBaseType = RemoveAllOptionals(GetSqlInCollectionItemType(sqlIn.Collection().Ref().GetTypeAnn())); - collection = ctx.Builder(pos) - .Callable("Map") - .Callable(0, "FlatMap") - .Add(0, collection) - .Lambda(1) - .Param("item") - .Callable("StrictCast") - .Arg(0, "item") - .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(collectionBaseType), ctx)) - .Seal() - .Seal() - .Seal() - .Lambda(1) - .Param("tuple") - .List() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (size_t i = 0; i < sources.size(); ++i) { - parent - .Callable(i, "Nth") - .Arg(0, "tuple") - .Atom(1, ToString(sources[i]), TNodeFlags::Default) - .Seal(); - } - return parent; - }) - .Seal() - .Seal() - .Seal() - .Build(); - } - - TExprNode::TPtr body; - if (!hasNot) { - // IN = collection of point intervals - body = ctx.Builder(pos) - .Callable("Take") - .Callable(0, "FlatMap") - .Add(0, collection) - .Lambda(1) - .Param("item") - .Callable("RangeMultiply") - .Callable(0, "Uint64") - .Atom(0, ToString(settings.MaxRanges), TNodeFlags::Default) - .Seal() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (haveTuples) { - const auto& types = compositeKeyType->Cast<TTupleExprType>()->GetItems(); - for (size_t i = 0; i < types.size(); ++i) { - const bool needWidenKey = settings.MergeAdjacentPointRanges && - i + 1 == types.size() && lastKey == keys.back(); - parent - .Callable(i + 1, "RangeFor") - .Atom(0, needWidenKey ? "===" : "==", TNodeFlags::Default) - .Callable(1, "Nth") - .Arg(0, "item") - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Add(2, ExpandType(pos, *types[i], ctx)) - .Seal(); - } - } else { - const bool needWidenKey = settings.MergeAdjacentPointRanges && - lastKey == keys.back(); - parent - .Callable(1, "RangeFor") - .Atom(0, needWidenKey ? "===" : "==", TNodeFlags::Default) - .Arg(1, "item") - .Add(2, ExpandType(pos, *compositeKeyType, ctx)) - .Seal(); - } - return parent; - }) - .Seal() - .Seal() - .Seal() - .Callable(1, "Uint64") - // +1 is essential here - RangeMultiply will detect overflow by this extra item - .Atom(0, ToString(settings.MaxRanges + 1), TNodeFlags::Default) - .Seal() - .Seal() - .Build(); - body = ctx.Builder(pos) - .Callable("RangeUnion") - .Callable(0, "RangeMultiply") - .Callable(0, "Uint64") - .Atom(0, ToString(settings.MaxRanges), TNodeFlags::Default) - .Seal() - .Add(1, body) - .Seal() - .Seal() - .Build(); - } else { - YQL_ENSURE(false, "not supported yet, should be rejected earlier"); - } - - auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { inputCollection }), std::move(body)); - - auto optCollection = GetSqlInCollectionAsOptList(sqlIn.Collection().Ptr(), ctx); - return ctx.Builder(pos) - .Callable("IfPresent") - .Add(0, optCollection) - .Add(1, lambda) - .Callable(2, "RangeEmpty") - .Add(0, ExpandType(pos, *compositeKeyType, ctx)) - .Seal() - .Seal() - .Build(); - } - - YQL_ENSURE(!hasNot); - YQL_ENSURE(opNode->IsCallable({"<", "<=", ">", ">=", "==", "!="})); - YQL_ENSURE(keys.size() == 1); - TStringBuf op = opNode->Content(); - if (op == "==" && lastKey == keys.back() && settings.MergeAdjacentPointRanges) { - op = "==="; - } - return ctx.Builder(pos) - .Callable("RangeFor") - .Atom(0, op, TNodeFlags::Default) - .Add(1, opNode->TailPtr()) - .Add(2, ExpandType(pos, *firstKeyType, ctx)) - .Seal() - .Build(); -} - -TExprNode::TPtr BuildFullRange(TPositionHandle pos, const TStructExprType& rowType, const TVector<TString>& indexKeys, TExprContext& ctx) { - YQL_ENSURE(!indexKeys.empty()); - TExprNodeList components; - for (auto key : indexKeys) { - auto idx = rowType.FindItem(key); - YQL_ENSURE(idx); - const TTypeAnnotationNode* optKeyType = ctx.MakeType<TOptionalExprType>(rowType.GetItems()[*idx]->GetItemType()); - auto nullNode = ctx.NewCallable(pos, "Nothing", { ExpandType(pos, *optKeyType, ctx) }); - - components.push_back(nullNode); - } - - components.push_back(ctx.NewCallable(pos, "Int32", { ctx.NewAtom(pos, "0", TNodeFlags::Default) })); - auto boundary = ctx.NewList(pos, std::move(components)); - return ctx.Builder(pos) - .Callable("AsRange") - .List(0) - .Add(0, boundary) - .Add(1, boundary) - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr RebuildAsRangeRest(const TStructExprType& rowType, const TExprNode& range, TExprContext& ctx) { - if (range.IsCallable({"RangeRest", "Range"})) { - return ctx.RenameNode(range, "RangeRest"); - } - - TPositionHandle pos = range.Pos(); - auto row = ctx.NewArgument(pos, "row"); - TExprNode::TPtr predicate; - if (range.IsCallable("RangeConst")) { - predicate = range.HeadPtr(); - } else { - YQL_ENSURE(range.IsCallable({"RangeOr", "RangeAnd"})); - - TExprNodeList predicates; - for (auto& child : range.ChildrenList()) { - auto rebuilt = RebuildAsRangeRest(rowType, *child, ctx); - predicates.push_back(ctx.Builder(pos).Apply(rebuilt->TailPtr()).With(0, row).Seal().Build()); - } - predicate = ctx.NewCallable(pos, range.IsCallable("RangeOr") ? "Or" : "And", std::move(predicates)); - } - - auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { row }), std::move(predicate)); - return ctx.NewCallable(pos, "RangeRest", { ExpandType(pos, rowType, ctx), lambda }); -} - -struct TIndexRange { - size_t Begin = 0; - size_t End = 0; - bool IsPoint = false; - - bool IsEmpty() const { - return Begin >= End; - } - -}; - -bool operator<(const TIndexRange& a, const TIndexRange& b) { - if (a.IsEmpty()) { - return false; - } - if (b.IsEmpty()) { - return true; - } - - if (a.Begin != b.Begin) { - return a.Begin < b.Begin; - } - - return a.End < b.End; -} - -TIndexRange ExtractIndexRangeFromKeys(const TVector<TString>& keys, const THashMap<TString, size_t>& indexKeysOrder, bool isPoint) { - YQL_ENSURE(!keys.empty()); - - auto firstIt = indexKeysOrder.find(keys.front()); - auto lastIt = indexKeysOrder.find(keys.back()); - YQL_ENSURE(firstIt != indexKeysOrder.end() && lastIt != indexKeysOrder.end()); - - TIndexRange result; - result.Begin = firstIt->second; - result.End = lastIt->second + 1; - result.IsPoint = isPoint; - - return result; -} - -TExprNode::TPtr DoRebuildRangeForIndexKeys(const TStructExprType& rowType, const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder, - TIndexRange& resultIndexRange, TExprContext& ctx) -{ - resultIndexRange = {}; - if (range->IsCallable("RangeRest")) { - return range; - } - - if (range->IsCallable("RangeConst")) { - resultIndexRange.Begin = 0; - resultIndexRange.End = 1; - return range; - } - - if (range->IsCallable("Range")) { - auto keys = GetColumnsFromRange(*range, indexKeysOrder); - if (!keys.empty()) { - resultIndexRange = ExtractIndexRangeFromKeys(keys, indexKeysOrder, false); - return range; - } - return ctx.RenameNode(*range, "RangeRest"); - } - - TMaybe<TIndexRange> commonIndexRange; - TExprNodeList rebuilt; - if (range->IsCallable("RangeOr")) { - rebuilt = range->ChildrenList(); - for (auto& child : rebuilt) { - TIndexRange childIndexRange; - child = DoRebuildRangeForIndexKeys(rowType, child, indexKeysOrder, childIndexRange, ctx); - if (childIndexRange.IsEmpty()) { - YQL_ENSURE(child->IsCallable("RangeRest")); - return RebuildAsRangeRest(rowType, *range, ctx); - } - - if (!commonIndexRange) { - commonIndexRange = childIndexRange; - continue; - } - - if (commonIndexRange->Begin != childIndexRange.Begin) { - return RebuildAsRangeRest(rowType, *range, ctx); - } - - commonIndexRange->End = std::max(commonIndexRange->End, childIndexRange.End); - } - } else { - YQL_ENSURE(range->IsCallable("RangeAnd")); - struct TNodeAndIndexRange { - TExprNode::TPtr Node; - TIndexRange IndexRange; - }; - - TVector<TNodeAndIndexRange> toRebuild; - for (const auto& child : range->ChildrenList()) { - toRebuild.emplace_back(); - TNodeAndIndexRange& curr = toRebuild.back(); - curr.Node = DoRebuildRangeForIndexKeys(rowType, child, indexKeysOrder, curr.IndexRange, ctx); - } - - std::stable_sort(toRebuild.begin(), toRebuild.end(), [&](const TNodeAndIndexRange& a, const TNodeAndIndexRange& b) { - // sort children by key order - // move RangeRest/RangeConst to the end while preserving their relative order - return a.IndexRange < b.IndexRange; - }); - - - TExprNodeList rests; - TVector<TExprNodeList> childrenChains; - THashMap<size_t, TSet<size_t>> chainIdxByEndIdx; - - for (auto& current : toRebuild) { - if (current.IndexRange.IsEmpty()) { - YQL_ENSURE(current.Node->IsCallable("RangeRest")); - rests.emplace_back(std::move(current.Node)); - continue; - } - const size_t beginIdx = current.IndexRange.Begin; - const size_t endIdx = current.IndexRange.End; - if (!commonIndexRange || beginIdx == commonIndexRange->Begin) { - if (!commonIndexRange) { - commonIndexRange = current.IndexRange; - } else { - commonIndexRange->End = std::max(commonIndexRange->End, endIdx); - } - chainIdxByEndIdx[endIdx].insert(childrenChains.size()); - childrenChains.emplace_back(); - childrenChains.back().push_back(current.Node); - } else { - auto it = chainIdxByEndIdx.find(beginIdx); - if (it == chainIdxByEndIdx.end()) { - rests.emplace_back(RebuildAsRangeRest(rowType, *current.Node, ctx)); - continue; - } - - YQL_ENSURE(!it->second.empty()); - const size_t tgtChainIdx = *it->second.begin(); - it->second.erase(tgtChainIdx); - if (it->second.empty()) { - chainIdxByEndIdx.erase(it); - } - - childrenChains[tgtChainIdx].push_back(current.Node); - chainIdxByEndIdx[endIdx].insert(tgtChainIdx); - commonIndexRange->End = std::max(commonIndexRange->End, endIdx); - } - } - - for (auto& chain : childrenChains) { - YQL_ENSURE(!chain.empty()); - rebuilt.push_back(ctx.NewCallable(range->Pos(), "RangeAnd", std::move(chain))); - } - - if (!rests.empty()) { - rebuilt.push_back(RebuildAsRangeRest(rowType, *ctx.NewCallable(range->Pos(), "RangeAnd", std::move(rests)), ctx)); - } - } - - TExprNode::TPtr result = ctx.ChangeChildren(*range, std::move(rebuilt)); - if (commonIndexRange) { - resultIndexRange = *commonIndexRange; - } else { - result = RebuildAsRangeRest(rowType, *result, ctx); - } - - return result; -} - -TExprNode::TPtr RebuildRangeForIndexKeys(const TStructExprType& rowType, const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder, - size_t& usedPrefixLen, TExprContext& ctx) -{ - TIndexRange resultIndexRange; - auto result = DoRebuildRangeForIndexKeys(rowType, range, indexKeysOrder, resultIndexRange, ctx); - YQL_ENSURE(result); - if (!resultIndexRange.IsEmpty() && resultIndexRange.Begin != 0) { - resultIndexRange = {}; - result = RebuildAsRangeRest(rowType, *result, ctx); - } - usedPrefixLen = resultIndexRange.IsEmpty() ? 0 : resultIndexRange.End; - return result; -} - -TMaybe<size_t> CalcMaxRanges(const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder) { - if (range->IsCallable("RangeConst")) { - return 1; - } - - if (range->IsCallable("RangeRest")) { - return 0; - } - - if (range->IsCallable("Range")) { - auto opNode = GetOpFromRange(*range); - if (opNode->IsCallable("SqlIn")) { - TCoSqlIn sqlIn(opNode); - return GetSqlInCollectionSize(sqlIn.Collection().Ptr()); - } - - return opNode->IsCallable("!=") ? 2 : 1; - } - - if (range->IsCallable("RangeOr")) { - size_t result = 0; - for (auto& child : range->ChildrenList()) { - YQL_ENSURE(!child->IsCallable("RangeRest")); - auto childRanges = CalcMaxRanges(child, indexKeysOrder); - if (!childRanges) { - return {}; - } - result += *childRanges; - } - return result; - } - - YQL_ENSURE(range->IsCallable("RangeAnd")); - TMap<TVector<TString>, size_t> maxRangesByKey; - TVector<size_t> toMultiply; - for (auto& child : range->ChildrenList()) { - TMaybe<size_t> childRanges; - if (child->IsCallable("Range") || !child->IsCallable("RangeRest")) { - childRanges = CalcMaxRanges(child, indexKeysOrder); - if (!childRanges) { - return {}; - } - - if (child->IsCallable("Range")) { - auto key = GetColumnsFromRange(*child, indexKeysOrder); - maxRangesByKey[key] = std::max(maxRangesByKey[key], *childRanges); - } else { - toMultiply.push_back(*childRanges); - } - } - } - - for (const auto& [key, r] : maxRangesByKey) { - toMultiply.push_back(r); - } - - size_t result = 1; - for (auto p : toMultiply) { - YQL_ENSURE(p); - if (result > std::numeric_limits<size_t>::max() / p) { - // overflow - return {}; - } - result *= p; - } - - return result; -} - -TExprNode::TPtr MakePredicateFromPrunedRange(const TExprNode::TPtr& range, const TExprNode::TPtr& row, TExprContext& ctx) { - TPositionHandle pos = range->Pos(); - if (range->IsCallable("RangeRest")) { - return ctx.Builder(pos) - .Apply(range->TailPtr()) - .With(0, row) - .Seal() - .Build(); - } - - YQL_ENSURE(range->IsCallable({"RangeOr", "RangeAnd"})); - TExprNodeList children = range->ChildrenList(); - for (auto& child : children) { - child = MakePredicateFromPrunedRange(child, row, ctx); - } - - return ctx.NewCallable(pos, range->IsCallable("RangeOr") ? "Or" : "And", std::move(children)); -} - -TExprNode::TPtr BuildRestTrue(TPositionHandle pos, const TTypeAnnotationNode& rowType, TExprContext& ctx) { - return ctx.Builder(pos) - .Callable("RangeRest") - .Add(0, ExpandType(pos, rowType, ctx)) - .Lambda(1) - .Param("row") - .Callable("Bool") - .Atom(0, "true", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Build(); -} - -bool IsRestTrue(const TExprNode& node) { - if (node.IsCallable("RangeRest")) { - TCoLambda lambda(node.TailPtr()); - if (auto maybeBool = lambda.Body().Maybe<TCoBool>().Literal()) { - if (maybeBool.Cast().Value() == "true") { - return true; - } - } - } - return false; -} - -TExprNode::TPtr BuildRangeMultiply(TPositionHandle pos, size_t maxRanges, const TExprNodeList& toMultiply, TExprContext& ctx) { - TExprNodeList args; - args.reserve(toMultiply.size() + 1); - args.push_back(ctx.NewCallable(pos, "Uint64", { ctx.NewAtom(pos, ToString(maxRanges), TNodeFlags::Default) })); - args.insert(args.end(), toMultiply.begin(), toMultiply.end()); - return ctx.NewCallable(pos, "RangeMultiply", std::move(args)); -} - -TExprNode::TPtr DoBuildMultiColumnComputeNode(const TStructExprType& rowType, const TExprNode::TPtr& range, - const TVector<TString>& indexKeys, const THashMap<TString, size_t>& indexKeysOrder, - TExprNode::TPtr& prunedRange, TIndexRange& resultIndexRange, const TPredicateExtractorSettings& settings, - size_t usedPrefixLen, TExprContext& ctx) -{ - prunedRange = {}; - resultIndexRange = {}; - TPositionHandle pos = range->Pos(); - if (range->IsCallable("Range")) { - // top-level node or child of RangeOr - auto cols = GetColumnsFromRange(*range, indexKeysOrder); - resultIndexRange = ExtractIndexRangeFromKeys(cols, indexKeysOrder, IsPointRange(*range)); - - auto rawCols = GetRawColumnsFromRange(*range); - prunedRange = (rawCols.size() == cols.size()) ? - BuildRestTrue(pos, rowType, ctx) : - RebuildAsRangeRest(rowType, *range, ctx); - YQL_ENSURE(usedPrefixLen > 0 && usedPrefixLen <= indexKeys.size()); - return BuildSingleComputeRange(rowType, *range, indexKeysOrder, settings, indexKeys[usedPrefixLen - 1], ctx); - } - - if (range->IsCallable("RangeRest")) { - // top-level RangeRest - prunedRange = range; - return {}; - } - - if (range->IsCallable("RangeConst")) { - prunedRange = BuildRestTrue(pos, rowType, ctx); - resultIndexRange.Begin = 0; - resultIndexRange.End = 1; - resultIndexRange.IsPoint = false; - return ctx.Builder(pos) - .Callable("If") - .Add(0, range->HeadPtr()) - .Add(1, BuildFullRange(range->Pos(), rowType, { indexKeys.front() }, ctx)) - .Callable(2, "RangeEmpty") - .Seal() - .Seal() - .Build(); - } - - TExprNodeList output; - TExprNodeList prunedOutput; - if (range->IsCallable("RangeOr")) { - TVector<TIndexRange> childIndexRanges; - for (auto& child : range->ChildrenList()) { - prunedOutput.emplace_back(); - TIndexRange childIndexRange; - output.push_back(DoBuildMultiColumnComputeNode(rowType, child, indexKeys, indexKeysOrder, prunedOutput.back(), childIndexRange, settings, usedPrefixLen, ctx)); - childIndexRanges.push_back(childIndexRange); - YQL_ENSURE(!childIndexRange.IsEmpty()); - if (resultIndexRange.IsEmpty()) { - resultIndexRange = childIndexRange; - } else { - YQL_ENSURE(childIndexRange.Begin == resultIndexRange.Begin); - resultIndexRange.End = std::max(resultIndexRange.End, childIndexRange.End); - resultIndexRange.IsPoint = resultIndexRange.IsPoint && childIndexRange.IsPoint; - } - } - - for (size_t i = 0; i < output.size(); ++i) { - if (childIndexRanges[i].End < resultIndexRange.End) { - TVector<TString> alignKeys(indexKeys.begin() + childIndexRanges[i].End, indexKeys.begin() + resultIndexRange.End); - output[i] = BuildRangeMultiply(pos, settings.MaxRanges, { output[i], BuildFullRange(pos, rowType, alignKeys, ctx) }, ctx); - } - } - } else { - YQL_ENSURE(range->IsCallable("RangeAnd")); - TVector<TIndexRange> childIndexRanges; - bool needAlign = true; - for (const auto& child : range->ChildrenList()) { - prunedOutput.emplace_back(); - TIndexRange childIndexRange; - auto compute = DoBuildMultiColumnComputeNode(rowType, child, indexKeys, indexKeysOrder, prunedOutput.back(), childIndexRange, settings, usedPrefixLen, ctx); - if (!compute) { - continue; - } - output.push_back(compute); - childIndexRanges.push_back(childIndexRange); - YQL_ENSURE(!childIndexRange.IsEmpty()); - if (resultIndexRange.IsEmpty()) { - resultIndexRange = childIndexRange; - } else { - if (childIndexRange.Begin != resultIndexRange.Begin) { - needAlign = false; - if (!resultIndexRange.IsPoint) { - prunedOutput.back() = RebuildAsRangeRest(rowType, *child, ctx); - } - resultIndexRange.IsPoint = resultIndexRange.IsPoint && childIndexRange.IsPoint; - } else { - resultIndexRange.IsPoint = resultIndexRange.IsPoint || childIndexRange.IsPoint; - } - resultIndexRange.End = std::max(resultIndexRange.End, childIndexRange.End); - } - } - - if (needAlign) { - for (size_t i = 0; i < output.size(); ++i) { - if (childIndexRanges[i].End < resultIndexRange.End) { - TVector<TString> alignKeys(indexKeys.begin() + childIndexRanges[i].End, indexKeys.begin() + resultIndexRange.End); - output[i] = BuildRangeMultiply(pos, settings.MaxRanges, { output[i], BuildFullRange(pos, rowType, alignKeys, ctx) }, ctx); - } - } - } else { - output = { BuildRangeMultiply(pos, settings.MaxRanges, output, ctx) }; - } - } - YQL_ENSURE(!prunedOutput.empty()); - YQL_ENSURE(!output.empty()); - - prunedOutput.erase( - std::remove_if(prunedOutput.begin(), prunedOutput.end(), [](const auto& pruned) { return IsRestTrue(*pruned); }), - prunedOutput.end() - ); - - if (prunedOutput.empty()) { - prunedRange = BuildRestTrue(pos, rowType, ctx); - } else if (range->IsCallable("RangeOr")) { - prunedRange = RebuildAsRangeRest(rowType, *range, ctx); - } else { - prunedRange = ctx.NewCallable(pos, range->Content(), std::move(prunedOutput)); - } - - return ctx.NewCallable(pos, range->IsCallable("RangeOr") ? "RangeUnion" : "RangeIntersect", std::move(output)); -} - -TExprNode::TPtr BuildMultiColumnComputeNode(const TStructExprType& rowType, const TExprNode::TPtr& range, - const TVector<TString>& indexKeys, const THashMap<TString, size_t>& indexKeysOrder, - TExprNode::TPtr& prunedRange, const TPredicateExtractorSettings& settings, size_t usedPrefixLen, TExprContext& ctx) -{ - TIndexRange resultIndexRange; - auto result = DoBuildMultiColumnComputeNode(rowType, range, indexKeys, indexKeysOrder, prunedRange, resultIndexRange, settings, usedPrefixLen, ctx); - YQL_ENSURE(prunedRange); - if (result) { - YQL_ENSURE(!resultIndexRange.IsEmpty()); - YQL_ENSURE(resultIndexRange.Begin == 0); - YQL_ENSURE(usedPrefixLen == resultIndexRange.End); - - TPositionHandle pos = range->Pos(); - if (resultIndexRange.End < indexKeys.size()) { - result = BuildRangeMultiply(pos, settings.MaxRanges, - { result, BuildFullRange(pos, rowType, TVector<TString>(indexKeys.begin() + resultIndexRange.End, indexKeys.end()), ctx) }, ctx); - } - - if (!result->IsCallable("RangeUnion")) { - // normalize ranges - result = ctx.NewCallable(pos, "RangeUnion", { result }); - } - - if (!result->IsCallable("RangeMultiply")) { - // apply max_ranges limit - result = BuildRangeMultiply(pos, settings.MaxRanges, { result }, ctx); - } - - // convert to user format - result = ctx.NewCallable(pos, "RangeFinalize", { result }); - } - return result; -} - - -} // namespace - -bool TPredicateRangeExtractor::Prepare(const TExprNode::TPtr& filterLambdaNode, const TTypeAnnotationNode& rowType, - THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) -{ - possibleIndexKeys.clear(); - YQL_ENSURE(!FilterLambda, "Prepare() should be called only once"); - FilterLambda = filterLambdaNode; - YQL_ENSURE(rowType.GetKind() == ETypeAnnotationKind::Struct); - RowType = rowType.Cast<TStructExprType>(); - - TCoLambda filterLambda = OptimizeLambdaForRangeExtraction(filterLambdaNode, ctx, typesCtx); - - TExprNode::TPtr rowArg = filterLambda.Args().Arg(0).Ptr(); - TExprNode::TPtr pred; - - if (auto maybeCond = filterLambda.Body().Maybe<TCoConditionalValueBase>()) { - pred = maybeCond.Cast().Predicate().Ptr(); - } else { - pred = filterLambda.Body().Ptr(); - } - - const TTypeAnnotationNode* rowArgType = rowArg->GetTypeAnn(); - YQL_ENSURE(EnsureSpecificDataType(*pred, EDataSlot::Bool, ctx)); - YQL_ENSURE(rowArgType->GetKind() == ETypeAnnotationKind::Struct); - for (auto& item : rowArgType->Cast<TStructExprType>()->GetItems()) { - auto idx = RowType->FindItem(item->GetName()); - YQL_ENSURE(idx, - "RowType/lambda arg mismatch: column " << item->GetName() << " is missing in original table"); - YQL_ENSURE(IsSameAnnotation(*RowType->GetItems()[*idx]->GetItemType(), *item->GetItemType()), - "RowType/lambda arg mismatch: column " << item->GetName() << " has type: " << *item->GetItemType() << - " expecting: " << *RowType->GetItems()[*idx]->GetItemType()); - } - - TSet<TString> keysInScope; - DoBuildRanges(rowArg, pred, Settings, Range, keysInScope, ctx, false); - possibleIndexKeys.insert(keysInScope.begin(), keysInScope.end()); - - TOptimizeExprSettings settings(nullptr); - settings.VisitChanges = true; - // replace plain predicate in Range/RangeRest with lambda - auto status = OptimizeExpr(Range, Range, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { - if (node->IsCallable({"Range", "RangeRest"}) && node->ChildrenSize() == 1) { - auto pos = node->Pos(); - // use lambda in Range/RangeRest - auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { rowArg }), node->HeadPtr()); - return ctx.Builder(pos) - .Callable(node->Content()) - .Add(0, ExpandType(pos, *rowArgType, ctx)) - .Lambda(1) - .Param("row") - .Apply(lambda) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Build(); - } - return node; - }, ctx, settings); - return status == IGraphTransformer::TStatus::Ok; -} - -TPredicateRangeExtractor::TBuildResult TPredicateRangeExtractor::BuildComputeNode(const TVector<TString>& indexKeys, - TExprContext& ctx) const -{ - YQL_ENSURE(FilterLambda && Range && RowType, "Prepare() is not called"); - - TBuildResult result; - result.PrunedLambda = ctx.DeepCopyLambda(*FilterLambda); - - { - THashSet<TString> uniqIndexKeys; - for (auto& key : indexKeys) { - YQL_ENSURE(uniqIndexKeys.insert(key).second, "Duplicate index column " << key); - } - } - - THashMap<TString, size_t> indexKeysOrder; - TVector<TString> effectiveIndexKeys = indexKeys; - for (size_t i = 0; i < effectiveIndexKeys.size(); ++i) { - TMaybe<ui32> idx = RowType->FindItem(effectiveIndexKeys[i]); - if (idx) { - auto keyBaseType = RemoveAllOptionals(RowType->GetItems()[*idx]->GetItemType()); - if (!(keyBaseType->GetKind() == ETypeAnnotationKind::Data && keyBaseType->IsComparable() && keyBaseType->IsEquatable())) { - idx = {}; - } - } - if (!idx) { - effectiveIndexKeys.resize(i); - break; - } - indexKeysOrder[effectiveIndexKeys[i]] = i; - } - - if (effectiveIndexKeys.empty()) { - return result; - } - - TExprNode::TPtr rebuiltRange = RebuildRangeForIndexKeys(*RowType, Range, indexKeysOrder, result.UsedPrefixLen, ctx); - TExprNode::TPtr prunedRange; - result.ComputeNode = BuildMultiColumnComputeNode(*RowType, rebuiltRange, effectiveIndexKeys, indexKeysOrder, - prunedRange, Settings, result.UsedPrefixLen, ctx); - if (result.ComputeNode) { - result.ExpectedMaxRanges = CalcMaxRanges(rebuiltRange, indexKeysOrder); - if (result.ExpectedMaxRanges && *result.ExpectedMaxRanges < Settings.MaxRanges) { - // rebuild filter lambda with prunedRange predicate - TCoLambda lambda(result.PrunedLambda); - auto newPred = MakePredicateFromPrunedRange(prunedRange, lambda.Args().Arg(0).Ptr(), ctx); - - TExprNode::TPtr newBody; - if (auto maybeCond = lambda.Body().Maybe<TCoConditionalValueBase>()) { - newBody = ctx.ChangeChild(lambda.Body().Ref(), TCoConditionalValueBase::idx_Predicate, std::move(newPred)); - } else { - newBody = std::move(newPred); - } - - result.PrunedLambda = ctx.ChangeChild(lambda.Ref(), TCoLambda::idx_Body, std::move(newBody)); - } - } - return result; -} - -} // namespace NDetail - -IPredicateRangeExtractor::TPtr MakePredicateRangeExtractor(const TPredicateExtractorSettings& settings) { + pipeline.Add(CreateFunctorTransformer( + [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + DoOptimizeForRangeExtraction(input, output, true, ctx); + IGraphTransformer::TStatus result = IGraphTransformer::TStatus::Ok; + if (input != output) { + result = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); + } + return result; + } + ), "ExtractPredicate", issueCode); + + return pipeline.BuildWithNoArgChecks(false); +} + +TCoLambda OptimizeLambdaForRangeExtraction(const TExprNode::TPtr& filterLambdaNode, TExprContext& ctx, TTypeAnnotationContext& typesCtx) { + YQL_ENSURE(filterLambdaNode->IsLambda()); + TCoLambda filterLambda(filterLambdaNode); + YQL_ENSURE(filterLambda.Args().Size() == 1); + + const TTypeAnnotationNode* argType = filterLambda.Args().Arg(0).Ref().GetTypeAnn(); + TConstraintNode::TListType argConstraints = filterLambda.Args().Arg(0).Ref().GetAllConstraints(); + + YQL_ENSURE(argType && argType->GetKind() == ETypeAnnotationKind::Struct); + + auto transformer = CreateRangeExtractionOptimizer(typesCtx); + TExprNode::TPtr output = filterLambdaNode; + YQL_CLOG(DEBUG, Core) << "Begin optimizing lambda for range extraction"; + for (;;) { + const auto status = InstantTransform(*transformer, output, ctx, true); + if (status == IGraphTransformer::TStatus::Ok) { + break; + } + YQL_ENSURE(status != IGraphTransformer::TStatus::Error); + YQL_ENSURE(status.HasRestart); + YQL_ENSURE(UpdateLambdaAllArgumentsTypes(output, { argType }, ctx)); + YQL_ENSURE(UpdateLambdaConstraints(output, ctx, { argConstraints }) != IGraphTransformer::TStatus::Error); + } + YQL_CLOG(DEBUG, Core) << "Finish optimizing lambda for range extraction"; + + return TCoLambda(output); +} + +TExprNode::TPtr BuildSingleComputeRange(const TStructExprType& rowType, + const TExprNode& range, const THashMap<TString, size_t>& indexKeysOrder, + const TPredicateExtractorSettings& settings, const TString& lastKey, TExprContext& ctx) +{ + TVector<TString> keys = GetColumnsFromRange(range, indexKeysOrder); + YQL_ENSURE(!keys.empty()); + + auto idx = rowType.FindItem(keys.front()); + YQL_ENSURE(idx); + const TTypeAnnotationNode* firstKeyType = rowType.GetItems()[*idx]->GetItemType(); + + bool hasNot = false; + auto opNode = GetOpFromRange(range, hasNot); + TPositionHandle pos = opNode->Pos(); + if (opNode->IsCallable("Exists")) { + YQL_ENSURE(keys.size() == 1); + return ctx.Builder(pos) + .Callable("RangeFor") + .Atom(0, hasNot ? "NotExists" : "Exists", TNodeFlags::Default) + .Callable(1, "Void") + .Seal() + .Add(2, ExpandType(pos, *firstKeyType, ctx)) + .Seal() + .Build(); + } + + if (opNode->IsCallable("StartsWith")) { + YQL_ENSURE(keys.size() == 1); + return ctx.Builder(pos) + .Callable("RangeFor") + .Atom(0, hasNot ? "NotStartsWith" : "StartsWith", TNodeFlags::Default) + .Add(1, opNode->TailPtr()) + .Add(2, ExpandType(pos, *firstKeyType, ctx)) + .Seal() + .Build(); + } + + if (opNode->IsCallable("SqlIn")) { + TCoSqlIn sqlIn(opNode); + + const auto inputCollection = ctx.NewArgument(pos, "collection"); + const bool haveTuples = sqlIn.Lookup().Ref().IsList(); + + TTypeAnnotationNode::TListType keyTypes; + for (auto& key : keys) { + auto idx = rowType.FindItem(key); + YQL_ENSURE(idx); + keyTypes.push_back(rowType.GetItems()[*idx]->GetItemType()); + } + + const TTypeAnnotationNode* compositeKeyType = nullptr; + if (haveTuples) { + compositeKeyType = ctx.MakeType<TTupleExprType>(std::move(keyTypes)); + } else { + YQL_ENSURE(keyTypes.size() == 1); + compositeKeyType = keyTypes.front(); + } + + TExprNode::TPtr collection = inputCollection; + if (haveTuples) { + TVector<TString> rawKeys = GetRawColumnsFromRange(range); + YQL_ENSURE(rawKeys.size() >= keys.size()); + THashMap<TString, size_t> rawKeysPos; + for (size_t i = 0; i < rawKeys.size(); ++i) { + // in case of duplicate members in tuple prefer the first one + if (!rawKeysPos.contains(rawKeys[i])) { + rawKeysPos[rawKeys[i]] = i; + } + } + + // sourcePosition for given targetPosition + TVector<size_t> sources; + for (size_t target = 0; target < keys.size(); ++target) { + TString key = keys[target]; + auto it = rawKeysPos.find(key); + YQL_ENSURE(it != rawKeysPos.end()); + size_t source = it->second; + sources.push_back(source); + } + + // reorder and normalize collection + auto collectionBaseType = RemoveAllOptionals(GetSqlInCollectionItemType(sqlIn.Collection().Ref().GetTypeAnn())); + collection = ctx.Builder(pos) + .Callable("Map") + .Callable(0, "FlatMap") + .Add(0, collection) + .Lambda(1) + .Param("item") + .Callable("StrictCast") + .Arg(0, "item") + .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(collectionBaseType), ctx)) + .Seal() + .Seal() + .Seal() + .Lambda(1) + .Param("tuple") + .List() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (size_t i = 0; i < sources.size(); ++i) { + parent + .Callable(i, "Nth") + .Arg(0, "tuple") + .Atom(1, ToString(sources[i]), TNodeFlags::Default) + .Seal(); + } + return parent; + }) + .Seal() + .Seal() + .Seal() + .Build(); + } + + TExprNode::TPtr body; + if (!hasNot) { + // IN = collection of point intervals + body = ctx.Builder(pos) + .Callable("Take") + .Callable(0, "FlatMap") + .Add(0, collection) + .Lambda(1) + .Param("item") + .Callable("RangeMultiply") + .Callable(0, "Uint64") + .Atom(0, ToString(settings.MaxRanges), TNodeFlags::Default) + .Seal() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + if (haveTuples) { + const auto& types = compositeKeyType->Cast<TTupleExprType>()->GetItems(); + for (size_t i = 0; i < types.size(); ++i) { + const bool needWidenKey = settings.MergeAdjacentPointRanges && + i + 1 == types.size() && lastKey == keys.back(); + parent + .Callable(i + 1, "RangeFor") + .Atom(0, needWidenKey ? "===" : "==", TNodeFlags::Default) + .Callable(1, "Nth") + .Arg(0, "item") + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Add(2, ExpandType(pos, *types[i], ctx)) + .Seal(); + } + } else { + const bool needWidenKey = settings.MergeAdjacentPointRanges && + lastKey == keys.back(); + parent + .Callable(1, "RangeFor") + .Atom(0, needWidenKey ? "===" : "==", TNodeFlags::Default) + .Arg(1, "item") + .Add(2, ExpandType(pos, *compositeKeyType, ctx)) + .Seal(); + } + return parent; + }) + .Seal() + .Seal() + .Seal() + .Callable(1, "Uint64") + // +1 is essential here - RangeMultiply will detect overflow by this extra item + .Atom(0, ToString(settings.MaxRanges + 1), TNodeFlags::Default) + .Seal() + .Seal() + .Build(); + body = ctx.Builder(pos) + .Callable("RangeUnion") + .Callable(0, "RangeMultiply") + .Callable(0, "Uint64") + .Atom(0, ToString(settings.MaxRanges), TNodeFlags::Default) + .Seal() + .Add(1, body) + .Seal() + .Seal() + .Build(); + } else { + YQL_ENSURE(false, "not supported yet, should be rejected earlier"); + } + + auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { inputCollection }), std::move(body)); + + auto optCollection = GetSqlInCollectionAsOptList(sqlIn.Collection().Ptr(), ctx); + return ctx.Builder(pos) + .Callable("IfPresent") + .Add(0, optCollection) + .Add(1, lambda) + .Callable(2, "RangeEmpty") + .Add(0, ExpandType(pos, *compositeKeyType, ctx)) + .Seal() + .Seal() + .Build(); + } + + YQL_ENSURE(!hasNot); + YQL_ENSURE(opNode->IsCallable({"<", "<=", ">", ">=", "==", "!="})); + YQL_ENSURE(keys.size() == 1); + TStringBuf op = opNode->Content(); + if (op == "==" && lastKey == keys.back() && settings.MergeAdjacentPointRanges) { + op = "==="; + } + return ctx.Builder(pos) + .Callable("RangeFor") + .Atom(0, op, TNodeFlags::Default) + .Add(1, opNode->TailPtr()) + .Add(2, ExpandType(pos, *firstKeyType, ctx)) + .Seal() + .Build(); +} + +TExprNode::TPtr BuildFullRange(TPositionHandle pos, const TStructExprType& rowType, const TVector<TString>& indexKeys, TExprContext& ctx) { + YQL_ENSURE(!indexKeys.empty()); + TExprNodeList components; + for (auto key : indexKeys) { + auto idx = rowType.FindItem(key); + YQL_ENSURE(idx); + const TTypeAnnotationNode* optKeyType = ctx.MakeType<TOptionalExprType>(rowType.GetItems()[*idx]->GetItemType()); + auto nullNode = ctx.NewCallable(pos, "Nothing", { ExpandType(pos, *optKeyType, ctx) }); + + components.push_back(nullNode); + } + + components.push_back(ctx.NewCallable(pos, "Int32", { ctx.NewAtom(pos, "0", TNodeFlags::Default) })); + auto boundary = ctx.NewList(pos, std::move(components)); + return ctx.Builder(pos) + .Callable("AsRange") + .List(0) + .Add(0, boundary) + .Add(1, boundary) + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr RebuildAsRangeRest(const TStructExprType& rowType, const TExprNode& range, TExprContext& ctx) { + if (range.IsCallable({"RangeRest", "Range"})) { + return ctx.RenameNode(range, "RangeRest"); + } + + TPositionHandle pos = range.Pos(); + auto row = ctx.NewArgument(pos, "row"); + TExprNode::TPtr predicate; + if (range.IsCallable("RangeConst")) { + predicate = range.HeadPtr(); + } else { + YQL_ENSURE(range.IsCallable({"RangeOr", "RangeAnd"})); + + TExprNodeList predicates; + for (auto& child : range.ChildrenList()) { + auto rebuilt = RebuildAsRangeRest(rowType, *child, ctx); + predicates.push_back(ctx.Builder(pos).Apply(rebuilt->TailPtr()).With(0, row).Seal().Build()); + } + predicate = ctx.NewCallable(pos, range.IsCallable("RangeOr") ? "Or" : "And", std::move(predicates)); + } + + auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { row }), std::move(predicate)); + return ctx.NewCallable(pos, "RangeRest", { ExpandType(pos, rowType, ctx), lambda }); +} + +struct TIndexRange { + size_t Begin = 0; + size_t End = 0; + bool IsPoint = false; + + bool IsEmpty() const { + return Begin >= End; + } + +}; + +bool operator<(const TIndexRange& a, const TIndexRange& b) { + if (a.IsEmpty()) { + return false; + } + if (b.IsEmpty()) { + return true; + } + + if (a.Begin != b.Begin) { + return a.Begin < b.Begin; + } + + return a.End < b.End; +} + +TIndexRange ExtractIndexRangeFromKeys(const TVector<TString>& keys, const THashMap<TString, size_t>& indexKeysOrder, bool isPoint) { + YQL_ENSURE(!keys.empty()); + + auto firstIt = indexKeysOrder.find(keys.front()); + auto lastIt = indexKeysOrder.find(keys.back()); + YQL_ENSURE(firstIt != indexKeysOrder.end() && lastIt != indexKeysOrder.end()); + + TIndexRange result; + result.Begin = firstIt->second; + result.End = lastIt->second + 1; + result.IsPoint = isPoint; + + return result; +} + +TExprNode::TPtr DoRebuildRangeForIndexKeys(const TStructExprType& rowType, const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder, + TIndexRange& resultIndexRange, TExprContext& ctx) +{ + resultIndexRange = {}; + if (range->IsCallable("RangeRest")) { + return range; + } + + if (range->IsCallable("RangeConst")) { + resultIndexRange.Begin = 0; + resultIndexRange.End = 1; + return range; + } + + if (range->IsCallable("Range")) { + auto keys = GetColumnsFromRange(*range, indexKeysOrder); + if (!keys.empty()) { + resultIndexRange = ExtractIndexRangeFromKeys(keys, indexKeysOrder, false); + return range; + } + return ctx.RenameNode(*range, "RangeRest"); + } + + TMaybe<TIndexRange> commonIndexRange; + TExprNodeList rebuilt; + if (range->IsCallable("RangeOr")) { + rebuilt = range->ChildrenList(); + for (auto& child : rebuilt) { + TIndexRange childIndexRange; + child = DoRebuildRangeForIndexKeys(rowType, child, indexKeysOrder, childIndexRange, ctx); + if (childIndexRange.IsEmpty()) { + YQL_ENSURE(child->IsCallable("RangeRest")); + return RebuildAsRangeRest(rowType, *range, ctx); + } + + if (!commonIndexRange) { + commonIndexRange = childIndexRange; + continue; + } + + if (commonIndexRange->Begin != childIndexRange.Begin) { + return RebuildAsRangeRest(rowType, *range, ctx); + } + + commonIndexRange->End = std::max(commonIndexRange->End, childIndexRange.End); + } + } else { + YQL_ENSURE(range->IsCallable("RangeAnd")); + struct TNodeAndIndexRange { + TExprNode::TPtr Node; + TIndexRange IndexRange; + }; + + TVector<TNodeAndIndexRange> toRebuild; + for (const auto& child : range->ChildrenList()) { + toRebuild.emplace_back(); + TNodeAndIndexRange& curr = toRebuild.back(); + curr.Node = DoRebuildRangeForIndexKeys(rowType, child, indexKeysOrder, curr.IndexRange, ctx); + } + + std::stable_sort(toRebuild.begin(), toRebuild.end(), [&](const TNodeAndIndexRange& a, const TNodeAndIndexRange& b) { + // sort children by key order + // move RangeRest/RangeConst to the end while preserving their relative order + return a.IndexRange < b.IndexRange; + }); + + + TExprNodeList rests; + TVector<TExprNodeList> childrenChains; + THashMap<size_t, TSet<size_t>> chainIdxByEndIdx; + + for (auto& current : toRebuild) { + if (current.IndexRange.IsEmpty()) { + YQL_ENSURE(current.Node->IsCallable("RangeRest")); + rests.emplace_back(std::move(current.Node)); + continue; + } + const size_t beginIdx = current.IndexRange.Begin; + const size_t endIdx = current.IndexRange.End; + if (!commonIndexRange || beginIdx == commonIndexRange->Begin) { + if (!commonIndexRange) { + commonIndexRange = current.IndexRange; + } else { + commonIndexRange->End = std::max(commonIndexRange->End, endIdx); + } + chainIdxByEndIdx[endIdx].insert(childrenChains.size()); + childrenChains.emplace_back(); + childrenChains.back().push_back(current.Node); + } else { + auto it = chainIdxByEndIdx.find(beginIdx); + if (it == chainIdxByEndIdx.end()) { + rests.emplace_back(RebuildAsRangeRest(rowType, *current.Node, ctx)); + continue; + } + + YQL_ENSURE(!it->second.empty()); + const size_t tgtChainIdx = *it->second.begin(); + it->second.erase(tgtChainIdx); + if (it->second.empty()) { + chainIdxByEndIdx.erase(it); + } + + childrenChains[tgtChainIdx].push_back(current.Node); + chainIdxByEndIdx[endIdx].insert(tgtChainIdx); + commonIndexRange->End = std::max(commonIndexRange->End, endIdx); + } + } + + for (auto& chain : childrenChains) { + YQL_ENSURE(!chain.empty()); + rebuilt.push_back(ctx.NewCallable(range->Pos(), "RangeAnd", std::move(chain))); + } + + if (!rests.empty()) { + rebuilt.push_back(RebuildAsRangeRest(rowType, *ctx.NewCallable(range->Pos(), "RangeAnd", std::move(rests)), ctx)); + } + } + + TExprNode::TPtr result = ctx.ChangeChildren(*range, std::move(rebuilt)); + if (commonIndexRange) { + resultIndexRange = *commonIndexRange; + } else { + result = RebuildAsRangeRest(rowType, *result, ctx); + } + + return result; +} + +TExprNode::TPtr RebuildRangeForIndexKeys(const TStructExprType& rowType, const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder, + size_t& usedPrefixLen, TExprContext& ctx) +{ + TIndexRange resultIndexRange; + auto result = DoRebuildRangeForIndexKeys(rowType, range, indexKeysOrder, resultIndexRange, ctx); + YQL_ENSURE(result); + if (!resultIndexRange.IsEmpty() && resultIndexRange.Begin != 0) { + resultIndexRange = {}; + result = RebuildAsRangeRest(rowType, *result, ctx); + } + usedPrefixLen = resultIndexRange.IsEmpty() ? 0 : resultIndexRange.End; + return result; +} + +TMaybe<size_t> CalcMaxRanges(const TExprNode::TPtr& range, const THashMap<TString, size_t>& indexKeysOrder) { + if (range->IsCallable("RangeConst")) { + return 1; + } + + if (range->IsCallable("RangeRest")) { + return 0; + } + + if (range->IsCallable("Range")) { + auto opNode = GetOpFromRange(*range); + if (opNode->IsCallable("SqlIn")) { + TCoSqlIn sqlIn(opNode); + return GetSqlInCollectionSize(sqlIn.Collection().Ptr()); + } + + return opNode->IsCallable("!=") ? 2 : 1; + } + + if (range->IsCallable("RangeOr")) { + size_t result = 0; + for (auto& child : range->ChildrenList()) { + YQL_ENSURE(!child->IsCallable("RangeRest")); + auto childRanges = CalcMaxRanges(child, indexKeysOrder); + if (!childRanges) { + return {}; + } + result += *childRanges; + } + return result; + } + + YQL_ENSURE(range->IsCallable("RangeAnd")); + TMap<TVector<TString>, size_t> maxRangesByKey; + TVector<size_t> toMultiply; + for (auto& child : range->ChildrenList()) { + TMaybe<size_t> childRanges; + if (child->IsCallable("Range") || !child->IsCallable("RangeRest")) { + childRanges = CalcMaxRanges(child, indexKeysOrder); + if (!childRanges) { + return {}; + } + + if (child->IsCallable("Range")) { + auto key = GetColumnsFromRange(*child, indexKeysOrder); + maxRangesByKey[key] = std::max(maxRangesByKey[key], *childRanges); + } else { + toMultiply.push_back(*childRanges); + } + } + } + + for (const auto& [key, r] : maxRangesByKey) { + toMultiply.push_back(r); + } + + size_t result = 1; + for (auto p : toMultiply) { + YQL_ENSURE(p); + if (result > std::numeric_limits<size_t>::max() / p) { + // overflow + return {}; + } + result *= p; + } + + return result; +} + +TExprNode::TPtr MakePredicateFromPrunedRange(const TExprNode::TPtr& range, const TExprNode::TPtr& row, TExprContext& ctx) { + TPositionHandle pos = range->Pos(); + if (range->IsCallable("RangeRest")) { + return ctx.Builder(pos) + .Apply(range->TailPtr()) + .With(0, row) + .Seal() + .Build(); + } + + YQL_ENSURE(range->IsCallable({"RangeOr", "RangeAnd"})); + TExprNodeList children = range->ChildrenList(); + for (auto& child : children) { + child = MakePredicateFromPrunedRange(child, row, ctx); + } + + return ctx.NewCallable(pos, range->IsCallable("RangeOr") ? "Or" : "And", std::move(children)); +} + +TExprNode::TPtr BuildRestTrue(TPositionHandle pos, const TTypeAnnotationNode& rowType, TExprContext& ctx) { + return ctx.Builder(pos) + .Callable("RangeRest") + .Add(0, ExpandType(pos, rowType, ctx)) + .Lambda(1) + .Param("row") + .Callable("Bool") + .Atom(0, "true", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Build(); +} + +bool IsRestTrue(const TExprNode& node) { + if (node.IsCallable("RangeRest")) { + TCoLambda lambda(node.TailPtr()); + if (auto maybeBool = lambda.Body().Maybe<TCoBool>().Literal()) { + if (maybeBool.Cast().Value() == "true") { + return true; + } + } + } + return false; +} + +TExprNode::TPtr BuildRangeMultiply(TPositionHandle pos, size_t maxRanges, const TExprNodeList& toMultiply, TExprContext& ctx) { + TExprNodeList args; + args.reserve(toMultiply.size() + 1); + args.push_back(ctx.NewCallable(pos, "Uint64", { ctx.NewAtom(pos, ToString(maxRanges), TNodeFlags::Default) })); + args.insert(args.end(), toMultiply.begin(), toMultiply.end()); + return ctx.NewCallable(pos, "RangeMultiply", std::move(args)); +} + +TExprNode::TPtr DoBuildMultiColumnComputeNode(const TStructExprType& rowType, const TExprNode::TPtr& range, + const TVector<TString>& indexKeys, const THashMap<TString, size_t>& indexKeysOrder, + TExprNode::TPtr& prunedRange, TIndexRange& resultIndexRange, const TPredicateExtractorSettings& settings, + size_t usedPrefixLen, TExprContext& ctx) +{ + prunedRange = {}; + resultIndexRange = {}; + TPositionHandle pos = range->Pos(); + if (range->IsCallable("Range")) { + // top-level node or child of RangeOr + auto cols = GetColumnsFromRange(*range, indexKeysOrder); + resultIndexRange = ExtractIndexRangeFromKeys(cols, indexKeysOrder, IsPointRange(*range)); + + auto rawCols = GetRawColumnsFromRange(*range); + prunedRange = (rawCols.size() == cols.size()) ? + BuildRestTrue(pos, rowType, ctx) : + RebuildAsRangeRest(rowType, *range, ctx); + YQL_ENSURE(usedPrefixLen > 0 && usedPrefixLen <= indexKeys.size()); + return BuildSingleComputeRange(rowType, *range, indexKeysOrder, settings, indexKeys[usedPrefixLen - 1], ctx); + } + + if (range->IsCallable("RangeRest")) { + // top-level RangeRest + prunedRange = range; + return {}; + } + + if (range->IsCallable("RangeConst")) { + prunedRange = BuildRestTrue(pos, rowType, ctx); + resultIndexRange.Begin = 0; + resultIndexRange.End = 1; + resultIndexRange.IsPoint = false; + return ctx.Builder(pos) + .Callable("If") + .Add(0, range->HeadPtr()) + .Add(1, BuildFullRange(range->Pos(), rowType, { indexKeys.front() }, ctx)) + .Callable(2, "RangeEmpty") + .Seal() + .Seal() + .Build(); + } + + TExprNodeList output; + TExprNodeList prunedOutput; + if (range->IsCallable("RangeOr")) { + TVector<TIndexRange> childIndexRanges; + for (auto& child : range->ChildrenList()) { + prunedOutput.emplace_back(); + TIndexRange childIndexRange; + output.push_back(DoBuildMultiColumnComputeNode(rowType, child, indexKeys, indexKeysOrder, prunedOutput.back(), childIndexRange, settings, usedPrefixLen, ctx)); + childIndexRanges.push_back(childIndexRange); + YQL_ENSURE(!childIndexRange.IsEmpty()); + if (resultIndexRange.IsEmpty()) { + resultIndexRange = childIndexRange; + } else { + YQL_ENSURE(childIndexRange.Begin == resultIndexRange.Begin); + resultIndexRange.End = std::max(resultIndexRange.End, childIndexRange.End); + resultIndexRange.IsPoint = resultIndexRange.IsPoint && childIndexRange.IsPoint; + } + } + + for (size_t i = 0; i < output.size(); ++i) { + if (childIndexRanges[i].End < resultIndexRange.End) { + TVector<TString> alignKeys(indexKeys.begin() + childIndexRanges[i].End, indexKeys.begin() + resultIndexRange.End); + output[i] = BuildRangeMultiply(pos, settings.MaxRanges, { output[i], BuildFullRange(pos, rowType, alignKeys, ctx) }, ctx); + } + } + } else { + YQL_ENSURE(range->IsCallable("RangeAnd")); + TVector<TIndexRange> childIndexRanges; + bool needAlign = true; + for (const auto& child : range->ChildrenList()) { + prunedOutput.emplace_back(); + TIndexRange childIndexRange; + auto compute = DoBuildMultiColumnComputeNode(rowType, child, indexKeys, indexKeysOrder, prunedOutput.back(), childIndexRange, settings, usedPrefixLen, ctx); + if (!compute) { + continue; + } + output.push_back(compute); + childIndexRanges.push_back(childIndexRange); + YQL_ENSURE(!childIndexRange.IsEmpty()); + if (resultIndexRange.IsEmpty()) { + resultIndexRange = childIndexRange; + } else { + if (childIndexRange.Begin != resultIndexRange.Begin) { + needAlign = false; + if (!resultIndexRange.IsPoint) { + prunedOutput.back() = RebuildAsRangeRest(rowType, *child, ctx); + } + resultIndexRange.IsPoint = resultIndexRange.IsPoint && childIndexRange.IsPoint; + } else { + resultIndexRange.IsPoint = resultIndexRange.IsPoint || childIndexRange.IsPoint; + } + resultIndexRange.End = std::max(resultIndexRange.End, childIndexRange.End); + } + } + + if (needAlign) { + for (size_t i = 0; i < output.size(); ++i) { + if (childIndexRanges[i].End < resultIndexRange.End) { + TVector<TString> alignKeys(indexKeys.begin() + childIndexRanges[i].End, indexKeys.begin() + resultIndexRange.End); + output[i] = BuildRangeMultiply(pos, settings.MaxRanges, { output[i], BuildFullRange(pos, rowType, alignKeys, ctx) }, ctx); + } + } + } else { + output = { BuildRangeMultiply(pos, settings.MaxRanges, output, ctx) }; + } + } + YQL_ENSURE(!prunedOutput.empty()); + YQL_ENSURE(!output.empty()); + + prunedOutput.erase( + std::remove_if(prunedOutput.begin(), prunedOutput.end(), [](const auto& pruned) { return IsRestTrue(*pruned); }), + prunedOutput.end() + ); + + if (prunedOutput.empty()) { + prunedRange = BuildRestTrue(pos, rowType, ctx); + } else if (range->IsCallable("RangeOr")) { + prunedRange = RebuildAsRangeRest(rowType, *range, ctx); + } else { + prunedRange = ctx.NewCallable(pos, range->Content(), std::move(prunedOutput)); + } + + return ctx.NewCallable(pos, range->IsCallable("RangeOr") ? "RangeUnion" : "RangeIntersect", std::move(output)); +} + +TExprNode::TPtr BuildMultiColumnComputeNode(const TStructExprType& rowType, const TExprNode::TPtr& range, + const TVector<TString>& indexKeys, const THashMap<TString, size_t>& indexKeysOrder, + TExprNode::TPtr& prunedRange, const TPredicateExtractorSettings& settings, size_t usedPrefixLen, TExprContext& ctx) +{ + TIndexRange resultIndexRange; + auto result = DoBuildMultiColumnComputeNode(rowType, range, indexKeys, indexKeysOrder, prunedRange, resultIndexRange, settings, usedPrefixLen, ctx); + YQL_ENSURE(prunedRange); + if (result) { + YQL_ENSURE(!resultIndexRange.IsEmpty()); + YQL_ENSURE(resultIndexRange.Begin == 0); + YQL_ENSURE(usedPrefixLen == resultIndexRange.End); + + TPositionHandle pos = range->Pos(); + if (resultIndexRange.End < indexKeys.size()) { + result = BuildRangeMultiply(pos, settings.MaxRanges, + { result, BuildFullRange(pos, rowType, TVector<TString>(indexKeys.begin() + resultIndexRange.End, indexKeys.end()), ctx) }, ctx); + } + + if (!result->IsCallable("RangeUnion")) { + // normalize ranges + result = ctx.NewCallable(pos, "RangeUnion", { result }); + } + + if (!result->IsCallable("RangeMultiply")) { + // apply max_ranges limit + result = BuildRangeMultiply(pos, settings.MaxRanges, { result }, ctx); + } + + // convert to user format + result = ctx.NewCallable(pos, "RangeFinalize", { result }); + } + return result; +} + + +} // namespace + +bool TPredicateRangeExtractor::Prepare(const TExprNode::TPtr& filterLambdaNode, const TTypeAnnotationNode& rowType, + THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) +{ + possibleIndexKeys.clear(); + YQL_ENSURE(!FilterLambda, "Prepare() should be called only once"); + FilterLambda = filterLambdaNode; + YQL_ENSURE(rowType.GetKind() == ETypeAnnotationKind::Struct); + RowType = rowType.Cast<TStructExprType>(); + + TCoLambda filterLambda = OptimizeLambdaForRangeExtraction(filterLambdaNode, ctx, typesCtx); + + TExprNode::TPtr rowArg = filterLambda.Args().Arg(0).Ptr(); + TExprNode::TPtr pred; + + if (auto maybeCond = filterLambda.Body().Maybe<TCoConditionalValueBase>()) { + pred = maybeCond.Cast().Predicate().Ptr(); + } else { + pred = filterLambda.Body().Ptr(); + } + + const TTypeAnnotationNode* rowArgType = rowArg->GetTypeAnn(); + YQL_ENSURE(EnsureSpecificDataType(*pred, EDataSlot::Bool, ctx)); + YQL_ENSURE(rowArgType->GetKind() == ETypeAnnotationKind::Struct); + for (auto& item : rowArgType->Cast<TStructExprType>()->GetItems()) { + auto idx = RowType->FindItem(item->GetName()); + YQL_ENSURE(idx, + "RowType/lambda arg mismatch: column " << item->GetName() << " is missing in original table"); + YQL_ENSURE(IsSameAnnotation(*RowType->GetItems()[*idx]->GetItemType(), *item->GetItemType()), + "RowType/lambda arg mismatch: column " << item->GetName() << " has type: " << *item->GetItemType() << + " expecting: " << *RowType->GetItems()[*idx]->GetItemType()); + } + + TSet<TString> keysInScope; + DoBuildRanges(rowArg, pred, Settings, Range, keysInScope, ctx, false); + possibleIndexKeys.insert(keysInScope.begin(), keysInScope.end()); + + TOptimizeExprSettings settings(nullptr); + settings.VisitChanges = true; + // replace plain predicate in Range/RangeRest with lambda + auto status = OptimizeExpr(Range, Range, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + if (node->IsCallable({"Range", "RangeRest"}) && node->ChildrenSize() == 1) { + auto pos = node->Pos(); + // use lambda in Range/RangeRest + auto lambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { rowArg }), node->HeadPtr()); + return ctx.Builder(pos) + .Callable(node->Content()) + .Add(0, ExpandType(pos, *rowArgType, ctx)) + .Lambda(1) + .Param("row") + .Apply(lambda) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Build(); + } + return node; + }, ctx, settings); + return status == IGraphTransformer::TStatus::Ok; +} + +TPredicateRangeExtractor::TBuildResult TPredicateRangeExtractor::BuildComputeNode(const TVector<TString>& indexKeys, + TExprContext& ctx) const +{ + YQL_ENSURE(FilterLambda && Range && RowType, "Prepare() is not called"); + + TBuildResult result; + result.PrunedLambda = ctx.DeepCopyLambda(*FilterLambda); + + { + THashSet<TString> uniqIndexKeys; + for (auto& key : indexKeys) { + YQL_ENSURE(uniqIndexKeys.insert(key).second, "Duplicate index column " << key); + } + } + + THashMap<TString, size_t> indexKeysOrder; + TVector<TString> effectiveIndexKeys = indexKeys; + for (size_t i = 0; i < effectiveIndexKeys.size(); ++i) { + TMaybe<ui32> idx = RowType->FindItem(effectiveIndexKeys[i]); + if (idx) { + auto keyBaseType = RemoveAllOptionals(RowType->GetItems()[*idx]->GetItemType()); + if (!(keyBaseType->GetKind() == ETypeAnnotationKind::Data && keyBaseType->IsComparable() && keyBaseType->IsEquatable())) { + idx = {}; + } + } + if (!idx) { + effectiveIndexKeys.resize(i); + break; + } + indexKeysOrder[effectiveIndexKeys[i]] = i; + } + + if (effectiveIndexKeys.empty()) { + return result; + } + + TExprNode::TPtr rebuiltRange = RebuildRangeForIndexKeys(*RowType, Range, indexKeysOrder, result.UsedPrefixLen, ctx); + TExprNode::TPtr prunedRange; + result.ComputeNode = BuildMultiColumnComputeNode(*RowType, rebuiltRange, effectiveIndexKeys, indexKeysOrder, + prunedRange, Settings, result.UsedPrefixLen, ctx); + if (result.ComputeNode) { + result.ExpectedMaxRanges = CalcMaxRanges(rebuiltRange, indexKeysOrder); + if (result.ExpectedMaxRanges && *result.ExpectedMaxRanges < Settings.MaxRanges) { + // rebuild filter lambda with prunedRange predicate + TCoLambda lambda(result.PrunedLambda); + auto newPred = MakePredicateFromPrunedRange(prunedRange, lambda.Args().Arg(0).Ptr(), ctx); + + TExprNode::TPtr newBody; + if (auto maybeCond = lambda.Body().Maybe<TCoConditionalValueBase>()) { + newBody = ctx.ChangeChild(lambda.Body().Ref(), TCoConditionalValueBase::idx_Predicate, std::move(newPred)); + } else { + newBody = std::move(newPred); + } + + result.PrunedLambda = ctx.ChangeChild(lambda.Ref(), TCoLambda::idx_Body, std::move(newBody)); + } + } + return result; +} + +} // namespace NDetail + +IPredicateRangeExtractor::TPtr MakePredicateRangeExtractor(const TPredicateExtractorSettings& settings) { return MakeHolder<NDetail::TPredicateRangeExtractor>(settings); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.h b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.h index 571f5aef82..8e2d693f72 100644 --- a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.h +++ b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.h @@ -1,28 +1,28 @@ -#pragma once - -#include "extract_predicate.h" - -namespace NYql::NDetail { - -class TPredicateRangeExtractor : public IPredicateRangeExtractor { -public: - explicit TPredicateRangeExtractor(const TPredicateExtractorSettings& settings = {}) - : Settings(settings) - {} - - bool Prepare(const TExprNode::TPtr& filterLambdaNode, const TTypeAnnotationNode& rowType, - THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) override final; - - TExprNode::TPtr GetPreparedRange() const { - return Range; - } - - TBuildResult BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx) const override final; -private: - const TPredicateExtractorSettings Settings; - TExprNode::TPtr FilterLambda; - const TStructExprType* RowType = nullptr; - TExprNode::TPtr Range; -}; - -} +#pragma once + +#include "extract_predicate.h" + +namespace NYql::NDetail { + +class TPredicateRangeExtractor : public IPredicateRangeExtractor { +public: + explicit TPredicateRangeExtractor(const TPredicateExtractorSettings& settings = {}) + : Settings(settings) + {} + + bool Prepare(const TExprNode::TPtr& filterLambdaNode, const TTypeAnnotationNode& rowType, + THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) override final; + + TExprNode::TPtr GetPreparedRange() const { + return Range; + } + + TBuildResult BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx) const override final; +private: + const TPredicateExtractorSettings Settings; + TExprNode::TPtr FilterLambda; + const TStructExprType* RowType = nullptr; + TExprNode::TPtr Range; +}; + +} diff --git a/ydb/library/yql/core/extract_predicate/ya.make b/ydb/library/yql/core/extract_predicate/ya.make index 12193371bf..0f426e8b71 100644 --- a/ydb/library/yql/core/extract_predicate/ya.make +++ b/ydb/library/yql/core/extract_predicate/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - extract_predicate_dbg.cpp - extract_predicate_dbg.h - extract_predicate_impl.cpp - extract_predicate_impl.h - extract_predicate.h -) - -PEERDIR( +LIBRARY() + +OWNER(g:yql) + +SRCS( + extract_predicate_dbg.cpp + extract_predicate_dbg.h + extract_predicate_impl.cpp + extract_predicate_impl.h + extract_predicate.h +) + +PEERDIR( ydb/library/yql/core/services -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp index 7676505e43..b74abc1c34 100644 --- a/ydb/library/yql/core/facade/yql_facade.cpp +++ b/ydb/library/yql/core/facade/yql_facade.cpp @@ -124,17 +124,17 @@ TProgramFactory::TProgramFactory( , GatewaysConfig_(nullptr) , Runner_(runner) { - AddCredentialsTable(std::make_shared<TCredentialTable>()); + AddCredentialsTable(std::make_shared<TCredentialTable>()); } void TProgramFactory::UnrepeatableRandom() { UseUnrepeatableRandom = true; } -void TProgramFactory::EnableRangeComputeFor() { - EnableRangeComputeFor_ = true; -} - +void TProgramFactory::EnableRangeComputeFor() { + EnableRangeComputeFor_ = true; +} + void TProgramFactory::AddUserDataTable(const TUserDataTable& userDataTable) { for (auto& p : userDataTable) { if (!UserDataTable_.emplace(p).second) { @@ -198,7 +198,7 @@ TProgramPtr TProgramFactory::Create( // make UserDataTable_ copy here return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_, UserDataTable_, CredentialTables_, UserCredentials_, moduleResolver, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, - GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_); + GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_); } /////////////////////////////////////////////////////////////////////////////// @@ -222,8 +222,8 @@ TProgram::TProgram( const TString& filename, const TString& sourceCode, const TString& sessionId, - const TString& runner, - bool enableRangeComputeFor + const TString& runner, + bool enableRangeComputeFor ) : FunctionRegistry_(functionRegistry) , RandomProvider_(randomProvider) @@ -248,7 +248,7 @@ TProgram::TProgram( , SessionId_(sessionId) , ResultFormat_(NYson::EYsonFormat::Binary) , OutputFormat_(NYson::EYsonFormat::Pretty) - , EnableRangeComputeFor_(enableRangeComputeFor) + , EnableRangeComputeFor_(enableRangeComputeFor) { if (SessionId_.empty()) { SessionId_ = CreateGuidAsString(); @@ -344,15 +344,15 @@ bool TProgram::ExtractQueryParametersMetadata() { return true; } -bool TProgram::FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules) { +bool TProgram::FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules) { if (!astRes.Issues.Empty()) { if (!ExprCtx_) { ExprCtx_.Reset(new TExprContext(NextUniqueId_)); } auto& iManager = ExprCtx_->IssueManager; - if (warningRules) { - for (auto warningRule: *warningRules) { - iManager.AddWarningRule(warningRule); + if (warningRules) { + for (auto warningRule: *warningRules) { + iManager.AddWarningRule(warningRule); } } iManager.AddScope([this]() { @@ -413,8 +413,8 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) YQL_ENSURE(SourceSyntax_ == ESourceSyntax::Unknown); SourceSyntax_ = ESourceSyntax::Sql; SyntaxVersion_ = settings.SyntaxVersion; - NYql::TWarningRules warningRules; - return FillParseResult(SqlToYql(SourceCode_, settings, &warningRules), &warningRules); + NYql::TWarningRules warningRules; + return FillParseResult(SqlToYql(SourceCode_, settings, &warningRules), &warningRules); } bool TProgram::Compile(const TString& username) { @@ -576,7 +576,7 @@ TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutput .AddExpressionEvaluation(*FunctionRegistry_) .AddIOAnnotation() .AddTypeAnnotation() - .Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput") + .Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput") .Build(); TFuture<void> openSession = OpenSession(username); @@ -645,10 +645,10 @@ TProgram::TFutureStatus TProgram::OptimizeAsync( .AddIOAnnotation() .AddTypeAnnotation() .AddPostTypeAnnotation() - .Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput") + .Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput") .AddOptimization() .Add(CreatePlanInfoTransformer(*TypeCtx_), "PlanInfo") - .Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput") + .Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput") .Add(TPlanOutputTransformer::Sync(tracePlan, GetPlanBuilder(), OutputFormat_), "PlanOutput") .Build(); @@ -711,11 +711,11 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig( pipelineConf.AfterTypeAnnotation(&pipeline); pipeline.AddOptimization(); - if (EnableRangeComputeFor_) { - pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), - "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); - } - + if (EnableRangeComputeFor_) { + pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), + "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); + } + pipeline.Add(CreatePlanInfoTransformer(*TypeCtx_), "PlanInfo"); pipelineConf.AfterOptimize(&pipeline); @@ -780,26 +780,26 @@ TProgram::TFutureStatus TProgram::RunAsync( ExprStream_ = exprOut; PlanStream_ = tracePlan; - TTransformationPipeline pipeline(TypeCtx_); - pipeline.AddServiceTransformers(); - pipeline.AddParametersEvaluation(*FunctionRegistry_); - pipeline.AddPreTypeAnnotation(); - pipeline.AddExpressionEvaluation(*FunctionRegistry_); - pipeline.AddIOAnnotation(); - pipeline.AddTypeAnnotation(); - pipeline.AddPostTypeAnnotation(); - pipeline.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput"); - pipeline.AddOptimization(); - if (EnableRangeComputeFor_) { - pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), - "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); - } - pipeline.Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput"); - pipeline.Add(TPlanOutputTransformer::Sync(tracePlan, GetPlanBuilder(), OutputFormat_), "PlanOutput"); - pipeline.AddRun(ProgressWriter_); - - Transformer_ = pipeline.Build(); - + TTransformationPipeline pipeline(TypeCtx_); + pipeline.AddServiceTransformers(); + pipeline.AddParametersEvaluation(*FunctionRegistry_); + pipeline.AddPreTypeAnnotation(); + pipeline.AddExpressionEvaluation(*FunctionRegistry_); + pipeline.AddIOAnnotation(); + pipeline.AddTypeAnnotation(); + pipeline.AddPostTypeAnnotation(); + pipeline.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput"); + pipeline.AddOptimization(); + if (EnableRangeComputeFor_) { + pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), + "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); + } + pipeline.Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput"); + pipeline.Add(TPlanOutputTransformer::Sync(tracePlan, GetPlanBuilder(), OutputFormat_), "PlanOutput"); + pipeline.AddRun(ProgressWriter_); + + Transformer_ = pipeline.Build(); + TFuture<void> openSession = OpenSession(username); if (!openSession.Initialized()) { return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); @@ -863,10 +863,10 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig( pipelineConf.AfterTypeAnnotation(&pipeline); pipeline.AddOptimization(); - if (EnableRangeComputeFor_) { - pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), - "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); - } + if (EnableRangeComputeFor_) { + pipeline.Add(MakeExpandRangeComputeForTransformer(pipeline.GetTypeAnnotationContext()), + "ExpandRangeComputeFor", TIssuesIds::CORE_EXEC); + } pipelineConf.AfterOptimize(&pipeline); pipeline.AddRun(ProgressWriter_); @@ -1006,7 +1006,7 @@ TMaybe<TString> TProgram::GetQueryAst() { astStream.Reserve(DEFAULT_AST_BUF_SIZE); if (ExprRoot_) { - auto ast = ConvertToAst(*ExprRoot_, *ExprCtx_, TExprAnnotationFlags::None, true); + auto ast = ConvertToAst(*ExprRoot_, *ExprCtx_, TExprAnnotationFlags::None, true); ast.Root->PrettyPrintTo(astStream, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine); return astStream.Str(); } else if (AstRoot_) { @@ -1361,19 +1361,19 @@ TFuture<void> TProgram::OpenSession(const TString& username) return WaitExceptionOrAll(openFutures); } -void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan) { +void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan) { TVector<TTransformStage> printTransformers; const auto issueCode = TIssuesIds::DEFAULT_ERROR; if (exprOut) { printTransformers.push_back(TTransformStage( - TExprOutputTransformer::Sync(ExprRoot_, exprOut), + TExprOutputTransformer::Sync(ExprRoot_, exprOut), "ExprOutput", issueCode)); } if (planOut) { - if (cleanPlan) { - GetPlanBuilder().Clear(); - } + if (cleanPlan) { + GetPlanBuilder().Clear(); + } printTransformers.push_back(TTransformStage( TPlanOutputTransformer::Sync(planOut, GetPlanBuilder(), OutputFormat_), "PlanOutput", diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h index 409dbf0433..1f8f47681d 100644 --- a/ydb/library/yql/core/facade/yql_facade.h +++ b/ydb/library/yql/core/facade/yql_facade.h @@ -52,7 +52,7 @@ public: void SetUdfResolver(IUdfResolver::TPtr udfResolver); void SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet); void SetFileStorage(TFileStoragePtr fileStorage); - void EnableRangeComputeFor(); + void EnableRangeComputeFor(); TProgramPtr Create( const TFile& file, @@ -80,7 +80,7 @@ private: TUdfIndexPackageSet::TPtr UdfIndexPackageSet_; TFileStoragePtr FileStorage_; TString Runner_; - bool EnableRangeComputeFor_ = false; + bool EnableRangeComputeFor_ = false; }; /////////////////////////////////////////////////////////////////////////////// @@ -176,7 +176,7 @@ public: } } - void Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan = false); + void Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanPlan = false); inline void PrintErrorsTo(IOutputStream& out) const { if (ExprCtx_) { @@ -192,10 +192,10 @@ public: return ExprRoot_; } - inline TExprContext& ExprCtx() const { - return *ExprCtx_; - } - + inline TExprContext& ExprCtx() const { + return *ExprCtx_; + } + inline bool HasResults() const { return ResultProviderConfig_ && !ResultProviderConfig_->CommittedResults.empty(); @@ -311,8 +311,8 @@ private: const TString& filename, const TString& sourceCode, const TString& sessionId, - const TString& runner, - bool enableRangeComputeFor); + const TString& runner, + bool enableRangeComputeFor); TTypeAnnotationContextPtr BuildTypeAnnotationContext(const TString& username); TTypeAnnotationContextPtr GetAnnotationContext() const; @@ -328,7 +328,7 @@ private: TFutureStatus RemoteKikimrOptimize(const TString& cluster, const IPipelineConfigurator* pipelineConf); TFutureStatus RemoteKikimrRun(const TString& cluster, const IPipelineConfigurator* pipelineConf); - bool FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules = nullptr); + bool FillParseResult(NYql::TAstParseResult&& astRes, NYql::TWarningRules* warningRules = nullptr); TString GetSessionId() const; TString TakeSessionId(); @@ -384,7 +384,7 @@ private: IOutputStream* PlanStream_ = nullptr; TOperationProgressWriter ProgressWriter_ = [](const TOperationProgress&) {}; TString ExtractedQueryParametersMetadataYson_; - const bool EnableRangeComputeFor_; + const bool EnableRangeComputeFor_; }; } // namspace NYql diff --git a/ydb/library/yql/core/file_storage/file_storage.cpp b/ydb/library/yql/core/file_storage/file_storage.cpp index 9ce62b6164..631adbc688 100644 --- a/ydb/library/yql/core/file_storage/file_storage.cpp +++ b/ydb/library/yql/core/file_storage/file_storage.cpp @@ -80,7 +80,7 @@ public: if (srcLength == -1) { ythrow TSystemError() << "cannot get file length: " << file; } - + YQL_ENSURE(srcLength == length); return std::make_pair(static_cast<ui64>(length), md5); }); @@ -138,12 +138,12 @@ public: TUnbufferedFileOutput out(outFile); auto result = std::make_pair(TransferData(&in, &out), md5); outFile.Close(); - + i64 length = GetFileLength(dstFile.c_str()); if (length == -1) { ythrow TSystemError() << "cannot get file length: " << dstFile; } - + YQL_ENSURE(data.size() == static_cast<ui64>(length)); return result; }); diff --git a/ydb/library/yql/core/file_storage/storage.cpp b/ydb/library/yql/core/file_storage/storage.cpp index 19fb1ab057..247bff5242 100644 --- a/ydb/library/yql/core/file_storage/storage.cpp +++ b/ydb/library/yql/core/file_storage/storage.cpp @@ -13,7 +13,7 @@ #include <util/generic/ptr.h> #include <util/generic/utility.h> #include <util/system/file.h> -#include <util/system/file_lock.h> +#include <util/system/file_lock.h> #include <util/system/fs.h> #include <util/system/maxlen.h> #include <util/system/mutex.h> @@ -162,11 +162,11 @@ public: StorageDir.MkDirs(MODE0711); ProcessTempDir.MkDirs(MODE0711); -#ifdef _linux_ - ProcessTempDirLock.Reset(new TFileLock(ProcessTempDir / ".lockfile")); - ProcessTempDirLock->Acquire(); - // We never explicitly release this lock. It will be released when all file handles (including those in child processes) will be closed -#endif +#ifdef _linux_ + ProcessTempDirLock.Reset(new TFileLock(ProcessTempDir / ".lockfile")); + ProcessTempDirLock->Acquire(); + // We never explicitly release this lock. It will be released when all file handles (including those in child processes) will be closed +#endif if (!IsTemp) { LoadStats(); @@ -340,18 +340,18 @@ private: if (!IsProcessAlive(oldPid)) { // cleanup of previously not cleaned hardlinks directory try { -#ifdef _linux_ - TFileLock childLock(childPath / ".lockfile"); - TTryGuard guard(childLock); -#else - bool guard = true; -#endif - if (guard) { - childPath.ForceDelete(); - } else { - YQL_LOG(WARN) << "Not cleaning dead process dir " << childPath - << ": " << "directory is still locked, skipping"; - } +#ifdef _linux_ + TFileLock childLock(childPath / ".lockfile"); + TTryGuard guard(childLock); +#else + bool guard = true; +#endif + if (guard) { + childPath.ForceDelete(); + } else { + YQL_LOG(WARN) << "Not cleaning dead process dir " << childPath + << ": " << "directory is still locked, skipping"; + } } catch (...) { YQL_LOG(WARN) << "Error cleaning dead process dir " << childPath << ": " << CurrentExceptionMessage(); @@ -436,7 +436,7 @@ private: TMutex CleanupLock; const TFsPath StorageDir; const TFsPath ProcessTempDir; - THolder<TFileLock> ProcessTempDirLock; + THolder<TFileLock> ProcessTempDirLock; const bool IsTemp; const ui64 MaxFiles; const ui64 MaxSize; diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index 60305745a3..a2ed91d640 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -33,10 +33,10 @@ message TIssuesIds { CORE_NON_STREAM_BATCH_UDF = 1105; CORE_FLATTEN_BY_OPT = 1106; CORE_IMPLICIT_BITCAST = 1107; - CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE = 1108; - CORE_LEGACY_RANK_FOR_NULLABLE_KEYS = 1109; + CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE = 1108; + CORE_LEGACY_RANK_FOR_NULLABLE_KEYS = 1109; CORE_LEGACY_REGEX_ENGINE = 1110; - CORE_ALIAS_SHADOWS_COLUMN = 1111; + CORE_ALIAS_SHADOWS_COLUMN = 1111; // core errors CORE_GC_NODES_LIMIT_EXCEEDED = 1500; @@ -93,7 +93,7 @@ message TIssuesIds { YT_CONCURRENT_TABLE_MODIF = 3009; YT_LATE_TABLE_XLOCK = 3010; YT_SORT_ORDER_CHANGE = 3011; - YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED = 3012; + YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED = 3012; YT_WARN_TABLE_DOES_NOT_EXIST = 3013; YT_ROWSPEC_HIDES_FIELDS = 3014; YT_ROWSPEC_DIFF_SORT = 3015; @@ -107,32 +107,32 @@ message TIssuesIds { YQL_ORDER_BY_WITHOUT_LIMIT_IN_SUBQUERY = 4504; YQL_DEPRECATED_DOUBLE_QUOTE_IN_BRACKETS = 4505; YQL_DEPRECATED_JSON_UDF = 4506; - YQL_MISSING_IS_BEFORE_NOT_NULL = 4507; - YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX = 4508; + YQL_MISSING_IS_BEFORE_NOT_NULL = 4507; + YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX = 4508; YQL_DEPRECATED_DATETIME2 = 4509; YQL_S_EXPRESSIONS_CALL = 4510; YQL_DEPRECATED_INTERVAL_CONSTANT = 4511; YQL_DEPRECATED_FUNCTION_OR_SIGNATURE = 4512; - YQL_DEPRECATED_INLINE_ACTION_TERMINATOR = 4513; - YQL_MULTIWAY_JOIN_WITH_USING = 4514; - YQL_DEPRECATED_V0_SYNTAX = 4515; - YQL_UNNAMED_COLUMN = 4516; + YQL_DEPRECATED_INLINE_ACTION_TERMINATOR = 4513; + YQL_MULTIWAY_JOIN_WITH_USING = 4514; + YQL_DEPRECATED_V0_SYNTAX = 4515; + YQL_UNNAMED_COLUMN = 4516; YQL_SOURCE_SELECT_COLUMN_MISMATCH = 4517; YQL_DEPRECATED_PRAGMA = 4518; - YQL_EMPTY_WINDOW_FRAME = 4520; - YQL_RANK_WITHOUT_ORDER_BY = 4521; - YQL_LIMIT_ORDER_BY_WITH_UNION = 4522; - YQL_DISCARD_INTO_RESULT_BY_WITH_UNION = 4523; + YQL_EMPTY_WINDOW_FRAME = 4520; + YQL_RANK_WITHOUT_ORDER_BY = 4521; + YQL_LIMIT_ORDER_BY_WITH_UNION = 4522; + YQL_DISCARD_INTO_RESULT_BY_WITH_UNION = 4523; YQL_DEPRECATED_UDF_FUNCTION = 4524; - YQL_EMPTY_TABLENAME_RESULT = 4525; - YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT = 4526; - YQL_UNUSED_SYMBOL = 4527; + YQL_EMPTY_TABLENAME_RESULT = 4525; + YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT = 4526; + YQL_UNUSED_SYMBOL = 4527; YQL_MIXED_TZ = 4528; - YQL_OPERATION_WILL_RETURN_NULL = 4529; + YQL_OPERATION_WILL_RETURN_NULL = 4529; YQL_JSON_QUERY_RETURNING_JSON_IS_DEPRECATED = 4530; YQL_DEPRECATED_LIST_FLATMAP_OPTIONAL = 4531; - YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY = 4532; - YQL_TABLE_BINDING_DUPLICATE = 4533; + YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY = 4532; + YQL_TABLE_BINDING_DUPLICATE = 4533; // yql parser errors YQL_NOT_ALLOWED_IN_DISCOVERY = 4600; diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index cc2b89626f..897d5d6f9a 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -104,22 +104,22 @@ ids { severity: S_WARNING } ids { - code: CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE - severity: S_WARNING -} -ids { - code: CORE_LEGACY_RANK_FOR_NULLABLE_KEYS - severity: S_WARNING -} -ids { + code: CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE + severity: S_WARNING +} +ids { + code: CORE_LEGACY_RANK_FOR_NULLABLE_KEYS + severity: S_WARNING +} +ids { code: CORE_LEGACY_REGEX_ENGINE severity: S_WARNING } ids { - code: CORE_ALIAS_SHADOWS_COLUMN - severity: S_WARNING -} -ids { + code: CORE_ALIAS_SHADOWS_COLUMN + severity: S_WARNING +} +ids { code: CORE_GC_NODES_LIMIT_EXCEEDED severity: S_ERROR } @@ -320,15 +320,15 @@ ids { code: YQL_DEPRECATED_JSON_UDF severity: S_WARNING } -ids { - code: YQL_MISSING_IS_BEFORE_NOT_NULL - severity: S_WARNING -} ids { - code: YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX - severity: S_WARNING -} -ids { + code: YQL_MISSING_IS_BEFORE_NOT_NULL + severity: S_WARNING +} +ids { + code: YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX + severity: S_WARNING +} +ids { code: YQL_DEPRECATED_INTERVAL_CONSTANT severity: S_WARNING } @@ -389,10 +389,10 @@ ids { severity: S_WARNING } ids { - code: YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED - severity: S_ERROR -} -ids { + code: YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED + severity: S_ERROR +} +ids { code: YT_WARN_TABLE_DOES_NOT_EXIST severity: S_WARNING } @@ -420,21 +420,21 @@ ids { code: YQL_DEPRECATED_FUNCTION_OR_SIGNATURE severity: S_WARNING } -ids { - code: YQL_DEPRECATED_INLINE_ACTION_TERMINATOR - severity: S_WARNING -} -ids { - code: YQL_MULTIWAY_JOIN_WITH_USING - severity: S_WARNING -} -ids { - code: YQL_DEPRECATED_V0_SYNTAX - severity: S_WARNING -} -ids { - code: YQL_UNNAMED_COLUMN - severity: S_WARNING +ids { + code: YQL_DEPRECATED_INLINE_ACTION_TERMINATOR + severity: S_WARNING +} +ids { + code: YQL_MULTIWAY_JOIN_WITH_USING + severity: S_WARNING +} +ids { + code: YQL_DEPRECATED_V0_SYNTAX + severity: S_WARNING +} +ids { + code: YQL_UNNAMED_COLUMN + severity: S_WARNING } ids { code: YQL_SOURCE_SELECT_COLUMN_MISMATCH @@ -496,7 +496,7 @@ ids { code: JSONPATH_DIVISION_BY_ZERO severity: S_ERROR } -ids { +ids { code: JSONPATH_BINARY_OPERATION_RESULT_INFINITY severity: S_ERROR } @@ -541,45 +541,45 @@ ids { severity: S_ERROR } ids { - code: YQL_EMPTY_WINDOW_FRAME - severity: S_WARNING -} -ids { - code: YQL_RANK_WITHOUT_ORDER_BY - severity: S_WARNING -} -ids { - code: YQL_LIMIT_ORDER_BY_WITH_UNION - severity: S_WARNING -} -ids { - code: YQL_DISCARD_INTO_RESULT_BY_WITH_UNION - severity: S_WARNING + code: YQL_EMPTY_WINDOW_FRAME + severity: S_WARNING +} +ids { + code: YQL_RANK_WITHOUT_ORDER_BY + severity: S_WARNING +} +ids { + code: YQL_LIMIT_ORDER_BY_WITH_UNION + severity: S_WARNING +} +ids { + code: YQL_DISCARD_INTO_RESULT_BY_WITH_UNION + severity: S_WARNING } ids { code: YQL_DEPRECATED_UDF_FUNCTION severity: S_WARNING } -ids { - code: YQL_EMPTY_TABLENAME_RESULT - severity: S_WARNING -} -ids { - code: YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT - severity: S_WARNING -} -ids { - code: YQL_UNUSED_SYMBOL - severity: S_WARNING +ids { + code: YQL_EMPTY_TABLENAME_RESULT + severity: S_WARNING +} +ids { + code: YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT + severity: S_WARNING +} +ids { + code: YQL_UNUSED_SYMBOL + severity: S_WARNING } ids { code: YQL_MIXED_TZ severity: S_WARNING } -ids { - code: YQL_OPERATION_WILL_RETURN_NULL - severity: S_WARNING -} +ids { + code: YQL_OPERATION_WILL_RETURN_NULL + severity: S_WARNING +} ids { code: YQL_JSON_QUERY_RETURNING_JSON_IS_DEPRECATED severity: S_WARNING @@ -589,14 +589,14 @@ ids { severity: S_WARNING } ids { - code: YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY - severity: S_WARNING -} -ids { - code: YQL_TABLE_BINDING_DUPLICATE - severity: S_WARNING -} -ids { + code: YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY + severity: S_WARNING +} +ids { + code: YQL_TABLE_BINDING_DUPLICATE + severity: S_WARNING +} +ids { code: DQ_GATEWAY_ERROR severity: S_ERROR } diff --git a/ydb/library/yql/core/peephole_opt/ya.make b/ydb/library/yql/core/peephole_opt/ya.make index 8b843a6c79..9afca2a7b9 100644 --- a/ydb/library/yql/core/peephole_opt/ya.make +++ b/ydb/library/yql/core/peephole_opt/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( vvvv - g:yql + g:yql g:yql_ydb_core -) - -SRCS( - yql_opt_peephole_physical.h - yql_opt_peephole_physical.cpp -) - -PEERDIR( +) + +SRCS( + yql_opt_peephole_physical.h + yql_opt_peephole_physical.cpp +) + +PEERDIR( ydb/library/yql/core ydb/library/yql/core/common_opt ydb/library/yql/core/type_ann -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp index d3095afaf5..ce22b12571 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -20,8 +20,8 @@ #include <util/generic/xrange.h> -#include <library/cpp/yson/writer.h> - +#include <library/cpp/yson/writer.h> + namespace NYql { namespace { @@ -34,7 +34,7 @@ using TPeepHoleOptimizerMap = std::unordered_map<std::string_view, TPeepHoleOpti using TNonDeterministicOptimizerPtr = TExprNode::TPtr (*const)(const TExprNode::TPtr&, TExprContext&, TTypeAnnotationContext& types); using TNonDeterministicOptimizerMap = std::unordered_map<std::string_view, TNonDeterministicOptimizerPtr>; -TExprNode::TPtr MakeNothing(TPositionHandle pos, const TTypeAnnotationNode& type, TExprContext& ctx) { +TExprNode::TPtr MakeNothing(TPositionHandle pos, const TTypeAnnotationNode& type, TExprContext& ctx) { return ctx.NewCallable(pos, "Nothing", {ExpandType(pos, *ctx.MakeType<TOptionalExprType>(&type), ctx)}); } @@ -591,10 +591,10 @@ TExprNode::TPtr PeepHoleDictFromKeysToDict(const TExprNode::TPtr& node, TExprCon } TExprNode::TPtr ExpandEquiJoin(const TExprNode::TPtr& input, TExprContext& ctx) { - YQL_ENSURE(input->ChildrenSize() >= 4); + YQL_ENSURE(input->ChildrenSize() >= 4); return ExpandEquiJoinImpl(*input, ctx); -} - +} + template <bool Strong> bool CastMayFail(const TTypeAnnotationNode* source, const TTypeAnnotationNode* target) { return NUdf::ECastOptions::MayFail & CastResult<Strong>(source, target); @@ -1591,262 +1591,262 @@ TExprNode::TPtr ExpandAlterTo(const TExprNode::TPtr& node, TExprContext& ctx) { return ctx.ReplaceNode(node->Child(2)->TailPtr(), node->Child(2)->Head().Head(), std::move(casted)); } -TExprNode::TPtr BuildDictOverListOfStructs(TPositionHandle pos, const TExprNode::TPtr& collection, - const TTypeAnnotationNode*& dictKeyType, TExprContext& ctx) -{ - auto collectionType = collection->GetTypeAnn(); - YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::List); - - auto listItemType = collectionType->Cast<TListExprType>()->GetItemType(); - YQL_ENSURE(listItemType->GetKind() == ETypeAnnotationKind::Struct); - - auto structType = listItemType->Cast<TStructExprType>(); - YQL_ENSURE(structType->GetSize() == 1); - auto memberName = structType->GetItems()[0]->GetName(); - dictKeyType = structType->GetItems()[0]->GetItemType(); - - return ctx.Builder(pos) - .Callable("ToDict") - .Add(0, collection) - .Lambda(1) // keyExtractor - .Param("item") - .Callable("Member") - .Arg(0, "item") - .Atom(1, memberName) - .Seal() - .Seal() - .Lambda(2) // payloadExtractor - .Param("item") - .Callable("Void") - .Seal() - .Seal() - .List(3) +TExprNode::TPtr BuildDictOverListOfStructs(TPositionHandle pos, const TExprNode::TPtr& collection, + const TTypeAnnotationNode*& dictKeyType, TExprContext& ctx) +{ + auto collectionType = collection->GetTypeAnn(); + YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::List); + + auto listItemType = collectionType->Cast<TListExprType>()->GetItemType(); + YQL_ENSURE(listItemType->GetKind() == ETypeAnnotationKind::Struct); + + auto structType = listItemType->Cast<TStructExprType>(); + YQL_ENSURE(structType->GetSize() == 1); + auto memberName = structType->GetItems()[0]->GetName(); + dictKeyType = structType->GetItems()[0]->GetItemType(); + + return ctx.Builder(pos) + .Callable("ToDict") + .Add(0, collection) + .Lambda(1) // keyExtractor + .Param("item") + .Callable("Member") + .Arg(0, "item") + .Atom(1, memberName) + .Seal() + .Seal() + .Lambda(2) // payloadExtractor + .Param("item") + .Callable("Void") + .Seal() + .Seal() + .List(3) .Atom(0, "Hashed", TNodeFlags::Default) .Atom(1, "One", TNodeFlags::Default) .Atom(2, "Compact", TNodeFlags::Default) - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildDictOverList(TPositionHandle pos, const TExprNode::TPtr& collection, TExprContext& ctx) { - auto collectionType = collection->GetTypeAnn(); - YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::List); - - return ctx.Builder(pos) - .Callable("ToDict") - .Add(0, collection) - .Lambda(1) // keyExtractor - .Param("item") - .Arg("item") - .Seal() - .Lambda(2) // payloadExtractor - .Param("item") - .Callable("Void") - .Seal() - .Seal() - .List(3) + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildDictOverList(TPositionHandle pos, const TExprNode::TPtr& collection, TExprContext& ctx) { + auto collectionType = collection->GetTypeAnn(); + YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::List); + + return ctx.Builder(pos) + .Callable("ToDict") + .Add(0, collection) + .Lambda(1) // keyExtractor + .Param("item") + .Arg("item") + .Seal() + .Lambda(2) // payloadExtractor + .Param("item") + .Callable("Void") + .Seal() + .Seal() + .List(3) .Atom(0, "Hashed", TNodeFlags::Default) .Atom(1, "One", TNodeFlags::Default) .Atom(2, "Compact", TNodeFlags::Default) - .Seal() - .Seal() - .Build(); -} - + .Seal() + .Seal() + .Build(); +} + TExprNode::TPtr BuildDictOverTuple(TExprNode::TPtr&& collection, const TTypeAnnotationNode*& dictKeyType, TExprContext& ctx) -{ +{ if (!collection->GetTypeAnn()->Cast<TTupleExprType>()->GetSize()) { dictKeyType = nullptr; - return nullptr; - } - + return nullptr; + } + dictKeyType = CommonTypeForChildren(*collection, ctx); YQL_ENSURE(dictKeyType, "Uncompatible colllection elements."); const auto pos = collection->Pos(); return ctx.NewCallable(pos, "DictFromKeys", {ExpandType(pos, *dictKeyType, ctx), std::move(collection)}); -} - -TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) { - auto collection = input->HeadPtr(); - auto lookup = input->ChildPtr(1); - auto options = input->ChildPtr(2); - - const bool ansiIn = HasSetting(*options, "ansi"); - - auto collectionType = collection->GetTypeAnn(); - TExprNode::TPtr dict; - const TTypeAnnotationNode* dictKeyType = nullptr; - if (collectionType->GetKind() == ETypeAnnotationKind::List) - { - if (collectionType->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct) { +} + +TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) { + auto collection = input->HeadPtr(); + auto lookup = input->ChildPtr(1); + auto options = input->ChildPtr(2); + + const bool ansiIn = HasSetting(*options, "ansi"); + + auto collectionType = collection->GetTypeAnn(); + TExprNode::TPtr dict; + const TTypeAnnotationNode* dictKeyType = nullptr; + if (collectionType->GetKind() == ETypeAnnotationKind::List) + { + if (collectionType->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct) { YQL_CLOG(DEBUG, CorePeepHole) << "IN List of Structs"; - dict = BuildDictOverListOfStructs(input->Pos(), collection, dictKeyType, ctx); - } else { + dict = BuildDictOverListOfStructs(input->Pos(), collection, dictKeyType, ctx); + } else { YQL_CLOG(DEBUG, CorePeepHole) << "IN List"; - dict = BuildDictOverList(input->Pos(), collection, ctx); - dictKeyType = collectionType->Cast<TListExprType>()->GetItemType(); - } - } else if (collectionType->GetKind() == ETypeAnnotationKind::Tuple) { - if (ansiIn && collectionType->Cast<TTupleExprType>()->GetSize()) { - return ctx.Builder(input->Pos()) - .Callable("SqlIn") + dict = BuildDictOverList(input->Pos(), collection, ctx); + dictKeyType = collectionType->Cast<TListExprType>()->GetItemType(); + } + } else if (collectionType->GetKind() == ETypeAnnotationKind::Tuple) { + if (ansiIn && collectionType->Cast<TTupleExprType>()->GetSize()) { + return ctx.Builder(input->Pos()) + .Callable("SqlIn") .Callable(0, "AsListStrict") .Add(collection->ChildrenList()) .Seal() .Add(1, std::move(lookup)) .Add(2, std::move(options)) - .Seal() - .Build(); - } + .Seal() + .Build(); + } YQL_CLOG(DEBUG, CorePeepHole) << "IN Tuple"; dict = BuildDictOverTuple(std::move(collection), dictKeyType, ctx); } else if (collectionType->GetKind() == ETypeAnnotationKind::EmptyDict) { YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyDict"; } else if (collectionType->GetKind() == ETypeAnnotationKind::EmptyList) { YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyList"; - } else { - YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::Dict); + } else { + YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::Dict); YQL_CLOG(DEBUG, CorePeepHole) << "IN Dict"; - dict = collection; - dictKeyType = collectionType->Cast<TDictExprType>()->GetKeyType(); - } - + dict = collection; + dictKeyType = collectionType->Cast<TDictExprType>()->GetKeyType(); + } + const auto lookupType = lookup->GetTypeAnn(); const auto falseNode = MakeBool<false>(input->Pos(), ctx); - const auto justFalseNode = ctx.NewCallable(input->Pos(), "Just", { falseNode }); - - if (ansiIn && !dict) { + const auto justFalseNode = ctx.NewCallable(input->Pos(), "Just", { falseNode }); + + if (ansiIn && !dict) { YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with statically deduced empty collection"; - return lookupType->HasOptionalOrNull() ? justFalseNode: falseNode; - } - - TExprNode::TPtr contains = falseNode; - if (!dictKeyType) { + return lookupType->HasOptionalOrNull() ? justFalseNode: falseNode; + } + + TExprNode::TPtr contains = falseNode; + if (!dictKeyType) { YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to statically deduced empty collection"; } else if (NUdf::ECastOptions::Impossible & CastResult<true>(lookupType, dictKeyType)) { YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to uncompatible type of lookup (" << *lookupType - << ") and collection item (" << *dictKeyType << ")"; - } else { - YQL_ENSURE(dict); - contains = ctx.Builder(input->Pos()) + << ") and collection item (" << *dictKeyType << ")"; + } else { + YQL_ENSURE(dict); + contains = ctx.Builder(input->Pos()) .Callable("Contains") - .Add(0, dict) - .Add(1, lookup) - .Seal() - .Build(); - } - - const bool nullableCollectionItems = IsSqlInCollectionItemsNullable(NNodes::TCoSqlIn(input)); - if (ansiIn && (nullableCollectionItems || lookupType->HasOptionalOrNull())) { + .Add(0, dict) + .Add(1, lookup) + .Seal() + .Build(); + } + + const bool nullableCollectionItems = IsSqlInCollectionItemsNullable(NNodes::TCoSqlIn(input)); + if (ansiIn && (nullableCollectionItems || lookupType->HasOptionalOrNull())) { YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with nullable items in collection or lookup"; - YQL_ENSURE(dict); - const auto trueNode = MakeBool(input->Pos(), true, ctx); - const auto nullNode = MakeNull(input->Pos(), ctx); - - const auto nakedLookupType = RemoveAllOptionals(lookupType); - if (nakedLookupType->GetKind() == ETypeAnnotationKind::Data || - nakedLookupType->GetKind() == ETypeAnnotationKind::Null) - { - return ctx.Builder(input->Pos()) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, lookup) - .Seal() - .Callable(1, "If") - .Callable(0, "Not") - .Callable(0, "HasItems") - .Add(0, dict) - .Seal() - .Seal() - .Add(1, falseNode) - .Add(2, nullNode) - .Seal() - .Callable(2, "If") - .Add(0, contains) - .Add(1, trueNode) - .Callable(2, "If") - .Callable(0, "HasNull") - .Callable(0, "DictKeys") - .Add(0, dict) - .Seal() - .Seal() - .Add(1, nullNode) - .Add(2, falseNode) - .Seal() - .Seal() - .Seal() - .Build(); - } - - // a IN (b1, b2, ...) -> false OR (a == b1 OR a == b2 OR ...) - auto inViaEqualChain = ctx.Builder(input->Pos()) - .Callable("Fold") - .Callable(0, "TakeWhileInclusive") - .Callable(0, "DictKeys") - .Add(0, dict) - .Seal() - .Lambda(1) - .Param("collectionItem") - .Callable("Coalesce") - .Callable(0, "!=") - .Arg(0, "collectionItem") - .Add(1, lookup) - .Seal() - .Add(1, trueNode) - .Seal() - .Seal() - .Seal() - .Add(1, justFalseNode) - .Lambda(2) - .Param("collectionItem") - .Param("result") - .Callable("Or") - .Arg(0, "result") - .Callable(1, "==") - .Arg(0, "collectionItem") - .Add(1, lookup) - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - - return ctx.Builder(input->Pos()) - .Callable("If") - .Callable(0, "Or") - .Callable(0, "HasNull") - .Add(0, lookup) - .Seal() - .Callable(1, "HasNull") - .Callable(0, "DictKeys") - .Add(0, dict) - .Seal() - .Seal() - .Seal() - .Add(1, inViaEqualChain) - .Add(2, contains) - .Seal() - .Build(); - } - - auto result = contains; - if (lookupType->HasOptionalOrNull()) { - result = ctx.Builder(input->Pos()) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, lookup) - .Seal() - .Callable(1, "Null") - .Seal() - .Add(2, result) - .Seal() - .Build(); - } - - return result; -} - + YQL_ENSURE(dict); + const auto trueNode = MakeBool(input->Pos(), true, ctx); + const auto nullNode = MakeNull(input->Pos(), ctx); + + const auto nakedLookupType = RemoveAllOptionals(lookupType); + if (nakedLookupType->GetKind() == ETypeAnnotationKind::Data || + nakedLookupType->GetKind() == ETypeAnnotationKind::Null) + { + return ctx.Builder(input->Pos()) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, lookup) + .Seal() + .Callable(1, "If") + .Callable(0, "Not") + .Callable(0, "HasItems") + .Add(0, dict) + .Seal() + .Seal() + .Add(1, falseNode) + .Add(2, nullNode) + .Seal() + .Callable(2, "If") + .Add(0, contains) + .Add(1, trueNode) + .Callable(2, "If") + .Callable(0, "HasNull") + .Callable(0, "DictKeys") + .Add(0, dict) + .Seal() + .Seal() + .Add(1, nullNode) + .Add(2, falseNode) + .Seal() + .Seal() + .Seal() + .Build(); + } + + // a IN (b1, b2, ...) -> false OR (a == b1 OR a == b2 OR ...) + auto inViaEqualChain = ctx.Builder(input->Pos()) + .Callable("Fold") + .Callable(0, "TakeWhileInclusive") + .Callable(0, "DictKeys") + .Add(0, dict) + .Seal() + .Lambda(1) + .Param("collectionItem") + .Callable("Coalesce") + .Callable(0, "!=") + .Arg(0, "collectionItem") + .Add(1, lookup) + .Seal() + .Add(1, trueNode) + .Seal() + .Seal() + .Seal() + .Add(1, justFalseNode) + .Lambda(2) + .Param("collectionItem") + .Param("result") + .Callable("Or") + .Arg(0, "result") + .Callable(1, "==") + .Arg(0, "collectionItem") + .Add(1, lookup) + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + + return ctx.Builder(input->Pos()) + .Callable("If") + .Callable(0, "Or") + .Callable(0, "HasNull") + .Add(0, lookup) + .Seal() + .Callable(1, "HasNull") + .Callable(0, "DictKeys") + .Add(0, dict) + .Seal() + .Seal() + .Seal() + .Add(1, inViaEqualChain) + .Add(2, contains) + .Seal() + .Build(); + } + + auto result = contains; + if (lookupType->HasOptionalOrNull()) { + result = ctx.Builder(input->Pos()) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, lookup) + .Seal() + .Callable(1, "Null") + .Seal() + .Add(2, result) + .Seal() + .Build(); + } + + return result; +} + template <ui8 FirstLambdaIndex = 1u, ui8 LastLambdaIndex = FirstLambdaIndex> TExprNode::TPtr CleckClosureOnUpperLambdaOverList(const TExprNode::TPtr& input, TExprContext& ctx) { if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) { @@ -1912,20 +1912,20 @@ TExprNode::TPtr ExpandFilter(const TExprNode::TPtr& input, TExprContext& ctx) { return input; } -IGraphTransformer::TStatus PeepHoleCommonStage(const TExprNode::TPtr& input, TExprNode::TPtr& output, +IGraphTransformer::TStatus PeepHoleCommonStage(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, const TPeepHoleOptimizerMap& optimizers) -{ - TOptimizeExprSettings settings(&types); - settings.CustomInstantTypeTransformer = types.CustomInstantTypeTransformer.Get(); - +{ + TOptimizeExprSettings settings(&types); + settings.CustomInstantTypeTransformer = types.CustomInstantTypeTransformer.Get(); + return OptimizeExpr(input, output, [&optimizers](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { if (const auto rule = optimizers.find(node->Content()); optimizers.cend() != rule) return (rule->second)(node, ctx); return node; - }, ctx, settings); -} - -IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExprNode::TPtr& output, + }, ctx, settings); +} + +IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, bool* hasNonDeterministicFunctions, const TPeepHoleOptimizerMap& optimizers, const TNonDeterministicOptimizerMap& nonDetOptimizers) { @@ -1947,19 +1947,19 @@ IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExp } void AddStandardTransformers(TTransformationPipeline& pipelene, IGraphTransformer* typeAnnotator) { - auto issueCode = TIssuesIds::CORE_EXEC; + auto issueCode = TIssuesIds::CORE_EXEC; pipelene.AddServiceTransformers(issueCode); if (typeAnnotator) { pipelene.Add(*typeAnnotator, "TypeAnnotation", issueCode); } else { pipelene.AddTypeAnnotationTransformer(issueCode); } - + pipelene.AddPostTypeAnnotation(true, issueCode); pipelene.Add(TExprLogTransformer::Sync("PeepHoleOpt", NLog::EComponent::CorePeepHole, NLog::ELevel::TRACE), "PeepHoleOptTrace", issueCode, "PeepHoleOptTrace"); } - + template<bool FlatOrMulti> TExprNode::TPtr FuseNarrowMap(const TExprNode& node, TExprContext& ctx) { YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node.Content() << " with " << node.Head().Content(); @@ -2499,9 +2499,9 @@ TExprNode::TPtr ExpandAggrMinMax(const TExprNode::TPtr& node, TExprContext& ctx) template <bool Ordered, bool EnableNewOptimizers> TExprNode::TPtr OptimizeMap(const TExprNode::TPtr& node, TExprContext& ctx) { - const auto& arg = node->Tail().Head().Head(); + const auto& arg = node->Tail().Head().Head(); if constexpr (EnableNewOptimizers) { - if (!arg.IsUsedInDependsOn() && ETypeAnnotationKind::Optional == node->GetTypeAnn()->GetKind()) { + if (!arg.IsUsedInDependsOn() && ETypeAnnotationKind::Optional == node->GetTypeAnn()->GetKind()) { YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " over Optional"; return ctx.Builder(node->Pos()) .Callable("IfPresent") @@ -2519,10 +2519,10 @@ TExprNode::TPtr OptimizeMap(const TExprNode::TPtr& node, TExprContext& ctx) { .Build(); } - if (const auto& input = node->Head(); !arg.IsUsedInDependsOn() && input.IsCallable("AsList")) { + if (const auto& input = node->Head(); !arg.IsUsedInDependsOn() && input.IsCallable("AsList")) { TNodeSet uniqueItems(input.ChildrenSize()); input.ForEachChild([&uniqueItems](const TExprNode& item){ uniqueItems.emplace(&item); }); - if (uniqueItems.size() < 0x10U) { + if (uniqueItems.size() < 0x10U) { YQL_CLOG(DEBUG, CorePeepHole) << "Eliminate " << node->Content() << " over list of " << uniqueItems.size(); auto list = input.ChildrenList(); for (auto& item : list) { @@ -2552,7 +2552,7 @@ TExprNode::TPtr OptimizeMap(const TExprNode::TPtr& node, TExprContext& ctx) { } } - if (1U == node->Head().UseCount() && !arg.IsUsedInDependsOn()) { + if (1U == node->Head().UseCount() && !arg.IsUsedInDependsOn()) { if (node->Head().IsCallable({"Map", "OrderedMap"})) { YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node->Content() << " over " << node->Head().Content(); auto lambda = ctx.Builder(node->Pos()) @@ -4433,22 +4433,22 @@ TExprNode::TPtr ExpandSkipNullFields(const TExprNode::TPtr& node, TExprContext& return node; } -TExprNode::TPtr ExpandConstraintsOf(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content(); - +TExprNode::TPtr ExpandConstraintsOf(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content(); + TString json; TStringOutput out(json); NJson::TJsonWriter jsonWriter(&out, true); node->Head().GetConstraintSet().ToJson(jsonWriter); jsonWriter.Flush(); - - return ctx.Builder(node->Pos()) + + return ctx.Builder(node->Pos()) .Callable("Json") .Atom(0, json, TNodeFlags::MultilineContent) - .Seal() - .Build(); -} - + .Seal() + .Build(); +} + TExprNode::TPtr OptimizeMapJoinCore(const TExprNode::TPtr& node, TExprContext& ctx) { if (const auto& input = node->Head(); input.IsCallable("NarrowMap") && input.Tail().Tail().IsCallable("AsStruct")) { YQL_CLOG(DEBUG, CorePeepHole) << "Swap " << node->Content() << " with " << input.Content(); @@ -4509,19 +4509,19 @@ TExprNode::TPtr OptimizeCommonJoinCore(const TExprNode::TPtr& node, TExprContext return node; } -TExprNode::TPtr DoBuildTablePath(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_ENSURE(node->Head().IsCallable("String"), - "BuildTablePath: expecting string literal as first argument, got: " << node->Head().Content()); - YQL_ENSURE(node->Tail().IsCallable("String"), - "BuildTablePath: expecting string literal as second argument, got: " << node->Tail().Content()); - - return ctx.Builder(node->Pos()) - .Callable("String") - .Atom(0, BuildTablePath(node->Head().Head().Content(), node->Tail().Head().Content())) - .Seal() - .Build(); -} - +TExprNode::TPtr DoBuildTablePath(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_ENSURE(node->Head().IsCallable("String"), + "BuildTablePath: expecting string literal as first argument, got: " << node->Head().Content()); + YQL_ENSURE(node->Tail().IsCallable("String"), + "BuildTablePath: expecting string literal as second argument, got: " << node->Tail().Content()); + + return ctx.Builder(node->Pos()) + .Callable("String") + .Atom(0, BuildTablePath(node->Head().Head().Content(), node->Tail().Head().Content())) + .Seal() + .Build(); +} + template <bool Equality> TExprNode::TPtr ReduceBothArgs(const TExprNode& node, TExprContext& ctx) { YQL_CLOG(DEBUG, CorePeepHole) << "Expand '" << node.Content() << "' with both Optionals."; @@ -4696,36 +4696,36 @@ TExprNode::TPtr SqlEqualTuples(const TExprNode& node, TExprContext& ctx) { const auto lSize = node.Head().GetTypeAnn()->Cast<TTupleExprType>()->GetSize(); const auto rSize = node.Tail().GetTypeAnn()->Cast<TTupleExprType>()->GetSize(); - const auto size = std::max(lSize, rSize); + const auto size = std::max(lSize, rSize); TExprNode::TListType compares; - compares.reserve(size); + compares.reserve(size); - TExprNode::TPtr nullNode = MakeNull(node.Pos(), ctx); + TExprNode::TPtr nullNode = MakeNull(node.Pos(), ctx); for (ui32 i = 0U; i < size; ++i) { - TExprNode::TPtr left = (i >= lSize) ? nullNode : ctx.Builder(node.Pos()) - .Callable("Nth") - .Add(0, node.HeadPtr()) - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Build(); - - TExprNode::TPtr right = (i >= rSize) ? nullNode : ctx.Builder(node.Pos()) - .Callable("Nth") - .Add(0, node.TailPtr()) - .Atom(1, ToString(i), TNodeFlags::Default) - .Seal() - .Build(); - + TExprNode::TPtr left = (i >= lSize) ? nullNode : ctx.Builder(node.Pos()) + .Callable("Nth") + .Add(0, node.HeadPtr()) + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Build(); + + TExprNode::TPtr right = (i >= rSize) ? nullNode : ctx.Builder(node.Pos()) + .Callable("Nth") + .Add(0, node.TailPtr()) + .Atom(1, ToString(i), TNodeFlags::Default) + .Seal() + .Build(); + compares.emplace_back(ctx.Builder(node.Pos()) .Callable(node.Content()) - .Add(0, left) - .Add(1, right) + .Add(0, left) + .Add(1, right) .Seal() .Build()); } - if (compares.empty()) { + if (compares.empty()) { return MakeBool<Equals>(node.Pos(), ctx); } @@ -4890,43 +4890,43 @@ TExprNode::TPtr SqlEqualStructs(const TExprNode& node, TExprContext& ctx) { TExprNode::TListType compares; compares.reserve(std::max(lType->GetSize(), rType->GetSize())); - TExprNode::TPtr nullNode = MakeNull(node.Pos(), ctx); + TExprNode::TPtr nullNode = MakeNull(node.Pos(), ctx); for (const auto& item : lType->GetItems()) { - const auto& name = item->GetName(); - TExprNode::TPtr right = !rType->FindItem(name) ? nullNode : ctx.Builder(node.Pos()) - .Callable("Member") - .Add(0, node.TailPtr()) - .Atom(1, name) - .Seal() - .Build(); - - compares.emplace_back(ctx.Builder(node.Pos()) - .Callable(node.Content()) - .Callable(0, "Member") - .Add(0, node.HeadPtr()) - .Atom(1, name) - .Seal() - .Add(1, right) - .Seal() - .Build()); - } - - for (const auto& item : rType->GetItems()) { - const auto& name = item->GetName(); - if (!lType->FindItem(name)) { + const auto& name = item->GetName(); + TExprNode::TPtr right = !rType->FindItem(name) ? nullNode : ctx.Builder(node.Pos()) + .Callable("Member") + .Add(0, node.TailPtr()) + .Atom(1, name) + .Seal() + .Build(); + + compares.emplace_back(ctx.Builder(node.Pos()) + .Callable(node.Content()) + .Callable(0, "Member") + .Add(0, node.HeadPtr()) + .Atom(1, name) + .Seal() + .Add(1, right) + .Seal() + .Build()); + } + + for (const auto& item : rType->GetItems()) { + const auto& name = item->GetName(); + if (!lType->FindItem(name)) { compares.emplace_back(ctx.Builder(node.Pos()) .Callable(node.Content()) - .Add(0, nullNode) + .Add(0, nullNode) .Callable(1, "Member") .Add(0, node.TailPtr()) .Atom(1, name) .Seal() .Seal() - .Build()); + .Build()); } } - if (compares.empty()) { + if (compares.empty()) { return MakeBool<Equals>(node.Pos(), ctx); } @@ -5354,7 +5354,7 @@ TExprNode::TPtr SqlCompareVariants(const TExprNode& node, TExprContext& ctx) { const auto& lItems = lTuple->GetItems(); const auto& rItems = rTuple->GetItems(); for (ui32 i = 0U; i < tupleType->GetSize(); ++i) { - variants.emplace_back(ctx.NewAtom(node.Pos(), ToString(i), TNodeFlags::Default), + variants.emplace_back(ctx.NewAtom(node.Pos(), ToString(i), TNodeFlags::Default), !IsDistinct && ECompareOptions::Optional == CanCompare<true>(lItems[i], rItems[i])); } @@ -5696,7 +5696,7 @@ struct TPeepHoleRules { {"SafeCast", &ExpandCast<false>}, {"StrictCast", &ExpandCast<true>}, {"AlterTo", &ExpandAlterTo}, - {"SqlIn", &ExpandSqlIn}, + {"SqlIn", &ExpandSqlIn}, {"Lookup", &RewriteSearchByKeyForTypesMismatch<false>}, {"Contains", &RewriteSearchByKeyForTypesMismatch<true>}, {"ListHas", &ExpandListHas}, @@ -5714,8 +5714,8 @@ struct TPeepHoleRules { {"Fold1Map", &CleckClosureOnUpperLambdaOverList<1U, 2U>}, {"Chain1Map", &CleckClosureOnUpperLambdaOverList<1U, 2U>}, {"CalcOverWindow", &ExpandCalcOverWindow}, - {"CalcOverSessionWindow", &ExpandCalcOverWindow}, - {"CalcOverWindowGroup", &ExpandCalcOverWindow}, + {"CalcOverSessionWindow", &ExpandCalcOverWindow}, + {"CalcOverWindowGroup", &ExpandCalcOverWindow}, {"PartitionsByKeys", &ExpandPartitionsByKeys}, {"DictItems", &MapForOptionalContainer}, {"DictKeys", &MapForOptionalContainer}, @@ -5737,7 +5737,7 @@ struct TPeepHoleRules { {"CombineByKey", &ExpandCombineByKey}, {"SkipNullMembers", &ExpandSkipNullFields<EnableNewOptimizers, false>}, {"SkipNullElements", &ExpandSkipNullFields<EnableNewOptimizers, true>}, - {"ConstraintsOf", &ExpandConstraintsOf}, + {"ConstraintsOf", &ExpandConstraintsOf}, {"==", &ExpandSqlEqual<true, false>}, {"!=", &ExpandSqlEqual<false, false>}, {"IsNotDistinctFrom", &ExpandSqlEqual<true, true>}, @@ -5752,9 +5752,9 @@ struct TPeepHoleRules { {"AggrGreater", &ExpandAggrCompare<false, false>}, {"AggrLessOrEqual", &ExpandAggrCompare<true, true>}, {"AggrGreaterOrEqual", &ExpandAggrCompare<false, true>}, - {"RangeEmpty", &ExpandRangeEmpty}, - {"AsRange", &ExpandAsRange}, - {"RangeFor", &ExpandRangeFor}, + {"RangeEmpty", &ExpandRangeEmpty}, + {"AsRange", &ExpandAsRange}, + {"RangeFor", &ExpandRangeFor}, {"PgCall", &ExpandPgCall}, }; @@ -5788,7 +5788,7 @@ struct TPeepHoleRules { {"FlattenMembers", &ExpandFlattenMembers}, {"FlattenStructs", &ExpandFlattenStructs}, {"FlattenByColumns", &ExpandFlattenByColumns}, - {"CastStruct", &ExpandCastStruct}, + {"CastStruct", &ExpandCastStruct}, {"Filter", &ExpandFilter}, {"OrderedFilter", &ExpandFilter}, {"TakeWhile", &ExpandFilter<false>}, @@ -5811,7 +5811,7 @@ struct TPeepHoleRules { {"NarrowFlatMap", &OptimizeNarrowFlatMap}, {"NarrowMultiMap", &OptimizeWideMaps}, {"MapJoinCore", &OptimizeMapJoinCore}, - {"CommonJoinCore", &OptimizeCommonJoinCore}, + {"CommonJoinCore", &OptimizeCommonJoinCore}, {"BuildTablePath", &DoBuildTablePath}, {"Exists", &OptimizeExists}, {"SqueezeToDict", &OptimizeSqueezeToDict} @@ -5847,55 +5847,55 @@ struct TPeepHoleRules { template <bool EnableNewOptimizers> THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, const TPeepholeSettings& peepholeSettings) -{ +{ TTransformationPipeline pipeline(&types); if (peepholeSettings.CommonConfig) { peepholeSettings.CommonConfig->AfterCreate(&pipeline); } - + AddStandardTransformers(pipeline, typeAnnotator); if (peepholeSettings.CommonConfig) { peepholeSettings.CommonConfig->AfterTypeAnnotation(&pipeline); } - - auto issueCode = TIssuesIds::CORE_EXEC; - + + auto issueCode = TIssuesIds::CORE_EXEC; + pipeline.AddCommonOptimization(issueCode); pipeline.Add(CreateFunctorTransformer( [&types](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { return PeepHoleCommonStage(input, output, ctx, types, TPeepHoleRules<EnableNewOptimizers>::Instance().CommonStageRules); - } - ), - "PeepHoleCommon", + } + ), + "PeepHoleCommon", issueCode); - + if (peepholeSettings.CommonConfig) { peepholeSettings.CommonConfig->AfterOptimize(&pipeline); } return pipeline.BuildWithNoArgChecks(false); -} - +} + template <bool EnableNewOptimizers> THolder<IGraphTransformer> CreatePeepHoleFinalStageTransformer(TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, bool* hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings) -{ +{ TTransformationPipeline pipeline(&types); if (peepholeSettings.FinalConfig) { peepholeSettings.FinalConfig->AfterCreate(&pipeline); } - + AddStandardTransformers(pipeline, typeAnnotator); if (peepholeSettings.FinalConfig) { peepholeSettings.FinalConfig->AfterTypeAnnotation(&pipeline); } - - auto issueCode = TIssuesIds::CORE_EXEC; - + + auto issueCode = TIssuesIds::CORE_EXEC; + pipeline.Add( - CreateFunctorTransformer( + CreateFunctorTransformer( [&types, hasNonDeterministicFunctions, withFinalRules = peepholeSettings.WithFinalStageRules, withNonDeterministicRules = peepholeSettings.WithNonDeterministicRules](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { auto stageRules = TPeepHoleRules<EnableNewOptimizers>::Instance().SimplifyStageRules; @@ -5908,41 +5908,41 @@ THolder<IGraphTransformer> CreatePeepHoleFinalStageTransformer(TTypeAnnotationCo TPeepHoleRules<EnableNewOptimizers>::Instance().FinalStageNonDetRules : TNonDeterministicOptimizerMap{}; return PeepHoleFinalStage(input, output, ctx, types, hasNonDeterministicFunctions, stageRules, nonDetStageRules); - } - ), - "PeepHoleFinal", + } + ), + "PeepHoleFinal", issueCode); - + if (peepholeSettings.FinalConfig) { peepholeSettings.FinalConfig->AfterOptimize(&pipeline); } return pipeline.BuildWithNoArgChecks(false); -} - -} - +} + +} + IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, IGraphTransformer& commonTransformer, IGraphTransformer& finalTransformer) -{ - output = input; - +{ + output = input; + for (bool isFinal = false;;) { const auto status = InstantTransform(isFinal ? finalTransformer : commonTransformer, output, ctx, input->IsLambda()); if (status == IGraphTransformer::TStatus::Error || (status.HasRestart && input->IsLambda())) { - return status; - } - if (status == IGraphTransformer::TStatus::Ok) { - if (isFinal) { - break; - } - isFinal = true; - } - } - - return IGraphTransformer::TStatus::Ok; -} - + return status; + } + if (status == IGraphTransformer::TStatus::Ok) { + if (isFinal) { + break; + } + isFinal = true; + } + } + + return IGraphTransformer::TStatus::Ok; +} + template <bool EnableNewOptimizers> IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, @@ -5964,7 +5964,7 @@ THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr ty [common = std::move(commonTransformer), final = std::move(finalTransformer)](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) -> IGraphTransformer::TStatus { return DoPeepHoleOptimizeNode(input, output, ctx, *common, *final); }); -} +} template IGraphTransformer::TStatus PeepHoleOptimizeNode<true>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h index ce98014dec..a357c5b395 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h @@ -15,7 +15,7 @@ struct TPeepholeSettings { }; template <bool EnableNewOptimizers> -IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, +IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings = {}); diff --git a/ydb/library/yql/core/services/yql_eval_expr.cpp b/ydb/library/yql/core/services/yql_eval_expr.cpp index 3c1700adad..811f22a605 100644 --- a/ydb/library/yql/core/services/yql_eval_expr.cpp +++ b/ydb/library/yql/core/services/yql_eval_expr.cpp @@ -57,7 +57,7 @@ bool CheckPendingArgs(const TExprNode& root, TNodeSet& visited, TNodeMap<const T hasUnresolvedTypes = true; } } else if (!externalWorlds.count(&root)) { - ctx.AddError(TIssue(ctx.GetPosition(root.Pos()), TStringBuilder() << "Failed to evaluate unresolved argument: " << root.Content() << ". Did you use a column?")); + ctx.AddError(TIssue(ctx.GetPosition(root.Pos()), TStringBuilder() << "Failed to evaluate unresolved argument: " << root.Content() << ". Did you use a column?")); return false; } } @@ -376,7 +376,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp [&](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) -> IGraphTransformer::TStatus { output = input; if (!input->GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Lambda is not allowed as argument for function: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Lambda is not allowed as argument for function: " << input->Content())); return IGraphTransformer::TStatus::Error; } @@ -405,8 +405,8 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp return IGraphTransformer::TStatus::Ok; }), "TopLevelType", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "Ensure type of expression is correct"); - const bool forSubGraph = true; - pipeline.AddPostTypeAnnotation(forSubGraph); + const bool forSubGraph = true; + pipeline.AddPostTypeAnnotation(forSubGraph); pipeline.Add(TExprLogTransformer::Sync("EvalExpressionOpt", NLog::EComponent::CoreEval, NLog::ELevel::TRACE), "EvalOptTrace", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "EvalOptTrace"); pipeline.AddOptimization(false); @@ -468,14 +468,14 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp } if (!calcProvider) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Only pure expressions are supported")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Only pure expressions are supported")); return IGraphTransformer::TStatus::Error; } if (!calcWorldRoot) { calcWorldRoot = ctx.NewWorld(input->Pos()); calcWorldRoot->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - calcWorldRoot->SetState(TExprNode::EState::ConstrComplete); + calcWorldRoot->SetState(TExprNode::EState::ConstrComplete); } } @@ -503,7 +503,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp settings.VisitChanges = true; auto status = OptimizeExpr(output, output, [&](const TExprNode::TPtr& node, TExprContext& ctx)->TExprNode::TPtr { TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); }); if (node->IsCallable("EvaluateIf!")) { @@ -530,7 +530,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp } if (!node->Child(1)->IsCallable("Bool") || node->Child(1)->ChildrenSize() != 1) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(1)->Pos()), TStringBuilder() << "Expected literal bool")); + ctx.AddError(TIssue(ctx.GetPosition(node->Child(1)->Pos()), TStringBuilder() << "Expected literal bool")); return nullptr; } @@ -538,7 +538,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp ui8 predValue; if (predAtom->Flags() & TNodeFlags::BinaryContent) { if (predAtom->Content().size() != 1) { - ctx.AddError(TIssue(ctx.GetPosition(predAtom->Pos()), TStringBuilder() << "Incorrect literal bool value")); + ctx.AddError(TIssue(ctx.GetPosition(predAtom->Pos()), TStringBuilder() << "Incorrect literal bool value")); return nullptr; } @@ -616,13 +616,13 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp } if (!list->IsCallable("List") && !list->IsCallable("AsList")) { - ctx.AddError(TIssue(ctx.GetPosition(list->Pos()), TStringBuilder() << "Expected (optional) literal list")); + ctx.AddError(TIssue(ctx.GetPosition(list->Pos()), TStringBuilder() << "Expected (optional) literal list")); return nullptr; } auto itemsCount = list->ChildrenSize() - (list->IsCallable("List") ? 1 : 0); if (itemsCount > types.EvaluateForLimit) { - ctx.AddError(TIssue(ctx.GetPosition(list->Pos()), TStringBuilder() << "Too large list for EVALUATE FOR, allowed: " << + ctx.AddError(TIssue(ctx.GetPosition(list->Pos()), TStringBuilder() << "Too large list for EVALUATE FOR, allowed: " << types.EvaluateForLimit << ", got: " << itemsCount)); return nullptr; } @@ -799,7 +799,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp TExprNode::TListType keys; for (const auto& eachKey : node->Children()) { if (!eachKey->IsCallable("Key")) { - ctx.AddError(TIssue(ctx.GetPosition(eachKey->Pos()), TStringBuilder() << "Expected Key")); + ctx.AddError(TIssue(ctx.GetPosition(eachKey->Pos()), TStringBuilder() << "Expected Key")); return nullptr; } @@ -809,7 +809,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp if (!eachKey->Child(0)->IsList() || eachKey->Child(0)->ChildrenSize() != 2 || !eachKey->Child(0)->Child(0)->IsAtom() || eachKey->Child(0)->Child(0)->Content() != "table") { - ctx.AddError(TIssue(ctx.GetPosition(eachKey->Pos()), TStringBuilder() << "Invalid Key")); + ctx.AddError(TIssue(ctx.GetPosition(eachKey->Pos()), TStringBuilder() << "Invalid Key")); return nullptr; } @@ -819,19 +819,19 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp } if (list->IsCallable("List") && list->ChildrenSize() == 0) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Invalid literal list value")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Invalid literal list value")); return nullptr; } if (!list->IsCallable("List") && !list->IsCallable("AsList")) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected literal list")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected literal list")); return nullptr; } for (ui32 i = list->IsCallable("List") ? 1 : 0; i < list->ChildrenSize(); ++i) { auto name = list->ChildPtr(i); if (!name->IsCallable("String")) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected literal string as table name")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected literal string as table name")); return nullptr; } @@ -1005,7 +1005,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp if (isAtomPipeline) { if (isOptionalAtom) { if (dataNode.IsEntity() || dataNode.AsList().empty()) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to get atom from an empty optional")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to get atom from an empty optional")); return nullptr; } @@ -1027,7 +1027,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp TProgramBuilder pgmBuilder(env, functionRegistry); TType* mkqlType = NCommon::BuildType(*clonedArg->GetTypeAnn(), pgmBuilder, err); if (!mkqlType) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to process type: " << err.Str())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to process type: " << err.Str())); return nullptr; } @@ -1035,13 +1035,13 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp THolderFactory holderFactory(alloc.Ref(), memInfo); auto value = NCommon::ParseYsonNodeInResultFormat(holderFactory, dataNode, mkqlType, &err); if (!value) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to parse data: " << err.Str())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to parse data: " << err.Str())); return nullptr; } if (isTypePipeline) { auto yson = TStringBuf(value->AsStringRef()); - auto type = NCommon::ParseTypeFromYson(yson, ctx, ctx.GetPosition(node->Pos())); + auto type = NCommon::ParseTypeFromYson(yson, ctx, ctx.GetPosition(node->Pos())); if (!type) { return nullptr; } diff --git a/ydb/library/yql/core/services/yql_eval_params.cpp b/ydb/library/yql/core/services/yql_eval_params.cpp index d1a0115c54..dce9df4892 100644 --- a/ydb/library/yql/core/services/yql_eval_params.cpp +++ b/ydb/library/yql/core/services/yql_eval_params.cpp @@ -94,7 +94,7 @@ bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& auto name = param->Child(0)->Content(); if (!param->GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Failed to type check parameter: " << name)); + ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Failed to type check parameter: " << name)); return nullptr; } @@ -102,7 +102,7 @@ bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext& if (!type) { type = param->GetTypeAnn(); } else if (!IsSameAnnotation(*type, *param->GetTypeAnn())) { - ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Mismatch of types: " << *type << " != " << *param->GetTypeAnn() + ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Mismatch of types: " << *type << " != " << *param->GetTypeAnn() << " for parameter: " << name)); return nullptr; } diff --git a/ydb/library/yql/core/services/yql_plan.cpp b/ydb/library/yql/core/services/yql_plan.cpp index 86e1d11576..4d7cfffaf5 100644 --- a/ydb/library/yql/core/services/yql_plan.cpp +++ b/ydb/library/yql/core/services/yql_plan.cpp @@ -146,9 +146,9 @@ ui32 FillLevels(THashMap<ui32, TLevelContext>& basicNodesMap, ui32 current, THas return 0; } - auto findPtr = basicNodesMap.FindPtr(current); - YQL_ENSURE(findPtr); - auto& ctx = *findPtr; + auto findPtr = basicNodesMap.FindPtr(current); + YQL_ENSURE(findPtr); + auto& ctx = *findPtr; if (ctx.Node->Level) { return ctx.Node->Level; } @@ -157,18 +157,18 @@ ui32 FillLevels(THashMap<ui32, TLevelContext>& basicNodesMap, ui32 current, THas ui32 maxLevel = 0; bool hasOutputs = false; for (auto& child : ctx.Inputs) { - auto findPtr = basicNodesMap.FindPtr(child); - YQL_ENSURE(findPtr); - auto& childCtx = *findPtr; + auto findPtr = basicNodesMap.FindPtr(child); + YQL_ENSURE(findPtr); + auto& childCtx = *findPtr; hasOutputs = hasOutputs || !childCtx.Outputs.empty(); maxLevel = Max(maxLevel, FillLevels(basicNodesMap, child, visited)); } ctx.Node->Level = maxLevel + (hasOutputs ? 2 : 1); for (auto& child : ctx.Outputs) { - auto findPtr = basicNodesMap.FindPtr(child); - YQL_ENSURE(findPtr); - auto& childCtx = *findPtr; + auto findPtr = basicNodesMap.FindPtr(child); + YQL_ENSURE(findPtr); + auto& childCtx = *findPtr; childCtx.Node->Level = ctx.Node->Level + 1; } @@ -537,7 +537,7 @@ public: for (auto& link : basicLinks) { basicNodesMap[link.Target].Inputs.push_back(link.Source); auto target = basicNodesMap.FindPtr(link.Target); - YQL_ENSURE(target); + YQL_ENSURE(target); if (target->Node->Type == TBasicNode::EType::Output) { basicNodesMap[link.Source].Outputs.push_back(link.Target); } @@ -570,16 +570,16 @@ public: for (auto& pin : pins) { if (pin.DataSource) { auto providerName = pin.DataSource->Child(0)->Content(); - auto providerPtr = Types_.DataSourceMap.FindPtr(providerName); - YQL_ENSURE(providerPtr); - UpdateProviders(providers, pin.DataSource, providerPtr->Get()); + auto providerPtr = Types_.DataSourceMap.FindPtr(providerName); + YQL_ENSURE(providerPtr); + UpdateProviders(providers, pin.DataSource, providerPtr->Get()); } if (pin.DataSink) { auto providerName = pin.DataSink->Child(0)->Content(); - auto providerPtr = Types_.DataSinkMap.FindPtr(providerName); - YQL_ENSURE(providerPtr); - UpdateProviders(providers, pin.DataSink, providerPtr->Get()); + auto providerPtr = Types_.DataSinkMap.FindPtr(providerName); + YQL_ENSURE(providerPtr); + UpdateProviders(providers, pin.DataSink, providerPtr->Get()); } } } @@ -587,16 +587,16 @@ public: IDataProvider& GetProvider(const TPinInfo& pin, TTypeAnnotationContext& types) { if (pin.DataSource) { auto providerName = pin.DataSource->Child(0)->Content(); - auto providerPtr = types.DataSourceMap.FindPtr(providerName); - YQL_ENSURE(providerPtr && *providerPtr); - return **providerPtr; + auto providerPtr = types.DataSourceMap.FindPtr(providerName); + YQL_ENSURE(providerPtr && *providerPtr); + return **providerPtr; } if (pin.DataSink) { auto providerName = pin.DataSink->Child(0)->Content(); - auto providerPtr = types.DataSinkMap.FindPtr(providerName); - YQL_ENSURE(providerPtr && *providerPtr); - return **providerPtr; + auto providerPtr = types.DataSinkMap.FindPtr(providerName); + YQL_ENSURE(providerPtr && *providerPtr); + return **providerPtr; } YQL_ENSURE(false, "Expected either datasource or sink"); @@ -605,20 +605,20 @@ public: TProviderInfo& FindProvider(TProviderInfoMap& providers, const TPinInfo& pin) const { if (pin.DataSource) { auto providerName = pin.DataSource->Child(0)->Content(); - auto providerPtr = Types_.DataSourceMap.FindPtr(providerName); - YQL_ENSURE(providerPtr && *providerPtr); - auto infoPtr = providers.FindPtr((*providerPtr)->GetPlanFormatter().GetProviderPath(*pin.DataSource)); - YQL_ENSURE(infoPtr); - return *infoPtr; + auto providerPtr = Types_.DataSourceMap.FindPtr(providerName); + YQL_ENSURE(providerPtr && *providerPtr); + auto infoPtr = providers.FindPtr((*providerPtr)->GetPlanFormatter().GetProviderPath(*pin.DataSource)); + YQL_ENSURE(infoPtr); + return *infoPtr; } if (pin.DataSink) { auto providerName = pin.DataSink->Child(0)->Content(); - auto providerPtr = Types_.DataSinkMap.FindPtr(providerName); - YQL_ENSURE(providerPtr && *providerPtr); - auto infoPtr = providers.FindPtr((*providerPtr)->GetPlanFormatter().GetProviderPath(*pin.DataSink)); - YQL_ENSURE(infoPtr); - return *infoPtr; + auto providerPtr = Types_.DataSinkMap.FindPtr(providerName); + YQL_ENSURE(providerPtr && *providerPtr); + auto infoPtr = providers.FindPtr((*providerPtr)->GetPlanFormatter().GetProviderPath(*pin.DataSink)); + YQL_ENSURE(infoPtr); + return *infoPtr; } YQL_ENSURE(false, "Expected either datasource or sink"); diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.cpp b/ydb/library/yql/core/services/yql_transform_pipeline.cpp index 89e2cdf0fd..ae425f34c8 100644 --- a/ydb/library/yql/core/services/yql_transform_pipeline.cpp +++ b/ydb/library/yql/core/services/yql_transform_pipeline.cpp @@ -124,10 +124,10 @@ TTransformationPipeline& TTransformationPipeline::AddPostTypeAnnotation(bool for issueCode)); Transformers_.push_back(TTransformStage( CreateFunctorTransformer( - [forSubGraph, coStore = TypeAnnotationContext_->ColumnOrderStorage](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - return EliminateCommonSubExpressions(input, output, ctx, forSubGraph, *coStore); - } - ), + [forSubGraph, coStore = TypeAnnotationContext_->ColumnOrderStorage](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + return EliminateCommonSubExpressions(input, output, ctx, forSubGraph, *coStore); + } + ), "CSEE", issueCode)); @@ -147,13 +147,13 @@ TTransformationPipeline& TTransformationPipeline::AddCommonOptimization(EYqlIssu } TTransformationPipeline& TTransformationPipeline::AddFinalCommonOptimization(EYqlIssueCode issueCode) { - Transformers_.push_back(TTransformStage( - CreateCommonOptFinalTransformer(TypeAnnotationContext_.Get()), - "FinalCommonOptimization", - issueCode)); - return *this; -} - + Transformers_.push_back(TTransformStage( + CreateCommonOptFinalTransformer(TypeAnnotationContext_.Get()), + "FinalCommonOptimization", + issueCode)); + return *this; +} + TTransformationPipeline& TTransformationPipeline::AddOptimization(bool checkWorld, bool withFinalOptimization, EYqlIssueCode issueCode) { AddCommonOptimization(issueCode); Transformers_.push_back(TTransformStage( diff --git a/ydb/library/yql/core/sql_types/simple_types.cpp b/ydb/library/yql/core/sql_types/simple_types.cpp index 1a4fbf7563..467e9bbdac 100644 --- a/ydb/library/yql/core/sql_types/simple_types.cpp +++ b/ydb/library/yql/core/sql_types/simple_types.cpp @@ -1,82 +1,82 @@ -#include "simple_types.h" - -#include <util/generic/string.h> -#include <util/string/cast.h> - -#include <unordered_map> - -namespace NYql { - -std::optional<std::string_view> LookupSimpleTypeBySqlAlias(const std::string_view& alias, bool flexibleTypesEnabled) { - static const std::unordered_map<std::string_view, std::string_view> simpleTypes = { - {"void", "Void"}, - {"unit", "Unit"}, - {"generic", "Generic"}, - {"emptylist", "EmptyList"}, - {"emptydict", "EmptyDict"}, - - {"bool", "Bool"}, - - {"int8", "Int8"}, - {"int16", "Int16"}, - {"int32", "Int32"}, - {"int64", "Int64"}, - - {"tinyint", "Int8"}, - {"smallint", "Int16"}, - {"int", "Int32"}, - {"integer", "Int32"}, - {"bigint", "Int64"}, - - {"uint8", "Uint8"}, - {"uint16", "Uint16"}, - {"uint32", "Uint32"}, - {"uint64", "Uint64"}, - - {"float", "Float"}, - {"double", "Double"}, - - {"dynumber", "DyNumber"}, - - {"string", "String"}, - {"varchar", "String"}, - {"utf8", "Utf8"}, - - {"uuid", "Uuid"}, - {"yson", "Yson"}, - {"json", "Json"}, - {"jsondocument", "JsonDocument"}, - {"xml", "Xml"}, - {"yaml", "Yaml"}, - - {"date", "Date"}, - {"datetime", "Datetime"}, - {"timestamp", "Timestamp"}, - {"interval", "Interval"}, - - {"tzdate", "TzDate"}, - {"tzdatetime", "TzDatetime"}, - {"tztimestamp", "TzTimestamp"}, - }; - - // new types (or aliases) should be added here - static const std::unordered_map<std::string_view, std::string_view> newSimpleTypes = { - {"text", "Utf8"}, - {"bytes", "String"}, - }; - - auto normalized = to_lower(ToString(alias)); - if (auto it = simpleTypes.find(normalized); it != simpleTypes.end()) { - return it->second; - } - - if (flexibleTypesEnabled) { - if (auto it = newSimpleTypes.find(normalized); it != newSimpleTypes.end()) { - return it->second; - } - } - return {}; -} - - -} +#include "simple_types.h" + +#include <util/generic/string.h> +#include <util/string/cast.h> + +#include <unordered_map> + +namespace NYql { + +std::optional<std::string_view> LookupSimpleTypeBySqlAlias(const std::string_view& alias, bool flexibleTypesEnabled) { + static const std::unordered_map<std::string_view, std::string_view> simpleTypes = { + {"void", "Void"}, + {"unit", "Unit"}, + {"generic", "Generic"}, + {"emptylist", "EmptyList"}, + {"emptydict", "EmptyDict"}, + + {"bool", "Bool"}, + + {"int8", "Int8"}, + {"int16", "Int16"}, + {"int32", "Int32"}, + {"int64", "Int64"}, + + {"tinyint", "Int8"}, + {"smallint", "Int16"}, + {"int", "Int32"}, + {"integer", "Int32"}, + {"bigint", "Int64"}, + + {"uint8", "Uint8"}, + {"uint16", "Uint16"}, + {"uint32", "Uint32"}, + {"uint64", "Uint64"}, + + {"float", "Float"}, + {"double", "Double"}, + + {"dynumber", "DyNumber"}, + + {"string", "String"}, + {"varchar", "String"}, + {"utf8", "Utf8"}, + + {"uuid", "Uuid"}, + {"yson", "Yson"}, + {"json", "Json"}, + {"jsondocument", "JsonDocument"}, + {"xml", "Xml"}, + {"yaml", "Yaml"}, + + {"date", "Date"}, + {"datetime", "Datetime"}, + {"timestamp", "Timestamp"}, + {"interval", "Interval"}, + + {"tzdate", "TzDate"}, + {"tzdatetime", "TzDatetime"}, + {"tztimestamp", "TzTimestamp"}, + }; + + // new types (or aliases) should be added here + static const std::unordered_map<std::string_view, std::string_view> newSimpleTypes = { + {"text", "Utf8"}, + {"bytes", "String"}, + }; + + auto normalized = to_lower(ToString(alias)); + if (auto it = simpleTypes.find(normalized); it != simpleTypes.end()) { + return it->second; + } + + if (flexibleTypesEnabled) { + if (auto it = newSimpleTypes.find(normalized); it != newSimpleTypes.end()) { + return it->second; + } + } + return {}; +} + + +} diff --git a/ydb/library/yql/core/sql_types/simple_types.h b/ydb/library/yql/core/sql_types/simple_types.h index 99c6c67744..dfff83b726 100644 --- a/ydb/library/yql/core/sql_types/simple_types.h +++ b/ydb/library/yql/core/sql_types/simple_types.h @@ -1,12 +1,12 @@ -#pragma once - -#include <optional> -#include <string_view> - -namespace NYql { - -// simple type is a type which is not parameterized by other types (or parameters) -// Void, Unit, Generic, EmptyList, EmptyDict and all Data types (except for Decimal) are simple types -std::optional<std::string_view> LookupSimpleTypeBySqlAlias(const std::string_view& alias, bool flexibleTypesEnabled); - -} +#pragma once + +#include <optional> +#include <string_view> + +namespace NYql { + +// simple type is a type which is not parameterized by other types (or parameters) +// Void, Unit, Generic, EmptyList, EmptyDict and all Data types (except for Decimal) are simple types +std::optional<std::string_view> LookupSimpleTypeBySqlAlias(const std::string_view& alias, bool flexibleTypesEnabled); + +} diff --git a/ydb/library/yql/core/sql_types/ya.make b/ydb/library/yql/core/sql_types/ya.make index 8cbfce1eae..57c191020b 100644 --- a/ydb/library/yql/core/sql_types/ya.make +++ b/ydb/library/yql/core/sql_types/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - simple_types.h - simple_types.cpp -) - -END() +LIBRARY() + +OWNER(g:yql) + +SRCS( + simple_types.h + simple_types.cpp +) + +END() diff --git a/ydb/library/yql/core/type_ann/type_ann_columnorder.cpp b/ydb/library/yql/core/type_ann/type_ann_columnorder.cpp index a80a8dc399..98485de1b7 100644 --- a/ydb/library/yql/core/type_ann/type_ann_columnorder.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_columnorder.cpp @@ -1,44 +1,44 @@ -#include "type_ann_columnorder.h" - +#include "type_ann_columnorder.h" + #include <ydb/library/yql/core/yql_type_helpers.h> #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/core/yql_join.h> - -namespace NYql { -namespace NTypeAnnImpl { - -namespace { -void FilterColumnOrderByType(TVector<TString>& columnOrder, const TTypeAnnotationNode& type) { - TSet<TStringBuf> typeColumns = GetColumnsOfStructOrSequenceOfStruct(type); - EraseIf(columnOrder, [&](const TString& col) { return !typeColumns.contains(col); }); -} - -void DivePrefixes(TVector<TString>& columnOrder, const TVector<TString>& prefixes) { - TVector<TString> outputColumnOrder; - THashSet<TString> outputSet; - for (auto& col : columnOrder) { - for (auto& prefix : prefixes) { - if (col.StartsWith(prefix)) { - TString outputColumn = col.substr(prefix.length()); - if (!outputSet.contains(outputColumn)) { - outputColumnOrder.push_back(outputColumn); - outputSet.insert(outputColumn); - } - break; - } - } - } - std::swap(columnOrder, outputColumnOrder); -} - -void AddPrefix(TVector<TString>& columnOrder, const TString& prefix) { - for (auto& col : columnOrder) { - col = prefix + col; - } -} - -} // namespace - + +namespace NYql { +namespace NTypeAnnImpl { + +namespace { +void FilterColumnOrderByType(TVector<TString>& columnOrder, const TTypeAnnotationNode& type) { + TSet<TStringBuf> typeColumns = GetColumnsOfStructOrSequenceOfStruct(type); + EraseIf(columnOrder, [&](const TString& col) { return !typeColumns.contains(col); }); +} + +void DivePrefixes(TVector<TString>& columnOrder, const TVector<TString>& prefixes) { + TVector<TString> outputColumnOrder; + THashSet<TString> outputSet; + for (auto& col : columnOrder) { + for (auto& prefix : prefixes) { + if (col.StartsWith(prefix)) { + TString outputColumn = col.substr(prefix.length()); + if (!outputSet.contains(outputColumn)) { + outputColumnOrder.push_back(outputColumn); + outputSet.insert(outputColumn); + } + break; + } + } + } + std::swap(columnOrder, outputColumnOrder); +} + +void AddPrefix(TVector<TString>& columnOrder, const TString& prefix) { + for (auto& col : columnOrder) { + col = prefix + col; + } +} + +} // namespace + IGraphTransformer::TStatus OrderForPgSetItem(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); TVector<TString> columnOrder; @@ -72,207 +72,207 @@ IGraphTransformer::TStatus OrderForPgSetItem(const TExprNode::TPtr& node, TExprN return ctx.Types.SetColumnOrder(*node, columnOrder, ctx.Expr); } -IGraphTransformer::TStatus OrderForAssumeColumnOrder(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - TVector<TString> columnOrder; - for (auto& col : node->Tail().ChildrenList()) { - columnOrder.push_back(TString(col->Content())); - } - - return ctx.Types.SetColumnOrder(*node, columnOrder, ctx.Expr); -} - -IGraphTransformer::TStatus OrderForSqlProject(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList) { - return IGraphTransformer::TStatus::Ok; - } - - auto inputOrder = ctx.Types.LookupColumnOrder(node->Head()); - const bool hasStar = AnyOf(node->Child(1)->ChildrenList(), - [](const TExprNode::TPtr& node) { return node->IsCallable("SqlProjectStarItem"); }); - - if (hasStar && !inputOrder) { - return IGraphTransformer::TStatus::Ok; - } - - TVector<TString> resultColumnOrder; - for (const auto& item : node->Child(1)->ChildrenList()) { - TString name(item->Child(1)->Content()); - if (item->IsCallable("SqlProjectItem")) { - resultColumnOrder.push_back(name); - continue; - } - - YQL_ENSURE(inputOrder); - TVector<TString> starOutput = *inputOrder; - - if (item->ChildrenSize() < 4) { - // legacy star without options - column order is not supported - return IGraphTransformer::TStatus::Ok; - } - - if (auto dive = GetSetting(*item->Child(3), "divePrefix")) { - TVector<TString> divePrefixes; - for (auto& prefix : dive->Child(1)->ChildrenList()) { - YQL_ENSURE(prefix->IsAtom()); - divePrefixes.push_back(TString(prefix->Content())); - } - Sort(divePrefixes, [](const auto& left, const auto& right) { return right < left; }); - DivePrefixes(starOutput, divePrefixes); - } else if (auto addPrefix = GetSetting(*item->Child(3), "addPrefix")) { - YQL_ENSURE(addPrefix->Child(1)->IsAtom()); - AddPrefix(starOutput, TString(addPrefix->Child(1)->Content())); - } - - FilterColumnOrderByType(starOutput, *item->GetTypeAnn()); - resultColumnOrder.insert(resultColumnOrder.end(), starOutput.begin(), starOutput.end()); - } - return ctx.Types.SetColumnOrder(*node, resultColumnOrder, ctx.Expr); -} - -IGraphTransformer::TStatus OrderForMergeExtend(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - YQL_ENSURE(node->ChildrenSize()); - auto common = ctx.Types.LookupColumnOrder(node->Head()); - if (!common) { - return IGraphTransformer::TStatus::Ok; - } - - for (ui32 i = 1; i < node->ChildrenSize(); i++) { - auto current = ctx.Types.LookupColumnOrder(*node->Child(i)); - if (!current || current != common) { - return IGraphTransformer::TStatus::Ok; - } - } - - return ctx.Types.SetColumnOrder(*node, *common, ctx.Expr); -} - -IGraphTransformer::TStatus OrderForUnionAll(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - YQL_ENSURE(node->ChildrenSize()); - auto common = ctx.Types.LookupColumnOrder(node->Head()); - if (!common) { - return IGraphTransformer::TStatus::Ok; - } - - for (ui32 i = 1; i < node->ChildrenSize(); i++) { - auto input = node->Child(i); - auto current = ctx.Types.LookupColumnOrder(*input); - if (!current) { - return IGraphTransformer::TStatus::Ok; - } - - bool truncated = false; - for (size_t i = 0; i < Min(common->size(), current->size()); ++i) { - if ((*current)[i] != (*common)[i]) { - common->resize(i); - truncated = true; - break; - } - } - if (!truncated && current->size() > common->size()) { - common = current; - } - } - - if (common->size() > 0) { - auto allColumns = GetColumnsOfStructOrSequenceOfStruct(*node->GetTypeAnn()); - for (auto& col : *common) { - auto it = allColumns.find(col); - YQL_ENSURE(it != allColumns.end()); - allColumns.erase(it); - } - - for (auto& remain : allColumns) { - common->push_back(TString(remain)); - } - return ctx.Types.SetColumnOrder(*node, *common, ctx.Expr); - } - - return IGraphTransformer::TStatus::Ok; -} - -IGraphTransformer::TStatus OrderForEquiJoin(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - const size_t numLists = node->ChildrenSize() - 2; - const auto joinTree = node->Child(numLists); - const auto optionsNode = node->Child(numLists + 1);; - TVector<TMaybe<TColumnOrder>> inputColumnOrder; - TJoinLabels labels; - for (size_t i = 0; i < numLists; ++i) { - auto& list = node->Child(i)->Head(); - inputColumnOrder.push_back(ctx.Types.LookupColumnOrder(list)); - if (auto err = labels.Add(ctx.Expr, *node->Child(i)->Child(1), - list.GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>())) - { - ctx.Expr.AddError(*err); - return IGraphTransformer::TStatus::Error; - } - } - - TJoinOptions options; - auto status = ValidateEquiJoinOptions(node->Pos(), *optionsNode, options, ctx.Expr); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - if (options.Flatten) { - // will be done after ExpandFlattenEquiJoin - return IGraphTransformer::TStatus::Ok; - } - - auto columnTypes = GetJoinColumnTypes(*joinTree, labels, ctx.Expr); - YQL_ENSURE(labels.Inputs.size() == inputColumnOrder.size()); - size_t idx = 0; - TVector<TString> resultColumnOrder; - for (const auto& label : labels.Inputs) { - auto columnOrder = inputColumnOrder[idx++]; - if (!columnOrder) { - bool survivesJoin = AnyOf(label.EnumerateAllColumns(), [&](const TString& outputColumn) { - return columnTypes.contains(outputColumn); - }); - if (survivesJoin) { - // no column order on output - return IGraphTransformer::TStatus::Ok; - } - continue; - } - - for (auto col : *columnOrder) { - TString fullName = label.FullName(col); - if (columnTypes.contains(fullName)) { - auto it = options.RenameMap.find(fullName); - if (it != options.RenameMap.end()) { - for (auto target : it->second) { - resultColumnOrder.push_back(TString(target)); - } - } else { - resultColumnOrder.push_back(fullName); - } - } - } - } - - return ctx.Types.SetColumnOrder(*node, resultColumnOrder, ctx.Expr); -}; - -IGraphTransformer::TStatus OrderFromFirst(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - if (auto columnOrder = ctx.Types.LookupColumnOrder(node->Head())) { - return ctx.Types.SetColumnOrder(*node, *columnOrder, ctx.Expr); - } - return IGraphTransformer::TStatus::Ok; -}; - -IGraphTransformer::TStatus OrderFromFirstAndOutputType(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - if (auto columnOrder = ctx.Types.LookupColumnOrder(node->Head())) { - FilterColumnOrderByType(*columnOrder, *node->GetTypeAnn()); - return ctx.Types.SetColumnOrder(*node, *columnOrder, ctx.Expr); - } - return IGraphTransformer::TStatus::Ok; -} - -} // namespace NTypeAnnImpl -} // namespace NYql +IGraphTransformer::TStatus OrderForAssumeColumnOrder(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + TVector<TString> columnOrder; + for (auto& col : node->Tail().ChildrenList()) { + columnOrder.push_back(TString(col->Content())); + } + + return ctx.Types.SetColumnOrder(*node, columnOrder, ctx.Expr); +} + +IGraphTransformer::TStatus OrderForSqlProject(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList) { + return IGraphTransformer::TStatus::Ok; + } + + auto inputOrder = ctx.Types.LookupColumnOrder(node->Head()); + const bool hasStar = AnyOf(node->Child(1)->ChildrenList(), + [](const TExprNode::TPtr& node) { return node->IsCallable("SqlProjectStarItem"); }); + + if (hasStar && !inputOrder) { + return IGraphTransformer::TStatus::Ok; + } + + TVector<TString> resultColumnOrder; + for (const auto& item : node->Child(1)->ChildrenList()) { + TString name(item->Child(1)->Content()); + if (item->IsCallable("SqlProjectItem")) { + resultColumnOrder.push_back(name); + continue; + } + + YQL_ENSURE(inputOrder); + TVector<TString> starOutput = *inputOrder; + + if (item->ChildrenSize() < 4) { + // legacy star without options - column order is not supported + return IGraphTransformer::TStatus::Ok; + } + + if (auto dive = GetSetting(*item->Child(3), "divePrefix")) { + TVector<TString> divePrefixes; + for (auto& prefix : dive->Child(1)->ChildrenList()) { + YQL_ENSURE(prefix->IsAtom()); + divePrefixes.push_back(TString(prefix->Content())); + } + Sort(divePrefixes, [](const auto& left, const auto& right) { return right < left; }); + DivePrefixes(starOutput, divePrefixes); + } else if (auto addPrefix = GetSetting(*item->Child(3), "addPrefix")) { + YQL_ENSURE(addPrefix->Child(1)->IsAtom()); + AddPrefix(starOutput, TString(addPrefix->Child(1)->Content())); + } + + FilterColumnOrderByType(starOutput, *item->GetTypeAnn()); + resultColumnOrder.insert(resultColumnOrder.end(), starOutput.begin(), starOutput.end()); + } + return ctx.Types.SetColumnOrder(*node, resultColumnOrder, ctx.Expr); +} + +IGraphTransformer::TStatus OrderForMergeExtend(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + YQL_ENSURE(node->ChildrenSize()); + auto common = ctx.Types.LookupColumnOrder(node->Head()); + if (!common) { + return IGraphTransformer::TStatus::Ok; + } + + for (ui32 i = 1; i < node->ChildrenSize(); i++) { + auto current = ctx.Types.LookupColumnOrder(*node->Child(i)); + if (!current || current != common) { + return IGraphTransformer::TStatus::Ok; + } + } + + return ctx.Types.SetColumnOrder(*node, *common, ctx.Expr); +} + +IGraphTransformer::TStatus OrderForUnionAll(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + YQL_ENSURE(node->ChildrenSize()); + auto common = ctx.Types.LookupColumnOrder(node->Head()); + if (!common) { + return IGraphTransformer::TStatus::Ok; + } + + for (ui32 i = 1; i < node->ChildrenSize(); i++) { + auto input = node->Child(i); + auto current = ctx.Types.LookupColumnOrder(*input); + if (!current) { + return IGraphTransformer::TStatus::Ok; + } + + bool truncated = false; + for (size_t i = 0; i < Min(common->size(), current->size()); ++i) { + if ((*current)[i] != (*common)[i]) { + common->resize(i); + truncated = true; + break; + } + } + if (!truncated && current->size() > common->size()) { + common = current; + } + } + + if (common->size() > 0) { + auto allColumns = GetColumnsOfStructOrSequenceOfStruct(*node->GetTypeAnn()); + for (auto& col : *common) { + auto it = allColumns.find(col); + YQL_ENSURE(it != allColumns.end()); + allColumns.erase(it); + } + + for (auto& remain : allColumns) { + common->push_back(TString(remain)); + } + return ctx.Types.SetColumnOrder(*node, *common, ctx.Expr); + } + + return IGraphTransformer::TStatus::Ok; +} + +IGraphTransformer::TStatus OrderForEquiJoin(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + const size_t numLists = node->ChildrenSize() - 2; + const auto joinTree = node->Child(numLists); + const auto optionsNode = node->Child(numLists + 1);; + TVector<TMaybe<TColumnOrder>> inputColumnOrder; + TJoinLabels labels; + for (size_t i = 0; i < numLists; ++i) { + auto& list = node->Child(i)->Head(); + inputColumnOrder.push_back(ctx.Types.LookupColumnOrder(list)); + if (auto err = labels.Add(ctx.Expr, *node->Child(i)->Child(1), + list.GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>())) + { + ctx.Expr.AddError(*err); + return IGraphTransformer::TStatus::Error; + } + } + + TJoinOptions options; + auto status = ValidateEquiJoinOptions(node->Pos(), *optionsNode, options, ctx.Expr); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + if (options.Flatten) { + // will be done after ExpandFlattenEquiJoin + return IGraphTransformer::TStatus::Ok; + } + + auto columnTypes = GetJoinColumnTypes(*joinTree, labels, ctx.Expr); + YQL_ENSURE(labels.Inputs.size() == inputColumnOrder.size()); + size_t idx = 0; + TVector<TString> resultColumnOrder; + for (const auto& label : labels.Inputs) { + auto columnOrder = inputColumnOrder[idx++]; + if (!columnOrder) { + bool survivesJoin = AnyOf(label.EnumerateAllColumns(), [&](const TString& outputColumn) { + return columnTypes.contains(outputColumn); + }); + if (survivesJoin) { + // no column order on output + return IGraphTransformer::TStatus::Ok; + } + continue; + } + + for (auto col : *columnOrder) { + TString fullName = label.FullName(col); + if (columnTypes.contains(fullName)) { + auto it = options.RenameMap.find(fullName); + if (it != options.RenameMap.end()) { + for (auto target : it->second) { + resultColumnOrder.push_back(TString(target)); + } + } else { + resultColumnOrder.push_back(fullName); + } + } + } + } + + return ctx.Types.SetColumnOrder(*node, resultColumnOrder, ctx.Expr); +}; + +IGraphTransformer::TStatus OrderFromFirst(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + if (auto columnOrder = ctx.Types.LookupColumnOrder(node->Head())) { + return ctx.Types.SetColumnOrder(*node, *columnOrder, ctx.Expr); + } + return IGraphTransformer::TStatus::Ok; +}; + +IGraphTransformer::TStatus OrderFromFirstAndOutputType(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + if (auto columnOrder = ctx.Types.LookupColumnOrder(node->Head())) { + FilterColumnOrderByType(*columnOrder, *node->GetTypeAnn()); + return ctx.Types.SetColumnOrder(*node, *columnOrder, ctx.Expr); + } + return IGraphTransformer::TStatus::Ok; +} + +} // namespace NTypeAnnImpl +} // namespace NYql diff --git a/ydb/library/yql/core/type_ann/type_ann_columnorder.h b/ydb/library/yql/core/type_ann/type_ann_columnorder.h index bf96ebc949..dc9e9c0a56 100644 --- a/ydb/library/yql/core/type_ann/type_ann_columnorder.h +++ b/ydb/library/yql/core/type_ann/type_ann_columnorder.h @@ -1,21 +1,21 @@ -#pragma once - -#include "type_ann_impl.h" - +#pragma once + +#include "type_ann_impl.h" + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/yql_graph_transformer.h> - -namespace NYql { -namespace NTypeAnnImpl { - + +namespace NYql { +namespace NTypeAnnImpl { + IGraphTransformer::TStatus OrderForPgSetItem(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderForAssumeColumnOrder(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderForSqlProject(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderForMergeExtend(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderForUnionAll(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderForEquiJoin(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); - -IGraphTransformer::TStatus OrderFromFirst(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -IGraphTransformer::TStatus OrderFromFirstAndOutputType(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); -} // namespace NTypeAnnImpl -} // namespace NYql +IGraphTransformer::TStatus OrderForAssumeColumnOrder(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +IGraphTransformer::TStatus OrderForSqlProject(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +IGraphTransformer::TStatus OrderForMergeExtend(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +IGraphTransformer::TStatus OrderForUnionAll(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +IGraphTransformer::TStatus OrderForEquiJoin(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); + +IGraphTransformer::TStatus OrderFromFirst(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +IGraphTransformer::TStatus OrderFromFirstAndOutputType(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExtContext& ctx); +} // namespace NTypeAnnImpl +} // namespace NYql diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index a0c8f116b9..5846e6cb10 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -2,7 +2,7 @@ #include "type_ann_expr.h" #include "type_ann_impl.h" #include "type_ann_list.h" -#include "type_ann_columnorder.h" +#include "type_ann_columnorder.h" #include "type_ann_wide.h" #include "type_ann_types.h" @@ -53,7 +53,7 @@ namespace NTypeAnnImpl { }; template <typename T> - bool IsValidSmallData(TExprNode& atomNode, const TStringBuf& type, TExprContext& ctx, NKikimr::NUdf::EDataSlot slot, TMaybe<TString>& plainValue) { + bool IsValidSmallData(TExprNode& atomNode, const TStringBuf& type, TExprContext& ctx, NKikimr::NUdf::EDataSlot slot, TMaybe<TString>& plainValue) { bool isValid; if (atomNode.Flags() & TNodeFlags::BinaryContent) { if (atomNode.Content().size() != NKikimr::NUdf::GetDataTypeInfo(slot).FixedSize) { @@ -61,9 +61,9 @@ namespace NTypeAnnImpl { } else { const auto data = *reinterpret_cast<const T*>(atomNode.Content().data()); isValid = NKikimr::NMiniKQL::IsValidValue(slot, NKikimr::NUdf::TUnboxedValuePod(data)); - if (isValid) { - plainValue = ToString(data); - } + if (isValid) { + plainValue = ToString(data); + } } } else { isValid = NKikimr::NMiniKQL::IsValidStringValue(slot, atomNode.Content()); @@ -77,15 +77,15 @@ namespace NTypeAnnImpl { && NKikimr::NMiniKQL::IsValidValue(slot, NKikimr::NUdf::TUnboxedValuePod(data)); } } - - if (slot == NKikimr::NUdf::EDataSlot::Bool && isValid && atomNode.Content() != "true" && atomNode.Content() != "false") { - bool value = AsciiEqualsIgnoreCase(atomNode.Content(), "true") || atomNode.Content() == "1"; - plainValue = value ? "true" : "false"; - } + + if (slot == NKikimr::NUdf::EDataSlot::Bool && isValid && atomNode.Content() != "true" && atomNode.Content() != "false") { + bool value = AsciiEqualsIgnoreCase(atomNode.Content(), "true") || atomNode.Content() == "1"; + plainValue = value ? "true" : "false"; + } } if (!isValid) { - ctx.AddError(TIssue(ctx.GetPosition(atomNode.Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.AddError(TIssue(ctx.GetPosition(atomNode.Pos()), TStringBuilder() << "Bad atom format for type: " << type << ", value: " << TString(atomNode.Content()).Quote())); } @@ -101,16 +101,16 @@ namespace NTypeAnnImpl { return true; } - TMaybe<TString> SerializeTzComponents(bool isValid, ui64 value, ui16 tzId) { - if (isValid) { - return ToString(value) + "," + ToString(*NKikimr::NMiniKQL::FindTimezoneIANAName(tzId)); - } - return {}; - } - + TMaybe<TString> SerializeTzComponents(bool isValid, ui64 value, ui16 tzId) { + if (isValid) { + return ToString(value) + "," + ToString(*NKikimr::NMiniKQL::FindTimezoneIANAName(tzId)); + } + return {}; + } + template <typename T> - bool IsValidTzData(TExprNode& atomNode, const TStringBuf& type, TExprContext& ctx, NKikimr::NUdf::EDataSlot slot, TMaybe<TString>& plainValue) { - plainValue = {}; + bool IsValidTzData(TExprNode& atomNode, const TStringBuf& type, TExprContext& ctx, NKikimr::NUdf::EDataSlot slot, TMaybe<TString>& plainValue) { + plainValue = {}; bool isValid; if (atomNode.Flags() & TNodeFlags::BinaryContent) { // just deserialize @@ -119,21 +119,21 @@ namespace NTypeAnnImpl { ui16 value; ui16 tzId; isValid = NKikimr::NMiniKQL::DeserializeTzDate(atomNode.Content(), value, tzId); - plainValue = SerializeTzComponents(isValid, value, tzId); + plainValue = SerializeTzComponents(isValid, value, tzId); break; } case sizeof(ui32): { ui32 value; ui16 tzId; isValid = NKikimr::NMiniKQL::DeserializeTzDatetime(atomNode.Content(), value, tzId); - plainValue = SerializeTzComponents(isValid, value, tzId); + plainValue = SerializeTzComponents(isValid, value, tzId); break; } case sizeof(ui64): { ui64 value; ui16 tzId; isValid = NKikimr::NMiniKQL::DeserializeTzTimestamp(atomNode.Content(), value, tzId); - plainValue = SerializeTzComponents(isValid, value, tzId); + plainValue = SerializeTzComponents(isValid, value, tzId); break; } } @@ -148,7 +148,7 @@ namespace NTypeAnnImpl { } if (!isValid) { - ctx.AddError(TIssue(ctx.GetPosition(atomNode.Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.AddError(TIssue(ctx.GetPosition(atomNode.Pos()), TStringBuilder() << "Bad atom format for type: " << type << ", value: " << TString(atomNode.Content()).Quote())); } @@ -156,7 +156,7 @@ namespace NTypeAnnImpl { } // TODO: Use ExpandType - TExprNode::TPtr MakeNothingData(TExprContext& ctx, TPositionHandle pos, TStringBuf data) { + TExprNode::TPtr MakeNothingData(TExprContext& ctx, TPositionHandle pos, TStringBuf data) { return ctx.Builder(pos) .Callable("Nothing") .Callable(0, "OptionalType") @@ -522,7 +522,7 @@ namespace NTypeAnnImpl { THashMap<TString, TAnnotationFunc> Functions; THashMap<TString, TExtendedAnnotationFunc> ExtFunctions; - THashMap<TString, TExtendedAnnotationFunc> ColumnOrderFunctions; + THashMap<TString, TExtendedAnnotationFunc> ColumnOrderFunctions; THashMap<TString, ETypeAnnotationKind> TypeKinds; THashSet<TString> AllNames; @@ -542,8 +542,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -578,7 +578,7 @@ namespace NTypeAnnImpl { } if (!NYql::NDecimal::IsValid(input->Head().Content())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " << input->Content() << ", value: " << TString{input->Head().Content()}.Quote())); return IGraphTransformer::TStatus::Error; @@ -598,7 +598,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - TMaybe<TString> textValue; + TMaybe<TString> textValue; if (input->Content() == "Bool") { if (!IsValidSmallData<bool>(input->Head(), input->Content(), ctx.Expr, NKikimr::NUdf::EDataSlot::Bool, textValue)) { return IGraphTransformer::TStatus::Error; @@ -645,21 +645,21 @@ namespace NTypeAnnImpl { } } else if (input->Content() == "Yson") { if (!NDom::IsValidYson(input->Head().Content())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " << input->Content() << ", value: " << TString(input->Head().Content()).Quote())); return IGraphTransformer::TStatus::Error; } } else if (input->Content() == "Json") { if (!NDom::IsValidJson(input->Head().Content())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " << input->Content() << ", value: " << TString(input->Head().Content()).Quote())); return IGraphTransformer::TStatus::Error; } } else if (input->Content() == "Utf8") { if (!IsUtf8(input->Head().Content())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " << input->Content() << ", value: " << TString(input->Head().Content()).Quote())); return IGraphTransformer::TStatus::Error; @@ -700,7 +700,7 @@ namespace NTypeAnnImpl { } } else if (input->Content() == "Uuid") { if (input->Head().Content().size() != 16) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: " << input->Content() << ", value: " << TString(input->Head().Content()).Quote())); return IGraphTransformer::TStatus::Error; @@ -718,16 +718,16 @@ namespace NTypeAnnImpl { ythrow yexception() << "Unknown data type: " << input->Content(); } - if (textValue) { - // need to replace binary arg with text one - output = ctx.Expr.Builder(input->Pos()) - .Callable(input->Content()) - .Atom(0, *textValue) - .Seal() - .Build(); - return IGraphTransformer::TStatus::Repeat; - } - + if (textValue) { + // need to replace binary arg with text one + output = ctx.Expr.Builder(input->Pos()) + .Callable(input->Content()) + .Atom(0, *textValue) + .Seal() + .Build(); + return IGraphTransformer::TStatus::Repeat; + } + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(input->Content()))); return IGraphTransformer::TStatus::Ok; } @@ -741,7 +741,7 @@ namespace NTypeAnnImpl { } if (child->ChildrenSize() != 1 && child->ChildrenSize() != 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected tuple of size 1 or 2, but got: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected tuple of size 1 or 2, but got: " << input->ChildrenSize())); return IGraphTransformer::TStatus::Error; } @@ -779,7 +779,7 @@ namespace NTypeAnnImpl { auto worldType = input->Head().GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[0]; if (worldType->GetKind() != ETypeAnnotationKind::World) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected world type as type of first tuple element, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected world type as type of first tuple element, but got: " << *worldType)); return IGraphTransformer::TStatus::Error; } @@ -800,7 +800,7 @@ namespace NTypeAnnImpl { auto worldType = input->Head().GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[0]; if (worldType->GetKind() != ETypeAnnotationKind::World) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected world type as type of first tuple element, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected world type as type of first tuple element, but got: " << *worldType)); return IGraphTransformer::TStatus::Error; } @@ -832,39 +832,39 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, const TStructExprType& structType, TString& errStr) - { - auto result = structType.FindItem(memberName); - if (!result) { - TStringBuilder sb; - sb << "Member not found: " << memberName; - auto mistype = structType.FindMistype(memberName); - if (mistype) { - YQL_ENSURE(mistype.GetRef() != memberName); - sb << ". Did you mean " << mistype.GetRef() << "?"; - } else { - auto dotPos = memberName.find_first_of('.'); - if (dotPos != TStringBuf::npos) { - TStringBuf rest = memberName.SubStr(dotPos + 1); - if (rest && structType.FindItem(rest)) { - sb << ". Did you mean " << rest << "?"; - } - } - } - errStr = sb; - } - return result; - } - - TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TStructExprType& structType, TContext& ctx) { - TString errStr; - auto result = FindOrReportMissingMember(memberName, structType, errStr); - if (!result) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), errStr)); - } - return result; - } - + TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, const TStructExprType& structType, TString& errStr) + { + auto result = structType.FindItem(memberName); + if (!result) { + TStringBuilder sb; + sb << "Member not found: " << memberName; + auto mistype = structType.FindMistype(memberName); + if (mistype) { + YQL_ENSURE(mistype.GetRef() != memberName); + sb << ". Did you mean " << mistype.GetRef() << "?"; + } else { + auto dotPos = memberName.find_first_of('.'); + if (dotPos != TStringBuf::npos) { + TStringBuf rest = memberName.SubStr(dotPos + 1); + if (rest && structType.FindItem(rest)) { + sb << ". Did you mean " << rest << "?"; + } + } + } + errStr = sb; + } + return result; + } + + TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TStructExprType& structType, TContext& ctx) { + TString errStr; + auto result = FindOrReportMissingMember(memberName, structType, errStr); + if (!result) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), errStr)); + } + return result; + } + IGraphTransformer::TStatus MemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -900,7 +900,7 @@ namespace NTypeAnnImpl { } auto memberName = input->Tail().Content(); - auto pos = FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -913,60 +913,60 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus SingleMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus SingleMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto arg = input->HeadPtr(); - - if (IsNull(*arg)) { - output = arg; - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* item = arg->GetTypeAnn(); - if (item->GetKind() == ETypeAnnotationKind::Optional) { - item = item->Cast<TOptionalExprType>()->GetItemType(); - } - - if (!EnsureStructType(arg->Pos(), *item, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto structType = item->Cast<TStructExprType>(); - - if (structType->GetSize() != 1) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expecting single member struct, but got: " << *item)); - return IGraphTransformer::TStatus::Error; - } - - output = ctx.Expr.Builder(input->Pos()) - .Callable("Member") - .Add(0, arg) - .Atom(1, structType->GetItems()[0]->GetName()) - .Seal() - .Build(); - return IGraphTransformer::TStatus::Repeat; - } - + + if (IsNull(*arg)) { + output = arg; + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* item = arg->GetTypeAnn(); + if (item->GetKind() == ETypeAnnotationKind::Optional) { + item = item->Cast<TOptionalExprType>()->GetItemType(); + } + + if (!EnsureStructType(arg->Pos(), *item, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto structType = item->Cast<TStructExprType>(); + + if (structType->GetSize() != 1) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expecting single member struct, but got: " << *item)); + return IGraphTransformer::TStatus::Error; + } + + output = ctx.Expr.Builder(input->Pos()) + .Callable("Member") + .Add(0, arg) + .Atom(1, structType->GetItems()[0]->GetName()) + .Seal() + .Build(); + return IGraphTransformer::TStatus::Repeat; + } + IGraphTransformer::TStatus SqlColumnWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - const bool columnOrType = input->IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType"}); - const bool isPlain = input->IsCallable({"SqlPlainColumn", "SqlPlainColumnOrType"}); - if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) { + const bool columnOrType = input->IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType"}); + const bool isPlain = input->IsCallable({"SqlPlainColumn", "SqlPlainColumnOrType"}); + if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureMaxArgsCount(*input, isPlain ? 2 : 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (!EnsureMaxArgsCount(*input, isPlain ? 2 : 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (IsNull(input->Head())) { - if (columnOrType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected (optional) struct type, but got Null")); - return IGraphTransformer::TStatus::Error; - } + if (columnOrType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected (optional) struct type, but got Null")); + return IGraphTransformer::TStatus::Error; + } output = input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; } @@ -978,10 +978,10 @@ namespace NTypeAnnImpl { sourceNameNode = input->ChildPtr(2); } - if (!EnsureStructOrOptionalStructType(*rowNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; + if (!EnsureStructOrOptionalStructType(*rowNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; } - const TStructExprType* structType = RemoveAllOptionals(rowNode->GetTypeAnn())->Cast<TStructExprType>(); + const TStructExprType* structType = RemoveAllOptionals(rowNode->GetTypeAnn())->Cast<TStructExprType>(); if (!EnsureAtom(*columnNameNode, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -990,63 +990,63 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - const auto columnName = columnNameNode->Content(); - const auto sourceName = sourceNameNode ? sourceNameNode->Content() : ""; + const auto columnName = columnNameNode->Content(); + const auto sourceName = sourceNameNode ? sourceNameNode->Content() : ""; - TString effectiveColumnName; + TString effectiveColumnName; if (sourceName) { - effectiveColumnName = DotJoin(sourceName, columnName); - } else if (isPlain) { - effectiveColumnName = ToString(columnName); + effectiveColumnName = DotJoin(sourceName, columnName); + } else if (isPlain) { + effectiveColumnName = ToString(columnName); } else { TStringBuf goalItemName; for (auto& item: structType->GetItems()) { const auto& itemName = item->GetName(); - TStringBuf columnItemName; - auto dotPos = itemName.find_first_of('.'); - if (dotPos == TStringBuf::npos) { - columnItemName = itemName; - } else { - columnItemName = itemName.SubStr(dotPos + 1); - } + TStringBuf columnItemName; + auto dotPos = itemName.find_first_of('.'); + if (dotPos == TStringBuf::npos) { + columnItemName = itemName; + } else { + columnItemName = itemName.SubStr(dotPos + 1); + } if (columnItemName == columnName) { if (goalItemName) { - auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "column name: " << + auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "column name: " << columnName << " conflicted without correlation name it may be one of: " << - goalItemName << ", " << itemName); - if (columnOrType) { - input->SetTypeAnn(ctx.Expr.MakeType<TErrorExprType>(issue)); - return IGraphTransformer::TStatus::Ok; - } - ctx.Expr.AddError(issue); + goalItemName << ", " << itemName); + if (columnOrType) { + input->SetTypeAnn(ctx.Expr.MakeType<TErrorExprType>(issue)); + return IGraphTransformer::TStatus::Ok; + } + ctx.Expr.AddError(issue); return IGraphTransformer::TStatus::Error; } else { goalItemName = itemName; } } } - effectiveColumnName = goalItemName ? goalItemName : columnName; - } - - TString errStr; - auto pos = FindOrReportMissingMember(effectiveColumnName, *structType, errStr); - if (columnOrType) { - if (!pos) { - input->SetTypeAnn(ctx.Expr.MakeType<TErrorExprType>(TIssue(ctx.Expr.GetPosition(input->Pos()), errStr))); - return IGraphTransformer::TStatus::Ok; - } - output = ctx.Expr.Builder(input->Pos()) - .Callable("SqlColumnFromType") - .Add(0, input->HeadPtr()) - .Atom(1, effectiveColumnName) - .Add(2, input->ChildPtr(1)) // original column/type name - .Seal() - .Build(); - return IGraphTransformer::TStatus::Repeat; - } - + effectiveColumnName = goalItemName ? goalItemName : columnName; + } + + TString errStr; + auto pos = FindOrReportMissingMember(effectiveColumnName, *structType, errStr); + if (columnOrType) { + if (!pos) { + input->SetTypeAnn(ctx.Expr.MakeType<TErrorExprType>(TIssue(ctx.Expr.GetPosition(input->Pos()), errStr))); + return IGraphTransformer::TStatus::Ok; + } + output = ctx.Expr.Builder(input->Pos()) + .Callable("SqlColumnFromType") + .Add(0, input->HeadPtr()) + .Atom(1, effectiveColumnName) + .Add(2, input->ChildPtr(1)) // original column/type name + .Seal() + .Build(); + return IGraphTransformer::TStatus::Repeat; + } + if (!pos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), errStr)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), errStr)); return IGraphTransformer::TStatus::Error; } output = ctx.Expr.Builder(input->Pos()) @@ -1057,43 +1057,43 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus SqlColumnFromTypeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureStructOrOptionalStructType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto columnNameNode = input->Child(1); - if (!EnsureAtom(*columnNameNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto originalTypeNode = input->Child(2); - if (!EnsureAtom(*originalTypeNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - bool isOptional = input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; - const TStructExprType& structType = *RemoveAllOptionals(input->Head().GetTypeAnn())->Cast<TStructExprType>(); - - auto pos = FindOrReportMissingMember(columnNameNode->Content(), input->Pos(), structType, ctx); - if (!pos) { - return IGraphTransformer::TStatus::Error; - } - - const TTypeAnnotationNode* resultType = structType.GetItems()[*pos]->GetItemType(); - if (isOptional && resultType->GetKind() != ETypeAnnotationKind::Optional && resultType->GetKind() != ETypeAnnotationKind::Null) { - resultType = ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()); - } - - input->SetTypeAnn(resultType); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus SqlColumnFromTypeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureStructOrOptionalStructType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto columnNameNode = input->Child(1); + if (!EnsureAtom(*columnNameNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto originalTypeNode = input->Child(2); + if (!EnsureAtom(*originalTypeNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + bool isOptional = input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional; + const TStructExprType& structType = *RemoveAllOptionals(input->Head().GetTypeAnn())->Cast<TStructExprType>(); + + auto pos = FindOrReportMissingMember(columnNameNode->Content(), input->Pos(), structType, ctx); + if (!pos) { + return IGraphTransformer::TStatus::Error; + } + + const TTypeAnnotationNode* resultType = structType.GetItems()[*pos]->GetItemType(); + if (isOptional && resultType->GetKind() != ETypeAnnotationKind::Optional && resultType->GetKind() != ETypeAnnotationKind::Null) { + resultType = ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()); + } + + input->SetTypeAnn(resultType); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus TryMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -1256,9 +1256,9 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus NormalizeAtomListForDiveOrSelect(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus NormalizeAtomListForDiveOrSelect(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { YQL_ENSURE(input->IsCallable({"DivePrefixMembers", "SelectMembers", "FilterMembers", "RemovePrefixMembers"})); - + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -1267,50 +1267,50 @@ namespace NTypeAnnImpl { if (!EnsureTuple(*prefixes, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - - for (auto& node : prefixes->Children()) { - if (!EnsureAtom(*node, ctx.Expr)) { + + for (auto& node : prefixes->Children()) { + if (!EnsureAtom(*node, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } } - auto descending = [](const TExprNode::TPtr& left, const TExprNode::TPtr& right) { - return left->Content() > right->Content(); - }; - - if (!IsSorted(prefixes->Children().begin(), prefixes->Children().end(), descending)) { - auto list = prefixes->ChildrenList(); - Sort(list, descending); - - output = ctx.Expr.Builder(input->Pos()) - .Callable(input->Content()) + auto descending = [](const TExprNode::TPtr& left, const TExprNode::TPtr& right) { + return left->Content() > right->Content(); + }; + + if (!IsSorted(prefixes->Children().begin(), prefixes->Children().end(), descending)) { + auto list = prefixes->ChildrenList(); + Sort(list, descending); + + output = ctx.Expr.Builder(input->Pos()) + .Callable(input->Content()) .Add(0, input->HeadPtr()) - .Add(1, ctx.Expr.NewList(prefixes->Pos(), std::move(list))) - .Seal() - .Build(); - - return IGraphTransformer::TStatus::Repeat; - } - - return IGraphTransformer::TStatus::Ok; - } - + .Add(1, ctx.Expr.NewList(prefixes->Pos(), std::move(list))) + .Seal() + .Build(); + + return IGraphTransformer::TStatus::Repeat; + } + + return IGraphTransformer::TStatus::Ok; + } + template<bool ByPrefix> - IGraphTransformer::TStatus SelectMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus SelectMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto structObj = input->Child(0); if (!EnsureStructOrOptionalStructType(*structObj, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - + auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + auto type = structObj->GetTypeAnn(); const bool optional = type->GetKind() == ETypeAnnotationKind::Optional; if (optional) { @@ -1320,7 +1320,7 @@ namespace NTypeAnnImpl { auto structExprType = type->Cast<TStructExprType>(); for (auto& field: structExprType->GetItems()) { const auto& fieldName = field->GetName(); - auto prefixes = input->Child(1); + auto prefixes = input->Child(1); for (const auto& prefixNode: prefixes->Children()) { const auto& prefix = prefixNode->Content(); if (ByPrefix ? fieldName.StartsWith(prefix) : fieldName == prefix) { @@ -1352,11 +1352,11 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - + auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + auto type = structObj->GetTypeAnn(); const bool optional = type->GetKind() == ETypeAnnotationKind::Optional; if (optional) { @@ -1366,7 +1366,7 @@ namespace NTypeAnnImpl { auto structExprType = type->Cast<TStructExprType>(); for (auto& field: structExprType->GetItems()) { const auto& fieldName = field->GetName(); - auto prefixes = input->Child(1); + auto prefixes = input->Child(1); for (const auto& prefixNode: prefixes->Children()) { const auto& prefix = prefixNode->Content(); if (fieldName.StartsWith(prefix)) { @@ -1399,7 +1399,7 @@ namespace NTypeAnnImpl { if ((*iter)->IsAtom()) { mode = (*iter)->Content(); if (mode != "auto" && mode != "optional" && mode != "list" && mode != "dict") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition((*iter)->Pos()), TStringBuilder() << "Unsupported flatten by mode: " << mode)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition((*iter)->Pos()), TStringBuilder() << "Unsupported flatten by mode: " << mode)); return IGraphTransformer::TStatus::Error; } @@ -1421,7 +1421,7 @@ namespace NTypeAnnImpl { TString alias; const auto& argType = child->Type(); if (argType != TExprNode::List && argType != TExprNode::Atom) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected atom or tuple, but got: " << argType)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected atom or tuple, but got: " << argType)); return IGraphTransformer::TStatus::Error; } TExprNode* columnNameNode = child.Get(); @@ -1436,12 +1436,12 @@ namespace NTypeAnnImpl { } alias = aliasNode->Content(); if (!aliases.emplace(alias).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode->Pos()), TStringBuilder() << "Duplicate flatten alias found: " << alias)); return IGraphTransformer::TStatus::Error; } if (flattenByColumns.contains(alias)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << "Collision between alias and column name: " << alias)); return IGraphTransformer::TStatus::Error; } @@ -1451,12 +1451,12 @@ namespace NTypeAnnImpl { } auto columnName = columnNameNode->Content(); if (!flattenByColumns.emplace(TString(columnName), alias).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << "Duplicate flatten field found: " << columnName)); return IGraphTransformer::TStatus::Error; } if (aliases.contains(columnName)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(columnNameNode->Pos()), TStringBuilder() << "Collision between alias and column name: " << columnName)); return IGraphTransformer::TStatus::Error; } @@ -1474,7 +1474,7 @@ namespace NTypeAnnImpl { auto flattenIter = flattenByColumns.find(fieldName); if (flattenIter == flattenByColumns.end()) { if (aliases.contains(fieldName)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Conflict flatten alias and column name: '" << fieldName << "'")); return IGraphTransformer::TStatus::Error; } @@ -1497,28 +1497,28 @@ namespace NTypeAnnImpl { } if (mode == "optional" && !fieldOptional) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Expected optional type in field of struct: '" << fieldName << "', but got: " << *field->GetItemType())); return IGraphTransformer::TStatus::Error; } if (mode == "list" && RemoveOptionalType(field->GetItemType())->GetKind() != ETypeAnnotationKind::List) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Expected (optional) list type in field of struct: '" << fieldName << "', but got: " << *field->GetItemType())); return IGraphTransformer::TStatus::Error; } if (mode == "dict" && RemoveOptionalType(field->GetItemType())->GetKind() != ETypeAnnotationKind::Dict) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Expected (optional) dict type in field of struct: '" << fieldName << "', but got: " << *field->GetItemType())); return IGraphTransformer::TStatus::Error; } if (mode == "auto" && fieldOptional && RemoveOptionalType(field->GetItemType())->GetKind() == ETypeAnnotationKind::List) { - auto issue = TIssue(ctx.Expr.GetPosition(structObj->Pos()), "Ambiguous FLATTEN BY statement, please choose FLATTEN LIST BY or FLATTEN OPTIONAL BY"); + auto issue = TIssue(ctx.Expr.GetPosition(structObj->Pos()), "Ambiguous FLATTEN BY statement, please choose FLATTEN LIST BY or FLATTEN OPTIONAL BY"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_FLATTEN_BY_OPT, issue); if (!ctx.Expr.AddWarning(issue)) { return IGraphTransformer::TStatus::Error; @@ -1526,7 +1526,7 @@ namespace NTypeAnnImpl { } if (mode == "auto" && fieldOptional && RemoveOptionalType(field->GetItemType())->GetKind() == ETypeAnnotationKind::Dict) { - auto issue = TIssue(ctx.Expr.GetPosition(structObj->Pos()), "Ambiguous FLATTEN BY statement, please choose FLATTEN DICT BY or FLATTEN OPTIONAL BY"); + auto issue = TIssue(ctx.Expr.GetPosition(structObj->Pos()), "Ambiguous FLATTEN BY statement, please choose FLATTEN DICT BY or FLATTEN OPTIONAL BY"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_FLATTEN_BY_OPT, issue); if (!ctx.Expr.AddWarning(issue)) { return IGraphTransformer::TStatus::Error; @@ -1556,7 +1556,7 @@ namespace NTypeAnnImpl { const auto payloadType = dictType->GetPayloadType(); flattenItemType = ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType({keyType, payloadType})); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Expected list, dict or optional types in field of struct: '" << fieldName << "', but got: " << *field->GetItemType())); return IGraphTransformer::TStatus::Error; @@ -1566,7 +1566,7 @@ namespace NTypeAnnImpl { flattenByColumns.erase(flattenIter); } if (!flattenByColumns.empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() << "Column for flatten \"" << flattenByColumns.begin()->first << "\" does not exist")); return IGraphTransformer::TStatus::Error; } @@ -1624,7 +1624,7 @@ namespace NTypeAnnImpl { } if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; } @@ -1688,10 +1688,10 @@ namespace NTypeAnnImpl { auto structType = input->Head().GetTypeAnn()->Cast<TStructExprType>(); auto memberName = input->Tail().Content(); - TVector<const TItemExprType*> newItems = structType->GetItems(); - EraseIf(newItems, [&](const auto& item) { return item->GetName() == memberName; }); + TVector<const TItemExprType*> newItems = structType->GetItems(); + EraseIf(newItems, [&](const auto& item) { return item->GetName() == memberName; }); - if (!Forced && !FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx)) { + if (!Forced && !FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx)) { return IGraphTransformer::TStatus::Error; } @@ -1873,7 +1873,7 @@ namespace NTypeAnnImpl { auto memberName = input->Child(1)->Content(); auto structType = input->Head().GetTypeAnn()->Cast<TStructExprType>(); - auto pos = FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -1932,13 +1932,13 @@ namespace NTypeAnnImpl { case ECompareOptions::Null: output = MakeBoolNothing(input->Pos(), ctx.Expr); return IGraphTransformer::TStatus::Repeat; - + case ECompareOptions::Uncomparable: - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Uncompatible types in compare: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Uncompatible types in compare: " << *input->Head().GetTypeAnn() << " '" << input->Content() << "' " << *input->Tail().GetTypeAnn())); return IGraphTransformer::TStatus::Error; - + case ECompareOptions::Comparable: input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); break; @@ -1971,7 +1971,7 @@ namespace NTypeAnnImpl { } if (!(IsDataTypeNumeric(dataType->GetSlot()) || IsDataTypeDecimal(dataType->GetSlot()) || dataType->GetSlot() == EDataSlot::Interval)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric, decimal or interval type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric, decimal or interval type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2015,7 +2015,7 @@ namespace NTypeAnnImpl { const bool isLeftNumeric = IsDataTypeNumeric(dataType1->GetSlot()); const bool isRightNumeric = IsDataTypeNumeric(dataType2->GetSlot()); if (isLeftNumeric != isRightNumeric) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of data types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of data types: " << *input->Head().GetTypeAnn() << " != " << *input->Child(index)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2027,7 +2027,7 @@ namespace NTypeAnnImpl { } else { if (!IsSameAnnotation(*dataType1, *dataType2)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of data types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of data types: " << *input->Head().GetTypeAnn() << " != " << *input->Child(index)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2053,7 +2053,7 @@ namespace NTypeAnnImpl { } if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " << *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2080,7 +2080,7 @@ namespace NTypeAnnImpl { } if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " << *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2095,28 +2095,28 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus DistinctFromWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!(EnsurePersistable(input->Head(), ctx.Expr) && EnsurePersistable(input->Tail(), ctx.Expr))) { - return IGraphTransformer::TStatus::Error; - } - - if (CanCompare<true>(input->Head().GetTypeAnn(), input->Tail().GetTypeAnn()) == ECompareOptions::Uncomparable) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Uncompatible types in compare: " << *input->Head().GetTypeAnn() << " '" << input->Content() << "' " << *input->Tail().GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); - input->SetUnorderedChildren(); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus DistinctFromWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!(EnsurePersistable(input->Head(), ctx.Expr) && EnsurePersistable(input->Tail(), ctx.Expr))) { + return IGraphTransformer::TStatus::Error; + } + + if (CanCompare<true>(input->Head().GetTypeAnn(), input->Tail().GetTypeAnn()) == ECompareOptions::Uncomparable) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Uncompatible types in compare: " << *input->Head().GetTypeAnn() << " '" << input->Content() << "' " << *input->Tail().GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); + input->SetUnorderedChildren(); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus AggrAddWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -2145,7 +2145,7 @@ namespace NTypeAnnImpl { } if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " << *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2153,7 +2153,7 @@ namespace NTypeAnnImpl { const bool isInterval = dataType1->GetSlot() == EDataSlot::Interval; if (!(IsDataTypeNumeric(dataType1->GetSlot()) || IsDataTypeDecimal(dataType1->GetSlot()) || isInterval)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric or decimal data type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric or decimal data type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2212,7 +2212,7 @@ namespace NTypeAnnImpl { if (!(*dataTypeOne == *dataTypeTwo)) { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot add different decimals." )); @@ -2222,7 +2222,7 @@ namespace NTypeAnnImpl { commonType = dataType[0]; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot add type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2281,7 +2281,7 @@ namespace NTypeAnnImpl { if (!(*dataTypeOne == *dataTypeTwo)) { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot substract different decimals." )); @@ -2291,7 +2291,7 @@ namespace NTypeAnnImpl { commonType = dataType[0]; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot substract type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2347,7 +2347,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot multiply type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2400,7 +2400,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot divide type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2450,7 +2450,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot mod type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2495,7 +2495,7 @@ namespace NTypeAnnImpl { if (!(*dataTypeOne == *dataTypeTwo)) { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot calculate with different decimals." )); @@ -2503,7 +2503,7 @@ namespace NTypeAnnImpl { } } else if (!IsDataTypeIntegral(dataType[1]->GetSlot())) { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot operate with decimal and " << *input->Tail().GetTypeAnn() )); @@ -2513,7 +2513,7 @@ namespace NTypeAnnImpl { commonType = dataType[0]; } else { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot use type " << *input->Head().GetTypeAnn() << " and " << *input->Tail().GetTypeAnn() )); @@ -2546,7 +2546,7 @@ namespace NTypeAnnImpl { } if (!IsDataTypeIntegral(dataType->GetSlot())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected integral data type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected integral data type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2595,7 +2595,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } } else if (!IsDataTypeNumeric(dataSlot)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric or decimal data type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric or decimal data type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2628,7 +2628,7 @@ namespace NTypeAnnImpl { auto dataSlot = dataType->GetSlot(); if (!(IsDataTypeNumeric(dataSlot) || IsDataTypeDecimal(dataSlot) || dataSlot == EDataSlot::Interval)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric, decimal or interval data type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric, decimal or interval data type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2660,7 +2660,7 @@ namespace NTypeAnnImpl { auto dataSlot = dataType[i]->GetSlot(); if (!IsDataTypeUnsigned(dataSlot)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected unsigned data type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected unsigned data type, but got: " << *input->Child(i)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2809,7 +2809,7 @@ namespace NTypeAnnImpl { } if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: " << *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2839,14 +2839,14 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } - for (ui32 i = 0; i < input->ChildrenSize(); ++i) { - auto child = input->Child(i); - if (!child->GetTypeAnn()) { - YQL_ENSURE(child->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() - << "Expected (optional) " << (i ? "Uint32" : "String") << ", but got lambda")); - return IGraphTransformer::TStatus::Error; - } + for (ui32 i = 0; i < input->ChildrenSize(); ++i) { + auto child = input->Child(i); + if (!child->GetTypeAnn()) { + YQL_ENSURE(child->Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + << "Expected (optional) " << (i ? "Uint32" : "String") << ", but got lambda")); + return IGraphTransformer::TStatus::Error; + } } auto originalType = input->Head().GetTypeAnn(); @@ -2866,7 +2866,7 @@ namespace NTypeAnnImpl { } const auto convertStatus1 = TryConvertTo(input->ChildRef(1), *expectedTypeOne, ctx.Expr); if (convertStatus1.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -2877,7 +2877,7 @@ namespace NTypeAnnImpl { } const auto convertStatus2 = TryConvertTo(input->ChildRef(2), *expectedTypeTwo, ctx.Expr); if (convertStatus2.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -2994,7 +2994,7 @@ namespace NTypeAnnImpl { auto expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32); auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -3048,7 +3048,7 @@ namespace NTypeAnnImpl { } else return IGraphTransformer::TStatus::Error; - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(input->Head().GetTypeAnn())); + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Ok; } @@ -3091,7 +3091,7 @@ namespace NTypeAnnImpl { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expecting list, optional or tuple argument, but got: " << *argType)); return IGraphTransformer::TStatus::Error; - } + } } IGraphTransformer::TStatus ToOptionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { @@ -3151,23 +3151,23 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus BuildTablePathWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureSpecificDataType(input->Head(), EDataSlot::String, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureSpecificDataType(input->Tail(), EDataSlot::String, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(input->Head().GetTypeAnn()); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus BuildTablePathWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureSpecificDataType(input->Head(), EDataSlot::String, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureSpecificDataType(input->Tail(), EDataSlot::String, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(input->Head().GetTypeAnn()); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus WithOptionalArgsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { @@ -3290,14 +3290,14 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus StructWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (input->ChildrenSize() > 0) { - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Struct) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected struct type, but got: " << *type)); + TStringBuilder() << "Expected struct type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -3341,7 +3341,7 @@ namespace NTypeAnnImpl { bool hasMissingMembers = false; for (auto& x : expectedMembers) { if (!foundMembers.contains(x.first)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Missing member: " << x.first)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Missing member: " << x.first)); hasMissingMembers = true; } } @@ -3441,7 +3441,7 @@ namespace NTypeAnnImpl { auto dataTypeName = input->Child(1)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(dataTypeName); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); return IGraphTransformer::TStatus::Error; } @@ -3489,7 +3489,7 @@ namespace NTypeAnnImpl { auto dataTypeName = input->Child(1)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(dataTypeName); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); return IGraphTransformer::TStatus::Error; } @@ -3540,7 +3540,7 @@ namespace NTypeAnnImpl { auto dataTypeName = input->Child(1)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(dataTypeName); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << dataTypeName)); return IGraphTransformer::TStatus::Error; } @@ -3595,8 +3595,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } EDataSlot targetSlot; @@ -3627,7 +3627,7 @@ namespace NTypeAnnImpl { } if (!CanConvert(sourceSlot, targetSlot)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot convert type " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot convert type " << *input->Head().GetTypeAnn() << " into " << NKikimr::NUdf::GetDataTypeInfo(targetSlot).Name)); return IGraphTransformer::TStatus::Error; } @@ -3652,8 +3652,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } EDataSlot targetSlot; @@ -3678,7 +3678,7 @@ namespace NTypeAnnImpl { const auto sourceSlot = dataType->GetSlot(); if (!((EDataSlot::Bool == sourceSlot || IsDataTypeIntegral(sourceSlot) || IsDataTypeDateOrTzDateOrInterval(sourceSlot)) && IsDataTypeIntegral(targetSlot))) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot bit cast type " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot bit cast type " << *input->Head().GetTypeAnn() << " into " << NKikimr::NUdf::GetDataTypeInfo(targetSlot).Name)); return IGraphTransformer::TStatus::Error; } @@ -3687,67 +3687,67 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus AlterToWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 4, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus AlterToWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 4, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + const auto& source = input->Head(); - auto& targetTypeNode = input->ChildRef(1); - auto& lambda = input->ChildRef(2); + auto& targetTypeNode = input->ChildRef(1); + auto& lambda = input->ChildRef(2); const auto& alterFailValue = input->Tail(); - + if (!EnsureComputable(source, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(targetTypeNode, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(targetTypeNode, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + if (!EnsureComputable(alterFailValue, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + const auto sourceType = source.GetTypeAnn(); - const auto targetType = targetTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - + const auto targetType = targetTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + const auto status = ConvertToLambda(lambda, ctx.Expr, 1); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(lambda, {targetType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!lambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(lambda, {targetType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!lambda->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + const auto alterSuccessType = lambda->GetTypeAnn(); const auto alterFailType = alterFailValue.GetTypeAnn(); - if (!IsSameAnnotation(*alterSuccessType, *alterFailType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of success/fail types, success type: " - << *alterSuccessType << ", fail type: " << *alterFailType)); - return IGraphTransformer::TStatus::Error; - } - - if (IsSameAnnotation(*sourceType, *targetType)) { + if (!IsSameAnnotation(*alterSuccessType, *alterFailType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of success/fail types, success type: " + << *alterSuccessType << ", fail type: " << *alterFailType)); + return IGraphTransformer::TStatus::Error; + } + + if (IsSameAnnotation(*sourceType, *targetType)) { output = ctx.Expr.ReplaceNode(lambda->TailPtr(), lambda->Head().Head(), input->HeadPtr()); - return IGraphTransformer::TStatus::Repeat; - } - + return IGraphTransformer::TStatus::Repeat; + } + if (NKikimr::NUdf::ECastOptions::Impossible & CastResult<true>(sourceType, targetType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Impossible alter " << *sourceType << " to " << *targetType)); - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(alterSuccessType); - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(alterSuccessType); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus ToIntegralWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -3759,8 +3759,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } EDataSlot targetSlot; @@ -3803,7 +3803,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } } else if (!IsDataTypeFloat(sourceSlot) || !(IsDataTypeIntegral(targetSlot) || targetSlot == EDataSlot::Bool)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot make an integral type " << NKikimr::NUdf::GetDataTypeInfo(targetSlot).Name + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot make an integral type " << NKikimr::NUdf::GetDataTypeInfo(targetSlot).Name << " from type " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3824,8 +3824,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeOrAtomRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } EDataSlot targetType; @@ -3834,7 +3834,7 @@ namespace NTypeAnnImpl { const auto targetTypeStr = input->Child(1)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(targetTypeStr); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetTypeStr)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetTypeStr)); return IGraphTransformer::TStatus::Error; } @@ -3862,8 +3862,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = (Strong ? &EnsureTypeRewrite : &EnsureTypeOrAtomRewrite)(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = (Strong ? &EnsureTypeRewrite : &EnsureTypeOrAtomRewrite)(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } const TTypeAnnotationNode* targetType = nullptr; @@ -3872,7 +3872,7 @@ namespace NTypeAnnImpl { const auto targetTypeStr = input->Child(1)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(targetTypeStr); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetTypeStr)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetTypeStr)); return IGraphTransformer::TStatus::Error; } @@ -3950,12 +3950,12 @@ namespace NTypeAnnImpl { } if (sourceDataType->GetSlot() == EDataSlot::Yson && targetDataType->GetSlot() == EDataSlot::String) { - auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Consider using ToBytes to get internal representation of Yson type," - " or Yson::ConvertToString* methods to get string values"); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_CAST_YSON_JSON_BYTES, issue); - if (!ctx.Expr.AddWarning(issue)) { - return IGraphTransformer::TStatus::Error; - } + auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Consider using ToBytes to get internal representation of Yson type," + " or Yson::ConvertToString* methods to get string values"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_CAST_YSON_JSON_BYTES, issue); + if (!ctx.Expr.AddWarning(issue)) { + return IGraphTransformer::TStatus::Error; + } } auto fromFeatures = NUdf::GetDataTypeInfo(sourceDataType->GetSlot()).Features; @@ -3980,7 +3980,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot cast type " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot cast type " << *sourceType << " into " << *targetType)); return IGraphTransformer::TStatus::Error; @@ -4005,7 +4005,7 @@ namespace NTypeAnnImpl { const bool isDecimal = IsDataTypeDecimal(sourceDataType->GetSlot()); const bool isInterval = sourceDataType->GetSlot() == EDataSlot::Interval; if (!(isDecimal || IsDataTypeNumeric(sourceDataType->GetSlot()) || isInterval)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric type, but got " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected numeric type, but got " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -4047,7 +4047,7 @@ namespace NTypeAnnImpl { } if (!ctx.Types.DeprecatedSQL) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unsafe timestamp cast restricted from SQL v1.")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unsafe timestamp cast restricted from SQL v1.")); return IGraphTransformer::TStatus::Error; } @@ -4075,8 +4075,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4116,8 +4116,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4211,7 +4211,7 @@ namespace NTypeAnnImpl { const TTypeAnnotationNode* commonItemType = nullptr; if (SilentInferCommonType(convertedArg2, *rightItemType, convertedArg1, *leftItemType, ctx.Expr, commonItemType, TConvertFlags().Set(NConvertFlags::AllowUnsafeConvert)) == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "uncompatible coalesce types, first type: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "uncompatible coalesce types, first type: " << *leftType << ", second type: " << *rightType)); return IGraphTransformer::TStatus::Error; } @@ -4255,86 +4255,86 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus CoalesceMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto inputStruct = input->HeadPtr(); - if (IsNull(*inputStruct)) { - output = inputStruct; - return IGraphTransformer::TStatus::Repeat; - } - - const TStructExprType* structType; - if (inputStruct->GetTypeAnn() && inputStruct->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) { - auto itemType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType(); - if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - structType = itemType->Cast<TStructExprType>(); - } else { - if (!EnsureStructType(*inputStruct, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - structType = inputStruct->GetTypeAnn()->Cast<TStructExprType>(); - } - - auto coalesceColumns = input->ChildPtr(1); - if (!EnsureTuple(*coalesceColumns, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TExprNodeList realCoalesceColumns; - THashSet<TStringBuf> uniqColumns; - for (auto coalesceColumn : coalesceColumns->ChildrenList()) { - if (!EnsureAtom(*coalesceColumn, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto col = coalesceColumn->Content(); - if (structType->FindItem(col) && !uniqColumns.contains(col)) { - realCoalesceColumns.push_back(coalesceColumn); - uniqColumns.insert(col); - } - } - - if (realCoalesceColumns.size() < 2) { - output = inputStruct; - } else { - TExprNodeList coalesceArgs; - for (const auto& column : realCoalesceColumns) { - coalesceArgs.push_back( - ctx.Expr.Builder(input->Pos()) - .Callable("Member") - .Add(0, inputStruct) - .Add(1, column) - .Seal() - .Build() - ); - } - - output = ctx.Expr.Builder(input->Pos()) - .Callable("ReplaceMember") - .Add(0, inputStruct) - .Add(1, realCoalesceColumns[0]) - .Add(2, ctx.Expr.NewCallable(input->Pos(), "Coalesce", std::move(coalesceArgs))) - .Seal() - .Build(); - - for (size_t i = 1; i < realCoalesceColumns.size(); ++i) { - output = ctx.Expr.Builder(input->Pos()) - .Callable("RemoveMember") - .Add(0, output) - .Add(1, realCoalesceColumns[i]) - .Seal() - .Build(); - } - } - - return IGraphTransformer::TStatus::Repeat; - } - + IGraphTransformer::TStatus CoalesceMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto inputStruct = input->HeadPtr(); + if (IsNull(*inputStruct)) { + output = inputStruct; + return IGraphTransformer::TStatus::Repeat; + } + + const TStructExprType* structType; + if (inputStruct->GetTypeAnn() && inputStruct->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) { + auto itemType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType(); + if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + structType = itemType->Cast<TStructExprType>(); + } else { + if (!EnsureStructType(*inputStruct, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + structType = inputStruct->GetTypeAnn()->Cast<TStructExprType>(); + } + + auto coalesceColumns = input->ChildPtr(1); + if (!EnsureTuple(*coalesceColumns, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TExprNodeList realCoalesceColumns; + THashSet<TStringBuf> uniqColumns; + for (auto coalesceColumn : coalesceColumns->ChildrenList()) { + if (!EnsureAtom(*coalesceColumn, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto col = coalesceColumn->Content(); + if (structType->FindItem(col) && !uniqColumns.contains(col)) { + realCoalesceColumns.push_back(coalesceColumn); + uniqColumns.insert(col); + } + } + + if (realCoalesceColumns.size() < 2) { + output = inputStruct; + } else { + TExprNodeList coalesceArgs; + for (const auto& column : realCoalesceColumns) { + coalesceArgs.push_back( + ctx.Expr.Builder(input->Pos()) + .Callable("Member") + .Add(0, inputStruct) + .Add(1, column) + .Seal() + .Build() + ); + } + + output = ctx.Expr.Builder(input->Pos()) + .Callable("ReplaceMember") + .Add(0, inputStruct) + .Add(1, realCoalesceColumns[0]) + .Add(2, ctx.Expr.NewCallable(input->Pos(), "Coalesce", std::move(coalesceArgs))) + .Seal() + .Build(); + + for (size_t i = 1; i < realCoalesceColumns.size(); ++i) { + output = ctx.Expr.Builder(input->Pos()) + .Callable("RemoveMember") + .Add(0, output) + .Add(1, realCoalesceColumns[i]) + .Seal() + .Build(); + } + } + + return IGraphTransformer::TStatus::Repeat; + } + IGraphTransformer::TStatus NanvlWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -4740,18 +4740,18 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (type->GetKind() == ETypeAnnotationKind::Null) { - output = ctx.Expr.NewCallable(input->Pos(), "Null", {}); - return IGraphTransformer::TStatus::Repeat; - } - + if (type->GetKind() == ETypeAnnotationKind::Null) { + output = ctx.Expr.NewCallable(input->Pos(), "Null", {}); + return IGraphTransformer::TStatus::Repeat; + } + if (type->GetKind() != ETypeAnnotationKind::Optional) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected optional or Null type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected optional or Null type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -4766,8 +4766,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4818,13 +4818,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Variant) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Expected variant type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Expected variant type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -4833,7 +4833,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TTypeAnnotationNode* itemType; if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct) { auto structType = varType->GetUnderlyingType()->Cast<TStructExprType>(); - auto pos = FindOrReportMissingMember(input->Child(1)->Content(), input->Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(input->Child(1)->Content(), input->Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -4843,12 +4843,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto tupleType = varType->GetUnderlyingType()->Cast<TTupleExprType>(); ui32 index = 0; if (!TryFromString(input->Child(1)->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; } @@ -4960,8 +4960,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4986,7 +4986,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!IsSameAnnotation(*input->Child(i)->Child(1)->GetTypeAnn(), *payloadType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Child(1)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Child(1)->Pos()), TStringBuilder() << "Mismatch type of payload, expected: " << *payloadType << ", but got: " << *input->Child(i)->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -4997,11 +4997,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } template <bool ContainsOrLookup, bool InList = false> - IGraphTransformer::TStatus ContainsLookupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus ContainsLookupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if constexpr (ContainsOrLookup) { if (IsDataOrOptionalOfData(input->Head().GetTypeAnn())) { return WithWrapper(input, output, ctx); @@ -5011,19 +5011,19 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (IsNull(input->Head())) { output = ContainsOrLookup ? MakeBool(input->Pos(), false, ctx.Expr) : input->HeadPtr(); return IGraphTransformer::TStatus::Repeat; - } - + } + auto dictType = input->Head().GetTypeAnn(); - if (!dictType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected (optional) " << (InList ? "List" : "Dict") << " type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (dictType->GetKind() == ETypeAnnotationKind::Optional) { - dictType = dictType->Cast<TOptionalExprType>()->GetItemType(); - } - + if (!dictType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected (optional) " << (InList ? "List" : "Dict") << " type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (dictType->GetKind() == ETypeAnnotationKind::Optional) { + dictType = dictType->Cast<TOptionalExprType>()->GetItemType(); + } + if constexpr (InList) { if (dictType->GetKind() == ETypeAnnotationKind::EmptyList) { output = MakeBool(input->Pos(), false, ctx.Expr); @@ -5048,12 +5048,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!EnsureDictType(input->Head().Pos(), *dictType, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - } - + } + if (!EnsurePersistable(input->Tail(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - + const auto keyType = InList ? dictType->Cast<TListExprType>()->GetItemType() : dictType->Cast<TDictExprType>()->GetKeyType(); const auto payloadType = InList ? keyType : dictType->Cast<TDictExprType>()->GetPayloadType(); const auto lookupType = input->Tail().GetTypeAnn(); @@ -5063,156 +5063,156 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (NKikimr::NUdf::ECastOptions::Impossible & CastResult<true>(lookupType, keyType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Can't lookup " << *lookupType << " in set of " << *keyType)); - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + if constexpr (ContainsOrLookup) { - input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); - } else { - input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(payloadType)); - } - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus SqlInWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto collection = input->Child(0); - const auto lookup = input->Child(1); - const auto options = input->Child(2); - - if (!EnsureComputable(*collection, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsurePersistable(*lookup, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureTuple(*options, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - for (const auto& option : options->Children()) { - if (!EnsureTupleSize(*option, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(option->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto optionName = option->Head().Content(); - static const THashSet<TStringBuf> supportedOptions = - {"isCompact", "tableSource", "nullsProcessed", "ansi", "warnNoAnsi"}; - if (!supportedOptions.contains(optionName)) { - ctx.Expr.AddError( - TIssue(ctx.Expr.GetPosition(option->Pos()), TStringBuilder() << "Unknown IN option '" << optionName)); - return IGraphTransformer::TStatus::Error; - } - } - - const auto lookupType = lookup->GetTypeAnn(); - const bool isAnsi = HasSetting(*options, "ansi"); - - auto collectionType = collection->GetTypeAnn(); - if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { - collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); - } - const auto collectionKind = collectionType->GetKind(); - - auto issueWarningAndRestart = [&](const TString& warnPrefix) { - auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), - warnPrefix + " Consider adding 'PRAGMA AnsiInForEmptyOrNullableItemsCollections;'"); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE, issue); - if (!ctx.Expr.AddWarning(issue)) { - return IGraphTransformer::TStatus::Error; - } - output = ctx.Expr.Builder(input->Pos()) - .Callable("SqlIn") - .Add(0, collection) - .Add(1, lookup) - .Add(2, RemoveSetting(*options, "warnNoAnsi", ctx.Expr)) - .Seal() - .Build(); - return IGraphTransformer::TStatus::Repeat; - }; - - const TTypeAnnotationNode* collectionItemType = nullptr; - if (!HasSetting(*options, "tableSource")) { - switch (collectionKind) { - case ETypeAnnotationKind::Tuple: - { - const auto tupleType = collectionType->Cast<TTupleExprType>(); - for (auto itemType : tupleType->GetItems()) { - YQL_ENSURE(itemType); - if (ECompareOptions::Uncomparable == CanCompare<true>(lookupType, itemType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Can't compare " << *lookupType << " with " << *itemType)); - return IGraphTransformer::TStatus::Error; - } - } - break; - } - case ETypeAnnotationKind::Dict: - collectionItemType = collectionType->Cast<TDictExprType>()->GetKeyType(); - break; - case ETypeAnnotationKind::List: - collectionItemType = collectionType->Cast<TListExprType>()->GetItemType(); - break; - case ETypeAnnotationKind::EmptyDict: - case ETypeAnnotationKind::EmptyList: - case ETypeAnnotationKind::Null: - break; - default: - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(collection->Pos()), - TStringBuilder() << "IN only supports lookup in (optional) Tuple, List or Dict, " - << "but got: " << *collectionType)); - return IGraphTransformer::TStatus::Error; - } - } else { - const TTypeAnnotationNode* listItemType = (collectionKind == ETypeAnnotationKind::List) ? - collectionType->Cast<TListExprType>()->GetItemType() : nullptr; - if (!listItemType || - listItemType->GetKind() != ETypeAnnotationKind::Struct || - listItemType->Cast<TStructExprType>()->GetSize() != 1) - { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(collection->Pos()), - TStringBuilder() << "expecting single column table source, but got: " << *collectionType)); - return IGraphTransformer::TStatus::Error; - } - - collectionItemType = listItemType->Cast<TStructExprType>()->GetItems()[0]->GetItemType(); - } - + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); + } else { + input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(payloadType)); + } + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus SqlInWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto collection = input->Child(0); + const auto lookup = input->Child(1); + const auto options = input->Child(2); + + if (!EnsureComputable(*collection, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsurePersistable(*lookup, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureTuple(*options, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + for (const auto& option : options->Children()) { + if (!EnsureTupleSize(*option, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(option->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto optionName = option->Head().Content(); + static const THashSet<TStringBuf> supportedOptions = + {"isCompact", "tableSource", "nullsProcessed", "ansi", "warnNoAnsi"}; + if (!supportedOptions.contains(optionName)) { + ctx.Expr.AddError( + TIssue(ctx.Expr.GetPosition(option->Pos()), TStringBuilder() << "Unknown IN option '" << optionName)); + return IGraphTransformer::TStatus::Error; + } + } + + const auto lookupType = lookup->GetTypeAnn(); + const bool isAnsi = HasSetting(*options, "ansi"); + + auto collectionType = collection->GetTypeAnn(); + if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { + collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); + } + const auto collectionKind = collectionType->GetKind(); + + auto issueWarningAndRestart = [&](const TString& warnPrefix) { + auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), + warnPrefix + " Consider adding 'PRAGMA AnsiInForEmptyOrNullableItemsCollections;'"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE, issue); + if (!ctx.Expr.AddWarning(issue)) { + return IGraphTransformer::TStatus::Error; + } + output = ctx.Expr.Builder(input->Pos()) + .Callable("SqlIn") + .Add(0, collection) + .Add(1, lookup) + .Add(2, RemoveSetting(*options, "warnNoAnsi", ctx.Expr)) + .Seal() + .Build(); + return IGraphTransformer::TStatus::Repeat; + }; + + const TTypeAnnotationNode* collectionItemType = nullptr; + if (!HasSetting(*options, "tableSource")) { + switch (collectionKind) { + case ETypeAnnotationKind::Tuple: + { + const auto tupleType = collectionType->Cast<TTupleExprType>(); + for (auto itemType : tupleType->GetItems()) { + YQL_ENSURE(itemType); + if (ECompareOptions::Uncomparable == CanCompare<true>(lookupType, itemType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Can't compare " << *lookupType << " with " << *itemType)); + return IGraphTransformer::TStatus::Error; + } + } + break; + } + case ETypeAnnotationKind::Dict: + collectionItemType = collectionType->Cast<TDictExprType>()->GetKeyType(); + break; + case ETypeAnnotationKind::List: + collectionItemType = collectionType->Cast<TListExprType>()->GetItemType(); + break; + case ETypeAnnotationKind::EmptyDict: + case ETypeAnnotationKind::EmptyList: + case ETypeAnnotationKind::Null: + break; + default: + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(collection->Pos()), + TStringBuilder() << "IN only supports lookup in (optional) Tuple, List or Dict, " + << "but got: " << *collectionType)); + return IGraphTransformer::TStatus::Error; + } + } else { + const TTypeAnnotationNode* listItemType = (collectionKind == ETypeAnnotationKind::List) ? + collectionType->Cast<TListExprType>()->GetItemType() : nullptr; + if (!listItemType || + listItemType->GetKind() != ETypeAnnotationKind::Struct || + listItemType->Cast<TStructExprType>()->GetSize() != 1) + { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(collection->Pos()), + TStringBuilder() << "expecting single column table source, but got: " << *collectionType)); + return IGraphTransformer::TStatus::Error; + } + + collectionItemType = listItemType->Cast<TStructExprType>()->GetItems()[0]->GetItemType(); + } + if (collectionItemType && ECompareOptions::Uncomparable == CanCompare<true>(lookupType, collectionItemType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Can't lookup " << *lookupType << " in collection of " << *collectionItemType - << ": types " << *lookupType << " and " << *collectionItemType << " are not comparable")); - return IGraphTransformer::TStatus::Error; - } - - const bool collectionItemIsNullable = IsSqlInCollectionItemsNullable(NNodes::TCoSqlIn(input)); - if (!isAnsi && HasSetting(*options, "warnNoAnsi") && (collectionItemIsNullable || lookupType->HasOptionalOrNull())) { - return issueWarningAndRestart("IN may produce unexpected result when used with nullable arguments."); - } - - const TTypeAnnotationNode* resultType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool); - const bool collectionIsOptionalOrNull = collection->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional || - collection->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Null; - - if (collectionIsOptionalOrNull || lookupType->HasOptionalOrNull() || (isAnsi && collectionItemIsNullable)) { - resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType); - } - - input->SetTypeAnn(resultType); - return IGraphTransformer::TStatus::Ok; - } - + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Can't lookup " << *lookupType << " in collection of " << *collectionItemType + << ": types " << *lookupType << " and " << *collectionItemType << " are not comparable")); + return IGraphTransformer::TStatus::Error; + } + + const bool collectionItemIsNullable = IsSqlInCollectionItemsNullable(NNodes::TCoSqlIn(input)); + if (!isAnsi && HasSetting(*options, "warnNoAnsi") && (collectionItemIsNullable || lookupType->HasOptionalOrNull())) { + return issueWarningAndRestart("IN may produce unexpected result when used with nullable arguments."); + } + + const TTypeAnnotationNode* resultType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool); + const bool collectionIsOptionalOrNull = collection->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional || + collection->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Null; + + if (collectionIsOptionalOrNull || lookupType->HasOptionalOrNull() || (isAnsi && collectionItemIsNullable)) { + resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType); + } + + input->SetTypeAnn(resultType); + return IGraphTransformer::TStatus::Ok; + } + template <EDictItems Mode> IGraphTransformer::TStatus DictItemsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { @@ -5280,12 +5280,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } IGraphTransformer::TStatus AsStructWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (input->IsCallable("AsStructUnordered")) { - // obsolete callable - output = ctx.Expr.RenameNode(*input, "AsStruct"); - return IGraphTransformer::TStatus::Repeat; - } - + if (input->IsCallable("AsStructUnordered")) { + // obsolete callable + output = ctx.Expr.RenameNode(*input, "AsStruct"); + return IGraphTransformer::TStatus::Repeat; + } + TVector<const TItemExprType*> items; for (auto& child : input->Children()) { if (!EnsureTupleSize(*child, 2, ctx.Expr)) { @@ -5373,7 +5373,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (item1 != arg1) { // need update all previous items for (ui32 index = 0; index < i; ++index) { - input->ChildRef(index) = ctx.Expr.ReplaceNode(TExprNode::TPtr(item1), *arg1, input->ChildPtr(index)); + input->ChildRef(index) = ctx.Expr.ReplaceNode(TExprNode::TPtr(item1), *arg1, input->ChildPtr(index)); } return IGraphTransformer::TStatus::Repeat; @@ -5383,7 +5383,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } } - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Cannot infer common type for : " << *commonType << " and " << *child->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -5421,11 +5421,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto typeNode = input->Child(0); auto keysListNode = input->Child(1); - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } const auto keyType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (!EnsureComputableType(typeNode->Pos(), *keyType, ctx.Expr)) { + if (!EnsureComputableType(typeNode->Pos(), *keyType, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -5675,7 +5675,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const auto thenType = lambda->GetTypeAnn(); const auto elseType = input->Tail().GetTypeAnn(); if (!IsSameAnnotation(*thenType, *elseType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of then/else types, then type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of then/else types, then type: " << *thenType << ", else type: " << *elseType)); return IGraphTransformer::TStatus::Error; } @@ -5754,169 +5754,169 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus StaticZipWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const TStructExprType* argStruct = nullptr; - TVector<TStringBuf> argNames; - const TTupleExprType* argTuple = nullptr; - - auto getMemberNames = [](const TStructExprType& type) { - TVector<TStringBuf> result; - for (auto& item : type.GetItems()) { - result.push_back(item->GetName()); - } - Sort(result); - return result; - }; - - for (ui32 i = 0; i < input->ChildrenSize(); ++i) { - auto child = input->Child(i); - const TTypeAnnotationNode* childType = child->GetTypeAnn(); - if (HasError(childType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!childType) { - YQL_ENSURE(child->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected either struct or tuple, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (argStruct) { - if (childType->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), - TStringBuilder() << "Expected all arguments to be of Struct type, but got: " << *childType << " for " << i << "th argument")); - return IGraphTransformer::TStatus::Error; - } - auto childNames = getMemberNames(*childType->Cast<TStructExprType>()); - if (childNames != argNames) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), - TStringBuilder() << "Struct members mismatch. Members for first argument: " << JoinSeq(",", argNames) - << ". Members for " << i << "th argument: " << JoinSeq(", ", childNames))); - return IGraphTransformer::TStatus::Error; - } - } else if (argTuple) { - if (childType->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), - TStringBuilder() << "Expected all arguments to be of Tuple type, but got: " << *childType << " for " << i << "th argument")); - return IGraphTransformer::TStatus::Error; - } - auto childSize = childType->Cast<TTupleExprType>()->GetSize(); - if (childSize != argTuple->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), - TStringBuilder() << "Tuple size mismatch. Got " << argTuple->GetSize() - << " for first argument and " << childSize << " for " << i << "th argument")); - return IGraphTransformer::TStatus::Error; - } - } else if (childType->GetKind() == ETypeAnnotationKind::Struct) { - argStruct = childType->Cast<TStructExprType>(); - argNames = getMemberNames(*argStruct); - } else if (childType->GetKind() == ETypeAnnotationKind::Tuple) { - argTuple = childType->Cast<TTupleExprType>(); - } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected either struct or tuple, but got: " << *childType)); - return IGraphTransformer::TStatus::Error; - } - } - - if (argStruct) { - TExprNodeList asStructArgs; - for (auto& name : argNames) { - auto nameAtom = ctx.Expr.NewAtom(input->Pos(), name); - TExprNodeList zippedValues; - for (auto child : input->ChildrenList()) { - zippedValues.push_back(ctx.Expr.NewCallable(child->Pos(), "Member", { child , nameAtom })); - } - auto zipped = ctx.Expr.NewList(input->Pos(), std::move(zippedValues)); - asStructArgs.push_back(ctx.Expr.NewList(input->Pos(), { nameAtom, zipped })); - } - output = ctx.Expr.NewCallable(input->Pos(), "AsStruct", std::move(asStructArgs)); - } else { - YQL_ENSURE(argTuple); - TExprNodeList tupleArgs; - for (size_t i = 0; i < argTuple->GetSize(); ++i) { - auto idxAtom = ctx.Expr.NewAtom(input->Pos(), ToString(i), TNodeFlags::Default); - TExprNodeList zippedValues; - for (auto child : input->ChildrenList()) { - zippedValues.push_back(ctx.Expr.NewCallable(child->Pos(), "Nth", { child , idxAtom })); - } - auto zipped = ctx.Expr.NewList(input->Pos(), std::move(zippedValues)); - tupleArgs.push_back(zipped); - } - output = ctx.Expr.NewList(input->Pos(), std::move(tupleArgs)); - } - - return IGraphTransformer::TStatus::Repeat; - } - - IGraphTransformer::TStatus TryRemoveAllOptionalsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto value = input->HeadPtr(); - if (HasError(value->GetTypeAnn(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!value->GetTypeAnn()) { - YQL_ENSURE(value->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(value->Pos()), TStringBuilder() << "Expected either struct or tuple, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - bool isStruct = value->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct; - bool isTuple = value->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple; - if (!isStruct && !isTuple) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(value->Pos()), TStringBuilder() << "Expected either struct or tuple, but got: " - << *value->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - const TTypeAnnotationNode* resultType = nullptr; - if (isTuple) { + IGraphTransformer::TStatus StaticZipWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const TStructExprType* argStruct = nullptr; + TVector<TStringBuf> argNames; + const TTupleExprType* argTuple = nullptr; + + auto getMemberNames = [](const TStructExprType& type) { + TVector<TStringBuf> result; + for (auto& item : type.GetItems()) { + result.push_back(item->GetName()); + } + Sort(result); + return result; + }; + + for (ui32 i = 0; i < input->ChildrenSize(); ++i) { + auto child = input->Child(i); + const TTypeAnnotationNode* childType = child->GetTypeAnn(); + if (HasError(childType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!childType) { + YQL_ENSURE(child->Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected either struct or tuple, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (argStruct) { + if (childType->GetKind() != ETypeAnnotationKind::Struct) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), + TStringBuilder() << "Expected all arguments to be of Struct type, but got: " << *childType << " for " << i << "th argument")); + return IGraphTransformer::TStatus::Error; + } + auto childNames = getMemberNames(*childType->Cast<TStructExprType>()); + if (childNames != argNames) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), + TStringBuilder() << "Struct members mismatch. Members for first argument: " << JoinSeq(",", argNames) + << ". Members for " << i << "th argument: " << JoinSeq(", ", childNames))); + return IGraphTransformer::TStatus::Error; + } + } else if (argTuple) { + if (childType->GetKind() != ETypeAnnotationKind::Tuple) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), + TStringBuilder() << "Expected all arguments to be of Tuple type, but got: " << *childType << " for " << i << "th argument")); + return IGraphTransformer::TStatus::Error; + } + auto childSize = childType->Cast<TTupleExprType>()->GetSize(); + if (childSize != argTuple->GetSize()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), + TStringBuilder() << "Tuple size mismatch. Got " << argTuple->GetSize() + << " for first argument and " << childSize << " for " << i << "th argument")); + return IGraphTransformer::TStatus::Error; + } + } else if (childType->GetKind() == ETypeAnnotationKind::Struct) { + argStruct = childType->Cast<TStructExprType>(); + argNames = getMemberNames(*argStruct); + } else if (childType->GetKind() == ETypeAnnotationKind::Tuple) { + argTuple = childType->Cast<TTupleExprType>(); + } else { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected either struct or tuple, but got: " << *childType)); + return IGraphTransformer::TStatus::Error; + } + } + + if (argStruct) { + TExprNodeList asStructArgs; + for (auto& name : argNames) { + auto nameAtom = ctx.Expr.NewAtom(input->Pos(), name); + TExprNodeList zippedValues; + for (auto child : input->ChildrenList()) { + zippedValues.push_back(ctx.Expr.NewCallable(child->Pos(), "Member", { child , nameAtom })); + } + auto zipped = ctx.Expr.NewList(input->Pos(), std::move(zippedValues)); + asStructArgs.push_back(ctx.Expr.NewList(input->Pos(), { nameAtom, zipped })); + } + output = ctx.Expr.NewCallable(input->Pos(), "AsStruct", std::move(asStructArgs)); + } else { + YQL_ENSURE(argTuple); + TExprNodeList tupleArgs; + for (size_t i = 0; i < argTuple->GetSize(); ++i) { + auto idxAtom = ctx.Expr.NewAtom(input->Pos(), ToString(i), TNodeFlags::Default); + TExprNodeList zippedValues; + for (auto child : input->ChildrenList()) { + zippedValues.push_back(ctx.Expr.NewCallable(child->Pos(), "Nth", { child , idxAtom })); + } + auto zipped = ctx.Expr.NewList(input->Pos(), std::move(zippedValues)); + tupleArgs.push_back(zipped); + } + output = ctx.Expr.NewList(input->Pos(), std::move(tupleArgs)); + } + + return IGraphTransformer::TStatus::Repeat; + } + + IGraphTransformer::TStatus TryRemoveAllOptionalsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto value = input->HeadPtr(); + if (HasError(value->GetTypeAnn(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!value->GetTypeAnn()) { + YQL_ENSURE(value->Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(value->Pos()), TStringBuilder() << "Expected either struct or tuple, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + bool isStruct = value->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct; + bool isTuple = value->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple; + if (!isStruct && !isTuple) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(value->Pos()), TStringBuilder() << "Expected either struct or tuple, but got: " + << *value->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + const TTypeAnnotationNode* resultType = nullptr; + if (isTuple) { auto outputTypes = value->GetTypeAnn()->Cast<TTupleExprType>()->GetItems(); - for (auto& item : outputTypes) { - if (item->GetKind() == ETypeAnnotationKind::Optional) { - item = item->Cast<TOptionalExprType>()->GetItemType(); - } - } - - resultType = ctx.Expr.MakeType<TTupleExprType>(std::move(outputTypes)); + for (auto& item : outputTypes) { + if (item->GetKind() == ETypeAnnotationKind::Optional) { + item = item->Cast<TOptionalExprType>()->GetItemType(); + } + } + + resultType = ctx.Expr.MakeType<TTupleExprType>(std::move(outputTypes)); } else if (isStruct) { auto outputTypes = value->GetTypeAnn()->Cast<TStructExprType>()->GetItems(); - for (auto& item : outputTypes) { - if (item->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) { + for (auto& item : outputTypes) { + if (item->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) { item = ctx.Expr.MakeType<TItemExprType>(item->GetName(), item->GetItemType()->Cast<TOptionalExprType>()->GetItemType()); - } - } - - resultType = ctx.Expr.MakeType<TStructExprType>(std::move(outputTypes)); - } - + } + } + + resultType = ctx.Expr.MakeType<TStructExprType>(std::move(outputTypes)); + } + resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType); auto type = ExpandType(input->Pos(), *resultType, ctx.Expr); output = ctx.Expr.NewCallable(input->Pos(), "StrictCast", {std::move(value), std::move(type)}); return IGraphTransformer::TStatus::Repeat; - } - - IGraphTransformer::TStatus HasNullWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + } + + IGraphTransformer::TStatus HasNullWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureComputable(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus TypeOfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -5931,24 +5931,24 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus ConstraintsOfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + IGraphTransformer::TStatus ConstraintsOfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Json)); - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus InstanceOfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } input->SetTypeAnn(input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType()); @@ -5960,8 +5960,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } const auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -6055,8 +6055,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto param = input->HeadPtr(); @@ -6088,8 +6088,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } TString message; @@ -6126,7 +6126,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto targetType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (Strict) { if (!IsSameAnnotation(*valueType, *targetType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch types: " << *valueType << " != " << *targetType << message)); return IGraphTransformer::TStatus::Error; } @@ -6134,7 +6134,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto arg = ctx.Expr.NewArgument(input->Pos(), "arg"); auto status = TrySilentConvertTo(arg, *valueType, *targetType, ctx.Expr, TConvertFlags().Set(NConvertFlags::AllowUnsafeConvert)); if (status.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot convert type " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot convert type " << *valueType << " into " << *targetType << message)); return IGraphTransformer::TStatus::Error; } @@ -6155,7 +6155,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto nodePtr = input->HeadPtr(); if (!nodePtr->IsPersistable()) { - auto issue = TIssue(ctx.Expr.GetPosition(nodePtr->Pos()), "Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable"); + auto issue = TIssue(ctx.Expr.GetPosition(nodePtr->Pos()), "Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_NON_PERSISTABLE_ENTITY, issue); if (!ctx.Expr.AddWarning(issue)) { return IGraphTransformer::TStatus::Error; @@ -6206,7 +6206,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } ui32 tupleSize = 0; if (!TryFromString(input->Child(1)->Content(), tupleSize)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected number, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected number, but got: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -6357,7 +6357,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> TMaybe<bool> isHashed; TMaybe<ui64> itemsCount; bool isCompact; - TMaybe<TIssue> error = ParseToDictSettings(*input, ctx.Expr, isMany, isHashed, itemsCount, isCompact); + TMaybe<TIssue> error = ParseToDictSettings(*input, ctx.Expr, isMany, isHashed, itemsCount, isCompact); if (error) { ctx.Expr.AddError(*error); return IGraphTransformer::TStatus::Error; @@ -6516,13 +6516,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected error type, but got: " << *type)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected error type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -6545,7 +6545,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Child(1)->GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch types: " << *input->Head().GetTypeAnn() << " != " << *input->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -6570,7 +6570,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!IsSameAnnotation(*lambda->GetTypeAnn(), *innerType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch lambda return type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch lambda return type, " << *lambda->GetTypeAnn() << " != " << *innerType)); return IGraphTransformer::TStatus::Error; } @@ -6586,7 +6586,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const auto content = aliasNode.Content(); if (!ctx.Types.UserDataStorage->ContainsUserDataBlock(content)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "File not found: " << content)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "File not found: " << content)); return false; } return true; @@ -6599,7 +6599,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const auto content = aliasNode.Content(); if (!ctx.Types.UserDataStorage->ContainsUserDataFolder(content)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "Folder not found: " << content)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(aliasNode.Pos()), TStringBuilder() << "Folder not found: " << content)); return false; } @@ -6711,8 +6711,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TTypeAnnotationNode* userType = nullptr; if (input->ChildrenSize() > 2) { if (!input->Child(2)->IsCallable("Void")) { - if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } userType = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -6735,8 +6735,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> // (4) cached callable type const TCallableExprType* cachedType = nullptr; if (input->ChildrenSize() > 4) { - if (auto status = EnsureTypeRewrite(input->ChildRef(4), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(4), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(4)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -6750,8 +6750,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> // (5) cached run config type const TTypeAnnotationNode* cachedRunConfigType = nullptr; if (input->ChildrenSize() > 5) { - if (auto status = EnsureTypeRewrite(input->ChildRef(5), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(5), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } cachedRunConfigType = input->Child(5)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -6837,7 +6837,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } else if (cachedRunConfigType->GetKind() == ETypeAnnotationKind::Optional) { runConfigValue = ctx.Expr.NewCallable(input->Pos(), "Nothing", { runConfigTypeNode }); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Missing run config value for type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Missing run config value for type: " << *cachedRunConfigType << " in function " << name)); return IGraphTransformer::TStatus::Error; } @@ -6864,7 +6864,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto status = TryConvertTo(input->ChildRef(1), *cachedRunConfigType, ctx.Expr); if (status.Level != IGraphTransformer::TStatus::Ok) { if (status.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Mismatch type of run config in UDF function " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Mismatch type of run config in UDF function " << name << ", typeConfig:" << typeConfig)); } @@ -6915,8 +6915,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> // function type const TCallableExprType* callableType; { - if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } const TTypeAnnotationNode* tn = input->Child(2)->GetTypeAnn(); const TTypeExprType* type = tn->Cast<TTypeExprType>(); @@ -6938,7 +6938,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto moduleName = input->Head().Content(); auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName); if (scriptType == NKikimr::NMiniKQL::EScriptType::Unknown) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unknown script type: " << moduleName)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unknown script type: " << moduleName)); return IGraphTransformer::TStatus::Error; } @@ -6996,7 +6996,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> double val = 0.; if (!TryFromString(setting->Child(1)->Content(), val) || val == 0.) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() << "Bad " << TString{nameNode->Content()}.Quote() << " setting value: " << setting->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -7004,13 +7004,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> else if (nameNode->Content() == "extraMem") { ui64 val = 0; if (!TryFromString(setting->Child(1)->Content(), val)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() << "Bad " << TString{nameNode->Content()}.Quote() << " setting value: " << setting->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() << "Unsupported setting: " << nameNode->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() << "Unsupported setting: " << nameNode->Content())); return IGraphTransformer::TStatus::Error; } } @@ -7087,11 +7087,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> combinedStatus = combinedStatus.Combine(convertStatus); if (combinedStatus.Level == IGraphTransformer::TStatus::Error) { if (!srcType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type argument #" << i + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type argument #" << i << ", source type: lambda, target type: " << *arg.Type)); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type argument #" << i + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type argument #" << i << ", source type: " << *srcType << ", target type: " << *arg.Type)); } return IGraphTransformer::TStatus::Error; @@ -7174,13 +7174,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto structSize = structType->GetSize(); auto totalArgs = tupleSize + structSize; if (totalArgs > type->GetArgumentsSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Too many arguments, expected at most: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Too many arguments, expected at most: " << type->GetArgumentsSize() << ", but got: " << totalArgs)); return IGraphTransformer::TStatus::Error; } if (totalArgs < (type->GetArgumentsSize() - type->GetOptionalArgumentsCount())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Too few arguments, expected at least: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Too few arguments, expected at least: " << (type->GetArgumentsSize() - type->GetOptionalArgumentsCount()) << ", but got: " << totalArgs)); return IGraphTransformer::TStatus::Error; } @@ -7216,7 +7216,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (hasError) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Mismatch positional argument types, expected: " << *static_cast<const TTypeAnnotationNode*>(expectedTupleType) << ", but got: " << *input->Child(1)->GetTypeAnn())); @@ -7234,12 +7234,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> for (const auto& structItem : structType->GetItems()) { auto foundIndex = type->ArgumentIndexByName(structItem->GetName()); if (!foundIndex) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unknown argument name: " << structItem->GetName())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unknown argument name: " << structItem->GetName())); return IGraphTransformer::TStatus::Error; } if (*foundIndex < tupleSize) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument with name " << structItem->GetName() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument with name " << structItem->GetName() << " was already used for positional argument #" << (1 + *foundIndex))); return IGraphTransformer::TStatus::Error; } @@ -7254,11 +7254,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!usedIndices.contains(i)) { const auto& arg = type->GetArguments()[i]; if (arg.Name.empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument # " << (i + 1) + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument # " << (i + 1) << " is required, but has not been set")); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument '" << arg.Name + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Argument '" << arg.Name << "' is required, but has not been set")); } @@ -7296,7 +7296,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (hasError) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Mismatch named argument types, expected: " << *static_cast<const TTypeAnnotationNode*>(expectedStructType) << ", but got: " << *input->Child(2)->GetTypeAnn())); @@ -7400,126 +7400,126 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus PositionalArgsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus SqlCallWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 2, 4, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*input->Child(0), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto udfName = input->ChildPtr(0); - - if (!EnsureTupleMinSize(*input->Child(1), 1, ctx.Expr) || !EnsureTupleMaxSize(*input->Child(1), 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto& positionalArgsNode = input->Child(1)->Head(); - if (!EnsureCallable(positionalArgsNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!positionalArgsNode.IsCallable("PositionalArgs")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Head().Pos()), - TStringBuilder() << "Expecting PositionalArgs callable, but got: " << positionalArgsNode.Content())); - return IGraphTransformer::TStatus::Error; - } - - TExprNodeList positionalArgs = positionalArgsNode.ChildrenList(); - TExprNode::TPtr namedArgs; - if (input->Child(1)->ChildrenSize() == 2) { - namedArgs = input->Child(1)->TailPtr(); - if (!EnsureStructType(*namedArgs, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } - - TExprNode::TPtr customUserType; - if (input->ChildrenSize() > 2) { - if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - customUserType = input->ChildPtr(2); - } else { - customUserType = ctx.Expr.NewCallable(input->Pos(), "TupleType", {}); - } - - TExprNode::TPtr typeConfig; - if (input->ChildrenSize() == 4) { - typeConfig = input->TailPtr(); - if (!EnsureAtom(*typeConfig, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } - - TExprNode::TPtr udf = ctx.Expr.Builder(input->Pos()) - .Callable("Udf") - .Add(0, udfName) - .Callable(1, "Void").Seal() - .Callable(2, "TupleType") - .Callable(0, "TupleType") - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - size_t idx = 0; - for (auto& arg : positionalArgs) { - parent - .Callable(idx++, "TypeOf") - .Add(0, arg) - .Seal(); - } - return parent; - }) - .Seal() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (namedArgs) { - parent - .Callable(1, "TypeOf") - .Add(0, namedArgs) - .Seal(); - } else { - parent - .Callable(1, "StructType") - .Seal(); - } - return parent; - }) - .Add(2, customUserType) - .Seal() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (typeConfig) { - parent.Add(3, typeConfig); - } - return parent; - }) - .Seal() - .Build(); - - TExprNodeList applyArgs = { udf }; - if (namedArgs) { - applyArgs.push_back(ctx.Expr.NewList(input->Pos(), std::move(positionalArgs))); - applyArgs.push_back(namedArgs); - } else { - applyArgs.insert(applyArgs.end(), positionalArgs.begin(), positionalArgs.end()); - } - - output = ctx.Expr.NewCallable(input->Pos(), namedArgs ? "NamedApply" : "Apply", std::move(applyArgs)); - return IGraphTransformer::TStatus::Repeat; - } - + IGraphTransformer::TStatus PositionalArgsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus SqlCallWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 2, 4, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*input->Child(0), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto udfName = input->ChildPtr(0); + + if (!EnsureTupleMinSize(*input->Child(1), 1, ctx.Expr) || !EnsureTupleMaxSize(*input->Child(1), 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto& positionalArgsNode = input->Child(1)->Head(); + if (!EnsureCallable(positionalArgsNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!positionalArgsNode.IsCallable("PositionalArgs")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Head().Pos()), + TStringBuilder() << "Expecting PositionalArgs callable, but got: " << positionalArgsNode.Content())); + return IGraphTransformer::TStatus::Error; + } + + TExprNodeList positionalArgs = positionalArgsNode.ChildrenList(); + TExprNode::TPtr namedArgs; + if (input->Child(1)->ChildrenSize() == 2) { + namedArgs = input->Child(1)->TailPtr(); + if (!EnsureStructType(*namedArgs, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } + + TExprNode::TPtr customUserType; + if (input->ChildrenSize() > 2) { + if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + customUserType = input->ChildPtr(2); + } else { + customUserType = ctx.Expr.NewCallable(input->Pos(), "TupleType", {}); + } + + TExprNode::TPtr typeConfig; + if (input->ChildrenSize() == 4) { + typeConfig = input->TailPtr(); + if (!EnsureAtom(*typeConfig, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } + + TExprNode::TPtr udf = ctx.Expr.Builder(input->Pos()) + .Callable("Udf") + .Add(0, udfName) + .Callable(1, "Void").Seal() + .Callable(2, "TupleType") + .Callable(0, "TupleType") + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t idx = 0; + for (auto& arg : positionalArgs) { + parent + .Callable(idx++, "TypeOf") + .Add(0, arg) + .Seal(); + } + return parent; + }) + .Seal() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + if (namedArgs) { + parent + .Callable(1, "TypeOf") + .Add(0, namedArgs) + .Seal(); + } else { + parent + .Callable(1, "StructType") + .Seal(); + } + return parent; + }) + .Add(2, customUserType) + .Seal() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + if (typeConfig) { + parent.Add(3, typeConfig); + } + return parent; + }) + .Seal() + .Build(); + + TExprNodeList applyArgs = { udf }; + if (namedArgs) { + applyArgs.push_back(ctx.Expr.NewList(input->Pos(), std::move(positionalArgs))); + applyArgs.push_back(namedArgs); + } else { + applyArgs.insert(applyArgs.end(), positionalArgs.begin(), positionalArgs.end()); + } + + output = ctx.Expr.NewCallable(input->Pos(), namedArgs ? "NamedApply" : "Apply", std::move(applyArgs)); + return IGraphTransformer::TStatus::Repeat; + } + IGraphTransformer::TStatus CallableWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -7555,7 +7555,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!IsSameAnnotation(*lambda->GetTypeAnn(), *callableType->GetReturnType())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type: " << *lambda->GetTypeAnn() << " != " << *callableType->GetReturnType())); return IGraphTransformer::TStatus::Error; } @@ -7612,13 +7612,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected struct type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected struct type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -7628,14 +7628,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> for (auto item : newStructType->GetItems()) { auto oldItem = oldStructType->FindItem(item->GetName()); if (!oldItem) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to find member with name: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to find member with name: " << item->GetName() << " in struct " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } auto oldItemType = oldStructType->GetItems()[oldItem.GetRef()]->GetItemType(); if (!IsSameAnnotation(*item->GetItemType(), *oldItemType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert member with name: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert member with name: " << item->GetName() << " from type " << *oldItemType << " to type " << *item->GetItemType())); return IGraphTransformer::TStatus::Error; } @@ -7676,12 +7676,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto tupleType = variantType->GetUnderlyingType()->Cast<TTupleExprType>(); ui32 index = 0; if (!TryFromString(input->Child(1)->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; } @@ -7689,7 +7689,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> input->SetTypeAnn(tupleType->GetItems()[index]); } else { auto structType = variantType->GetUnderlyingType()->Cast<TStructExprType>(); - auto pos = FindOrReportMissingMember(input->Child(1)->Content(), input->Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(input->Child(1)->Content(), input->Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -7731,7 +7731,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto firstType = tupleType->GetItems()[0]; for (size_t i = 1; i < tupleType->GetSize(); ++i) { if (firstType != tupleType->GetItems()[i]) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *tupleType->GetItems()[i]))); return IGraphTransformer::TStatus::Error; } @@ -7743,7 +7743,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto firstType = structType->GetItems()[0]->GetItemType(); for (size_t i = 1; i < structType->GetSize(); ++i) { if (firstType != structType->GetItems()[i]->GetItemType()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *structType->GetItems()[i]->GetItemType()))); return IGraphTransformer::TStatus::Error; } @@ -7794,12 +7794,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (tupleType) { ui32 index = 0; if (!TryFromString(child->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; @@ -7808,7 +7808,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> itemType = tupleType->GetItems()[index]; itemIndex = index; } else { - auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -7819,10 +7819,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (usedFields[itemIndex]) { if (tupleType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Position " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Position " << itemIndex << " was already used")); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Member " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Member " << structType->GetItems()[itemIndex]->GetName() << " was already used")); } @@ -7835,7 +7835,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> ++index; if (index == input->ChildrenSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Expected lambda after this argument")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Expected lambda after this argument")); return IGraphTransformer::TStatus::Error; } @@ -7857,7 +7857,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> currentType = lambda->GetTypeAnn(); } else { if (index != input->ChildrenSize() - 1) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Default value should be in the end")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Default value should be in the end")); return IGraphTransformer::TStatus::Error; } @@ -7874,7 +7874,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> resultType = currentType; } else { if (!IsSameAnnotation(*resultType, *currentType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "mismatch of handler/default types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "mismatch of handler/default types: " << *currentType << " != " << *resultType)); return IGraphTransformer::TStatus::Error; } @@ -7883,7 +7883,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!hasDefaultValue && usedCount != usedFields.size()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Not all alternatives are handled, total: " << usedFields.size() << ", handled: " << usedCount)); return IGraphTransformer::TStatus::Error; @@ -8082,7 +8082,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> else if (unpacked->GetKind() == ETypeAnnotationKind::Variant) { output = ctx.Expr.NewCallable(input->Pos(), "Guess", { input->ChildPtr(1), input->ChildPtr(2) }); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Expected (optional) tuple or variant based on it, but got: " << *input->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -8097,7 +8097,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> else if (unpacked->GetKind() == ETypeAnnotationKind::Variant) { output = ctx.Expr.NewCallable(input->Pos(), "Guess", { input->ChildPtr(1), input->ChildPtr(2) }); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected (optional) struct/tuple or variant based on it, but got: " << *input->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -8112,7 +8112,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } else if (unpacked->GetKind() == ETypeAnnotationKind::EmptyList || unpacked->GetKind() == ETypeAnnotationKind::EmptyDict) { output = ctx.Expr.NewCallable(input->Pos(), "Null", {}); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected (optional) list or dict, but got: " << *input->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -8136,7 +8136,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> size_t listArg = 0; if (!TryFromString<size_t>(input->Child(lastPos)->Content(), listArg) || listArg >= input->ChildrenSize() - 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(lastPos)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(lastPos)->Pos()), TStringBuilder() << "Invalid value of list argument position: " << input->Child(lastPos)->Content())); return IGraphTransformer::TStatus::Error; } @@ -8148,7 +8148,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TCallableExprType* callableType = input->Head().GetTypeAnn()->Cast<TCallableExprType>(); if (applyChildren.size() < callableType->GetArgumentsSize() + 1 - callableType->GetOptionalArgumentsCount()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Invalid number of arguments " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Invalid number of arguments " << (applyChildren.size() - 1) << " to use with callable type " << FormatType(callableType))); return IGraphTransformer::TStatus::Error; } @@ -8194,7 +8194,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto lambda = input->HeadPtr(); const auto args = lambda->Child(0); if (input->ChildrenSize() - 2 != args->ChildrenSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Different arguments count, lambda has " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Different arguments count, lambda has " << args->ChildrenSize() << " arguments, but provided " << (input->ChildrenSize() - 2))); return IGraphTransformer::TStatus::Error; } @@ -8231,109 +8231,109 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - TPositionHandle pos = input->Pos(); - - TExprNode::TPtr extractKeyLambda = input->ChildPtr(1); - TExprNode::TPtr udf = input->ChildPtr(2); - TExprNode::TPtr udfInput = input->ChildPtr(3); - - if (udf->IsCallable("SqlReduceUdf")) { - const TTypeAnnotationNode* positionalArgsUdfType = nullptr; - if (extractKeyLambda->IsAtom()) { - if (!udfInput->GetTypeAnn()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), TStringBuilder() << "Lambda is not expected as last argument of " << input->Content())); - return IGraphTransformer::TStatus::Error; - } - positionalArgsUdfType = ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{udfInput->GetTypeAnn()}); - } else { - const TTypeAnnotationNode* itemType = nullptr; - if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &itemType)) { - return IGraphTransformer::TStatus::Error; - } - - auto& keyExtractor = input->ChildRef(1U); - auto& udfInputLambda = input->ChildRef(3U); - - auto status = ConvertToLambda(keyExtractor, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(udfInputLambda, ctx.Expr, 1)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(keyExtractor, { itemType }, ctx.Expr) || - !UpdateLambdaAllArgumentsTypes(udfInputLambda, { itemType }, ctx.Expr)) - { - return IGraphTransformer::TStatus::Error; - } - - if (!keyExtractor->GetTypeAnn() || !udfInputLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - positionalArgsUdfType = ctx.Expr.MakeType<TTupleExprType>( - TTypeAnnotationNode::TListType{ keyExtractor->GetTypeAnn(), ctx.Expr.MakeType<TStreamExprType>(udfInputLambda->GetTypeAnn()) } - ); - } - - TExprNodeList udfArgs; - // name - udfArgs.push_back(udf->HeadPtr()); - // runConfig - udfArgs.push_back(ctx.Expr.NewCallable(pos, "Void", {})); - // userType - udfArgs.push_back( - ctx.Expr.Builder(pos) - .Callable("TupleType") - .Add(0, ExpandType(pos, *positionalArgsUdfType, ctx.Expr)) - .Callable(1, "StructType") - .Seal() - .Add(2, udf->ChildPtr(1)) - .Seal() - .Build() - ); - - if (udf->ChildrenSize() == 3) { - // typeConfig - udfArgs.push_back(udf->TailPtr()); - } - output = ctx.Expr.ChangeChild(*input, 2, ctx.Expr.NewCallable(pos, "Udf", std::move(udfArgs))); - return IGraphTransformer::TStatus::Repeat; - } - - if (extractKeyLambda->IsAtom()) { - TExprNode::TPtr applied; - if (udf->IsLambda()) { - applied = ctx.Expr.Builder(pos) - .Apply(udf) - .With(0, udfInput) - .Seal() - .Build(); - } else { - applied = ctx.Expr.Builder(pos) - .Callable("Apply") - .Add(0, udf) - .Add(1, udfInput) - .Seal() - .Build(); - } - if (extractKeyLambda->Content() == "byAll") { - output = ctx.Expr.NewCallable(pos, "ToSequence", { applied }); - } else if (extractKeyLambda->Content() == "byAllList") { - output = ctx.Expr.Builder(pos) - .Callable("ForwardList") - .Callable(0, "ToStream") - .Add(0, applied) - .Seal() - .Seal() - .Build(); - } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), TStringBuilder() << "Expected 'byAll' ot 'byAllList' as second argument")); - return IGraphTransformer::TStatus::Error; - } - - return IGraphTransformer::TStatus::Repeat; - } - + TPositionHandle pos = input->Pos(); + + TExprNode::TPtr extractKeyLambda = input->ChildPtr(1); + TExprNode::TPtr udf = input->ChildPtr(2); + TExprNode::TPtr udfInput = input->ChildPtr(3); + + if (udf->IsCallable("SqlReduceUdf")) { + const TTypeAnnotationNode* positionalArgsUdfType = nullptr; + if (extractKeyLambda->IsAtom()) { + if (!udfInput->GetTypeAnn()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), TStringBuilder() << "Lambda is not expected as last argument of " << input->Content())); + return IGraphTransformer::TStatus::Error; + } + positionalArgsUdfType = ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{udfInput->GetTypeAnn()}); + } else { + const TTypeAnnotationNode* itemType = nullptr; + if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &itemType)) { + return IGraphTransformer::TStatus::Error; + } + + auto& keyExtractor = input->ChildRef(1U); + auto& udfInputLambda = input->ChildRef(3U); + + auto status = ConvertToLambda(keyExtractor, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(udfInputLambda, ctx.Expr, 1)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(keyExtractor, { itemType }, ctx.Expr) || + !UpdateLambdaAllArgumentsTypes(udfInputLambda, { itemType }, ctx.Expr)) + { + return IGraphTransformer::TStatus::Error; + } + + if (!keyExtractor->GetTypeAnn() || !udfInputLambda->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + positionalArgsUdfType = ctx.Expr.MakeType<TTupleExprType>( + TTypeAnnotationNode::TListType{ keyExtractor->GetTypeAnn(), ctx.Expr.MakeType<TStreamExprType>(udfInputLambda->GetTypeAnn()) } + ); + } + + TExprNodeList udfArgs; + // name + udfArgs.push_back(udf->HeadPtr()); + // runConfig + udfArgs.push_back(ctx.Expr.NewCallable(pos, "Void", {})); + // userType + udfArgs.push_back( + ctx.Expr.Builder(pos) + .Callable("TupleType") + .Add(0, ExpandType(pos, *positionalArgsUdfType, ctx.Expr)) + .Callable(1, "StructType") + .Seal() + .Add(2, udf->ChildPtr(1)) + .Seal() + .Build() + ); + + if (udf->ChildrenSize() == 3) { + // typeConfig + udfArgs.push_back(udf->TailPtr()); + } + output = ctx.Expr.ChangeChild(*input, 2, ctx.Expr.NewCallable(pos, "Udf", std::move(udfArgs))); + return IGraphTransformer::TStatus::Repeat; + } + + if (extractKeyLambda->IsAtom()) { + TExprNode::TPtr applied; + if (udf->IsLambda()) { + applied = ctx.Expr.Builder(pos) + .Apply(udf) + .With(0, udfInput) + .Seal() + .Build(); + } else { + applied = ctx.Expr.Builder(pos) + .Callable("Apply") + .Add(0, udf) + .Add(1, udfInput) + .Seal() + .Build(); + } + if (extractKeyLambda->Content() == "byAll") { + output = ctx.Expr.NewCallable(pos, "ToSequence", { applied }); + } else if (extractKeyLambda->Content() == "byAllList") { + output = ctx.Expr.Builder(pos) + .Callable("ForwardList") + .Callable(0, "ToStream") + .Add(0, applied) + .Seal() + .Seal() + .Build(); + } else { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), TStringBuilder() << "Expected 'byAll' ot 'byAllList' as second argument")); + return IGraphTransformer::TStatus::Error; + } + + return IGraphTransformer::TStatus::Repeat; + } + if (!EnsureListType(input->Head(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -8344,7 +8344,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const auto callableType = input->Child(2)->GetTypeAnn()->Cast<TCallableExprType>(); if (callableType->GetArgumentsSize() != 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected callable with 2 arguments")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected callable with 2 arguments")); return IGraphTransformer::TStatus::Error; } @@ -8666,88 +8666,88 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Repeat; } - IGraphTransformer::TStatus SqlReduceUdfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureMinMaxArgsCount(*input, 2, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (input->ChildrenSize() == 3 && !EnsureAtom(input->Tail(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus SqlProjectWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (IsEmptyList(input->Head())) { - output = input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* itemType = nullptr; - if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &itemType)) { - return IGraphTransformer::TStatus::Error; - } - - if (!itemType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected Struct as a sequence item type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - - if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureTupleMinSize(*input->Child(1), 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TVector<const TItemExprType*> allItems; - for (auto& item : input->Child(1)->Children()) { - if (!item->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(item->Pos()), - TStringBuilder() << "Expected SqlProjectItem or SqlProjectStarItem as argument")); - return IGraphTransformer::TStatus::Error; - } - - if (item->IsCallable("SqlProjectStarItem")) { - auto& structItems = item->GetTypeAnn()->Cast<TStructExprType>()->GetItems(); - allItems.insert(allItems.end(), structItems.begin(), structItems.end()); - } else { - YQL_ENSURE(item->Child(1)->IsAtom()); - allItems.push_back(ctx.Expr.MakeType<TItemExprType>(item->Child(1)->Content(), item->GetTypeAnn())); - } - } - - auto resultStructType = ctx.Expr.MakeType<TStructExprType>(allItems); - if (!resultStructType->Validate(input->Pos(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto resultType = MakeSequenceType(input->Child(0)->GetTypeAnn()->GetKind(), - static_cast<const TTypeAnnotationNode&>(*resultStructType), ctx.Expr); - - input->SetTypeAnn(resultType); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus SqlReduceUdfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureMinMaxArgsCount(*input, 2, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (input->ChildrenSize() == 3 && !EnsureAtom(input->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus SqlProjectWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (IsEmptyList(input->Head())) { + output = input->HeadPtr(); + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* itemType = nullptr; + if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &itemType)) { + return IGraphTransformer::TStatus::Error; + } + + if (!itemType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected Struct as a sequence item type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + + if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureTupleMinSize(*input->Child(1), 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TVector<const TItemExprType*> allItems; + for (auto& item : input->Child(1)->Children()) { + if (!item->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(item->Pos()), + TStringBuilder() << "Expected SqlProjectItem or SqlProjectStarItem as argument")); + return IGraphTransformer::TStatus::Error; + } + + if (item->IsCallable("SqlProjectStarItem")) { + auto& structItems = item->GetTypeAnn()->Cast<TStructExprType>()->GetItems(); + allItems.insert(allItems.end(), structItems.begin(), structItems.end()); + } else { + YQL_ENSURE(item->Child(1)->IsAtom()); + allItems.push_back(ctx.Expr.MakeType<TItemExprType>(item->Child(1)->Content(), item->GetTypeAnn())); + } + } + + auto resultStructType = ctx.Expr.MakeType<TStructExprType>(allItems); + if (!resultStructType->Validate(input->Pos(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto resultType = MakeSequenceType(input->Child(0)->GetTypeAnn()->GetKind(), + static_cast<const TTypeAnnotationNode&>(*resultStructType), ctx.Expr); + + input->SetTypeAnn(resultType); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus PgStarWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { @@ -9077,8 +9077,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> bool hasType = false; if (!input->Child(1)->IsCallable("Void")) { hasType = true; - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } } @@ -9114,8 +9114,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> bool hasType = false; if (!input->Child(0)->IsCallable("Void")) { hasType = true; - if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } } @@ -9161,8 +9161,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> bool hasType = false; if (!input->Child(0)->IsCallable("Void")) { hasType = true; - if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } } @@ -10618,190 +10618,190 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return ctx.Types.SetColumnOrder(*input, resultColumnOrder, ctx.Expr); } - IGraphTransformer::TStatus SqlProjectItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - YQL_ENSURE(input->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})); - const bool isStar = input->IsCallable("SqlProjectStarItem"); - if (!EnsureMinMaxArgsCount(*input, 3, 4, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (input->ChildrenSize() == 4) { - TExprNode::TPtr normalized; - auto status = NormalizeKeyValueTuples(input->ChildPtr(3), 0, normalized, ctx.Expr); - if (status.Level == IGraphTransformer::TStatus::Repeat) { - output = ctx.Expr.ChangeChild(*input, 3, std::move(normalized)); - return status; - } - - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - } - - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - const auto seqType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (seqType->GetKind() == ETypeAnnotationKind::EmptyList) { - input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - const TTypeAnnotationNode* itemType = nullptr; - if (!EnsureNewSeqType<false>(input->Head().Pos(), *seqType, ctx.Expr, &itemType)) { - return IGraphTransformer::TStatus::Error; - } - - YQL_ENSURE(itemType); - if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*input->Child(1), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (input->ChildrenSize() == 4) { - // validate options - THashSet<TStringBuf> seenOptions; - for (auto& optionNode : input->Child(3)->ChildrenList()) { - if (!EnsureTupleMinSize(*optionNode, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TStringBuf name = optionNode->Child(0)->Content(); - if (!seenOptions.insert(name).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), - TStringBuilder() << "Duplicate option " << name)); - return IGraphTransformer::TStatus::Error; - } - - if (isStar && name == "divePrefix") { - if (!EnsureTupleSize(*optionNode, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureTupleOfAtoms(*optionNode->Child(1), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - THashSet<TStringBuf> prefixes; - for (auto& prefix : optionNode->Child(1)->ChildrenList()) { - if (!prefixes.insert(prefix->Content()).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), - TStringBuilder() << "Duplicate prefix " << prefix->Content())); - return IGraphTransformer::TStatus::Error; - } - } - } else if (isStar && name == "addPrefix") { - if (!EnsureTupleSize(*optionNode, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureAtom(*optionNode->Child(1), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } else if (!isStar && name == "warnShadow") { - // no params - if (!EnsureTupleSize(*optionNode, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), - TStringBuilder() << "Unknown option: " << name)); - return IGraphTransformer::TStatus::Error; - } - } - - if (isStar && seenOptions.size() > 1) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(3)->Pos()), - TStringBuilder() << "Options addPrefix and divePrefix cannot be used at the same time")); - return IGraphTransformer::TStatus::Error; - } - - if (seenOptions.contains("warnShadow")) { - auto alias = input->Child(1)->Content(); - if (itemType->Cast<TStructExprType>()->FindItem(alias)) { - auto issue = TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), - TStringBuilder() << "Alias `" << alias << "` shadows column with the same name. It looks like comma is missed here. " - "If not, it is recommended to use ... AS `" << alias << "` to avoid confusion"); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_ALIAS_SHADOWS_COLUMN, issue); - if (!ctx.Expr.AddWarning(issue)) { - return IGraphTransformer::TStatus::Error; - } - } - auto newChildren = input->ChildrenList(); - // drop options - newChildren.pop_back(); - output = ctx.Expr.ChangeChildren(*input, std::move(newChildren)); - return IGraphTransformer::TStatus::Repeat; - } - } - - auto& lambda = input->ChildRef(2); - auto convertStatus = ConvertToLambda(lambda, ctx.Expr, 1); - if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { - return convertStatus; - } - - if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto lambdaResult = lambda->GetTypeAnn(); - if (!lambdaResult) { - return IGraphTransformer::TStatus::Repeat; - } - - if (isStar && !EnsureStructType(lambda->Pos(), *lambdaResult, ctx.Expr)) { - // lambda should return struct - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(lambdaResult); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus SqlTypeFromYsonWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto type = NCommon::ParseTypeFromYson(input->Head().Content(), ctx.Expr, ctx.Expr.GetPosition(input->Pos())); - if (!type) { - return IGraphTransformer::TStatus::Error; - } - - output = ExpandType(input->Pos(), *type, ctx.Expr); - return IGraphTransformer::TStatus::Repeat; - } - - IGraphTransformer::TStatus SqlColumnOrderFromYsonWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TVector<TString> topLevelColumns; - auto type = NCommon::ParseOrderAwareTypeFromYson(input->Head().Content(), topLevelColumns, ctx.Expr, ctx.Expr.GetPosition(input->Pos())); - if (!type) { - return IGraphTransformer::TStatus::Error; - } - - TExprNodeList items; - for (auto& col : topLevelColumns) { - items.push_back(ctx.Expr.NewAtom(input->Pos(), col)); - } - - output = ctx.Expr.NewList(input->Pos(), std::move(items)); - return IGraphTransformer::TStatus::Repeat; - } - + IGraphTransformer::TStatus SqlProjectItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + YQL_ENSURE(input->IsCallable({"SqlProjectItem", "SqlProjectStarItem"})); + const bool isStar = input->IsCallable("SqlProjectStarItem"); + if (!EnsureMinMaxArgsCount(*input, 3, 4, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (input->ChildrenSize() == 4) { + TExprNode::TPtr normalized; + auto status = NormalizeKeyValueTuples(input->ChildPtr(3), 0, normalized, ctx.Expr); + if (status.Level == IGraphTransformer::TStatus::Repeat) { + output = ctx.Expr.ChangeChild(*input, 3, std::move(normalized)); + return status; + } + + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + } + + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + const auto seqType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (seqType->GetKind() == ETypeAnnotationKind::EmptyList) { + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + const TTypeAnnotationNode* itemType = nullptr; + if (!EnsureNewSeqType<false>(input->Head().Pos(), *seqType, ctx.Expr, &itemType)) { + return IGraphTransformer::TStatus::Error; + } + + YQL_ENSURE(itemType); + if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*input->Child(1), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (input->ChildrenSize() == 4) { + // validate options + THashSet<TStringBuf> seenOptions; + for (auto& optionNode : input->Child(3)->ChildrenList()) { + if (!EnsureTupleMinSize(*optionNode, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TStringBuf name = optionNode->Child(0)->Content(); + if (!seenOptions.insert(name).second) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), + TStringBuilder() << "Duplicate option " << name)); + return IGraphTransformer::TStatus::Error; + } + + if (isStar && name == "divePrefix") { + if (!EnsureTupleSize(*optionNode, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureTupleOfAtoms(*optionNode->Child(1), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + THashSet<TStringBuf> prefixes; + for (auto& prefix : optionNode->Child(1)->ChildrenList()) { + if (!prefixes.insert(prefix->Content()).second) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), + TStringBuilder() << "Duplicate prefix " << prefix->Content())); + return IGraphTransformer::TStatus::Error; + } + } + } else if (isStar && name == "addPrefix") { + if (!EnsureTupleSize(*optionNode, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureAtom(*optionNode->Child(1), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } else if (!isStar && name == "warnShadow") { + // no params + if (!EnsureTupleSize(*optionNode, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } else { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(optionNode->Pos()), + TStringBuilder() << "Unknown option: " << name)); + return IGraphTransformer::TStatus::Error; + } + } + + if (isStar && seenOptions.size() > 1) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(3)->Pos()), + TStringBuilder() << "Options addPrefix and divePrefix cannot be used at the same time")); + return IGraphTransformer::TStatus::Error; + } + + if (seenOptions.contains("warnShadow")) { + auto alias = input->Child(1)->Content(); + if (itemType->Cast<TStructExprType>()->FindItem(alias)) { + auto issue = TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), + TStringBuilder() << "Alias `" << alias << "` shadows column with the same name. It looks like comma is missed here. " + "If not, it is recommended to use ... AS `" << alias << "` to avoid confusion"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_ALIAS_SHADOWS_COLUMN, issue); + if (!ctx.Expr.AddWarning(issue)) { + return IGraphTransformer::TStatus::Error; + } + } + auto newChildren = input->ChildrenList(); + // drop options + newChildren.pop_back(); + output = ctx.Expr.ChangeChildren(*input, std::move(newChildren)); + return IGraphTransformer::TStatus::Repeat; + } + } + + auto& lambda = input->ChildRef(2); + auto convertStatus = ConvertToLambda(lambda, ctx.Expr, 1); + if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { + return convertStatus; + } + + if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto lambdaResult = lambda->GetTypeAnn(); + if (!lambdaResult) { + return IGraphTransformer::TStatus::Repeat; + } + + if (isStar && !EnsureStructType(lambda->Pos(), *lambdaResult, ctx.Expr)) { + // lambda should return struct + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(lambdaResult); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus SqlTypeFromYsonWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto type = NCommon::ParseTypeFromYson(input->Head().Content(), ctx.Expr, ctx.Expr.GetPosition(input->Pos())); + if (!type) { + return IGraphTransformer::TStatus::Error; + } + + output = ExpandType(input->Pos(), *type, ctx.Expr); + return IGraphTransformer::TStatus::Repeat; + } + + IGraphTransformer::TStatus SqlColumnOrderFromYsonWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TVector<TString> topLevelColumns; + auto type = NCommon::ParseOrderAwareTypeFromYson(input->Head().Content(), topLevelColumns, ctx.Expr, ctx.Expr.GetPosition(input->Pos())); + if (!type) { + return IGraphTransformer::TStatus::Error; + } + + TExprNodeList items; + for (auto& col : topLevelColumns) { + items.push_back(ctx.Expr.NewAtom(input->Pos(), col)); + } + + output = ctx.Expr.NewList(input->Pos(), std::move(items)); + return IGraphTransformer::TStatus::Repeat; + } + IGraphTransformer::TStatus AutoDemuxListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -10866,43 +10866,43 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!EnsureDependsOnTail(*input, ctx.Expr, 3)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } auto typeArg = input->Child(0); - - const auto queueType = typeArg->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (!queueType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(typeArg->Pos()), TStringBuilder() << "Expecting computable type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureComputableType(typeArg->Pos(), *queueType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + + const auto queueType = typeArg->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (!queueType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(typeArg->Pos()), TStringBuilder() << "Expecting computable type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureComputableType(typeArg->Pos(), *queueType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto& capacityArg = input->ChildRef(1); auto& initSizeArg = input->ChildRef(2); - - if (!capacityArg->IsCallable("Uint64") && !capacityArg->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(capacityArg->Pos()), TStringBuilder() << "Queue capacity should be Uint64 literal or Void")); - return IGraphTransformer::TStatus::Error; - } - - if (!initSizeArg->IsCallable("Uint64")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(initSizeArg->Pos()), TStringBuilder() << "Queue initial size should be Uint64 literal")); - return IGraphTransformer::TStatus::Error; - } - - if (capacityArg->IsCallable("Uint64")) { - auto capacity = FromString<ui64>(capacityArg->Child(0)->Content()); - auto initSize = FromString<ui64>(initSizeArg->Child(0)->Content()); - if (initSize > capacity) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Queue initial size should not be bigger than capacity")); - return IGraphTransformer::TStatus::Error; - } - } - + + if (!capacityArg->IsCallable("Uint64") && !capacityArg->IsCallable("Void")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(capacityArg->Pos()), TStringBuilder() << "Queue capacity should be Uint64 literal or Void")); + return IGraphTransformer::TStatus::Error; + } + + if (!initSizeArg->IsCallable("Uint64")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(initSizeArg->Pos()), TStringBuilder() << "Queue initial size should be Uint64 literal")); + return IGraphTransformer::TStatus::Error; + } + + if (capacityArg->IsCallable("Uint64")) { + auto capacity = FromString<ui64>(capacityArg->Child(0)->Content()); + auto initSize = FromString<ui64>(initSizeArg->Child(0)->Content()); + if (initSize > capacity) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Queue initial size should not be bigger than capacity")); + return IGraphTransformer::TStatus::Error; + } + } + input->SetTypeAnn(ctx.Expr.MakeType<TResourceExprType>(TStringBuilder() << NKikimr::NMiniKQL::ResourceQueuePrefix << FormatType(queueType))); return IGraphTransformer::TStatus::Ok; @@ -10916,7 +10916,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const auto resourceTag = resourceType->GetTag(); using NKikimr::NMiniKQL::ResourceQueuePrefix; if (!resourceTag.StartsWith(ResourceQueuePrefix)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(resourceArg->Pos()), "You should use resource from QueueCreate")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(resourceArg->Pos()), "You should use resource from QueueCreate")); return false; } auto typeExpr = ctx.Expr.Builder(resourceArg->Pos()).Callable("ParseType") @@ -10993,35 +10993,35 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus QueueRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - if (!EnsureDependsOnTail(*input, ctx.Expr, 3)) { - return IGraphTransformer::TStatus::Error; - } - - auto resourceArg = input->Child(0); - auto& beginArg = input->ChildRef(1); - auto& endArg = input->ChildRef(2); - - const TTypeAnnotationNode* expectedValueType; - if (!EnsureQueueResource(resourceArg, expectedValueType, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto expectedIndexType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - auto convertStatus = TryConvertTo(beginArg, *expectedIndexType, ctx.Expr); - convertStatus = convertStatus.Combine(TryConvertTo(endArg, *expectedIndexType, ctx.Expr)); - if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { - return convertStatus; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TOptionalExprType>(expectedValueType))); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus QueueRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + if (!EnsureDependsOnTail(*input, ctx.Expr, 3)) { + return IGraphTransformer::TStatus::Error; + } + + auto resourceArg = input->Child(0); + auto& beginArg = input->ChildRef(1); + auto& endArg = input->ChildRef(2); + + const TTypeAnnotationNode* expectedValueType; + if (!EnsureQueueResource(resourceArg, expectedValueType, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto expectedIndexType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); + auto convertStatus = TryConvertTo(beginArg, *expectedIndexType, ctx.Expr); + convertStatus = convertStatus.Combine(TryConvertTo(endArg, *expectedIndexType, ctx.Expr)); + if (convertStatus.Level != IGraphTransformer::TStatus::Ok) { + return convertStatus; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TOptionalExprType>(expectedValueType))); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus PreserveStreamWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto streamArg = input->Child(0); @@ -11036,15 +11036,15 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TTypeAnnotationNode* streamType = streamArg->GetTypeAnn(); const TTypeAnnotationNode* itemType = streamType->Cast<TStreamExprType>()->GetItemType(); if (!IsSameAnnotation(*itemType, *expectedValueType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of stream and queue types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of stream and queue types: " << *itemType << " != " << *expectedValueType)); return IGraphTransformer::TStatus::Error; } - auto outpaceArg = input->Child(2); - if (!outpaceArg->IsCallable("Uint64")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(outpaceArg->Pos()), TStringBuilder() << "Outpace arg should be Uint64 literal")); - return IGraphTransformer::TStatus::Error; + auto outpaceArg = input->Child(2); + if (!outpaceArg->IsCallable("Uint64")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(outpaceArg->Pos()), TStringBuilder() << "Outpace arg should be Uint64 literal")); + return IGraphTransformer::TStatus::Error; } input->SetTypeAnn(streamType); @@ -11065,22 +11065,22 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus SeqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - for (auto& arg : input->Children()) { - if (!EnsureComputable(*arg, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } - - input->SetTypeAnn(input->Tail().GetTypeAnn()); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus SeqWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + for (auto& arg : input->Children()) { + if (!EnsureComputable(*arg, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(input->Tail().GetTypeAnn()); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus ByteStringWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -11109,8 +11109,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } input->SetTypeAnn(input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()); @@ -11122,14 +11122,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto dataType = node.Content(); auto slot = NKikimr::NUdf::FindDataSlot(dataType); if (!slot) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unknown datatype: " << dataType)); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unknown datatype: " << dataType)); return {}; } return slot; } else { if (!node.GetTypeAnn() || node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either atom or type")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either atom or type")); return {}; } @@ -11174,7 +11174,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (isDefault) { auto convertStatus = TryConvertTo(input->ChildRef(3), *targetTypeExpr, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Default value not correspond weak field type: " << targetType)); return IGraphTransformer::TStatus::Error; } @@ -11299,7 +11299,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto checkType = fieldType->GetKind() == ETypeAnnotationKind::Optional ? fieldType : ctx.Expr.MakeType<TOptionalExprType>(fieldType); if (!IsSameAnnotation(*targetTypeExpr, *checkType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "incompatible WeakField types: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "incompatible WeakField types: " << GetTypeDiff(*targetTypeExpr, *checkType))); return IGraphTransformer::TStatus::Error; } @@ -11392,7 +11392,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto targetType = targetTypeNode->Content(); auto targetSlot = NKikimr::NUdf::FindDataSlot(targetType); if (!targetSlot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(targetTypeNode->Pos()), TStringBuilder() << "Unknown datatype: " << targetType)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(targetTypeNode->Pos()), TStringBuilder() << "Unknown datatype: " << targetType)); return IGraphTransformer::TStatus::Error; } auto targetTypeExpr = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(*targetSlot)); @@ -11434,7 +11434,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto sourceSlot = sourceType->GetSlot(); if (sourceSlot != EDataSlot::String && sourceSlot != EDataSlot::Yson) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected string, yson or optional of it")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected string, yson or optional of it")); return IGraphTransformer::TStatus::Error; } @@ -11445,7 +11445,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> auto targetType = input->Child(1)->Content(); auto targetSlot = NKikimr::NUdf::FindDataSlot(targetType); if (!targetSlot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetType)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown datatype: " << targetType)); return IGraphTransformer::TStatus::Error; } @@ -11828,7 +11828,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> const TTypeAnnotationNode* expectedType = isOptional2 ? optUi16Type : ui16Type; auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -11878,11 +11878,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } - if (TCoJsonValue::idx_ReturningType < input->ChildrenSize()) { - auto status = EnsureTypeRewrite(input->ChildRef(TCoJsonValue::idx_ReturningType), ctx.Expr); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } + if (TCoJsonValue::idx_ReturningType < input->ChildrenSize()) { + auto status = EnsureTypeRewrite(input->ChildRef(TCoJsonValue::idx_ReturningType), ctx.Expr); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } } TCoJsonValue jsonValue(input); @@ -11903,7 +11903,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Error; } resultSlot = returningTypeAnn->Cast<TDataExprType>()->GetSlot(); - + if (!IsDataTypeNumeric(resultSlot) && !IsDataTypeDate(resultSlot) && resultSlot != EDataSlot::Utf8 @@ -12102,587 +12102,587 @@ template <NKikimr::NUdf::EDataSlot DataSlot> return IGraphTransformer::TStatus::Ok; } - bool IsValidTypeForRanges(const TTypeAnnotationNode* type) { - YQL_ENSURE(type); - if (type->GetKind() != ETypeAnnotationKind::Optional) { - return false; - } - type = RemoveAllOptionals(type); - YQL_ENSURE(type); - return type->GetKind() == ETypeAnnotationKind::Data && type->IsComparable() && type->IsEquatable(); - } - - bool EnsureValidRangeBoundary(TPositionHandle pos, const TTypeAnnotationNode* type, TExprContext& ctx) { - if (!type) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got lambda")); - return false; - } - - if (type->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got: " << *type)); - return false; - } - - const auto& components = type->Cast<TTupleExprType>()->GetItems(); - if (components.size() < 3 || components.size() % 2 == 0) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected tuple of minimal size 3 with odd number of components but got: " << *type)); - return false; - } - - for (size_t i = 0; i < components.size(); ++i) { - auto itemType = components[i]; - YQL_ENSURE(itemType); - if (i % 2 == 0) { - if (itemType->GetKind() != ETypeAnnotationKind::Data || itemType->Cast<TDataExprType>()->GetSlot() != EDataSlot::Int32) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected " << i << - "th component of range boundary tuple to be Int32, but got: " << *itemType)); - return false; - } - } else if (!IsValidTypeForRanges(itemType)) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected " << i << - "th component of range boundary tuple to be (multi) optional of " - "comparable and equatable Data type, but got: " << *itemType)); - return false; - } - } - return true; - } - - bool EnsureValidRange(TPositionHandle pos, const TTypeAnnotationNode* type, TExprContext& ctx) { - if (!type) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got lambda")); - return false; - } - - if (type->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got: " << *type)); - return false; - } - - const auto& components = type->Cast<TTupleExprType>()->GetItems(); - if (components.size() != 2) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple of size 2, but got: " << *type)); - return false; - } - - if (!EnsureValidRangeBoundary(pos, components.front(), ctx)) { - return false; - } - - YQL_ENSURE(components.front() && components.back()); - if (!IsSameAnnotation(*components.front(), *components.back())) { - ctx.AddError(TIssue(ctx.GetPosition(pos), + bool IsValidTypeForRanges(const TTypeAnnotationNode* type) { + YQL_ENSURE(type); + if (type->GetKind() != ETypeAnnotationKind::Optional) { + return false; + } + type = RemoveAllOptionals(type); + YQL_ENSURE(type); + return type->GetKind() == ETypeAnnotationKind::Data && type->IsComparable() && type->IsEquatable(); + } + + bool EnsureValidRangeBoundary(TPositionHandle pos, const TTypeAnnotationNode* type, TExprContext& ctx) { + if (!type) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got lambda")); + return false; + } + + if (type->GetKind() != ETypeAnnotationKind::Tuple) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got: " << *type)); + return false; + } + + const auto& components = type->Cast<TTupleExprType>()->GetItems(); + if (components.size() < 3 || components.size() % 2 == 0) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected tuple of minimal size 3 with odd number of components but got: " << *type)); + return false; + } + + for (size_t i = 0; i < components.size(); ++i) { + auto itemType = components[i]; + YQL_ENSURE(itemType); + if (i % 2 == 0) { + if (itemType->GetKind() != ETypeAnnotationKind::Data || itemType->Cast<TDataExprType>()->GetSlot() != EDataSlot::Int32) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected " << i << + "th component of range boundary tuple to be Int32, but got: " << *itemType)); + return false; + } + } else if (!IsValidTypeForRanges(itemType)) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected " << i << + "th component of range boundary tuple to be (multi) optional of " + "comparable and equatable Data type, but got: " << *itemType)); + return false; + } + } + return true; + } + + bool EnsureValidRange(TPositionHandle pos, const TTypeAnnotationNode* type, TExprContext& ctx) { + if (!type) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got lambda")); + return false; + } + + if (type->GetKind() != ETypeAnnotationKind::Tuple) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple type, but got: " << *type)); + return false; + } + + const auto& components = type->Cast<TTupleExprType>()->GetItems(); + if (components.size() != 2) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected tuple of size 2, but got: " << *type)); + return false; + } + + if (!EnsureValidRangeBoundary(pos, components.front(), ctx)) { + return false; + } + + YQL_ENSURE(components.front() && components.back()); + if (!IsSameAnnotation(*components.front(), *components.back())) { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Range begin/end type mismatch. Begin: " << *components.front() - << " End: " << *components.back())); - return false; - } - return true; - } - - bool EnsureValidUserRange(TPositionHandle pos, const TTypeAnnotationNode& range, - const TTypeAnnotationNode*& resultRange, TExprContext& ctx) - { - resultRange = nullptr; - if (range.GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected range to be tuple, but got: " << range)); - return false; - } - - auto rangeTuple = range.Cast<TTupleExprType>(); - if (rangeTuple->GetSize() != 2) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected range tuple to be of size 2, but got: " << rangeTuple->GetSize())); - return false; - } - - if (!IsSameAnnotation(*rangeTuple->GetItems().front(), *rangeTuple->GetItems().back())) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected both component of range to be of same type, but got: " - << *rangeTuple->GetItems().front() << " and " << *rangeTuple->GetItems().back())); - return false; - } - - auto boundaryType = rangeTuple->GetItems().front(); - if (boundaryType->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected range boundary to be tuple, but got: " << *boundaryType)); - return false; - } - - auto boundaryTuple = boundaryType->Cast<TTupleExprType>(); - if (boundaryTuple->GetSize() < 2) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected range boundary tuple to consist of at least 2 components, but got: " << boundaryTuple->GetSize())); - return false; - } - - // User range boundary encoding: AsTuple(x, ..., z, 0/1) - 0/1 means included/excluded. - // For column type T, boundary value x should have type T?, top level NULL means infinity. - // Infinity sign is implicit - infinity in left boundary always means minus infinity and plus infinity if in right boundary - TTypeAnnotationNode::TListType resultBoundaryItems; - auto int32Type = ctx.MakeType<TDataExprType>(EDataSlot::Int32); - const auto& items = boundaryTuple->GetItems(); - for (size_t i = 0; i < items.size() - 1; ++i) { - if (!IsValidTypeForRanges(items[i])) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected " << i << "th component of range boundary tuple to be (multi) optional of " - "comparable and equatable Data type, but got: " << *items[i])); - return false; - } - resultBoundaryItems.push_back(int32Type); - resultBoundaryItems.push_back(items[i]); - } - - YQL_ENSURE(items.back()); - if (items.back()->GetKind() != ETypeAnnotationKind::Data || items.back()->Cast<TDataExprType>()->GetSlot() != EDataSlot::Int32) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "Expected last component of range boundary tuple to be Int32, " - "but got: " << *items.back())); - return false; - } - - resultBoundaryItems.push_back(int32Type); - auto resultBoundary = ctx.MakeType<TTupleExprType>(resultBoundaryItems); - resultRange = ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundary, resultBoundary}); - return true; - } - - IGraphTransformer::TStatus AsRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (input->ChildrenSize() == 0) { - output = ctx.Expr.RenameNode(*input, "RangeEmpty"); - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* rangeType = nullptr; - const TTypeAnnotationNode* resultRangeType = nullptr; - for (ui32 i = 0; i < input->ChildrenSize(); ++i) { - auto child = input->Child(i); - TPositionHandle pos = child->Pos(); - auto argType = child->GetTypeAnn(); - if (!argType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), - TStringBuilder() << "Expected range tuple as " << i << "th argument, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (!rangeType) { - rangeType = argType; - if (!EnsureValidUserRange(pos, *rangeType, resultRangeType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - } - - if (!IsSameAnnotation(*rangeType, *argType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), - TStringBuilder() << "Expected all arguments to be of same type, but got: " << *rangeType - << " as first argument and " << *argType << " as " << i << "th")); - return IGraphTransformer::TStatus::Error; - } - } - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeCreateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureListType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto rangeType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - const TTypeAnnotationNode* resultRangeType = nullptr; - if (!EnsureValidUserRange(input->Head().Pos(), *rangeType, resultRangeType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeEmptyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (input->ChildrenSize() == 0) { - input->SetTypeAnn(ctx.Expr.MakeType<TEmptyListExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - TTypeAnnotationNode::TListType optKeys; - auto keyType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (keyType->GetKind() == ETypeAnnotationKind::Tuple) { - for (auto& type : keyType->Cast<TTupleExprType>()->GetItems()) { - optKeys.push_back(ctx.Expr.MakeType<TOptionalExprType>(type)); - } - } else { - optKeys.push_back(ctx.Expr.MakeType<TOptionalExprType>(keyType)); - } - - auto int32Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32); - TTypeAnnotationNode::TListType resultBoundaryItems; - for (auto& optKeyType : optKeys) { - if (!IsValidTypeForRanges(optKeyType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected (multi) optional of comparable and equatable Data type, but got: " << *optKeyType)); - return IGraphTransformer::TStatus::Error; - } - resultBoundaryItems.push_back(int32Type); - resultBoundaryItems.push_back(optKeyType); - } - - resultBoundaryItems.push_back(int32Type); - - const TTypeAnnotationNode* resultBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultBoundaryItems); - const TTypeAnnotationNode* resultRangeType = - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundaryType, resultBoundaryType}); - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeForWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TStringBuf op = input->Head().Content(); - static const THashSet<TStringBuf> ops = {"==", "!=", "<=", "<", ">=", ">", "Exists", "NotExists", "===", "StartsWith", "NotStartsWith"}; - if (!ops.contains(op)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Unknown operation: " << op)); - return IGraphTransformer::TStatus::Error; - } - - auto valueType = input->Child(1)->GetTypeAnn(); - if (!valueType) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), - TStringBuilder() << "Expecting (optional) Data as second argument, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - const TTypeAnnotationNode* valueBaseType = nullptr; - if (op != "Exists" && op != "NotExists") { - valueBaseType = RemoveAllOptionals(valueType); - YQL_ENSURE(valueBaseType); - if (valueBaseType->GetKind() != ETypeAnnotationKind::Data) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), - TStringBuilder() << "Expecting (optional) Data as second argument, but got: " << *valueType)); - return IGraphTransformer::TStatus::Error; - } - } else if (!EnsureVoidType(*input->Child(1), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->TailRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto keyType = input->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - YQL_ENSURE(keyType); - auto optKeyType = ctx.Expr.MakeType<TOptionalExprType>(keyType); - if (!IsValidTypeForRanges(optKeyType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected (optional) of comparable and equatable Data type, but got: " << *keyType)); - return IGraphTransformer::TStatus::Error; - } - - if (valueBaseType && CanCompare<false>(RemoveAllOptionals(keyType), valueBaseType) != ECompareOptions::Comparable) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Uncompatible key and value types: " << *keyType << " and " << *valueType)); - return IGraphTransformer::TStatus::Error; - } - - auto int32Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32); - TTypeAnnotationNode::TListType resultBoundaryItems; - resultBoundaryItems.push_back(int32Type); - resultBoundaryItems.push_back(optKeyType); - resultBoundaryItems.push_back(int32Type); - - const TTypeAnnotationNode* resultBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultBoundaryItems); - const TTypeAnnotationNode* resultRangeType = - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundaryType, resultBoundaryType}); - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeUnionWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureListType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto argType = input->Head().GetTypeAnn(); - auto rangeType = argType->Cast<TListExprType>()->GetItemType(); - if (!EnsureValidRange(input->Head().Pos(), rangeType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - for (ui32 i = 1; i < input->ChildrenSize(); ++i) { - TPositionHandle pos = input->Child(i)->Pos(); - auto type = input->Child(i)->GetTypeAnn(); - if (!type) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), - TStringBuilder() << "Expected " << *argType << " as argument #" << i << ", but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (!IsSameAnnotation(*type, *argType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), - TStringBuilder() << "Expected " << *argType << " as argument #" << i << ", but got: " << *type)); - return IGraphTransformer::TStatus::Error; - } - } - - input->SetTypeAnn(argType); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeMultiplyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureSpecificDataType(input->Head(), EDataSlot::Uint64, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TTypeAnnotationNode::TListType resultComponents; - for (ui32 i = 1; i < input->ChildrenSize(); ++i) { - if (!EnsureListType(*input->Child(i), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto rangeType = input->Child(i)->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - if (!EnsureValidRange(input->Child(i)->Pos(), rangeType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto& components = rangeType->Cast<TTupleExprType>()->GetItems().front()->Cast<TTupleExprType>()->GetItems(); - YQL_ENSURE(components.size() >= 3); - resultComponents.insert(resultComponents.end(), components.begin(), components.end() - 1); - } - resultComponents.push_back(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32)); - - auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultComponents); - auto resultRangeType = - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeFinalizeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureListType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto argType = input->Head().GetTypeAnn(); - auto rangeType = argType->Cast<TListExprType>()->GetItemType(); - if (!EnsureValidRange(input->Head().Pos(), rangeType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto& components = rangeType->Cast<TTupleExprType>()->GetItems().front()->Cast<TTupleExprType>()->GetItems(); - TTypeAnnotationNode::TListType resultComponents; - for (size_t i = 0; i < components.size(); ++i) { - // take odd and last component - if (i % 2 == 1 || i + 1 == components.size()) { - resultComponents.push_back(components[i]); - } - } - - auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultComponents); - auto resultRangeType = - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RangeComputeForWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto rowType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - YQL_ENSURE(rowType); - if (!EnsureStructType(input->Head().Pos(), *rowType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto& lambdaNode = input->ChildRef(1); - const auto status = ConvertToLambda(lambdaNode, ctx.Expr, 1); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaNode, { rowType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!lambdaNode->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - // extract_predicate library supports TCoConditionalValueBase in lambda root or just plain predicate lambda - using namespace NNodes; - TCoLambda lambda(lambdaNode); - if (!lambda.Body().Maybe<TCoConditionalValueBase>() && !EnsureSpecificDataType(lambda.Ref(), EDataSlot::Bool, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto& tupleOfAtomsNode = input->Tail(); - if (!EnsureTupleMinSize(tupleOfAtomsNode, 1, ctx.Expr) || !EnsureTupleOfAtoms(tupleOfAtomsNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - THashSet<TStringBuf> indexKeys; - const TStructExprType& structType = *rowType->Cast<TStructExprType>(); - TTypeAnnotationNode::TListType rangeBoundaryTypes; - for (auto& keyNode : tupleOfAtomsNode.ChildrenList()) { - TStringBuf key = keyNode->Content(); - if (!indexKeys.insert(key).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyNode->Pos()), - TStringBuilder() << "Duplicate index column '" << key << "'")); - return IGraphTransformer::TStatus::Error; - } - - auto pos = FindOrReportMissingMember(key, keyNode->Pos(), structType, ctx); - if (!pos) { - return IGraphTransformer::TStatus::Error; - } - - auto keyType = structType.GetItems()[*pos]->GetItemType(); - auto optKeyType = ctx.Expr.MakeType<TOptionalExprType>(keyType); - if (!IsValidTypeForRanges(optKeyType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyNode->Pos()), - TStringBuilder() << "Unsupported index column type: expecting Data or (multi) optional of Data, " - << "got: " << *keyType << " for column '" << key << "'")); - return IGraphTransformer::TStatus::Error; - } - - rangeBoundaryTypes.push_back(optKeyType); - } - rangeBoundaryTypes.push_back(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32)); - - auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(rangeBoundaryTypes); - auto resultRangeType = - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); - auto resultRangesType = ctx.Expr.MakeType<TListExprType>(resultRangeType); - auto serializedLambdaType = - ctx.Expr.MakeType<TTaggedExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String), "AST"); - - input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>( - ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangesType, serializedLambdaType}))); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus RoundWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureDataType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->TailRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto dstType = input->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (!EnsureDataType(input->Tail().Pos(), *dstType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto srcType = input->Head().GetTypeAnn(); - if (CanCompare<false>(srcType, dstType) != ECompareOptions::Comparable) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Uncompatible types in rounding: " << *srcType << " " << input->Content() << " to " << *dstType)); - return IGraphTransformer::TStatus::Error; - } - - if (IsSameAnnotation(*srcType, *dstType)) { - output = ctx.Expr.NewCallable(input->Pos(), "Just", { input->HeadPtr() }); - return IGraphTransformer::TStatus::Repeat; - } - - auto sSlot = srcType->Cast<TDataExprType>()->GetSlot(); - auto tSlot = dstType->Cast<TDataExprType>()->GetSlot(); - auto resultType = ctx.Expr.MakeType<TOptionalExprType>(dstType); - - const auto cast = NUdf::GetCastResult(sSlot, tSlot); - if (!cast) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Unsupported types in rounding: " << *srcType << " " << input->Content() << " to " << *dstType)); - return IGraphTransformer::TStatus::Error; - } - - if (!(*cast & (NUdf::ECastOptions::MayFail | NUdf::ECastOptions::MayLoseData | NUdf::ECastOptions::AnywayLoseData))) { - output = ctx.Expr.NewCallable(input->Pos(), "SafeCast", - { input->HeadPtr(), ExpandType(input->Pos(), *resultType, ctx.Expr) }); - return IGraphTransformer::TStatus::Repeat; - } - - input->SetTypeAnn(resultType); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus NextValueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureStringOrUtf8Type(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->Head().GetTypeAnn())); - return IGraphTransformer::TStatus::Ok; - } - + << " End: " << *components.back())); + return false; + } + return true; + } + + bool EnsureValidUserRange(TPositionHandle pos, const TTypeAnnotationNode& range, + const TTypeAnnotationNode*& resultRange, TExprContext& ctx) + { + resultRange = nullptr; + if (range.GetKind() != ETypeAnnotationKind::Tuple) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected range to be tuple, but got: " << range)); + return false; + } + + auto rangeTuple = range.Cast<TTupleExprType>(); + if (rangeTuple->GetSize() != 2) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected range tuple to be of size 2, but got: " << rangeTuple->GetSize())); + return false; + } + + if (!IsSameAnnotation(*rangeTuple->GetItems().front(), *rangeTuple->GetItems().back())) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected both component of range to be of same type, but got: " + << *rangeTuple->GetItems().front() << " and " << *rangeTuple->GetItems().back())); + return false; + } + + auto boundaryType = rangeTuple->GetItems().front(); + if (boundaryType->GetKind() != ETypeAnnotationKind::Tuple) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected range boundary to be tuple, but got: " << *boundaryType)); + return false; + } + + auto boundaryTuple = boundaryType->Cast<TTupleExprType>(); + if (boundaryTuple->GetSize() < 2) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected range boundary tuple to consist of at least 2 components, but got: " << boundaryTuple->GetSize())); + return false; + } + + // User range boundary encoding: AsTuple(x, ..., z, 0/1) - 0/1 means included/excluded. + // For column type T, boundary value x should have type T?, top level NULL means infinity. + // Infinity sign is implicit - infinity in left boundary always means minus infinity and plus infinity if in right boundary + TTypeAnnotationNode::TListType resultBoundaryItems; + auto int32Type = ctx.MakeType<TDataExprType>(EDataSlot::Int32); + const auto& items = boundaryTuple->GetItems(); + for (size_t i = 0; i < items.size() - 1; ++i) { + if (!IsValidTypeForRanges(items[i])) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected " << i << "th component of range boundary tuple to be (multi) optional of " + "comparable and equatable Data type, but got: " << *items[i])); + return false; + } + resultBoundaryItems.push_back(int32Type); + resultBoundaryItems.push_back(items[i]); + } + + YQL_ENSURE(items.back()); + if (items.back()->GetKind() != ETypeAnnotationKind::Data || items.back()->Cast<TDataExprType>()->GetSlot() != EDataSlot::Int32) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "Expected last component of range boundary tuple to be Int32, " + "but got: " << *items.back())); + return false; + } + + resultBoundaryItems.push_back(int32Type); + auto resultBoundary = ctx.MakeType<TTupleExprType>(resultBoundaryItems); + resultRange = ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundary, resultBoundary}); + return true; + } + + IGraphTransformer::TStatus AsRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (input->ChildrenSize() == 0) { + output = ctx.Expr.RenameNode(*input, "RangeEmpty"); + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* rangeType = nullptr; + const TTypeAnnotationNode* resultRangeType = nullptr; + for (ui32 i = 0; i < input->ChildrenSize(); ++i) { + auto child = input->Child(i); + TPositionHandle pos = child->Pos(); + auto argType = child->GetTypeAnn(); + if (!argType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), + TStringBuilder() << "Expected range tuple as " << i << "th argument, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (!rangeType) { + rangeType = argType; + if (!EnsureValidUserRange(pos, *rangeType, resultRangeType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } + + if (!IsSameAnnotation(*rangeType, *argType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), + TStringBuilder() << "Expected all arguments to be of same type, but got: " << *rangeType + << " as first argument and " << *argType << " as " << i << "th")); + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeCreateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureListType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto rangeType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + const TTypeAnnotationNode* resultRangeType = nullptr; + if (!EnsureValidUserRange(input->Head().Pos(), *rangeType, resultRangeType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeEmptyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (input->ChildrenSize() == 0) { + input->SetTypeAnn(ctx.Expr.MakeType<TEmptyListExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + TTypeAnnotationNode::TListType optKeys; + auto keyType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (keyType->GetKind() == ETypeAnnotationKind::Tuple) { + for (auto& type : keyType->Cast<TTupleExprType>()->GetItems()) { + optKeys.push_back(ctx.Expr.MakeType<TOptionalExprType>(type)); + } + } else { + optKeys.push_back(ctx.Expr.MakeType<TOptionalExprType>(keyType)); + } + + auto int32Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32); + TTypeAnnotationNode::TListType resultBoundaryItems; + for (auto& optKeyType : optKeys) { + if (!IsValidTypeForRanges(optKeyType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected (multi) optional of comparable and equatable Data type, but got: " << *optKeyType)); + return IGraphTransformer::TStatus::Error; + } + resultBoundaryItems.push_back(int32Type); + resultBoundaryItems.push_back(optKeyType); + } + + resultBoundaryItems.push_back(int32Type); + + const TTypeAnnotationNode* resultBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultBoundaryItems); + const TTypeAnnotationNode* resultRangeType = + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundaryType, resultBoundaryType}); + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeForWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TStringBuf op = input->Head().Content(); + static const THashSet<TStringBuf> ops = {"==", "!=", "<=", "<", ">=", ">", "Exists", "NotExists", "===", "StartsWith", "NotStartsWith"}; + if (!ops.contains(op)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Unknown operation: " << op)); + return IGraphTransformer::TStatus::Error; + } + + auto valueType = input->Child(1)->GetTypeAnn(); + if (!valueType) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), + TStringBuilder() << "Expecting (optional) Data as second argument, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + const TTypeAnnotationNode* valueBaseType = nullptr; + if (op != "Exists" && op != "NotExists") { + valueBaseType = RemoveAllOptionals(valueType); + YQL_ENSURE(valueBaseType); + if (valueBaseType->GetKind() != ETypeAnnotationKind::Data) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), + TStringBuilder() << "Expecting (optional) Data as second argument, but got: " << *valueType)); + return IGraphTransformer::TStatus::Error; + } + } else if (!EnsureVoidType(*input->Child(1), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->TailRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto keyType = input->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + YQL_ENSURE(keyType); + auto optKeyType = ctx.Expr.MakeType<TOptionalExprType>(keyType); + if (!IsValidTypeForRanges(optKeyType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected (optional) of comparable and equatable Data type, but got: " << *keyType)); + return IGraphTransformer::TStatus::Error; + } + + if (valueBaseType && CanCompare<false>(RemoveAllOptionals(keyType), valueBaseType) != ECompareOptions::Comparable) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Uncompatible key and value types: " << *keyType << " and " << *valueType)); + return IGraphTransformer::TStatus::Error; + } + + auto int32Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32); + TTypeAnnotationNode::TListType resultBoundaryItems; + resultBoundaryItems.push_back(int32Type); + resultBoundaryItems.push_back(optKeyType); + resultBoundaryItems.push_back(int32Type); + + const TTypeAnnotationNode* resultBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultBoundaryItems); + const TTypeAnnotationNode* resultRangeType = + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultBoundaryType, resultBoundaryType}); + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeUnionWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureListType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto argType = input->Head().GetTypeAnn(); + auto rangeType = argType->Cast<TListExprType>()->GetItemType(); + if (!EnsureValidRange(input->Head().Pos(), rangeType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + for (ui32 i = 1; i < input->ChildrenSize(); ++i) { + TPositionHandle pos = input->Child(i)->Pos(); + auto type = input->Child(i)->GetTypeAnn(); + if (!type) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), + TStringBuilder() << "Expected " << *argType << " as argument #" << i << ", but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (!IsSameAnnotation(*type, *argType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), + TStringBuilder() << "Expected " << *argType << " as argument #" << i << ", but got: " << *type)); + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(argType); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeMultiplyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureSpecificDataType(input->Head(), EDataSlot::Uint64, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TTypeAnnotationNode::TListType resultComponents; + for (ui32 i = 1; i < input->ChildrenSize(); ++i) { + if (!EnsureListType(*input->Child(i), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto rangeType = input->Child(i)->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + if (!EnsureValidRange(input->Child(i)->Pos(), rangeType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto& components = rangeType->Cast<TTupleExprType>()->GetItems().front()->Cast<TTupleExprType>()->GetItems(); + YQL_ENSURE(components.size() >= 3); + resultComponents.insert(resultComponents.end(), components.begin(), components.end() - 1); + } + resultComponents.push_back(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32)); + + auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultComponents); + auto resultRangeType = + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeFinalizeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureListType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto argType = input->Head().GetTypeAnn(); + auto rangeType = argType->Cast<TListExprType>()->GetItemType(); + if (!EnsureValidRange(input->Head().Pos(), rangeType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto& components = rangeType->Cast<TTupleExprType>()->GetItems().front()->Cast<TTupleExprType>()->GetItems(); + TTypeAnnotationNode::TListType resultComponents; + for (size_t i = 0; i < components.size(); ++i) { + // take odd and last component + if (i % 2 == 1 || i + 1 == components.size()) { + resultComponents.push_back(components[i]); + } + } + + auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(resultComponents); + auto resultRangeType = + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultRangeType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RangeComputeForWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto rowType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + YQL_ENSURE(rowType); + if (!EnsureStructType(input->Head().Pos(), *rowType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto& lambdaNode = input->ChildRef(1); + const auto status = ConvertToLambda(lambdaNode, ctx.Expr, 1); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaNode, { rowType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!lambdaNode->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + // extract_predicate library supports TCoConditionalValueBase in lambda root or just plain predicate lambda + using namespace NNodes; + TCoLambda lambda(lambdaNode); + if (!lambda.Body().Maybe<TCoConditionalValueBase>() && !EnsureSpecificDataType(lambda.Ref(), EDataSlot::Bool, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto& tupleOfAtomsNode = input->Tail(); + if (!EnsureTupleMinSize(tupleOfAtomsNode, 1, ctx.Expr) || !EnsureTupleOfAtoms(tupleOfAtomsNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + THashSet<TStringBuf> indexKeys; + const TStructExprType& structType = *rowType->Cast<TStructExprType>(); + TTypeAnnotationNode::TListType rangeBoundaryTypes; + for (auto& keyNode : tupleOfAtomsNode.ChildrenList()) { + TStringBuf key = keyNode->Content(); + if (!indexKeys.insert(key).second) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyNode->Pos()), + TStringBuilder() << "Duplicate index column '" << key << "'")); + return IGraphTransformer::TStatus::Error; + } + + auto pos = FindOrReportMissingMember(key, keyNode->Pos(), structType, ctx); + if (!pos) { + return IGraphTransformer::TStatus::Error; + } + + auto keyType = structType.GetItems()[*pos]->GetItemType(); + auto optKeyType = ctx.Expr.MakeType<TOptionalExprType>(keyType); + if (!IsValidTypeForRanges(optKeyType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyNode->Pos()), + TStringBuilder() << "Unsupported index column type: expecting Data or (multi) optional of Data, " + << "got: " << *keyType << " for column '" << key << "'")); + return IGraphTransformer::TStatus::Error; + } + + rangeBoundaryTypes.push_back(optKeyType); + } + rangeBoundaryTypes.push_back(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int32)); + + auto resultRangeBoundaryType = ctx.Expr.MakeType<TTupleExprType>(rangeBoundaryTypes); + auto resultRangeType = + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangeBoundaryType, resultRangeBoundaryType}); + auto resultRangesType = ctx.Expr.MakeType<TListExprType>(resultRangeType); + auto serializedLambdaType = + ctx.Expr.MakeType<TTaggedExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String), "AST"); + + input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>( + ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{resultRangesType, serializedLambdaType}))); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus RoundWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureDataType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->TailRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto dstType = input->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (!EnsureDataType(input->Tail().Pos(), *dstType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto srcType = input->Head().GetTypeAnn(); + if (CanCompare<false>(srcType, dstType) != ECompareOptions::Comparable) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Uncompatible types in rounding: " << *srcType << " " << input->Content() << " to " << *dstType)); + return IGraphTransformer::TStatus::Error; + } + + if (IsSameAnnotation(*srcType, *dstType)) { + output = ctx.Expr.NewCallable(input->Pos(), "Just", { input->HeadPtr() }); + return IGraphTransformer::TStatus::Repeat; + } + + auto sSlot = srcType->Cast<TDataExprType>()->GetSlot(); + auto tSlot = dstType->Cast<TDataExprType>()->GetSlot(); + auto resultType = ctx.Expr.MakeType<TOptionalExprType>(dstType); + + const auto cast = NUdf::GetCastResult(sSlot, tSlot); + if (!cast) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Unsupported types in rounding: " << *srcType << " " << input->Content() << " to " << *dstType)); + return IGraphTransformer::TStatus::Error; + } + + if (!(*cast & (NUdf::ECastOptions::MayFail | NUdf::ECastOptions::MayLoseData | NUdf::ECastOptions::AnywayLoseData))) { + output = ctx.Expr.NewCallable(input->Pos(), "SafeCast", + { input->HeadPtr(), ExpandType(input->Pos(), *resultType, ctx.Expr) }); + return IGraphTransformer::TStatus::Repeat; + } + + input->SetTypeAnn(resultType); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus NextValueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureStringOrUtf8Type(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->Head().GetTypeAnn())); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus AssumeAllMembersNullableAtOnceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 1, ctx.Expr)) { @@ -12724,12 +12724,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["TakeWhileInclusive"] = &InclusiveFilterWrapper<false>; Functions["SkipWhileInclusive"] = &InclusiveFilterWrapper<true>; Functions["Member"] = &MemberWrapper; - Functions["SingleMember"] = &SingleMemberWrapper; + Functions["SingleMember"] = &SingleMemberWrapper; Functions["SqlColumn"] = &SqlColumnWrapper; - Functions["SqlPlainColumn"] = &SqlColumnWrapper; - Functions["SqlColumnOrType"] = &SqlColumnWrapper; - Functions["SqlPlainColumnOrType"] = &SqlColumnWrapper; - Functions["SqlColumnFromType"] = &SqlColumnFromTypeWrapper; + Functions["SqlPlainColumn"] = &SqlColumnWrapper; + Functions["SqlColumnOrType"] = &SqlColumnWrapper; + Functions["SqlPlainColumnOrType"] = &SqlColumnWrapper; + Functions["SqlColumnFromType"] = &SqlColumnFromTypeWrapper; Functions["Nth"] = &NthWrapper; Functions["FlattenMembers"] = &FlattenMembersWrapper; Functions["SelectMembers"] = &SelectMembersWrapper<true>; @@ -12780,8 +12780,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["AggrGreaterOrEqual"] = &AggrCompareWrapper<true, true>; Functions["AggrMin"] = &AggrMinMaxWrapper; Functions["AggrMax"] = &AggrMinMaxWrapper; - Functions["IsNotDistinctFrom"] = &DistinctFromWrapper; - Functions["IsDistinctFrom"] = &DistinctFromWrapper;; + Functions["IsNotDistinctFrom"] = &DistinctFromWrapper; + Functions["IsDistinctFrom"] = &DistinctFromWrapper;; Functions["Abs"] = &AbsWrapper; Functions["ShiftLeft"] = &ShiftWrapper; Functions["RotLeft"] = &ShiftWrapper; @@ -12871,14 +12871,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["TopSort"] = &TopWrapper; Functions["KeepTop"] = &KeepTopWrapper; Functions["Unordered"] = &UnorderedWrapper; - Functions["UnorderedSubquery"] = &UnorderedWrapper; - Functions["SortTraits"] = &SortTraitsWrapper; - Functions["SessionWindowTraits"] = &SessionWindowTraitsWrapper; + Functions["UnorderedSubquery"] = &UnorderedWrapper; + Functions["SortTraits"] = &SortTraitsWrapper; + Functions["SessionWindowTraits"] = &SessionWindowTraitsWrapper; Functions["FromString"] = &FromStringWrapper; Functions["StrictFromString"] = &StrictFromStringWrapper; Functions["FromBytes"] = &FromBytesWrapper; Functions["Convert"] = &ConvertWrapper; - Functions["AlterTo"] = &AlterToWrapper; + Functions["AlterTo"] = &AlterToWrapper; Functions["ToIntegral"] = &ToIntegralWrapper; Functions["Cast"] = &OldCastWrapper; Functions["SafeCast"] = &CastWrapper<false>; @@ -12890,7 +12890,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["StablePickle"] = &PickleWrapper; Functions["Unpickle"] = &UnpickleWrapper; Functions["Coalesce"] = &CoalesceWrapper; - Functions["CoalesceMembers"] = &CoalesceMembersWrapper; + Functions["CoalesceMembers"] = &CoalesceMembersWrapper; Functions["Nvl"] = &NvlWrapper; Functions["Nanvl"] = &NanvlWrapper; Functions["Unwrap"] = &UnwrapWrapper; @@ -12939,13 +12939,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["AsVariant"] = &AsVariantWrapper; Functions["AsEnum"] = &AsEnumWrapper; Functions["Contains"] = &ContainsLookupWrapper<true>; - Functions["SqlIn"] = &SqlInWrapper; + Functions["SqlIn"] = &SqlInWrapper; Functions["Lookup"] = &ContainsLookupWrapper<false>; Functions["DictItems"] = &DictItemsWrapper<EDictItems::Both>; Functions["DictKeys"] = &DictItemsWrapper<EDictItems::Keys>; Functions["DictPayloads"] = &DictItemsWrapper<EDictItems::Payloads>; Functions["AsStruct"] = &AsStructWrapper; - Functions["AsStructUnordered"] = &AsStructWrapper; + Functions["AsStructUnordered"] = &AsStructWrapper; Functions["AsDict"] = &AsDictWrapper<false, false>; Functions["AsDictStrict"] = &AsDictWrapper<true, false>; Functions["AsSet"] = &AsDictWrapper<false, true>; @@ -12957,11 +12957,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions[ForName] = &ForWorldWrapper; Functions["IfPresent"] = &IfPresentWrapper; Functions["StaticMap"] = &StaticMapWrapper; - Functions["StaticZip"] = &StaticZipWrapper; - Functions["TryRemoveAllOptionals"] = &TryRemoveAllOptionalsWrapper; - Functions["HasNull"] = &HasNullWrapper; + Functions["StaticZip"] = &StaticZipWrapper; + Functions["TryRemoveAllOptionals"] = &TryRemoveAllOptionalsWrapper; + Functions["HasNull"] = &HasNullWrapper; Functions["TypeOf"] = &TypeOfWrapper; - Functions["ConstraintsOf"] = &ConstraintsOfWrapper; + Functions["ConstraintsOf"] = &ConstraintsOfWrapper; Functions["InstanceOf"] = &InstanceOfWrapper; Functions["SourceOf"] = &SourceOfWrapper; Functions["MatchType"] = &MatchTypeWrapper; @@ -13008,8 +13008,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["DictPayloadType"] = &TypeArgWrapper<ETypeArgument::DictPayload>; Functions["Apply"] = &ApplyWrapper; Functions["NamedApply"] = &NamedApplyWrapper; - Functions["PositionalArgs"] = &PositionalArgsWrapper; - Functions["SqlCall"] = &SqlCallWrapper; + Functions["PositionalArgs"] = &PositionalArgsWrapper; + Functions["SqlCall"] = &SqlCallWrapper; Functions["Callable"] = &CallableWrapper; Functions["CallableType"] = &TypeWrapper<ETypeAnnotationKind::Callable>; Functions["CallableResultType"] = &TypeArgWrapper<ETypeArgument::CallableResult>; @@ -13022,18 +13022,18 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["AggregationTraits"] = &AggregationTraitsWrapper; Functions["MultiAggregate"] = &MultiAggregateWrapper; Functions["Aggregate"] = &AggregateWrapper; - Functions["SqlAggregateAll"] = &SqlAggregateAllWrapper; + Functions["SqlAggregateAll"] = &SqlAggregateAllWrapper; Functions["WinOnRows"] = &WinOnRowsWrapper; Functions["WinOnRange"] = &WinOnRangeWrapper; Functions["WindowTraits"] = &WindowTraitsWrapper; Functions["CalcOverWindow"] = &CalcOverWindowWrapper; - Functions["CalcOverSessionWindow"] = &CalcOverWindowWrapper; - Functions["CalcOverWindowGroup"] = &CalcOverWindowGroupWrapper; + Functions["CalcOverSessionWindow"] = &CalcOverWindowWrapper; + Functions["CalcOverWindowGroup"] = &CalcOverWindowGroupWrapper; Functions["Lag"] = &WinLeadLagWrapper; Functions["Lead"] = &WinLeadLagWrapper; Functions["RowNumber"] = &WinRowNumberWrapper; - Functions["Rank"] = &WinRankWrapper; - Functions["DenseRank"] = &WinRankWrapper; + Functions["Rank"] = &WinRankWrapper; + Functions["DenseRank"] = &WinRankWrapper; Functions["Ascending"] = &PresortWrapper; Functions["Descending"] = &PresortWrapper; Functions["IsKeySwitch"] = &IsKeySwitchWrapper; @@ -13056,13 +13056,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["SqlReduce"] = &SqlReduceWrapper; Functions["SqlExternalFunction"] = &SqlExternalFunctionWrapper; Functions["SqlExtractKey"] = &SqlExtractKeyWrapper; - Functions["SqlReduceUdf"] = &SqlReduceUdfWrapper; - Functions["SqlProject"] = &SqlProjectWrapper; - Functions["SqlTypeFromYson"] = &SqlTypeFromYsonWrapper; - Functions["SqlColumnOrderFromYson"] = &SqlColumnOrderFromYsonWrapper; - Functions["OrderedSqlProject"] = &SqlProjectWrapper; - Functions["SqlProjectItem"] = &SqlProjectItemWrapper; - Functions["SqlProjectStarItem"] = &SqlProjectItemWrapper; + Functions["SqlReduceUdf"] = &SqlReduceUdfWrapper; + Functions["SqlProject"] = &SqlProjectWrapper; + Functions["SqlTypeFromYson"] = &SqlTypeFromYsonWrapper; + Functions["SqlColumnOrderFromYson"] = &SqlColumnOrderFromYsonWrapper; + Functions["OrderedSqlProject"] = &SqlProjectWrapper; + Functions["SqlProjectItem"] = &SqlProjectItemWrapper; + Functions["SqlProjectStarItem"] = &SqlProjectItemWrapper; Functions["PgStar"] = &PgStarWrapper; Functions["PgCall"] = &PgCallWrapper; Functions["PgAgg"] = &PgAggWrapper; @@ -13082,7 +13082,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["QueueCreate"] = &QueueCreateWrapper; Functions["QueuePop"] = &QueuePopWrapper; Functions["DependsOn"] = &DependsOnWrapper; - Functions["Seq"] = &SeqWrapper; + Functions["Seq"] = &SeqWrapper; Functions["Parameter"] = &ParameterWrapper; Functions["WeakField"] = &WeakFieldWrapper; Functions["TryWeakMemberFromDict"] = &TryWeakMemberFromDictWrapper; @@ -13136,10 +13136,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["EvaluateExpr"] = &RestartEvaluationWrapper; Functions["EvaluateType"] = &RestartEvaluationWrapper; Functions["EvaluateCode"] = &RestartEvaluationWrapper; - Functions["EvaluateExprIfPure"] = &EvaluateExprIfPureWrapper; + Functions["EvaluateExprIfPure"] = &EvaluateExprIfPureWrapper; Functions["ToFlow"] = &ToFlowWrapper; Functions["FromFlow"] = &FromFlowWrapper; - Functions["BuildTablePath"] = &BuildTablePathWrapper; + Functions["BuildTablePath"] = &BuildTablePathWrapper; Functions["WithOptionalArgs"] = &WithOptionalArgsWrapper; Functions["DecimalDiv"] = &DecimalBinaryWrapper; @@ -13188,20 +13188,20 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["NarrowFlatMap"] = &NarrowFlatMapWrapper; Functions["NarrowMultiMap"] = &NarrowMultiMapWrapper; - Functions["AsRange"] = &AsRangeWrapper; - Functions["RangeCreate"] = &RangeCreateWrapper; - Functions["RangeEmpty"] = &RangeEmptyWrapper; - Functions["RangeFor"] = &RangeForWrapper; - Functions["RangeUnion"] = &RangeUnionWrapper; - Functions["RangeIntersect"] = &RangeUnionWrapper; - Functions["RangeMultiply"] = &RangeMultiplyWrapper; - Functions["RangeFinalize"] = &RangeFinalizeWrapper; - Functions["RangeComputeFor"] = &RangeComputeForWrapper; - - Functions["RoundUp"] = &RoundWrapper; - Functions["RoundDown"] = &RoundWrapper; - Functions["NextValue"] = &NextValueWrapper; - + Functions["AsRange"] = &AsRangeWrapper; + Functions["RangeCreate"] = &RangeCreateWrapper; + Functions["RangeEmpty"] = &RangeEmptyWrapper; + Functions["RangeFor"] = &RangeForWrapper; + Functions["RangeUnion"] = &RangeUnionWrapper; + Functions["RangeIntersect"] = &RangeUnionWrapper; + Functions["RangeMultiply"] = &RangeMultiplyWrapper; + Functions["RangeFinalize"] = &RangeFinalizeWrapper; + Functions["RangeComputeFor"] = &RangeComputeForWrapper; + + Functions["RoundUp"] = &RoundWrapper; + Functions["RoundDown"] = &RoundWrapper; + Functions["NextValue"] = &NextValueWrapper; + ExtFunctions["PgSelect"] = &PgSelectWrapper; ExtFunctions["PgSetItem"] = &PgSetItemWrapper; ExtFunctions["TablePath"] = &TablePathWrapper; @@ -13219,7 +13219,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> ExtFunctions["CurrentActorId"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::String>; ExtFunctions["QueuePush"] = &QueuePushWrapper; ///< Ext for ParseTypeWrapper compatibility ExtFunctions["QueuePeek"] = &QueuePeekWrapper; ///< Ext for ParseTypeWrapper compatibility - ExtFunctions["QueueRange"] = &QueueRangeWrapper; ///< Ext for ParseTypeWrapper compatibility + ExtFunctions["QueueRange"] = &QueueRangeWrapper; ///< Ext for ParseTypeWrapper compatibility ExtFunctions["PreserveStream"] = &PreserveStreamWrapper; ExtFunctions["FilePath"] = &FilePathWrapper; ExtFunctions["FileContent"] = &FileContentWrapper; @@ -13238,27 +13238,27 @@ template <NKikimr::NUdf::EDataSlot DataSlot> ExtFunctions["JsonExists"] = &JsonExistsWrapper; ExtFunctions["JsonQuery"] = &JsonQueryWrapper; ExtFunctions["JsonVariables"] = &JsonVariablesWrapper; - ExtFunctions["AssumeColumnOrder"] = &AssumeColumnOrderWrapper; - ExtFunctions["AssumeColumnOrderPartial"] = &AssumeColumnOrderWrapper; - ExtFunctions["UnionAllPositional"] = &UnionAllPositionalWrapper; + ExtFunctions["AssumeColumnOrder"] = &AssumeColumnOrderWrapper; + ExtFunctions["AssumeColumnOrderPartial"] = &AssumeColumnOrderWrapper; + ExtFunctions["UnionAllPositional"] = &UnionAllPositionalWrapper; ColumnOrderFunctions["PgSetItem"] = &OrderForPgSetItem; - ColumnOrderFunctions["AssumeColumnOrder"] = &OrderForAssumeColumnOrder; - - ColumnOrderFunctions["SqlProject"] = ColumnOrderFunctions["OrderedSqlProject"] = &OrderForSqlProject; - ColumnOrderFunctions["SqlAggregateAll"] = &OrderFromFirst; - - ColumnOrderFunctions["Merge"] = ColumnOrderFunctions["Extend"] = &OrderForMergeExtend; - ColumnOrderFunctions[RightName] = &OrderFromFirst; - ColumnOrderFunctions["UnionAll"] = &OrderForUnionAll; - ColumnOrderFunctions["EquiJoin"] = &OrderForEquiJoin; - - ColumnOrderFunctions["RemovePrefixMembers"] = &OrderFromFirstAndOutputType; - ColumnOrderFunctions["Sort"] = ColumnOrderFunctions["Take"] = ColumnOrderFunctions["Skip"] = - ColumnOrderFunctions["Filter"] = ColumnOrderFunctions["OrderedFilter"] = &OrderFromFirst; - ColumnOrderFunctions["AssumeSorted"] = ColumnOrderFunctions["Unordered"] = - ColumnOrderFunctions["UnorderedSubquery"] = ColumnOrderFunctions["AssumeUniq"] = &OrderFromFirst; - + ColumnOrderFunctions["AssumeColumnOrder"] = &OrderForAssumeColumnOrder; + + ColumnOrderFunctions["SqlProject"] = ColumnOrderFunctions["OrderedSqlProject"] = &OrderForSqlProject; + ColumnOrderFunctions["SqlAggregateAll"] = &OrderFromFirst; + + ColumnOrderFunctions["Merge"] = ColumnOrderFunctions["Extend"] = &OrderForMergeExtend; + ColumnOrderFunctions[RightName] = &OrderFromFirst; + ColumnOrderFunctions["UnionAll"] = &OrderForUnionAll; + ColumnOrderFunctions["EquiJoin"] = &OrderForEquiJoin; + + ColumnOrderFunctions["RemovePrefixMembers"] = &OrderFromFirstAndOutputType; + ColumnOrderFunctions["Sort"] = ColumnOrderFunctions["Take"] = ColumnOrderFunctions["Skip"] = + ColumnOrderFunctions["Filter"] = ColumnOrderFunctions["OrderedFilter"] = &OrderFromFirst; + ColumnOrderFunctions["AssumeSorted"] = ColumnOrderFunctions["Unordered"] = + ColumnOrderFunctions["UnorderedSubquery"] = ColumnOrderFunctions["AssumeUniq"] = &OrderFromFirst; + for (ui32 i = 0; i < NKikimr::NUdf::DataSlotCount; ++i) { auto name = TString(NKikimr::NUdf::GetDataTypeInfo((EDataSlot)i).Name); Functions[name] = &DataConstructorWrapper; @@ -13273,10 +13273,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot> AllNames.insert(func.first); } - for (auto& func : ExtFunctions) { - AllNames.insert(func.first); - } - + for (auto& func : ExtFunctions) { + AllNames.insert(func.first); + } + AllNames.insert(TString(CommitName)); AllNames.insert(TString(ReadName)); AllNames.insert(TString(WriteName)); @@ -13354,27 +13354,27 @@ template <NKikimr::NUdf::EDataSlot DataSlot> TMaybe<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { auto& functions = TSyncFunctionsMap::Instance().Functions; auto& extFunctions = TSyncFunctionsMap::Instance().ExtFunctions; - auto& columnOrderFunctions = TSyncFunctionsMap::Instance().ColumnOrderFunctions; + auto& columnOrderFunctions = TSyncFunctionsMap::Instance().ColumnOrderFunctions; auto name = input->Content(); - IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; + IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; if (auto func = functions.FindPtr(name)) { TContext funcCtx(ctx); - status = (*func)(input, output, funcCtx); + status = (*func)(input, output, funcCtx); } else if (auto func = extFunctions.FindPtr(name)) { TExtContext funcCtx(ctx, Types); - status = (*func)(input, output, funcCtx); + status = (*func)(input, output, funcCtx); } else { return Nothing(); } - - if (status == IGraphTransformer::TStatus::Ok && Types.OrderedColumns && !Types.LookupColumnOrder(*input)) { - if (auto func = columnOrderFunctions.FindPtr(name)) { - TExtContext funcCtx(ctx, Types); - status = (*func)(input, output, funcCtx); - } - } - - return status; + + if (status == IGraphTransformer::TStatus::Ok && Types.OrderedColumns && !Types.LookupColumnOrder(*input)) { + if (auto func = columnOrderFunctions.FindPtr(name)) { + TExtContext funcCtx(ctx, Types); + status = (*func)(input, output, funcCtx); + } + } + + return status; } TMaybe<IGraphTransformer::TStatus> ProcessList(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&) { @@ -13389,7 +13389,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> IGraphTransformer::TStatus ValidateProviderCommitResult(const TExprNode::TPtr& input, TExprContext& ctx) { if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink commit result")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink commit result")); return TStatus::Error; } return TStatus::Ok; @@ -13400,7 +13400,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple || input->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() != 2 || input->GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[0]->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasource read result")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasource read result")); return TStatus::Error; } return TStatus::Ok; @@ -13408,7 +13408,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> IGraphTransformer::TStatus ValidateProviderWriteResult(const TExprNode::TPtr& input, TExprContext& ctx) { if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink write result")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink write result")); return TStatus::Error; } return TStatus::Ok; @@ -13416,7 +13416,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> IGraphTransformer::TStatus ValidateProviderConfigureResult(const TExprNode::TPtr& input, TExprContext& ctx) { if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != input->Head().GetTypeAnn()->GetKind()) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad provider configure result")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad provider configure result")); return TStatus::Error; } return TStatus::Ok; diff --git a/ydb/library/yql/core/type_ann/type_ann_expr.cpp b/ydb/library/yql/core/type_ann/type_ann_expr.cpp index 193687502d..f67d3b4120 100644 --- a/ydb/library/yql/core/type_ann/type_ann_expr.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_expr.cpp @@ -56,7 +56,7 @@ public: Processed.clear(); if (status == TStatus::Ok) { Types.ExpectedTypes.clear(); - Types.ExpectedColumnOrders.clear(); + Types.ExpectedColumnOrders.clear(); } HasRenames = false; @@ -79,7 +79,7 @@ public: output = input; TStatus combinedStatus = TStatus::Ok; for (const auto& callable : CallableInputs) { - callable->SetState(TExprNode::EState::TypePending); + callable->SetState(TExprNode::EState::TypePending); TExprNode::TPtr callableOutput; auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx); Y_VERIFY(callableOutput); @@ -87,7 +87,7 @@ public: YQL_ENSURE(callableOutput == callable); combinedStatus = combinedStatus.Combine(status); if (status.Level == TStatus::Error) { - callable->SetState(TExprNode::EState::Error); + callable->SetState(TExprNode::EState::Error); } } @@ -133,7 +133,7 @@ private: return TStatus::Repeat; } - switch (start->GetState()) { + switch (start->GetState()) { case TExprNode::EState::Initial: return TStatus(TStatus::Repeat, true); case TExprNode::EState::TypeInProgress: @@ -170,7 +170,7 @@ private: auto input = start; for (;;) { - TIssueScopeGuard issueScope(ctx.IssueManager, [this, input, &ctx]() -> TIssuePtr { + TIssueScopeGuard issueScope(ctx.IssueManager, [this, input, &ctx]() -> TIssuePtr { TStringBuilder str; str << "At "; switch (input->Type()) { @@ -209,7 +209,7 @@ private: str << "unknown"; } - return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), str); + return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), str); }); if (input->Type() == TExprNode::Callable) { @@ -225,7 +225,7 @@ private: }; TStatus retStatus = TStatus::Error; - switch (input->GetState()) { + switch (input->GetState()) { case TExprNode::EState::Initial: break; case TExprNode::EState::TypeInProgress: @@ -247,12 +247,12 @@ private: YQL_ENSURE(false, "Unknown state"); } - input->SetState(TExprNode::EState::TypePending); + input->SetState(TExprNode::EState::TypePending); switch (input->Type()) { case TExprNode::Atom: { input->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - CheckExpected(*input, ctx); + CheckExpected(*input, ctx); return TStatus::Ok; } @@ -274,7 +274,7 @@ private: if (combinedStatus != TStatus::Ok) { if (combinedStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (updatedChildren) { input->ChangeChildrenInplace(std::move(newChildren)); @@ -289,7 +289,7 @@ private: bool isUnit = false; for (auto& child : input->Children()) { if (!EnsureComposable(*child, ctx)) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); return TStatus::Error; } @@ -302,7 +302,7 @@ private: input->SetTypeAnn(isUnit ? (const TTypeAnnotationNode*)ctx.MakeType<TUnitExprType>() : ctx.MakeType<TTupleExprType>(children)); - CheckExpected(*input, ctx); + CheckExpected(*input, ctx); return TStatus::Ok; } @@ -318,7 +318,7 @@ private: auto argStatus = TransformNode(input->HeadPtr(), out, ctx); UpdateStatusIfChanged(argStatus, input->HeadPtr(), out); if (argStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); return argStatus; } @@ -342,7 +342,7 @@ private: if (combinedStatus != TStatus::Ok) { if (combinedStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (updatedChildren) { input->ChangeChildrenInplace(std::move(newChildren)); @@ -364,7 +364,7 @@ private: } if (input->GetTypeAnn()) { - CheckExpected(*input, ctx); + CheckExpected(*input, ctx); } return TStatus::Ok; @@ -373,8 +373,8 @@ private: case TExprNode::Argument: if (input->GetTypeAnn()) { if (input->Type() == TExprNode::Lambda) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Unable to use lambda as argument")); - input->SetState(TExprNode::EState::Error); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Unable to use lambda as argument")); + input->SetState(TExprNode::EState::Error); return TStatus::Error; } } @@ -403,7 +403,7 @@ private: if (combinedStatus != TStatus::Ok) { if (combinedStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (updatedChildren) { input->ChangeChildrenInplace(std::move(newChildren)); @@ -425,23 +425,23 @@ private: } if (status == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); return status; } if (status == TStatus::Ok) { if (!input->GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Node is not annotated yet")); - input->SetState(TExprNode::EState::Error); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Node is not annotated yet")); + input->SetState(TExprNode::EState::Error); return TStatus::Error; } - input->SetState(TExprNode::EState::TypeComplete); - CheckExpected(*input, ctx); + input->SetState(TExprNode::EState::TypeComplete); + CheckExpected(*input, ctx); } else if (status == TStatus::Async) { CallableInputs.push_back(input); - input->SetState(TExprNode::EState::TypeInProgress); + input->SetState(TExprNode::EState::TypeInProgress); } else { RepeatCallableCount[input.Get()->Content()] += 1; if (output != input.Get()) { @@ -459,7 +459,7 @@ private: case TExprNode::World: { input->SetTypeAnn(ctx.MakeType<TWorldExprType>()); - CheckExpected(*input, ctx); + CheckExpected(*input, ctx); return TStatus::Ok; } @@ -467,7 +467,7 @@ private: { if (input->Children().empty()) { if (input->GetTypeAnn()) { - input->SetState(TExprNode::EState::TypeComplete); + input->SetState(TExprNode::EState::TypeComplete); return TStatus::Ok; } @@ -485,7 +485,7 @@ private: if (combinedStatus != TStatus::Ok) { if (combinedStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } return combinedStatus; @@ -513,32 +513,32 @@ private: } } - void CheckExpected(const TExprNode& input, TExprContext& ctx) { - Y_UNUSED(ctx); + void CheckExpected(const TExprNode& input, TExprContext& ctx) { + Y_UNUSED(ctx); auto it = Types.ExpectedTypes.find(input.UniqueId()); if (it != Types.ExpectedTypes.end()) { YQL_ENSURE(IsSameAnnotation(*input.GetTypeAnn(), *it->second), "Rewrite error, type should be : " << *it->second << ", but it is: " << *input.GetTypeAnn() << " for node " << input.Content()); } - - auto coIt = Types.ExpectedColumnOrders.find(input.UniqueId()); - if (coIt != Types.ExpectedColumnOrders.end()) { - TColumnOrder oldColumnOrder = coIt->second; - TMaybe<TColumnOrder> newColumnOrder = Types.LookupColumnOrder(input); - if (!newColumnOrder) { - // keep column order after rewrite - // TODO: check if needed - auto status = Types.SetColumnOrder(input, oldColumnOrder, ctx); - YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); - } else { - YQL_ENSURE(newColumnOrder == oldColumnOrder, - "Rewrite error, column order should be: " - << FormatColumnOrder(oldColumnOrder) << ", but it is: " - << FormatColumnOrder(newColumnOrder) << " for node " - << input.Content()); - } - } + + auto coIt = Types.ExpectedColumnOrders.find(input.UniqueId()); + if (coIt != Types.ExpectedColumnOrders.end()) { + TColumnOrder oldColumnOrder = coIt->second; + TMaybe<TColumnOrder> newColumnOrder = Types.LookupColumnOrder(input); + if (!newColumnOrder) { + // keep column order after rewrite + // TODO: check if needed + auto status = Types.SetColumnOrder(input, oldColumnOrder, ctx); + YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); + } else { + YQL_ENSURE(newColumnOrder == oldColumnOrder, + "Rewrite error, column order should be: " + << FormatColumnOrder(oldColumnOrder) << ", but it is: " + << FormatColumnOrder(newColumnOrder) << " for node " + << input.Content()); + } + } } private: @@ -557,8 +557,8 @@ private: IGraphTransformer::TStatus CheckWholeProgramType(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { output = input; if (input->Type() == TExprNode::Lambda || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Return must be world")); - input->SetState(TExprNode::EState::Error); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Return must be world")); + input->SetState(TExprNode::EState::Error); return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/type_ann/type_ann_impl.h b/ydb/library/yql/core/type_ann/type_ann_impl.h index ddf7881c84..3d940ebd73 100644 --- a/ydb/library/yql/core/type_ann/type_ann_impl.h +++ b/ydb/library/yql/core/type_ann/type_ann_impl.h @@ -33,8 +33,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus GroupingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TStructExprType& structType, TContext& ctx); + TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TStructExprType& structType, TContext& ctx); - TExprNode::TPtr MakeNothingData(TExprContext& ctx, TPositionHandle pos, TStringBuf data); + TExprNode::TPtr MakeNothingData(TExprContext& ctx, TPositionHandle pos, TStringBuf data); } // namespace NTypeAnnImpl } // namespace NYql diff --git a/ydb/library/yql/core/type_ann/type_ann_join.cpp b/ydb/library/yql/core/type_ann/type_ann_join.cpp index e8b404daf8..18516a8ab8 100644 --- a/ydb/library/yql/core/type_ann/type_ann_join.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_join.cpp @@ -34,7 +34,7 @@ namespace NTypeAnnImpl { auto joinKind = input->Child(4)->Content(); if (joinKind != "Inner" && joinKind != "Left" && joinKind != "Right" && joinKind != "Full") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(4)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(4)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind << ", supported: Inner, Right, Left, Full")); return IGraphTransformer::TStatus::Error; } @@ -106,7 +106,7 @@ namespace NTypeAnnImpl { } if (!IsSameAnnotation(*lambda1->GetTypeAnn(), *lambda2->GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of key extractors types, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of key extractors types, " << *lambda1->GetTypeAnn() << " != " << *lambda2->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -240,9 +240,9 @@ namespace NTypeAnnImpl { } const size_t numLists = input->ChildrenSize() - 2; - auto optionsNode = input->Child(input->ChildrenSize() - 1); - TJoinOptions options; - auto status = ValidateEquiJoinOptions(input->Pos(), *optionsNode, options, ctx.Expr); + auto optionsNode = input->Child(input->ChildrenSize() - 1); + TJoinOptions options; + auto status = ValidateEquiJoinOptions(input->Pos(), *optionsNode, options, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -263,14 +263,14 @@ namespace NTypeAnnImpl { const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType(); if (itemType->GetKind() != ETypeAnnotationKind::Struct) { ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(list.Pos()), + ctx.Expr.GetPosition(list.Pos()), TStringBuilder() << "Expected list of struct" )); return IGraphTransformer::TStatus::Error; } auto structType = itemType->Cast<TStructExprType>(); - if (!options.KeepSysColumns && AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) { + if (!options.KeepSysColumns && AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) { if (updatedChildren.empty()) { updatedChildren = input->ChildrenList(); } @@ -286,10 +286,10 @@ namespace NTypeAnnImpl { ); continue; } - if (auto err = labels.Add(ctx.Expr, *listPair.Child(1), structType)) { + if (auto err = labels.Add(ctx.Expr, *listPair.Child(1), structType)) { ctx.Expr.AddError(*err); ctx.Expr.AddError(TIssue( - ctx.Expr.GetPosition(input->Child(idx)->Pos()), + ctx.Expr.GetPosition(input->Child(idx)->Pos()), TStringBuilder() << "Failed to parse labels of struct as second element of " << idx << " argument" )); return IGraphTransformer::TStatus::Error; @@ -302,7 +302,7 @@ namespace NTypeAnnImpl { auto joins = input->Child(input->ChildrenSize() - 2); const TStructExprType* resultType = nullptr; - status = EquiJoinAnnotation(input->Pos(), resultType, labels, *joins, options, ctx.Expr); + status = EquiJoinAnnotation(input->Pos(), resultType, labels, *joins, options, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -335,7 +335,7 @@ namespace NTypeAnnImpl { const auto joinKind = input->Child(2)->Content(); if (joinKind != "Inner" && joinKind != "Left" && joinKind != "LeftSemi" && joinKind != "LeftOnly") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind << ", supported: Inner, Left, LeftSemi, LeftOnly")); return IGraphTransformer::TStatus::Error; } @@ -399,12 +399,12 @@ namespace NTypeAnnImpl { const auto oldPos = GetFieldPosition(leftItemType, oldName->Content()); if (!oldPos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(oldName->Pos()), TStringBuilder() << "Unknown column: " << oldName->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(oldName->Pos()), TStringBuilder() << "Unknown column: " << oldName->Content())); return IGraphTransformer::TStatus::Error; } if (newName->Content().empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(newName->Pos()), "Empty column is not allowed")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(newName->Pos()), "Empty column is not allowed")); return IGraphTransformer::TStatus::Error; } @@ -443,12 +443,12 @@ namespace NTypeAnnImpl { const auto oldPos = rightStructType ? GetFieldPosition(*rightStructType, oldName->Content()) : GetFieldPosition(*rightTupleType, oldName->Content()); if (!oldPos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(oldName->Pos()), TStringBuilder() << "Unknown column: " << oldName->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(oldName->Pos()), TStringBuilder() << "Unknown column: " << oldName->Content())); return IGraphTransformer::TStatus::Error; } if (newName->Content().empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(newName->Pos()), "Empty column is not allowed")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(newName->Pos()), "Empty column is not allowed")); return IGraphTransformer::TStatus::Error; } @@ -458,7 +458,7 @@ namespace NTypeAnnImpl { } auto columnType = rightStructType ? GetFieldType(*rightStructType, *oldPos) : GetFieldType(*rightTupleType, *oldPos); - if (joinKind == "Left" && !columnType->IsOptionalOrNull()) { + if (joinKind == "Left" && !columnType->IsOptionalOrNull()) { columnType = ctx.Expr.MakeType<TOptionalExprType>(columnType); } @@ -530,7 +530,7 @@ namespace NTypeAnnImpl { if (joinKind != "Inner" && joinKind != "Left" && joinKind != "Right" && joinKind != "Full" && joinKind != "LeftOnly" && joinKind != "RightOnly" && joinKind != "Exclusion" && joinKind != "LeftSemi" && joinKind != "RightSemi" && joinKind != "Cross") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown join kind: " << joinKind << ", supported: Inner, Right, Left, Full, LeftOnly, RightOnly, Exclusion, LeftSemi, RightSemi, Cross")); return IGraphTransformer::TStatus::Error; } @@ -565,17 +565,17 @@ namespace NTypeAnnImpl { const auto pos = GetFieldPosition(inputItemType, child->Content()); if (!pos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (!leftColumns.insert(child->Content()).second || !fullColumns.insert(child->Content()).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (const auto inputColumnType = GetFieldType(inputItemType, *pos); !inputColumnType->IsOptionalOrNull()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected optional or null type for column: " << child->Content() << ", but got: " << *inputColumnType)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected optional or null type for column: " << child->Content() << ", but got: " << *inputColumnType)); return IGraphTransformer::TStatus::Error; } } @@ -597,17 +597,17 @@ namespace NTypeAnnImpl { const auto pos = GetFieldPosition(inputItemType, child->Content()); if (!pos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (!rightColumns.insert(child->Content()).second || !fullColumns.insert(child->Content()).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (const auto inputColumnType = GetFieldType(inputItemType, *pos); !inputColumnType->IsOptionalOrNull()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected optional or null type for column: " << child->Content() << ", but got: " << *inputColumnType)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected optional or null type for column: " << child->Content() << ", but got: " << *inputColumnType)); return IGraphTransformer::TStatus::Error; } } @@ -623,23 +623,23 @@ namespace NTypeAnnImpl { } if (!fullColumns.contains(child->Content())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (!requiredColumns.insert(child->Content()).second) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplication of column: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (leftColumns.contains(child->Content())) { if (IsLeftJoinSideOptional(joinKind)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Required column " << child->Content() << " cannot be at the left side for the join kind: " << joinKind)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Required column " << child->Content() << " cannot be at the left side for the join kind: " << joinKind)); return IGraphTransformer::TStatus::Error; } } else { if (IsRightJoinSideOptional(joinKind)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Required column " << child->Content() << " cannot be at the right side for the join kind: " << joinKind)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Required column " << child->Content() << " cannot be at the right side for the join kind: " << joinKind)); return IGraphTransformer::TStatus::Error; } } @@ -655,7 +655,7 @@ namespace NTypeAnnImpl { } if (const auto pos = GetFieldPosition(inputItemType, child->Content()); !pos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown column: " << child->Content())); return IGraphTransformer::TStatus::Error; } } @@ -664,7 +664,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - TSet<TStringBuf> seenOptions; + TSet<TStringBuf> seenOptions; for (const auto& child : input->Child(6)->Children()) { if (!EnsureTupleMinSize(*child, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -676,9 +676,9 @@ namespace NTypeAnnImpl { if (const auto optionName = child->Head().Content(); !seenOptions.insert(optionName).second) { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Head().Pos()), TStringBuilder() << - "Duplicate option: " << optionName)); - return IGraphTransformer::TStatus::Error; - } + "Duplicate option: " << optionName)); + return IGraphTransformer::TStatus::Error; + } else if (optionName == "sorted") { if (!EnsureTupleSize(*child, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -689,7 +689,7 @@ namespace NTypeAnnImpl { } if (const auto side = child->Child(1)->Content(); side != "left" && side != "right") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), TStringBuilder() << "Unknown sorted side, expected left or right, but got: " << side)); return IGraphTransformer::TStatus::Error; } @@ -704,41 +704,41 @@ namespace NTypeAnnImpl { } if (ui64 memLimit = 0ULL; !TryFromString(child->Child(1)->Content(), memLimit)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), TStringBuilder() << "Bad memLimit value: " << child->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } } - else if (optionName == "any") { - if (!EnsureTupleSize(*child, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureTuple(*child->Child(1), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - for (auto sideNode : child->Child(1)->Children()) { - if (!EnsureAtom(*sideNode, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto side = sideNode->Content(); - if (side != "left" && side != "right") { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sideNode->Pos()), TStringBuilder() << - "Unknown any side, expected left or right, but got: " << side)); - return IGraphTransformer::TStatus::Error; - } - } - } - else if (optionName == "forceSortedMerge") { - if (!EnsureTupleSize(*child, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - } + else if (optionName == "any") { + if (!EnsureTupleSize(*child, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureTuple(*child->Child(1), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + for (auto sideNode : child->Child(1)->Children()) { + if (!EnsureAtom(*sideNode, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto side = sideNode->Content(); + if (side != "left" && side != "right") { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sideNode->Pos()), TStringBuilder() << + "Unknown any side, expected left or right, but got: " << side)); + return IGraphTransformer::TStatus::Error; + } + } + } + else if (optionName == "forceSortedMerge") { + if (!EnsureTupleSize(*child, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown option name: " << optionName)); return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp index 8250224402..b145cde389 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp @@ -9,14 +9,14 @@ #include <ydb/library/yql/core/yql_type_helpers.h> #include <util/generic/algorithm.h> -#include <util/string/join.h> +#include <util/string/join.h> namespace NYql { namespace NTypeAnnImpl { using namespace NNodes; -namespace { +namespace { bool IsEmptyList(const TExprNode::TPtr& x) { return x->GetTypeAnn() && x->GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList; }; @@ -118,7 +118,7 @@ namespace { } TNodeOnNodeOwnedMap factoryReplaces; - factoryReplaces[traitsFactory->Head().Child(0)] = listType ? ExpandType(node.Pos(), *listType->GetItemType(), ctx) : node.HeadPtr(); + factoryReplaces[traitsFactory->Head().Child(0)] = listType ? ExpandType(node.Pos(), *listType->GetItemType(), ctx) : node.HeadPtr(); factoryReplaces[traitsFactory->Head().Child(1)] = extractor; auto traits = ctx.ReplaceNodes(traitsFactory->ChildPtr(1), factoryReplaces); @@ -295,88 +295,88 @@ namespace { } } - if (!listType) { + if (!listType) { ui32 defValueIndex = onWindow ? 5 : 7; - if (!traits->Child(defValueIndex)->IsCallable("Null")) { + if (!traits->Child(defValueIndex)->IsCallable("Null")) { isNullDefValue = false; - isLambdaDefValue = isLambdaDefValue || traits->ChildPtr(defValueIndex)->IsLambda(); + isLambdaDefValue = isLambdaDefValue || traits->ChildPtr(defValueIndex)->IsLambda(); if (structType) { - defValueArgs.push_back(ctx.NewList(node.Pos(), { memberAtom, traits->ChildPtr(defValueIndex) })); + defValueArgs.push_back(ctx.NewList(node.Pos(), { memberAtom, traits->ChildPtr(defValueIndex) })); } else { - defValueArgs.push_back(traits->ChildPtr(defValueIndex)); + defValueArgs.push_back(traits->ChildPtr(defValueIndex)); } } } } - TExprNode::TPtr defValue; - if (listType || isNullDefValue) { - defValue = ctx.NewCallable(node.Pos(), "Null", {}); - } else if (!isLambdaDefValue) { - defValue = structType ? - ctx.NewCallable(node.Pos(), "AsStruct", std::move(defValueArgs)) : - ctx.NewList(node.Pos(), std::move(defValueArgs)); - } else { - if (structType) { - defValue = ctx.Builder(node.Pos()) - .Lambda() - .Param("type") - .Callable("AsStruct") - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < defValueArgs.size(); ++i) { - const auto& x = defValueArgs[i]; - auto list = parent.List(i); - list.Add(0, x->HeadPtr()); - auto value = x->ChildPtr(1); - if (value->IsLambda()) { - list.Apply(1, value) - .With(0) - .Callable("StructMemberType") - .Arg(0, "type") - .Add(1, x->HeadPtr()) - .Seal() - .Done().Seal(); - } else { - list.Add(1, value); - } - - list.Seal(); - } - - return parent; - }) - .Seal() - .Seal() - .Build(); - } else { - defValue = ctx.Builder(node.Pos()) - .Lambda() - .Param("type") - .List() - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < defValueArgs.size(); ++i) { - const auto& value = defValueArgs[i]; - if (value->IsLambda()) { - parent.Apply(i, value) - .With(0) - .Callable("TupleElementType") - .Arg(0, "type") - .Atom(1, ToString(i)) - .Seal() - .Done().Seal(); - } else { - parent.Add(i, value); - } - } - - return parent; - }) - .Seal() - .Seal() - .Build(); - } - } - + TExprNode::TPtr defValue; + if (listType || isNullDefValue) { + defValue = ctx.NewCallable(node.Pos(), "Null", {}); + } else if (!isLambdaDefValue) { + defValue = structType ? + ctx.NewCallable(node.Pos(), "AsStruct", std::move(defValueArgs)) : + ctx.NewList(node.Pos(), std::move(defValueArgs)); + } else { + if (structType) { + defValue = ctx.Builder(node.Pos()) + .Lambda() + .Param("type") + .Callable("AsStruct") + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (ui32 i = 0; i < defValueArgs.size(); ++i) { + const auto& x = defValueArgs[i]; + auto list = parent.List(i); + list.Add(0, x->HeadPtr()); + auto value = x->ChildPtr(1); + if (value->IsLambda()) { + list.Apply(1, value) + .With(0) + .Callable("StructMemberType") + .Arg(0, "type") + .Add(1, x->HeadPtr()) + .Seal() + .Done().Seal(); + } else { + list.Add(1, value); + } + + list.Seal(); + } + + return parent; + }) + .Seal() + .Seal() + .Build(); + } else { + defValue = ctx.Builder(node.Pos()) + .Lambda() + .Param("type") + .List() + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (ui32 i = 0; i < defValueArgs.size(); ++i) { + const auto& value = defValueArgs[i]; + if (value->IsLambda()) { + parent.Apply(i, value) + .With(0) + .Callable("TupleElementType") + .Arg(0, "type") + .Atom(1, ToString(i)) + .Seal() + .Done().Seal(); + } else { + parent.Add(i, value); + } + } + + return parent; + }) + .Seal() + .Seal() + .Build(); + } + } + if (onWindow) { return ctx.NewCallable(node.Pos(), "WindowTraits", { node.HeadPtr(), @@ -399,8 +399,8 @@ namespace { ctx.NewArguments(node.Pos(), { finishArg1 }), !tupleType ? ctx.NewCallable(node.Pos(), structType ? "AsStruct" : "Map", std::move(finishBodyArgs)) : - ctx.NewList(node.Pos(), std::move(finishBodyArgs))), - defValue }); + ctx.NewList(node.Pos(), std::move(finishBodyArgs))), + defValue }); } else { return ctx.NewCallable(node.Pos(), "AggregationTraits", { node.HeadPtr(), @@ -442,189 +442,189 @@ namespace { defValue }); } } - - IGraphTransformer::TStatus ValidateCalcOverWindowArgs(TVector<const TItemExprType*>& outputStructType, - const TStructExprType& inputStructType, const TExprNode& partitionBy, const TExprNode& sortSpec, const TExprNode& winList, - const TExprNode::TPtr& sessionSpec, const TExprNode::TPtr& sessionColumns, TExprContext& ctx) - { - YQL_ENSURE(sessionSpec ? bool(sessionColumns) : !sessionColumns); - if (!EnsureTuple(partitionBy, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - for (auto& child : partitionBy.Children()) { - if (!EnsureAtom(*child, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto item = inputStructType.FindItem(child->Content()); - if (!item) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << - child->Content() << ", type: " << static_cast<const TTypeAnnotationNode&>(inputStructType))); - return IGraphTransformer::TStatus::Error; - } - - auto columnType = inputStructType.GetItems()[*item]->GetItemType(); - if (!columnType->IsHashable() || !columnType->IsEquatable()) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for key column: " << - child->Content() << ", but got: " << *columnType)); - return IGraphTransformer::TStatus::Error; - } - } - - if (!sortSpec.IsCallable({"Void", "SortTraits"})) { - ctx.AddError(TIssue(ctx.GetPosition(sortSpec.Pos()), "Expected sort traits or Void")); - return IGraphTransformer::TStatus::Error; - } - - if (sessionSpec && !sessionSpec->IsCallable({"Void", "SessionWindowTraits"})) { - ctx.AddError(TIssue(ctx.GetPosition(sessionSpec->Pos()), "Expected SessionWindowTraits or Void")); - return IGraphTransformer::TStatus::Error; - } - - if (sessionColumns) { - if (!EnsureTuple(*sessionColumns, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (sessionColumns->ChildrenSize()) { - if (!sessionSpec->IsCallable("SessionWindowTraits")) { - ctx.AddError(TIssue(ctx.GetPosition(sessionSpec->Pos()), "Got non-empty session columns list without session traits")); - return IGraphTransformer::TStatus::Error; - } - - TCoSessionWindowTraits traits(sessionSpec); - - TVector<const TItemExprType*> sessionStructItems; - sessionStructItems.push_back(ctx.MakeType<TItemExprType>("start", traits.Calculate().Ref().GetTypeAnn())); - sessionStructItems.push_back(ctx.MakeType<TItemExprType>("state", traits.InitState().Ref().GetTypeAnn())); - auto sessionStructType = ctx.MakeType<TStructExprType>(sessionStructItems); - - for (auto& column : sessionColumns->ChildrenList()) { - if (!EnsureAtom(*column, ctx)) { - return IGraphTransformer::TStatus::Error; - } - outputStructType.push_back(ctx.MakeType<TItemExprType>(column->Content(), sessionStructType)); - } - } - } - - if (!EnsureTuple(winList, ctx)) { - return IGraphTransformer::TStatus::Error; - } - for (auto winOn: winList.Children()) { - if (!winOn->IsCallable("WinOnRows")) { - ctx.AddError(TIssue(ctx.GetPosition(winOn->Pos()), "Expected window params based in callable WinOnRows")); - return IGraphTransformer::TStatus::Error; - } - - bool frameCanBeEmpty = false; - { - TWindowFrameSettings frameSettings; - YQL_ENSURE(ParseWindowFrameSettings(*winOn, frameSettings, ctx)); - frameCanBeEmpty = !frameSettings.NeverEmpty; - } - - for (auto iterFunc = winOn->Children().begin() + 1; iterFunc != winOn->Children().end(); ++iterFunc) { - auto func = *iterFunc; - YQL_ENSURE(func->IsList()); - YQL_ENSURE(func->Child(0)->IsAtom()); - - const auto paramName = func->Child(0)->Content(); - const auto calcSpec = func->Child(1); - YQL_ENSURE(calcSpec->IsCallable({"Lag", "Lead", "RowNumber", "Rank", "DenseRank", "WindowTraits"})); - - auto traitsInputTypeNode = calcSpec->Child(0); - YQL_ENSURE(traitsInputTypeNode->GetTypeAnn()); - const TTypeAnnotationNode* traitsInputItemType = traitsInputTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (!calcSpec->IsCallable("WindowTraits")) { - traitsInputItemType = traitsInputItemType->Cast<TListExprType>()->GetItemType(); - } - - const TStructExprType* traitsInputStruct = traitsInputItemType->Cast<TStructExprType>(); - // traitsInputStruct should be subset of inputStruct - for (auto& item : traitsInputStruct->GetItems()) { - auto name = item->GetName(); - auto type = item->GetItemType(); - if (auto idx = inputStructType.FindItem(name)) { - if (inputStructType.GetItems()[*idx]->GetItemType() == type) { - continue; - } - } - ctx.AddError(TIssue(ctx.GetPosition(traitsInputTypeNode->Pos()), TStringBuilder() << "Invalid " << - calcSpec->Content() << " traits input type: " << *traitsInputItemType << ", expecting subset of " << - static_cast<const TTypeAnnotationNode&>(inputStructType))); - return IGraphTransformer::TStatus::Error; - } - - if (calcSpec->IsCallable("WindowTraits")) { - auto finishType = calcSpec->Child(4)->GetTypeAnn(); - if (frameCanBeEmpty) { - bool isOptional = finishType->GetKind() == ETypeAnnotationKind::Optional; - - auto defVal = calcSpec->Child(5); - if (defVal->IsCallable("Null") && !isOptional) { - finishType = ctx.MakeType<TOptionalExprType>(finishType); - } - - if (!defVal->IsCallable("Null") && defVal->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional - && finishType->GetKind() == ETypeAnnotationKind::Optional) { - finishType = finishType->Cast<TOptionalExprType>()->GetItemType(); - } - } - outputStructType.push_back(ctx.MakeType<TItemExprType>(paramName, finishType)); - } else { - outputStructType.push_back(ctx.MakeType<TItemExprType>(paramName, calcSpec->GetTypeAnn())); - } - } - } - - return IGraphTransformer::TStatus::Ok; - } - - - IGraphTransformer::TStatus ValidateTraitsDefaultValue(const TExprNode::TPtr& input, ui32 defaultValueIndex, - const TTypeAnnotationNode& finishType, TStringBuf finishName, TExprNode::TPtr& output, TContext& ctx) - { - auto defaultValue = input->Child(defaultValueIndex); + + IGraphTransformer::TStatus ValidateCalcOverWindowArgs(TVector<const TItemExprType*>& outputStructType, + const TStructExprType& inputStructType, const TExprNode& partitionBy, const TExprNode& sortSpec, const TExprNode& winList, + const TExprNode::TPtr& sessionSpec, const TExprNode::TPtr& sessionColumns, TExprContext& ctx) + { + YQL_ENSURE(sessionSpec ? bool(sessionColumns) : !sessionColumns); + if (!EnsureTuple(partitionBy, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + for (auto& child : partitionBy.Children()) { + if (!EnsureAtom(*child, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto item = inputStructType.FindItem(child->Content()); + if (!item) { + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << + child->Content() << ", type: " << static_cast<const TTypeAnnotationNode&>(inputStructType))); + return IGraphTransformer::TStatus::Error; + } + + auto columnType = inputStructType.GetItems()[*item]->GetItemType(); + if (!columnType->IsHashable() || !columnType->IsEquatable()) { + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for key column: " << + child->Content() << ", but got: " << *columnType)); + return IGraphTransformer::TStatus::Error; + } + } + + if (!sortSpec.IsCallable({"Void", "SortTraits"})) { + ctx.AddError(TIssue(ctx.GetPosition(sortSpec.Pos()), "Expected sort traits or Void")); + return IGraphTransformer::TStatus::Error; + } + + if (sessionSpec && !sessionSpec->IsCallable({"Void", "SessionWindowTraits"})) { + ctx.AddError(TIssue(ctx.GetPosition(sessionSpec->Pos()), "Expected SessionWindowTraits or Void")); + return IGraphTransformer::TStatus::Error; + } + + if (sessionColumns) { + if (!EnsureTuple(*sessionColumns, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (sessionColumns->ChildrenSize()) { + if (!sessionSpec->IsCallable("SessionWindowTraits")) { + ctx.AddError(TIssue(ctx.GetPosition(sessionSpec->Pos()), "Got non-empty session columns list without session traits")); + return IGraphTransformer::TStatus::Error; + } + + TCoSessionWindowTraits traits(sessionSpec); + + TVector<const TItemExprType*> sessionStructItems; + sessionStructItems.push_back(ctx.MakeType<TItemExprType>("start", traits.Calculate().Ref().GetTypeAnn())); + sessionStructItems.push_back(ctx.MakeType<TItemExprType>("state", traits.InitState().Ref().GetTypeAnn())); + auto sessionStructType = ctx.MakeType<TStructExprType>(sessionStructItems); + + for (auto& column : sessionColumns->ChildrenList()) { + if (!EnsureAtom(*column, ctx)) { + return IGraphTransformer::TStatus::Error; + } + outputStructType.push_back(ctx.MakeType<TItemExprType>(column->Content(), sessionStructType)); + } + } + } + + if (!EnsureTuple(winList, ctx)) { + return IGraphTransformer::TStatus::Error; + } + for (auto winOn: winList.Children()) { + if (!winOn->IsCallable("WinOnRows")) { + ctx.AddError(TIssue(ctx.GetPosition(winOn->Pos()), "Expected window params based in callable WinOnRows")); + return IGraphTransformer::TStatus::Error; + } + + bool frameCanBeEmpty = false; + { + TWindowFrameSettings frameSettings; + YQL_ENSURE(ParseWindowFrameSettings(*winOn, frameSettings, ctx)); + frameCanBeEmpty = !frameSettings.NeverEmpty; + } + + for (auto iterFunc = winOn->Children().begin() + 1; iterFunc != winOn->Children().end(); ++iterFunc) { + auto func = *iterFunc; + YQL_ENSURE(func->IsList()); + YQL_ENSURE(func->Child(0)->IsAtom()); + + const auto paramName = func->Child(0)->Content(); + const auto calcSpec = func->Child(1); + YQL_ENSURE(calcSpec->IsCallable({"Lag", "Lead", "RowNumber", "Rank", "DenseRank", "WindowTraits"})); + + auto traitsInputTypeNode = calcSpec->Child(0); + YQL_ENSURE(traitsInputTypeNode->GetTypeAnn()); + const TTypeAnnotationNode* traitsInputItemType = traitsInputTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (!calcSpec->IsCallable("WindowTraits")) { + traitsInputItemType = traitsInputItemType->Cast<TListExprType>()->GetItemType(); + } + + const TStructExprType* traitsInputStruct = traitsInputItemType->Cast<TStructExprType>(); + // traitsInputStruct should be subset of inputStruct + for (auto& item : traitsInputStruct->GetItems()) { + auto name = item->GetName(); + auto type = item->GetItemType(); + if (auto idx = inputStructType.FindItem(name)) { + if (inputStructType.GetItems()[*idx]->GetItemType() == type) { + continue; + } + } + ctx.AddError(TIssue(ctx.GetPosition(traitsInputTypeNode->Pos()), TStringBuilder() << "Invalid " << + calcSpec->Content() << " traits input type: " << *traitsInputItemType << ", expecting subset of " << + static_cast<const TTypeAnnotationNode&>(inputStructType))); + return IGraphTransformer::TStatus::Error; + } + + if (calcSpec->IsCallable("WindowTraits")) { + auto finishType = calcSpec->Child(4)->GetTypeAnn(); + if (frameCanBeEmpty) { + bool isOptional = finishType->GetKind() == ETypeAnnotationKind::Optional; + + auto defVal = calcSpec->Child(5); + if (defVal->IsCallable("Null") && !isOptional) { + finishType = ctx.MakeType<TOptionalExprType>(finishType); + } + + if (!defVal->IsCallable("Null") && defVal->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional + && finishType->GetKind() == ETypeAnnotationKind::Optional) { + finishType = finishType->Cast<TOptionalExprType>()->GetItemType(); + } + } + outputStructType.push_back(ctx.MakeType<TItemExprType>(paramName, finishType)); + } else { + outputStructType.push_back(ctx.MakeType<TItemExprType>(paramName, calcSpec->GetTypeAnn())); + } + } + } + + return IGraphTransformer::TStatus::Ok; + } + + + IGraphTransformer::TStatus ValidateTraitsDefaultValue(const TExprNode::TPtr& input, ui32 defaultValueIndex, + const TTypeAnnotationNode& finishType, TStringBuf finishName, TExprNode::TPtr& output, TContext& ctx) + { + auto defaultValue = input->Child(defaultValueIndex); if (!defaultValue->IsCallable({"Null", "EmptyList"})) { - if (defaultValue->IsLambda()) { - if (!EnsureMinArgsCount(defaultValue->Head(), 0, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureMaxArgsCount(defaultValue->Head(), 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto newDefault = defaultValue->ChildPtr(1); - if (defaultValue->Head().ChildrenSize() == 1) { - newDefault = ctx.Expr.ReplaceNode(std::move(newDefault), defaultValue->Head().Head(), - ExpandType(input->Pos(), finishType, ctx.Expr)); - } - - output = ctx.Expr.ChangeChild(*input, defaultValueIndex, std::move(newDefault)); - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureComputable(*defaultValue, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto finishItemType = RemoveOptionalType(&finishType); + if (defaultValue->IsLambda()) { + if (!EnsureMinArgsCount(defaultValue->Head(), 0, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureMaxArgsCount(defaultValue->Head(), 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto newDefault = defaultValue->ChildPtr(1); + if (defaultValue->Head().ChildrenSize() == 1) { + newDefault = ctx.Expr.ReplaceNode(std::move(newDefault), defaultValue->Head().Head(), + ExpandType(input->Pos(), finishType, ctx.Expr)); + } + + output = ctx.Expr.ChangeChild(*input, defaultValueIndex, std::move(newDefault)); + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureComputable(*defaultValue, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto finishItemType = RemoveOptionalType(&finishType); if (finishItemType->GetKind() != ETypeAnnotationKind::Null && finishItemType->GetKind() != ETypeAnnotationKind::EmptyList && !IsSameAnnotation(*finishItemType, *defaultValue->GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(defaultValue->Pos()), - TStringBuilder() << "Mismatch of default value type and " << finishName - << " type: " << *defaultValue->GetTypeAnn() << " != " << finishType)); - return IGraphTransformer::TStatus::Error; - } - } - - return IGraphTransformer::TStatus::Ok; - } -} // namespace - + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(defaultValue->Pos()), + TStringBuilder() << "Mismatch of default value type and " << finishName + << " type: " << *defaultValue->GetTypeAnn() << " != " << finishType)); + return IGraphTransformer::TStatus::Error; + } + } + + return IGraphTransformer::TStatus::Ok; + } +} // namespace + IGraphTransformer::TStatus FilterWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureMinMaxArgsCount(*input, 2U, 3U, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -871,7 +871,7 @@ namespace { return IGraphTransformer::TStatus::Error; } if (!lambda->Child(1)->IsList()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Expected literal tuple as result for update lambda for FoldMap")); return IGraphTransformer::TStatus::Error; } @@ -879,7 +879,7 @@ namespace { auto updatedListType = tupleExprType->GetItems()[0]; auto stateType = tupleExprType->GetItems()[1]; if (!IsSameAnnotation(*initState.GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of update lambda state and init state type: " << *stateType << " != " << *initState.GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -908,7 +908,7 @@ namespace { } auto status = ConvertToLambda(initLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } @@ -925,7 +925,7 @@ namespace { return IGraphTransformer::TStatus::Error; } if (!initLambda->Child(1)->IsList()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(initLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(initLambda->Pos()), TStringBuilder() << "Expected literal tuple as result for init lambda for Fold1Map")); return IGraphTransformer::TStatus::Error; } @@ -941,13 +941,13 @@ namespace { return IGraphTransformer::TStatus::Repeat; } if (!IsSameAnnotation(*lambdaType, *updateLambdaType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of update and init lambda types: " << *updateLambdaType << " != " << *lambdaType)); return IGraphTransformer::TStatus::Error; } if (!updateLambda->Child(1)->IsList()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Expected literal tuple as result for update lambda for Fold1Map")); return IGraphTransformer::TStatus::Error; } @@ -975,7 +975,7 @@ namespace { } auto status = ConvertToLambda(initLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } @@ -1211,7 +1211,7 @@ namespace { } if (type->GetKind() != ETypeAnnotationKind::List && type->GetKind() != ETypeAnnotationKind::EmptyList) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (empty) list or optional of (empty) list, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -1314,42 +1314,42 @@ namespace { } auto type = input->Head().GetTypeAnn(); - TExprNode::TPtr fold1Input = input->HeadPtr(); + TExprNode::TPtr fold1Input = input->HeadPtr(); if (type->GetKind() == ETypeAnnotationKind::Optional) { type = type->Cast<TOptionalExprType>()->GetItemType(); - fold1Input = ctx.Expr.Builder(input->Head().Pos()) - .Callable("Coalesce") - .Add(0, input->HeadPtr()) - .Callable(1, "List") - .Add(0, ExpandType(input->Head().Pos(), *type, ctx.Expr)) - .Seal() - .Seal() - .Build(); + fold1Input = ctx.Expr.Builder(input->Head().Pos()) + .Callable("Coalesce") + .Add(0, input->HeadPtr()) + .Callable(1, "List") + .Add(0, ExpandType(input->Head().Pos(), *type, ctx.Expr)) + .Seal() + .Seal() + .Build(); } - output = ctx.Expr.Builder(input->Pos()) + output = ctx.Expr.Builder(input->Pos()) .Callable("Fold1") - .Add(0, fold1Input) - .Lambda(1) - .Param("item") - .Arg("item") - .Seal() + .Add(0, fold1Input) + .Lambda(1) + .Param("item") + .Arg("item") + .Seal() .Add(2, std::move(updateLambda)) - .Seal() - .Build(); - + .Seal() + .Build(); + const auto itemType = type->Cast<TListExprType>()->GetItemType(); - if (itemType->GetKind() == ETypeAnnotationKind::Optional) { - // remove extra optional level created by Fold1 - output = ctx.Expr.Builder(input->Pos()) - .Callable("FlatMap") - .Add(0, output) - .Lambda(1) - .Param("item") - .Arg("item") - .Seal() - .Seal() - .Build(); + if (itemType->GetKind() == ETypeAnnotationKind::Optional) { + // remove extra optional level created by Fold1 + output = ctx.Expr.Builder(input->Pos()) + .Callable("FlatMap") + .Add(0, output) + .Lambda(1) + .Param("item") + .Arg("item") + .Seal() + .Seal() + .Build(); } return IGraphTransformer::TStatus::Repeat; @@ -1525,16 +1525,16 @@ namespace { } auto listType = input->Head().GetTypeAnn(); - if (HasError(listType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!listType) { - YQL_ENSURE(input->Head().IsLambda()); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - + if (HasError(listType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!listType) { + YQL_ENSURE(input->Head().IsLambda()); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + if (listType->GetKind() == ETypeAnnotationKind::Optional) { listType = listType->Cast<TOptionalExprType>()->GetItemType(); } @@ -1553,7 +1553,7 @@ namespace { bool isOptionalItem; const TDataExprType* dataType; if (!EnsureDataOrOptionalOfData(input->Head().Pos(), itemType, isOptionalItem, dataType, ctx.Expr) || !EnsureSpecificDataType(input->Head().Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "List of " << *itemType << " instead of boolean.")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "List of " << *itemType << " instead of boolean.")); return IGraphTransformer::TStatus::Error; } @@ -1677,7 +1677,7 @@ namespace { if (!input->Head().GetTypeAnn()) { YQL_ENSURE(input->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list or dict type, but got lambda")); return IGraphTransformer::TStatus::Error; } @@ -1705,7 +1705,7 @@ namespace { } if (type->GetKind() != ETypeAnnotationKind::List && type->GetKind() != ETypeAnnotationKind::Dict) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list or dict type, but got: " << *originalType)); return IGraphTransformer::TStatus::Error; } @@ -1726,7 +1726,7 @@ namespace { if (!input->Head().GetTypeAnn()) { YQL_ENSURE(input->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list, but got lambda")); return IGraphTransformer::TStatus::Error; } @@ -1744,7 +1744,7 @@ namespace { } if (type->GetKind() != ETypeAnnotationKind::List) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) list, but got: " << *originalType)); return IGraphTransformer::TStatus::Error; } @@ -1760,8 +1760,8 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -1889,7 +1889,7 @@ namespace { } else if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) { itemType = input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected list or stream type, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -1989,7 +1989,7 @@ namespace { const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); auto convertStatus = TryConvertTo(input->TailRef(), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -2022,7 +2022,7 @@ namespace { ui64 memoryLimitBytes = 0; if (!TryFromString(input->Child(1)->Content(), memoryLimitBytes)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -2036,7 +2036,7 @@ namespace { inputStreams = underlyingType->Cast<TTupleExprType>(); if (inputStreams->GetSize() < 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Expected at least 2 input streams")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Expected at least 2 input streams")); return IGraphTransformer::TStatus::Error; } @@ -2080,11 +2080,11 @@ namespace { } if (lambdaInputItemTypes.empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), "At least one stream index must be specified")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), "At least one stream index must be specified")); } if (i + 1 == input->ChildrenSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), "Expected handler")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), "Expected handler")); return IGraphTransformer::TStatus::Error; } @@ -2108,17 +2108,17 @@ namespace { } const TTypeAnnotationNode* lambdaItemType = nullptr; - if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) { - if (!EnsureStreamType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - lambdaItemType = lambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); - } else { - YQL_ENSURE(input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow); - if (!EnsureFlowType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - lambdaItemType = lambda->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType(); + if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) { + if (!EnsureStreamType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + lambdaItemType = lambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); + } else { + YQL_ENSURE(input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow); + if (!EnsureFlowType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + lambdaItemType = lambda->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType(); } if (!EnsurePersistableType(lambda->Pos(), *lambdaItemType, ctx.Expr)) { @@ -2132,7 +2132,7 @@ namespace { const auto underlyingType = lambdaItemType->Cast<TVariantExprType>()->GetUnderlyingType(); if (!EnsureTupleType(lambda->Pos(), *underlyingType, ctx.Expr)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), "Lambda output variant must be tuple based")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), "Lambda output variant must be tuple based")); return IGraphTransformer::TStatus::Error; } @@ -2156,7 +2156,7 @@ namespace { if (!input->Head().GetTypeAnn()) { YQL_ENSURE(input->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) (empty) list or dict type, but got lambda")); return IGraphTransformer::TStatus::Error; } @@ -2179,7 +2179,7 @@ namespace { } if (type->GetKind() != ETypeAnnotationKind::List && type->GetKind() != ETypeAnnotationKind::Dict) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected (optional) (empty) list or dict type, but got: " << *originalType)); return IGraphTransformer::TStatus::Error; } @@ -2276,7 +2276,7 @@ namespace { } auto err = TIssue( - ctx.Expr.GetPosition(pos), + ctx.Expr.GetPosition(pos), TStringBuilder() << "Uncompatible member " << item->GetName() << " types: " << *p.first << " and " << *item->GetItemType() @@ -2334,19 +2334,19 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { - Y_UNUSED(output); - if (!ctx.Types.OrderedColumns) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() - << "Unable to handle positional UNION ALL with column order disabled")); - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - TColumnOrder resultColumnOrder; + IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { + Y_UNUSED(output); + if (!ctx.Types.OrderedColumns) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() + << "Unable to handle positional UNION ALL with column order disabled")); + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + TColumnOrder resultColumnOrder; const TStructExprType* resultStructType = nullptr; auto status = InferPositionalUnionType(input->Pos(), input->ChildrenList(), resultColumnOrder, resultStructType, ctx); if (status != IGraphTransformer::TStatus::Ok) { @@ -2359,85 +2359,85 @@ namespace { IGraphTransformer::TStatus InferPositionalUnionType(TPositionHandle pos, const TExprNode::TListType& children, TColumnOrder& resultColumnOrder, const TStructExprType*& resultStructType, TExtContext& ctx) { - TVector<const TTypeAnnotationNode*> resultTypes; - size_t idx = 0; + TVector<const TTypeAnnotationNode*> resultTypes; + size_t idx = 0; for (const auto& child : children) { - if (!EnsureListType(*child, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto itemType = child->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - YQL_ENSURE(itemType); - if (!EnsureStructType(child->Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto childColumnOrder = ctx.Types.LookupColumnOrder(*child); - if (!childColumnOrder) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() - << "Input #" << idx << " does not have ordered columns. " - << "Consider making column order explicit by using SELECT with column names")); - return IGraphTransformer::TStatus::Error; - } - - TVector<const TTypeAnnotationNode*> childTypes; - auto structType = itemType->Cast<TStructExprType>(); - for (const auto& col : *childColumnOrder) { - auto itemIdx = structType->FindItem(col); - YQL_ENSURE(itemIdx); - childTypes.push_back(structType->GetItems()[*itemIdx]->GetItemType()); - } - - if (idx == 0) { - resultColumnOrder = *childColumnOrder; - resultTypes = childTypes; - } else { - if (childTypes.size() != resultTypes.size()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() - << "Different column counts in UNION ALL inputs: input #0 has " << resultTypes.size() - << " column, input #" << idx << " has " << childTypes.size() << " columns")); - return IGraphTransformer::TStatus::Error; - } - for (size_t i = 0; i < childTypes.size(); ++i) { + if (!EnsureListType(*child, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto itemType = child->GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + YQL_ENSURE(itemType); + if (!EnsureStructType(child->Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto childColumnOrder = ctx.Types.LookupColumnOrder(*child); + if (!childColumnOrder) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + << "Input #" << idx << " does not have ordered columns. " + << "Consider making column order explicit by using SELECT with column names")); + return IGraphTransformer::TStatus::Error; + } + + TVector<const TTypeAnnotationNode*> childTypes; + auto structType = itemType->Cast<TStructExprType>(); + for (const auto& col : *childColumnOrder) { + auto itemIdx = structType->FindItem(col); + YQL_ENSURE(itemIdx); + childTypes.push_back(structType->GetItems()[*itemIdx]->GetItemType()); + } + + if (idx == 0) { + resultColumnOrder = *childColumnOrder; + resultTypes = childTypes; + } else { + if (childTypes.size() != resultTypes.size()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + << "Different column counts in UNION ALL inputs: input #0 has " << resultTypes.size() + << " column, input #" << idx << " has " << childTypes.size() << " columns")); + return IGraphTransformer::TStatus::Error; + } + for (size_t i = 0; i < childTypes.size(); ++i) { auto arg1 = ctx.Expr.NewArgument(child->Pos(), "arg1"); auto arg2 = ctx.Expr.NewArgument(child ->Pos(), "arg2"); - const TTypeAnnotationNode* commonType = nullptr; - auto status = SilentInferCommonType(arg1, *resultTypes[i], arg2, *childTypes[i], ctx.Expr, commonType); + const TTypeAnnotationNode* commonType = nullptr; + auto status = SilentInferCommonType(arg1, *resultTypes[i], arg2, *childTypes[i], ctx.Expr, commonType); if (status == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() - << "Can not infer common type for result column " << resultColumnOrder[i] - << ": no common type for " << *resultTypes[i] << " and column " << (*childColumnOrder)[i] - << " type " << *childTypes[i])); - return IGraphTransformer::TStatus::Error; - } - YQL_ENSURE(commonType); - resultTypes[i] = commonType; - } - } - idx++; - } - - YQL_ENSURE(resultColumnOrder.size() == resultTypes.size()); - TVector<const TItemExprType*> structItems; - for (size_t i = 0; i < resultTypes.size(); ++i) { - structItems.push_back(ctx.Expr.MakeType<TItemExprType>(resultColumnOrder[i], resultTypes[i])); - } - + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + << "Can not infer common type for result column " << resultColumnOrder[i] + << ": no common type for " << *resultTypes[i] << " and column " << (*childColumnOrder)[i] + << " type " << *childTypes[i])); + return IGraphTransformer::TStatus::Error; + } + YQL_ENSURE(commonType); + resultTypes[i] = commonType; + } + } + idx++; + } + + YQL_ENSURE(resultColumnOrder.size() == resultTypes.size()); + TVector<const TItemExprType*> structItems; + for (size_t i = 0; i < resultTypes.size(); ++i) { + structItems.push_back(ctx.Expr.MakeType<TItemExprType>(resultColumnOrder[i], resultTypes[i])); + } + resultStructType = ctx.Expr.MakeType<TStructExprType>(structItems); if (!resultStructType->Validate(pos, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - + return IGraphTransformer::TStatus::Error; + } + return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus ListAutomapArgs(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx, - TStringBuf name) - { + } + + IGraphTransformer::TStatus ListAutomapArgs(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx, + TStringBuf name) + { bool isSomeOptional = false; for (ui32 i = 0; i < input->ChildrenSize(); ++i) { if (!input->Child(i)->GetTypeAnn()) { YQL_ENSURE(input->Child(i)->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Expected (optional) list type, but got lambda")); return IGraphTransformer::TStatus::Error; } @@ -2456,7 +2456,7 @@ namespace { if (inputType->GetKind() != ETypeAnnotationKind::List && inputType->GetKind() != ETypeAnnotationKind::EmptyList) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Expected (optional) (empty) list type, but got: " << *input->Child(i)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2510,20 +2510,20 @@ namespace { bool hasEmptyList = false; for (const auto& child : input->Children()) { - if (!child->GetTypeAnn()) { - YQL_ENSURE(child->Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() - << "Expected (optional) list type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (IsNull(*child)) { - output = child; - return IGraphTransformer::TStatus::Repeat; - } + if (!child->GetTypeAnn()) { + YQL_ENSURE(child->Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() + << "Expected (optional) list type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (IsNull(*child)) { + output = child; + return IGraphTransformer::TStatus::Repeat; + } hasEmptyList = hasEmptyList || ETypeAnnotationKind::EmptyList == child->GetTypeAnn()->GetKind(); - } - + } + if (hasEmptyList) { auto children = input->ChildrenList(); children.erase(std::remove_if(children.begin(), children.end(), @@ -2531,15 +2531,15 @@ namespace { output = ctx.Expr.ChangeChildren(*input, std::move(children)); return IGraphTransformer::TStatus::Repeat; } - + if constexpr (!IsStrict) { if (const auto commonType = CommonTypeForChildren(*input, ctx.Expr)) { if (const auto status = ConvertChildrenToType(input, commonType, ctx.Expr); status != IGraphTransformer::TStatus::Ok) - return status; + return status; } else return IGraphTransformer::TStatus::Error; - } - + } + return ListAutomapArgs(input, output, ctx, "OrderedExtend"); } @@ -2547,15 +2547,15 @@ namespace { template IGraphTransformer::TStatus ListExtendWrapper<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListUnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - return ListAutomapArgs(input, output, ctx, "UnionAll"); + return ListAutomapArgs(input, output, ctx, "UnionAll"); } IGraphTransformer::TStatus ListZipWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - return ListAutomapArgs(input, output, ctx, "Zip"); + return ListAutomapArgs(input, output, ctx, "Zip"); } IGraphTransformer::TStatus ListZipAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - return ListAutomapArgs(input, output, ctx, "ZipAll"); + return ListAutomapArgs(input, output, ctx, "ZipAll"); } bool ValidateSortDirections(TExprNode& direction, TExprContext& ctx, bool& isTuple) { @@ -2580,7 +2580,7 @@ namespace { } if (!isOkAscending) { - ctx.AddError(TIssue(ctx.GetPosition(direction.Pos()), "Expected either Bool or tuple of Bool")); + ctx.AddError(TIssue(ctx.GetPosition(direction.Pos()), "Expected either Bool or tuple of Bool")); return false; } @@ -2610,13 +2610,13 @@ namespace { const auto directionTupleSize = direction->GetTypeAnn()->Cast<TTupleExprType>()->GetSize(); if (directionTupleSize == 0 || directionTupleSize >= 2) { if (lambda->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "expected tuple type as lambda result, but got: " << *lambda->GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "expected tuple type as lambda result, but got: " << *lambda->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } const auto lambdaTupleSize = lambda->GetTypeAnn()->Cast<TTupleExprType>()->GetSize(); if (lambdaTupleSize != directionTupleSize) { - ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda result and ascending parameters sizes: " << lambdaTupleSize << " != " << directionTupleSize)); return IGraphTransformer::TStatus::Error; @@ -2625,7 +2625,7 @@ namespace { } if (!lambda->GetTypeAnn()->IsComparable()) { - ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected comparable type, but got: " << *lambda->GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected comparable type, but got: " << *lambda->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -2674,7 +2674,7 @@ namespace { const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); const auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -2746,14 +2746,14 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + const auto& listTypeNode = input->Head(); auto& listType = *listTypeNode.GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (listType.GetKind() == ETypeAnnotationKind::EmptyList) { @@ -2765,189 +2765,189 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto status = ValidateSortTraits(listType.Cast<TListExprType>()->GetItemType(), input->Child(1), input->TailRef(), ctx.Expr); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 4, 5, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - const auto listTypeNode = input->HeadPtr(); - const auto sortTraits = input->ChildPtr(1); - - auto& listType = *listTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - if (listType.GetKind() == ETypeAnnotationKind::EmptyList) { - output = ctx.Expr.NewCallable(input->Pos(), "Void", {}); - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureListType(listTypeNode->Pos(), listType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!sortTraits->IsCallable({"Void", "SortTraits"})) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sortTraits->Pos()), - TStringBuilder() << "Expecting SortTraits or Void as second argument")); - return IGraphTransformer::TStatus::Error; - } - - auto itemType = listType.Cast<TListExprType>()->GetItemType(); - - if (input->ChildrenSize() == 4) { - // legacy SessionWindowTraits - auto& sessionKey = input->ChildRef(2); - auto& sessionPred = input->TailRef(); - - auto status = ConvertToLambda(sessionKey, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(sessionPred, ctx.Expr, 2)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(sessionKey, { itemType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto sessionKeyType = sessionKey->GetTypeAnn(); - if (!sessionKeyType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureComputableType(sessionKey->Pos(), *sessionKeyType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(sessionPred, { sessionKeyType, sessionKeyType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto sessionPredType = sessionPred->GetTypeAnn(); - if (!sessionPredType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (sessionPredType->GetKind() != ETypeAnnotationKind::Data || - sessionPredType->Cast<TDataExprType>()->GetSlot() != EDataSlot::Bool) - { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sessionPred->Pos()), - TStringBuilder() << "Expected Bool as return type of session timeout predicate, but got: " << *sessionPredType)); - return IGraphTransformer::TStatus::Error; - } - - // rewrite in new format - auto initLambda = ctx.Expr.Builder(input->Pos()) - .Lambda() - .Param("row") - .Apply(sessionKey) - .With(0, "row") - .Seal() - .Seal() - .Build(); - - auto calculateLambda = ctx.Expr.Builder(input->Pos()) - .Lambda() - .Param("row") - .Param("state") - .Apply(sessionKey) - .With(0, "row") - .Seal() - .Seal() - .Build(); - - auto updateLambda = ctx.Expr.Builder(input->Pos()) - .Lambda() - .Param("row") - .Param("state") - .List() - .Apply(0, sessionPred) - .With(0, "state") - .With(1) - .Apply(sessionKey) - .With(0, "row") - .Seal() - .Done() - .Seal() - .Apply(1, sessionKey) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Build(); - - output = ctx.Expr.ChangeChildren(*input, { listTypeNode, sortTraits, initLambda, updateLambda, calculateLambda }); - return IGraphTransformer::TStatus::Repeat; - } - - auto& initLambda = input->ChildRef(2); - auto& updateLambda = input->ChildRef(3); - auto& calculateLambda = input->ChildRef(4); - - auto status = ConvertToLambda(initLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); - status = status.Combine(ConvertToLambda(calculateLambda, ctx.Expr, 2)); + auto status = ValidateSortTraits(listType.Cast<TListExprType>()->GetItemType(), input->Child(1), input->TailRef(), ctx.Expr); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 4, 5, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + const auto listTypeNode = input->HeadPtr(); + const auto sortTraits = input->ChildPtr(1); + + auto& listType = *listTypeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + if (listType.GetKind() == ETypeAnnotationKind::EmptyList) { + output = ctx.Expr.NewCallable(input->Pos(), "Void", {}); + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureListType(listTypeNode->Pos(), listType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!sortTraits->IsCallable({"Void", "SortTraits"})) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sortTraits->Pos()), + TStringBuilder() << "Expecting SortTraits or Void as second argument")); + return IGraphTransformer::TStatus::Error; + } + + auto itemType = listType.Cast<TListExprType>()->GetItemType(); + + if (input->ChildrenSize() == 4) { + // legacy SessionWindowTraits + auto& sessionKey = input->ChildRef(2); + auto& sessionPred = input->TailRef(); + + auto status = ConvertToLambda(sessionKey, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(sessionPred, ctx.Expr, 2)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(sessionKey, { itemType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto sessionKeyType = sessionKey->GetTypeAnn(); + if (!sessionKeyType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureComputableType(sessionKey->Pos(), *sessionKeyType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(sessionPred, { sessionKeyType, sessionKeyType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto sessionPredType = sessionPred->GetTypeAnn(); + if (!sessionPredType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (sessionPredType->GetKind() != ETypeAnnotationKind::Data || + sessionPredType->Cast<TDataExprType>()->GetSlot() != EDataSlot::Bool) + { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sessionPred->Pos()), + TStringBuilder() << "Expected Bool as return type of session timeout predicate, but got: " << *sessionPredType)); + return IGraphTransformer::TStatus::Error; + } + + // rewrite in new format + auto initLambda = ctx.Expr.Builder(input->Pos()) + .Lambda() + .Param("row") + .Apply(sessionKey) + .With(0, "row") + .Seal() + .Seal() + .Build(); + + auto calculateLambda = ctx.Expr.Builder(input->Pos()) + .Lambda() + .Param("row") + .Param("state") + .Apply(sessionKey) + .With(0, "row") + .Seal() + .Seal() + .Build(); + + auto updateLambda = ctx.Expr.Builder(input->Pos()) + .Lambda() + .Param("row") + .Param("state") + .List() + .Apply(0, sessionPred) + .With(0, "state") + .With(1) + .Apply(sessionKey) + .With(0, "row") + .Seal() + .Done() + .Seal() + .Apply(1, sessionKey) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Build(); + + output = ctx.Expr.ChangeChildren(*input, { listTypeNode, sortTraits, initLambda, updateLambda, calculateLambda }); + return IGraphTransformer::TStatus::Repeat; + } + + auto& initLambda = input->ChildRef(2); + auto& updateLambda = input->ChildRef(3); + auto& calculateLambda = input->ChildRef(4); + + auto status = ConvertToLambda(initLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(calculateLambda, ctx.Expr, 2)); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } - // initLambda - if (!UpdateLambdaAllArgumentsTypes(initLambda, { itemType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto sessionStateType = initLambda->GetTypeAnn(); - if (!sessionStateType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureComputableType(initLambda->Pos(), *sessionStateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - // calculateLambda - if (!UpdateLambdaAllArgumentsTypes(calculateLambda, { itemType, sessionStateType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto sessionValueType = calculateLambda->GetTypeAnn(); - if (!sessionValueType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureComputableType(initLambda->Pos(), *sessionValueType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - // updateLambda - if (!UpdateLambdaAllArgumentsTypes(updateLambda, { itemType, sessionStateType }, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto updateLambdaType = updateLambda->GetTypeAnn(); - if (!updateLambdaType) { - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* expectedType = - ctx.Expr.MakeType<TTupleExprType>( - TTypeAnnotationNode::TListType{ ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool), sessionStateType }); - - if (!IsSameAnnotation(*expectedType, *updateLambdaType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), - TStringBuilder() << "Expected " << *expectedType << " as return type of update lambda, but got: " << *updateLambdaType)); - return IGraphTransformer::TStatus::Error; - } - + // initLambda + if (!UpdateLambdaAllArgumentsTypes(initLambda, { itemType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto sessionStateType = initLambda->GetTypeAnn(); + if (!sessionStateType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureComputableType(initLambda->Pos(), *sessionStateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + // calculateLambda + if (!UpdateLambdaAllArgumentsTypes(calculateLambda, { itemType, sessionStateType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto sessionValueType = calculateLambda->GetTypeAnn(); + if (!sessionValueType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureComputableType(initLambda->Pos(), *sessionValueType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + // updateLambda + if (!UpdateLambdaAllArgumentsTypes(updateLambda, { itemType, sessionStateType }, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto updateLambdaType = updateLambda->GetTypeAnn(); + if (!updateLambdaType) { + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* expectedType = + ctx.Expr.MakeType<TTupleExprType>( + TTypeAnnotationNode::TListType{ ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool), sessionStateType }); + + if (!IsSameAnnotation(*expectedType, *updateLambdaType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), + TStringBuilder() << "Expected " << *expectedType << " as return type of update lambda, but got: " << *updateLambdaType)); + return IGraphTransformer::TStatus::Error; + } + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); return IGraphTransformer::TStatus::Ok; } @@ -2984,7 +2984,7 @@ namespace { if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple && lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items")); return IGraphTransformer::TStatus::Error; } @@ -3000,7 +3000,7 @@ namespace { auto retKind = lambdaListHandler->GetTypeAnn()->GetKind(); if (retKind != ETypeAnnotationKind::List && retKind != ETypeAnnotationKind::Optional && retKind != ETypeAnnotationKind::Stream) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() << "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3035,7 +3035,7 @@ namespace { } if (input->Child(2)->IsCallable("Void") != input->Child(3)->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(3)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(3)->Pos()), TStringBuilder() << "Direction and sort key extractor should be specified at the same time")); return IGraphTransformer::TStatus::Error; } @@ -3063,7 +3063,7 @@ namespace { } if (!lambdaKeySelector->GetTypeAnn()->IsHashable() || !lambdaKeySelector->GetTypeAnn()->IsEquatable()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), TStringBuilder() << "Expected hashable and equatable type for key, but got: " << *lambdaKeySelector->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3092,7 +3092,7 @@ namespace { if (retKind != ETypeAnnotationKind::List && retKind != ETypeAnnotationKind::Optional && retKind != ETypeAnnotationKind::Stream) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() << "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3204,7 +3204,7 @@ namespace { const TTypeAnnotationNode* expectedType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -3272,7 +3272,7 @@ namespace { } if (!IsSameAnnotation(*lambda->GetTypeAnn(), *input->Child(1)->GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *lambda->GetTypeAnn() << " != " << *input->Child(1)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3337,7 +3337,7 @@ namespace { } if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *updateLambda->GetTypeAnn() << "!= " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -3383,12 +3383,12 @@ namespace { bool isOptional; const TDataExprType* dataType; if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean.")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean.")); return IGraphTransformer::TStatus::Error; } if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *updateLambda->GetTypeAnn() << " != " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -3440,12 +3440,12 @@ namespace { bool isOptional; const TDataExprType* dataType; if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean.")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean.")); return IGraphTransformer::TStatus::Error; } if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *updateLambda->GetTypeAnn() << "!= " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -3485,7 +3485,7 @@ namespace { } if (!IsSameAnnotation(*lambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *lambda->GetTypeAnn() << " != " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -3494,7 +3494,7 @@ namespace { auto& loadLambda = input->ChildRef(4); if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << "Save and load lambdas must be specified at the same time")); return IGraphTransformer::TStatus::Error; } @@ -3526,13 +3526,13 @@ namespace { } if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); return IGraphTransformer::TStatus::Error; } } - auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze, use Condense instead."); + auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze, use Condense instead."); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue); if (!ctx.Expr.AddWarning(issue)) { return IGraphTransformer::TStatus::Error; @@ -3580,7 +3580,7 @@ namespace { } if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, " << *updateLambda->GetTypeAnn() << "!= " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -3589,7 +3589,7 @@ namespace { auto& loadLambda = input->ChildRef(4); if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << "Save and load lambdas must be specified at the same time")); return IGraphTransformer::TStatus::Error; } @@ -3621,13 +3621,13 @@ namespace { } if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); return IGraphTransformer::TStatus::Error; } } - auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze1, use Condense1 instead."); + auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze1, use Condense1 instead."); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue); if (!ctx.Expr.AddWarning(issue)) { return IGraphTransformer::TStatus::Error; @@ -3731,7 +3731,7 @@ namespace { if (input->ChildrenSize() > 1) { auto convertStatus = TryConvertTo(input->ChildRef(1), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -3743,7 +3743,7 @@ namespace { if (input->ChildrenSize() > 2) { auto convertStatus = TryConvertTo(input->ChildRef(2), *expectedType, ctx.Expr); if (convertStatus.Level == IGraphTransformer::TStatus::Error) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "Mismatch argument types")); return IGraphTransformer::TStatus::Error; } @@ -3765,8 +3765,8 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -3783,7 +3783,7 @@ namespace { auto itemType = listType->GetItemType(); for (size_t i = 1; i < input->ChildrenSize(); ++i) { if (!IsSameAnnotation(*itemType, *input->Child(i)->GetTypeAnn())) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type of list item, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type of list item, " << *itemType << " != " << *input->Child(i)->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3843,7 +3843,7 @@ namespace { if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple && lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items")); return IGraphTransformer::TStatus::Error; } @@ -3871,7 +3871,7 @@ namespace { } if (!IsSameAnnotation(*lambdaUpdateHandler->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdateHandler->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdateHandler->Pos()), TStringBuilder() << "Mismatch of update lambda return type and state type, " << *lambdaUpdateHandler->GetTypeAnn() << "!= " << *stateType)); return IGraphTransformer::TStatus::Error; @@ -3921,7 +3921,7 @@ namespace { const TTypeAnnotationNode* itemType = nullptr; if (firstArgKind == ETypeAnnotationKind::List) { itemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - } else { + } else { itemType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType(); } @@ -3930,7 +3930,7 @@ namespace { auto structType = itemType->Cast<TStructExprType>(); auto pos = structType->FindItem(input->Tail().Content()); if (!pos) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Member with name " << input->Tail().Content() << " is not found in " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3940,13 +3940,13 @@ namespace { else if (itemType->GetKind() == ETypeAnnotationKind::Tuple) { ui32 index = 0; if (!TryFromString(input->Tail().Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } auto tupleType = itemType->Cast<TTupleExprType>(); if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << tupleType->GetSize() << ", type: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3954,7 +3954,7 @@ namespace { extractedType = tupleType->GetItems()[index]; } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected either list of struct or list tuples, but got: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -3973,11 +3973,11 @@ namespace { return IGraphTransformer::TStatus::Error; } - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + const TTypeAnnotationNode* inputItemType = nullptr; if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &inputItemType)) { return IGraphTransformer::TStatus::Error; @@ -3990,8 +3990,8 @@ namespace { const auto structType = inputItemType->Cast<TStructExprType>(); TVector<const TItemExprType*> resItems; for (auto& x : input->Tail().Children()) { - YQL_ENSURE(x->IsAtom()); - auto pos = FindOrReportMissingMember(x->Content(), input->Head().Pos(), *structType, ctx); + YQL_ENSURE(x->IsAtom()); + auto pos = FindOrReportMissingMember(x->Content(), input->Head().Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } @@ -4013,7 +4013,7 @@ namespace { return IGraphTransformer::TStatus::Error; } - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -4030,105 +4030,105 @@ namespace { const auto structType = inputItemType->Cast<TStructExprType>(); for (auto& x : input->Tail().Children()) { YQL_ENSURE(x->IsAtom()); - auto pos = FindOrReportMissingMember(x->Content(), input->Head().Pos(), *structType, ctx); + auto pos = FindOrReportMissingMember(x->Content(), input->Head().Pos(), *structType, ctx); + if (!pos) { + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(input->Head().GetTypeAnn()); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureTupleOfAtoms(input->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (HasError(input->Head().GetTypeAnn(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!input->Head().GetTypeAnn()) { + YQL_ENSURE(input->Head().Type() == TExprNode::Lambda); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), + TStringBuilder() << "Expected either struct or optional/list/stream/flow of struct, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList) { + output = input->HeadPtr(); + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* itemType = nullptr; + if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) { + itemType = input->Head().GetTypeAnn(); + } else if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &itemType)) { + return IGraphTransformer::TStatus::Error; + } + + + if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const TStructExprType* structType = itemType->Cast<TStructExprType>(); + + auto inputColumns = GetColumnsOfStructOrSequenceOfStruct(*input->Head().GetTypeAnn()); + auto columnOrder = input->Tail().ChildrenList(); + + for (auto& column : columnOrder) { + YQL_ENSURE(column->IsAtom()); + auto pos = FindOrReportMissingMember(column->Content(), input->Head().Pos(), *structType, ctx); if (!pos) { return IGraphTransformer::TStatus::Error; } + inputColumns.erase(inputColumns.find(column->Content())); + } + + if (input->IsCallable("AssumeColumnOrderPartial")) { + for (auto& c : inputColumns) { + columnOrder.push_back(ctx.Expr.NewAtom(input->Pos(), c)); + } + + output = ctx.Expr.Builder(input->Pos()) + .Callable("AssumeColumnOrder") + .Add(0, input->HeadPtr()) + .Add(1, ctx.Expr.NewList(input->Tail().Pos(), std::move(columnOrder))) + .Seal() + .Build(); + return IGraphTransformer::TStatus::Repeat; + } + + if (!inputColumns.empty()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), + TStringBuilder() << "Unordered columns " << JoinSeq(", ", inputColumns) + << " in input of type " << *input->Head().GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + if (!ctx.Types.OrderedColumns) { + output = input->HeadPtr(); + return IGraphTransformer::TStatus::Repeat; } input->SetTypeAnn(input->Head().GetTypeAnn()); return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureTupleOfAtoms(input->Tail(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (HasError(input->Head().GetTypeAnn(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!input->Head().GetTypeAnn()) { - YQL_ENSURE(input->Head().Type() == TExprNode::Lambda); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), - TStringBuilder() << "Expected either struct or optional/list/stream/flow of struct, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList) { - output = input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* itemType = nullptr; - if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) { - itemType = input->Head().GetTypeAnn(); - } else if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &itemType)) { - return IGraphTransformer::TStatus::Error; - } - - - if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const TStructExprType* structType = itemType->Cast<TStructExprType>(); - - auto inputColumns = GetColumnsOfStructOrSequenceOfStruct(*input->Head().GetTypeAnn()); - auto columnOrder = input->Tail().ChildrenList(); - - for (auto& column : columnOrder) { - YQL_ENSURE(column->IsAtom()); - auto pos = FindOrReportMissingMember(column->Content(), input->Head().Pos(), *structType, ctx); - if (!pos) { - return IGraphTransformer::TStatus::Error; - } - inputColumns.erase(inputColumns.find(column->Content())); - } - - if (input->IsCallable("AssumeColumnOrderPartial")) { - for (auto& c : inputColumns) { - columnOrder.push_back(ctx.Expr.NewAtom(input->Pos(), c)); - } - - output = ctx.Expr.Builder(input->Pos()) - .Callable("AssumeColumnOrder") - .Add(0, input->HeadPtr()) - .Add(1, ctx.Expr.NewList(input->Tail().Pos(), std::move(columnOrder))) - .Seal() - .Build(); - return IGraphTransformer::TStatus::Repeat; - } - - if (!inputColumns.empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), - TStringBuilder() << "Unordered columns " << JoinSeq(", ", inputColumns) - << " in input of type " << *input->Head().GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (!ctx.Types.OrderedColumns) { - output = input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - - input->SetTypeAnn(input->Head().GetTypeAnn()); - return IGraphTransformer::TStatus::Ok; - } - IGraphTransformer::TStatus AggregationTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 8, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4191,7 +4191,7 @@ namespace { } if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *combineStateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " << *combineStateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -4236,7 +4236,7 @@ namespace { } if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *reduceStateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " << *reduceStateType << ", but got: " << *lambdaMerge->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -4255,9 +4255,9 @@ namespace { return IGraphTransformer::TStatus::Error; } - status = ValidateTraitsDefaultValue(input, 7, *finishType, "finish", output, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - return status; + status = ValidateTraitsDefaultValue(input, 7, *finishType, "finish", output, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + return status; } input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); @@ -4269,8 +4269,8 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto rowType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -4333,19 +4333,19 @@ namespace { } auto inputStructType = inputItemType->Cast<TStructExprType>(); - if (input->Child(1)->IsCallable("Void")) { - TExprNodeList names; - names.reserve(inputStructType->GetSize()); - for (const auto& item : inputStructType->GetItems()) { - names.emplace_back(ctx.Expr.NewAtom(input->Child(1)->Pos(), item->GetName())); - } - output = ctx.Expr.ChangeChild(*input, 1, ctx.Expr.NewList(input->Child(1)->Pos(), std::move(names))); - return IGraphTransformer::TStatus::Repeat; - } - - auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); - if (status != IGraphTransformer::TStatus::Ok) { - return status; + if (input->Child(1)->IsCallable("Void")) { + TExprNodeList names; + names.reserve(inputStructType->GetSize()); + for (const auto& item : inputStructType->GetItems()) { + names.emplace_back(ctx.Expr.NewAtom(input->Child(1)->Pos(), item->GetName())); + } + output = ctx.Expr.ChangeChild(*input, 1, ctx.Expr.NewList(input->Child(1)->Pos(), std::move(names))); + return IGraphTransformer::TStatus::Repeat; + } + + auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + if (status != IGraphTransformer::TStatus::Ok) { + return status; } if (input->ChildrenSize() < 4U) { @@ -4353,27 +4353,27 @@ namespace { children.push_back(ctx.Expr.NewList(input->Pos(), {})); output = ctx.Expr.ChangeChildren(*input, std::move(children)); return IGraphTransformer::TStatus::Repeat; - } else { - TExprNode::TPtr normalized; - status = NormalizeKeyValueTuples(input->ChildPtr(3), 0, normalized, ctx.Expr); - if (status.Level == IGraphTransformer::TStatus::Repeat) { - output = ctx.Expr.ChangeChild(*input, 3, std::move(normalized)); - return status; - } - - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } + } else { + TExprNode::TPtr normalized; + status = NormalizeKeyValueTuples(input->ChildPtr(3), 0, normalized, ctx.Expr); + if (status.Level == IGraphTransformer::TStatus::Repeat) { + output = ctx.Expr.ChangeChild(*input, 3, std::move(normalized)); + return status; + } + + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } } bool isHopping = false; - const auto settings = input->Child(3); + const auto settings = input->Child(3); if (!EnsureTuple(*settings, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; + return IGraphTransformer::TStatus::Error; } - TVector<const TItemExprType*> rowColumns; - TMaybe<TStringBuf> sessionColumnName; + TVector<const TItemExprType*> rowColumns; + TMaybe<TStringBuf> sessionColumnName; for (const auto& setting : settings->Children()) { if (!EnsureTupleMinSize(*setting, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -4384,96 +4384,96 @@ namespace { } const auto settingName = setting->Head().Content(); - if (settingName == "hopping") { - isHopping = true; - if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!setting->Child(1)->IsCallable("HoppingTraits")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), - TStringBuilder() << "Expected HoppingTraits callable")); - return IGraphTransformer::TStatus::Error; - } - rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>("_yql_time", - ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)))); - } else if (settingName == "session") { - if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto value = setting->ChildPtr(1); - if (!EnsureTupleSize(*value, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto sessionOutputColumn = value->Child(0); - if (!EnsureAtom(*sessionOutputColumn, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (sessionOutputColumn->Content().Empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sessionOutputColumn->Pos()), - TStringBuilder() << "Session output column name can not be empty")); - return IGraphTransformer::TStatus::Error; - } - - sessionColumnName = sessionOutputColumn->Content(); - - auto traits = value->Child(1); - if (!traits->IsCallable("SessionWindowTraits")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), - TStringBuilder() << "Expected SessionWindowTraits callable")); - return IGraphTransformer::TStatus::Error; - } - - auto sessionKeyType = traits->Child(TCoSessionWindowTraits::idx_Calculate)->GetTypeAnn(); - rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>(*sessionColumnName, sessionKeyType)); - - bool seenAsKeyColumn = AnyOf(input->Child(1)->ChildrenList(), [&](const auto& keyColum) { - return sessionColumnName == keyColum->Content(); - }); - - if (!seenAsKeyColumn) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), - TStringBuilder() << "Session column " << *sessionColumnName << " is not listed in key columns")); - return IGraphTransformer::TStatus::Error; - } + if (settingName == "hopping") { + isHopping = true; + if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!setting->Child(1)->IsCallable("HoppingTraits")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), + TStringBuilder() << "Expected HoppingTraits callable")); + return IGraphTransformer::TStatus::Error; + } + rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>("_yql_time", + ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)))); + } else if (settingName == "session") { + if (!EnsureTupleSize(*setting, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto value = setting->ChildPtr(1); + if (!EnsureTupleSize(*value, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto sessionOutputColumn = value->Child(0); + if (!EnsureAtom(*sessionOutputColumn, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (sessionOutputColumn->Content().Empty()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(sessionOutputColumn->Pos()), + TStringBuilder() << "Session output column name can not be empty")); + return IGraphTransformer::TStatus::Error; + } + + sessionColumnName = sessionOutputColumn->Content(); + + auto traits = value->Child(1); + if (!traits->IsCallable("SessionWindowTraits")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), + TStringBuilder() << "Expected SessionWindowTraits callable")); + return IGraphTransformer::TStatus::Error; + } + + auto sessionKeyType = traits->Child(TCoSessionWindowTraits::idx_Calculate)->GetTypeAnn(); + rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>(*sessionColumnName, sessionKeyType)); + + bool seenAsKeyColumn = AnyOf(input->Child(1)->ChildrenList(), [&](const auto& keyColum) { + return sessionColumnName == keyColum->Content(); + }); + + if (!seenAsKeyColumn) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Child(1)->Pos()), + TStringBuilder() << "Session column " << *sessionColumnName << " is not listed in key columns")); + return IGraphTransformer::TStatus::Error; + } } else if (settingName == "compact") { if (!EnsureTupleSize(*setting, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } } else { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Head().Pos()), - TStringBuilder() << "Unexpected setting: " << settingName)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(setting->Head().Pos()), + TStringBuilder() << "Unexpected setting: " << settingName)); + return IGraphTransformer::TStatus::Error; + } + } + + for (auto& child : input->Child(1)->Children()) { + if (!EnsureAtom(*child, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (sessionColumnName == child->Content()) { + continue; + } + + auto item = FindOrReportMissingMember(child->Content(), child->Pos(), *inputStructType, ctx); + if (!item) { + return IGraphTransformer::TStatus::Error; + } + + auto columnType = inputStructType->GetItems()[*item]->GetItemType(); + if (!columnType->IsHashable() || !columnType->IsEquatable()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for key column: " << + child->Content() << ", but got: " << *columnType)); return IGraphTransformer::TStatus::Error; } + + rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>(child->Content(), columnType)); } - for (auto& child : input->Child(1)->Children()) { - if (!EnsureAtom(*child, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (sessionColumnName == child->Content()) { - continue; - } - - auto item = FindOrReportMissingMember(child->Content(), child->Pos(), *inputStructType, ctx); - if (!item) { - return IGraphTransformer::TStatus::Error; - } - - auto columnType = inputStructType->GetItems()[*item]->GetItemType(); - if (!columnType->IsHashable() || !columnType->IsEquatable()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for key column: " << - child->Content() << ", but got: " << *columnType)); - return IGraphTransformer::TStatus::Error; - } - - rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>(child->Content(), columnType)); - } - for (ui32 childIndex = 0; childIndex < input->Child(2)->ChildrenSize(); ++childIndex) { const auto& child = input->Child(2)->ChildRef(childIndex); if (!EnsureTupleMinSize(*child, 2, ctx.Expr)) { @@ -4503,7 +4503,7 @@ namespace { } if (!child->Child(1)->IsCallable("AggregationTraits")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), "Expected aggregation traits")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), "Expected aggregation traits")); return IGraphTransformer::TStatus::Error; } @@ -4515,14 +4515,14 @@ namespace { auto item = inputStructType->FindItem(child->Child(2)->Content()); if (!item) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << child->Child(2)->Content() << ", type: " << *input->Head().GetTypeAnn())); return IGraphTransformer::TStatus::Error; } distinctColumnType = inputStructType->GetItems()[*item]->GetItemType(); if (!distinctColumnType->IsHashable() || !distinctColumnType->IsEquatable()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for distinct column: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Expected hashable and equatable type for distinct column: " << child->Child(2)->Content() << ", but got: " << *distinctColumnType)); return IGraphTransformer::TStatus::Error; } @@ -4537,7 +4537,7 @@ namespace { const auto tupleType = finishType->Cast<TTupleExprType>(); if (tupleType->GetSize() != child->Head().ChildrenSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Child(6)->Pos()), + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Child(6)->Pos()), TStringBuilder() << "Expected tuple type of size: " << child->Head().ChildrenSize() << ", but got: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; } @@ -4577,44 +4577,44 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus SqlAggregateAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (IsEmptyList(input->Head())) { - output = input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - - bool isStream; - if (!EnsureSeqType(input->Head(), ctx.Expr, &isStream)) { - return IGraphTransformer::TStatus::Error; - } - - auto inputItemType = isStream - ? input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType() - : input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - - if (!EnsureStructType(input->Head().Pos(), *inputItemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto inputStructType = inputItemType->Cast<TStructExprType>(); - for (auto& item : inputStructType->GetItems()) { - auto columnName = item->GetName(); - auto columnType = item->GetItemType(); - if (!columnType->IsHashable() || !columnType->IsEquatable()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Expected hashable and equatable type for key column: " << columnName << ", but got: " << *columnType)); - return IGraphTransformer::TStatus::Error; - } - } - - input->SetTypeAnn(input->Head().GetTypeAnn()); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus SqlAggregateAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (IsEmptyList(input->Head())) { + output = input->HeadPtr(); + return IGraphTransformer::TStatus::Repeat; + } + + bool isStream; + if (!EnsureSeqType(input->Head(), ctx.Expr, &isStream)) { + return IGraphTransformer::TStatus::Error; + } + + auto inputItemType = isStream + ? input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType() + : input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + + if (!EnsureStructType(input->Head().Pos(), *inputItemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto inputStructType = inputItemType->Cast<TStructExprType>(); + for (auto& item : inputStructType->GetItems()) { + auto columnName = item->GetName(); + auto columnType = item->GetItemType(); + if (!columnType->IsHashable() || !columnType->IsEquatable()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Expected hashable and equatable type for key column: " << columnName << ", but got: " << *columnType)); + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(input->Head().GetTypeAnn()); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus FilterNullMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureMinMaxArgsCount(*input, 1, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -4623,8 +4623,8 @@ namespace { const TTypeAnnotationNode* itemType = nullptr; if (!EnsureNewSeqType<true>(input->Head(), ctx.Expr, &itemType)) { return IGraphTransformer::TStatus::Error; - } - + } + if (!EnsureStructType(input->Head().Pos(), *itemType, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -4651,7 +4651,7 @@ namespace { return IGraphTransformer::TStatus::Repeat; } } else { - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -4729,7 +4729,7 @@ namespace { return IGraphTransformer::TStatus::Repeat; } } else { - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -4742,7 +4742,7 @@ namespace { const auto it = columnTypes.find(name); if (columnTypes.cend() == it) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(members->Pos()), TStringBuilder() << "Unknown column: " << name)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(members->Pos()), TStringBuilder() << "Unknown column: " << name)); return IGraphTransformer::TStatus::Error; } @@ -4795,7 +4795,7 @@ namespace { return IGraphTransformer::TStatus::Repeat; } } else { - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -4806,12 +4806,12 @@ namespace { for (const auto& child : elements->Children()) { ui32 index = 0U; if (!TryFromString(child->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (index >= columnTypes.size()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << columnTypes.size())); return IGraphTransformer::TStatus::Error; } @@ -4868,7 +4868,7 @@ namespace { return IGraphTransformer::TStatus::Repeat; } } else { - const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); if (status != IGraphTransformer::TStatus::Ok) { return status; } @@ -4879,12 +4879,12 @@ namespace { for (const auto& child : elements->Children()) { ui32 index = 0U; if (!TryFromString(child->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (index >= columnTypes.size()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Index out of range. Index: " << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Index out of range. Index: " << index << ", size: " << columnTypes.size())); return IGraphTransformer::TStatus::Error; } @@ -4911,65 +4911,65 @@ namespace { if (!EnsureMinArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - - auto frameSettingsNode = input->ChildPtr(0); - if (frameSettingsNode->IsList()) { - TExprNode::TPtr normalizedFrameSettings; - auto status = NormalizeKeyValueTuples(frameSettingsNode, 0, normalizedFrameSettings, ctx.Expr); - if (status.Level == IGraphTransformer::TStatus::Repeat) { - output = ctx.Expr.ChangeChild(*input, 0, std::move(normalizedFrameSettings)); - return status; - } - - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - } - - TWindowFrameSettings frameSettings; - if (!ParseWindowFrameSettings(*input, frameSettings, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto status = NormalizeKeyValueTuples(input, 1, output, ctx.Expr); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - - for (ui32 i = 1; i < input->ChildrenSize(); ++i) { - if (!EnsureTupleSize(*input->Child(i), 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto currColumn = input->Child(i)->Child(0)->Content(); - auto calcSpec = input->Child(i)->Child(1); + + auto frameSettingsNode = input->ChildPtr(0); + if (frameSettingsNode->IsList()) { + TExprNode::TPtr normalizedFrameSettings; + auto status = NormalizeKeyValueTuples(frameSettingsNode, 0, normalizedFrameSettings, ctx.Expr); + if (status.Level == IGraphTransformer::TStatus::Repeat) { + output = ctx.Expr.ChangeChild(*input, 0, std::move(normalizedFrameSettings)); + return status; + } + + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + } + + TWindowFrameSettings frameSettings; + if (!ParseWindowFrameSettings(*input, frameSettings, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto status = NormalizeKeyValueTuples(input, 1, output, ctx.Expr); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + + for (ui32 i = 1; i < input->ChildrenSize(); ++i) { + if (!EnsureTupleSize(*input->Child(i), 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto currColumn = input->Child(i)->Child(0)->Content(); + auto calcSpec = input->Child(i)->Child(1); if (!calcSpec->IsCallable({"WindowTraits", "Lag", "Lead", "RowNumber", "Rank", "DenseRank", "Void"})) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(calcSpec->Pos()), - "Invalid traits or special function for calculation on window")); - return IGraphTransformer::TStatus::Error; - } - - if (i + 1 < input->ChildrenSize()) { - auto nextColumn = input->Child(i + 1)->Child(0)->Content(); - if (currColumn == nextColumn) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i + 1)->Child(0)->Pos()), - TStringBuilder() << "Duplicate column '" << nextColumn << "'")); - return IGraphTransformer::TStatus::Error; - } - } - } - + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(calcSpec->Pos()), + "Invalid traits or special function for calculation on window")); + return IGraphTransformer::TStatus::Error; + } + + if (i + 1 < input->ChildrenSize()) { + auto nextColumn = input->Child(i + 1)->Child(0)->Content(); + if (currColumn == nextColumn) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i + 1)->Child(0)->Pos()), + TStringBuilder() << "Duplicate column '" << nextColumn << "'")); + return IGraphTransformer::TStatus::Error; + } + } + } + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); return IGraphTransformer::TStatus::Ok; } IGraphTransformer::TStatus WinOnRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), "WinOnRange is not supported yet")); - return IGraphTransformer::TStatus::Error; + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), "WinOnRange is not supported yet")); + return IGraphTransformer::TStatus::Error; } IGraphTransformer::TStatus WindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 6, ctx.Expr)) { + if (!EnsureArgsCount(*input, 6, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } auto& item = input->ChildRef(0); @@ -4978,8 +4978,8 @@ namespace { auto& lambdaShift = input->ChildRef(3); auto& lambdaCurrent = input->ChildRef(4); - if (auto status = EnsureTypeRewrite(item, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(item, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = item->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -5036,7 +5036,7 @@ namespace { } if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -5100,24 +5100,24 @@ namespace { return IGraphTransformer::TStatus::Error; } - status = ValidateTraitsDefaultValue(input, 5, *currentType, "current", output, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - + status = ValidateTraitsDefaultValue(input, 5, *currentType, "current", output, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); return IGraphTransformer::TStatus::Ok; } IGraphTransformer::TStatus CalcOverWindowWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - const bool isSession = input->IsCallable("CalcOverSessionWindow"); - if (isSession && input->ChildrenSize() == 5) { - auto children = input->ChildrenList(); - children.push_back(ctx.Expr.NewList(input->Pos(), {})); - output = ctx.Expr.ChangeChildren(*input, std::move(children)); - return IGraphTransformer::TStatus::Repeat; - } - if (!EnsureArgsCount(*input, isSession ? 6 : 4, ctx.Expr)) { + const bool isSession = input->IsCallable("CalcOverSessionWindow"); + if (isSession && input->ChildrenSize() == 5) { + auto children = input->ChildrenList(); + children.push_back(ctx.Expr.NewList(input->Pos(), {})); + output = ctx.Expr.ChangeChildren(*input, std::move(children)); + return IGraphTransformer::TStatus::Repeat; + } + if (!EnsureArgsCount(*input, isSession ? 6 : 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -5135,102 +5135,102 @@ namespace { return IGraphTransformer::TStatus::Error; } - auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); - if (isSession) { - status = status.Combine(NormalizeTupleOfAtoms(input, 5, output, ctx.Expr, /*deduplicate=*/false)); - } - - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - + auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr); + if (isSession) { + status = status.Combine(NormalizeTupleOfAtoms(input, 5, output, ctx.Expr, /*deduplicate=*/false)); + } + + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto inputStructType = inputListType->Cast<TStructExprType>(); + TVector<const TItemExprType*> outputColumns = inputStructType->GetItems(); + + status = ValidateCalcOverWindowArgs(outputColumns, *inputStructType, *input->Child(1), *input->Child(2), *input->Child(3), + isSession ? input->ChildPtr(4) : TExprNode::TPtr(), isSession ? input->ChildPtr(5) : TExprNode::TPtr(), ctx.Expr); + + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto rowType = ctx.Expr.MakeType<TStructExprType>(outputColumns); + if (!rowType->Validate(input->Pos(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(rowType)); + return IGraphTransformer::TStatus::Ok; + } + + IGraphTransformer::TStatus CalcOverWindowGroupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureListType(input->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto inputListType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); + if (!EnsureStructType(input->Head().Pos(), *inputListType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto& calcs = input->ChildRef(1); + if (!EnsureTuple(*calcs, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; + TExprNodeList calcsList = calcs->ChildrenList(); + for (auto& calc : calcsList) { + if (!EnsureTupleMinSize(*calc, 3, ctx.Expr) || !EnsureTupleMaxSize(*calc, 5, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (calc->ChildrenSize() < 5) { + auto calcItems = calc->ChildrenList(); + if (calc->ChildrenSize() == 3) { + calcItems.push_back(ctx.Expr.NewCallable(calc->Pos(), "Void", {})); + } + if (calc->ChildrenSize() == 4) { + calcItems.push_back(ctx.Expr.NewList(calc->Pos(), {})); + } + + calc = ctx.Expr.ChangeChildren(*calc, std::move(calcItems)); + status = status.Combine(IGraphTransformer::TStatus::Repeat); + } else { + status = status.Combine(NormalizeTupleOfAtoms(calc, TCoCalcOverWindowTuple::idx_SessionColumns, calc, + ctx.Expr, /*deduplicate=*/false)); + } + + if (status.Level == IGraphTransformer::TStatus::Error) { + return status; + } + } + + if (status.Level != IGraphTransformer::TStatus::Ok) { + if (status.Level == IGraphTransformer::TStatus::Repeat) { + calcs = ctx.Expr.NewList(calcs->Pos(), std::move(calcsList)); + } + return status; + } + auto inputStructType = inputListType->Cast<TStructExprType>(); - TVector<const TItemExprType*> outputColumns = inputStructType->GetItems(); - - status = ValidateCalcOverWindowArgs(outputColumns, *inputStructType, *input->Child(1), *input->Child(2), *input->Child(3), - isSession ? input->ChildPtr(4) : TExprNode::TPtr(), isSession ? input->ChildPtr(5) : TExprNode::TPtr(), ctx.Expr); - - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto rowType = ctx.Expr.MakeType<TStructExprType>(outputColumns); - if (!rowType->Validate(input->Pos(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(rowType)); - return IGraphTransformer::TStatus::Ok; - } - - IGraphTransformer::TStatus CalcOverWindowGroupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 2, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureListType(input->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto inputListType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType(); - if (!EnsureStructType(input->Head().Pos(), *inputListType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto& calcs = input->ChildRef(1); - if (!EnsureTuple(*calcs, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; - TExprNodeList calcsList = calcs->ChildrenList(); - for (auto& calc : calcsList) { - if (!EnsureTupleMinSize(*calc, 3, ctx.Expr) || !EnsureTupleMaxSize(*calc, 5, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (calc->ChildrenSize() < 5) { - auto calcItems = calc->ChildrenList(); - if (calc->ChildrenSize() == 3) { - calcItems.push_back(ctx.Expr.NewCallable(calc->Pos(), "Void", {})); - } - if (calc->ChildrenSize() == 4) { - calcItems.push_back(ctx.Expr.NewList(calc->Pos(), {})); - } - - calc = ctx.Expr.ChangeChildren(*calc, std::move(calcItems)); - status = status.Combine(IGraphTransformer::TStatus::Repeat); - } else { - status = status.Combine(NormalizeTupleOfAtoms(calc, TCoCalcOverWindowTuple::idx_SessionColumns, calc, - ctx.Expr, /*deduplicate=*/false)); - } - - if (status.Level == IGraphTransformer::TStatus::Error) { - return status; - } - } - - if (status.Level != IGraphTransformer::TStatus::Ok) { - if (status.Level == IGraphTransformer::TStatus::Repeat) { - calcs = ctx.Expr.NewList(calcs->Pos(), std::move(calcsList)); - } - return status; - } - - auto inputStructType = inputListType->Cast<TStructExprType>(); - TVector<const TItemExprType*> outputColumns = inputStructType->GetItems(); - - for (auto& calc : calcs->Children()) { - status = ValidateCalcOverWindowArgs(outputColumns, *inputStructType, *calc->Child(0), *calc->Child(1), *calc->Child(2), - calc->ChildPtr(3), calc->ChildPtr(4), ctx.Expr); - - if (status != IGraphTransformer::TStatus::Ok) { - return status; - } - } - - auto rowType = ctx.Expr.MakeType<TStructExprType>(outputColumns); + TVector<const TItemExprType*> outputColumns = inputStructType->GetItems(); + + for (auto& calc : calcs->Children()) { + status = ValidateCalcOverWindowArgs(outputColumns, *inputStructType, *calc->Child(0), *calc->Child(1), *calc->Child(2), + calc->ChildPtr(3), calc->ChildPtr(4), ctx.Expr); + + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + } + + auto rowType = ctx.Expr.MakeType<TStructExprType>(outputColumns); if (!rowType->Validate(input->Pos(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -5247,8 +5247,8 @@ namespace { auto& lambdaValue = input->ChildRef(1); auto winOffsetType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Int64); - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto rowListType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (rowListType->GetKind() == ETypeAnnotationKind::EmptyList) { @@ -5266,7 +5266,7 @@ namespace { auto status = ConvertToLambda(lambdaValue, ctx.Expr, 1); if (argCount > 2) { - status = status.Combine(TryConvertTo(input->ChildRef(2), *winOffsetType, ctx.Expr)); + status = status.Combine(TryConvertTo(input->ChildRef(2), *winOffsetType, ctx.Expr)); } if (status.Level != IGraphTransformer::TStatus::Ok) { return status; @@ -5277,7 +5277,7 @@ namespace { if (!lambdaValue->GetTypeAnn()) { return IGraphTransformer::TStatus::Repeat; } - const bool isOptional = lambdaValue->GetTypeAnn()->IsOptionalOrNull(); + const bool isOptional = lambdaValue->GetTypeAnn()->IsOptionalOrNull(); input->SetTypeAnn(isOptional ? lambdaValue->GetTypeAnn() : ctx.Expr.MakeType<TOptionalExprType>(lambdaValue->GetTypeAnn())); return IGraphTransformer::TStatus::Ok; } @@ -5287,123 +5287,123 @@ namespace { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64)); return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto& lambdaValue = input->ChildRef(1); - auto options = input->Child(2); - - if (!EnsureTuple(*options, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - for (const auto& option : options->Children()) { - if (!EnsureTupleSize(*option, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(option->Head(), ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - const auto optionName = option->Head().Content(); - static const THashSet<TStringBuf> supportedOptions = - {"ansi", "warnNoAnsi"}; - if (!supportedOptions.contains(optionName)) { - ctx.Expr.AddError( - TIssue(ctx.Expr.GetPosition(option->Pos()), - TStringBuilder() << "Unknown " << input->Content() << "option '" << optionName)); - return IGraphTransformer::TStatus::Error; - } - } - - auto rowListType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto& lambdaValue = input->ChildRef(1); + auto options = input->Child(2); + + if (!EnsureTuple(*options, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + for (const auto& option : options->Children()) { + if (!EnsureTupleSize(*option, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(option->Head(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + const auto optionName = option->Head().Content(); + static const THashSet<TStringBuf> supportedOptions = + {"ansi", "warnNoAnsi"}; + if (!supportedOptions.contains(optionName)) { + ctx.Expr.AddError( + TIssue(ctx.Expr.GetPosition(option->Pos()), + TStringBuilder() << "Unknown " << input->Content() << "option '" << optionName)); + return IGraphTransformer::TStatus::Error; + } + } + + auto rowListType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (rowListType->GetKind() == ETypeAnnotationKind::EmptyList) { output = ctx.Expr.NewCallable(input->Pos(), "Void", {}); return IGraphTransformer::TStatus::Repeat; } - if (!EnsureListType(input->Head().Pos(), *rowListType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto rowType = rowListType->Cast<TListExprType>()->GetItemType(); - if (!EnsureStructType(input->Head().Pos(), *rowType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto status = ConvertToLambda(lambdaValue, ctx.Expr, 1); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaValue, {rowType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto keyType = lambdaValue->GetTypeAnn(); - if (!keyType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureEquatableType(lambdaValue->Pos(), *keyType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - bool isAnsi = HasSetting(*options, "ansi"); - if (!isAnsi && keyType->HasOptionalOrNull() && keyType->GetKind() != ETypeAnnotationKind::Optional) { - // this case is just broken in legacy RANK()/DENSE_RANK() - ctx.Expr.AddError( - TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Legacy " << input->Content() << " does not support key type " << *keyType << ".\n" - << "Please use PRAGMA AnsiRankForNullableKeys;")); - return IGraphTransformer::TStatus::Error; - } - - if (HasSetting(*options, "warnNoAnsi")) { - if (!isAnsi && keyType->HasOptionalOrNull()) { - auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << input->Content() << " will not produce ANSI-compliant result here.\n" - << "Consider adding 'PRAGMA AnsiRankForNullableKeys;' or \n" - << "'PRAGMA DisableAnsiRankForNullableKeys;' to keep current behavior and silence this warning"); - SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_LEGACY_RANK_FOR_NULLABLE_KEYS, issue); - if (!ctx.Expr.AddWarning(issue)) { - return IGraphTransformer::TStatus::Error; - } - } - - output = ctx.Expr.ChangeChild(*input, 2, RemoveSetting(*options, "warnNoAnsi", ctx.Expr)); - return IGraphTransformer::TStatus::Repeat; - } - - const TTypeAnnotationNode* outputType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); - if (!isAnsi && keyType->GetKind() == ETypeAnnotationKind::Optional) { - outputType = ctx.Expr.MakeType<TOptionalExprType>(outputType); - } - - input->SetTypeAnn(outputType); - return IGraphTransformer::TStatus::Ok; - } - + if (!EnsureListType(input->Head().Pos(), *rowListType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto rowType = rowListType->Cast<TListExprType>()->GetItemType(); + if (!EnsureStructType(input->Head().Pos(), *rowType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto status = ConvertToLambda(lambdaValue, ctx.Expr, 1); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaValue, {rowType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto keyType = lambdaValue->GetTypeAnn(); + if (!keyType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureEquatableType(lambdaValue->Pos(), *keyType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + bool isAnsi = HasSetting(*options, "ansi"); + if (!isAnsi && keyType->HasOptionalOrNull() && keyType->GetKind() != ETypeAnnotationKind::Optional) { + // this case is just broken in legacy RANK()/DENSE_RANK() + ctx.Expr.AddError( + TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Legacy " << input->Content() << " does not support key type " << *keyType << ".\n" + << "Please use PRAGMA AnsiRankForNullableKeys;")); + return IGraphTransformer::TStatus::Error; + } + + if (HasSetting(*options, "warnNoAnsi")) { + if (!isAnsi && keyType->HasOptionalOrNull()) { + auto issue = TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << input->Content() << " will not produce ANSI-compliant result here.\n" + << "Consider adding 'PRAGMA AnsiRankForNullableKeys;' or \n" + << "'PRAGMA DisableAnsiRankForNullableKeys;' to keep current behavior and silence this warning"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_LEGACY_RANK_FOR_NULLABLE_KEYS, issue); + if (!ctx.Expr.AddWarning(issue)) { + return IGraphTransformer::TStatus::Error; + } + } + + output = ctx.Expr.ChangeChild(*input, 2, RemoveSetting(*options, "warnNoAnsi", ctx.Expr)); + return IGraphTransformer::TStatus::Repeat; + } + + const TTypeAnnotationNode* outputType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64); + if (!isAnsi && keyType->GetKind() == ETypeAnnotationKind::Optional) { + outputType = ctx.Expr.MakeType<TOptionalExprType>(outputType); + } + + input->SetTypeAnn(outputType); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -5427,7 +5427,7 @@ namespace { const TTypeAnnotationNode* timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)); if (!IsSameAnnotation(*lambdaTimeExtractor->GetTypeAnn(), *timeType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaTimeExtractor->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaTimeExtractor->Pos()), TStringBuilder() << "Mismatch hopping window time extractor lambda output type, expected: " << *timeType << ", but got: " << *lambdaTimeExtractor->GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -5439,19 +5439,19 @@ namespace { auto type = param->GetTypeAnn(); if (type->GetKind() == ETypeAnnotationKind::Optional) { if (param->IsCallable("Nothing")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Hopping window parameter value cannot be evaluated")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Hopping window parameter value cannot be evaluated")); return IGraphTransformer::TStatus::Error; } type = type->Cast<TOptionalExprType>()->GetItemType(); } if (!IsSameAnnotation(*type, *intervalType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), TStringBuilder() << "Mismatch hopping window parameter type, expected: " << *intervalType << ", but got: " << *type)); return IGraphTransformer::TStatus::Error; } if (!IsPureIsolatedLambda(*param)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Parameter is not a pure expression")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Parameter is not a pure expression")); return IGraphTransformer::TStatus::Error; } return IGraphTransformer::TStatus::Ok; @@ -5562,7 +5562,7 @@ namespace { } if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -5575,7 +5575,7 @@ namespace { } if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -5587,13 +5587,13 @@ namespace { return IGraphTransformer::TStatus::Repeat; } if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() << "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << "Save and load lambdas must be specified at the same time")); return IGraphTransformer::TStatus::Error; } @@ -5625,7 +5625,7 @@ namespace { } if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " << *loadLambda->GetTypeAnn() << " != " << *stateType)); return IGraphTransformer::TStatus::Error; } @@ -5871,7 +5871,7 @@ namespace { return IGraphTransformer::TStatus::Repeat; } if (!IsSameAnnotation(*updateHandler->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateHandler->Pos()), "Mismatch of update lambda return type and state type")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateHandler->Pos()), "Mismatch of update lambda return type and state type")); return IGraphTransformer::TStatus::Error; } @@ -5937,7 +5937,7 @@ namespace { } if (!keyExtractor->GetTypeAnn()->IsHashable() || !keyExtractor->GetTypeAnn()->IsEquatable()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyExtractor->Pos()), TStringBuilder() + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyExtractor->Pos()), TStringBuilder() << "Expected hashable and equatable type for key, but got: " << *keyExtractor->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h index 2ff2bedcd3..608b813fe8 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.h +++ b/ydb/library/yql/core/type_ann/type_ann_list.h @@ -55,7 +55,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus HasItemsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus UnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); template <bool IsStrict> IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListUnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -65,8 +65,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus GroupByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus PartitionByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus PartitionsByKeysWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -87,11 +87,11 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus ExtractWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ExtractMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus AssumeUniqueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); + IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); IGraphTransformer::TStatus AggregationTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus MultiAggregateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus AggregateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus SqlAggregateAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus SqlAggregateAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus FilterNullMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SkipNullMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus FilterNullElementsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); @@ -100,10 +100,10 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus WinOnRangeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus CalcOverWindowWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus CalcOverWindowGroupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus CalcOverWindowGroupWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/ydb/library/yql/core/type_ann/type_ann_types.cpp b/ydb/library/yql/core/type_ann/type_ann_types.cpp index 2c528e6a57..c27c0c433f 100644 --- a/ydb/library/yql/core/type_ann/type_ann_types.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_types.cpp @@ -56,7 +56,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus EnsureCodeOrListOfCode(TExprNode::TPtr& node, TExprContext& ctx) { if (!node->GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Expected code or list of code, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Expected code or list of code, but got lambda")); return IGraphTransformer::TStatus::Error; } @@ -77,7 +77,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Repeat; } - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected code or list of code, but got: " << *node->GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected code or list of code, but got: " << *node->GetTypeAnn())); return IGraphTransformer::TStatus::Error; } @@ -117,8 +117,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -165,13 +165,13 @@ namespace NTypeAnnImpl { ui32 row = 0; if (!TryFromString(input->Child(0)->Content(), row)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(0)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(0)->Content())); return IGraphTransformer::TStatus::Error; } ui32 column = 0; if (!TryFromString(input->Child(1)->Content(), column)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -196,7 +196,7 @@ namespace NTypeAnnImpl { auto name = input->Child(0)->Content(); auto slot = NKikimr::NUdf::FindDataSlot(name); if (!slot) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unknown data type: " << name)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Unknown data type: " << name)); return IGraphTransformer::TStatus::Error; } else if (*slot == EDataSlot::Decimal) { if (!EnsureArgsCount(*input, 3, ctx.Expr)) { @@ -226,8 +226,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -247,8 +247,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -268,8 +268,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -289,8 +289,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -307,10 +307,10 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus TypeWrapper<ETypeAnnotationKind::Tuple>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); TTypeAnnotationNode::TListType items; - for (size_t i = 0; i < input->ChildrenSize(); ++i) { - auto& child = input->ChildRef(i); - if (auto status = EnsureTypeRewrite(child, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + for (size_t i = 0; i < input->ChildrenSize(); ++i) { + auto& child = input->ChildRef(i); + if (auto status = EnsureTypeRewrite(child, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto elemType = child->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (!EnsureInspectableType(child->Pos(), *elemType, ctx.Expr)) { @@ -329,10 +329,10 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus TypeWrapper<ETypeAnnotationKind::Multi>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); TTypeAnnotationNode::TListType items; - for (size_t i = 0; i < input->ChildrenSize(); ++i) { - auto& child = input->ChildRef(i); - if (auto status = EnsureTypeRewrite(child, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + for (size_t i = 0; i < input->ChildrenSize(); ++i) { + auto& child = input->ChildRef(i); + if (auto status = EnsureTypeRewrite(child, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto elemType = child->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -352,8 +352,8 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus TypeWrapper<ETypeAnnotationKind::Struct>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); TVector<const TItemExprType*> items; - for (size_t i = 0; i < input->ChildrenSize(); ++i) { - auto& child = input->ChildRef(i); + for (size_t i = 0; i < input->ChildrenSize(); ++i) { + auto& child = input->ChildRef(i); if (!EnsureTupleSize(*child, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -363,12 +363,12 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - auto& typeNode = child->ChildRef(1); - if (auto status = EnsureTypeRewrite(typeNode, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - if (status == IGraphTransformer::TStatus::Repeat) { - child = ctx.Expr.ShallowCopy(*child); - } - return status; + auto& typeNode = child->ChildRef(1); + if (auto status = EnsureTypeRewrite(typeNode, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + if (status == IGraphTransformer::TStatus::Repeat) { + child = ctx.Expr.ShallowCopy(*child); + } + return status; } auto memberType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -406,12 +406,12 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(0), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } - if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto keyType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -501,8 +501,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -512,8 +512,8 @@ namespace NTypeAnnImpl { } if (type->GetKind() != ETypeAnnotationKind::Optional) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), - TStringBuilder() << "Expected optional type, but got: " << *type)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), + TStringBuilder() << "Expected optional type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -533,8 +533,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -544,8 +544,8 @@ namespace NTypeAnnImpl { } if (type->GetKind() != ETypeAnnotationKind::List) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), - TStringBuilder() << "Expected list type, but got: " << *type)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), + TStringBuilder() << "Expected list type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -565,14 +565,14 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Stream) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), - TStringBuilder() << "Expected stream type, but got: " << *type)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), + TStringBuilder() << "Expected stream type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -586,8 +586,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -596,20 +596,20 @@ namespace NTypeAnnImpl { auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), - TStringBuilder() << "Expected tuple type, but got: " << *type)); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), + TStringBuilder() << "Expected tuple type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } ui32 index = 0; if (!TryFromString(input->Child(1)->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } auto tupleType = type->Cast<TTupleExprType>(); if (index >= tupleType->GetSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect tuple index: " << index + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect tuple index: " << index << ", tuple size: " << tupleType->GetSize())); return IGraphTransformer::TStatus::Error; @@ -625,8 +625,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -635,7 +635,7 @@ namespace NTypeAnnImpl { auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected struct type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected struct type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -650,7 +650,7 @@ namespace NTypeAnnImpl { } if (!foundMember) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect member name: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect member name: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -665,8 +665,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -676,7 +676,7 @@ namespace NTypeAnnImpl { } if (type->GetKind() != ETypeAnnotationKind::Dict) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected dict type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected dict type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -691,8 +691,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -702,7 +702,7 @@ namespace NTypeAnnImpl { } if (type->GetKind() != ETypeAnnotationKind::Dict) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected dict type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected dict type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -733,7 +733,7 @@ namespace NTypeAnnImpl { } if (!TryFromString(input->Child(0)->Child(0)->Content(), optionalArgsCount)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(0)->Child(0)->Content())); return IGraphTransformer::TStatus::Error; } @@ -760,12 +760,12 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - auto& typeChild = child->ChildRef(0); - if (auto status = EnsureTypeRewrite(typeChild, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - if (status == IGraphTransformer::TStatus::Repeat) { - input->ChildRef(index) = ctx.Expr.ShallowCopy(*child); - } - return status; + auto& typeChild = child->ChildRef(0); + if (auto status = EnsureTypeRewrite(typeChild, ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + if (status == IGraphTransformer::TStatus::Repeat) { + input->ChildRef(index) = ctx.Expr.ShallowCopy(*child); + } + return status; } auto type = typeChild->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -794,7 +794,7 @@ namespace NTypeAnnImpl { } if (!TryFromString<ui64>(flagsChild->Content(), arg.Flags)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(flagsChild->Pos()), TStringBuilder() << "Expected unsigned number, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(flagsChild->Pos()), TStringBuilder() << "Expected unsigned number, but got: " << flagsChild->Content())); } } @@ -818,13 +818,13 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Callable) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -839,8 +839,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -849,20 +849,20 @@ namespace NTypeAnnImpl { auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Callable) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } ui32 index = 0; if (!TryFromString(input->Child(1)->Content(), index)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Failed to convert to integer: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } auto callableType = type->Cast<TCallableExprType>(); if (index >= callableType->GetArgumentsSize()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect argument index: " << index + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Incorrect argument index: " << index << ", total arguments: " << callableType->GetArgumentsSize())); return IGraphTransformer::TStatus::Error; } @@ -883,15 +883,15 @@ namespace NTypeAnnImpl { TMemoryPool pool(4096); TIssues issues; - auto parsedType = ParseType(input->Child(0)->Content(), pool, issues, ctx.Expr.GetPosition(input->Child(0)->Pos())); + auto parsedType = ParseType(input->Child(0)->Content(), pool, issues, ctx.Expr.GetPosition(input->Child(0)->Pos())); if (!parsedType) { ctx.Expr.IssueManager.AddIssues(issues); return IGraphTransformer::TStatus::Error; } - auto inputPos = ctx.Expr.GetPosition(input->Pos()); - auto astRoot = TAstNode::NewList(inputPos, pool, - TAstNode::NewList(inputPos, pool, + auto inputPos = ctx.Expr.GetPosition(input->Pos()); + auto astRoot = TAstNode::NewList(inputPos, pool, + TAstNode::NewList(inputPos, pool, TAstNode::NewLiteralAtom(inputPos, TStringBuf("return"), pool), parsedType)); TExprNode::TPtr exprRoot; if (!CompileExpr(*astRoot, exprRoot, ctx.Expr, nullptr)) { @@ -916,10 +916,10 @@ namespace NTypeAnnImpl { } auto resType = MakeTypeHandleResourceType(ctx.Expr); - if (input->Child(0)->GetTypeAnn() != resType) { - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; - } + if (input->Child(0)->GetTypeAnn() != resType) { + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; + } } input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)); @@ -932,8 +932,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr)); @@ -974,16 +974,16 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -1014,8 +1014,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } if (!EnsureAtom(*input->Child(1), ctx.Expr)) { @@ -1023,7 +1023,7 @@ namespace NTypeAnnImpl { } if (input->Child(1)->Content().empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Empty member name is not allowed")); + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), "Empty member name is not allowed")); return IGraphTransformer::TStatus::Error; } @@ -1046,7 +1046,7 @@ namespace NTypeAnnImpl { } if (!found && !force) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Unknown member name: " << input->Child(1)->Content())); return IGraphTransformer::TStatus::Error; } @@ -1083,8 +1083,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(child->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(child->ChildRef(1), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = child->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -1130,8 +1130,8 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Error; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto underlyingType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); @@ -1160,13 +1160,13 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { - return status; + if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { + return status; } auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Variant) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected variant type, but got: " + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected variant type, but got: " << *type)); return IGraphTransformer::TStatus::Error; } @@ -1530,7 +1530,7 @@ namespace NTypeAnnImpl { .List(1) .Atom(0, "Name") .Callable(1, "Coalesce") - .Add(0, input->ChildrenSize() > 1 ? input->ChildPtr(1) : ctx.Expr.NewCallable(TPosition(), "Null", {})) + .Add(0, input->ChildrenSize() > 1 ? input->ChildPtr(1) : ctx.Expr.NewCallable(TPosition(), "Null", {})) .Callable(1, "String") .Atom(0, "") .Seal() @@ -1539,7 +1539,7 @@ namespace NTypeAnnImpl { .List(2) .Atom(0, "Flags") .Callable(1, "Coalesce") - .Add(0, input->ChildrenSize() > 2 ? input->ChildPtr(2) : ctx.Expr.NewCallable(TPosition(), "Null", {})) + .Add(0, input->ChildrenSize() > 2 ? input->ChildPtr(2) : ctx.Expr.NewCallable(TPosition(), "Null", {})) .Callable(1, "List") .Callable(0, "ListType") .Callable(0, "DataType") @@ -1646,20 +1646,20 @@ return IGraphTransformer::TStatus::Repeat; return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); } - IGraphTransformer::TStatus EvaluateExprIfPureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!input->Head().GetTypeAnn()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Lambda is unexpected here")); - return IGraphTransformer::TStatus::Error; - } - - output = IsPureIsolatedLambda(input->Head()) ? ctx.Expr.RenameNode(*input, "EvaluateExpr") : input->HeadPtr(); - return IGraphTransformer::TStatus::Repeat; - } - + IGraphTransformer::TStatus EvaluateExprIfPureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!input->Head().GetTypeAnn()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Lambda is unexpected here")); + return IGraphTransformer::TStatus::Error; + } + + output = IsPureIsolatedLambda(input->Head()) ? ctx.Expr.RenameNode(*input, "EvaluateExpr") : input->HeadPtr(); + return IGraphTransformer::TStatus::Repeat; + } + template <> IGraphTransformer::TStatus MakeCodeWrapper<TExprNode::World>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); diff --git a/ydb/library/yql/core/type_ann/type_ann_types.h b/ydb/library/yql/core/type_ann/type_ann_types.h index 929d574553..8c79ed458d 100644 --- a/ydb/library/yql/core/type_ann/type_ann_types.h +++ b/ydb/library/yql/core/type_ann/type_ann_types.h @@ -50,7 +50,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus FormatCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ReprCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus RestartEvaluationWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus EvaluateExprIfPureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus EvaluateExprIfPureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); template <TExprNode::EType> IGraphTransformer::TStatus MakeCodeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/ydb/library/yql/core/type_ann/ya.make b/ydb/library/yql/core/type_ann/ya.make index 118d39aed8..be843b5abc 100644 --- a/ydb/library/yql/core/type_ann/ya.make +++ b/ydb/library/yql/core/type_ann/ya.make @@ -9,8 +9,8 @@ OWNER( SRCS( type_ann_core.cpp type_ann_core.h - type_ann_columnorder.cpp - type_ann_columnorder.h + type_ann_columnorder.cpp + type_ann_columnorder.h type_ann_expr.cpp type_ann_expr.h type_ann_join.cpp diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make index 081773c322..29169996f5 100644 --- a/ydb/library/yql/core/ya.make +++ b/ydb/library/yql/core/ya.make @@ -35,8 +35,8 @@ SRCS( yql_opt_aggregate.cpp yql_opt_proposed_by_data.cpp yql_opt_proposed_by_data.h - yql_opt_range.cpp - yql_opt_range.h + yql_opt_range.cpp + yql_opt_range.h yql_opt_rewrite_io.cpp yql_opt_rewrite_io.h yql_opt_utils.cpp @@ -45,8 +45,8 @@ SRCS( yql_opt_window.h yql_type_annotation.cpp yql_type_annotation.h - yql_type_helpers.cpp - yql_type_helpers.h + yql_type_helpers.cpp + yql_type_helpers.h yql_udf_index.cpp yql_udf_index.h yql_udf_index_package_set.cpp diff --git a/ydb/library/yql/core/yql_callable_transform.h b/ydb/library/yql/core/yql_callable_transform.h index b6f43b4347..d508152029 100644 --- a/ydb/library/yql/core/yql_callable_transform.h +++ b/ydb/library/yql/core/yql_callable_transform.h @@ -38,7 +38,7 @@ public: auto name = input->Content(); TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << name); + return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << name); }); TStatus status = TStatus::Ok; @@ -168,7 +168,7 @@ protected: auto datasinkName = input.Child(1)->Child(0)->Content(); auto datasink = Types.DataSinkMap.FindPtr(datasinkName); if (!datasink) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; } @@ -191,7 +191,7 @@ protected: auto datasourceName = input.Child(1)->Child(0)->Content(); auto datasource = Types.DataSourceMap.FindPtr(datasourceName); if (!datasource) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); return nullptr; } @@ -214,7 +214,7 @@ protected: auto datasinkName = input.Child(1)->Child(0)->Content(); auto datasink = Types.DataSinkMap.FindPtr(datasinkName); if (!datasink) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; } @@ -238,7 +238,7 @@ protected: auto datasourceName = input.Child(1)->Child(0)->Content(); auto datasource = Types.DataSourceMap.FindPtr(datasourceName); if (!datasource) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); return nullptr; } @@ -249,7 +249,7 @@ protected: auto datasinkName = input.Child(1)->Child(0)->Content(); auto datasink = Types.DataSinkMap.FindPtr(datasinkName); if (!datasink) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); + ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; } @@ -264,7 +264,7 @@ protected: auto status = static_cast<TDerived*>(this)->GetTransformer(dataProvider).Transform(input, output, ctx); if (status.Level == IGraphTransformer::TStatus::Async) { if (InstantOnly) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Async status is not allowed for instant transform, provider name: " << dataProvider.GetName())); return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h index f5a03c7cd7..3ffbdc041d 100644 --- a/ydb/library/yql/core/yql_data_provider.h +++ b/ydb/library/yql/core/yql_data_provider.h @@ -59,21 +59,21 @@ public: virtual TString GetOperationDisplayName(const TExprNode& node) = 0; }; -class ITrackableNodeProcessor { -public: - virtual ~ITrackableNodeProcessor() = default; - - struct TExprNodeAndId - { - TExprNode::TPtr Node; - TString Id; - }; - - virtual void GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) = 0; - virtual void GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) = 0; - virtual IGraphTransformer& GetCleanupTransformer() = 0; -}; - +class ITrackableNodeProcessor { +public: + virtual ~ITrackableNodeProcessor() = default; + + struct TExprNodeAndId + { + TExprNode::TPtr Node; + TString Id; + }; + + virtual void GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) = 0; + virtual void GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) = 0; + virtual IGraphTransformer& GetCleanupTransformer() = 0; +}; + class IDqIntegration; class IOptimizationContext; @@ -162,9 +162,9 @@ public: //-- plan virtual IGraphTransformer& GetPlanInfoTransformer() = 0; virtual IPlanFormatter& GetPlanFormatter() = 0; - - //-- garbage collection - virtual ITrackableNodeProcessor& GetTrackableNodeProcessor() = 0; + + //-- garbage collection + virtual ITrackableNodeProcessor& GetTrackableNodeProcessor() = 0; // DQ virtual IDqIntegration* GetDqIntegration() = 0; diff --git a/ydb/library/yql/core/yql_execution.cpp b/ydb/library/yql/core/yql_execution.cpp index b6a78cfb11..eb138b1b4f 100644 --- a/ydb/library/yql/core/yql_execution.cpp +++ b/ydb/library/yql/core/yql_execution.cpp @@ -6,7 +6,7 @@ #include <ydb/library/yql/utils/yql_panic.h> #include <util/string/builder.h> -#include <util/string/join.h> +#include <util/string/join.h> #include <util/generic/queue.h> @@ -64,15 +64,15 @@ public: }); } - auto status = CollectUnusedNodes(*input, ctx); - if (status == TStatus::Error) { - return status; - } - - status = status.Combine(ExecuteNode(input, output, ctx, 0)); + auto status = CollectUnusedNodes(*input, ctx); + if (status == TStatus::Error) { + return status; + } + + status = status.Combine(ExecuteNode(input, output, ctx, 0)); for (auto node: FreshPendingNodes) { - if (TExprNode::EState::ExecutionPending == node->GetState()) { - node->SetState(TExprNode::EState::ConstrComplete); + if (TExprNode::EState::ExecutionPending == node->GetState()) { + node->SetState(TExprNode::EState::ConstrComplete); } } FreshPendingNodes.clear(); @@ -80,7 +80,7 @@ public: return TStatus::Error; } YQL_CLOG(INFO, CoreExecution) << "Finish, output #" << output->UniqueId() << ", status: " << status; - + if (status != TStatus::Ok || !WithFinalize) { return status; } @@ -114,16 +114,16 @@ public: } for (auto& item : completed) { - auto collectedIt = CollectingNodes.find(item.Node); - if (collectedIt != CollectingNodes.end()) { - YQL_CLOG(INFO, CoreExecution) << "Completed async cleanup for node #" << item.Node->UniqueId(); - TExprNode::TPtr callableOutput; - auto status = item.DataProvider->GetTrackableNodeProcessor().GetCleanupTransformer().ApplyAsyncChanges(item.Node, callableOutput, ctx); - combinedStatus = combinedStatus.Combine(status); - CollectingNodes.erase(collectedIt); - continue; - } - + auto collectedIt = CollectingNodes.find(item.Node); + if (collectedIt != CollectingNodes.end()) { + YQL_CLOG(INFO, CoreExecution) << "Completed async cleanup for node #" << item.Node->UniqueId(); + TExprNode::TPtr callableOutput; + auto status = item.DataProvider->GetTrackableNodeProcessor().GetCleanupTransformer().ApplyAsyncChanges(item.Node, callableOutput, ctx); + combinedStatus = combinedStatus.Combine(status); + CollectingNodes.erase(collectedIt); + continue; + } + YQL_CLOG(INFO, CoreExecution) << "Completed async execution for node #" << item.Node->UniqueId(); TExprNode::TPtr callableOutput; auto status = item.DataProvider->GetCallableExecutionTransformer().ApplyAsyncChanges(item.Node, callableOutput, ctx); @@ -131,23 +131,23 @@ public: YQL_ENSURE(status != TStatus::Async); combinedStatus = combinedStatus.Combine(status); if (status.Level == TStatus::Error) { - item.Node->SetState(TExprNode::EState::Error); + item.Node->SetState(TExprNode::EState::Error); } else if (status.Level == TStatus::Repeat) { if (callableOutput != item.Node) { YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << item.Node->UniqueId() << " to #" << callableOutput->UniqueId() << " in ApplyAsyncChanges()"; NewNodes[item.Node] = callableOutput; - combinedStatus = combinedStatus.Combine(TStatus(TStatus::Repeat, true)); + combinedStatus = combinedStatus.Combine(TStatus(TStatus::Repeat, true)); FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput); } } if (callableOutput == item.Node) { - YQL_CLOG(INFO, CoreExecution) << "State is " << item.Node->GetState() + YQL_CLOG(INFO, CoreExecution) << "State is " << item.Node->GetState() << " after apply async changes for node #" << item.Node->UniqueId(); } - if (item.Node->GetState() == TExprNode::EState::ExecutionComplete || - item.Node->GetState() == TExprNode::EState::Error) + if (item.Node->GetState() == TExprNode::EState::ExecutionComplete || + item.Node->GetState() == TExprNode::EState::Error) { FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput); } @@ -206,10 +206,10 @@ public: State->HasResult = false; NewNodes.clear(); FinalizingTransformer.Reset(); - - TrackableNodes.clear(); - CollectingNodes.clear(); - ProvidersCache.clear(); + + TrackableNodes.clear(); + CollectingNodes.clear(); + ProvidersCache.clear(); } TStatus ExecuteNode(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExprContext& ctx, ui32 depth) { @@ -221,7 +221,7 @@ public: changed = true; } - switch (output->GetState()) { + switch (output->GetState()) { case TExprNode::EState::Initial: case TExprNode::EState::TypeInProgress: case TExprNode::EState::TypePending: @@ -238,7 +238,7 @@ public: break; case TExprNode::EState::ExecutionComplete: YQL_ENSURE(output->HasResult()); - OnNodeExecutionComplete(output, ctx); + OnNodeExecutionComplete(output, ctx); return changed ? TStatus(TStatus::Repeat, true) : TStatus(TStatus::Ok); case TExprNode::EState::Error: return TStatus::Error; @@ -251,8 +251,8 @@ public: case TExprNode::Argument: case TExprNode::Arguments: case TExprNode::Lambda: - ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), TStringBuilder() << "Failed to execute node with type: " << output->Type())); - output->SetState(TExprNode::EState::Error); + ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), TStringBuilder() << "Failed to execute node with type: " << output->Type())); + output->SetState(TExprNode::EState::Error); return TStatus::Error; case TExprNode::List: @@ -263,16 +263,16 @@ public: ? ExecuteCallable(output, output, ctx, depth) : ExecuteList(output, ctx); if (status.Level == TStatus::Error) { - output->SetState(TExprNode::EState::Error); + output->SetState(TExprNode::EState::Error); } else if (status.Level == TStatus::Ok) { - output->SetState(TExprNode::EState::ExecutionComplete); - OnNodeExecutionComplete(output, ctx); + output->SetState(TExprNode::EState::ExecutionComplete); + OnNodeExecutionComplete(output, ctx); YQL_ENSURE(output->HasResult()); } else if (status.Level == TStatus::Repeat) { if (!status.HasRestart) { - output->SetState(TExprNode::EState::ExecutionPending); + output->SetState(TExprNode::EState::ExecutionPending); status = ExecuteChildren(output, output, ctx, depth + 1); - if (TExprNode::EState::ExecutionPending == output->GetState()) { + if (TExprNode::EState::ExecutionPending == output->GetState()) { FreshPendingNodes.push_back(output.Get()); } if (status.Level != TStatus::Repeat) { @@ -285,14 +285,14 @@ public: } return TStatus(TStatus::Repeat, output != prevOutput); } else if (status.Level == TStatus::Async) { - output->SetState(TExprNode::EState::ExecutionInProgress); + output->SetState(TExprNode::EState::ExecutionInProgress); } return status; } case TExprNode::World: - output->SetState(TExprNode::EState::ExecutionComplete); + output->SetState(TExprNode::EState::ExecutionComplete); return TStatus::Ok; default: @@ -306,15 +306,15 @@ public: bool newNode = false; for (auto& child : node->Children()) { auto newChild = child; - if (child->GetState() == TExprNode::EState::ExecutionRequired) { + if (child->GetState() == TExprNode::EState::ExecutionRequired) { auto childStatus = ExecuteNode(child, newChild, ctx, depth); if (childStatus.Level == TStatus::Error) return childStatus; combinedStatus = combinedStatus.Combine(childStatus); - } else if (child->GetState() == TExprNode::EState::ExecutionInProgress) { + } else if (child->GetState() == TExprNode::EState::ExecutionInProgress) { combinedStatus = combinedStatus.Combine(TStatus::Async); - } else if (child->GetState() == TExprNode::EState::ExecutionPending) { + } else if (child->GetState() == TExprNode::EState::ExecutionPending) { combinedStatus = combinedStatus.Combine(TStatus::Repeat); } newChildren.push_back(newChild); @@ -325,11 +325,11 @@ public: if (combinedStatus.Level == TStatus::Ok) { Y_VERIFY_DEBUG(!newNode); - node->SetState(TExprNode::EState::ConstrComplete); + node->SetState(TExprNode::EState::ConstrComplete); return ExecuteNode(node, output, ctx, depth - 1); } else { if (combinedStatus.Level == TStatus::Error) { - node->SetState(TExprNode::EState::Error); + node->SetState(TExprNode::EState::Error); } if (newNode) { output = ctx.ChangeChildren(*node, std::move(newChildren)); @@ -342,7 +342,7 @@ public: TStatus ExecuteList(const TExprNode::TPtr& node, TExprContext& ctx) { IGraphTransformer::TStatus combinedStatus = IGraphTransformer::TStatus::Ok; for (ui32 i = 0; i < node->ChildrenSize(); ++i) { - combinedStatus = combinedStatus.Combine(RequireChild(*node, i)); + combinedStatus = combinedStatus.Combine(RequireChild(*node, i)); } if (combinedStatus.Level != IGraphTransformer::TStatus::Ok) { @@ -353,30 +353,30 @@ public: return TStatus::Ok; } - IDataProvider* GetDataProvider(const TExprNode& node) const { - IDataProvider* dataProvider = nullptr; - for (const auto& x : Types.DataSources) { - if (x->CanExecute(node)) { - dataProvider = x.Get(); - break; - } - } - - if (!dataProvider) { - for (const auto& x : Types.DataSinks) { - if (x->CanExecute(node)) { - dataProvider = x.Get(); - } - } - } - return dataProvider; - } - + IDataProvider* GetDataProvider(const TExprNode& node) const { + IDataProvider* dataProvider = nullptr; + for (const auto& x : Types.DataSources) { + if (x->CanExecute(node)) { + dataProvider = x.Get(); + break; + } + } + + if (!dataProvider) { + for (const auto& x : Types.DataSinks) { + if (x->CanExecute(node)) { + dataProvider = x.Get(); + } + } + } + return dataProvider; + } + TStatus ExecuteCallable(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExprContext& ctx, ui32 depth) { YQL_CLOG(TRACE, CoreExecution) << '{' << depth << "}, callable #" << node->UniqueId() << " <" << node->Content() << '>'; if (node->Content() == CommitName) { - auto requireStatus = RequireChild(*node, 0); + auto requireStatus = RequireChild(*node, 0); if (requireStatus.Level != TStatus::Ok) { return requireStatus; } @@ -391,8 +391,8 @@ public: StartNode(category, *node); AddCallable(node, (*datasink).Get(), ctx); } else { - if (output->GetState() == TExprNode::EState::ExecutionComplete || - output->GetState() == TExprNode::EState::Error) + if (output->GetState() == TExprNode::EState::ExecutionComplete || + output->GetState() == TExprNode::EState::Error) { StartNode(category, *node); FinishNode(category, *node, *output); @@ -407,7 +407,7 @@ public: } if (node->Content() == LeftName) { - auto requireStatus = RequireChild(*node, 0); + auto requireStatus = RequireChild(*node, 0); if (requireStatus.Level != TStatus::Ok) { return requireStatus; } @@ -417,7 +417,7 @@ public: } if (node->Content() == RightName) { - auto requireStatus = RequireChild(*node, 0); + auto requireStatus = RequireChild(*node, 0); if (requireStatus.Level != TStatus::Ok) { return requireStatus; } @@ -426,9 +426,9 @@ public: return TStatus::Ok; } - IDataProvider* dataProvider = GetDataProvider(*node); + IDataProvider* dataProvider = GetDataProvider(*node); if (!dataProvider) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute callable with name: " << node->Content())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute callable with name: " << node->Content())); return TStatus::Error; } @@ -439,8 +439,8 @@ public: StartNode(dataProvider->GetName(), *node); AddCallable(node, dataProvider, ctx); } else { - if (output->GetState() == TExprNode::EState::ExecutionComplete || - output->GetState() == TExprNode::EState::Error) + if (output->GetState() == TExprNode::EState::ExecutionComplete || + output->GetState() == TExprNode::EState::Error) { StartNode(dataProvider->GetName(), *node); FinishNode(dataProvider->GetName(), *node, *output); @@ -454,11 +454,11 @@ public: Y_UNUSED(ctx); YQL_CLOG(INFO, CoreExecution) << "Register async execution for node #" << node->UniqueId(); auto future = dataProvider->GetCallableExecutionTransformer().GetAsyncFuture(*node); - SubscribeAsyncFuture(node, dataProvider, future); - } - - void SubscribeAsyncFuture(const TExprNode::TPtr& node, IDataProvider* dataProvider, const NThreading::TFuture<void>& future) - { + SubscribeAsyncFuture(node, dataProvider, future); + } + + void SubscribeAsyncFuture(const TExprNode::TPtr& node, IDataProvider* dataProvider, const NThreading::TFuture<void>& future) + { auto state = State; future.Subscribe([state, node=node.Get(), dataProvider](const NThreading::TFuture<void>& future) { YQL_ENSURE(!future.HasException()); @@ -503,7 +503,7 @@ public: auto progIt = Progresses.find(*publicId); YQL_ENSURE(progIt != Progresses.end()); - auto newState = (node.GetState() == TExprNode::EState::ExecutionComplete) + auto newState = (node.GetState() == TExprNode::EState::ExecutionComplete) ? TOperationProgress::EState::Finished : TOperationProgress::EState::Failed; @@ -515,115 +515,115 @@ public: } } - void OnNodeExecutionComplete(const TExprNode::TPtr& node, TExprContext& ctx) { - auto nodeId = node->UniqueId(); - YQL_CLOG(INFO, CoreExecution) << "Node #" << nodeId << "<" << node->Content() << "> finished execution"; - - auto dataProvider = GetDataProvider(*node); - if (!dataProvider) { - return; - } - - TVector<ITrackableNodeProcessor::TExprNodeAndId> createdNodes; - dataProvider->GetTrackableNodeProcessor().GetCreatedNodes(*node, createdNodes, ctx); - - TVector<TString> ids; - for (const auto& c : createdNodes) { - auto& info = TrackableNodes[c.Id]; - info.Provider = dataProvider; - info.Node = c.Node; - ids.push_back(c.Id); - } - YQL_CLOG(INFO, CoreExecution) << "Node #" << nodeId << "<" << node->Content() << "> created " << ids.size() - << " trackable nodes: " << JoinSeq(", ", ids); - } - - TStatus CollectUnusedNodes(const TExprNode& root, TExprContext& ctx) { - if (TrackableNodes.empty()) { - return TStatus::Ok; - } - - YQL_CLOG(TRACE, CoreExecution) << "Collecting unused nodes on root #" << root.UniqueId(); - - THashSet<ui64> visited; - THashSet<TString> usedIds; - VisitExpr(root, [&](const TExprNode& node) { - if (node.GetState() == TExprNode::EState::ExecutionComplete) { - return false; - } - - auto nodeId = node.UniqueId(); - visited.insert(nodeId); - - TIntrusivePtr<IDataProvider> dataProvider; - auto providerIt = ProvidersCache.find(nodeId); - if (providerIt != ProvidersCache.end()) { - YQL_ENSURE(providerIt->second); - dataProvider = providerIt->second; - } else { - dataProvider = GetDataProvider(node); - if (dataProvider) { - ProvidersCache[nodeId] = dataProvider; - } - } - - if (dataProvider) { - TVector<TString> usedNodes; - dataProvider->GetTrackableNodeProcessor().GetUsedNodes(node, usedNodes); - usedIds.insert(usedNodes.begin(), usedNodes.end()); - } - - return true; - }); - - for (auto i = ProvidersCache.begin(); i != ProvidersCache.end();) { - if (visited.count(i->first) == 0) { - ProvidersCache.erase(i++); - } else { - ++i; - } - } - - THashMap<TIntrusivePtr<IDataProvider>, TExprNode::TListType> toCollect; - for (auto i = TrackableNodes.begin(); i != TrackableNodes.end();) { - TString id = i->first; - TTrackableNodeInfo info = i->second; - if (!usedIds.contains(id)) { - YQL_ENSURE(info.Node); - YQL_ENSURE(info.Provider); - YQL_CLOG(INFO, CoreExecution) << "Marking node " << id << " for collection"; - toCollect[info.Provider].push_back(info.Node); - TrackableNodes.erase(i++); - } else { - ++i; - } - } - - TStatus collectStatus = TStatus::Ok; - for (auto& i : toCollect) { - const auto& provider = i.first; - YQL_ENSURE(!i.second.empty()); - auto pos = i.second.front()->Pos(); - TExprNode::TPtr collectNode = ctx.NewList(pos, std::move(i.second)); - TExprNode::TPtr output; - TStatus status = provider->GetTrackableNodeProcessor().GetCleanupTransformer().Transform(collectNode, output, ctx); - YQL_ENSURE(status != TStatus::Repeat); - - collectStatus = collectStatus.Combine(status); - if (status == TStatus::Error) { - break; - } - - if (status == TStatus::Async) { - CollectingNodes[collectNode.Get()] = collectNode; - auto future = provider->GetTrackableNodeProcessor().GetCleanupTransformer().GetAsyncFuture(*collectNode); - SubscribeAsyncFuture(collectNode, provider.Get(), future); - } - } - - return collectStatus; - } - + void OnNodeExecutionComplete(const TExprNode::TPtr& node, TExprContext& ctx) { + auto nodeId = node->UniqueId(); + YQL_CLOG(INFO, CoreExecution) << "Node #" << nodeId << "<" << node->Content() << "> finished execution"; + + auto dataProvider = GetDataProvider(*node); + if (!dataProvider) { + return; + } + + TVector<ITrackableNodeProcessor::TExprNodeAndId> createdNodes; + dataProvider->GetTrackableNodeProcessor().GetCreatedNodes(*node, createdNodes, ctx); + + TVector<TString> ids; + for (const auto& c : createdNodes) { + auto& info = TrackableNodes[c.Id]; + info.Provider = dataProvider; + info.Node = c.Node; + ids.push_back(c.Id); + } + YQL_CLOG(INFO, CoreExecution) << "Node #" << nodeId << "<" << node->Content() << "> created " << ids.size() + << " trackable nodes: " << JoinSeq(", ", ids); + } + + TStatus CollectUnusedNodes(const TExprNode& root, TExprContext& ctx) { + if (TrackableNodes.empty()) { + return TStatus::Ok; + } + + YQL_CLOG(TRACE, CoreExecution) << "Collecting unused nodes on root #" << root.UniqueId(); + + THashSet<ui64> visited; + THashSet<TString> usedIds; + VisitExpr(root, [&](const TExprNode& node) { + if (node.GetState() == TExprNode::EState::ExecutionComplete) { + return false; + } + + auto nodeId = node.UniqueId(); + visited.insert(nodeId); + + TIntrusivePtr<IDataProvider> dataProvider; + auto providerIt = ProvidersCache.find(nodeId); + if (providerIt != ProvidersCache.end()) { + YQL_ENSURE(providerIt->second); + dataProvider = providerIt->second; + } else { + dataProvider = GetDataProvider(node); + if (dataProvider) { + ProvidersCache[nodeId] = dataProvider; + } + } + + if (dataProvider) { + TVector<TString> usedNodes; + dataProvider->GetTrackableNodeProcessor().GetUsedNodes(node, usedNodes); + usedIds.insert(usedNodes.begin(), usedNodes.end()); + } + + return true; + }); + + for (auto i = ProvidersCache.begin(); i != ProvidersCache.end();) { + if (visited.count(i->first) == 0) { + ProvidersCache.erase(i++); + } else { + ++i; + } + } + + THashMap<TIntrusivePtr<IDataProvider>, TExprNode::TListType> toCollect; + for (auto i = TrackableNodes.begin(); i != TrackableNodes.end();) { + TString id = i->first; + TTrackableNodeInfo info = i->second; + if (!usedIds.contains(id)) { + YQL_ENSURE(info.Node); + YQL_ENSURE(info.Provider); + YQL_CLOG(INFO, CoreExecution) << "Marking node " << id << " for collection"; + toCollect[info.Provider].push_back(info.Node); + TrackableNodes.erase(i++); + } else { + ++i; + } + } + + TStatus collectStatus = TStatus::Ok; + for (auto& i : toCollect) { + const auto& provider = i.first; + YQL_ENSURE(!i.second.empty()); + auto pos = i.second.front()->Pos(); + TExprNode::TPtr collectNode = ctx.NewList(pos, std::move(i.second)); + TExprNode::TPtr output; + TStatus status = provider->GetTrackableNodeProcessor().GetCleanupTransformer().Transform(collectNode, output, ctx); + YQL_ENSURE(status != TStatus::Repeat); + + collectStatus = collectStatus.Combine(status); + if (status == TStatus::Error) { + break; + } + + if (status == TStatus::Async) { + CollectingNodes[collectNode.Get()] = collectNode; + auto future = provider->GetTrackableNodeProcessor().GetCleanupTransformer().GetAsyncFuture(*collectNode); + SubscribeAsyncFuture(collectNode, provider.Get(), future); + } + } + + return collectStatus; + } + private: TTypeAnnotationContext& Types; TOperationProgressWriter Writer; @@ -632,16 +632,16 @@ private: TNodeOnNodeOwnedMap NewNodes; TAutoPtr<IGraphTransformer> FinalizingTransformer; THashMap<ui32, TOperationProgress> Progresses; - - struct TTrackableNodeInfo - { - TIntrusivePtr<IDataProvider> Provider; - TExprNode::TPtr Node; - }; - - THashMap<TString, TTrackableNodeInfo> TrackableNodes; - TNodeOnNodeOwnedMap CollectingNodes; - THashMap<ui64, TIntrusivePtr<IDataProvider>> ProvidersCache; + + struct TTrackableNodeInfo + { + TIntrusivePtr<IDataProvider> Provider; + TExprNode::TPtr Node; + }; + + THashMap<TString, TTrackableNodeInfo> TrackableNodes; + TNodeOnNodeOwnedMap CollectingNodes; + THashMap<ui64, TIntrusivePtr<IDataProvider>> ProvidersCache; TExprNode::TListType FreshPendingNodes; }; @@ -662,7 +662,7 @@ IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprCo if (node->Content() == CommitName) { auto datasink = types.DataSinkMap.FindPtr(node->Child(1)->Child(0)->Content()); if (!datasink) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Unknown datasink: " << node->Child(1)->Child(0)->Content())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Unknown datasink: " << node->Child(1)->Child(0)->Content())); return TStatus::Error; } @@ -699,13 +699,13 @@ IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprCo } if (!dataProvider) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute callable with name: " << node->Content() + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute callable with name: " << node->Content() << ", you possibly used cross provider/cluster operations or pulled not materialized result in refselect mode")); return TStatus::Error; } if (node->ChildrenSize() < 2) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Executable callable should have at least 2 children")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Executable callable should have at least 2 children")); return TStatus::Error; } @@ -722,12 +722,12 @@ IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprCo IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited) { using TStatus = IGraphTransformer::TStatus; - if (node->GetState() == TExprNode::EState::ExecutionComplete) { + if (node->GetState() == TExprNode::EState::ExecutionComplete) { return TStatus::Ok; } if (!node->GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Node has no type annotation")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Node has no type annotation")); return TStatus::Error; } @@ -737,7 +737,7 @@ IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprC case TExprNode::Argument: case TExprNode::Arguments: case TExprNode::Lambda: - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute node with type: " << node->Type())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute node with type: " << node->Type())); return TStatus::Error; case TExprNode::List: @@ -791,7 +791,7 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio GatherParents(*input, parentsMap); bool hasErrors = false; THashSet<TIssue> added; - auto funcCheckExecution = [&](const THashSet<TStringBuf>& notAllowList, bool collectCalcOverWindow, const TExprNode::TPtr& node) { + auto funcCheckExecution = [&](const THashSet<TStringBuf>& notAllowList, bool collectCalcOverWindow, const TExprNode::TPtr& node) { if (node->IsCallable("ErrorType")) { hasErrors = true; const auto err = node->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TErrorExprType>()->GetError(); @@ -799,23 +799,23 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio ctx.AddError(err); } } else if (node->IsCallable("TablePath") || node->IsCallable("TableRecord")) { - auto issue = TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << node->Content() << " will be empty"); + auto issue = TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << node->Content() << " will be empty"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_FREE_TABLE_PATH_RECORD, issue); if (!ctx.AddWarning(issue)) { hasErrors = true; } } else if (node->IsCallable("UnsafeTimestampCast")) { - auto issue = TIssue(ctx.GetPosition(node->Pos()), "Unsafe conversion integral value to Timestamp, consider using date types"); + auto issue = TIssue(ctx.GetPosition(node->Pos()), "Unsafe conversion integral value to Timestamp, consider using date types"); SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_CAST_INTEGRAL_TO_TIMESTAMP_UNSAFE, issue); if (!ctx.AddWarning(issue)) { hasErrors = true; } - } else if (collectCalcOverWindow && node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})) { + } else if (collectCalcOverWindow && node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})) { overWinNodes.emplace(node.Get()); return false; } else if (node->IsCallable(notAllowList)) { hasErrors = true; - const auto err = TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Can't execute " << node->Content()); + const auto err = TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Can't execute " << node->Content()); if (added.insert(err).second) { ctx.AddError(err); } @@ -838,7 +838,7 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio if (usageCount > 1 && !node->GetConstraint<TEmptyConstraintNode>()) { hasErrors = true; - const auto err = TIssue(ctx.GetPosition(node->Pos()), "Multiple stream clients"); + const auto err = TIssue(ctx.GetPosition(node->Pos()), "Multiple stream clients"); if (added.insert(err).second) { ctx.AddError(err); } @@ -848,16 +848,16 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio return true; }; - static const THashSet<TStringBuf> noExecutionList = {"InstanceOf", "Lag", "Lead", "Rank", "DenseRank", "RowNumber"}; + static const THashSet<TStringBuf> noExecutionList = {"InstanceOf", "Lag", "Lead", "Rank", "DenseRank", "RowNumber"}; static const THashSet<TStringBuf> noExecutionListForCalcOverWindow = {"InstanceOf"}; VisitExpr(input, [funcCheckExecution](const TExprNode::TPtr& node) { - bool collectCalcOverWindow = true; - return funcCheckExecution(noExecutionList, collectCalcOverWindow, node); + bool collectCalcOverWindow = true; + return funcCheckExecution(noExecutionList, collectCalcOverWindow, node); }); for (auto overWin: overWinNodes) { VisitExpr(overWin, [funcCheckExecution](const TExprNode::TPtr& node) { - bool collectCalcOverWindow = false; - return funcCheckExecution(noExecutionListForCalcOverWindow, collectCalcOverWindow, node); + bool collectCalcOverWindow = false; + return funcCheckExecution(noExecutionListForCalcOverWindow, collectCalcOverWindow, node); }); } @@ -866,7 +866,7 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio }; IGraphTransformer::TStatus RequireChild(const TExprNode& node, ui32 index) { - switch (node.Child(index)->GetState()) { + switch (node.Child(index)->GetState()) { case TExprNode::EState::Error: case TExprNode::EState::ExecutionComplete: return IGraphTransformer::TStatus::Ok; @@ -877,7 +877,7 @@ IGraphTransformer::TStatus RequireChild(const TExprNode& node, ui32 index) { break; } - node.Child(index)->SetState(TExprNode::EState::ExecutionRequired); + node.Child(index)->SetState(TExprNode::EState::ExecutionRequired); return IGraphTransformer::TStatus::Repeat; } diff --git a/ydb/library/yql/core/yql_expr_constraint.cpp b/ydb/library/yql/core/yql_expr_constraint.cpp index 0c80d5baa5..78c8f68776 100644 --- a/ydb/library/yql/core/yql_expr_constraint.cpp +++ b/ydb/library/yql/core/yql_expr_constraint.cpp @@ -81,11 +81,11 @@ public: , SubGraph(subGraph) { Functions["Unordered"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["UnorderedSubquery"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions["UnorderedSubquery"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; Functions["Sort"] = &TCallableConstraintTransformer::SortWrap; Functions["AssumeSorted"] = &TCallableConstraintTransformer::AssumeSortedWrap; Functions["AssumeUnique"] = &TCallableConstraintTransformer::AssumeUniqueWrap; - Functions["AssumeColumnOrder"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions["AssumeColumnOrder"] = &TCallableConstraintTransformer::CopyAllFrom<0>; Functions["AssumeAllMembersNullableAtOnce"] = &TCallableConstraintTransformer::CopyAllFrom<0>; Functions["Top"] = &TCallableConstraintTransformer::TopWrap<false>; Functions["TopSort"] = &TCallableConstraintTransformer::TopWrap<true>; @@ -1046,7 +1046,7 @@ private: } if (const auto status = UpdateLambdaConstraints(input->TailRef(), ctx, {argConstraints}); status != TStatus::Ok) { - return status; + return status; } TSet<TStringBuf> except; @@ -1512,7 +1512,7 @@ private: std::transform(optionals.cbegin(), optionals.cend(), std::back_inserter(constraints), [](const TExprNode::TPtr& node){ return node->GetAllConstraints(); }); if (const auto status = UpdateLambdaConstraints(input->ChildRef(lambdaIndex), ctx, constraints); status != TStatus::Ok) { - return status; + return status; } if (std::any_of(optionals.cbegin(), optionals.cend(), [] (const TExprNode::TPtr& node) { return bool(node->GetConstraint<TEmptyConstraintNode>()); })) { @@ -2006,7 +2006,7 @@ private: if (inputPassthrough) argConstraints.emplace_back(inputPassthrough); if (const auto status = UpdateLambdaConstraints(input->ChildRef(1), ctx, {argConstraints}); status != TStatus::Ok) { - return status; + return status; } } @@ -2097,7 +2097,7 @@ private: } if (const auto status = UpdateLambdaConstraints(input->ChildRef(1), ctx, argConstraints); status != TStatus::Ok) { - return status; + return status; } const auto initLambda = input->Child(1); @@ -2116,7 +2116,7 @@ private: if (const auto status = UpdateLambdaConstraints(input->ChildRef(2), ctx, argConstraints) .Combine(UpdateLambdaConstraints(input->TailRef(), ctx, argConstraints)); status != TStatus::Ok) { - return status; + return status; } const TPassthroughConstraintNode* commonPassthrough = nullptr; @@ -2238,7 +2238,7 @@ private: } if (const auto status = UpdateLambdaConstraints(input->ChildRef(TCoPartitionByKeyBase::idx_ListHandlerLambda), ctx, {argConstraints}); status != TStatus::Ok) { - return status; + return status; } const auto handlerLambda = input->Child(TCoPartitionByKeyBase::idx_ListHandlerLambda); @@ -2455,7 +2455,7 @@ private: if (inputPassthrough) argConstraints.emplace_back(inputPassthrough); if (const auto status = UpdateLambdaConstraints(input->ChildRef(1), ctx, {argConstraints}); status != TStatus::Ok) { - return status; + return status; } } @@ -2777,7 +2777,7 @@ public: output = input; TStatus combinedStatus = TStatus::Ok; for (const auto& callable : CallableInputs) { - callable->SetState(TExprNode::EState::ConstrPending); + callable->SetState(TExprNode::EState::ConstrPending); TExprNode::TPtr callableOutput; auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx); Y_VERIFY(callableOutput); @@ -2785,7 +2785,7 @@ public: YQL_ENSURE(callableOutput == callable); combinedStatus = combinedStatus.Combine(status); if (status.Level == TStatus::Error) { - callable->SetState(TExprNode::EState::Error); + callable->SetState(TExprNode::EState::Error); } } @@ -2807,7 +2807,7 @@ private: return TStatus::Repeat; } - switch (start->GetState()) { + switch (start->GetState()) { case TExprNode::EState::Initial: case TExprNode::EState::TypeInProgress: case TExprNode::EState::TypePending: @@ -2818,7 +2818,7 @@ private: return IGraphTransformer::TStatus::Async; case TExprNode::EState::ConstrPending: if (start->Type() == TExprNode::Lambda) { - if (start->Child(0)->GetState() != TExprNode::EState::ConstrComplete) { + if (start->Child(0)->GetState() != TExprNode::EState::ConstrComplete) { return TStatus::Ok; } else if (start->Child(0)->ChildrenSize() == 0) { break; @@ -2845,7 +2845,7 @@ private: auto input = start; for (;;) { - TIssueScopeGuard issueScope(ctx.IssueManager, [this, input, &ctx]() -> TIssuePtr { + TIssueScopeGuard issueScope(ctx.IssueManager, [this, input, &ctx]() -> TIssuePtr { TStringBuilder str; str << "At "; switch (input->Type()) { @@ -2884,7 +2884,7 @@ private: str << "unknown"; } - return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), str); + return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), str); }); if (input->Type() == TExprNode::Callable) { @@ -2900,7 +2900,7 @@ private: }; TStatus retStatus = TStatus::Error; - switch (input->GetState()) { + switch (input->GetState()) { case TExprNode::EState::Initial: case TExprNode::EState::TypeInProgress: case TExprNode::EState::TypePending: @@ -2922,11 +2922,11 @@ private: YQL_ENSURE(false, "Unknown state"); } - input->SetState(TExprNode::EState::ConstrPending); + input->SetState(TExprNode::EState::ConstrPending); switch (input->Type()) { case TExprNode::Atom: case TExprNode::World: - input->SetState(TExprNode::EState::ConstrComplete); + input->SetState(TExprNode::EState::ConstrComplete); CheckExpected(*input); return TStatus::Ok; @@ -2955,7 +2955,7 @@ private: auto argStatus = TransformNode(input->HeadPtr(), out, ctx); UpdateStatusIfChanged(argStatus, input->HeadPtr(), out); if (argStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); return argStatus; } @@ -2979,7 +2979,7 @@ private: retStatus = argStatus.Combine(bodyStatus); if (retStatus != TStatus::Ok) { if (retStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (updatedChildren) { output = ctx.DeepCopyLambda(*input, std::move(newBody)); @@ -2997,7 +2997,7 @@ private: } case TExprNode::Argument: - if (input->GetState() != TExprNode::EState::ConstrComplete) { + if (input->GetState() != TExprNode::EState::ConstrComplete) { return TStatus::Repeat; } @@ -3006,7 +3006,7 @@ private: case TExprNode::Arguments: { if (input->Children().empty()) { - if (TExprNode::EState::ConstrComplete == input->GetState()) { + if (TExprNode::EState::ConstrComplete == input->GetState()) { return TStatus::Ok; } return TStatus::Repeat; @@ -3023,10 +3023,10 @@ private: if (retStatus != TStatus::Ok) { if (retStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } } else { - input->SetState(TExprNode::EState::ConstrComplete); + input->SetState(TExprNode::EState::ConstrComplete); } return retStatus; } @@ -3044,18 +3044,18 @@ private: CurrentFunctions.top().second = true; retStatus = CallableTransformer->Transform(input, output, ctx); if (retStatus == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (retStatus == TStatus::Ok) { // Sanity check for (size_t i = 0; i < input->ChildrenSize(); ++i) { - YQL_ENSURE(input->Child(i)->GetState() >= TExprNode::EState::ConstrComplete, + YQL_ENSURE(input->Child(i)->GetState() >= TExprNode::EState::ConstrComplete, "Child with index " << i << " of callable " << TString{input->Content()}.Quote() << " has bad state after constraint transform"); } - input->SetState(TExprNode::EState::ConstrComplete); + input->SetState(TExprNode::EState::ConstrComplete); CheckExpected(*input); } else if (retStatus == TStatus::Async) { CallableInputs.push_back(input); - input->SetState(TExprNode::EState::ConstrInProgress); + input->SetState(TExprNode::EState::ConstrInProgress); } else { if (output != input.Get()) { processedPair.first->second = output; @@ -3092,7 +3092,7 @@ private: if (combinedStatus != TStatus::Ok) { if (combinedStatus.Level == TStatus::Error) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); } else if (updatedChildren) { output = ctx.ChangeChildren(*input, std::move(newChildren)); @@ -3111,7 +3111,7 @@ private: void CheckExpected(const TExprNode& input) { if (const auto it = Types.ExpectedConstraints.find(input.UniqueId()); it != Types.ExpectedConstraints.cend()) { for (const TConstraintNode* expectedConstr: it->second) { - if (!Types.DisableConstraintCheck.contains(expectedConstr->GetName())) { + if (!Types.DisableConstraintCheck.contains(expectedConstr->GetName())) { const auto newConstr = input.GetConstraint(expectedConstr->GetName()); // Constraint Multi(0:{Empty},1:{Empty}, ..., N:{Empty}) can be reduced to Empty if (expectedConstr->GetName() == TMultiConstraintNode::Name()) { @@ -3154,17 +3154,17 @@ TAutoPtr<IGraphTransformer> CreateDefCallableConstraintTransformer() { IGraphTransformer::TStatus UpdateLambdaConstraints(const TExprNode& lambda) { auto args = lambda.Child(0); for (const auto& arg: args->Children()) { - if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) { - arg->SetState(TExprNode::EState::ConstrComplete); + if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) { + arg->SetState(TExprNode::EState::ConstrComplete); } YQL_ENSURE(arg->GetAllConstraints().empty()); } - if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) { - args->SetState(TExprNode::EState::ConstrComplete); + if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) { + args->SetState(TExprNode::EState::ConstrComplete); } - if (lambda.GetState() != TExprNode::EState::ConstrComplete) { + if (lambda.GetState() != TExprNode::EState::ConstrComplete) { return IGraphTransformer::TStatus::Repeat; } @@ -3179,11 +3179,11 @@ IGraphTransformer::TStatus UpdateLambdaConstraints(TExprNode::TPtr& lambda, TExp size_t i = 0; for (const auto constrList: constraints) { const auto arg = args->Child(i++); - if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) { + if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) { for (const auto c: constrList) { arg->AddConstraint(c); } - arg->SetState(TExprNode::EState::ConstrComplete); + arg->SetState(TExprNode::EState::ConstrComplete); } else { if (constrList.size() != arg->GetAllConstraints().size() || !AllOf(constrList, [arg] (const TConstraintNode* c) { return arg->GetConstraint(c->GetName()) == c; })) { updateArgs = true; @@ -3210,7 +3210,7 @@ IGraphTransformer::TStatus UpdateLambdaConstraints(TExprNode::TPtr& lambda, TExp auto newArgs = ctx.NewArguments(args->Pos(), std::move(argsChildren)); newArgs->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - newArgs->SetState(TExprNode::EState::ConstrComplete); + newArgs->SetState(TExprNode::EState::ConstrComplete); const auto type = lambda->GetTypeAnn(); lambda = ctx.NewLambda(lambda->Pos(), std::move(newArgs), ctx.ReplaceNodes<true>(GetLambdaBody(*lambda), replaces)); lambda->SetTypeAnn(type); @@ -3218,11 +3218,11 @@ IGraphTransformer::TStatus UpdateLambdaConstraints(TExprNode::TPtr& lambda, TExp return IGraphTransformer::TStatus::Repeat; } - if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) { - args->SetState(TExprNode::EState::ConstrComplete); + if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) { + args->SetState(TExprNode::EState::ConstrComplete); } - if (lambda->GetState() != TExprNode::EState::ConstrComplete) { + if (lambda->GetState() != TExprNode::EState::ConstrComplete) { return IGraphTransformer::TStatus::Repeat; } diff --git a/ydb/library/yql/core/yql_expr_csee.cpp b/ydb/library/yql/core/yql_expr_csee.cpp index 8a919293dd..b0e7525521 100644 --- a/ydb/library/yql/core/yql_expr_csee.cpp +++ b/ydb/library/yql/core/yql_expr_csee.cpp @@ -6,7 +6,7 @@ #include <util/generic/hash_set.h> #include <util/digest/murmur.h> -#include <util/system/env.h> +#include <util/system/env.h> #include <tuple> namespace NYql { @@ -14,43 +14,43 @@ namespace NYql { namespace { static constexpr bool UseDeterminsticHash = false; - ui64 CseeHash(const void* data, size_t size, ui64 initHash) { - return MurmurHash<ui64>(data, size, initHash); - } - - template<typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> - ui64 CseeHash(T value, ui64 initHash) { - // workaround Coverity warning for Murmur when sizeof(T) < 8 - ui64 val = static_cast<ui64>(value); - return MurmurHash<ui64>(&val, sizeof(val), initHash); - } - - struct TLambdaFrame { + ui64 CseeHash(const void* data, size_t size, ui64 initHash) { + return MurmurHash<ui64>(data, size, initHash); + } + + template<typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr> + ui64 CseeHash(T value, ui64 initHash) { + // workaround Coverity warning for Murmur when sizeof(T) < 8 + ui64 val = static_cast<ui64>(value); + return MurmurHash<ui64>(&val, sizeof(val), initHash); + } + + struct TLambdaFrame { TLambdaFrame(const TExprNode* lambda, const TLambdaFrame* prev) - : Lambda(lambda) - , Prev(prev) - {} - - TLambdaFrame() = default; - - const TExprNode* Lambda = nullptr; + : Lambda(lambda) + , Prev(prev) + {} + + TLambdaFrame() = default; + + const TExprNode* Lambda = nullptr; const TLambdaFrame* Prev = nullptr; - }; - - bool IsArgInScope(const TLambdaFrame& frame, const TExprNode& arg) { + }; + + bool IsArgInScope(const TLambdaFrame& frame, const TExprNode& arg) { for (auto curr = &frame; curr; curr = curr->Prev) { if (const auto lambda = curr->Lambda) { - YQL_ENSURE(lambda->IsLambda()); + YQL_ENSURE(lambda->IsLambda()); for (ui32 i = 0U; i < lambda->Head().ChildrenSize(); ++i) { if (lambda->Head().Child(i) == &arg) { - return true; - } - } - } - } - return false; - } - + return true; + } + } + } + } + return false; + } + ui16 GetDependencyLevel(const TExprNode& node) { if (const auto lambda = node.GetDependencyScope()->first) { return 1 + GetDependencyLevel(*lambda); @@ -90,7 +90,7 @@ namespace { return EDependencyScope::None; } - ui64 CalculateHash(ui16 depth, TExprNode& node, const TLambdaFrame& currFrame, const TColumnOrderStorage& coStore) { + ui64 CalculateHash(ui16 depth, TExprNode& node, const TLambdaFrame& currFrame, const TColumnOrderStorage& coStore) { const auto dependency = CheckDependencyScope(currFrame, node); switch (dependency) { case EDependencyScope::None: @@ -113,91 +113,91 @@ namespace { } ui64 hash = node.GetTypeAnn()->GetHash(); - hash = CseeHash(ui32(node.Type()), hash); + hash = CseeHash(ui32(node.Type()), hash); for (auto c: node.GetAllConstraints()) { - hash = CseeHash(c->GetHash(), hash); + hash = CseeHash(c->GetHash(), hash); } - hash = AddColumnOrderHash(coStore.Lookup(node.UniqueId()), hash); + hash = AddColumnOrderHash(coStore.Lookup(node.UniqueId()), hash); switch (node.Type()) { case TExprNode::Atom: { if constexpr (UseDeterminsticHash) { - hash = CseeHash(node.Content().data(), node.Content().size(), hash); + hash = CseeHash(node.Content().data(), node.Content().size(), hash); } else { // can hash ptr due to intern const char* ptr = node.Content().data(); - hash = CseeHash(&ptr, sizeof(ptr), hash); + hash = CseeHash(&ptr, sizeof(ptr), hash); } - hash = CseeHash(node.GetFlagsToCompare(), hash); + hash = CseeHash(node.GetFlagsToCompare(), hash); break; } case TExprNode::Callable: if constexpr (UseDeterminsticHash) { - hash = CseeHash(node.Content().data(), node.Content().size(), hash); + hash = CseeHash(node.Content().data(), node.Content().size(), hash); } else { // can hash ptr due to intern const char* ptr = node.Content().data(); - hash = CseeHash(&ptr, sizeof(ptr), hash); + hash = CseeHash(&ptr, sizeof(ptr), hash); } - [[fallthrough]]; + [[fallthrough]]; case TExprNode::List: { const auto size = node.ChildrenSize(); - hash = CseeHash(size, hash); + hash = CseeHash(size, hash); if (node.UnorderedChildren()) { TSmallVec<ui64> hashes; hashes.reserve(size); for (ui32 i = 0U; i < node.ChildrenSize(); ++i) { - hashes.emplace_back(CalculateHash(depth, *node.Child(i), currFrame, coStore)); + hashes.emplace_back(CalculateHash(depth, *node.Child(i), currFrame, coStore)); }; std::sort(hashes.begin(), hashes.end()); hash = std::accumulate(hashes.cbegin(), hashes.cend(), ~hash, [] (ui64 hash, ui64 childHash) { - return CseeHash(childHash, hash); + return CseeHash(childHash, hash); }); } else { for (ui32 i = 0U; i < node.ChildrenSize(); ++i) { - const auto childHash = CalculateHash(depth, *node.Child(i), currFrame, coStore); - hash = CseeHash(childHash, hash); + const auto childHash = CalculateHash(depth, *node.Child(i), currFrame, coStore); + hash = CseeHash(childHash, hash); } } break; } case TExprNode::Lambda: { if (const ui32 size = node.ChildrenSize()) - hash = CseeHash(size, hash); + hash = CseeHash(size, hash); const auto& args = node.Head(); - hash = CseeHash(args.ChildrenSize(), hash); + hash = CseeHash(args.ChildrenSize(), hash); for (ui32 i = 0; i < args.ChildrenSize(); ++i) { const auto& arg = *args.Child(i); - hash = CseeHash(arg.GetTypeAnn()->GetHash(), hash); + hash = CseeHash(arg.GetTypeAnn()->GetHash(), hash); for (auto c: arg.GetAllConstraints()) { - hash = CseeHash(c->GetHash(), hash); + hash = CseeHash(c->GetHash(), hash); } } - TLambdaFrame newFrame(&node, &currFrame); + TLambdaFrame newFrame(&node, &currFrame); for (ui32 i = 1U; i < node.ChildrenSize(); ++i) { - const auto lambdaHash = CalculateHash(depth + 1, *node.Child(i), newFrame, coStore); - hash = CseeHash(lambdaHash, hash); + const auto lambdaHash = CalculateHash(depth + 1, *node.Child(i), newFrame, coStore); + hash = CseeHash(lambdaHash, hash); } break; } case TExprNode::Argument: switch (dependency) { case EDependencyScope::Inner: { - hash = CseeHash(GetDependencyLevel(node), hash); - hash = CseeHash(node.GetArgIndex(), hash); + hash = CseeHash(GetDependencyLevel(node), hash); + hash = CseeHash(node.GetArgIndex(), hash); break; } case EDependencyScope::Outer: { if constexpr (UseDeterminsticHash) { - hash = CseeHash(node.UniqueId(), hash); + hash = CseeHash(node.UniqueId(), hash); } else { const auto ptr = &node; - hash = CseeHash(&ptr, sizeof(ptr), hash); + hash = CseeHash(&ptr, sizeof(ptr), hash); } break; } @@ -232,9 +232,9 @@ namespace { return hash; } - bool EqualNodes(const TExprNode& left, TLambdaFrame& currLeftFrame, const TExprNode& right, TLambdaFrame& currRightFrame, - TNodeSet& visited, const TColumnOrderStorage& coStore) - { + bool EqualNodes(const TExprNode& left, TLambdaFrame& currLeftFrame, const TExprNode& right, TLambdaFrame& currRightFrame, + TNodeSet& visited, const TColumnOrderStorage& coStore) + { if (&left == &right) { return true; } @@ -255,10 +255,10 @@ namespace { return false; } - if (coStore.Lookup(left.UniqueId()) != coStore.Lookup(right.UniqueId())) { - return false; - } - + if (coStore.Lookup(left.UniqueId()) != coStore.Lookup(right.UniqueId())) { + return false; + } + switch (left.Type()) { case TExprNode::Atom: // compare pointers due to intern @@ -277,10 +277,10 @@ namespace { if (left.UnorderedChildren() && right.UnorderedChildren()) { if (2U == left.ChildrenSize()) { - return EqualNodes(left.Head(), currLeftFrame, right.Head(), currRightFrame, visited, coStore) - && EqualNodes(left.Tail(), currLeftFrame, right.Tail(), currRightFrame, visited, coStore) - || EqualNodes(left.Head(), currLeftFrame, right.Tail(), currRightFrame, visited, coStore) - && EqualNodes(left.Tail(), currLeftFrame, right.Head(), currRightFrame, visited, coStore); + return EqualNodes(left.Head(), currLeftFrame, right.Head(), currRightFrame, visited, coStore) + && EqualNodes(left.Tail(), currLeftFrame, right.Tail(), currRightFrame, visited, coStore) + || EqualNodes(left.Head(), currLeftFrame, right.Tail(), currRightFrame, visited, coStore) + && EqualNodes(left.Tail(), currLeftFrame, right.Head(), currRightFrame, visited, coStore); } else { TSmallVec<const TExprNode*> lNodes, rNodes; lNodes.reserve(left.ChildrenSize()); @@ -294,14 +294,14 @@ namespace { std::sort(rNodes.begin(), rNodes.end(), order); for (ui32 i = 0; i < lNodes.size(); ++i) { - if (!EqualNodes(*lNodes[i], currLeftFrame, *rNodes[i], currRightFrame, visited, coStore)) { + if (!EqualNodes(*lNodes[i], currLeftFrame, *rNodes[i], currRightFrame, visited, coStore)) { return false; } } } } else { for (ui32 i = 0; i < left.ChildrenSize(); ++i) { - if (!EqualNodes(*left.Child(i), currLeftFrame, *right.Child(i), currRightFrame, visited, coStore)) { + if (!EqualNodes(*left.Child(i), currLeftFrame, *right.Child(i), currRightFrame, visited, coStore)) { return false; } } @@ -334,11 +334,11 @@ namespace { } } - TLambdaFrame newLeftFrame(&left, &currLeftFrame); - TLambdaFrame newRightFrame(&right, &currRightFrame); - + TLambdaFrame newLeftFrame(&left, &currLeftFrame); + TLambdaFrame newRightFrame(&right, &currRightFrame); + for (ui32 i = 1U; i < left.ChildrenSize(); ++i) { - if (!EqualNodes(*left.Child(i), newLeftFrame, *right.Child(i), newRightFrame, visited, coStore)) + if (!EqualNodes(*left.Child(i), newLeftFrame, *right.Child(i), newRightFrame, visited, coStore)) return false; } return true; @@ -347,7 +347,7 @@ namespace { if (currLeftFrame.Lambda && currRightFrame.Lambda && IsArgInScope(currLeftFrame, left) && IsArgInScope(currRightFrame, right)) { const ui16 leftRelativeLevel = GetDependencyLevel(left); const ui16 rightRelativeLevel = GetDependencyLevel(right); - return leftRelativeLevel == rightRelativeLevel && left.GetArgIndex() == right.GetArgIndex(); + return leftRelativeLevel == rightRelativeLevel && left.GetArgIndex() == right.GetArgIndex(); } else { return &left == &right; } @@ -401,7 +401,7 @@ namespace { } } - [[fallthrough]]; + [[fallthrough]]; case TExprNode::List: if (left.ChildrenSize() != right.ChildrenSize()) { return (int)left.ChildrenSize() - (int)right.ChildrenSize(); @@ -432,8 +432,8 @@ namespace { return 0; } case TExprNode::Argument: - if (left.GetArgIndex() != right.GetArgIndex()) { - return (int)left.GetArgIndex() - (int)right.GetArgIndex(); + if (left.GetArgIndex() != right.GetArgIndex()) { + return (int)left.GetArgIndex() - (int)right.GetArgIndex(); } return (int)left.GetDependencyScope()->first->GetLambdaLevel() - (int)right.GetDependencyScope()->first->GetLambdaLevel(); @@ -460,8 +460,8 @@ namespace { } return; default: break; - } - + } + const auto ins = (insideDependsOn ? visitedInsideDependsOn : visited).emplace(&node, TNodeSet{}); if (!ins.second) { closures.insert(ins.first->second.cbegin(), ins.first->second.cend()); @@ -478,7 +478,7 @@ namespace { } internal.erase(&node); } else { - insideDependsOn = insideDependsOn || node.IsCallable("DependsOn"); + insideDependsOn = insideDependsOn || node.IsCallable("DependsOn"); node.ForEachChild(std::bind(&CalculateCompletness, std::placeholders::_1, insideDependsOn, level, std::ref(internal), std::ref(visited), std::ref(visitedInsideDependsOn))); } @@ -498,21 +498,21 @@ namespace { closures.insert(internal.cbegin(), internal.cend()); } - ui64 CalcHash(TExprNode& node, const TColumnOrderStorage& coStore) { + ui64 CalcHash(TExprNode& node, const TColumnOrderStorage& coStore) { TLambdaFrame frame; - return CalculateHash(0, node, frame, coStore); + return CalculateHash(0, node, frame, coStore); } - bool EqualNodes(const TExprNode& left, const TExprNode& right, const TColumnOrderStorage& coStore) { + bool EqualNodes(const TExprNode& left, const TExprNode& right, const TColumnOrderStorage& coStore) { TNodeSet visited; TLambdaFrame frame; - return EqualNodes(left, frame, right, frame, visited, coStore); + return EqualNodes(left, frame, right, frame, visited, coStore); } TExprNode::TPtr VisitNode(TExprNode& node, TExprNode* currentLambda, ui16 level, std::unordered_multimap<ui64, TExprNode*>& uniqueNodes, std::unordered_multimap<ui64, TExprNode*>& incompleteNodes, - TNodeMap<TExprNode*>& renames, const TColumnOrderStorage& coStore) { + TNodeMap<TExprNode*>& renames, const TColumnOrderStorage& coStore) { if (node.Type() == TExprNode::Argument) { return nullptr; @@ -523,17 +523,17 @@ namespace { return find.first->second; } - const auto hash = CalcHash(node, coStore); + const auto hash = CalcHash(node, coStore); if (node.Type() == TExprNode::Lambda) { for (ui32 i = 1U; i < node.ChildrenSize(); ++i) { - if (auto newNode = VisitNode(*node.Child(i), &node, level + 1U, uniqueNodes, incompleteNodes, renames, coStore)) { + if (auto newNode = VisitNode(*node.Child(i), &node, level + 1U, uniqueNodes, incompleteNodes, renames, coStore)) { node.ChildRef(i) = std::move(newNode); } } } else { for (ui32 i = 0; i < node.ChildrenSize(); ++i) { - if (auto newNode = VisitNode(*node.Child(i), currentLambda, level, uniqueNodes, incompleteNodes, renames, coStore)) { + if (auto newNode = VisitNode(*node.Child(i), currentLambda, level, uniqueNodes, incompleteNodes, renames, coStore)) { node.ChildRef(i) = std::move(newNode); } } @@ -551,8 +551,8 @@ namespace { continue; } - if (!EqualNodes(node, *iter->second, coStore)) { -#ifndef NDEBUG + if (!EqualNodes(node, *iter->second, coStore)) { +#ifndef NDEBUG if (!GetEnv("YQL_ALLOW_CSEE_HASH_COLLISION")) { YQL_ENSURE(false, "Node -BEGIN-\n" << node.Dump() << "-END-" << " has same hash as -BEGIN-\n" << iter->second->Dump() << "-END-"); @@ -591,15 +591,15 @@ IGraphTransformer::TStatus UpdateCompletness(const TExprNode::TPtr& input, TExpr } IGraphTransformer::TStatus EliminateCommonSubExpressions(const TExprNode::TPtr& input, TExprNode::TPtr& output, - TExprContext& ctx, bool forSubGraph, const TColumnOrderStorage& coStore) + TExprContext& ctx, bool forSubGraph, const TColumnOrderStorage& coStore) { - YQL_PROFILE_SCOPE(DEBUG, forSubGraph ? "EliminateCommonSubExpressionsForSubGraph" : "EliminateCommonSubExpressions"); + YQL_PROFILE_SCOPE(DEBUG, forSubGraph ? "EliminateCommonSubExpressionsForSubGraph" : "EliminateCommonSubExpressions"); output = input; TNodeMap<TExprNode*> renames; //Cerr << "INPUT\n" << output->Dump() << "\n"; std::unordered_multimap<ui64, TExprNode*> incompleteNodes; - const auto newNode = VisitNode(*output, nullptr, 0, ctx.UniqueNodes, incompleteNodes, renames, coStore); - YQL_ENSURE(forSubGraph || !newNode); + const auto newNode = VisitNode(*output, nullptr, 0, ctx.UniqueNodes, incompleteNodes, renames, coStore); + YQL_ENSURE(forSubGraph || !newNode); //Cerr << "OUTPUT\n" << output->Dump() << "\n"; return IGraphTransformer::TStatus::Ok; } diff --git a/ydb/library/yql/core/yql_expr_csee.h b/ydb/library/yql/core/yql_expr_csee.h index 81395a8f9d..177041d998 100644 --- a/ydb/library/yql/core/yql_expr_csee.h +++ b/ydb/library/yql/core/yql_expr_csee.h @@ -1,12 +1,12 @@ #pragma once #include "yql_graph_transformer.h" -#include "yql_type_annotation.h" +#include "yql_type_annotation.h" namespace NYql { -IGraphTransformer::TStatus EliminateCommonSubExpressions(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, - bool forSubGraph, const TColumnOrderStorage& coStore); +IGraphTransformer::TStatus EliminateCommonSubExpressions(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, + bool forSubGraph, const TColumnOrderStorage& coStore); IGraphTransformer::TStatus UpdateCompletness(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx); // Calculate order between two given nodes. Must be used only after CSEE pass or UpdateCompletness. diff --git a/ydb/library/yql/core/yql_expr_optimize.cpp b/ydb/library/yql/core/yql_expr_optimize.cpp index 9160c74620..677ca7fa53 100644 --- a/ydb/library/yql/core/yql_expr_optimize.cpp +++ b/ydb/library/yql/core/yql_expr_optimize.cpp @@ -83,10 +83,10 @@ namespace { settings.Types->ExpectedTypes.emplace(dst.UniqueId(), src.GetTypeAnn()); settings.Types->ExpectedConstraints.emplace(dst.UniqueId(), src.GetAllConstraints()); - auto columnOrder = settings.Types->LookupColumnOrder(src); - if (columnOrder) { - settings.Types->ExpectedColumnOrders.emplace(dst.UniqueId(), *columnOrder); - } + auto columnOrder = settings.Types->LookupColumnOrder(src); + if (columnOrder) { + settings.Types->ExpectedColumnOrders.emplace(dst.UniqueId(), *columnOrder); + } } template<typename TContext> @@ -263,11 +263,11 @@ namespace { TExprNode::TListType newChildren; newChildren.reserve(node->ChildrenSize()); bool hasRenames = false; - - for (auto& child : node->Children()) { + + for (auto& child : node->Children()) { bool childChanged = false; - auto newChild = OptimizeNode(child, childChanged, ctx, level + 1); - + auto newChild = OptimizeNode(child, childChanged, ctx, level + 1); + if (!newChild) return nullptr; @@ -586,7 +586,7 @@ IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode:: } if (!node->Child(2)->IsCallable("AsStruct") || node->Child(2)->ChildrenSize() != 0) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(2)->Pos()), "Expected empty AsStruct in NamedApply")); + ctx.AddError(TIssue(ctx.GetPosition(node->Child(2)->Pos()), "Expected empty AsStruct in NamedApply")); return nullptr; } @@ -596,7 +596,7 @@ IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode:: } if (!node->Child(i)->IsCallable("DependsOn")) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(i)->Pos()), "Expected DependsOn")); + ctx.AddError(TIssue(ctx.GetPosition(node->Child(i)->Pos()), "Expected DependsOn")); return nullptr; } } @@ -660,7 +660,7 @@ IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode:: const auto f = child->Flags(); if ((TNodeFlags::MultilineContent | TNodeFlags::BinaryContent) & f) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), "Can't combine binary or multiline atoms.")); + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), "Can't combine binary or multiline atoms.")); return nullptr; } @@ -685,7 +685,7 @@ IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode:: ui32 index = 0U; if (const auto& indexValue = node->Tail().Content(); !TryFromString(indexValue, index)) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(1)->Pos()), TStringBuilder() << "Index '" << indexValue << "' isn't UI32.")); + ctx.AddError(TIssue(ctx.GetPosition(node->Child(1)->Pos()), TStringBuilder() << "Index '" << indexValue << "' isn't UI32.")); return nullptr; } @@ -709,7 +709,7 @@ IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode:: return nullptr; } - if (node->Tail().IsCallable("Apply")) { + if (node->Tail().IsCallable("Apply")) { return node; } diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index 1832076c00..f2b793af8d 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -1,7 +1,7 @@ #include "yql_expr_type_annotation.h" #include "yql_opt_proposed_by_data.h" #include "yql_opt_rewrite_io.h" -#include "yql_opt_utils.h" +#include "yql_opt_utils.h" #include <ydb/library/yql/public/udf/udf_data_type.h> #include <ydb/library/yql/minikql/dom/json.h> @@ -459,7 +459,7 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& columnTransforms[newField->GetName()] = field; } - if (flags.Test(NConvertFlags::DisableTruncation) && usedFields != from->GetSize()) { + if (flags.Test(NConvertFlags::DisableTruncation) && usedFields != from->GetSize()) { return IGraphTransformer::TStatus::Error; } @@ -518,7 +518,7 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& } auto fromElement = item->GetItemType(); auto toElement = toUnderlying->GetItems()[*toIndex]->GetItemType(); - auto arg = ctx.NewArgument(TPositionHandle(), "arg"); + auto arg = ctx.NewArgument(TPositionHandle(), "arg"); auto status1 = TryConvertToImpl(ctx, arg, *fromElement, *toElement, flags); if (status1.Level == IGraphTransformer::TStatus::Error) { return status1; @@ -558,7 +558,7 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& if (i == targetIndex) { continue; // Already checked when converting targetItem } - auto arg = ctx.NewArgument(TPositionHandle(), "arg"); + auto arg = ctx.NewArgument(TPositionHandle(), "arg"); auto status1 = TryConvertToImpl(ctx, arg, *fromUnderlying->GetItems()[i], *toUnderlying->GetItems()[i], flags); if (status1.Level == IGraphTransformer::TStatus::Error) { @@ -812,48 +812,48 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr& auto to = expectedType.Cast<TDictExprType>(); auto oldKeyType = from->GetKeyType(); - auto oldPayloadType = from->GetPayloadType(); + auto oldPayloadType = from->GetPayloadType(); auto newKeyType = to->GetKeyType(); - auto newPayloadType = to->GetPayloadType(); - - TExprNode::TListType valueTransforms; - if (node->IsCallable("Dict")) { - valueTransforms.push_back(ExpandType(node->Pos(), *to, ctx)); - } - - for (ui32 i = node->IsCallable("Dict") ? 1 : 0; i < node->ChildrenSize(); ++i) { - auto valueKey = node->Child(i)->ChildPtr(0); - auto valuePayload = node->Child(i)->ChildPtr(1); - auto status = TryConvertToImpl(ctx, valueKey, *oldKeyType, *newKeyType, flags); - status = status.Combine(TryConvertToImpl(ctx, valuePayload, *oldPayloadType, *newPayloadType, flags)); - if (status.Level == IGraphTransformer::TStatus::Error) { - return status; + auto newPayloadType = to->GetPayloadType(); + + TExprNode::TListType valueTransforms; + if (node->IsCallable("Dict")) { + valueTransforms.push_back(ExpandType(node->Pos(), *to, ctx)); + } + + for (ui32 i = node->IsCallable("Dict") ? 1 : 0; i < node->ChildrenSize(); ++i) { + auto valueKey = node->Child(i)->ChildPtr(0); + auto valuePayload = node->Child(i)->ChildPtr(1); + auto status = TryConvertToImpl(ctx, valueKey, *oldKeyType, *newKeyType, flags); + status = status.Combine(TryConvertToImpl(ctx, valuePayload, *oldPayloadType, *newPayloadType, flags)); + if (status.Level == IGraphTransformer::TStatus::Error) { + return status; } - valueTransforms.push_back(ctx.NewList(node->Pos(), { valueKey, valuePayload })); + valueTransforms.push_back(ctx.NewList(node->Pos(), { valueKey, valuePayload })); } - - node = ctx.ChangeChildren(*node, std::move(valueTransforms)); - return IGraphTransformer::TStatus::Repeat; + + node = ctx.ChangeChildren(*node, std::move(valueTransforms)); + return IGraphTransformer::TStatus::Repeat; } else if (expectedType.GetKind() == ETypeAnnotationKind::Dict && sourceType.GetKind() == ETypeAnnotationKind::Dict) { auto from = sourceType.Cast<TDictExprType>(); auto to = expectedType.Cast<TDictExprType>(); auto oldKeyType = from->GetKeyType(); - auto oldPayloadType = from->GetPayloadType(); + auto oldPayloadType = from->GetPayloadType(); auto newKeyType = to->GetKeyType(); - auto newPayloadType = to->GetPayloadType(); - - auto arg = ctx.NewArgument(node->Pos(), "item"); - auto key = ctx.NewCallable(node->Pos(), "Nth", { arg, ctx.NewAtom(node->Pos(), "0") }); - auto value = ctx.NewCallable(node->Pos(), "Nth", { arg, ctx.NewAtom(node->Pos(), "1") }); - auto status = TryConvertToImpl(ctx, key, *oldKeyType, *newKeyType, flags); - status = status.Combine(TryConvertToImpl(ctx, value, *oldPayloadType, *newPayloadType, flags)); - if (status.Level != IGraphTransformer::TStatus::Error) { - auto body = ctx.NewList(node->Pos(), { key, value }); - auto lambda = ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { arg }), std::move(body)); - node = RebuildDict(node, lambda, ctx); - return IGraphTransformer::TStatus::Repeat; + auto newPayloadType = to->GetPayloadType(); + + auto arg = ctx.NewArgument(node->Pos(), "item"); + auto key = ctx.NewCallable(node->Pos(), "Nth", { arg, ctx.NewAtom(node->Pos(), "0") }); + auto value = ctx.NewCallable(node->Pos(), "Nth", { arg, ctx.NewAtom(node->Pos(), "1") }); + auto status = TryConvertToImpl(ctx, key, *oldKeyType, *newKeyType, flags); + status = status.Combine(TryConvertToImpl(ctx, value, *oldPayloadType, *newPayloadType, flags)); + if (status.Level != IGraphTransformer::TStatus::Error) { + auto body = ctx.NewList(node->Pos(), { key, value }); + auto lambda = ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { arg }), std::move(body)); + node = RebuildDict(node, lambda, ctx); + return IGraphTransformer::TStatus::Repeat; } } else if (expectedType.GetKind() == ETypeAnnotationKind::Tagged && sourceType.GetKind() == ETypeAnnotationKind::Tagged) { auto from = sourceType.Cast<TTaggedExprType>(); @@ -1429,24 +1429,24 @@ const TTaggedExprType* CommonType(TPositionHandle pos, const TTaggedExprType* on return nullptr; } -TExprNode::TPtr TryExpandSimpleType(TPositionHandle pos, const TStringBuf& type, TExprContext& ctx) { - TExprNode::TPtr result; - // backend is always ready for pragma FlexibleTypes; - const bool flexibleTypes = true; - if (auto found = LookupSimpleTypeBySqlAlias(type, flexibleTypes)) { - auto typeName = ToString(*found); - if (NKikimr::NUdf::FindDataSlot(typeName)) { - result = ctx.NewCallable(pos, "DataType", { ctx.NewAtom(pos, typeName, TNodeFlags::Default )}); - } else { - result = ctx.NewCallable(pos, typeName + ToString("Type"), {}); - } - } else { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Unknown type name: '" << type << "'")); - } - - return result; -} - +TExprNode::TPtr TryExpandSimpleType(TPositionHandle pos, const TStringBuf& type, TExprContext& ctx) { + TExprNode::TPtr result; + // backend is always ready for pragma FlexibleTypes; + const bool flexibleTypes = true; + if (auto found = LookupSimpleTypeBySqlAlias(type, flexibleTypes)) { + auto typeName = ToString(*found); + if (NKikimr::NUdf::FindDataSlot(typeName)) { + result = ctx.NewCallable(pos, "DataType", { ctx.NewAtom(pos, typeName, TNodeFlags::Default )}); + } else { + result = ctx.NewCallable(pos, typeName + ToString("Type"), {}); + } + } else { + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Unknown type name: '" << type << "'")); + } + + return result; +} + } // namespace template <bool Strong> @@ -1695,13 +1695,13 @@ const TTypeAnnotationNode* CommonTypeForChildren(const TExprNode& node, TExprCon } size_t GetOptionalLevel(const TTypeAnnotationNode* type) { - size_t level = 0; - YQL_ENSURE(type); - while (type->GetKind() == ETypeAnnotationKind::Optional) { - type = type->Cast<TOptionalExprType>()->GetItemType(); - ++level; - } - return level; + size_t level = 0; + YQL_ENSURE(type); + while (type->GetKind() == ETypeAnnotationKind::Optional) { + type = type->Cast<TOptionalExprType>()->GetItemType(); + ++level; + } + return level; } void ClearExprTypeAnnotations(TExprNode& root) { @@ -1731,16 +1731,16 @@ void EnsureAllNodesTypeAnnotated(const TExprNode& root) { } } -namespace { - -bool IsDataOrOptionalOfData(TPosition pos, const TTypeAnnotationNode* typeAnnotation, bool& isOptional, - const TDataExprType*& dataType, TIssue& err, bool& hasErrorType) -{ - err = {}; - hasErrorType = false; - +namespace { + +bool IsDataOrOptionalOfData(TPosition pos, const TTypeAnnotationNode* typeAnnotation, bool& isOptional, + const TDataExprType*& dataType, TIssue& err, bool& hasErrorType) +{ + err = {}; + hasErrorType = false; + if (!typeAnnotation) { - err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got lambda"); + err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got lambda"); return false; } @@ -1751,20 +1751,20 @@ bool IsDataOrOptionalOfData(TPosition pos, const TTypeAnnotationNode* typeAnnota } if (typeAnnotation->GetKind() != ETypeAnnotationKind::Optional) { - if (!HasError(typeAnnotation, err)) { - err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got: " << *typeAnnotation); - } else { - hasErrorType = true; + if (!HasError(typeAnnotation, err)) { + err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got: " << *typeAnnotation); + } else { + hasErrorType = true; } return false; } auto itemType = typeAnnotation->Cast<TOptionalExprType>()->GetItemType(); if (itemType->GetKind() != ETypeAnnotationKind::Data) { - if (!HasError(itemType, err)) { - err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got optional of: " << *itemType); - } else { - hasErrorType = true; + if (!HasError(itemType, err)) { + err = TIssue(pos, TStringBuilder() << "Expected data or optional of data, but got optional of: " << *itemType); + } else { + hasErrorType = true; } return false; } @@ -1774,15 +1774,15 @@ bool IsDataOrOptionalOfData(TPosition pos, const TTypeAnnotationNode* typeAnnota return true; } -} - -bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isOptional, const TDataExprType*& dataType) -{ - TIssue err; - bool hasErrorType; - return IsDataOrOptionalOfData({}, typeAnnotation, isOptional, dataType, err, hasErrorType); -} - +} + +bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isOptional, const TDataExprType*& dataType) +{ + TIssue err; + bool hasErrorType; + return IsDataOrOptionalOfData({}, typeAnnotation, isOptional, dataType, err, hasErrorType); +} + bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation) { bool isOptional; const TDataExprType* dataType; @@ -1791,7 +1791,7 @@ bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation) { bool EnsureArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx) { if (node.ChildrenSize() != expectedArgs) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected " << expectedArgs << " argument(s), but got " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected " << expectedArgs << " argument(s), but got " << node.ChildrenSize())); return false; } @@ -1801,7 +1801,7 @@ bool EnsureArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx bool EnsureMinArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx) { if (node.ChildrenSize() < expectedArgs) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected at least " << expectedArgs << " argument(s), but got " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected at least " << expectedArgs << " argument(s), but got " << node.ChildrenSize())); return false; } @@ -1811,7 +1811,7 @@ bool EnsureMinArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& bool EnsureMaxArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx) { if (node.ChildrenSize() > expectedArgs) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected at most " << expectedArgs << " argument(s), but got " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected at most " << expectedArgs << " argument(s), but got " << node.ChildrenSize())); return false; } @@ -1823,18 +1823,18 @@ bool EnsureMinMaxArgsCount(const TExprNode& node, ui32 minArgs, ui32 maxArgs, TE return EnsureMinArgsCount(node, minArgs, ctx) && EnsureMaxArgsCount(node, maxArgs, ctx); } -bool EnsureCallableMinArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { +bool EnsureCallableMinArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { if (args < expectedArgs) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at least " << expectedArgs << " argument(s), but got " << args)); + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at least " << expectedArgs << " argument(s), but got " << args)); return false; } return true; } -bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { +bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx) { if (args > expectedArgs) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at most " << expectedArgs << " argument(s), but got " << args)); + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Callable expected at most " << expectedArgs << " argument(s), but got " << args)); return false; } @@ -1843,7 +1843,7 @@ bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expe bool EnsureAtom(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Atom) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected atom, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected atom, but got: " << node.Type())); return false; } @@ -1852,7 +1852,7 @@ bool EnsureAtom(const TExprNode& node, TExprContext& ctx) { bool EnsureCallable(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Callable) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable, but got: " << node.Type())); return false; } @@ -1861,34 +1861,34 @@ bool EnsureCallable(const TExprNode& node, TExprContext& ctx) { bool EnsureTuple(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); + return false; + } + + return true; +} + +bool EnsureTupleOfAtoms(const TExprNode& node, TExprContext& ctx) { + if (!EnsureTuple(node, ctx)) { return false; } + for (const auto& atom : node.ChildrenList()) { + if (!EnsureAtom(*atom, ctx)) { + return false; + } + } return true; } -bool EnsureTupleOfAtoms(const TExprNode& node, TExprContext& ctx) { - if (!EnsureTuple(node, ctx)) { - return false; - } - - for (const auto& atom : node.ChildrenList()) { - if (!EnsureAtom(*atom, ctx)) { - return false; - } - } - return true; -} - bool EnsureTupleSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); return false; } if (node.ChildrenSize() != expectedSize) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple size: " << expectedSize << ", but got: " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple size: " << expectedSize << ", but got: " << node.ChildrenSize())); return false; } @@ -1898,12 +1898,12 @@ bool EnsureTupleSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx bool EnsureTupleMinSize(const TExprNode& node, ui32 minSize, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); return false; } if (node.ChildrenSize() < minSize) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple of at least size: " << minSize << ", but got: " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple of at least size: " << minSize << ", but got: " << node.ChildrenSize())); return false; } @@ -1913,12 +1913,12 @@ bool EnsureTupleMinSize(const TExprNode& node, ui32 minSize, TExprContext& ctx) bool EnsureTupleMaxSize(const TExprNode& node, ui32 maxSize, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type())); return false; } if (node.ChildrenSize() > maxSize) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple of at most size: " << maxSize << ", but got: " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple of at most size: " << maxSize << ", but got: " << node.ChildrenSize())); return false; } @@ -1929,21 +1929,21 @@ bool EnsureTupleMaxSize(const TExprNode& node, ui32 maxSize, TExprContext& ctx) bool EnsureTupleType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureTupleType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureTupleType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got: " << type)); return false; } @@ -1951,24 +1951,24 @@ bool EnsureTupleType(TPositionHandle position, const TTypeAnnotationNode& type, } bool EnsureTupleTypeSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx) { - YQL_ENSURE(node.GetTypeAnn() || node.Type() == TExprNode::Lambda); - return EnsureTupleTypeSize(node.Pos(), node.GetTypeAnn(), expectedSize, ctx); -} - -bool EnsureTupleTypeSize(TPositionHandle position, const TTypeAnnotationNode* type, ui32 expectedSize, TExprContext& ctx) { - if (HasError(type, ctx) || !type) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got lambda")); + YQL_ENSURE(node.GetTypeAnn() || node.Type() == TExprNode::Lambda); + return EnsureTupleTypeSize(node.Pos(), node.GetTypeAnn(), expectedSize, ctx); +} + +bool EnsureTupleTypeSize(TPositionHandle position, const TTypeAnnotationNode* type, ui32 expectedSize, TExprContext& ctx) { + if (HasError(type, ctx) || !type) { + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got lambda")); return false; } - if (type->GetKind() != ETypeAnnotationKind::Tuple) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got: " << *type)); + if (type->GetKind() != ETypeAnnotationKind::Tuple) { + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type, but got: " << *type)); return false; } - auto tupleSize = type->Cast<TTupleExprType>()->GetSize(); + auto tupleSize = type->Cast<TTupleExprType>()->GetSize(); if (tupleSize != expectedSize) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type of size: " << expectedSize << ", but got: " + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected tuple type of size: " << expectedSize << ", but got: " << tupleSize)); return false; } @@ -2003,16 +2003,16 @@ bool EnsureMultiType(TPositionHandle position, const TTypeAnnotationNode& type, bool EnsureVariantType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected variant type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected variant type, but got lambda")); return false; } return EnsureVariantType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Variant) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected variant type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected variant type, but got: " << type)); return false; } @@ -2022,16 +2022,16 @@ bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type bool EnsureDataType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda")); return false; } return EnsureDataType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Data) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type, but got: " << type)); return false; } @@ -2040,7 +2040,7 @@ bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, T bool EnsureLambda(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Lambda) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected lambda, but got: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected lambda, but got: " << node.Type())); return false; } @@ -2118,14 +2118,14 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext& } if (!withTypes || HasError(node->GetTypeAnn(), ctx) || node->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Callable) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected lambda, but got: " << node->Type())); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected lambda, but got: " << node->Type())); return IGraphTransformer::TStatus::Error; } auto callableType = node->GetTypeAnn()->Cast<TCallableExprType>(); if (minArgumentsCount != Max<ui32>() && (minArgumentsCount > callableType->GetArgumentsSize() || minArgumentsCount < callableType->GetArgumentsSize() - callableType->GetOptionalArgumentsCount())) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert to lambda with " + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert to lambda with " << minArgumentsCount << " arguments from callable type " << *node->GetTypeAnn())); return IGraphTransformer::TStatus::Error; @@ -2147,7 +2147,7 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext& bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda")); return false; } @@ -2157,13 +2157,13 @@ bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, T } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got: " << *node.GetTypeAnn())); return false; } auto dataSlot = node.GetTypeAnn()->Cast<TDataExprType>()->GetSlot(); if (dataSlot != expectedDataSlot) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type: " << NKikimr::NUdf::GetDataTypeInfo(expectedDataSlot).Name << ", but got: " << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type: " << NKikimr::NUdf::GetDataTypeInfo(expectedDataSlot).Name << ", but got: " << *node.GetTypeAnn())); return false; } @@ -2171,15 +2171,15 @@ bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, T return true; } -bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx) { +bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Data) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type, but got: " << type)); return false; } auto dataSlot = type.Cast<TDataExprType>()->GetSlot(); if (dataSlot != expectedDataSlot) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type: " << NKikimr::NUdf::GetDataTypeInfo(expectedDataSlot).Name << ", but got: " << + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected data type: " << NKikimr::NUdf::GetDataTypeInfo(expectedDataSlot).Name << ", but got: " << type)); return false; } @@ -2224,21 +2224,21 @@ bool EnsureStringOrUtf8Type(TPositionHandle position, const TTypeAnnotationNode& bool EnsureStructType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Struct) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected struct type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected struct type, but got: " << type)); return false; } @@ -2250,8 +2250,8 @@ bool EnsureTypeWithStructType(const TExprNode& node, TExprContext& ctx) { return false; } auto nodeType = node.GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - YQL_ENSURE(nodeType); - if (!EnsureStructType(node.Pos(), *nodeType, ctx)) { + YQL_ENSURE(nodeType); + if (!EnsureStructType(node.Pos(), *nodeType, ctx)) { return false; } return true; @@ -2259,7 +2259,7 @@ bool EnsureTypeWithStructType(const TExprNode& node, TExprContext& ctx) { bool EnsureComposable(const TExprNode& node, TExprContext& ctx) { if (!node.IsComposable()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Composable required. World, datasink, datasource and lambda are not composable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Composable required. World, datasink, datasource and lambda are not composable")); return false; } @@ -2269,16 +2269,16 @@ bool EnsureComposable(const TExprNode& node, TExprContext& ctx) { bool EnsureComposableType(const TExprNode& node, TExprContext& ctx) { if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected composable type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected composable type, but got lambda")); return false; } return EnsureComposableType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (!type.IsComposable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), "Composable type required. World, datasink, datasource types are not composable")); + ctx.AddError(TIssue(ctx.GetPosition(position), "Composable type required. World, datasink, datasource types are not composable")); return false; } @@ -2288,12 +2288,12 @@ bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& t bool EnsureWorldType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got: " << *node.GetTypeAnn())); return false; } @@ -2303,12 +2303,12 @@ bool EnsureWorldType(const TExprNode& node, TExprContext& ctx) { bool EnsureDataSource(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda")); return false; } if (!node.IsCallable("DataSource")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable")); return false; } @@ -2318,12 +2318,12 @@ bool EnsureDataSource(const TExprNode& node, TExprContext& ctx) { bool EnsureDataSink(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda")); return false; } if (!node.IsCallable("DataSink")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable")); return false; } @@ -2333,12 +2333,12 @@ bool EnsureDataSink(const TExprNode& node, TExprContext& ctx) { bool EnsureDataProvider(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource or datasink callable, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource or datasink callable, but got lambda")); return false; } if (!node.IsCallable("DataSource") && !node.IsCallable("DataSink")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource or datasink callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource or datasink callable")); return false; } @@ -2348,18 +2348,18 @@ bool EnsureDataProvider(const TExprNode& node, TExprContext& ctx) { bool EnsureSpecificDataSource(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda")); return false; } if (!node.IsCallable("DataSource")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable")); return false; } auto category = node.Head().Content(); if (category != expectedCategory) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource category: " << expectedCategory << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource category: " << expectedCategory << ", but got: " << category)); return false; } @@ -2370,18 +2370,18 @@ bool EnsureSpecificDataSource(const TExprNode& node, TStringBuf expectedCategory bool EnsureSpecificDataSink(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda")); return false; } if (!node.IsCallable("DataSink")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable")); return false; } auto category = node.Head().Content(); if (category != expectedCategory) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink category: " << expectedCategory << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink category: " << expectedCategory << ", but got: " << category)); return false; } @@ -2392,21 +2392,21 @@ bool EnsureSpecificDataSink(const TExprNode& node, TStringBuf expectedCategory, bool EnsureListType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureListType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureListType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::List) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list type, but got: " << type)); return false; } @@ -2441,14 +2441,14 @@ bool EnsureListOrEmptyType(TPositionHandle position, const TTypeAnnotationNode& bool EnsureListOfVoidType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list of void type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list of void type, but got lambda")); return false; } return EnsureListOfVoidType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (!EnsureListType(position, type, ctx)) { return false; } @@ -2457,7 +2457,7 @@ bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& t YQL_ENSURE(listType); if (listType->GetItemType()->GetKind() != ETypeAnnotationKind::Void) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list of void type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list of void type, but got: " << type)); return false; } @@ -2467,21 +2467,21 @@ bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& t bool EnsureStreamType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Stream) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected stream type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected stream type, but got: " << type)); return false; } @@ -2491,21 +2491,21 @@ bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type, bool EnsureFlowType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureFlowType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureFlowType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Flow) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected flow type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected flow type, but got: " << type)); return false; } @@ -2539,16 +2539,16 @@ bool EnsureWideFlowType(TPositionHandle position, const TTypeAnnotationNode& typ bool EnsureOptionalType(const TExprNode& node, TExprContext& ctx) { if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected optional type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected optional type, but got lambda")); return false; } return EnsureOptionalType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected optional type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected optional type, but got: " << type)); return false; } @@ -2556,59 +2556,59 @@ bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& typ } bool EnsureType(const TExprNode& node, TExprContext& ctx) { - YQL_ENSURE(!node.IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType", "SqlColumnFromType"}), - "Unexpected " << node.Content() << " it should be processed earlier"); + YQL_ENSURE(!node.IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType", "SqlColumnFromType"}), + "Unexpected " << node.Content() << " it should be processed earlier"); if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got: " << *node.GetTypeAnn())); return false; } return true; } -IGraphTransformer::TStatus EnsureTypeRewrite(TExprNode::TPtr& node, TExprContext& ctx) { - const TTypeAnnotationNode* type = node->GetTypeAnn(); - if (!type) { - YQL_ENSURE(node->Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected type, but got lambda")); - return IGraphTransformer::TStatus::Error; - } - - if (node->IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType", "SqlColumnFromType"})) { - ui32 typeNameIdx = node->IsCallable("SqlColumnFromType") ? 2 : 1; - auto typeNameNode = node->Child(typeNameIdx); - YQL_ENSURE(typeNameNode->IsAtom()); - TStringBuf typeName = typeNameNode->Content(); - auto typeNode = TryExpandSimpleType(node->Pos(), typeName, ctx); - if (!typeNode) { - return IGraphTransformer::TStatus::Error; - } - node = typeNode; - return IGraphTransformer::TStatus::Repeat; - } - - if (type->GetKind() != ETypeAnnotationKind::Type) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected type, but got: " << *type)); - return IGraphTransformer::TStatus::Error; - } - - return IGraphTransformer::TStatus::Ok; -} - -IGraphTransformer::TStatus EnsureTypeOrAtomRewrite(TExprNode::TPtr& node, TExprContext& ctx) { - if (node->Type() != TExprNode::Atom) { - return EnsureTypeRewrite(node, ctx); - } - - return IGraphTransformer::TStatus::Ok; -} - +IGraphTransformer::TStatus EnsureTypeRewrite(TExprNode::TPtr& node, TExprContext& ctx) { + const TTypeAnnotationNode* type = node->GetTypeAnn(); + if (!type) { + YQL_ENSURE(node->Type() == TExprNode::Lambda); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected type, but got lambda")); + return IGraphTransformer::TStatus::Error; + } + + if (node->IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType", "SqlColumnFromType"})) { + ui32 typeNameIdx = node->IsCallable("SqlColumnFromType") ? 2 : 1; + auto typeNameNode = node->Child(typeNameIdx); + YQL_ENSURE(typeNameNode->IsAtom()); + TStringBuf typeName = typeNameNode->Content(); + auto typeNode = TryExpandSimpleType(node->Pos(), typeName, ctx); + if (!typeNode) { + return IGraphTransformer::TStatus::Error; + } + node = typeNode; + return IGraphTransformer::TStatus::Repeat; + } + + if (type->GetKind() != ETypeAnnotationKind::Type) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Expected type, but got: " << *type)); + return IGraphTransformer::TStatus::Error; + } + + return IGraphTransformer::TStatus::Ok; +} + +IGraphTransformer::TStatus EnsureTypeOrAtomRewrite(TExprNode::TPtr& node, TExprContext& ctx) { + if (node->Type() != TExprNode::Atom) { + return EnsureTypeRewrite(node, ctx); + } + + return IGraphTransformer::TStatus::Ok; +} + bool EnsureDryType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (const auto dry = DryType(&type, ctx); !(dry && IsSameAnnotation(*dry, type))) { ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected dry type, but got: " << type)); @@ -2630,21 +2630,21 @@ bool EnsureDryType(const TExprNode& node, TExprContext& ctx) { bool EnsureDictType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Dict) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Dict) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected dict type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected dict type, but got: " << type)); return false; } @@ -2654,12 +2654,12 @@ bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, T bool EnsureVoidType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Void) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got: " << *node.GetTypeAnn())); return false; } @@ -2669,12 +2669,12 @@ bool EnsureVoidType(const TExprNode& node, TExprContext& ctx) { bool EnsureVoidLiteral(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void literal, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void literal, but got lambda")); return false; } if (!node.IsCallable("Void")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void literal, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void literal, but got: " << *node.GetTypeAnn())); return false; } @@ -2684,16 +2684,16 @@ bool EnsureVoidLiteral(const TExprNode& node, TExprContext& ctx) { bool EnsureCallableType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable type, but got lambda")); return false; } return EnsureCallableType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx) || type.GetKind() != ETypeAnnotationKind::Callable) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected callable type, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected callable type, but got: " << type)); return false; } @@ -2703,12 +2703,12 @@ bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& typ bool EnsureResourceType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Resource) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got: " << *node.GetTypeAnn())); return false; } @@ -2718,12 +2718,12 @@ bool EnsureResourceType(const TExprNode& node, TExprContext& ctx) { bool EnsureTaggedType(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got lambda")); return false; } if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tagged) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got: " << *node.GetTypeAnn())); return false; } @@ -2733,7 +2733,7 @@ bool EnsureTaggedType(const TExprNode& node, TExprContext& ctx) { bool EnsureOneOrTupleOfDataOrOptionalOfData(const TExprNode& node, TExprContext& ctx) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either data (optional of data) or non-empty tuple of data (optional of data), but got lambda")); return false; } @@ -2741,55 +2741,55 @@ bool EnsureOneOrTupleOfDataOrOptionalOfData(const TExprNode& node, TExprContext& return EnsureOneOrTupleOfDataOrOptionalOfData(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureOneOrTupleOfDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureOneOrTupleOfDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { bool ok = false; bool isOptional = false; const TDataExprType* dataType = nullptr; - TIssue err; - bool hasErrorType = false; - TPosition pos = ctx.GetPosition(position); + TIssue err; + bool hasErrorType = false; + TPosition pos = ctx.GetPosition(position); if (type.GetKind() == ETypeAnnotationKind::Tuple) { for (auto& child: type.Cast<TTupleExprType>()->GetItems()) { - ok = IsDataOrOptionalOfData(pos, child, isOptional, dataType, err, hasErrorType); + ok = IsDataOrOptionalOfData(pos, child, isOptional, dataType, err, hasErrorType); if (!ok) { break; } } } else { - ok = IsDataOrOptionalOfData(pos, &type, isOptional, dataType, err, hasErrorType); + ok = IsDataOrOptionalOfData(pos, &type, isOptional, dataType, err, hasErrorType); } if (!ok) { - ctx.AddError(err); - if (!hasErrorType) { - ctx.AddError(TIssue(pos, TStringBuilder() - << "Expected either data (optional of data) or non-empty tuple of data (optional of data), but got: " << type)); - } + ctx.AddError(err); + if (!hasErrorType) { + ctx.AddError(TIssue(pos, TStringBuilder() + << "Expected either data (optional of data) or non-empty tuple of data (optional of data), but got: " << type)); + } } return ok; } -bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { - if (HasError(&type, ctx)) { - return false; - } - +bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { + if (HasError(&type, ctx)) { + return false; + } + if (!type.IsComparable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected comparable type, i.e. combination of Data, Optional, List or Tuple, but got:" << type)); return false; } return true; } -bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { - if (HasError(&type, ctx)) { - return false; - } - +bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { + if (HasError(&type, ctx)) { + return false; + } + if (!type.IsEquatable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected equatable type, i.e. combination of Data, Optional, List, Dict, Tuple, Struct, or Variant, but got:" << type)); return false; } @@ -2827,8 +2827,8 @@ bool IsInstantEqual(const TTypeAnnotationNode& type) { return true; } -static bool EnsureKeyProperty(TPositionHandle position, const TTypeAnnotationNode* keyType, - const std::function<bool(TPositionHandle, EDataSlot, TExprContext&)>& propertyCheck, TExprContext& ctx) { +static bool EnsureKeyProperty(TPositionHandle position, const TTypeAnnotationNode* keyType, + const std::function<bool(TPositionHandle, EDataSlot, TExprContext&)>& propertyCheck, TExprContext& ctx) { if (HasError(keyType, ctx)) { return false; @@ -2860,15 +2860,15 @@ static bool EnsureKeyProperty(TPositionHandle position, const TTypeAnnotationNod return true; } -bool EnsureHashableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { +bool EnsureHashableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { return EnsureKeyProperty(position, keyType, EnsureHashableDataType, ctx); } -bool EnsureComparableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { +bool EnsureComparableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { return EnsureKeyProperty(position, keyType, EnsureComparableDataType, ctx); } -bool EnsureEquatableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { +bool EnsureEquatableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) { return EnsureKeyProperty(position, keyType, EnsureEquatableDataType, ctx); } @@ -2921,52 +2921,52 @@ bool EnsureDataOrOptionalOfData(const TExprNode& node, bool& isOptional, const T YQL_ENSURE(node.Type() == TExprNode::Lambda); } - return EnsureDataOrOptionalOfData(node.Pos(), node.GetTypeAnn(), isOptional, dataType, ctx); + return EnsureDataOrOptionalOfData(node.Pos(), node.GetTypeAnn(), isOptional, dataType, ctx); } -bool EnsureDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode* type, - bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx) -{ - TIssue err; - bool hasErrorType; - if (!IsDataOrOptionalOfData(ctx.GetPosition(position), type, isOptional, dataType, err, hasErrorType)) { - ctx.AddError(err); +bool EnsureDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode* type, + bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx) +{ + TIssue err; + bool hasErrorType; + if (!IsDataOrOptionalOfData(ctx.GetPosition(position), type, isOptional, dataType, err, hasErrorType)) { + ctx.AddError(err); return false; } - return true; -} + return true; +} -bool EnsurePersistable(const TExprNode& node, TExprContext& ctx) { - if (HasError(node.GetTypeAnn(), ctx)) { +bool EnsurePersistable(const TExprNode& node, TExprContext& ctx) { + if (HasError(node.GetTypeAnn(), ctx)) { return false; } if (!node.IsPersistable()) { if (node.GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected persistable data, but got: " << *node.GetTypeAnn())); } else { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected persistable data, but got lambda")); } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Persistable required. Atom, type, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Persistable required. Atom, type, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable")); return false; } return true; } -bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { - if (HasError(&type, ctx)) { - return false; - } - +bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { + if (HasError(&type, ctx)) { + return false; + } + if (!type.IsPersistable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected persistable data, but got: " << type)); - ctx.AddError(TIssue(ctx.GetPosition(position), "Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable")); + ctx.AddError(TIssue(ctx.GetPosition(position), "Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable")); return false; } @@ -2976,15 +2976,15 @@ bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode& bool EnsureComputable(const TExprNode& node, TExprContext& ctx) { if (!node.IsComputable()) { if (node.GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected computable data, but got: " << *node.GetTypeAnn())); } else { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected computable data, but got lambda")); } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Computable required. Atom, key, world, datasink, datasource, type, lambda are not computable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Computable required. Atom, key, world, datasink, datasource, type, lambda are not computable")); return false; } @@ -2994,39 +2994,39 @@ bool EnsureComputable(const TExprNode& node, TExprContext& ctx) { bool EnsureInspectable(const TExprNode& node, TExprContext& ctx) { if (!node.IsInspectable()) { if (node.GetTypeAnn()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected inspectable data, but got: " << *node.GetTypeAnn())); } else { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected inspectable data, but got lambda")); } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Inspectable required. World, datasink, datasource, lambda are not inspectable")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Inspectable required. World, datasink, datasource, lambda are not inspectable")); return false; } return true; } -bool EnsureInspectableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureInspectableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (!type.IsInspectable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected inspectable data, but got: " << type)); - ctx.AddError(TIssue(ctx.GetPosition(position), "Inspectable required. Atom, key, world, datasink, datasource, lambda are not inspectable")); + ctx.AddError(TIssue(ctx.GetPosition(position), "Inspectable required. Atom, key, world, datasink, datasource, lambda are not inspectable")); return false; } return true; } -bool EnsureComputableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureComputableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (!type.IsComputable()) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected computable data, but got: " << type)); - ctx.AddError(TIssue(ctx.GetPosition(position), "Computable required. Atom, key, world, datasink, datasource, type, lambda are not computable")); + ctx.AddError(TIssue(ctx.GetPosition(position), "Computable required. Atom, key, world, datasink, datasource, type, lambda are not computable")); return false; } @@ -3040,13 +3040,13 @@ bool EnsureListOrOptionalType(const TExprNode& node, TExprContext& ctx) { if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got lambda")); return false; } auto kind = node.GetTypeAnn()->GetKind(); if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got: " << *node.GetTypeAnn())); return false; } @@ -3061,20 +3061,20 @@ bool EnsureListOrOptionalListType(const TExprNode& node, TExprContext& ctx) { const auto type = node.GetTypeAnn(); if (!type) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got lambda")); return false; } const auto kind = type->GetKind(); if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got: " << *type)); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got: " << *type)); return false; } if (kind == ETypeAnnotationKind::Optional) { const auto itemType = type->Cast<TOptionalExprType>()->GetItemType(); if (itemType->GetKind() != ETypeAnnotationKind::List) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got: " << *itemType)); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional of list, but got: " << *itemType)); return false; } } @@ -3089,14 +3089,14 @@ bool EnsureSeqType(const TExprNode& node, TExprContext& ctx, bool* isStream) { if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or stream, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or stream, but got lambda")); return false; } return EnsureSeqType(node.Pos(), *node.GetTypeAnn(), ctx, isStream); } -bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx, bool* isStream) { +bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx, bool* isStream) { if (HasError(&type, ctx)) { return false; } @@ -3110,7 +3110,7 @@ bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TE return true; default: - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list or stream, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list or stream, but got: " << type)); return false; } @@ -3123,27 +3123,27 @@ bool EnsureSeqOrOptionalType(const TExprNode& node, TExprContext& ctx) { if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list, stream or optional, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list, stream or optional, but got lambda")); return false; } auto kind = node.GetTypeAnn()->GetKind(); if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Stream && kind != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list, stream or optional, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list, stream or optional, but got: " << *node.GetTypeAnn())); return false; } return true; } -bool EnsureSeqOrOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { +bool EnsureSeqOrOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { if (HasError(&type, ctx)) { return false; } auto kind = type.GetKind(); if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Stream && kind != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list, stream or optional, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected list, stream or optional, but got: " << type)); return false; } @@ -3227,21 +3227,21 @@ bool EnsureStructOrOptionalStructType(const TExprNode& node, TExprContext& ctx) if (!node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either struct or optional of struct, but got lambda")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either struct or optional of struct, but got lambda")); return false; } return EnsureStructOrOptionalStructType(node.Pos(), *node.GetTypeAnn(), ctx); } -bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { - if (HasError(&type, ctx)) { - return false; - } - +bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) { + if (HasError(&type, ctx)) { + return false; + } + auto kind = type.GetKind(); if (kind != ETypeAnnotationKind::Struct && kind != ETypeAnnotationKind::Optional) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected either struct or optional of struct, but got: " << type)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected either struct or optional of struct, but got: " << type)); return false; } if (kind == ETypeAnnotationKind::Optional) { @@ -3262,12 +3262,12 @@ bool EnsureDependsOn(const TExprNode& node, TExprContext& ctx) { } if (!node.IsCallable()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected DependsOn, but got node with type: " << node.Type())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected DependsOn, but got node with type: " << node.Type())); return false; } if (!node.IsCallable("DependsOn")) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected DependsOn, but got callable: " << node.Content())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected DependsOn, but got callable: " << node.Content())); return false; } @@ -3296,7 +3296,7 @@ bool EnsureTypeHandleResourceType(const TExprNode& node, TExprContext& ctx) { } if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != TypeResourceTag) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn())); return false; } @@ -3313,7 +3313,7 @@ bool EnsureCodeResourceType(const TExprNode& node, TExprContext& ctx) { } if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != CodeResourceTag) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn())); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn())); return false; } @@ -3343,7 +3343,7 @@ IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnota auto callableType = expectedType.Cast<TCallableExprType>(); auto lambdaArgsCount = node->Head().ChildrenSize(); if (lambdaArgsCount != callableType->GetArgumentsSize()) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Wrong number of lambda arguments: " + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Wrong number of lambda arguments: " << lambdaArgsCount << ", failed to convert lambda to " << expectedType)); return IGraphTransformer::TStatus::Error; } @@ -3353,7 +3353,7 @@ IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnota return IGraphTransformer::TStatus::Repeat; } - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert lambda to " << expectedType)); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert lambda to " << expectedType)); return IGraphTransformer::TStatus::Error; } @@ -3368,7 +3368,7 @@ IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnota auto status = TryConvertToImpl(ctx, node, sourceType, expectedType, flags); if (status.Level == IGraphTransformer::TStatus::Error) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert type: " << + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert type: " << sourceType << " to " << expectedType)); } @@ -3562,10 +3562,10 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T auto type1Opt = ctx.MakeType<TOptionalExprType>(&type1); auto prev = node1; node1 = ctx.NewCallable(node1->Pos(), "Just", { std::move(node1) }); - - const TTypeAnnotationNode* commonItemType; + + const TTypeAnnotationNode* commonItemType; if (SilentInferCommonType(node1, *type1Opt, node2, type2, ctx, commonItemType, flags) != IGraphTransformer::TStatus::Error) { - commonType = commonItemType; + commonType = commonItemType; return IGraphTransformer::TStatus::Repeat; } @@ -3576,10 +3576,10 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T auto type2Opt = ctx.MakeType<TOptionalExprType>(&type2); auto prev = node2; node2 = ctx.NewCallable(node2->Pos(), "Just", { std::move(node2) }); - - const TTypeAnnotationNode* commonItemType; + + const TTypeAnnotationNode* commonItemType; if (SilentInferCommonType(node1, type1, node2, *type2Opt, ctx, commonItemType, flags) != IGraphTransformer::TStatus::Error) { - commonType = commonItemType; + commonType = commonItemType; return IGraphTransformer::TStatus::Repeat; } @@ -3617,13 +3617,13 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T if (type1.GetKind() == ETypeAnnotationKind::List && type2.GetKind() == ETypeAnnotationKind::List || type1.GetKind() == ETypeAnnotationKind::Optional && type2.GetKind() == ETypeAnnotationKind::Optional) { - const bool isList = type1.GetKind() == ETypeAnnotationKind::List; - - auto item1type = isList ? + const bool isList = type1.GetKind() == ETypeAnnotationKind::List; + + auto item1type = isList ? type1.Cast<TListExprType>()->GetItemType() : type1.Cast<TOptionalExprType>()->GetItemType(); - auto item2type = isList ? + auto item2type = isList ? type2.Cast<TListExprType>()->GetItemType() : type2.Cast<TOptionalExprType>()->GetItemType(); @@ -3651,11 +3651,11 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T .Build(); } - if (isList) { - commonType = ctx.MakeType<TListExprType>(commonItemType); - } else { - commonType = ctx.MakeType<TOptionalExprType>(commonItemType); - } + if (isList) { + commonType = ctx.MakeType<TListExprType>(commonItemType); + } else { + commonType = ctx.MakeType<TOptionalExprType>(commonItemType); + } return IGraphTransformer::TStatus::Repeat; } } @@ -3716,7 +3716,7 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T if (pos1 && pos2) { auto member1 = structType1->GetItems()[*pos1]; auto member2 = structType2->GetItems()[*pos2]; - auto atom = ctx.NewAtom(TPositionHandle(), x); + auto atom = ctx.NewAtom(TPositionHandle(), x); auto arg1 = ctx.NewCallable(node1->Pos(), "Member", { node1, atom }); auto arg2 = ctx.NewCallable(node2->Pos(), "Member", { node2, atom }); const TTypeAnnotationNode* commonItemType; @@ -3731,7 +3731,7 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T } else if (pos1) { auto member1 = structType1->GetItems()[*pos1]; auto commonItemType = member1->GetItemType(); - auto atom = ctx.NewAtom(TPositionHandle(), x); + auto atom = ctx.NewAtom(TPositionHandle(), x); TExprNode::TPtr arg1, arg2; if (commonItemType->GetKind() != ETypeAnnotationKind::Null) { bool addJust = false; @@ -3756,7 +3756,7 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T } else if (pos2) { auto member2 = structType2->GetItems()[*pos2]; auto commonItemType = member2->GetItemType(); - auto atom = ctx.NewAtom(TPositionHandle(), x); + auto atom = ctx.NewAtom(TPositionHandle(), x); TExprNode::TPtr arg1, arg2; if (commonItemType->GetKind() != ETypeAnnotationKind::Null) { bool addJust = false; @@ -4025,57 +4025,57 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input, const TTypeAnnotationNode* targetType, TExprContext& ctx) { if (!input->ChildrenSize()) { - return IGraphTransformer::TStatus::Ok; - } - + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; for (auto i = 0U; i < input->ChildrenSize(); ++i) { - const auto child = input->Child(i); - if (!EnsureComputable(*child, ctx)) { - return IGraphTransformer::TStatus::Error; - } - + const auto child = input->Child(i); + if (!EnsureComputable(*child, ctx)) { + return IGraphTransformer::TStatus::Error; + } + status = status.Combine(TryConvertTo(input->ChildRef(i), *targetType, ctx)); if (IGraphTransformer::TStatus::Error == status) break; - } - + } + return status; -} - -bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node) { - auto collectionType = node.Collection().Ref().GetTypeAnn(); - if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { - collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); - } - - const auto collectionKind = collectionType->GetKind(); - bool result = false; - switch (collectionKind) { - case ETypeAnnotationKind::Tuple: - { - const auto tupleType = collectionType->Cast<TTupleExprType>(); - result = AnyOf(tupleType->GetItems(), [](const auto& item) { return item->HasOptionalOrNull(); } ); - break; - } - case ETypeAnnotationKind::Dict: - result = collectionType->Cast<TDictExprType>()->GetKeyType()->HasOptionalOrNull(); - break; - case ETypeAnnotationKind::List: - result = collectionType->Cast<TListExprType>()->GetItemType()->HasOptionalOrNull(); - break; - case ETypeAnnotationKind::EmptyDict: - case ETypeAnnotationKind::EmptyList: - case ETypeAnnotationKind::Null: - break; - default: - YQL_ENSURE(false, "Unexpected collection type: " << *collectionType); - } - - return result; -} - - +} + +bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node) { + auto collectionType = node.Collection().Ref().GetTypeAnn(); + if (collectionType->GetKind() == ETypeAnnotationKind::Optional) { + collectionType = collectionType->Cast<TOptionalExprType>()->GetItemType(); + } + + const auto collectionKind = collectionType->GetKind(); + bool result = false; + switch (collectionKind) { + case ETypeAnnotationKind::Tuple: + { + const auto tupleType = collectionType->Cast<TTupleExprType>(); + result = AnyOf(tupleType->GetItems(), [](const auto& item) { return item->HasOptionalOrNull(); } ); + break; + } + case ETypeAnnotationKind::Dict: + result = collectionType->Cast<TDictExprType>()->GetKeyType()->HasOptionalOrNull(); + break; + case ETypeAnnotationKind::List: + result = collectionType->Cast<TListExprType>()->GetItemType()->HasOptionalOrNull(); + break; + case ETypeAnnotationKind::EmptyDict: + case ETypeAnnotationKind::EmptyList: + case ETypeAnnotationKind::Null: + break; + default: + YQL_ENSURE(false, "Unexpected collection type: " << *collectionType); + } + + return result; +} + + ui32 GetNumericDataTypeLevel(EDataSlot dataSlot) { if (dataSlot == EDataSlot::Uint8) return 0; @@ -4285,7 +4285,7 @@ void ExtractIntegralValue(const TExprNode& constructor, bool negate, bool& hasSi } else { hasSign = atom.Content().StartsWith('-'); - auto strValue = hasSign + auto strValue = hasSign ? atom.Content().Tail(1) : atom.Content(); @@ -4426,7 +4426,7 @@ bool IsDataTypeString(EDataSlot dataSlot) { return NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::StringType; } -bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { +bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanCompare)) { TStringBuilder sb; bool isYson = dataSlot == EDataSlot::Yson; @@ -4440,25 +4440,25 @@ bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExp } else { sb << "Expected comparable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name; } - ctx.AddError(TIssue(ctx.GetPosition(position), sb)); + ctx.AddError(TIssue(ctx.GetPosition(position), sb)); return false; } return true; } -bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { +bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanEquate)) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected equatable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected equatable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name)); return false; } return true; } -bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { +bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) { if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanHash)) { - ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected hashable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name)); + ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected hashable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name)); return false; } @@ -4520,28 +4520,28 @@ TString NormalizeName(const TStringBuf& name) { } bool HasError(const TTypeAnnotationNode* type, TExprContext& ctx) { - TIssue errIssue; - if (HasError(type, errIssue)) { - ctx.AddError(errIssue); - return true; - } - return false; -} - -bool HasError(const TTypeAnnotationNode* type, TIssue& errIssue) { - errIssue = {}; - + TIssue errIssue; + if (HasError(type, errIssue)) { + ctx.AddError(errIssue); + return true; + } + return false; +} + +bool HasError(const TTypeAnnotationNode* type, TIssue& errIssue) { + errIssue = {}; + if (!type) { return false; } if (type->GetKind() == ETypeAnnotationKind::Error) { - errIssue = type->Cast<TErrorExprType>()->GetError(); + errIssue = type->Cast<TErrorExprType>()->GetError(); return true; } if (type->GetKind() == ETypeAnnotationKind::Type) { - return HasError(type->Cast<TTypeExprType>()->GetType(), errIssue); + return HasError(type->Cast<TTypeExprType>()->GetType(), errIssue); } return false; @@ -4813,89 +4813,89 @@ TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& return ret; } -bool IsSystemMember(const TStringBuf& memberName) { +bool IsSystemMember(const TStringBuf& memberName) { return memberName.StartsWith(TStringBuf("_yql_")); } -IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, - bool deduplicate) -{ - if (!EnsureTupleOfAtoms(*input->Child(index), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto atomList = input->Child(index)->ChildrenList(); - bool needRestart = false; - auto getKey = [](const auto& node) { return node->Content(); }; - auto cmp = [&getKey](const auto& a, const auto& b) { return getKey(a) < getKey(b); }; - if (!IsSorted(atomList.begin(), atomList.end(), cmp)) { - if (deduplicate) { - SortUniqueBy(atomList, getKey); - } else { - Sort(atomList, cmp); - } - needRestart = true; - } else if (deduplicate) { - auto dups = UniqueBy(atomList.begin(), atomList.end(), getKey); - if (dups != atomList.end()) { - needRestart = true; - atomList.erase(dups, atomList.end()); - } - } - - if (needRestart) { - output = ctx.ChangeChild(*input, index, ctx.NewList(input->Child(index)->Pos(), std::move(atomList))); - return IGraphTransformer::TStatus::Repeat; - } - - return IGraphTransformer::TStatus::Ok; -} - -IGraphTransformer::TStatus NormalizeKeyValueTuples(const TExprNode::TPtr& input, ui32 startIndex, TExprNode::TPtr& output, - TExprContext &ctx, bool deduplicate) -{ - YQL_ENSURE(input->IsCallable() || input->IsList()); - YQL_ENSURE(startIndex <= input->ChildrenSize()); - - auto children = input->ChildrenList(); - auto begin = children.begin() + startIndex; - auto end = children.end(); - - for (auto item = begin; item != end; ++item) { - auto node = *item; - if (!EnsureTupleMinSize(*node, 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureAtom(*node->Child(0), ctx)) { - return IGraphTransformer::TStatus::Error; - } - } - - bool needRestart = false; - auto getKey = [](const auto& node) { return node->Child(0)->Content(); }; - auto compare = [&getKey](const auto& a, const auto& b) { return getKey(a) < getKey(b); }; - if (!IsSorted(begin, end, compare)) { - StableSort(begin, end, compare); - needRestart = true; - } - - if (deduplicate) { - auto dups = UniqueBy(begin, end, getKey); - if (dups != end) { - needRestart = true; - children.erase(dups, end); - } - } - - if (needRestart) { - output = input->IsCallable() ? ctx.NewCallable(input->Pos(), input->Content(), std::move(children)) : - ctx.NewList(input->Pos(), std::move(children)); - return IGraphTransformer::TStatus::Repeat; - } - - return IGraphTransformer::TStatus::Ok; -} - +IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, + bool deduplicate) +{ + if (!EnsureTupleOfAtoms(*input->Child(index), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto atomList = input->Child(index)->ChildrenList(); + bool needRestart = false; + auto getKey = [](const auto& node) { return node->Content(); }; + auto cmp = [&getKey](const auto& a, const auto& b) { return getKey(a) < getKey(b); }; + if (!IsSorted(atomList.begin(), atomList.end(), cmp)) { + if (deduplicate) { + SortUniqueBy(atomList, getKey); + } else { + Sort(atomList, cmp); + } + needRestart = true; + } else if (deduplicate) { + auto dups = UniqueBy(atomList.begin(), atomList.end(), getKey); + if (dups != atomList.end()) { + needRestart = true; + atomList.erase(dups, atomList.end()); + } + } + + if (needRestart) { + output = ctx.ChangeChild(*input, index, ctx.NewList(input->Child(index)->Pos(), std::move(atomList))); + return IGraphTransformer::TStatus::Repeat; + } + + return IGraphTransformer::TStatus::Ok; +} + +IGraphTransformer::TStatus NormalizeKeyValueTuples(const TExprNode::TPtr& input, ui32 startIndex, TExprNode::TPtr& output, + TExprContext &ctx, bool deduplicate) +{ + YQL_ENSURE(input->IsCallable() || input->IsList()); + YQL_ENSURE(startIndex <= input->ChildrenSize()); + + auto children = input->ChildrenList(); + auto begin = children.begin() + startIndex; + auto end = children.end(); + + for (auto item = begin; item != end; ++item) { + auto node = *item; + if (!EnsureTupleMinSize(*node, 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureAtom(*node->Child(0), ctx)) { + return IGraphTransformer::TStatus::Error; + } + } + + bool needRestart = false; + auto getKey = [](const auto& node) { return node->Child(0)->Content(); }; + auto compare = [&getKey](const auto& a, const auto& b) { return getKey(a) < getKey(b); }; + if (!IsSorted(begin, end, compare)) { + StableSort(begin, end, compare); + needRestart = true; + } + + if (deduplicate) { + auto dups = UniqueBy(begin, end, getKey); + if (dups != end) { + needRestart = true; + children.erase(dups, end); + } + } + + if (needRestart) { + output = input->IsCallable() ? ctx.NewCallable(input->Pos(), input->Content(), std::move(children)) : + ctx.NewList(input->Pos(), std::move(children)); + return IGraphTransformer::TStatus::Repeat; + } + + return IGraphTransformer::TStatus::Ok; +} + std::optional<ui32> GetFieldPosition(const TMultiExprType& multiType, const TStringBuf& field) { if (ui32 pos; TryFromString(field, pos) && pos < multiType.GetSize()) return {pos}; diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h index 0c30c2d5eb..a865aa27cd 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.h +++ b/ydb/library/yql/core/yql_expr_type_annotation.h @@ -52,7 +52,7 @@ IGraphTransformer::TStatus CheckWholeProgramType(const TExprNode::TPtr& input, T void ClearExprTypeAnnotations(TExprNode& root); bool AreAllNodesTypeAnnotated(const TExprNode& root); void EnsureAllNodesTypeAnnotated(const TExprNode& root); -bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isOptional, const TDataExprType*& dataType); +bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isOptional, const TDataExprType*& dataType); bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation); bool UpdateLambdaAllArgumentsTypes(TExprNode::TPtr& lambda, const std::vector<const TTypeAnnotationNode*>& argumentsAnnotations, TExprContext& ctx); bool UpdateLambdaArgumentsType(const TExprNode& lambda, TExprContext& ctx); @@ -60,12 +60,12 @@ bool EnsureArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx bool EnsureMinArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx); bool EnsureMaxArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx); bool EnsureMinMaxArgsCount(const TExprNode& node, ui32 minArgs, ui32 maxArgs, TExprContext& ctx); -bool EnsureCallableMinArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx); -bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx); +bool EnsureCallableMinArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx); +bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expectedArgs, TExprContext& ctx); bool EnsureAtom(const TExprNode& node, TExprContext& ctx); bool EnsureCallable(const TExprNode& node, TExprContext& ctx); bool EnsureTuple(const TExprNode& node, TExprContext& ctx); -bool EnsureTupleOfAtoms(const TExprNode& node, TExprContext& ctx); +bool EnsureTupleOfAtoms(const TExprNode& node, TExprContext& ctx); bool EnsureLambda(const TExprNode& node, TExprContext& ctx); IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext& ctx, ui32 argumentsCount, ui32 maxArgumentsCount = Max<ui32>(), bool withTypes = true); @@ -73,21 +73,21 @@ bool EnsureTupleSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx bool EnsureTupleMinSize(const TExprNode& node, ui32 minSize, TExprContext& ctx); bool EnsureTupleMaxSize(const TExprNode& node, ui32 maxSize, TExprContext& ctx); bool EnsureTupleType(const TExprNode& node, TExprContext& ctx); -bool EnsureTupleType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureTupleType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureTupleTypeSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx); -bool EnsureTupleTypeSize(TPositionHandle position, const TTypeAnnotationNode* type, ui32 expectedSize, TExprContext& ctx); +bool EnsureTupleTypeSize(TPositionHandle position, const TTypeAnnotationNode* type, ui32 expectedSize, TExprContext& ctx); bool EnsureMultiType(const TExprNode& node, TExprContext& ctx); bool EnsureMultiType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureVariantType(const TExprNode& node, TExprContext& ctx); -bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureDataType(const TExprNode& node, TExprContext& ctx); -bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional = false); -bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx); +bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx); bool EnsureStringOrUtf8Type(const TExprNode& node, TExprContext& ctx); bool EnsureStringOrUtf8Type(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureStructType(const TExprNode& node, TExprContext& ctx); -bool EnsureStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureTypeWithStructType(const TExprNode& node, TExprContext& ctx); bool EnsureComposable(const TExprNode& node, TExprContext& ctx); bool EnsureComposableType(const TExprNode& node, TExprContext& ctx); @@ -98,57 +98,57 @@ bool EnsureDataProvider(const TExprNode& node, TExprContext& ctx); bool EnsureSpecificDataSource(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx); bool EnsureSpecificDataSink(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx); bool EnsureListType(const TExprNode& node, TExprContext& ctx); -bool EnsureListType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureListType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureListOrEmptyType(const TExprNode& node, TExprContext& ctx); bool EnsureListOrEmptyType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureListOfVoidType(const TExprNode& node, TExprContext& ctx); -bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureStreamType(const TExprNode& node, TExprContext& ctx); -bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureFlowType(const TExprNode& node, TExprContext& ctx); -bool EnsureFlowType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureFlowType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureWideFlowType(const TExprNode& node, TExprContext& ctx); bool EnsureWideFlowType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureOptionalType(const TExprNode& node, TExprContext& ctx); -bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureType(const TExprNode& node, TExprContext& ctx); -IGraphTransformer::TStatus EnsureTypeRewrite(TExprNode::TPtr& node, TExprContext& ctx); -IGraphTransformer::TStatus EnsureTypeOrAtomRewrite(TExprNode::TPtr& node, TExprContext& ctx); +IGraphTransformer::TStatus EnsureTypeRewrite(TExprNode::TPtr& node, TExprContext& ctx); +IGraphTransformer::TStatus EnsureTypeOrAtomRewrite(TExprNode::TPtr& node, TExprContext& ctx); bool EnsureDryType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureDryType(const TExprNode& node, TExprContext& ctx); bool EnsureDictType(const TExprNode& node, TExprContext& ctx); -bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureVoidType(const TExprNode& node, TExprContext& ctx); bool EnsureVoidLiteral(const TExprNode& node, TExprContext& ctx); bool EnsureCallableType(const TExprNode& node, TExprContext& ctx); -bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureResourceType(const TExprNode& node, TExprContext& ctx); bool EnsureTaggedType(const TExprNode& node, TExprContext& ctx); -bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureOneOrTupleOfDataOrOptionalOfData(const TExprNode& node, TExprContext& ctx); -bool EnsureOneOrTupleOfDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); -bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); -bool EnsureComparableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); -bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); -bool EnsureEquatableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); -bool EnsureHashableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); +bool EnsureOneOrTupleOfDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureComparableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); +bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureEquatableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); +bool EnsureHashableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx); bool EnsureDataOrOptionalOfData(const TExprNode& node, bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx); -bool EnsureDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode* type, bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx); +bool EnsureDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode* type, bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx); bool EnsurePersistable(const TExprNode& node, TExprContext& ctx); -bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureComputable(const TExprNode& node, TExprContext& ctx); -bool EnsureComputableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureComputableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureInspectable(const TExprNode& node, TExprContext& ctx); -bool EnsureInspectableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureInspectableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureListOrOptionalType(const TExprNode& node, TExprContext& ctx); bool EnsureListOrOptionalListType(const TExprNode& node, TExprContext& ctx); bool EnsureStructOrOptionalStructType(const TExprNode& node, TExprContext& ctx); -bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureSeqType(const TExprNode& node, TExprContext& ctx, bool* isStream = nullptr); -bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx, bool* isStream = nullptr); +bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx, bool* isStream = nullptr); bool EnsureSeqOrOptionalType(const TExprNode& node, TExprContext& ctx); -bool EnsureSeqOrOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +bool EnsureSeqOrOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); template <bool WithOptional, bool WithList = true, bool WithStream = true> bool EnsureNewSeqType(const TExprNode& node, TExprContext& ctx, const TTypeAnnotationNode** itemType = nullptr); template <bool WithOptional, bool WithList = true, bool WithStream = true> @@ -220,8 +220,8 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const T const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx); -bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node); - +bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node); + bool IsDataTypeNumeric(EDataSlot dataSlot); bool IsDataTypeFloat(EDataSlot dataSlot); bool IsDataTypeIntegral(EDataSlot dataSlot); @@ -250,14 +250,14 @@ bool AllowIntegralConversion(NNodes::TCoIntegralCtor node, bool negate, EDataSlo TString* atomValue = nullptr); void ExtractIntegralValue(const TExprNode& constructor, bool negate, bool& hasSign, bool& isSigned, ui64& value); bool IsDataTypeString(EDataSlot dataSlot); -bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); -bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); -bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); +bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); +bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); +bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx); TMaybe<TIssue> NormalizeName(TPosition position, TString& name); TString NormalizeName(const TStringBuf& name); bool HasError(const TTypeAnnotationNode* type, TExprContext& ctx); -bool HasError(const TTypeAnnotationNode* type, TIssue& errIssue); +bool HasError(const TTypeAnnotationNode* type, TIssue& errIssue); bool IsNull(const TExprNode& node); bool IsNull(const TTypeAnnotationNode& type); bool IsEmptyList(const TExprNode& node); @@ -265,15 +265,15 @@ bool IsEmptyList(const TTypeAnnotationNode& type); bool IsInstantEqual(const TTypeAnnotationNode& type); TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right); -TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); +TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); + +bool IsSystemMember(const TStringBuf& memberName); -bool IsSystemMember(const TStringBuf& memberName); +IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, + bool deduplicte = true); +IGraphTransformer::TStatus NormalizeKeyValueTuples(const TExprNode::TPtr& input, ui32 startIndex, TExprNode::TPtr& output, + TExprContext& ctx, bool deduplicate = false); -IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx, - bool deduplicte = true); -IGraphTransformer::TStatus NormalizeKeyValueTuples(const TExprNode::TPtr& input, ui32 startIndex, TExprNode::TPtr& output, - TExprContext& ctx, bool deduplicate = false); - std::optional<ui32> GetFieldPosition(const TMultiExprType& tupleType, const TStringBuf& field); std::optional<ui32> GetFieldPosition(const TTupleExprType& tupleType, const TStringBuf& field); std::optional<ui32> GetFieldPosition(const TStructExprType& structType, const TStringBuf& field); diff --git a/ydb/library/yql/core/yql_graph_transformer.cpp b/ydb/library/yql/core/yql_graph_transformer.cpp index b140c501e3..26193c4c40 100644 --- a/ydb/library/yql/core/yql_graph_transformer.cpp +++ b/ydb/library/yql/core/yql_graph_transformer.cpp @@ -5,14 +5,14 @@ namespace NYql { -namespace { - +namespace { + class TCompositeGraphTransformer : public TGraphTransformerBase { public: - TCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes, bool doCheckArguments) + TCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes, bool doCheckArguments) : Stages(stages) , UseIssueScopes(useIssueScopes) - , DoCheckArguments(doCheckArguments) + , DoCheckArguments(doCheckArguments) { if (UseIssueScopes) { for (const auto& stage : Stages) { @@ -23,12 +23,12 @@ public: void Rewind() override { for (auto& stage : Stages) { - stage.GetTransformer().Rewind(); + stage.GetTransformer().Rewind(); } Index = 0; LastIssueScope = Nothing(); - CheckArgumentsCount = 0; + CheckArgumentsCount = 0; } TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override { @@ -52,21 +52,21 @@ public: if (UseIssueScopes) { UpdateIssueScope(ctx.IssueManager); } - auto status = Stages[Index].GetTransformer().Transform(input, output, ctx); -#ifndef NDEBUG - if (DoCheckArguments && output && output != input) { - try { - CheckArguments(*output); - ++CheckArgumentsCount; - } catch (yexception& e) { - e << "at CheckArguments() pass #" << CheckArgumentsCount - << ", stage '" << Stages[Index].Name << "'"; - throw; - } - } -#else - Y_UNUSED(DoCheckArguments); - Y_UNUSED(CheckArgumentsCount); + auto status = Stages[Index].GetTransformer().Transform(input, output, ctx); +#ifndef NDEBUG + if (DoCheckArguments && output && output != input) { + try { + CheckArguments(*output); + ++CheckArgumentsCount; + } catch (yexception& e) { + e << "at CheckArguments() pass #" << CheckArgumentsCount + << ", stage '" << Stages[Index].Name << "'"; + throw; + } + } +#else + Y_UNUSED(DoCheckArguments); + Y_UNUSED(CheckArgumentsCount); #endif status = HandleStatus(status); return status; @@ -74,12 +74,12 @@ public: NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) override { YQL_ENSURE(Index < Stages.size()); - return Stages[Index].GetTransformer().GetAsyncFuture(input); + return Stages[Index].GetTransformer().GetAsyncFuture(input); } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override { YQL_ENSURE(Index < Stages.size()); - const auto status = Stages[Index].GetTransformer().ApplyAsyncChanges(input, output, ctx); + const auto status = Stages[Index].GetTransformer().ApplyAsyncChanges(input, output, ctx); return HandleStatus(status); } @@ -92,7 +92,7 @@ public: for (size_t i = 0; i < Stages.size(); ++i) { auto& stagePair = Statistics.Stages[i]; stagePair.first = Stages[i].Name; - stagePair.second = Stages[i].GetTransformer().GetStatistics(); + stagePair.second = Stages[i].GetTransformer().GetStatistics(); } return Statistics; @@ -136,29 +136,29 @@ private: protected: TVector<TTransformStage> Stages; const bool UseIssueScopes; - const bool DoCheckArguments; + const bool DoCheckArguments; size_t Index = 0; TMaybe<EYqlIssueCode> LastIssueScope; - ui64 CheckArgumentsCount = 0; + ui64 CheckArgumentsCount = 0; }; -void AddTooManyTransformationsError(TPositionHandle pos, const TStringBuf& where, TExprContext& ctx) { - ctx.AddError(TIssue(ctx.GetPosition(pos), - TStringBuilder() << "YQL: Internal core error! " << where << " take too much iteration: " - << ctx.RepeatTransformLimit - << ". You may set RepeatTransformLimit as flags for config provider.")); -} - -} - +void AddTooManyTransformationsError(TPositionHandle pos, const TStringBuf& where, TExprContext& ctx) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + TStringBuilder() << "YQL: Internal core error! " << where << " take too much iteration: " + << ctx.RepeatTransformLimit + << ". You may set RepeatTransformLimit as flags for config provider.")); +} + +} + TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes) { - return new TCompositeGraphTransformer(stages, useIssueScopes, /* doCheckArguments = */ true); + return new TCompositeGraphTransformer(stages, useIssueScopes, /* doCheckArguments = */ true); +} + +TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const TVector<TTransformStage>& stages, bool useIssueScopes) { + return new TCompositeGraphTransformer(stages, useIssueScopes, /* doCheckArguments = */ false); } -TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const TVector<TTransformStage>& stages, bool useIssueScopes) { - return new TCompositeGraphTransformer(stages, useIssueScopes, /* doCheckArguments = */ false); -} - namespace { class TChoiceGraphTransformer : public TCompositeGraphTransformer { @@ -230,7 +230,7 @@ TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( IGraphTransformer::TStatus SyncTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx) { try { - for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { + for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { TExprNode::TPtr newRoot; auto status = transformer.Transform(root, newRoot, ctx); if (newRoot) { @@ -271,17 +271,17 @@ IGraphTransformer::TStatus SyncTransform(IGraphTransformer& transformer, TExprNo YQL_ENSURE(false, "Unknown status"); } } - AddTooManyTransformationsError(root->Pos(), "SyncTransform", ctx); + AddTooManyTransformationsError(root->Pos(), "SyncTransform", ctx); } catch (const std::exception& e) { ctx.AddError(ExceptionToIssue(e)); } - return IGraphTransformer::TStatus::Error; + return IGraphTransformer::TStatus::Error; } IGraphTransformer::TStatus InstantTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx, bool breakOnRestart) { try { - for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { + for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { TExprNode::TPtr newRoot; auto status = transformer.Transform(root, newRoot, ctx); if (newRoot) { @@ -299,18 +299,18 @@ IGraphTransformer::TStatus InstantTransform(IGraphTransformer& transformer, TExp continue; case IGraphTransformer::TStatus::Async: - ctx.AddError(TIssue(ctx.GetPosition(root->Pos()), "Instant transform can not be delayed")); + ctx.AddError(TIssue(ctx.GetPosition(root->Pos()), "Instant transform can not be delayed")); return IGraphTransformer::TStatus::Error; default: YQL_ENSURE(false, "Unknown status"); } } - AddTooManyTransformationsError(root->Pos(), "InstantTransform", ctx); + AddTooManyTransformationsError(root->Pos(), "InstantTransform", ctx); } catch (const std::exception& e) { ctx.AddError(ExceptionToIssue(e)); } - return IGraphTransformer::TStatus::Error; + return IGraphTransformer::TStatus::Error; } NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx, @@ -338,7 +338,7 @@ NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransform(IGraphTransformer } return NThreading::MakeFuture(status); } - for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { + for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { TExprNode::TPtr newRoot; auto status = transformer.Transform(root, newRoot, ctx); if (newRoot) { @@ -359,8 +359,8 @@ NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransform(IGraphTransformer } break; } - if (ctx.RepeatTransformCounter >= ctx.RepeatTransformLimit) { - AddTooManyTransformationsError(root->Pos(), "AsyncTransform", ctx); + if (ctx.RepeatTransformCounter >= ctx.RepeatTransformLimit) { + AddTooManyTransformationsError(root->Pos(), "AsyncTransform", ctx); return NThreading::MakeFuture(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error)); } } diff --git a/ydb/library/yql/core/yql_graph_transformer.h b/ydb/library/yql/core/yql_graph_transformer.h index ab37f51e93..60d87727dd 100644 --- a/ydb/library/yql/core/yql_graph_transformer.h +++ b/ydb/library/yql/core/yql_graph_transformer.h @@ -56,7 +56,7 @@ public: , Padding(0) {} - [[nodiscard]] + [[nodiscard]] TStatus Combine(TStatus other) const { const bool hasRestart = HasRestart || other.HasRestart; return TStatus((TStatus::ELevel)Max(Level, other.Level), hasRestart); @@ -194,32 +194,32 @@ struct TTransformStage { TString IssueMessage; TTransformStage(const TAutoPtr<IGraphTransformer>& transformer, const TString& name, EYqlIssueCode issueCode, const TString& issueMessage = {}) - : Name(name) + : Name(name) , IssueCode(issueCode) , IssueMessage(issueMessage) - , RawTransformer(transformer.Get()) - , Transformer(transformer) + , RawTransformer(transformer.Get()) + , Transformer(transformer) {} - - TTransformStage(IGraphTransformer& transformer, const TString& name, EYqlIssueCode issueCode, const TString& issueMessage = {}) - : Name(name) - , IssueCode(issueCode) - , IssueMessage(issueMessage) - , RawTransformer(&transformer) - {} - - IGraphTransformer& GetTransformer() const - { - return *RawTransformer; - } -private: - IGraphTransformer* const RawTransformer; - const TAutoPtr<IGraphTransformer> Transformer; + + TTransformStage(IGraphTransformer& transformer, const TString& name, EYqlIssueCode issueCode, const TString& issueMessage = {}) + : Name(name) + , IssueCode(issueCode) + , IssueMessage(issueMessage) + , RawTransformer(&transformer) + {} + + IGraphTransformer& GetTransformer() const + { + return *RawTransformer; + } +private: + IGraphTransformer* const RawTransformer; + const TAutoPtr<IGraphTransformer> Transformer; }; TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes); -TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const TVector<TTransformStage>& stages, bool useIssueScopes); - +TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const TVector<TTransformStage>& stages, bool useIssueScopes); + TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, const TTransformStage& left, @@ -333,7 +333,7 @@ WrapFutureCallback(const TFuture& future, const TCallback& callback, const TStri TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { return MakeIntrusive<TIssue>( - ctx.GetPosition(input->Pos()), + ctx.GetPosition(input->Pos()), message.empty() ? TStringBuilder() << "Execution of node: " << input->Content() : message); @@ -341,7 +341,7 @@ WrapFutureCallback(const TFuture& future, const TCallback& callback, const TStri res.ReportIssues(ctx.IssueManager); if (!res.Success()) { - input->SetState(TExprNode::EState::Error); + input->SetState(TExprNode::EState::Error); return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); } else { @@ -355,7 +355,7 @@ template <typename TFuture, typename TResultExtractor> std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> WrapFuture(const TFuture& future, const TResultExtractor& extractor, const TString& message = "") { return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) { - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(extractor(res, input, ctx)); return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok); }, message); @@ -366,7 +366,7 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> WrapModifyFuture(const TFuture& future, const TResultExtractor& extractor, const TString& message = "") { return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { TExprNode::TPtr resultNode = extractor(res, input, output, ctx); - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); output->SetResult(std::move(resultNode)); if (input != output) { return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); diff --git a/ydb/library/yql/core/yql_join.cpp b/ydb/library/yql/core/yql_join.cpp index 5559aed075..9a9ded7a32 100644 --- a/ydb/library/yql/core/yql_join.cpp +++ b/ydb/library/yql/core/yql_join.cpp @@ -1,407 +1,407 @@ #include "yql_join.h" #include "yql_expr_type_annotation.h" -#include "yql_opt_utils.h" +#include "yql_opt_utils.h" #include <util/string/cast.h> -#include <util/string/join.h> - +#include <util/string/join.h> + namespace NYql { using namespace NNodes; namespace { const TTypeAnnotationNode* AddOptionalType(const TTypeAnnotationNode* type, TExprContext& ctx) { - if (type->IsOptionalOrNull()) { + if (type->IsOptionalOrNull()) { return type; } return ctx.MakeType<TOptionalExprType>(type); } - - struct TJoinState { - bool Used = false; - }; - - IGraphTransformer::TStatus ParseJoinKeys(const TExprNode& side, TVector<std::pair<TStringBuf, TStringBuf>>& keys, - TVector<const TTypeAnnotationNode*>& keyTypes, const TJoinLabels& labels, - TExprContext& ctx, bool isCross) { - if (!EnsureTuple(side, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - for (auto& child : side.Children()) { - if (!EnsureAtom(*child, ctx)) { - return IGraphTransformer::TStatus::Error; - } - } - - if (isCross) { - if (side.ChildrenSize() != 0) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Expected empty list")); - return IGraphTransformer::TStatus::Error; - } - } else { - if (side.ChildrenSize() < 2 || (side.ChildrenSize() % 2) != 0) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Expected non-empty list of atoms with even length")); - return IGraphTransformer::TStatus::Error; - } - } - - keys.clear(); - for (ui32 i = 0; i < side.ChildrenSize(); i += 2) { - auto table = side.Child(i)->Content(); - auto column = side.Child(i + 1)->Content(); - auto key = std::make_pair(table, column); - keys.push_back(key); - } - - for (auto& key : keys) { - auto keyType = labels.FindColumn(key.first, key.second); - if (!keyType) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Unknown column: " << key.second << " in correlation name: " << key.first)); - return IGraphTransformer::TStatus::Error; - } - - if (!(*keyType)->IsHashable() || !(*keyType)->IsEquatable()) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Unsupported type of column: " << key.second << " in correlation name: " << key.first - << ", type: " << *(*keyType))); - return IGraphTransformer::TStatus::Error; - } - - keyTypes.push_back(*keyType); - } - - return IGraphTransformer::TStatus::Ok; - } - - struct TGLobalJoinState { - ui32 NestedJoins = 0; - }; - - IGraphTransformer::TStatus ParseJoins(const TJoinLabels& labels, - const TExprNode& joins, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, - TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx); - - bool AddEquiJoinLinkOptionHint(TStringBuf side, TSet<TStringBuf>& hints, const TExprNode& hintNode, TExprContext& ctx) { - if (!EnsureAtom(hintNode, ctx)) { - return false; - } - - auto hint = hintNode.Content(); - auto pos = ctx.GetPosition(hintNode.Pos()); - if (hint == "unique" || hint == "small") { - if (hints.contains(hint == "small" ? "unique" : "small")) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Hints 'unique' and 'small' are not compatible")); - return false; - } - hints.insert(hint); - } else if (hint == "any") { - hints.insert(hint); - } else { - ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown hint: '" << hint << "' for " << side << " side")); - return false; - } - - return true; - } - - IGraphTransformer::TStatus ParseJoinScope(const TJoinLabels& labels, - const TExprNode& side, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, - TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx) { - if (side.IsAtom()) { - auto label = side.Content(); - auto input = labels.FindInput(label); - if (!input) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Unknown correlation name: " << label)); - return IGraphTransformer::TStatus::Error; - } - - for (auto& x : (*input)->Tables) { - scope.insert(x); - } - - return IGraphTransformer::TStatus::Ok; - } - - if (globalState.NestedJoins + 2 == labels.Inputs.size()) { - ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), - TStringBuilder() << "Too many nested joins, expected exactly: " << (labels.Inputs.size() - 2))); - return IGraphTransformer::TStatus::Error; - } - - ++globalState.NestedJoins; - return ParseJoins(labels, side, joinsStates, scope, globalState, strictKeys, ctx); - } - - IGraphTransformer::TStatus ParseJoins(const TJoinLabels& labels, - const TExprNode& joins, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, - TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx) { - if (!EnsureTupleSize(joins, 6, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*joins.Child(0), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto joinType = joins.Child(0)->Content(); - if (joinType != "Inner" && joinType != "Left" && joinType != "Right" && joinType != "Full" - && joinType != "LeftOnly" && joinType != "RightOnly" && joinType != "Exclusion" - && joinType != "LeftSemi" && joinType != "RightSemi" && joinType != "Cross") { - ctx.AddError(TIssue(ctx.GetPosition(joins.Child(0)->Pos()), TStringBuilder() << "Unsupported join type: " << joinType)); - return IGraphTransformer::TStatus::Error; - } - - THashSet<TStringBuf> myLeftScope; - THashSet<TStringBuf> myRightScope; - auto status = ParseJoinScope(labels, *joins.Child(1), joinsStates, myLeftScope, globalState, strictKeys, ctx); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - status = ParseJoinScope(labels, *joins.Child(2), joinsStates, myRightScope, globalState, strictKeys, ctx); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - TVector<std::pair<TStringBuf, TStringBuf>> leftKeys; - TVector<const TTypeAnnotationNode*> leftKeyTypes; - status = ParseJoinKeys(*joins.Child(3), leftKeys, leftKeyTypes, labels, ctx, joinType == "Cross"); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - for (auto& x : leftKeys) { - if (!myLeftScope.contains(x.first)) { - ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), - TStringBuilder() << "Correlation name " << x.first << " is out of scope")); - return IGraphTransformer::TStatus::Error; - } - - auto leftIndex = *labels.FindInputIndex(x.first); - joinsStates[leftIndex].Used = true; - } - - TVector<std::pair<TStringBuf, TStringBuf>> rightKeys; - TVector<const TTypeAnnotationNode*> rightKeyTypes; - status = ParseJoinKeys(*joins.Child(4), rightKeys, rightKeyTypes, labels, ctx, joinType == "Cross"); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - for (auto& x : rightKeys) { - if (!myRightScope.contains(x.first)) { - ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), - TStringBuilder() << "Correlation name " << x.first << " is out of scope")); - return IGraphTransformer::TStatus::Error; - } - - auto rightIndex = *labels.FindInputIndex(x.first); - joinsStates[rightIndex].Used = true; - } - - if (leftKeys.size() != rightKeys.size()) { - ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), - TStringBuilder() << "Mismatch of key column count in equality between " << leftKeys[0].first - << " and " << rightKeys[0].first)); - return IGraphTransformer::TStatus::Error; - } - - for (ui32 i = 0; i < leftKeyTypes.size(); ++i) { - if (strictKeys && leftKeyTypes[i] != rightKeyTypes[i]) { - ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), - TStringBuilder() << "Strict key type match requested, but keys have different types: (" - << leftKeys[i].first << "." << leftKeys[i].second - << " has type: " << *leftKeyTypes[i] << ", " << rightKeys[i].first << "." << rightKeys[i].second - << " has type: " << *rightKeyTypes[i] << ")")); - return IGraphTransformer::TStatus::Error; - } + + struct TJoinState { + bool Used = false; + }; + + IGraphTransformer::TStatus ParseJoinKeys(const TExprNode& side, TVector<std::pair<TStringBuf, TStringBuf>>& keys, + TVector<const TTypeAnnotationNode*>& keyTypes, const TJoinLabels& labels, + TExprContext& ctx, bool isCross) { + if (!EnsureTuple(side, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + for (auto& child : side.Children()) { + if (!EnsureAtom(*child, ctx)) { + return IGraphTransformer::TStatus::Error; + } + } + + if (isCross) { + if (side.ChildrenSize() != 0) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Expected empty list")); + return IGraphTransformer::TStatus::Error; + } + } else { + if (side.ChildrenSize() < 2 || (side.ChildrenSize() % 2) != 0) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Expected non-empty list of atoms with even length")); + return IGraphTransformer::TStatus::Error; + } + } + + keys.clear(); + for (ui32 i = 0; i < side.ChildrenSize(); i += 2) { + auto table = side.Child(i)->Content(); + auto column = side.Child(i + 1)->Content(); + auto key = std::make_pair(table, column); + keys.push_back(key); + } + + for (auto& key : keys) { + auto keyType = labels.FindColumn(key.first, key.second); + if (!keyType) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Unknown column: " << key.second << " in correlation name: " << key.first)); + return IGraphTransformer::TStatus::Error; + } + + if (!(*keyType)->IsHashable() || !(*keyType)->IsEquatable()) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Unsupported type of column: " << key.second << " in correlation name: " << key.first + << ", type: " << *(*keyType))); + return IGraphTransformer::TStatus::Error; + } + + keyTypes.push_back(*keyType); + } + + return IGraphTransformer::TStatus::Ok; + } + + struct TGLobalJoinState { + ui32 NestedJoins = 0; + }; + + IGraphTransformer::TStatus ParseJoins(const TJoinLabels& labels, + const TExprNode& joins, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, + TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx); + + bool AddEquiJoinLinkOptionHint(TStringBuf side, TSet<TStringBuf>& hints, const TExprNode& hintNode, TExprContext& ctx) { + if (!EnsureAtom(hintNode, ctx)) { + return false; + } + + auto hint = hintNode.Content(); + auto pos = ctx.GetPosition(hintNode.Pos()); + if (hint == "unique" || hint == "small") { + if (hints.contains(hint == "small" ? "unique" : "small")) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Hints 'unique' and 'small' are not compatible")); + return false; + } + hints.insert(hint); + } else if (hint == "any") { + hints.insert(hint); + } else { + ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown hint: '" << hint << "' for " << side << " side")); + return false; + } + + return true; + } + + IGraphTransformer::TStatus ParseJoinScope(const TJoinLabels& labels, + const TExprNode& side, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, + TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx) { + if (side.IsAtom()) { + auto label = side.Content(); + auto input = labels.FindInput(label); + if (!input) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Unknown correlation name: " << label)); + return IGraphTransformer::TStatus::Error; + } + + for (auto& x : (*input)->Tables) { + scope.insert(x); + } + + return IGraphTransformer::TStatus::Ok; + } + + if (globalState.NestedJoins + 2 == labels.Inputs.size()) { + ctx.AddError(TIssue(ctx.GetPosition(side.Pos()), + TStringBuilder() << "Too many nested joins, expected exactly: " << (labels.Inputs.size() - 2))); + return IGraphTransformer::TStatus::Error; + } + + ++globalState.NestedJoins; + return ParseJoins(labels, side, joinsStates, scope, globalState, strictKeys, ctx); + } + + IGraphTransformer::TStatus ParseJoins(const TJoinLabels& labels, + const TExprNode& joins, TVector<TJoinState>& joinsStates, THashSet<TStringBuf>& scope, + TGLobalJoinState& globalState, bool strictKeys, TExprContext& ctx) { + if (!EnsureTupleSize(joins, 6, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*joins.Child(0), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto joinType = joins.Child(0)->Content(); + if (joinType != "Inner" && joinType != "Left" && joinType != "Right" && joinType != "Full" + && joinType != "LeftOnly" && joinType != "RightOnly" && joinType != "Exclusion" + && joinType != "LeftSemi" && joinType != "RightSemi" && joinType != "Cross") { + ctx.AddError(TIssue(ctx.GetPosition(joins.Child(0)->Pos()), TStringBuilder() << "Unsupported join type: " << joinType)); + return IGraphTransformer::TStatus::Error; + } + + THashSet<TStringBuf> myLeftScope; + THashSet<TStringBuf> myRightScope; + auto status = ParseJoinScope(labels, *joins.Child(1), joinsStates, myLeftScope, globalState, strictKeys, ctx); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + status = ParseJoinScope(labels, *joins.Child(2), joinsStates, myRightScope, globalState, strictKeys, ctx); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + TVector<std::pair<TStringBuf, TStringBuf>> leftKeys; + TVector<const TTypeAnnotationNode*> leftKeyTypes; + status = ParseJoinKeys(*joins.Child(3), leftKeys, leftKeyTypes, labels, ctx, joinType == "Cross"); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + for (auto& x : leftKeys) { + if (!myLeftScope.contains(x.first)) { + ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), + TStringBuilder() << "Correlation name " << x.first << " is out of scope")); + return IGraphTransformer::TStatus::Error; + } + + auto leftIndex = *labels.FindInputIndex(x.first); + joinsStates[leftIndex].Used = true; + } + + TVector<std::pair<TStringBuf, TStringBuf>> rightKeys; + TVector<const TTypeAnnotationNode*> rightKeyTypes; + status = ParseJoinKeys(*joins.Child(4), rightKeys, rightKeyTypes, labels, ctx, joinType == "Cross"); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + for (auto& x : rightKeys) { + if (!myRightScope.contains(x.first)) { + ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), + TStringBuilder() << "Correlation name " << x.first << " is out of scope")); + return IGraphTransformer::TStatus::Error; + } + + auto rightIndex = *labels.FindInputIndex(x.first); + joinsStates[rightIndex].Used = true; + } + + if (leftKeys.size() != rightKeys.size()) { + ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), + TStringBuilder() << "Mismatch of key column count in equality between " << leftKeys[0].first + << " and " << rightKeys[0].first)); + return IGraphTransformer::TStatus::Error; + } + + for (ui32 i = 0; i < leftKeyTypes.size(); ++i) { + if (strictKeys && leftKeyTypes[i] != rightKeyTypes[i]) { + ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), + TStringBuilder() << "Strict key type match requested, but keys have different types: (" + << leftKeys[i].first << "." << leftKeys[i].second + << " has type: " << *leftKeyTypes[i] << ", " << rightKeys[i].first << "." << rightKeys[i].second + << " has type: " << *rightKeyTypes[i] << ")")); + return IGraphTransformer::TStatus::Error; + } if (ECompareOptions::Uncomparable == CanCompare<true>(leftKeyTypes[i], rightKeyTypes[i])) { - ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), + ctx.AddError(TIssue(ctx.GetPosition(joins.Pos()), TStringBuilder() << "Cannot compare key columns (" << leftKeys[i].first << "." << leftKeys[i].second - << " has type: " << *leftKeyTypes[i] << ", " << rightKeys[i].first << "." << rightKeys[i].second - << " has type: " << *rightKeyTypes[i] << ")")); - return IGraphTransformer::TStatus::Error; - } - } - - if (joinType == "Cross") { - for (auto& x : myLeftScope) { - auto leftIndex = *labels.FindInputIndex(x); - joinsStates[leftIndex].Used = true; - } - - for (auto& x : myRightScope) { - auto rightIndex = *labels.FindInputIndex(x); - joinsStates[rightIndex].Used = true; - } - } - - scope.clear(); - if (joinType != "RightOnly" && joinType != "RightSemi") { - scope.insert(myLeftScope.begin(), myLeftScope.end()); - } - - if (joinType != "LeftOnly" && joinType != "LeftSemi") { - scope.insert(myRightScope.begin(), myRightScope.end()); - } - - auto linkOptions = joins.Child(5); - if (!EnsureTuple(*linkOptions, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - TMaybe<TSet<TStringBuf>> leftHints; - TMaybe<TSet<TStringBuf>> rightHints; - bool forceSortedMerge = false; - for (auto child : linkOptions->Children()) { - if (!EnsureTupleMinSize(*child, 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*child->Child(0), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto optionName = child->Child(0)->Content(); - if (optionName == "left" || optionName == "right") { - if (!EnsureTupleSize(*child, 2, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto& hints = optionName == "left" ? leftHints : rightHints; - if (hints) { - ctx.AddError(TIssue(ctx.GetPosition(child->Child(0)->Pos()), TStringBuilder() << - "Duplication of hints for " << optionName << " side")); - return IGraphTransformer::TStatus::Error; - } - - hints.ConstructInPlace(); - if (child->Child(1)->IsAtom()) { - if (!AddEquiJoinLinkOptionHint(optionName, *hints, *child->Child(1), ctx)) { - return IGraphTransformer::TStatus::Error; - } - } else { - if (!EnsureTuple(*child->Child(1), ctx)) { - return IGraphTransformer::TStatus::Error; - } - for (auto hint : child->Child(1)->Children()) { - if (!AddEquiJoinLinkOptionHint(optionName, *hints, *hint, ctx)) { - return IGraphTransformer::TStatus::Error; - } - } - } - } - else if (optionName == "forceSortedMerge") { - if (!EnsureTupleSize(*child, 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } - if (forceSortedMerge) { - ctx.AddError(TIssue(ctx.GetPosition(child->Child(0)->Pos()), TStringBuilder() << - "Duplicate " << optionName << " link option")); - return IGraphTransformer::TStatus::Error; - } - forceSortedMerge = true; - } - else { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << - "Unknown option name: " << optionName)); - return IGraphTransformer::TStatus::Error; - } - } - - return IGraphTransformer::TStatus::Ok; - } - - struct TFlattenState { - TString Table; - TVector<const TTypeAnnotationNode*> AllTypes; - }; - - void CollectEquiJoinKeyColumnsFromLeaf(const TExprNode& columns, THashMap<TStringBuf, THashSet<TStringBuf>>& tableKeysMap) { - YQL_ENSURE(columns.ChildrenSize() % 2 == 0); - for (ui32 i = 0; i < columns.ChildrenSize(); i += 2) { - auto table = columns.Child(i)->Content(); - auto column = columns.Child(i + 1)->Content(); - tableKeysMap[table].insert(column); - } - } - - void CollectEquiJoinKeyColumns(const TExprNode& joinTree, THashMap<TStringBuf, THashSet<TStringBuf>>& tableKeysMap) { - auto& left = *joinTree.Child(1); - if (!left.IsAtom()) { - CollectEquiJoinKeyColumns(left, tableKeysMap); - } - - auto& right = *joinTree.Child(2); - if (!right.IsAtom()) { - CollectEquiJoinKeyColumns(right, tableKeysMap); - } - - CollectEquiJoinKeyColumnsFromLeaf(*joinTree.Child(3), tableKeysMap); - CollectEquiJoinKeyColumnsFromLeaf(*joinTree.Child(4), tableKeysMap); - } - - bool CollectEquiJoinOnlyParents(const TExprNode& current, const TExprNode* prev, ui32 depth, - TVector<TEquiJoinParent>& results, const TExprNode* extractMembersInScope, - const TParentsMap& parents) - { - if (depth == 0) { - if (!prev || !TCoEquiJoin::Match(¤t)) { - return false; - } - - TCoEquiJoin equiJoin(¤t); - for (ui32 i = 0; i + 2 < equiJoin.ArgCount(); ++i) { - auto joinInput = equiJoin.Arg(i).Cast<TCoEquiJoinInput>(); - auto list = joinInput.List(); - if (list.Raw() == prev) { - results.emplace_back(equiJoin.Raw(), i, extractMembersInScope); - return true; - } - } - YQL_ENSURE(false, "Unable to locate FlatMap in EquiJoin"); - } - - auto it = parents.find(¤t); - if (it == parents.end() || it->second.empty()) { - return false; - } - - const TExprNode* extractMembers = extractMembersInScope; - bool currentIsExtractMembers = TCoExtractMembers::Match(¤t); - if (currentIsExtractMembers) { - if (extractMembers) { - // repeatable extract members should not actually happen - return false; - } - extractMembers = current.Child(1); - } - - auto nextPrev = (TCoFlatMapBase::Match(¤t) || currentIsExtractMembers) ? ¤t : prev; - - for (auto parent : it->second) { - if (!CollectEquiJoinOnlyParents(*parent, nextPrev, currentIsExtractMembers ? depth : (depth - 1), results, - extractMembers, parents)) - { - return false; - } - } - - return true; - } + << " has type: " << *leftKeyTypes[i] << ", " << rightKeys[i].first << "." << rightKeys[i].second + << " has type: " << *rightKeyTypes[i] << ")")); + return IGraphTransformer::TStatus::Error; + } + } + + if (joinType == "Cross") { + for (auto& x : myLeftScope) { + auto leftIndex = *labels.FindInputIndex(x); + joinsStates[leftIndex].Used = true; + } + + for (auto& x : myRightScope) { + auto rightIndex = *labels.FindInputIndex(x); + joinsStates[rightIndex].Used = true; + } + } + + scope.clear(); + if (joinType != "RightOnly" && joinType != "RightSemi") { + scope.insert(myLeftScope.begin(), myLeftScope.end()); + } + + if (joinType != "LeftOnly" && joinType != "LeftSemi") { + scope.insert(myRightScope.begin(), myRightScope.end()); + } + + auto linkOptions = joins.Child(5); + if (!EnsureTuple(*linkOptions, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + TMaybe<TSet<TStringBuf>> leftHints; + TMaybe<TSet<TStringBuf>> rightHints; + bool forceSortedMerge = false; + for (auto child : linkOptions->Children()) { + if (!EnsureTupleMinSize(*child, 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*child->Child(0), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto optionName = child->Child(0)->Content(); + if (optionName == "left" || optionName == "right") { + if (!EnsureTupleSize(*child, 2, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto& hints = optionName == "left" ? leftHints : rightHints; + if (hints) { + ctx.AddError(TIssue(ctx.GetPosition(child->Child(0)->Pos()), TStringBuilder() << + "Duplication of hints for " << optionName << " side")); + return IGraphTransformer::TStatus::Error; + } + + hints.ConstructInPlace(); + if (child->Child(1)->IsAtom()) { + if (!AddEquiJoinLinkOptionHint(optionName, *hints, *child->Child(1), ctx)) { + return IGraphTransformer::TStatus::Error; + } + } else { + if (!EnsureTuple(*child->Child(1), ctx)) { + return IGraphTransformer::TStatus::Error; + } + for (auto hint : child->Child(1)->Children()) { + if (!AddEquiJoinLinkOptionHint(optionName, *hints, *hint, ctx)) { + return IGraphTransformer::TStatus::Error; + } + } + } + } + else if (optionName == "forceSortedMerge") { + if (!EnsureTupleSize(*child, 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } + if (forceSortedMerge) { + ctx.AddError(TIssue(ctx.GetPosition(child->Child(0)->Pos()), TStringBuilder() << + "Duplicate " << optionName << " link option")); + return IGraphTransformer::TStatus::Error; + } + forceSortedMerge = true; + } + else { + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << + "Unknown option name: " << optionName)); + return IGraphTransformer::TStatus::Error; + } + } + + return IGraphTransformer::TStatus::Ok; + } + + struct TFlattenState { + TString Table; + TVector<const TTypeAnnotationNode*> AllTypes; + }; + + void CollectEquiJoinKeyColumnsFromLeaf(const TExprNode& columns, THashMap<TStringBuf, THashSet<TStringBuf>>& tableKeysMap) { + YQL_ENSURE(columns.ChildrenSize() % 2 == 0); + for (ui32 i = 0; i < columns.ChildrenSize(); i += 2) { + auto table = columns.Child(i)->Content(); + auto column = columns.Child(i + 1)->Content(); + tableKeysMap[table].insert(column); + } + } + + void CollectEquiJoinKeyColumns(const TExprNode& joinTree, THashMap<TStringBuf, THashSet<TStringBuf>>& tableKeysMap) { + auto& left = *joinTree.Child(1); + if (!left.IsAtom()) { + CollectEquiJoinKeyColumns(left, tableKeysMap); + } + + auto& right = *joinTree.Child(2); + if (!right.IsAtom()) { + CollectEquiJoinKeyColumns(right, tableKeysMap); + } + + CollectEquiJoinKeyColumnsFromLeaf(*joinTree.Child(3), tableKeysMap); + CollectEquiJoinKeyColumnsFromLeaf(*joinTree.Child(4), tableKeysMap); + } + + bool CollectEquiJoinOnlyParents(const TExprNode& current, const TExprNode* prev, ui32 depth, + TVector<TEquiJoinParent>& results, const TExprNode* extractMembersInScope, + const TParentsMap& parents) + { + if (depth == 0) { + if (!prev || !TCoEquiJoin::Match(¤t)) { + return false; + } + + TCoEquiJoin equiJoin(¤t); + for (ui32 i = 0; i + 2 < equiJoin.ArgCount(); ++i) { + auto joinInput = equiJoin.Arg(i).Cast<TCoEquiJoinInput>(); + auto list = joinInput.List(); + if (list.Raw() == prev) { + results.emplace_back(equiJoin.Raw(), i, extractMembersInScope); + return true; + } + } + YQL_ENSURE(false, "Unable to locate FlatMap in EquiJoin"); + } + + auto it = parents.find(¤t); + if (it == parents.end() || it->second.empty()) { + return false; + } + + const TExprNode* extractMembers = extractMembersInScope; + bool currentIsExtractMembers = TCoExtractMembers::Match(¤t); + if (currentIsExtractMembers) { + if (extractMembers) { + // repeatable extract members should not actually happen + return false; + } + extractMembers = current.Child(1); + } + + auto nextPrev = (TCoFlatMapBase::Match(¤t) || currentIsExtractMembers) ? ¤t : prev; + + for (auto parent : it->second) { + if (!CollectEquiJoinOnlyParents(*parent, nextPrev, currentIsExtractMembers ? depth : (depth - 1), results, + extractMembers, parents)) + { + return false; + } + } + + return true; + } } -TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStructExprType* structType) { +TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStructExprType* structType) { Tables.clear(); InputType = structType; if (auto atom = TMaybeNode<TCoAtom>(&node)) { - if (auto err = ValidateLabel(ctx, atom.Cast())) { + if (auto err = ValidateLabel(ctx, atom.Cast())) { return err; } @@ -411,11 +411,11 @@ TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStru } else if (auto tuple = TMaybeNode<TCoAtomList>(&node)) { if (tuple.Cast().Size() == 0) { - return TIssue(ctx.GetPosition(node.Pos()), "Empty list of correlation names are not allowed"); + return TIssue(ctx.GetPosition(node.Pos()), "Empty list of correlation names are not allowed"); } for (const auto& child : tuple.Cast()) { - if (auto err = ValidateLabel(ctx, child)) { + if (auto err = ValidateLabel(ctx, child)) { return err; } @@ -426,7 +426,7 @@ TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStru auto prevLabel = Tables[0]; for (ui32 i = 1; i < Tables.size(); ++i) { if (Tables[i] == prevLabel) { - return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Duplication of correlation names: " << prevLabel); + return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Duplication of correlation names: " << prevLabel); } prevLabel = Tables[i]; @@ -437,34 +437,34 @@ TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStru auto name = column->GetName(); auto pos = name.find('.'); if (pos == TString::npos) { - return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected columns name as table.name, but got: " << name); + return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected columns name as table.name, but got: " << name); } auto table = name.substr(0, pos); if (!BinarySearch(Tables.begin(), Tables.end(), table)) { - return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unknown table name: " << table); + return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unknown table name: " << table); } auto columnName = name.substr(pos + 1); if (columnName.empty()) { - return TIssue(ctx.GetPosition(node.Pos()), "Empty correlation name is not allowed"); + return TIssue(ctx.GetPosition(node.Pos()), "Empty correlation name is not allowed"); } } return {}; } else { - return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either atom or list, but got" << node.Type()); + return TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either atom or list, but got" << node.Type()); } } -TMaybe<TIssue> TJoinLabel::ValidateLabel(TExprContext& ctx, const TCoAtom& label) { +TMaybe<TIssue> TJoinLabel::ValidateLabel(TExprContext& ctx, const TCoAtom& label) { if (label.Value().empty()) { - return TIssue(ctx.GetPosition(label.Pos()), "Empty correlation name is not allowed"); + return TIssue(ctx.GetPosition(label.Pos()), "Empty correlation name is not allowed"); } if (label.Value().Contains('.')) { - return TIssue(ctx.GetPosition(label.Pos()), "Dot symbol is not allowed in the correlation name"); + return TIssue(ctx.GetPosition(label.Pos()), "Dot symbol is not allowed in the correlation name"); } return {}; @@ -539,18 +539,18 @@ TVector<TString> TJoinLabel::EnumerateAllMembers() const { return result; } -TMaybe<TIssue> TJoinLabels::Add(TExprContext& ctx, TExprNode& node, const TStructExprType* structType) { +TMaybe<TIssue> TJoinLabels::Add(TExprContext& ctx, TExprNode& node, const TStructExprType* structType) { ui32 index = Inputs.size(); Inputs.emplace_back(); TJoinLabel& label = Inputs.back(); - if (auto err = label.Parse(ctx, node, structType)) { + if (auto err = label.Parse(ctx, node, structType)) { return err; } for (auto& table : label.Tables) { if (!InputByTable.insert({ table, index }).second) { return TIssue( - ctx.GetPosition(node.Pos()), + ctx.GetPosition(node.Pos()), TStringBuilder() << "Duplication of table name " << table); } } @@ -618,95 +618,95 @@ TVector<TString> TJoinLabels::EnumerateColumns(const TStringBuf& table) const { return result; } -IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandle, const TExprNode& optionsNode, - TJoinOptions& options, TExprContext& ctx) +IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandle, const TExprNode& optionsNode, + TJoinOptions& options, TExprContext& ctx) { - auto position = ctx.GetPosition(positionHandle); - if (!EnsureTuple(optionsNode, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - options = TJoinOptions{}; - - THashSet<TStringBuf> renameTargetSet; - bool hasRename = false; - for (auto child : optionsNode.Children()) { - if (!EnsureTupleMinSize(*child, 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*child->Child(0), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto optionName = child->Child(0)->Content(); - if (optionName == "rename") { - hasRename = true; - if (!EnsureTupleSize(*child, 3, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*child->Child(1), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - if (!EnsureAtom(*child->Child(2), ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto& v = options.RenameMap[child->Child(1)->Content()]; - if (!child->Child(2)->Content().empty()) { - if (!renameTargetSet.insert(child->Child(2)->Content()).second) { - ctx.AddError(TIssue(position, TStringBuilder() << - "Duplicated target column: " << child->Child(2)->Content())); - return IGraphTransformer::TStatus::Error; - } - - v.push_back(child->Child(2)->Content()); - } - } else if (optionName == "flatten") { - options.Flatten = true; + auto position = ctx.GetPosition(positionHandle); + if (!EnsureTuple(optionsNode, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + options = TJoinOptions{}; + + THashSet<TStringBuf> renameTargetSet; + bool hasRename = false; + for (auto child : optionsNode.Children()) { + if (!EnsureTupleMinSize(*child, 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*child->Child(0), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto optionName = child->Child(0)->Content(); + if (optionName == "rename") { + hasRename = true; + if (!EnsureTupleSize(*child, 3, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*child->Child(1), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(*child->Child(2), ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto& v = options.RenameMap[child->Child(1)->Content()]; + if (!child->Child(2)->Content().empty()) { + if (!renameTargetSet.insert(child->Child(2)->Content()).second) { + ctx.AddError(TIssue(position, TStringBuilder() << + "Duplicated target column: " << child->Child(2)->Content())); + return IGraphTransformer::TStatus::Error; + } + + v.push_back(child->Child(2)->Content()); + } + } else if (optionName == "flatten") { + options.Flatten = true; } else if (optionName == "keep_sys") { - options.KeepSysColumns = true; - } else if (optionName == "strict_keys") { - options.StrictKeys = true; - } else if (optionName == "preferred_sort") { - THashSet<TStringBuf> sortBySet; - TVector<TStringBuf> sortBy; - if (!EnsureTupleSize(*child, 2, ctx)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureTupleMinSize(*child->Child(1), 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } - for (auto column : child->Child(1)->Children()) { - if (!EnsureAtom(*column, ctx)) { - return IGraphTransformer::TStatus::Error; - } - if (!sortBySet.insert(column->Content()).second) { - ctx.AddError(TIssue(ctx.GetPosition(column->Pos()), TStringBuilder() << - "Duplicated preferred_sort column: " << column->Content())); - return IGraphTransformer::TStatus::Error; - } - sortBy.push_back(column->Content()); - } - if (!options.PreferredSortSets.insert(sortBy).second) { - ctx.AddError(TIssue(ctx.GetPosition(child->Child(1)->Pos()), TStringBuilder() << - "Duplicated preferred_sort set: " << JoinSeq(", ", sortBy))); - } - } else { - ctx.AddError(TIssue(position, TStringBuilder() << - "Unknown option name: " << optionName)); - return IGraphTransformer::TStatus::Error; - } - - if (hasRename && options.Flatten) { - ctx.AddError(TIssue(position, TStringBuilder() << - "Options flatten and rename are incompatible with each other")); - return IGraphTransformer::TStatus::Error; - } - } - + options.KeepSysColumns = true; + } else if (optionName == "strict_keys") { + options.StrictKeys = true; + } else if (optionName == "preferred_sort") { + THashSet<TStringBuf> sortBySet; + TVector<TStringBuf> sortBy; + if (!EnsureTupleSize(*child, 2, ctx)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureTupleMinSize(*child->Child(1), 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } + for (auto column : child->Child(1)->Children()) { + if (!EnsureAtom(*column, ctx)) { + return IGraphTransformer::TStatus::Error; + } + if (!sortBySet.insert(column->Content()).second) { + ctx.AddError(TIssue(ctx.GetPosition(column->Pos()), TStringBuilder() << + "Duplicated preferred_sort column: " << column->Content())); + return IGraphTransformer::TStatus::Error; + } + sortBy.push_back(column->Content()); + } + if (!options.PreferredSortSets.insert(sortBy).second) { + ctx.AddError(TIssue(ctx.GetPosition(child->Child(1)->Pos()), TStringBuilder() << + "Duplicated preferred_sort set: " << JoinSeq(", ", sortBy))); + } + } else { + ctx.AddError(TIssue(position, TStringBuilder() << + "Unknown option name: " << optionName)); + return IGraphTransformer::TStatus::Error; + } + + if (hasRename && options.Flatten) { + ctx.AddError(TIssue(position, TStringBuilder() << + "Options flatten and rename are incompatible with each other")); + return IGraphTransformer::TStatus::Error; + } + } + return IGraphTransformer::TStatus::Ok; } @@ -715,131 +715,131 @@ IGraphTransformer::TStatus EquiJoinAnnotation( const TStructExprType*& resultType, const TJoinLabels& labels, const TExprNode& joins, - const TJoinOptions& options, + const TJoinOptions& options, TExprContext& ctx ) { auto position = ctx.GetPosition(positionHandle); - if (labels.InputByTable.size() < 2) { - ctx.AddError(TIssue(position, TStringBuilder() << "Expected at least 2 table")); - return IGraphTransformer::TStatus::Error; - } - - TVector<TJoinState> joinsStates(labels.Inputs.size()); - TGLobalJoinState globalState; - THashSet<TStringBuf> scope; - auto parseStatus = ParseJoins(labels, joins, joinsStates, scope, globalState, options.StrictKeys, ctx); - if (parseStatus.Level != IGraphTransformer::TStatus::Ok) { - return parseStatus; - } - - if (globalState.NestedJoins + 2 != labels.Inputs.size()) { - ctx.AddError(TIssue(position, - TStringBuilder() << "Too few nested joins, expected exactly: " << (labels.Inputs.size() - 2))); - return IGraphTransformer::TStatus::Error; - } - - for (ui32 i = 0; i < joinsStates.size(); ++i) { - if (!joinsStates[i].Used) { - ctx.AddError(TIssue(position, TStringBuilder() << - "Input with correlation name(s) " << JoinSeq(", ", labels.Inputs[i].Tables) << " was not used")); - return IGraphTransformer::TStatus::Error; - } - } - - auto columnTypes = GetJoinColumnTypes(joins, labels, ctx); - TVector<const TItemExprType*> resultFields; - TMap<TString, TFlattenState> flattenFields; // column -> table + if (labels.InputByTable.size() < 2) { + ctx.AddError(TIssue(position, TStringBuilder() << "Expected at least 2 table")); + return IGraphTransformer::TStatus::Error; + } + + TVector<TJoinState> joinsStates(labels.Inputs.size()); + TGLobalJoinState globalState; + THashSet<TStringBuf> scope; + auto parseStatus = ParseJoins(labels, joins, joinsStates, scope, globalState, options.StrictKeys, ctx); + if (parseStatus.Level != IGraphTransformer::TStatus::Ok) { + return parseStatus; + } + + if (globalState.NestedJoins + 2 != labels.Inputs.size()) { + ctx.AddError(TIssue(position, + TStringBuilder() << "Too few nested joins, expected exactly: " << (labels.Inputs.size() - 2))); + return IGraphTransformer::TStatus::Error; + } + + for (ui32 i = 0; i < joinsStates.size(); ++i) { + if (!joinsStates[i].Used) { + ctx.AddError(TIssue(position, TStringBuilder() << + "Input with correlation name(s) " << JoinSeq(", ", labels.Inputs[i].Tables) << " was not used")); + return IGraphTransformer::TStatus::Error; + } + } + + auto columnTypes = GetJoinColumnTypes(joins, labels, ctx); + TVector<const TItemExprType*> resultFields; + TMap<TString, TFlattenState> flattenFields; // column -> table THashSet<TString> processedRenames; - for (auto it: labels.Inputs) { - for (auto item: it.InputType->GetItems()) { - TString fullName = it.FullName(item->GetName()); - auto type = columnTypes.FindPtr(fullName); - if (type) { - TVector<TStringBuf> fullNames; - fullNames.push_back(fullName); + for (auto it: labels.Inputs) { + for (auto item: it.InputType->GetItems()) { + TString fullName = it.FullName(item->GetName()); + auto type = columnTypes.FindPtr(fullName); + if (type) { + TVector<TStringBuf> fullNames; + fullNames.push_back(fullName); if (!processedRenames.contains(fullName)) { - auto renameIt = options.RenameMap.find(fullName); - if (renameIt != options.RenameMap.end()) { + auto renameIt = options.RenameMap.find(fullName); + if (renameIt != options.RenameMap.end()) { fullNames = renameIt->second; processedRenames.insert(fullName); } - } - - for (auto& fullName: fullNames) { - if (options.Flatten) { - auto tableName = it.TableName(fullName); - auto columnName = it.ColumnName(fullName); - auto iter = flattenFields.find(columnName); - if (iter != flattenFields.end()) { - if (AreSameJoinKeys(joins, tableName, columnName, iter->second.Table, columnName)) { - iter->second.AllTypes.push_back(*type); - continue; - } - - ctx.AddError(TIssue(position, TStringBuilder() << - "Conflict of flattening output on columns " << fullName << " and " << iter->second.Table - << "." << columnName)); - return IGraphTransformer::TStatus::Error; - } - - TFlattenState state; - state.AllTypes.push_back(*type); - state.Table = TString(tableName); - flattenFields.emplace(TString(columnName), state); - } else { - resultFields.push_back(ctx.MakeType<TItemExprType>(fullName, *type)); - } - } - } - } - } - - if (options.Flatten) { - for (auto& x : flattenFields) { - bool isOptional = true; - const TTypeAnnotationNode* commonType = nullptr; - for (auto type : x.second.AllTypes) { - if (type->GetKind() != ETypeAnnotationKind::Optional) { - isOptional = false; - } else { - type = type->Cast<TOptionalExprType>()->GetItemType(); - } - - if (!commonType) { - commonType = type; - } else { - auto arg1 = ctx.NewArgument(positionHandle, "a"); - auto arg2 = ctx.NewArgument(positionHandle, "b"); + } + + for (auto& fullName: fullNames) { + if (options.Flatten) { + auto tableName = it.TableName(fullName); + auto columnName = it.ColumnName(fullName); + auto iter = flattenFields.find(columnName); + if (iter != flattenFields.end()) { + if (AreSameJoinKeys(joins, tableName, columnName, iter->second.Table, columnName)) { + iter->second.AllTypes.push_back(*type); + continue; + } + + ctx.AddError(TIssue(position, TStringBuilder() << + "Conflict of flattening output on columns " << fullName << " and " << iter->second.Table + << "." << columnName)); + return IGraphTransformer::TStatus::Error; + } + + TFlattenState state; + state.AllTypes.push_back(*type); + state.Table = TString(tableName); + flattenFields.emplace(TString(columnName), state); + } else { + resultFields.push_back(ctx.MakeType<TItemExprType>(fullName, *type)); + } + } + } + } + } + + if (options.Flatten) { + for (auto& x : flattenFields) { + bool isOptional = true; + const TTypeAnnotationNode* commonType = nullptr; + for (auto type : x.second.AllTypes) { + if (type->GetKind() != ETypeAnnotationKind::Optional) { + isOptional = false; + } else { + type = type->Cast<TOptionalExprType>()->GetItemType(); + } + + if (!commonType) { + commonType = type; + } else { + auto arg1 = ctx.NewArgument(positionHandle, "a"); + auto arg2 = ctx.NewArgument(positionHandle, "b"); if (SilentInferCommonType(arg1, *commonType, arg2, *type, ctx, commonType, TConvertFlags().Set(NConvertFlags::AllowUnsafeConvert)) == IGraphTransformer::TStatus::Error) { - return IGraphTransformer::TStatus::Error; - } - } - } - - if (isOptional) { - commonType = ctx.MakeType<TOptionalExprType>(commonType); - } - - resultFields.push_back(ctx.MakeType<TItemExprType>(x.first, commonType)); - } - } - - resultType = ctx.MakeType<TStructExprType>(resultFields); - if (!resultType->Validate(position, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - return IGraphTransformer::TStatus::Ok; -} - -THashMap<TStringBuf, THashSet<TStringBuf>> CollectEquiJoinKeyColumnsByLabel(const TExprNode& joinTree) { - THashMap<TStringBuf, THashSet<TStringBuf>> result; - CollectEquiJoinKeyColumns(joinTree, result); - return result; -}; - + return IGraphTransformer::TStatus::Error; + } + } + } + + if (isOptional) { + commonType = ctx.MakeType<TOptionalExprType>(commonType); + } + + resultFields.push_back(ctx.MakeType<TItemExprType>(x.first, commonType)); + } + } + + resultType = ctx.MakeType<TStructExprType>(resultFields); + if (!resultType->Validate(position, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + return IGraphTransformer::TStatus::Ok; +} + +THashMap<TStringBuf, THashSet<TStringBuf>> CollectEquiJoinKeyColumnsByLabel(const TExprNode& joinTree) { + THashMap<TStringBuf, THashSet<TStringBuf>> result; + CollectEquiJoinKeyColumns(joinTree, result); + return result; +}; + bool IsLeftJoinSideOptional(const TStringBuf& joinType) { if (joinType == "Right" || joinType == "Full" || joinType == "Exclusion") { return true; @@ -856,7 +856,7 @@ bool IsRightJoinSideOptional(const TStringBuf& joinType) { return false; } -TExprNode::TPtr FilterOutNullJoinColumns(TPositionHandle pos, const TExprNode::TPtr& input, +TExprNode::TPtr FilterOutNullJoinColumns(TPositionHandle pos, const TExprNode::TPtr& input, const TJoinLabel& label, const TSet<TString>& optionalKeyColumns, TExprContext& ctx) { if (optionalKeyColumns.empty()) { return input; @@ -894,20 +894,20 @@ TMap<TStringBuf, TVector<TStringBuf>> LoadJoinRenameMap(const TExprNode& setting return res; } -TSet<TVector<TStringBuf>> LoadJoinSortSets(const TExprNode& settings) { - TSet<TVector<TStringBuf>> res; - for (const auto& child : settings.Children()) { - if (child->Child(0)->Content() == "preferred_sort") { - TVector<TStringBuf> sortBy; - for (auto column : child->Child(1)->Children()) { - sortBy.push_back(column->Content()); - } - res.insert(sortBy); - } - } - return res; -} - +TSet<TVector<TStringBuf>> LoadJoinSortSets(const TExprNode& settings) { + TSet<TVector<TStringBuf>> res; + for (const auto& child : settings.Children()) { + if (child->Child(0)->Content() == "preferred_sort") { + TVector<TStringBuf> sortBy; + for (auto column : child->Child(1)->Children()) { + sortBy.push_back(column->Content()); + } + res.insert(sortBy); + } + } + return res; +} + THashMap<TString, const TTypeAnnotationNode*> GetJoinColumnTypes(const TExprNode& joins, const TJoinLabels& labels, TExprContext& ctx) { return GetJoinColumnTypes(joins, labels, joins.Child(0)->Content(), ctx); @@ -1039,7 +1039,7 @@ std::pair<bool, bool> IsRequiredSide(const TExprNode::TPtr& joinTree, const TJoi return{ false, false }; } -void AppendEquiJoinRenameMap(TPositionHandle pos, const TMap<TStringBuf, TVector<TStringBuf>>& newRenameMap, +void AppendEquiJoinRenameMap(TPositionHandle pos, const TMap<TStringBuf, TVector<TStringBuf>>& newRenameMap, TExprNode::TListType& joinSettingNodes, TExprContext& ctx) { for (auto& x : newRenameMap) { if (x.second.empty()) { @@ -1069,27 +1069,27 @@ void AppendEquiJoinRenameMap(TPositionHandle pos, const TMap<TStringBuf, TVector } } -void AppendEquiJoinSortSets(TPositionHandle pos, const TSet<TVector<TStringBuf>>& newSortSets, - TExprNode::TListType& joinSettingNodes, TExprContext& ctx) -{ - for (auto& ss : newSortSets) { - YQL_ENSURE(!ss.empty()); - joinSettingNodes.push_back(ctx.Builder(pos) - .List() - .Atom(0, "preferred_sort") - .List(1) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < ss.size(); ++i) { - parent.Atom(i, ss[i]); - } - return parent; - }) - .Seal() - .Seal() - .Build()); - } -} - +void AppendEquiJoinSortSets(TPositionHandle pos, const TSet<TVector<TStringBuf>>& newSortSets, + TExprNode::TListType& joinSettingNodes, TExprContext& ctx) +{ + for (auto& ss : newSortSets) { + YQL_ENSURE(!ss.empty()); + joinSettingNodes.push_back(ctx.Builder(pos) + .List() + .Atom(0, "preferred_sort") + .List(1) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (ui32 i = 0; i < ss.size(); ++i) { + parent.Atom(i, ss[i]); + } + return parent; + }) + .Seal() + .Seal() + .Build()); + } +} + TMap<TStringBuf, TVector<TStringBuf>> UpdateUsedFieldsInRenameMap( const TMap<TStringBuf, TVector<TStringBuf>>& renameMap, const TSet<TStringBuf>& usedFields, @@ -1142,111 +1142,111 @@ TMap<TStringBuf, TVector<TStringBuf>> UpdateUsedFieldsInRenameMap( return newRenameMap; } -TVector<TEquiJoinParent> CollectEquiJoinOnlyParents(const TCoFlatMapBase& flatMap, const TParentsMap& parents) -{ - TVector<TEquiJoinParent> result; - if (!CollectEquiJoinOnlyParents(flatMap.Ref(), nullptr, 2, result, nullptr, parents)) { - result.clear(); - } +TVector<TEquiJoinParent> CollectEquiJoinOnlyParents(const TCoFlatMapBase& flatMap, const TParentsMap& parents) +{ + TVector<TEquiJoinParent> result; + if (!CollectEquiJoinOnlyParents(flatMap.Ref(), nullptr, 2, result, nullptr, parents)) { + result.clear(); + } + + return result; +} + +TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings) { + TEquiJoinLinkSettings result; + result.Pos = linkSettings.Pos(); + + auto collectHints = [](TSet<TString>& hints, const TExprNode& hintsNode) { + if (hintsNode.IsAtom()) { + hints.insert(ToString(hintsNode.Content())); + } else { + for (auto h : hintsNode.Children()) { + YQL_ENSURE(h->IsAtom()); + hints.insert(ToString(h->Content())); + } + } + }; + + if (auto left = GetSetting(linkSettings, "left")) { + collectHints(result.LeftHints, *left->Child(1)); + } - return result; + if (auto right = GetSetting(linkSettings, "right")) { + collectHints(result.RightHints, *right->Child(1)); + } + + result.ForceSortedMerge = HasSetting(linkSettings, "forceSortedMerge"); + return result; } - -TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings) { - TEquiJoinLinkSettings result; - result.Pos = linkSettings.Pos(); - - auto collectHints = [](TSet<TString>& hints, const TExprNode& hintsNode) { - if (hintsNode.IsAtom()) { - hints.insert(ToString(hintsNode.Content())); - } else { - for (auto h : hintsNode.Children()) { - YQL_ENSURE(h->IsAtom()); - hints.insert(ToString(h->Content())); - } - } - }; - - if (auto left = GetSetting(linkSettings, "left")) { - collectHints(result.LeftHints, *left->Child(1)); - } - - if (auto right = GetSetting(linkSettings, "right")) { - collectHints(result.RightHints, *right->Child(1)); - } - - result.ForceSortedMerge = HasSetting(linkSettings, "forceSortedMerge"); - return result; -} - -TExprNode::TPtr BuildEquiJoinLinkSettings(const TEquiJoinLinkSettings& linkSettings, TExprContext& ctx) { - auto builder = [&](const TStringBuf& side) -> TExprNode::TPtr { - return ctx.Builder(linkSettings.Pos) - .List() - .Atom(0, side) - .List(1) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - ui32 i = 0; - for (auto h : (side == "left" ? linkSettings.LeftHints : linkSettings.RightHints)) { - parent.Atom(i++, h); - } - return parent; - }) - .Seal() - .Seal() - .Build(); - }; - - TExprNode::TListType settings; - if (linkSettings.ForceSortedMerge) { - settings.push_back(ctx.NewList(linkSettings.Pos, { ctx.NewAtom(linkSettings.Pos, "forceSortedMerge", TNodeFlags::Default) })); - } - if (linkSettings.LeftHints) { - settings.push_back(builder("left")); - } - - if (linkSettings.RightHints) { - settings.push_back(builder("right")); - } - - return ctx.NewList(linkSettings.Pos, std::move(settings)); -} - -TExprNode::TPtr RemapNonConvertibleMemberForJoin(TPositionHandle pos, const TExprNode::TPtr& memberValue, - const TTypeAnnotationNode& memberType, const TTypeAnnotationNode& unifiedType, TExprContext& ctx) -{ - TExprNode::TPtr result = memberValue; - - if (&memberType != &unifiedType) { - result = ctx.Builder(pos) - .Callable("StrictCast") - .Add(0, std::move(result)) - .Add(1, ExpandType(pos, unifiedType, ctx)) - .Seal() - .Build(); - } - - if (RemoveOptionalType(&unifiedType)->GetKind() != ETypeAnnotationKind::Data) { - result = ctx.Builder(pos) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, result) - .Seal() - .Callable(1, "Null") - .Seal() - .Callable(2, "StablePickle") - .Add(0, result) - .Seal() - .Seal() - .Build(); - } - - return result; -} - + +TExprNode::TPtr BuildEquiJoinLinkSettings(const TEquiJoinLinkSettings& linkSettings, TExprContext& ctx) { + auto builder = [&](const TStringBuf& side) -> TExprNode::TPtr { + return ctx.Builder(linkSettings.Pos) + .List() + .Atom(0, side) + .List(1) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + ui32 i = 0; + for (auto h : (side == "left" ? linkSettings.LeftHints : linkSettings.RightHints)) { + parent.Atom(i++, h); + } + return parent; + }) + .Seal() + .Seal() + .Build(); + }; + + TExprNode::TListType settings; + if (linkSettings.ForceSortedMerge) { + settings.push_back(ctx.NewList(linkSettings.Pos, { ctx.NewAtom(linkSettings.Pos, "forceSortedMerge", TNodeFlags::Default) })); + } + if (linkSettings.LeftHints) { + settings.push_back(builder("left")); + } + + if (linkSettings.RightHints) { + settings.push_back(builder("right")); + } + + return ctx.NewList(linkSettings.Pos, std::move(settings)); +} + +TExprNode::TPtr RemapNonConvertibleMemberForJoin(TPositionHandle pos, const TExprNode::TPtr& memberValue, + const TTypeAnnotationNode& memberType, const TTypeAnnotationNode& unifiedType, TExprContext& ctx) +{ + TExprNode::TPtr result = memberValue; + + if (&memberType != &unifiedType) { + result = ctx.Builder(pos) + .Callable("StrictCast") + .Add(0, std::move(result)) + .Add(1, ExpandType(pos, unifiedType, ctx)) + .Seal() + .Build(); + } + + if (RemoveOptionalType(&unifiedType)->GetKind() != ETypeAnnotationKind::Data) { + result = ctx.Builder(pos) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, result) + .Seal() + .Callable(1, "Null") + .Seal() + .Callable(2, "StablePickle") + .Add(0, result) + .Seal() + .Seal() + .Build(); + } + + return result; +} + TExprNode::TPtr PrepareListForJoin(TExprNode::TPtr list, const TTypeAnnotationNode::TListType& keyTypes, TExprNode::TListType& keys, TExprNode::TListType& payloads, bool payload, bool optional, bool filter, TExprContext& ctx) { const auto pos = list->Pos(); - + if (keyTypes.empty() && 1U == keys.size()) { return payload ? ctx.Builder(pos) @@ -1539,4 +1539,4 @@ TExprNode::TPtr MakeDictForJoin(TExprNode::TPtr&& list, bool payload, bool multi template TExprNode::TPtr MakeDictForJoin<true>(TExprNode::TPtr&& list, bool payload, bool multi, TExprContext& ctx); template TExprNode::TPtr MakeDictForJoin<false>(TExprNode::TPtr&& list, bool payload, bool multi, TExprContext& ctx); -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/core/yql_join.h b/ydb/library/yql/core/yql_join.h index 8647728cd9..b01b950c2d 100644 --- a/ydb/library/yql/core/yql_join.h +++ b/ydb/library/yql/core/yql_join.h @@ -22,8 +22,8 @@ inline void SplitTableName(const TStringBuf& fullName, TStringBuf& table, TStrin } struct TJoinLabel { - TMaybe<TIssue> Parse(TExprContext& ctx, TExprNode& node, const TStructExprType* structType); - TMaybe<TIssue> ValidateLabel(TExprContext& ctx, const NNodes::TCoAtom& label); + TMaybe<TIssue> Parse(TExprContext& ctx, TExprNode& node, const TStructExprType* structType); + TMaybe<TIssue> ValidateLabel(TExprContext& ctx, const NNodes::TCoAtom& label); TString FullName(const TStringBuf& column) const; TStringBuf ColumnName(const TStringBuf& column) const; TStringBuf TableName(const TStringBuf& column) const; @@ -39,7 +39,7 @@ struct TJoinLabel { }; struct TJoinLabels { - TMaybe<TIssue> Add(TExprContext& ctx, TExprNode& node, const TStructExprType* structType); + TMaybe<TIssue> Add(TExprContext& ctx, TExprNode& node, const TStructExprType* structType); TMaybe<const TJoinLabel*> FindInput(const TStringBuf& table) const; TMaybe<ui32> FindInputIndex(const TStringBuf& table) const; TMaybe<const TTypeAnnotationNode*> FindColumn(const TStringBuf& table, const TStringBuf& column) const; @@ -50,42 +50,42 @@ struct TJoinLabels { THashMap<TStringBuf, ui32> InputByTable; }; -struct TJoinOptions { - THashMap<TStringBuf, TVector<TStringBuf>> RenameMap; - TSet<TVector<TStringBuf>> PreferredSortSets; - - bool Flatten = false; - bool KeepSysColumns = false; - bool StrictKeys = false; -}; - +struct TJoinOptions { + THashMap<TStringBuf, TVector<TStringBuf>> RenameMap; + TSet<TVector<TStringBuf>> PreferredSortSets; + + bool Flatten = false; + bool KeepSysColumns = false; + bool StrictKeys = false; +}; + IGraphTransformer::TStatus ValidateEquiJoinOptions( TPositionHandle positionHandle, - const TExprNode& optionsNode, - TJoinOptions& options, + const TExprNode& optionsNode, + TJoinOptions& options, TExprContext& ctx ); -IGraphTransformer::TStatus EquiJoinAnnotation( - TPositionHandle position, - const TStructExprType*& resultType, - const TJoinLabels& labels, - const TExprNode& joins, - const TJoinOptions& options, - TExprContext& ctx -); - -THashMap<TStringBuf, THashSet<TStringBuf>> CollectEquiJoinKeyColumnsByLabel(const TExprNode& joinTree); - +IGraphTransformer::TStatus EquiJoinAnnotation( + TPositionHandle position, + const TStructExprType*& resultType, + const TJoinLabels& labels, + const TExprNode& joins, + const TJoinOptions& options, + TExprContext& ctx +); + +THashMap<TStringBuf, THashSet<TStringBuf>> CollectEquiJoinKeyColumnsByLabel(const TExprNode& joinTree); + bool IsLeftJoinSideOptional(const TStringBuf& joinType); bool IsRightJoinSideOptional(const TStringBuf& joinType); -TExprNode::TPtr FilterOutNullJoinColumns(TPositionHandle pos, const TExprNode::TPtr& input, +TExprNode::TPtr FilterOutNullJoinColumns(TPositionHandle pos, const TExprNode::TPtr& input, const TJoinLabel& label, const TSet<TString>& optionalKeyColumns, TExprContext& ctx); TMap<TStringBuf, TVector<TStringBuf>> LoadJoinRenameMap(const TExprNode& settings); -TSet<TVector<TStringBuf>> LoadJoinSortSets(const TExprNode& settings); - +TSet<TVector<TStringBuf>> LoadJoinSortSets(const TExprNode& settings); + THashMap<TString, const TTypeAnnotationNode*> GetJoinColumnTypes(const TExprNode& joins, const TJoinLabels& labels, TExprContext& ctx); @@ -96,47 +96,47 @@ bool AreSameJoinKeys(const TExprNode& joins, const TStringBuf& table1, const TSt // returns (is required side + allow skip nulls); std::pair<bool, bool> IsRequiredSide(const TExprNode::TPtr& joinTree, const TJoinLabels& labels, ui32 inputIndex); -void AppendEquiJoinRenameMap(TPositionHandle pos, const TMap<TStringBuf, TVector<TStringBuf>>& newRenameMap, +void AppendEquiJoinRenameMap(TPositionHandle pos, const TMap<TStringBuf, TVector<TStringBuf>>& newRenameMap, + TExprNode::TListType& joinSettingNodes, TExprContext& ctx); + +void AppendEquiJoinSortSets(TPositionHandle pos, const TSet<TVector<TStringBuf>>& newSortSets, TExprNode::TListType& joinSettingNodes, TExprContext& ctx); -void AppendEquiJoinSortSets(TPositionHandle pos, const TSet<TVector<TStringBuf>>& newSortSets, - TExprNode::TListType& joinSettingNodes, TExprContext& ctx); - TMap<TStringBuf, TVector<TStringBuf>> UpdateUsedFieldsInRenameMap( const TMap<TStringBuf, TVector<TStringBuf>>& renameMap, const TSet<TStringBuf>& usedFields, const TStructExprType* structType ); - -struct TEquiJoinParent { - TEquiJoinParent(const TExprNode* node, ui32 index, const TExprNode* extractedMembers) - : Node(node) - , Index(index) - , ExtractedMembers(extractedMembers) - { - } - const TExprNode* Node; - ui32 Index; - const TExprNode* ExtractedMembers; -}; - -TVector<TEquiJoinParent> CollectEquiJoinOnlyParents(const NNodes::TCoFlatMapBase& flatMap, const TParentsMap& parents); - -struct TEquiJoinLinkSettings { - TPositionHandle Pos; - TSet<TString> LeftHints; - TSet<TString> RightHints; - // JOIN implementation may ignore this flags if SortedMerge strategy is not supported - bool ForceSortedMerge = true; -}; - -TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings); -TExprNode::TPtr BuildEquiJoinLinkSettings(const TEquiJoinLinkSettings& linkSettings, TExprContext& ctx); - -TExprNode::TPtr RemapNonConvertibleMemberForJoin(TPositionHandle pos, const TExprNode::TPtr& memberValue, - const TTypeAnnotationNode& memberType, const TTypeAnnotationNode& unifiedType, TExprContext& ctx); - + +struct TEquiJoinParent { + TEquiJoinParent(const TExprNode* node, ui32 index, const TExprNode* extractedMembers) + : Node(node) + , Index(index) + , ExtractedMembers(extractedMembers) + { + } + const TExprNode* Node; + ui32 Index; + const TExprNode* ExtractedMembers; +}; + +TVector<TEquiJoinParent> CollectEquiJoinOnlyParents(const NNodes::TCoFlatMapBase& flatMap, const TParentsMap& parents); + +struct TEquiJoinLinkSettings { + TPositionHandle Pos; + TSet<TString> LeftHints; + TSet<TString> RightHints; + // JOIN implementation may ignore this flags if SortedMerge strategy is not supported + bool ForceSortedMerge = true; +}; + +TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings); +TExprNode::TPtr BuildEquiJoinLinkSettings(const TEquiJoinLinkSettings& linkSettings, TExprContext& ctx); + +TExprNode::TPtr RemapNonConvertibleMemberForJoin(TPositionHandle pos, const TExprNode::TPtr& memberValue, + const TTypeAnnotationNode& memberType, const TTypeAnnotationNode& unifiedType, TExprContext& ctx); + TExprNode::TPtr PrepareListForJoin(TExprNode::TPtr list, const TTypeAnnotationNode::TListType& keyTypes, TExprNode::TListType& keys, TExprNode::TListType& payloads, bool payload, bool optional, bool filter, TExprContext& ctx); template<bool Squeeze = false> diff --git a/ydb/library/yql/core/yql_library_compiler.cpp b/ydb/library/yql/core/yql_library_compiler.cpp index 038ae79f30..79296792e6 100644 --- a/ydb/library/yql/core/yql_library_compiler.cpp +++ b/ydb/library/yql/core/yql_library_compiler.cpp @@ -10,7 +10,7 @@ namespace NYql { namespace { -bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges, TNodeSet& visited, TNodeSet& parents) +bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges, TNodeSet& visited, TNodeSet& parents) { if (!node.ChildrenSize()) { return true; @@ -33,7 +33,7 @@ bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& ha hasChanges = true; } - if (!ReplaceNodes(*node.Child(i), replaces, hasChanges, visited, parents)) { + if (!ReplaceNodes(*node.Child(i), replaces, hasChanges, visited, parents)) { child.Reset(); return false; } @@ -42,12 +42,12 @@ bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& ha return true; } -bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges) { - TNodeSet visited; - TNodeSet parents; - return ReplaceNodes(node, replaces, hasChanges, visited, parents); -} - +bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges) { + TNodeSet visited; + TNodeSet parents; + return ReplaceNodes(node, replaces, hasChanges, visited, parents); +} + TString Load(const TString& path) { TFile file(path, EOpenModeFlag::RdOnly); @@ -62,11 +62,11 @@ TString Load(const TString& path) bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx) { TExprNode::TListType tupleItems; - for (const auto& x : cohesion.Exports.Symbols()) { + for (const auto& x : cohesion.Exports.Symbols()) { tupleItems.push_back(x.second); } - auto root = ctx.NewList(TPositionHandle(), std::move(tupleItems)); + auto root = ctx.NewList(TPositionHandle(), std::move(tupleItems)); for (;;) { auto status = ExpandApply(root, root, ctx); if (status == IGraphTransformer::TStatus::Error) { @@ -80,7 +80,7 @@ bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx) { } ui32 index = 0; - for (auto& x : cohesion.Exports.Symbols(ctx)) { + for (auto& x : cohesion.Exports.Symbols(ctx)) { x.second = root->ChildPtr(index++); } @@ -133,7 +133,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, } if (import.second.first == lib.first) { - ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), + ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), TStringBuilder() << "Library '" << lib.first << "' tries to import itself.")); return false; } @@ -148,7 +148,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, } if (!exportTable) { - ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), + ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), TStringBuilder() << "Library '" << lib.first << "' has unresolved dependency from '" << import.second.first << "'.")); return false; } @@ -156,7 +156,7 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, if (const auto ex = exportTable->Symbols().find(import.second.second); exportTable->Symbols().cend() != ex) { replaces[import.first] = externalModule ? ctxToClone.DeepCopy(*ex->second, exportTable->ExprCtx(), clones, true, false) : ex->second; } else { - ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), + ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()), TStringBuilder() << "Library '" << lib.first << "' has unresolved symbol '" << import.second.second << "' from '" << import.second.first << "'.")); return false; } @@ -175,9 +175,9 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, for (bool hasChanges = !replaces.empty(); hasChanges;) { hasChanges = false; for (const auto& lib : libs) { - for (const auto& expo : lib.second.Exports.Symbols()) { + for (const auto& expo : lib.second.Exports.Symbols()) { if (!ReplaceNodes(*expo.second, replaces, hasChanges)) { - ctx.AddError(TIssue(ctxToClone.GetPosition(expo.second->Pos()), + ctx.AddError(TIssue(ctxToClone.GetPosition(expo.second->Pos()), TStringBuilder() << "Cross reference detected under '" << expo.first << "' in '" << lib.first << "'.")); return false; } diff --git a/ydb/library/yql/core/yql_opt_aggregate.cpp b/ydb/library/yql/core/yql_opt_aggregate.cpp index ce80d49adc..226339b8c5 100644 --- a/ydb/library/yql/core/yql_opt_aggregate.cpp +++ b/ydb/library/yql/core/yql_opt_aggregate.cpp @@ -1,90 +1,90 @@ #include "yql_opt_aggregate.h" #include "yql_opt_utils.h" -#include "yql_opt_window.h" +#include "yql_opt_window.h" #include "yql_expr_type_annotation.h" namespace NYql { TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, bool forceCompact) { auto list = node->HeadPtr(); - auto keyColumns = node->ChildPtr(1); + auto keyColumns = node->ChildPtr(1); auto aggregatedColumns = node->Child(2); - auto settings = node->Child(3); - + auto settings = node->Child(3); + YQL_ENSURE(!HasSetting(*settings, "hopping"), "Aggregate with hopping unsupported here."); - static const TStringBuf sessionStartMemberName = "_yql_group_session_start"; - const TExprNode::TPtr voidNode = ctx.NewCallable(node->Pos(), "Void", {}); - - TExprNode::TPtr sessionKey; - const TTypeAnnotationNode* sessionKeyType = nullptr; - const TTypeAnnotationNode* sessionParamsType = nullptr; - TExprNode::TPtr sessionInit; - TExprNode::TPtr sessionUpdate; - - TExprNode::TPtr sortKey = voidNode; - TExprNode::TPtr sortOrder = voidNode; - + static const TStringBuf sessionStartMemberName = "_yql_group_session_start"; + const TExprNode::TPtr voidNode = ctx.NewCallable(node->Pos(), "Void", {}); + + TExprNode::TPtr sessionKey; + const TTypeAnnotationNode* sessionKeyType = nullptr; + const TTypeAnnotationNode* sessionParamsType = nullptr; + TExprNode::TPtr sessionInit; + TExprNode::TPtr sessionUpdate; + + TExprNode::TPtr sortKey = voidNode; + TExprNode::TPtr sortOrder = voidNode; + bool effectiveCompact = forceCompact || HasSetting(*settings, "compact"); - - const TStructExprType* originalRowType = GetSeqItemType(node->Head().GetTypeAnn())->Cast<TStructExprType>(); - TVector<const TItemExprType*> rowItems = originalRowType->GetItems(); - - const auto sessionSetting = GetSetting(*settings, "session"); - TMaybe<TStringBuf> sessionOutputColumn; - if (sessionSetting) { - YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); - sessionOutputColumn = sessionSetting->Child(1)->Child(0)->Content(); - - // remove session column from other keys - TExprNodeList keyColumnsList = keyColumns->ChildrenList(); - EraseIf(keyColumnsList, [&](const auto& key) { return sessionOutputColumn == key->Content(); }); - keyColumns = ctx.NewList(keyColumns->Pos(), std::move(keyColumnsList)); - - const bool haveDistinct = AnyOf(aggregatedColumns->ChildrenList(), - [](const auto& child) { return child->ChildrenSize() == 3; }); - - TExprNode::TPtr sessionSortTraits; - ExtractSessionWindowParams(node->Pos(), sessionSetting->Child(1)->ChildPtr(1), sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, - sessionInit, sessionUpdate, ctx); - ExtractSortKeyAndOrder(node->Pos(), sessionSortTraits, sortKey, sortOrder, ctx); - - if (haveDistinct) { - auto keySelector = BuildKeySelector(node->Pos(), *originalRowType, keyColumns, ctx); - const auto sessionStartMemberLambda = AddSessionParamsMemberLambda(node->Pos(), sessionStartMemberName, "", keySelector, - sessionKey, sessionInit, sessionUpdate, ctx); - - list = ctx.Builder(node->Pos()) - .Callable("PartitionsByKeys") - .Add(0, list) - .Add(1, keySelector) - .Add(2, sortOrder) - .Add(3, sortKey) - .Lambda(4) - .Param("partitionedStream") - .Apply(sessionStartMemberLambda) - .With(0, "partitionedStream") - .Seal() - .Seal() - .Seal() - .Build(); - - auto keyColumnsList = keyColumns->ChildrenList(); - keyColumnsList.push_back(ctx.NewAtom(node->Pos(), sessionStartMemberName)); - keyColumns = ctx.NewList(node->Pos(), std::move(keyColumnsList)); - - rowItems.push_back(ctx.MakeType<TItemExprType>(sessionStartMemberName, sessionKeyType)); - - sortOrder = sortKey = voidNode; - sessionKey = sessionInit = sessionUpdate = {}; - sessionKeyType = nullptr; - } else { - effectiveCompact = true; - } - } - - const bool compact = effectiveCompact; - const auto rowType = ctx.MakeType<TStructExprType>(rowItems); + + const TStructExprType* originalRowType = GetSeqItemType(node->Head().GetTypeAnn())->Cast<TStructExprType>(); + TVector<const TItemExprType*> rowItems = originalRowType->GetItems(); + + const auto sessionSetting = GetSetting(*settings, "session"); + TMaybe<TStringBuf> sessionOutputColumn; + if (sessionSetting) { + YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); + sessionOutputColumn = sessionSetting->Child(1)->Child(0)->Content(); + + // remove session column from other keys + TExprNodeList keyColumnsList = keyColumns->ChildrenList(); + EraseIf(keyColumnsList, [&](const auto& key) { return sessionOutputColumn == key->Content(); }); + keyColumns = ctx.NewList(keyColumns->Pos(), std::move(keyColumnsList)); + + const bool haveDistinct = AnyOf(aggregatedColumns->ChildrenList(), + [](const auto& child) { return child->ChildrenSize() == 3; }); + + TExprNode::TPtr sessionSortTraits; + ExtractSessionWindowParams(node->Pos(), sessionSetting->Child(1)->ChildPtr(1), sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, + sessionInit, sessionUpdate, ctx); + ExtractSortKeyAndOrder(node->Pos(), sessionSortTraits, sortKey, sortOrder, ctx); + + if (haveDistinct) { + auto keySelector = BuildKeySelector(node->Pos(), *originalRowType, keyColumns, ctx); + const auto sessionStartMemberLambda = AddSessionParamsMemberLambda(node->Pos(), sessionStartMemberName, "", keySelector, + sessionKey, sessionInit, sessionUpdate, ctx); + + list = ctx.Builder(node->Pos()) + .Callable("PartitionsByKeys") + .Add(0, list) + .Add(1, keySelector) + .Add(2, sortOrder) + .Add(3, sortKey) + .Lambda(4) + .Param("partitionedStream") + .Apply(sessionStartMemberLambda) + .With(0, "partitionedStream") + .Seal() + .Seal() + .Seal() + .Build(); + + auto keyColumnsList = keyColumns->ChildrenList(); + keyColumnsList.push_back(ctx.NewAtom(node->Pos(), sessionStartMemberName)); + keyColumns = ctx.NewList(node->Pos(), std::move(keyColumnsList)); + + rowItems.push_back(ctx.MakeType<TItemExprType>(sessionStartMemberName, sessionKeyType)); + + sortOrder = sortKey = voidNode; + sessionKey = sessionInit = sessionUpdate = {}; + sessionKeyType = nullptr; + } else { + effectiveCompact = true; + } + } + + const bool compact = effectiveCompact; + const auto rowType = ctx.MakeType<TStructExprType>(rowItems); auto preMap = ctx.Builder(node->Pos()) .Lambda() @@ -99,7 +99,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, YQL_ENSURE(index, "Unknown column: " << keyColumn->Content()); auto type = rowType->GetItems()[*index]->GetItemType(); keyItemTypes.push_back(type); - needPickle = needPickle || !IsDataOrOptionalOfData(type); + needPickle = needPickle || !IsDataOrOptionalOfData(type); } const TTypeAnnotationNode* pickleType = nullptr; @@ -795,7 +795,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, } // If no aggregation functions than add addional combiner - if (aggregatedColumns->ChildrenSize() == 0 && keyColumns->ChildrenSize() > 0 && !sessionUpdate) { + if (aggregatedColumns->ChildrenSize() == 0 && keyColumns->ChildrenSize() > 0 && !sessionUpdate) { // Return key as-is auto uniqCombineInit = ctx.Builder(node->Pos()) .Lambda() @@ -878,7 +878,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Build(); } - ui32 index = 0U; + ui32 index = 0U; auto groupInit = ctx.Builder(node->Pos()) .Lambda() .Param("item") @@ -886,7 +886,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < keyColumns->ChildrenSize(); ++i) { parent - .List(index++) + .List(index++) .Add(0, keyColumns->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "item") @@ -894,16 +894,16 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } - if (sessionUpdate) { - parent - .List(index++) - .Atom(0, sessionStartMemberName) - .Callable(1, "Member") - .Arg(0, "item") - .Atom(1, sessionStartMemberName) - .Seal() - .Seal(); - } + if (sessionUpdate) { + parent + .List(index++) + .Atom(0, sessionStartMemberName) + .Callable(1, "Member") + .Arg(0, "item") + .Atom(1, sessionStartMemberName) + .Seal() + .Seal(); + } return parent; }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { @@ -914,7 +914,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, auto loadLambda = trait->Child(4); if (!distinctFields.empty()) { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Callable(1, "Map") .Callable(0, "Member") @@ -925,7 +925,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } else { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Apply(1, *loadLambda) .With(0) @@ -979,7 +979,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, if (distinctField) { const bool isFirst = *distinct2Columns[distinctField->Content()].begin() == i; if (isFirst) { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .List(1) .Callable(0, "NamedApply") @@ -1017,13 +1017,13 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } else { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Do(initApply) .Seal(); } } else { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Do(initApply) .Seal(); @@ -1036,7 +1036,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Build(); - index = 0; + index = 0; auto groupMerge = ctx.Builder(node->Pos()) .Lambda() .Param("item") @@ -1045,7 +1045,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < keyColumns->ChildrenSize(); ++i) { parent - .List(index++) + .List(index++) .Add(0, keyColumns->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "state") @@ -1053,16 +1053,16 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } - if (sessionUpdate) { - parent - .List(index++) - .Atom(0, sessionStartMemberName) - .Callable(1, "Member") - .Arg(0, "state") - .Atom(1, sessionStartMemberName) - .Seal() - .Seal(); - } + if (sessionUpdate) { + parent + .List(index++) + .Atom(0, sessionStartMemberName) + .Callable(1, "Member") + .Arg(0, "state") + .Atom(1, sessionStartMemberName) + .Seal() + .Seal(); + } return parent; }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { @@ -1074,7 +1074,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, auto mergeLambda = trait->Child(5); if (!distinctFields.empty()) { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Callable(1, "OptionalReduce") .Callable(0, "Map") @@ -1092,7 +1092,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } else { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Apply(1, *mergeLambda) .With(0) @@ -1214,7 +1214,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, return parent; }; - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Callable(1, "If") .Callable(0, "NamedApply") @@ -1272,7 +1272,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } else { - parent.List(index++) + parent.List(index++) .Add(0, initialColumnNames[i]) .Do(updateApply) .Seal(); @@ -1285,16 +1285,16 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Build(); - index = 0U; + index = 0U; auto groupSave = ctx.Builder(node->Pos()) .Lambda() .Param("state") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < keyColumns->ChildrenSize(); ++i) { - if (keyColumns->Child(i)->Content() == sessionStartMemberName) { - continue; - } + if (keyColumns->Child(i)->Content() == sessionStartMemberName) { + continue; + } parent .List(index++) .Add(0, keyColumns->ChildPtr(i)) @@ -1304,17 +1304,17 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal(); } - - if (sessionOutputColumn) { - parent - .List(index++) - .Atom(0, *sessionOutputColumn) - .Callable(1, "Member") - .Arg(0, "state") - .Atom(1, sessionStartMemberName) - .Seal() - .Seal(); - } + + if (sessionOutputColumn) { + parent + .List(index++) + .Atom(0, *sessionOutputColumn) + .Callable(1, "Member") + .Arg(0, "state") + .Atom(1, sessionStartMemberName) + .Seal() + .Seal(); + } return parent; }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { @@ -1416,101 +1416,101 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, groupInput = std::move(list); } - TExprNode::TPtr preprocessLambda; - TExprNode::TPtr condenseSwitch; - if (sessionUpdate) { - YQL_ENSURE(compact); - YQL_ENSURE(sessionKey); - YQL_ENSURE(sessionKeyType); - YQL_ENSURE(sessionInit); - - preprocessLambda = - AddSessionParamsMemberLambda(node->Pos(), sessionStartMemberName, "", keyExtractor, sessionKey, sessionInit, sessionUpdate, ctx); - - if (!IsDataOrOptionalOfData(sessionKeyType)) { - preprocessLambda = ctx.Builder(node->Pos()) - .Lambda() - .Param("stream") - .Callable("OrderedMap") - .Apply(0, preprocessLambda) - .With(0, "stream") - .Seal() - .Lambda(1) - .Param("item") - .Callable("ReplaceMember") - .Arg(0, "item") - .Atom(1, sessionStartMemberName) - .Callable(2, "StablePickle") - .Callable(0, "Member") - .Arg(0, "item") - .Atom(1, sessionStartMemberName) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - } - - condenseSwitch = ctx.Builder(node->Pos()) - .Lambda() - .Param("item") - .Param("state") - .Callable("Or") - .Callable(0, "AggrNotEquals") - .Apply(0, keyExtractor) - .With(0, "item") - .Seal() - .Apply(1, keyExtractor) - .With(0, "state") - .Seal() - .Seal() - .Callable(1, "AggrNotEquals") - .Callable(0, "Member") - .Arg(0, "item") - .Atom(1, sessionStartMemberName) - .Seal() - .Callable(1, "Member") - .Arg(0, "state") - .Atom(1, sessionStartMemberName) - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - } else { - YQL_ENSURE(!sessionKey); - preprocessLambda = MakeIdentityLambda(node->Pos(), ctx); - condenseSwitch = ctx.Builder(node->Pos()) - .Lambda() - .Param("item") - .Param("state") - .Callable("IsKeySwitch") - .Arg(0, "item") - .Arg(1, "state") - .Add(2, keyExtractor) - .Add(3, keyExtractor) - .Seal() - .Seal() - .Build(); - } - + TExprNode::TPtr preprocessLambda; + TExprNode::TPtr condenseSwitch; + if (sessionUpdate) { + YQL_ENSURE(compact); + YQL_ENSURE(sessionKey); + YQL_ENSURE(sessionKeyType); + YQL_ENSURE(sessionInit); + + preprocessLambda = + AddSessionParamsMemberLambda(node->Pos(), sessionStartMemberName, "", keyExtractor, sessionKey, sessionInit, sessionUpdate, ctx); + + if (!IsDataOrOptionalOfData(sessionKeyType)) { + preprocessLambda = ctx.Builder(node->Pos()) + .Lambda() + .Param("stream") + .Callable("OrderedMap") + .Apply(0, preprocessLambda) + .With(0, "stream") + .Seal() + .Lambda(1) + .Param("item") + .Callable("ReplaceMember") + .Arg(0, "item") + .Atom(1, sessionStartMemberName) + .Callable(2, "StablePickle") + .Callable(0, "Member") + .Arg(0, "item") + .Atom(1, sessionStartMemberName) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + } + + condenseSwitch = ctx.Builder(node->Pos()) + .Lambda() + .Param("item") + .Param("state") + .Callable("Or") + .Callable(0, "AggrNotEquals") + .Apply(0, keyExtractor) + .With(0, "item") + .Seal() + .Apply(1, keyExtractor) + .With(0, "state") + .Seal() + .Seal() + .Callable(1, "AggrNotEquals") + .Callable(0, "Member") + .Arg(0, "item") + .Atom(1, sessionStartMemberName) + .Seal() + .Callable(1, "Member") + .Arg(0, "state") + .Atom(1, sessionStartMemberName) + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + } else { + YQL_ENSURE(!sessionKey); + preprocessLambda = MakeIdentityLambda(node->Pos(), ctx); + condenseSwitch = ctx.Builder(node->Pos()) + .Lambda() + .Param("item") + .Param("state") + .Callable("IsKeySwitch") + .Arg(0, "item") + .Arg(1, "state") + .Add(2, keyExtractor) + .Add(3, keyExtractor) + .Seal() + .Seal() + .Build(); + } + auto grouper = ctx.Builder(node->Pos()) .Callable("PartitionsByKeys") .Add(0, std::move(groupInput)) .Add(1, keyExtractor) - .Add(2, sortOrder) - .Add(3, sortKey) + .Add(2, sortOrder) + .Add(3, sortKey) .Lambda(4) .Param("stream") .Callable("Map") .Callable(0, "Condense1") - .Apply(0, preprocessLambda) - .With(0, "stream") - .Seal() + .Apply(0, preprocessLambda) + .With(0, "stream") + .Seal() .Add(1, std::move(groupInit)) - .Add(2, condenseSwitch) + .Add(2, condenseSwitch) .Add(3, std::move(groupMerge)) .Seal() .Add(1, std::move(groupSave)) @@ -1518,7 +1518,7 @@ TExprNode::TPtr ExpandAggregate(const TExprNode::TPtr& node, TExprContext& ctx, .Seal() .Seal().Build(); - if (keyColumns->ChildrenSize() == 0 && !sessionSetting) { + if (keyColumns->ChildrenSize() == 0 && !sessionSetting) { return MakeSingleGroupRow(*node, grouper, ctx); } diff --git a/ydb/library/yql/core/yql_opt_range.cpp b/ydb/library/yql/core/yql_opt_range.cpp index d83d5225cf..ca60e9edf5 100644 --- a/ydb/library/yql/core/yql_opt_range.cpp +++ b/ydb/library/yql/core/yql_opt_range.cpp @@ -1,502 +1,502 @@ -#include "yql_opt_range.h" -#include "yql_opt_utils.h" -#include "yql_expr_type_annotation.h" - +#include "yql_opt_range.h" +#include "yql_opt_utils.h" +#include "yql_expr_type_annotation.h" + #include <ydb/library/yql/public/udf/tz/udf_tz.h> - -namespace NYql { -namespace { - -struct TRangeBoundary { - TExprNode::TPtr Value; // top level null means infinity - bool Included = false; -}; - -TExprNode::TPtr BuildBoundaryNode(TPositionHandle pos, const TRangeBoundary& boundary, TExprContext& ctx) { - YQL_ENSURE(boundary.Value); - return ctx.Builder(pos) - .List() - .Add(0, boundary.Value) - .Callable(1, "Int32") - .Atom(0, boundary.Included ? "1" : "0", TNodeFlags::Default) - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildRange(TPositionHandle pos, const TRangeBoundary& left, const TRangeBoundary& right, - TExprContext& ctx) -{ - return ctx.NewList(pos, { BuildBoundaryNode(pos, left, ctx), BuildBoundaryNode(pos, right, ctx) }); -} - -TExprNode::TPtr BuildRangeSingle(TPositionHandle pos, const TRangeBoundary& left, const TRangeBoundary& right, - TExprContext& ctx) -{ - return ctx.NewCallable(pos, "AsRange", { BuildRange(pos, left, right, ctx) }); -} - -bool HasNaNs(EDataSlot slot) { - return NUdf::GetDataTypeInfo(slot).Features & (NUdf::EDataTypeFeatures::FloatType | NUdf::EDataTypeFeatures::DecimalType); -} - -TExprNode::TPtr MakeNaNBoundary(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { - auto baseType = RemoveAllOptionals(keyType); - auto keySlot = baseType->Cast<TDataExprType>()->GetSlot(); - YQL_ENSURE(HasNaNs(keySlot)); - - return ctx.Builder(pos) - .Callable("SafeCast") - .Callable(0, NUdf::GetDataTypeInfo(keySlot).Name) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - parent.Atom(0, "nan", TNodeFlags::Default); - if (keySlot == EDataSlot::Decimal) { - auto decimalType = baseType->Cast<TDataExprParamsType>(); - parent.Atom(1, decimalType->GetParamOne()); - parent.Atom(2, decimalType->GetParamTwo()); - } - return parent; - }) - .Seal() - .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx)) - .Seal() - .Build(); -} - -TExprNode::TPtr TzRound(const TExprNode::TPtr& key, const TTypeAnnotationNode* keyType, bool down, TExprContext& ctx) { - TPositionHandle pos = key->Pos(); - const auto& timeZones = NUdf::GetTimezones(); - const size_t tzSize = timeZones.size(); - YQL_ENSURE(tzSize > 0); - size_t targetTzId = 0; - if (!down) { - for (targetTzId = tzSize - 1; targetTzId > 0; --targetTzId) { - if (!timeZones[targetTzId].empty()) { - break; - } - } - } - YQL_ENSURE(!timeZones[targetTzId].empty()); - return ctx.Builder(pos) - .Callable("Coalesce") - .Callable(0, "AddTimezone") - .Callable(0, "RemoveTimezone") - .Callable(0, "SafeCast") - .Add(0, key) - .Add(1, ExpandType(pos, *RemoveAllOptionals(keyType), ctx)) - .Seal() - .Seal() - .Callable(1, "Uint16") - .Atom(0, ToString(targetTzId), TNodeFlags::Default) - .Seal() - .Seal() - .Add(1, key) - .Seal() - .Build(); -} - -TRangeBoundary BuildPlusInf(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx, bool excludeNaN = true) { - TRangeBoundary result; - - auto optKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx); - - auto baseType = RemoveAllOptionals(keyType); - auto keySlot = baseType->Cast<TDataExprType>()->GetSlot(); - if (excludeNaN && HasNaNs(keySlot)) { - // exclude NaN value (NaN is ordered as largest floating point value) - result.Value = MakeNaNBoundary(pos, keyType, ctx); - } else { - result.Value = ctx.NewCallable(pos, "Nothing", { optKeyTypeNode }); - } - - result.Included = false; - return result; -} - -TRangeBoundary BuildMinusInf(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { - // start from first non-null value - auto optKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx); - auto optBaseKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(RemoveAllOptionals(keyType)), ctx); - auto largestNull = ctx.Builder(pos) - .Callable("SafeCast") - .Callable(0, "Nothing") - .Add(0, optBaseKeyTypeNode) - .Seal() - .Add(1, optKeyTypeNode) - .Seal() - .Build(); - - TRangeBoundary result; - result.Value = largestNull; - result.Included = false; - return result; -} - -TExprNode::TPtr MakeWidePointRangeLambda(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { - TStringBuf name; - TString maxValueStr; - - const TTypeAnnotationNode* baseKeyType = RemoveAllOptionals(keyType); - const auto keySlot = baseKeyType->Cast<TDataExprType>()->GetSlot(); - switch (keySlot) { - case EDataSlot::Int8: name = "Int8"; maxValueStr = ToString(Max<i8>()); break; - case EDataSlot::Uint8: name = "Uint8"; maxValueStr = ToString(Max<ui8>()); break; - case EDataSlot::Int16: name = "Int16"; maxValueStr = ToString(Max<i16>()); break; - case EDataSlot::Uint16: name = "Uint16"; maxValueStr = ToString(Max<ui16>()); break; - case EDataSlot::Int32: name = "Int32"; maxValueStr = ToString(Max<i32>()); break; - case EDataSlot::Uint32: name = "Uint32"; maxValueStr = ToString(Max<ui32>()); break; - case EDataSlot::Int64: name = "Int64"; maxValueStr = ToString(Max<i64>()); break; - case EDataSlot::Uint64: name = "Uint64"; maxValueStr = ToString(Max<ui64>()); break; - - case EDataSlot::Date: name = "Date"; maxValueStr = ToString(NUdf::MAX_DATE - 1); break; - case EDataSlot::Datetime: name = "Datetime"; maxValueStr = ToString(NUdf::MAX_DATETIME - 1); break; - default: - YQL_ENSURE(keySlot == EDataSlot::Timestamp); - name = "Timestamp"; - maxValueStr = ToString(NUdf::MAX_TIMESTAMP - 1); - } - - TExprNode::TPtr maxValue = ctx.NewCallable(pos, name, { ctx.NewAtom(pos, maxValueStr, TNodeFlags::Default) }); - TExprNode::TPtr addValue; - if (keySlot == EDataSlot::Date) { - addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "86400000000", TNodeFlags::Default) }); - } else if (keySlot == EDataSlot::Datetime) { - addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "1000000", TNodeFlags::Default) }); - } else if (keySlot == EDataSlot::Timestamp) { - addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "1", TNodeFlags::Default) }); - } else { - addValue = ctx.NewCallable(pos, name, { ctx.NewAtom(pos, "1", TNodeFlags::Default) }); - } - - TExprNode::TPtr key = ctx.NewArgument(pos, "optKey"); - TExprNode::TPtr castedKey = ctx.Builder(pos) - .Callable("SafeCast") - .Add(0, key) - .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(baseKeyType), ctx)) - .Seal() - .Build(); - - TExprNode::TPtr leftBoundary = ctx.Builder(pos) - .List() - .Add(0, key) - .Callable(1, "Int32") - .Atom(0, "1", TNodeFlags::Default) - .Seal() - .Seal() - .Build(); - - auto body = ctx.Builder(pos) - .Callable("AsRange") - .List(0) - .Add(0, leftBoundary) - .Callable(1, "If") - .Callable(0, "Coalesce") - .Callable(0, "==") - .Add(0, key) - .Add(1, maxValue) - .Seal() - .Callable(1, "Bool") - .Atom(0, "true", TNodeFlags::Default) - .Seal() - .Seal() - .Add(1, leftBoundary) - .List(2) - .Callable(0, "SafeCast") - .Callable(0, "+") - .Add(0, castedKey) - .Add(1, addValue) - .Seal() - .Add(1, ExpandType(pos, *keyType, ctx)) - .Seal() - .Callable(1, "Int32") - .Atom(0, "0", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); -} - -TExprNode::TPtr BuildNormalRangeLambdaRaw(TPositionHandle pos, const TTypeAnnotationNode* keyType, - TStringBuf op, TExprContext& ctx) -{ - // key is argument of Optional<keyType> type - const auto key = ctx.NewArgument(pos, "key"); - const auto keySlot = RemoveAllOptionals(keyType)->Cast<TDataExprType>()->GetSlot(); - const bool isTzKey = NUdf::GetDataTypeInfo(keySlot).Features & NUdf::EDataTypeFeatures::TzDateType; - const bool isIntegralOrDateKey = NUdf::GetDataTypeInfo(keySlot).Features & - (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType); - const auto downKey = isTzKey ? TzRound(key, keyType, true, ctx) : key; - const auto upKey = isTzKey ? TzRound(key, keyType, false, ctx) : key; - if (op == "!=") { - TRangeBoundary left, right; - left = BuildMinusInf(pos, keyType, ctx); - right.Value = downKey; - right.Included = false; - auto leftRange = BuildRange(pos, left, right, ctx); - - left.Value = upKey; - left.Included = false; - bool excludeNaN = false; - right = BuildPlusInf(pos, keyType, ctx, excludeNaN); - auto rightRange = BuildRange(pos, left, right, ctx); - - auto body = ctx.NewCallable(pos, "AsRange", { leftRange, rightRange }); - if (HasNaNs(keySlot)) { - // NaN is not equal to any value (including NaN itself), so we must include it - left.Included = true; - left.Value = MakeNaNBoundary(pos, keyType, ctx); - right = left; - auto nan = BuildRangeSingle(pos, left, right, ctx); - body = ctx.NewCallable(pos, "RangeUnion", { body, nan }); - } - return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); - } - - if (op == "===") { - if (isIntegralOrDateKey) { - return MakeWidePointRangeLambda(pos, keyType, ctx); - } - op = "=="; - } - - TRangeBoundary left, right; - if (op == "==") { - left.Value = downKey; - right.Value = upKey; - left.Included = right.Included = true; - } else if (op == "<" || op == "<=") { - left = BuildMinusInf(pos, keyType, ctx); - right.Value = (op == "<=") ? upKey : downKey; - right.Included = (op == "<="); - } else if (op == ">" || op == ">=") { - right = BuildPlusInf(pos, keyType, ctx); - left.Value = (op == ">=") ? downKey : upKey; - left.Included = (op == ">="); - } else if (op == "Exists" || op == "NotExists" ) { - YQL_ENSURE(keyType->GetKind() == ETypeAnnotationKind::Optional); - auto nullKey = ctx.Builder(pos) - .Callable("Just") - .Callable(0, "Nothing") - .Add(0, ExpandType(pos, *keyType, ctx)) - .Seal() - .Seal() - .Build(); - - if (op == "NotExists") { - left.Value = right.Value = nullKey; - left.Included = right.Included = true; - } else { - left.Value = nullKey; - left.Included = false; - // Exists should include NaN - bool excludeNaN = false; - right = BuildPlusInf(pos, keyType, ctx, excludeNaN); - } - } else { - YQL_ENSURE(false, "Unknown operation: " << op); - } - - TExprNode::TPtr body = BuildRangeSingle(pos, left, right, ctx); - if (op == "==" && HasNaNs(keySlot)) { - auto fullRangeWithoutNaNs = BuildRangeSingle(pos, - BuildMinusInf(pos, keyType, ctx), - BuildPlusInf(pos, keyType, ctx), ctx); - body = ctx.NewCallable(pos, "RangeIntersect", { body, fullRangeWithoutNaNs }); - } - - return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); -} - -} //namespace - -TExprNode::TPtr ExpandRangeEmpty(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_ENSURE(node->IsCallable("RangeEmpty")); - if (node->ChildrenSize() == 0) { - return ctx.RenameNode(*node, "EmptyList"); - } - - return ctx.NewCallable(node->Pos(), "List", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) }); -} - -TExprNode::TPtr ExpandAsRange(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_ENSURE(node->IsCallable("AsRange")); - YQL_ENSURE(node->ChildrenSize()); - - return ctx.NewCallable(node->Pos(), "RangeCreate", { ctx.RenameNode(*node, "AsList") }); -} - -TExprNode::TPtr ExpandRangeFor(const TExprNode::TPtr& node, TExprContext& ctx) { - TPositionHandle pos = node->Pos(); - - TStringBuf op = node->Head().Content(); - auto value = node->ChildPtr(1); - const TTypeAnnotationNode* valueType = value->GetTypeAnn(); - const TTypeAnnotationNode* keyType = node->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - - const TTypeAnnotationNode* optKeyType = ctx.MakeType<TOptionalExprType>(keyType); - const TTypeAnnotationNode* keyBaseType = RemoveAllOptionals(keyType); - const TTypeAnnotationNode* valueBaseType = RemoveAllOptionals(valueType); - - const TExprNode::TPtr castedToKey = ctx.NewCallable(pos, "StrictCast", - { value, ExpandType(pos, *optKeyType, ctx) }); - - const TExprNode::TPtr emptyRange = ctx.NewCallable(pos, "RangeEmpty", - { ExpandType(pos, *keyType, ctx) }); - - TExprNode::TPtr rangeForNullCast; - if (op == "==" || op == "===" || op == "StartsWith") { - rangeForNullCast = emptyRange; - } else if (op == "!=" || op == "NotStartsWith") { - bool excludeNaN = false; - rangeForNullCast = BuildRangeSingle(pos, - BuildMinusInf(pos, keyType, ctx), - BuildPlusInf(pos, keyType, ctx, excludeNaN), ctx); - } - - TExprNode::TPtr result; - if (op == "Exists" || op == "NotExists") { - result = ctx.Builder(pos) - .Apply(BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) - .With(0, value) // value is not actually used for Exists/NotExists - .Seal() - .Build(); - } else if (op == "==" || op == "!=" || op == "===") { - YQL_ENSURE(rangeForNullCast); - result = ctx.Builder(pos) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, castedToKey) - .Seal() - .Add(1, rangeForNullCast) - .Apply(2, BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) - .With(0, castedToKey) - .Seal() - .Seal() - .Build(); - } else if (op == "StartsWith" || op == "NotStartsWith") { - YQL_ENSURE(rangeForNullCast); - - const auto boundary = ctx.NewArgument(pos, "boundary"); - const auto next = ctx.NewCallable(pos, "NextValue", { boundary }); - - TExprNode::TPtr rangeForIfNext; - TExprNode::TPtr rangeForIfNotNext; - - if (op == "StartsWith") { - const auto geBoundary = ctx.Builder(pos) - .Callable("RangeFor") - .Atom(0, ">=", TNodeFlags::Default) - .Add(1, boundary) - .Add(2, node->TailPtr()) - .Seal() - .Build(); - - rangeForIfNext = ctx.Builder(pos) - // [boundary, next) - .Callable("RangeIntersect") - .Add(0, geBoundary) - .Callable(1, "RangeFor") - .Atom(0, "<", TNodeFlags::Default) - .Add(1, next) - .Add(2, node->TailPtr()) - .Seal() - .Seal() - .Build(); - - // [boundary, +inf) - rangeForIfNotNext = geBoundary; - } else { - const auto lessBoundary = ctx.Builder(pos) - .Callable("RangeFor") - .Atom(0, "<", TNodeFlags::Default) - .Add(1, boundary) - .Add(2, node->TailPtr()) - .Seal() - .Build(); - - rangeForIfNext = ctx.Builder(pos) - // (-inf, boundary) U [next, +inf) - .Callable("RangeUnion") - .Add(0, lessBoundary) - .Callable(1, "RangeFor") - .Atom(0, ">=", TNodeFlags::Default) - .Add(1, next) - .Add(2, node->TailPtr()) - .Seal() - .Seal() - .Build(); - - // (-inf, boundary) - rangeForIfNotNext = lessBoundary; - } - - auto body = ctx.Builder(pos) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, next) - .Seal() - .Add(1, rangeForIfNotNext) - .Add(2, rangeForIfNext) - .Seal() - .Build(); - - YQL_ENSURE(rangeForNullCast); - result = ctx.Builder(pos) - .Callable("IfPresent") - .Callable(0, "StrictCast") - .Add(0, value) - .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyBaseType), ctx)) - .Seal() - .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, { boundary}), std::move(body))) - .Add(2, rangeForNullCast) - .Seal() - .Build(); - } else { - YQL_ENSURE(op == "<" || op == ">" || op == "<=" || op == ">="); - result = ctx.Builder(pos) - .Callable("If") - .Callable(0, "HasNull") - .Add(0, castedToKey) - .Seal() - .Callable(1, "IfPresent") - .Callable(0, "SafeCast") - .Add(0, value) - .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(valueBaseType), ctx)) - .Seal() - .Lambda(1) - .Param("unwrappedValue") - .Callable("IfPresent") - .Callable(0, op.StartsWith("<") ? "RoundDown" : "RoundUp") - .Arg(0, "unwrappedValue") - .Add(1, ExpandType(pos, *keyBaseType, ctx)) - .Seal() - .Lambda(1) - .Param("roundedKey") - .Apply(BuildNormalRangeLambdaRaw(pos, keyType, op.StartsWith("<") ? "<=" : ">=", ctx)) - .With(0) - .Callable("SafeCast") - .Arg(0, "roundedKey") - .Add(1, ExpandType(pos, *optKeyType, ctx)) - .Seal() - .Done() - .Seal() - .Seal() - .Add(2, emptyRange) - .Seal() - .Seal() - .Add(2, emptyRange) - .Seal() - .Apply(2, BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) - .With(0, castedToKey) - .Seal() - .Seal() - .Build(); - } - - return result; -} - -} + +namespace NYql { +namespace { + +struct TRangeBoundary { + TExprNode::TPtr Value; // top level null means infinity + bool Included = false; +}; + +TExprNode::TPtr BuildBoundaryNode(TPositionHandle pos, const TRangeBoundary& boundary, TExprContext& ctx) { + YQL_ENSURE(boundary.Value); + return ctx.Builder(pos) + .List() + .Add(0, boundary.Value) + .Callable(1, "Int32") + .Atom(0, boundary.Included ? "1" : "0", TNodeFlags::Default) + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildRange(TPositionHandle pos, const TRangeBoundary& left, const TRangeBoundary& right, + TExprContext& ctx) +{ + return ctx.NewList(pos, { BuildBoundaryNode(pos, left, ctx), BuildBoundaryNode(pos, right, ctx) }); +} + +TExprNode::TPtr BuildRangeSingle(TPositionHandle pos, const TRangeBoundary& left, const TRangeBoundary& right, + TExprContext& ctx) +{ + return ctx.NewCallable(pos, "AsRange", { BuildRange(pos, left, right, ctx) }); +} + +bool HasNaNs(EDataSlot slot) { + return NUdf::GetDataTypeInfo(slot).Features & (NUdf::EDataTypeFeatures::FloatType | NUdf::EDataTypeFeatures::DecimalType); +} + +TExprNode::TPtr MakeNaNBoundary(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { + auto baseType = RemoveAllOptionals(keyType); + auto keySlot = baseType->Cast<TDataExprType>()->GetSlot(); + YQL_ENSURE(HasNaNs(keySlot)); + + return ctx.Builder(pos) + .Callable("SafeCast") + .Callable(0, NUdf::GetDataTypeInfo(keySlot).Name) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + parent.Atom(0, "nan", TNodeFlags::Default); + if (keySlot == EDataSlot::Decimal) { + auto decimalType = baseType->Cast<TDataExprParamsType>(); + parent.Atom(1, decimalType->GetParamOne()); + parent.Atom(2, decimalType->GetParamTwo()); + } + return parent; + }) + .Seal() + .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx)) + .Seal() + .Build(); +} + +TExprNode::TPtr TzRound(const TExprNode::TPtr& key, const TTypeAnnotationNode* keyType, bool down, TExprContext& ctx) { + TPositionHandle pos = key->Pos(); + const auto& timeZones = NUdf::GetTimezones(); + const size_t tzSize = timeZones.size(); + YQL_ENSURE(tzSize > 0); + size_t targetTzId = 0; + if (!down) { + for (targetTzId = tzSize - 1; targetTzId > 0; --targetTzId) { + if (!timeZones[targetTzId].empty()) { + break; + } + } + } + YQL_ENSURE(!timeZones[targetTzId].empty()); + return ctx.Builder(pos) + .Callable("Coalesce") + .Callable(0, "AddTimezone") + .Callable(0, "RemoveTimezone") + .Callable(0, "SafeCast") + .Add(0, key) + .Add(1, ExpandType(pos, *RemoveAllOptionals(keyType), ctx)) + .Seal() + .Seal() + .Callable(1, "Uint16") + .Atom(0, ToString(targetTzId), TNodeFlags::Default) + .Seal() + .Seal() + .Add(1, key) + .Seal() + .Build(); +} + +TRangeBoundary BuildPlusInf(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx, bool excludeNaN = true) { + TRangeBoundary result; + + auto optKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx); + + auto baseType = RemoveAllOptionals(keyType); + auto keySlot = baseType->Cast<TDataExprType>()->GetSlot(); + if (excludeNaN && HasNaNs(keySlot)) { + // exclude NaN value (NaN is ordered as largest floating point value) + result.Value = MakeNaNBoundary(pos, keyType, ctx); + } else { + result.Value = ctx.NewCallable(pos, "Nothing", { optKeyTypeNode }); + } + + result.Included = false; + return result; +} + +TRangeBoundary BuildMinusInf(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { + // start from first non-null value + auto optKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyType), ctx); + auto optBaseKeyTypeNode = ExpandType(pos, *ctx.MakeType<TOptionalExprType>(RemoveAllOptionals(keyType)), ctx); + auto largestNull = ctx.Builder(pos) + .Callable("SafeCast") + .Callable(0, "Nothing") + .Add(0, optBaseKeyTypeNode) + .Seal() + .Add(1, optKeyTypeNode) + .Seal() + .Build(); + + TRangeBoundary result; + result.Value = largestNull; + result.Included = false; + return result; +} + +TExprNode::TPtr MakeWidePointRangeLambda(TPositionHandle pos, const TTypeAnnotationNode* keyType, TExprContext& ctx) { + TStringBuf name; + TString maxValueStr; + + const TTypeAnnotationNode* baseKeyType = RemoveAllOptionals(keyType); + const auto keySlot = baseKeyType->Cast<TDataExprType>()->GetSlot(); + switch (keySlot) { + case EDataSlot::Int8: name = "Int8"; maxValueStr = ToString(Max<i8>()); break; + case EDataSlot::Uint8: name = "Uint8"; maxValueStr = ToString(Max<ui8>()); break; + case EDataSlot::Int16: name = "Int16"; maxValueStr = ToString(Max<i16>()); break; + case EDataSlot::Uint16: name = "Uint16"; maxValueStr = ToString(Max<ui16>()); break; + case EDataSlot::Int32: name = "Int32"; maxValueStr = ToString(Max<i32>()); break; + case EDataSlot::Uint32: name = "Uint32"; maxValueStr = ToString(Max<ui32>()); break; + case EDataSlot::Int64: name = "Int64"; maxValueStr = ToString(Max<i64>()); break; + case EDataSlot::Uint64: name = "Uint64"; maxValueStr = ToString(Max<ui64>()); break; + + case EDataSlot::Date: name = "Date"; maxValueStr = ToString(NUdf::MAX_DATE - 1); break; + case EDataSlot::Datetime: name = "Datetime"; maxValueStr = ToString(NUdf::MAX_DATETIME - 1); break; + default: + YQL_ENSURE(keySlot == EDataSlot::Timestamp); + name = "Timestamp"; + maxValueStr = ToString(NUdf::MAX_TIMESTAMP - 1); + } + + TExprNode::TPtr maxValue = ctx.NewCallable(pos, name, { ctx.NewAtom(pos, maxValueStr, TNodeFlags::Default) }); + TExprNode::TPtr addValue; + if (keySlot == EDataSlot::Date) { + addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "86400000000", TNodeFlags::Default) }); + } else if (keySlot == EDataSlot::Datetime) { + addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "1000000", TNodeFlags::Default) }); + } else if (keySlot == EDataSlot::Timestamp) { + addValue = ctx.NewCallable(pos, "Interval", { ctx.NewAtom(pos, "1", TNodeFlags::Default) }); + } else { + addValue = ctx.NewCallable(pos, name, { ctx.NewAtom(pos, "1", TNodeFlags::Default) }); + } + + TExprNode::TPtr key = ctx.NewArgument(pos, "optKey"); + TExprNode::TPtr castedKey = ctx.Builder(pos) + .Callable("SafeCast") + .Add(0, key) + .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(baseKeyType), ctx)) + .Seal() + .Build(); + + TExprNode::TPtr leftBoundary = ctx.Builder(pos) + .List() + .Add(0, key) + .Callable(1, "Int32") + .Atom(0, "1", TNodeFlags::Default) + .Seal() + .Seal() + .Build(); + + auto body = ctx.Builder(pos) + .Callable("AsRange") + .List(0) + .Add(0, leftBoundary) + .Callable(1, "If") + .Callable(0, "Coalesce") + .Callable(0, "==") + .Add(0, key) + .Add(1, maxValue) + .Seal() + .Callable(1, "Bool") + .Atom(0, "true", TNodeFlags::Default) + .Seal() + .Seal() + .Add(1, leftBoundary) + .List(2) + .Callable(0, "SafeCast") + .Callable(0, "+") + .Add(0, castedKey) + .Add(1, addValue) + .Seal() + .Add(1, ExpandType(pos, *keyType, ctx)) + .Seal() + .Callable(1, "Int32") + .Atom(0, "0", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); +} + +TExprNode::TPtr BuildNormalRangeLambdaRaw(TPositionHandle pos, const TTypeAnnotationNode* keyType, + TStringBuf op, TExprContext& ctx) +{ + // key is argument of Optional<keyType> type + const auto key = ctx.NewArgument(pos, "key"); + const auto keySlot = RemoveAllOptionals(keyType)->Cast<TDataExprType>()->GetSlot(); + const bool isTzKey = NUdf::GetDataTypeInfo(keySlot).Features & NUdf::EDataTypeFeatures::TzDateType; + const bool isIntegralOrDateKey = NUdf::GetDataTypeInfo(keySlot).Features & + (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType); + const auto downKey = isTzKey ? TzRound(key, keyType, true, ctx) : key; + const auto upKey = isTzKey ? TzRound(key, keyType, false, ctx) : key; + if (op == "!=") { + TRangeBoundary left, right; + left = BuildMinusInf(pos, keyType, ctx); + right.Value = downKey; + right.Included = false; + auto leftRange = BuildRange(pos, left, right, ctx); + + left.Value = upKey; + left.Included = false; + bool excludeNaN = false; + right = BuildPlusInf(pos, keyType, ctx, excludeNaN); + auto rightRange = BuildRange(pos, left, right, ctx); + + auto body = ctx.NewCallable(pos, "AsRange", { leftRange, rightRange }); + if (HasNaNs(keySlot)) { + // NaN is not equal to any value (including NaN itself), so we must include it + left.Included = true; + left.Value = MakeNaNBoundary(pos, keyType, ctx); + right = left; + auto nan = BuildRangeSingle(pos, left, right, ctx); + body = ctx.NewCallable(pos, "RangeUnion", { body, nan }); + } + return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); + } + + if (op == "===") { + if (isIntegralOrDateKey) { + return MakeWidePointRangeLambda(pos, keyType, ctx); + } + op = "=="; + } + + TRangeBoundary left, right; + if (op == "==") { + left.Value = downKey; + right.Value = upKey; + left.Included = right.Included = true; + } else if (op == "<" || op == "<=") { + left = BuildMinusInf(pos, keyType, ctx); + right.Value = (op == "<=") ? upKey : downKey; + right.Included = (op == "<="); + } else if (op == ">" || op == ">=") { + right = BuildPlusInf(pos, keyType, ctx); + left.Value = (op == ">=") ? downKey : upKey; + left.Included = (op == ">="); + } else if (op == "Exists" || op == "NotExists" ) { + YQL_ENSURE(keyType->GetKind() == ETypeAnnotationKind::Optional); + auto nullKey = ctx.Builder(pos) + .Callable("Just") + .Callable(0, "Nothing") + .Add(0, ExpandType(pos, *keyType, ctx)) + .Seal() + .Seal() + .Build(); + + if (op == "NotExists") { + left.Value = right.Value = nullKey; + left.Included = right.Included = true; + } else { + left.Value = nullKey; + left.Included = false; + // Exists should include NaN + bool excludeNaN = false; + right = BuildPlusInf(pos, keyType, ctx, excludeNaN); + } + } else { + YQL_ENSURE(false, "Unknown operation: " << op); + } + + TExprNode::TPtr body = BuildRangeSingle(pos, left, right, ctx); + if (op == "==" && HasNaNs(keySlot)) { + auto fullRangeWithoutNaNs = BuildRangeSingle(pos, + BuildMinusInf(pos, keyType, ctx), + BuildPlusInf(pos, keyType, ctx), ctx); + body = ctx.NewCallable(pos, "RangeIntersect", { body, fullRangeWithoutNaNs }); + } + + return ctx.NewLambda(pos, ctx.NewArguments(pos, { key }), std::move(body)); +} + +} //namespace + +TExprNode::TPtr ExpandRangeEmpty(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_ENSURE(node->IsCallable("RangeEmpty")); + if (node->ChildrenSize() == 0) { + return ctx.RenameNode(*node, "EmptyList"); + } + + return ctx.NewCallable(node->Pos(), "List", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) }); +} + +TExprNode::TPtr ExpandAsRange(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_ENSURE(node->IsCallable("AsRange")); + YQL_ENSURE(node->ChildrenSize()); + + return ctx.NewCallable(node->Pos(), "RangeCreate", { ctx.RenameNode(*node, "AsList") }); +} + +TExprNode::TPtr ExpandRangeFor(const TExprNode::TPtr& node, TExprContext& ctx) { + TPositionHandle pos = node->Pos(); + + TStringBuf op = node->Head().Content(); + auto value = node->ChildPtr(1); + const TTypeAnnotationNode* valueType = value->GetTypeAnn(); + const TTypeAnnotationNode* keyType = node->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + + const TTypeAnnotationNode* optKeyType = ctx.MakeType<TOptionalExprType>(keyType); + const TTypeAnnotationNode* keyBaseType = RemoveAllOptionals(keyType); + const TTypeAnnotationNode* valueBaseType = RemoveAllOptionals(valueType); + + const TExprNode::TPtr castedToKey = ctx.NewCallable(pos, "StrictCast", + { value, ExpandType(pos, *optKeyType, ctx) }); + + const TExprNode::TPtr emptyRange = ctx.NewCallable(pos, "RangeEmpty", + { ExpandType(pos, *keyType, ctx) }); + + TExprNode::TPtr rangeForNullCast; + if (op == "==" || op == "===" || op == "StartsWith") { + rangeForNullCast = emptyRange; + } else if (op == "!=" || op == "NotStartsWith") { + bool excludeNaN = false; + rangeForNullCast = BuildRangeSingle(pos, + BuildMinusInf(pos, keyType, ctx), + BuildPlusInf(pos, keyType, ctx, excludeNaN), ctx); + } + + TExprNode::TPtr result; + if (op == "Exists" || op == "NotExists") { + result = ctx.Builder(pos) + .Apply(BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) + .With(0, value) // value is not actually used for Exists/NotExists + .Seal() + .Build(); + } else if (op == "==" || op == "!=" || op == "===") { + YQL_ENSURE(rangeForNullCast); + result = ctx.Builder(pos) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, castedToKey) + .Seal() + .Add(1, rangeForNullCast) + .Apply(2, BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) + .With(0, castedToKey) + .Seal() + .Seal() + .Build(); + } else if (op == "StartsWith" || op == "NotStartsWith") { + YQL_ENSURE(rangeForNullCast); + + const auto boundary = ctx.NewArgument(pos, "boundary"); + const auto next = ctx.NewCallable(pos, "NextValue", { boundary }); + + TExprNode::TPtr rangeForIfNext; + TExprNode::TPtr rangeForIfNotNext; + + if (op == "StartsWith") { + const auto geBoundary = ctx.Builder(pos) + .Callable("RangeFor") + .Atom(0, ">=", TNodeFlags::Default) + .Add(1, boundary) + .Add(2, node->TailPtr()) + .Seal() + .Build(); + + rangeForIfNext = ctx.Builder(pos) + // [boundary, next) + .Callable("RangeIntersect") + .Add(0, geBoundary) + .Callable(1, "RangeFor") + .Atom(0, "<", TNodeFlags::Default) + .Add(1, next) + .Add(2, node->TailPtr()) + .Seal() + .Seal() + .Build(); + + // [boundary, +inf) + rangeForIfNotNext = geBoundary; + } else { + const auto lessBoundary = ctx.Builder(pos) + .Callable("RangeFor") + .Atom(0, "<", TNodeFlags::Default) + .Add(1, boundary) + .Add(2, node->TailPtr()) + .Seal() + .Build(); + + rangeForIfNext = ctx.Builder(pos) + // (-inf, boundary) U [next, +inf) + .Callable("RangeUnion") + .Add(0, lessBoundary) + .Callable(1, "RangeFor") + .Atom(0, ">=", TNodeFlags::Default) + .Add(1, next) + .Add(2, node->TailPtr()) + .Seal() + .Seal() + .Build(); + + // (-inf, boundary) + rangeForIfNotNext = lessBoundary; + } + + auto body = ctx.Builder(pos) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, next) + .Seal() + .Add(1, rangeForIfNotNext) + .Add(2, rangeForIfNext) + .Seal() + .Build(); + + YQL_ENSURE(rangeForNullCast); + result = ctx.Builder(pos) + .Callable("IfPresent") + .Callable(0, "StrictCast") + .Add(0, value) + .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(keyBaseType), ctx)) + .Seal() + .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, { boundary}), std::move(body))) + .Add(2, rangeForNullCast) + .Seal() + .Build(); + } else { + YQL_ENSURE(op == "<" || op == ">" || op == "<=" || op == ">="); + result = ctx.Builder(pos) + .Callable("If") + .Callable(0, "HasNull") + .Add(0, castedToKey) + .Seal() + .Callable(1, "IfPresent") + .Callable(0, "SafeCast") + .Add(0, value) + .Add(1, ExpandType(pos, *ctx.MakeType<TOptionalExprType>(valueBaseType), ctx)) + .Seal() + .Lambda(1) + .Param("unwrappedValue") + .Callable("IfPresent") + .Callable(0, op.StartsWith("<") ? "RoundDown" : "RoundUp") + .Arg(0, "unwrappedValue") + .Add(1, ExpandType(pos, *keyBaseType, ctx)) + .Seal() + .Lambda(1) + .Param("roundedKey") + .Apply(BuildNormalRangeLambdaRaw(pos, keyType, op.StartsWith("<") ? "<=" : ">=", ctx)) + .With(0) + .Callable("SafeCast") + .Arg(0, "roundedKey") + .Add(1, ExpandType(pos, *optKeyType, ctx)) + .Seal() + .Done() + .Seal() + .Seal() + .Add(2, emptyRange) + .Seal() + .Seal() + .Add(2, emptyRange) + .Seal() + .Apply(2, BuildNormalRangeLambdaRaw(pos, keyType, op, ctx)) + .With(0, castedToKey) + .Seal() + .Seal() + .Build(); + } + + return result; +} + +} diff --git a/ydb/library/yql/core/yql_opt_range.h b/ydb/library/yql/core/yql_opt_range.h index 627d842600..c814fbf9f0 100644 --- a/ydb/library/yql/core/yql_opt_range.h +++ b/ydb/library/yql/core/yql_opt_range.h @@ -1,11 +1,11 @@ -#pragma once +#pragma once #include <ydb/library/yql/ast/yql_expr.h> - -namespace NYql { - -TExprNode::TPtr ExpandRangeEmpty(const TExprNode::TPtr& node, TExprContext& ctx); -TExprNode::TPtr ExpandAsRange(const TExprNode::TPtr& node, TExprContext& ctx); -TExprNode::TPtr ExpandRangeFor(const TExprNode::TPtr& node, TExprContext& ctx); - -} - + +namespace NYql { + +TExprNode::TPtr ExpandRangeEmpty(const TExprNode::TPtr& node, TExprContext& ctx); +TExprNode::TPtr ExpandAsRange(const TExprNode::TPtr& node, TExprContext& ctx); +TExprNode::TPtr ExpandRangeFor(const TExprNode::TPtr& node, TExprContext& ctx); + +} + diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp index cc9998c8ab..69aa84a500 100644 --- a/ydb/library/yql/core/yql_opt_utils.cpp +++ b/ydb/library/yql/core/yql_opt_utils.cpp @@ -1,7 +1,7 @@ #include "yql_opt_utils.h" #include "yql_expr_optimize.h" #include "yql_expr_type_annotation.h" -#include "yql_type_annotation.h" +#include "yql_type_annotation.h" #include "yql_type_helpers.h" #include <ydb/library/yql/ast/yql_constraint.h> @@ -47,19 +47,19 @@ TExprNode::TPtr MakeBool(TPositionHandle position, bool value, TExprContext& ctx return value ? MakeBool<true>(position, ctx) : MakeBool<false>(position, ctx); } -TExprNode::TPtr MakeOptionalBool(TPositionHandle position, bool value, TExprContext& ctx) { - return ctx.NewCallable(position, "Just", { MakeBool(position, value, ctx)}); -} - -TExprNode::TPtr MakeIdentityLambda(TPositionHandle position, TExprContext& ctx) { - return ctx.Builder(position) - .Lambda() - .Param("arg") - .Arg("arg") - .Seal() - .Build(); -} - +TExprNode::TPtr MakeOptionalBool(TPositionHandle position, bool value, TExprContext& ctx) { + return ctx.NewCallable(position, "Just", { MakeBool(position, value, ctx)}); +} + +TExprNode::TPtr MakeIdentityLambda(TPositionHandle position, TExprContext& ctx) { + return ctx.Builder(position) + .Lambda() + .Param("arg") + .Arg("arg") + .Seal() + .Build(); +} + bool IsJustOrSingleAsList(const TExprNode& node) { return node.ChildrenSize() == 1U && node.IsCallable({"Just", "AsList"}); } @@ -111,13 +111,13 @@ bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& struct } bool IsPassthroughFlatMap(const TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) { - return IsPassthroughLambda(flatmap.Lambda(), passthroughFields, analyzeJustMember); -} + return IsPassthroughLambda(flatmap.Lambda(), passthroughFields, analyzeJustMember); +} + +bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) { + auto body = lambda.Body(); + auto arg = lambda.Args().Arg(0); -bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) { - auto body = lambda.Body(); - auto arg = lambda.Args().Arg(0); - TMaybeNode<TExprBase> outItem; if (IsJustOrSingleAsList(body.Ref())) { outItem = body.Ref().Child(0); @@ -173,38 +173,38 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* return false; } -bool IsTablePropsDependent(const TExprNode& node) { - bool found = false; - VisitExpr(node, [&found](const TExprNode& n) { +bool IsTablePropsDependent(const TExprNode& node) { + bool found = false; + VisitExpr(node, [&found](const TExprNode& n) { found = found || TCoTablePropBase::Match(&n) || (TCoMember::Match(&n) && TCoMember(&n).Name().Value().StartsWith("_yql_sys_")); - return !found; - }); - return found; -} - -TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { - auto columnOrder = typeCtx.LookupColumnOrder(src); - if (!columnOrder) { - return node; - } - - return ctx.Builder(node->Pos()) - .Callable("AssumeColumnOrder") - .Add(0, node) - .List(1) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - size_t index = 0; - for (auto& col : *columnOrder) { - parent - .Atom(index++, col); - } - return parent; - }) - .Seal() - .Seal() - .Build(); -} - + return !found; + }); + return found; +} + +TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { + auto columnOrder = typeCtx.LookupColumnOrder(src); + if (!columnOrder) { + return node; + } + + return ctx.Builder(node->Pos()) + .Callable("AssumeColumnOrder") + .Add(0, node) + .List(1) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t index = 0; + for (auto& col : *columnOrder) { + parent + .Atom(index++, col); + } + return parent; + }) + .Seal() + .Seal() + .Build(); +} + template<class TFieldsSet> bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TFieldsSet& usedFields, const TParentsMap& parentsMap, bool allowDependsOn) { if (arg.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { @@ -314,21 +314,21 @@ bool IsEmptyContainer(const TExprNode& node) { || (1U == node.ChildrenSize() && node.IsCallable({"List", "Nothing", "EmptyIterator", "Dict"})); } -const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type) { - if (!type || type->GetKind() != ETypeAnnotationKind::Optional) { - return type; - } - - return type->Cast<TOptionalExprType>()->GetItemType(); -} - -const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type) { - while (type && type->GetKind() == ETypeAnnotationKind::Optional) { - type = type->Cast<TOptionalExprType>()->GetItemType(); - } - return type; -} - +const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type) { + if (!type || type->GetKind() != ETypeAnnotationKind::Optional) { + return type; + } + + return type->Cast<TOptionalExprType>()->GetItemType(); +} + +const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type) { + while (type && type->GetKind() == ETypeAnnotationKind::Optional) { + type = type->Cast<TOptionalExprType>()->GetItemType(); + } + return type; +} + const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* type) { switch (type->GetKind()) { case ETypeAnnotationKind::List: return type->Cast<TListExprType>()->GetItemType(); @@ -350,23 +350,23 @@ TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) { return nullptr; } -bool HasSetting(const TExprNode& settings, const TStringBuf& name) { - return GetSetting(settings, name) != nullptr; -} - -bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names) { - for (auto& setting : settings.Children()) { - if (setting->ChildrenSize() != 0 && names.contains(setting->Head().Content())) { - return true; - } - } - return false; -} - -TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx) { +bool HasSetting(const TExprNode& settings, const TStringBuf& name) { + return GetSetting(settings, name) != nullptr; +} + +bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names) { + for (auto& setting : settings.Children()) { + if (setting->ChildrenSize() != 0 && names.contains(setting->Head().Content())) { + return true; + } + } + return false; +} + +TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx) { TExprNode::TListType children; for (auto setting : settings.Children()) { - if (setting->ChildrenSize() != 0 && setting->Head().Content() == name) { + if (setting->ChildrenSize() != 0 && setting->Head().Content() == name) { continue; } @@ -392,80 +392,80 @@ TExprNode::TPtr ReplaceSetting(const TExprNode& settings, TPositionHandle pos, c return ret; } -TExprNode::TPtr AddSetting(const TExprNode& settings, TPositionHandle pos, const TString& name, const TExprNode::TPtr& value, TExprContext& ctx) { - auto newChildren = settings.ChildrenList(); - newChildren.push_back(value ? ctx.NewList(pos, { ctx.NewAtom(pos, name), value }) : ctx.NewList(pos, { ctx.NewAtom(pos, name) })); - - auto ret = ctx.NewList(settings.Pos(), std::move(newChildren)); - return ret; -} - - -TExprNode::TPtr MergeSettings(const TExprNode& settings1, const TExprNode& settings2, TExprContext& ctx) { - auto newChildren = settings1.ChildrenList(); - newChildren.insert( - newChildren.cend(), - settings2.Children().begin(), - settings2.Children().end()); - auto ret = ctx.NewList(settings1.Pos(), std::move(newChildren)); - return ret; -} - -TMaybe<TIssue> ParseToDictSettings(const TExprNode& input, TExprContext& ctx, TMaybe<bool>& isMany, TMaybe<bool>& isHashed, TMaybe<ui64>& itemsCount, bool& isCompact) { - isCompact = false; - auto settings = input.Child(3); - if (settings->Type() != TExprNode::List) { - return TIssue(ctx.GetPosition(settings->Pos()), TStringBuilder() << "Expected tuple, but got: " << input.Type()); - } - - for (auto& child : settings->Children()) { - if (child->Type() == TExprNode::Atom) { - if (child->Content() == "One") { - isMany = false; - } - else if (child->Content() == "Many") { - isMany = true; - } - else if (child->Content() == "Sorted") { - isHashed = false; - } - else if (child->Content() == "Hashed") { - isHashed = true; - } - else if (child->Content() == "Compact") { - isCompact = true; - } - else { - return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Unsupported option: " << child->Content()); - } - } else if (child->Type() == TExprNode::List) { - if (child->ChildrenSize() != 2) { - return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected list with 2 elemenst"); - } - if (child->Child(0)->Content() == "ItemsCount") { - ui64 count = 0; - if (TryFromString(child->Child(1)->Content(), count)) { - itemsCount = count; - } else { - return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Bad 'ItemsCount' value: " << child->Child(1)->Content()); - } - } - else { - return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Bad option: " << child->Child(0)->Content()); - } - } else { - return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected atom or list, but got: " << input.Type()); - } - - } - - if (!isHashed || !isMany) { - return TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Both options must be specified: Sorted/Hashed and Many/One"); - } - - return TMaybe<TIssue>(); -} - +TExprNode::TPtr AddSetting(const TExprNode& settings, TPositionHandle pos, const TString& name, const TExprNode::TPtr& value, TExprContext& ctx) { + auto newChildren = settings.ChildrenList(); + newChildren.push_back(value ? ctx.NewList(pos, { ctx.NewAtom(pos, name), value }) : ctx.NewList(pos, { ctx.NewAtom(pos, name) })); + + auto ret = ctx.NewList(settings.Pos(), std::move(newChildren)); + return ret; +} + + +TExprNode::TPtr MergeSettings(const TExprNode& settings1, const TExprNode& settings2, TExprContext& ctx) { + auto newChildren = settings1.ChildrenList(); + newChildren.insert( + newChildren.cend(), + settings2.Children().begin(), + settings2.Children().end()); + auto ret = ctx.NewList(settings1.Pos(), std::move(newChildren)); + return ret; +} + +TMaybe<TIssue> ParseToDictSettings(const TExprNode& input, TExprContext& ctx, TMaybe<bool>& isMany, TMaybe<bool>& isHashed, TMaybe<ui64>& itemsCount, bool& isCompact) { + isCompact = false; + auto settings = input.Child(3); + if (settings->Type() != TExprNode::List) { + return TIssue(ctx.GetPosition(settings->Pos()), TStringBuilder() << "Expected tuple, but got: " << input.Type()); + } + + for (auto& child : settings->Children()) { + if (child->Type() == TExprNode::Atom) { + if (child->Content() == "One") { + isMany = false; + } + else if (child->Content() == "Many") { + isMany = true; + } + else if (child->Content() == "Sorted") { + isHashed = false; + } + else if (child->Content() == "Hashed") { + isHashed = true; + } + else if (child->Content() == "Compact") { + isCompact = true; + } + else { + return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Unsupported option: " << child->Content()); + } + } else if (child->Type() == TExprNode::List) { + if (child->ChildrenSize() != 2) { + return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected list with 2 elemenst"); + } + if (child->Child(0)->Content() == "ItemsCount") { + ui64 count = 0; + if (TryFromString(child->Child(1)->Content(), count)) { + itemsCount = count; + } else { + return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Bad 'ItemsCount' value: " << child->Child(1)->Content()); + } + } + else { + return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Bad option: " << child->Child(0)->Content()); + } + } else { + return TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected atom or list, but got: " << input.Type()); + } + + } + + if (!isHashed || !isMany) { + return TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Both options must be specified: Sorted/Hashed and Many/One"); + } + + return TMaybe<TIssue>(); +} + TExprNode::TPtr MakeSingleGroupRow(const TExprNode& aggregateNode, TExprNode::TPtr reduced, TExprContext& ctx) { auto pos = aggregateNode.Pos(); auto aggregatedColumns = aggregateNode.Child(2); @@ -781,10 +781,10 @@ TExprNode::TPtr ExpandFlattenStructs(const TExprNode::TPtr& node, TExprContext& } TExprNode::TPtr ExpandDivePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx) { - auto prefixes = node->Child(1)->Children(); + auto prefixes = node->Child(1)->Children(); MemberUpdaterFunc filterAndCutByPrefixFunc = [&prefixes](TString& memberName, const TTypeAnnotationNode*) { - for (const auto& p : prefixes) { - auto prefix = p->Content(); + for (const auto& p : prefixes) { + auto prefix = p->Content(); if (memberName.StartsWith(prefix)) { memberName = memberName.substr(prefix.length()); return true; @@ -928,20 +928,20 @@ TExprNode::TPtr ExpandFlattenByColumns(const TExprNode::TPtr& node, TExprContext return internalResult; } -TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); - auto targetType = node->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); - TExprNode::TListType items; - for (auto item : targetType->GetItems()) { - auto nameAtom = ctx.NewAtom(node->Pos(), item->GetName()); - auto tuple = ctx.NewList(node->Pos(), { - nameAtom, ctx.NewCallable(node->Pos(), "Member", { node->HeadPtr(), nameAtom }) }); - items.push_back(std::move(tuple)); - } - - return ctx.NewCallable(node->Pos(), "AsStruct", std::move(items)); -} - +TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_CLOG(DEBUG, Core) << "Expand " << node->Content(); + auto targetType = node->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); + TExprNode::TListType items; + for (auto item : targetType->GetItems()) { + auto nameAtom = ctx.NewAtom(node->Pos(), item->GetName()); + auto tuple = ctx.NewList(node->Pos(), { + nameAtom, ctx.NewCallable(node->Pos(), "Member", { node->HeadPtr(), nameAtom }) }); + items.push_back(std::move(tuple)); + } + + return ctx.NewCallable(node->Pos(), "AsStruct", std::move(items)); +} + void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns) { if (keySelectorBody->IsList()) { for (auto& child: keySelectorBody->Children()) { @@ -1009,97 +1009,97 @@ const TExprNode& SkipCallables(const TExprNode& node, const std::initializer_lis } return *p; } - -namespace { -TExprNode::TPtr ApplyWithCastStructForFirstArg(const TExprNode::TPtr& node, const TTypeAnnotationNode& targetType, TExprContext& ctx) { - YQL_ENSURE(node->IsLambda()); - TCoLambda lambda(node); - YQL_ENSURE(lambda.Args().Size() >= 1); - - TPositionHandle pos = lambda.Pos(); - - TExprNodeList args = lambda.Args().Ref().ChildrenList(); - TExprNode::TPtr body = lambda.Body().Ptr(); - - auto newArg = ctx.NewArgument(pos, "row"); - auto cast = ctx.NewCallable(pos, "CastStruct", { newArg, ExpandType(pos, targetType, ctx) }); - - body = ctx.ReplaceNodes(std::move(body), {{ args.front().Get(), cast }}); - args.front() = newArg; - - auto result = ctx.NewLambda(pos, ctx.NewArguments(pos, std::move(args)), std::move(body)); - return ctx.DeepCopyLambda(*result); + +namespace { +TExprNode::TPtr ApplyWithCastStructForFirstArg(const TExprNode::TPtr& node, const TTypeAnnotationNode& targetType, TExprContext& ctx) { + YQL_ENSURE(node->IsLambda()); + TCoLambda lambda(node); + YQL_ENSURE(lambda.Args().Size() >= 1); + + TPositionHandle pos = lambda.Pos(); + + TExprNodeList args = lambda.Args().Ref().ChildrenList(); + TExprNode::TPtr body = lambda.Body().Ptr(); + + auto newArg = ctx.NewArgument(pos, "row"); + auto cast = ctx.NewCallable(pos, "CastStruct", { newArg, ExpandType(pos, targetType, ctx) }); + + body = ctx.ReplaceNodes(std::move(body), {{ args.front().Get(), cast }}); + args.front() = newArg; + + auto result = ctx.NewLambda(pos, ctx.NewArguments(pos, std::move(args)), std::move(body)); + return ctx.DeepCopyLambda(*result); } - -} - -void ExtractSortKeyAndOrder(TPositionHandle pos, const TExprNode::TPtr& sortTraitsNode, TExprNode::TPtr& sortKey, TExprNode::TPtr& sortOrder, TExprContext& ctx) { - if (sortTraitsNode->IsCallable("SortTraits")) { - TCoSortTraits sortTraits(sortTraitsNode); - auto lambdaInputType = - sortTraits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); - sortOrder = sortTraits.SortDirections().Ptr(); - sortKey = ApplyWithCastStructForFirstArg(sortTraits.SortKeySelectorLambda().Ptr(), *lambdaInputType, ctx); - } else { - YQL_ENSURE(sortTraitsNode->IsCallable("Void")); - sortOrder = sortKey = ctx.NewCallable(pos, "Void", {}); - } -} - -void ExtractSessionWindowParams(TPositionHandle pos, const TExprNode::TPtr& sessionTraits, TExprNode::TPtr& sessionKey, - const TTypeAnnotationNode*& sessionKeyType, const TTypeAnnotationNode*& sessionParamsType, TExprNode::TPtr& sessionSortTraits, TExprNode::TPtr& sessionInit, - TExprNode::TPtr& sessionUpdate, TExprContext& ctx) -{ - sessionKey = sessionSortTraits = sessionInit = sessionUpdate = {}; - sessionKeyType = nullptr; - sessionParamsType = nullptr; - - if (sessionTraits && sessionTraits->IsCallable("SessionWindowTraits")) { - TCoSessionWindowTraits swt(sessionTraits); - auto lambdaInputType = - swt.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); - - sessionKeyType = swt.Calculate().Ref().GetTypeAnn(); - - TVector<const TItemExprType*> sessionParamItems; - sessionParamItems.push_back(ctx.MakeType<TItemExprType>("start", sessionKeyType)); - sessionParamItems.push_back(ctx.MakeType<TItemExprType>("state", swt.InitState().Ref().GetTypeAnn())); - sessionParamsType = ctx.MakeType<TStructExprType>(sessionParamItems); - - sessionSortTraits = swt.SortSpec().Ptr(); - sessionKey = ApplyWithCastStructForFirstArg(swt.Calculate().Ptr(), *lambdaInputType, ctx); - sessionInit = ApplyWithCastStructForFirstArg(swt.InitState().Ptr(), *lambdaInputType, ctx); - sessionUpdate = ApplyWithCastStructForFirstArg(swt.UpdateState().Ptr(), *lambdaInputType, ctx); - } else { - YQL_ENSURE(!sessionTraits || sessionTraits->IsCallable("Void")); - sessionSortTraits = ctx.NewCallable(pos, "Void", {}); - } -} - -TExprNode::TPtr BuildKeySelector(TPositionHandle pos, const TStructExprType& rowType, const TExprNode::TPtr& keyColumns, TExprContext& ctx) { - auto keyExtractorArg = ctx.NewArgument(pos, "item"); - TExprNode::TListType tupleItems; - for (auto column : keyColumns->Children()) { - auto itemType = rowType.GetItems()[*rowType.FindItem(column->Content())]->GetItemType(); - auto keyValue = ctx.NewCallable(pos, "Member", { keyExtractorArg, column }); - if (RemoveOptionalType(itemType)->GetKind() != ETypeAnnotationKind::Data) { - keyValue = ctx.NewCallable(pos, "StablePickle", { keyValue }); - } - - tupleItems.push_back(keyValue); - } - - TExprNode::TPtr tuple; - if (tupleItems.size() == 0) { - tuple = ctx.Builder(pos).Callable("Uint32").Atom(0, "0").Seal().Build(); - } else if (tupleItems.size() == 1) { - tuple = tupleItems[0]; - } else { - tuple = ctx.NewList(pos, std::move(tupleItems)); - } - return ctx.NewLambda(pos, ctx.NewArguments(pos, {keyExtractorArg}), std::move(tuple)); -} - + +} + +void ExtractSortKeyAndOrder(TPositionHandle pos, const TExprNode::TPtr& sortTraitsNode, TExprNode::TPtr& sortKey, TExprNode::TPtr& sortOrder, TExprContext& ctx) { + if (sortTraitsNode->IsCallable("SortTraits")) { + TCoSortTraits sortTraits(sortTraitsNode); + auto lambdaInputType = + sortTraits.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); + sortOrder = sortTraits.SortDirections().Ptr(); + sortKey = ApplyWithCastStructForFirstArg(sortTraits.SortKeySelectorLambda().Ptr(), *lambdaInputType, ctx); + } else { + YQL_ENSURE(sortTraitsNode->IsCallable("Void")); + sortOrder = sortKey = ctx.NewCallable(pos, "Void", {}); + } +} + +void ExtractSessionWindowParams(TPositionHandle pos, const TExprNode::TPtr& sessionTraits, TExprNode::TPtr& sessionKey, + const TTypeAnnotationNode*& sessionKeyType, const TTypeAnnotationNode*& sessionParamsType, TExprNode::TPtr& sessionSortTraits, TExprNode::TPtr& sessionInit, + TExprNode::TPtr& sessionUpdate, TExprContext& ctx) +{ + sessionKey = sessionSortTraits = sessionInit = sessionUpdate = {}; + sessionKeyType = nullptr; + sessionParamsType = nullptr; + + if (sessionTraits && sessionTraits->IsCallable("SessionWindowTraits")) { + TCoSessionWindowTraits swt(sessionTraits); + auto lambdaInputType = + swt.ListType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); + + sessionKeyType = swt.Calculate().Ref().GetTypeAnn(); + + TVector<const TItemExprType*> sessionParamItems; + sessionParamItems.push_back(ctx.MakeType<TItemExprType>("start", sessionKeyType)); + sessionParamItems.push_back(ctx.MakeType<TItemExprType>("state", swt.InitState().Ref().GetTypeAnn())); + sessionParamsType = ctx.MakeType<TStructExprType>(sessionParamItems); + + sessionSortTraits = swt.SortSpec().Ptr(); + sessionKey = ApplyWithCastStructForFirstArg(swt.Calculate().Ptr(), *lambdaInputType, ctx); + sessionInit = ApplyWithCastStructForFirstArg(swt.InitState().Ptr(), *lambdaInputType, ctx); + sessionUpdate = ApplyWithCastStructForFirstArg(swt.UpdateState().Ptr(), *lambdaInputType, ctx); + } else { + YQL_ENSURE(!sessionTraits || sessionTraits->IsCallable("Void")); + sessionSortTraits = ctx.NewCallable(pos, "Void", {}); + } +} + +TExprNode::TPtr BuildKeySelector(TPositionHandle pos, const TStructExprType& rowType, const TExprNode::TPtr& keyColumns, TExprContext& ctx) { + auto keyExtractorArg = ctx.NewArgument(pos, "item"); + TExprNode::TListType tupleItems; + for (auto column : keyColumns->Children()) { + auto itemType = rowType.GetItems()[*rowType.FindItem(column->Content())]->GetItemType(); + auto keyValue = ctx.NewCallable(pos, "Member", { keyExtractorArg, column }); + if (RemoveOptionalType(itemType)->GetKind() != ETypeAnnotationKind::Data) { + keyValue = ctx.NewCallable(pos, "StablePickle", { keyValue }); + } + + tupleItems.push_back(keyValue); + } + + TExprNode::TPtr tuple; + if (tupleItems.size() == 0) { + tuple = ctx.Builder(pos).Callable("Uint32").Atom(0, "0").Seal().Build(); + } else if (tupleItems.size() == 1) { + tuple = tupleItems[0]; + } else { + tuple = ctx.NewList(pos, std::move(tupleItems)); + } + return ctx.NewLambda(pos, ctx.NewArguments(pos, {keyExtractorArg}), std::move(tuple)); +} + template <bool Cannonize, bool EnableNewOptimizers> TExprNode::TPtr OptimizeIfPresent(const TExprNode::TPtr& node, TExprContext& ctx) { auto optionals = node->ChildrenList(); @@ -1249,7 +1249,7 @@ TExprNode::TPtr OptimizeIfPresent(const TExprNode::TPtr& node, TExprContext& ctx } return node; -} +} template TExprNode::TPtr OptimizeIfPresent<true, true>(const TExprNode::TPtr& node, TExprContext& ctx); template TExprNode::TPtr OptimizeIfPresent<false, true>(const TExprNode::TPtr& node, TExprContext& ctx); @@ -1291,21 +1291,21 @@ TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx) } return node; -} - -bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx) { - YQL_ENSURE(unourderedSubquery.IsCallable("UnorderedSubquery")); - - auto issueCode = EYqlIssueCode::TIssuesIds_EIssueCode_YQL_ORDER_BY_WITHOUT_LIMIT_IN_SUBQUERY; - auto issue = TIssue(ctx.GetPosition(unourderedSubquery.Head().Pos()), "ORDER BY in subquery will be ignored"); - auto subIssue = TIssue(ctx.GetPosition(unourderedSubquery.Pos()), "When used here"); - SetIssueCode(issueCode, issue); - SetIssueCode(issueCode, subIssue); - issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); - - return ctx.AddWarning(issue); } - + +bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx) { + YQL_ENSURE(unourderedSubquery.IsCallable("UnorderedSubquery")); + + auto issueCode = EYqlIssueCode::TIssuesIds_EIssueCode_YQL_ORDER_BY_WITHOUT_LIMIT_IN_SUBQUERY; + auto issue = TIssue(ctx.GetPosition(unourderedSubquery.Head().Pos()), "ORDER BY in subquery will be ignored"); + auto subIssue = TIssue(ctx.GetPosition(unourderedSubquery.Pos()), "When used here"); + SetIssueCode(issueCode, issue); + SetIssueCode(issueCode, subIssue); + issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); + + return ctx.AddWarning(issue); +} + TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx) { TNodeOnNodeOwnedMap replaces; const auto isStream = [] (const TTypeAnnotationNode* type) { @@ -1325,7 +1325,7 @@ TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::f return true; }); return ctx.ReplaceNodes(std::move(lambda), replaces); -} +} IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx) { output = input; diff --git a/ydb/library/yql/core/yql_opt_utils.h b/ydb/library/yql/core/yql_opt_utils.h index e73077e202..00635fea81 100644 --- a/ydb/library/yql/core/yql_opt_utils.h +++ b/ydb/library/yql/core/yql_opt_utils.h @@ -19,11 +19,11 @@ bool IsFilterFlatMap(const NNodes::TCoLambda& lambda); bool IsListReorder(const TExprNode& node); bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode); bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); -bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); -bool IsTablePropsDependent(const TExprNode& node); +bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); +bool IsTablePropsDependent(const TExprNode& node); + +TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); -TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); - // returns true if usedFields contains subset of fields template<class TFieldsSet> bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TFieldsSet& usedFields, const TParentsMap& parentsMap, bool allowDependsOn = true); @@ -37,20 +37,20 @@ bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx); bool IsEmptyOptional(const TExprNode& node); bool IsEmptyContainer(const TExprNode& node); -const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type); -const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type); +const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type); +const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type); const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* seq); - + TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name); -bool HasSetting(const TExprNode& settings, const TStringBuf& name); -bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names); -TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx); -TExprNode::TPtr AddSetting(const TExprNode& settings, TPositionHandle pos, const TString& name, const TExprNode::TPtr& value, TExprContext& ctx); -TExprNode::TPtr MergeSettings(const TExprNode& settings1, const TExprNode& settings2, TExprContext& ctx); +bool HasSetting(const TExprNode& settings, const TStringBuf& name); +bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names); +TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx); +TExprNode::TPtr AddSetting(const TExprNode& settings, TPositionHandle pos, const TString& name, const TExprNode::TPtr& value, TExprContext& ctx); +TExprNode::TPtr MergeSettings(const TExprNode& settings1, const TExprNode& settings2, TExprContext& ctx); TExprNode::TPtr ReplaceSetting(const TExprNode& settings, TPositionHandle pos, const TString& name, const TExprNode::TPtr& value, TExprContext& ctx); -TMaybe<TIssue> ParseToDictSettings(const TExprNode& node, TExprContext& ctx, TMaybe<bool>& isMany, TMaybe<bool>& isHashed, TMaybe<ui64>& itemsCount, bool& isCompact); - +TMaybe<TIssue> ParseToDictSettings(const TExprNode& node, TExprContext& ctx, TMaybe<bool>& isMany, TMaybe<bool>& isHashed, TMaybe<ui64>& itemsCount, bool& isCompact); + using MemberUpdaterFunc = std::function<bool (TString& memberName, const TTypeAnnotationNode* TypeAnnotation)>; bool UpdateStructMembers(TExprContext& ctx, const TExprNode::TPtr& node, const TStringBuf& goal, TExprNode::TListType& members, MemberUpdaterFunc updaterFunc = MemberUpdaterFunc(), const TTypeAnnotationNode* nodeType = nullptr); @@ -64,7 +64,7 @@ TExprNode::TPtr ExpandDivePrefixMembers(const TExprNode::TPtr& node, TExprContex TExprNode::TPtr ExpandAddMember(const TExprNode::TPtr& node, TExprContext& ctx); TExprNode::TPtr ExpandReplaceMember(const TExprNode::TPtr& node, TExprContext& ctx); TExprNode::TPtr ExpandFlattenByColumns(const TExprNode::TPtr& node, TExprContext& ctx); -TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx); +TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx); void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns); inline void ExtractSimpleKeys(const TExprNode& keySelectorLambda, TVector<TStringBuf>& columns) { @@ -76,27 +76,27 @@ TExprNode::TPtr MakeNull(TPositionHandle position, TExprContext& ctx); TExprNode::TPtr MakeConstMap(TPositionHandle position, const TExprNode::TPtr& input, const TExprNode::TPtr& value, TExprContext& ctx); TExprNode::TPtr MakeBoolNothing(TPositionHandle position, TExprContext& ctx); TExprNode::TPtr MakeBool(TPositionHandle position, bool value, TExprContext& ctx); -TExprNode::TPtr MakeOptionalBool(TPositionHandle position, bool value, TExprContext& ctx); +TExprNode::TPtr MakeOptionalBool(TPositionHandle position, bool value, TExprContext& ctx); template <bool Bool> TExprNode::TPtr MakeBool(TPositionHandle position, TExprContext& ctx); -TExprNode::TPtr MakeIdentityLambda(TPositionHandle position, TExprContext& ctx); +TExprNode::TPtr MakeIdentityLambda(TPositionHandle position, TExprContext& ctx); constexpr std::initializer_list<std::string_view> SkippableCallables = {"Unordered", "AssumeSorted", "AssumeUnique", "AssumeColumnOrder", "AssumeAllMembersNullableAtOnce"}; const TExprNode& SkipCallables(const TExprNode& node, const std::initializer_list<std::string_view>& skipCallables); -void ExtractSortKeyAndOrder(TPositionHandle pos, const TExprNode::TPtr& sortTraitsNode, TExprNode::TPtr& sortKey, TExprNode::TPtr& sortOrder, TExprContext& ctx); -void ExtractSessionWindowParams(TPositionHandle pos, const TExprNode::TPtr& sessionTraits, TExprNode::TPtr& sessionKey, - const TTypeAnnotationNode*& sessionKeyType, const TTypeAnnotationNode*& sessionParamsType, TExprNode::TPtr& sessionSortTraits, TExprNode::TPtr& sessionInit, - TExprNode::TPtr& sessionUpdate, TExprContext& ctx); - -TExprNode::TPtr BuildKeySelector(TPositionHandle pos, const TStructExprType& rowType, const TExprNode::TPtr& keyColumns, TExprContext& ctx); +void ExtractSortKeyAndOrder(TPositionHandle pos, const TExprNode::TPtr& sortTraitsNode, TExprNode::TPtr& sortKey, TExprNode::TPtr& sortOrder, TExprContext& ctx); +void ExtractSessionWindowParams(TPositionHandle pos, const TExprNode::TPtr& sessionTraits, TExprNode::TPtr& sessionKey, + const TTypeAnnotationNode*& sessionKeyType, const TTypeAnnotationNode*& sessionParamsType, TExprNode::TPtr& sessionSortTraits, TExprNode::TPtr& sessionInit, + TExprNode::TPtr& sessionUpdate, TExprContext& ctx); + +TExprNode::TPtr BuildKeySelector(TPositionHandle pos, const TStructExprType& rowType, const TExprNode::TPtr& keyColumns, TExprContext& ctx); template <bool Cannonize, bool EnableNewOptimizers = true> TExprNode::TPtr OptimizeIfPresent(const TExprNode::TPtr& node, TExprContext& ctx); TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx); - -bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx); + +bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx); TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx); IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx); diff --git a/ydb/library/yql/core/yql_opt_window.cpp b/ydb/library/yql/core/yql_opt_window.cpp index 026837c42e..5fc03d35d9 100644 --- a/ydb/library/yql/core/yql_opt_window.cpp +++ b/ydb/library/yql/core/yql_opt_window.cpp @@ -3,360 +3,360 @@ #include "yql_expr_type_annotation.h" #include <ydb/library/yql/utils/log/log.h> - + namespace NYql { using namespace NNodes; -namespace { - -const TStringBuf SessionStartMemberName = "_yql_window_session_start"; -const TStringBuf SessionParamsMemberName = "_yql_window_session_params"; - -enum class EFrameType : ui8 { - EMPTY, - LAGGING, - CURRENT, - LEADING, - FULL, - GENERIC, -}; - -EFrameType FrameType(const TWindowFrameSettings& settings) { - auto first = settings.First; - auto last = settings.Last; - - if (first.Defined() && last.Defined() && first > last) { - return EFrameType::EMPTY; - } - - if (!first.Defined()) { - if (!last.Defined()) { - return EFrameType::FULL; - } - if (*last < 0) { - return EFrameType::LAGGING; - } - - return *last > 0 ? EFrameType::LEADING : EFrameType::CURRENT; - } - - return EFrameType::GENERIC; -} - -TExprNode::TPtr ReplaceLastLambdaArgWithStringLiteral(const TExprNode& lambda, TStringBuf literal, TExprContext& ctx) { - YQL_ENSURE(lambda.IsLambda()); - TExprNodeList args = lambda.ChildPtr(0)->ChildrenList(); - YQL_ENSURE(!args.empty()); - - auto literalNode = ctx.Builder(lambda.Pos()) - .Callable("String") - .Atom(0, literal) - .Seal() - .Build(); - auto newBody = ctx.ReplaceNodes(lambda.ChildPtr(1), {{args.back().Get(), literalNode}}); - args.pop_back(); - return ctx.NewLambda(lambda.Pos(), ctx.NewArguments(lambda.Pos(), std::move(args)), std::move(newBody)); -} - -TExprNode::TPtr ReplaceFirstLambdaArgWithCastStruct(const TExprNode& lambda, const TTypeAnnotationNode& targetType, TExprContext& ctx) { - YQL_ENSURE(lambda.IsLambda()); - YQL_ENSURE(targetType.GetKind() == ETypeAnnotationKind::Struct); - TExprNodeList args = lambda.ChildPtr(0)->ChildrenList(); - YQL_ENSURE(!args.empty()); - - auto newArg = ctx.NewArgument(lambda.Pos(), "row"); - - auto cast = ctx.Builder(lambda.Pos()) - .Callable("MatchType") - .Add(0, newArg) - .Atom(1, "Optional", TNodeFlags::Default) - .Lambda(2) - .Param("row") - .Callable("Map") - .Arg(0, "row") - .Lambda(1) - .Param("unwrapped") - .Callable("CastStruct") - .Arg(0, "unwrapped") - .Add(1, ExpandType(lambda.Pos(), targetType, ctx)) - .Seal() - .Seal() - .Seal() - .Seal() - .Lambda(3) - .Param("row") - .Callable("CastStruct") - .Arg(0, "row") - .Add(1, ExpandType(lambda.Pos(), targetType, ctx)) - .Seal() - .Seal() - .Seal() - .Build(); - - auto newBody = ctx.ReplaceNodes(lambda.ChildPtr(1), {{args.front().Get(), cast}}); - args[0] = newArg; - return ctx.NewLambda(lambda.Pos(), ctx.NewArguments(lambda.Pos(), std::move(args)), std::move(newBody)); -} - -TExprNode::TPtr AddOptionalIfNotAlreadyOptionalOrNull(const TExprNode::TPtr& lambda, TExprContext& ctx) { - YQL_ENSURE(lambda->IsLambda()); - YQL_ENSURE(lambda->ChildPtr(0)->ChildrenSize() == 1); - - auto identity = MakeIdentityLambda(lambda->Pos(), ctx); - return ctx.Builder(lambda->Pos()) - .Lambda() - .Param("arg") - .Callable("MatchType") - .Apply(0, lambda) - .With(0, "arg") - .Seal() - .Atom(1, "Optional", TNodeFlags::Default) - .Add(2, identity) - .Atom(3, "Null", TNodeFlags::Default) - .Add(4, identity) - .Lambda(5) - .Param("result") - .Callable("Just") - .Arg(0, "result") - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -struct TRawTrait { - TPositionHandle Pos; - - // Init/Update/Default are set only for aggregations - TExprNode::TPtr InitLambda; - TExprNode::TPtr UpdateLambda; - TExprNode::TPtr DefaultValue; - - TExprNode::TPtr CalculateLambda; - TMaybe<i64> CalculateLambdaLead; // lead/lag for input to CalculateLambda; - - - const TTypeAnnotationNode* OutputType = nullptr; - - TWindowFrameSettings FrameSettings; -}; - - -struct TCalcOverWindowTraits { - TMap<TStringBuf, TRawTrait> RawTraits; - ui64 MaxDataOutpace = 0; - ui64 MaxDataLag = 0; - ui64 MaxUnboundedPrecedingLag = 0; - const TTypeAnnotationNode* LagQueueItemType = nullptr; -}; - - -TCalcOverWindowTraits ExtractCalcOverWindowTraits(const TExprNode::TPtr& frames, TExprContext& ctx) { - TCalcOverWindowTraits result; - - auto& maxDataOutpace = result.MaxDataOutpace; - auto& maxDataLag = result.MaxDataLag; - auto& maxUnboundedPrecedingLag = result.MaxUnboundedPrecedingLag; - - TVector<const TItemExprType*> lagQueueStructItems; - for (auto& winOn : frames->ChildrenList()) { - YQL_ENSURE(winOn->IsCallable("WinOnRows")); - - TWindowFrameSettings frameSettings; - YQL_ENSURE(ParseWindowFrameSettings(*winOn, frameSettings, ctx)); - - ui64 frameOutpace = 0; - ui64 frameLag = 0; - const EFrameType frameType = FrameType(frameSettings); - const auto frameFirst = frameSettings.First; - const auto frameLast = frameSettings.Last; - - if (frameType != EFrameType::EMPTY) { - if (!frameLast.Defined() || *frameLast > 0) { - frameOutpace = frameLast.Defined() ? ui64(*frameLast) : Max<ui64>(); - } - - if (frameFirst.Defined() && *frameFirst < 0) { - frameLag = ui64(0 - *frameFirst); - } - } - - const auto& winOnChildren = winOn->ChildrenList(); - YQL_ENSURE(winOnChildren.size() > 1); - for (size_t i = 1; i < winOnChildren.size(); ++i) { - auto item = winOnChildren[i]; - YQL_ENSURE(item->IsList()); - - auto nameNode = item->Child(0); - YQL_ENSURE(nameNode->IsAtom()); - - TStringBuf name = nameNode->Content(); - - YQL_ENSURE(!result.RawTraits.contains(name)); - - auto traits = item->Child(1); - - auto& rawTraits = result.RawTraits[name]; - rawTraits.FrameSettings = frameSettings; - rawTraits.Pos = traits->Pos(); - - if (traits->IsCallable("WindowTraits")) { - maxDataOutpace = Max(maxDataOutpace, frameOutpace); - maxDataLag = Max(maxDataLag, frameLag); - - auto initLambda = traits->ChildPtr(1); - auto updateLambda = traits->ChildPtr(2); - auto calculateLambda = traits->ChildPtr(4); - - rawTraits.OutputType = calculateLambda->GetTypeAnn(); - YQL_ENSURE(rawTraits.OutputType); - - if (initLambda->Child(0)->ChildrenSize() == 2) { - initLambda = ReplaceLastLambdaArgWithStringLiteral(*initLambda, name, ctx); - } - - if (updateLambda->Child(0)->ChildrenSize() == 3) { - updateLambda = ReplaceLastLambdaArgWithStringLiteral(*updateLambda, name, ctx); - } - - auto lambdaInputType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - initLambda = ReplaceFirstLambdaArgWithCastStruct(*initLambda, *lambdaInputType, ctx); - updateLambda = ReplaceFirstLambdaArgWithCastStruct(*updateLambda, *lambdaInputType, ctx); - - rawTraits.InitLambda = initLambda; - rawTraits.UpdateLambda = updateLambda; - rawTraits.CalculateLambda = calculateLambda; - rawTraits.DefaultValue = traits->ChildPtr(5); - - if (frameType == EFrameType::LAGGING) { - maxUnboundedPrecedingLag = Max(maxUnboundedPrecedingLag, ui64(abs(*frameLast))); - lagQueueStructItems.push_back(ctx.MakeType<TItemExprType>(name, rawTraits.OutputType)); - } - } else if (traits->IsCallable({"Lead", "Lag"})) { - i64 lead = 1; - if (traits->ChildrenSize() == 3) { - YQL_ENSURE(traits->Child(2)->IsCallable("Int64")); - lead = FromString<i64>(traits->Child(2)->Child(0)->Content()); - } - - if (traits->IsCallable("Lag")) { - lead = -lead; - } - - if (lead < 0) { - maxDataLag = Max(maxDataLag, ui64(abs(lead))); - } else { - maxDataOutpace = Max<ui64>(maxDataOutpace, lead); - } - - auto lambdaInputType = - traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); - - rawTraits.CalculateLambda = ReplaceFirstLambdaArgWithCastStruct(*traits->Child(1), *lambdaInputType, ctx); - rawTraits.CalculateLambdaLead = lead; - rawTraits.OutputType = traits->Child(1)->GetTypeAnn(); - YQL_ENSURE(rawTraits.OutputType); - } else { - YQL_ENSURE(traits->IsCallable({"RowNumber", "Rank", "DenseRank"})); - - rawTraits.CalculateLambda = traits; - rawTraits.OutputType = traits->IsCallable("RowNumber") ? - ctx.MakeType<TDataExprType>(EDataSlot::Uint64) : traits->Child(1)->GetTypeAnn(); - } - } - } - - result.LagQueueItemType = ctx.MakeType<TStructExprType>(lagQueueStructItems); - - return result; -} - -TExprNode::TPtr BuildUint64(TPositionHandle pos, ui64 value, TExprContext& ctx) { - return ctx.Builder(pos) - .Callable("Uint64") - .Atom(0, ToString(value)) - .Seal() - .Build(); -} - -TExprNode::TPtr BuildQueuePeek(TPositionHandle pos, const TExprNode::TPtr& queue, ui64 index, const TExprNode::TPtr& dependsOn, - TExprContext& ctx) -{ - return ctx.Builder(pos) - .Callable("QueuePeek") - .Add(0, queue) - .Add(1, BuildUint64(pos, index, ctx)) - .Callable(2, "DependsOn") - .Add(0, dependsOn) - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildQueueRange(TPositionHandle pos, const TExprNode::TPtr& queue, ui64 begin, ui64 end, - const TExprNode::TPtr& dependsOn, TExprContext& ctx) -{ - return ctx.Builder(pos) - .Callable("FlatMap") - .Callable(0, "QueueRange") - .Add(0, queue) - .Add(1, BuildUint64(pos, begin, ctx)) - .Add(2, BuildUint64(pos, end, ctx)) - .Callable(3, "DependsOn") - .Add(0, dependsOn) - .Seal() - .Seal() - .Lambda(1) - .Param("item") - .Arg("item") - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr BuildQueue(TPositionHandle pos, const TTypeAnnotationNode& itemType, ui64 queueSize, ui64 initSize, - const TExprNode::TPtr& dependsOn, TExprContext& ctx) -{ - TExprNode::TPtr size; - if (queueSize == Max<ui64>()) { - size = ctx.NewCallable(pos, "Void", {}); - } else { - size = BuildUint64(pos, queueSize, ctx); - } - - return ctx.Builder(pos) - .Callable("QueueCreate") - .Add(0, ExpandType(pos, itemType, ctx)) - .Add(1, size) - .Add(2, BuildUint64(pos, initSize, ctx)) - .Callable(3, "DependsOn") - .Add(0, dependsOn) - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr CoalesceQueueOutput(TPositionHandle pos, const TExprNode::TPtr& output, bool rawOutputIsOptional, - const TExprNode::TPtr& defaultValue, TExprContext& ctx) -{ - if (defaultValue->IsCallable("Null")) { - if (!rawOutputIsOptional) { - return output; - } - - return ctx.Builder(pos) - .Callable("FlatMap") - .Add(0, output) - .Lambda(1) - .Param("item") - .Arg("item") - .Seal() - .Seal() - .Build(); - } - +namespace { + +const TStringBuf SessionStartMemberName = "_yql_window_session_start"; +const TStringBuf SessionParamsMemberName = "_yql_window_session_params"; + +enum class EFrameType : ui8 { + EMPTY, + LAGGING, + CURRENT, + LEADING, + FULL, + GENERIC, +}; + +EFrameType FrameType(const TWindowFrameSettings& settings) { + auto first = settings.First; + auto last = settings.Last; + + if (first.Defined() && last.Defined() && first > last) { + return EFrameType::EMPTY; + } + + if (!first.Defined()) { + if (!last.Defined()) { + return EFrameType::FULL; + } + if (*last < 0) { + return EFrameType::LAGGING; + } + + return *last > 0 ? EFrameType::LEADING : EFrameType::CURRENT; + } + + return EFrameType::GENERIC; +} + +TExprNode::TPtr ReplaceLastLambdaArgWithStringLiteral(const TExprNode& lambda, TStringBuf literal, TExprContext& ctx) { + YQL_ENSURE(lambda.IsLambda()); + TExprNodeList args = lambda.ChildPtr(0)->ChildrenList(); + YQL_ENSURE(!args.empty()); + + auto literalNode = ctx.Builder(lambda.Pos()) + .Callable("String") + .Atom(0, literal) + .Seal() + .Build(); + auto newBody = ctx.ReplaceNodes(lambda.ChildPtr(1), {{args.back().Get(), literalNode}}); + args.pop_back(); + return ctx.NewLambda(lambda.Pos(), ctx.NewArguments(lambda.Pos(), std::move(args)), std::move(newBody)); +} + +TExprNode::TPtr ReplaceFirstLambdaArgWithCastStruct(const TExprNode& lambda, const TTypeAnnotationNode& targetType, TExprContext& ctx) { + YQL_ENSURE(lambda.IsLambda()); + YQL_ENSURE(targetType.GetKind() == ETypeAnnotationKind::Struct); + TExprNodeList args = lambda.ChildPtr(0)->ChildrenList(); + YQL_ENSURE(!args.empty()); + + auto newArg = ctx.NewArgument(lambda.Pos(), "row"); + + auto cast = ctx.Builder(lambda.Pos()) + .Callable("MatchType") + .Add(0, newArg) + .Atom(1, "Optional", TNodeFlags::Default) + .Lambda(2) + .Param("row") + .Callable("Map") + .Arg(0, "row") + .Lambda(1) + .Param("unwrapped") + .Callable("CastStruct") + .Arg(0, "unwrapped") + .Add(1, ExpandType(lambda.Pos(), targetType, ctx)) + .Seal() + .Seal() + .Seal() + .Seal() + .Lambda(3) + .Param("row") + .Callable("CastStruct") + .Arg(0, "row") + .Add(1, ExpandType(lambda.Pos(), targetType, ctx)) + .Seal() + .Seal() + .Seal() + .Build(); + + auto newBody = ctx.ReplaceNodes(lambda.ChildPtr(1), {{args.front().Get(), cast}}); + args[0] = newArg; + return ctx.NewLambda(lambda.Pos(), ctx.NewArguments(lambda.Pos(), std::move(args)), std::move(newBody)); +} + +TExprNode::TPtr AddOptionalIfNotAlreadyOptionalOrNull(const TExprNode::TPtr& lambda, TExprContext& ctx) { + YQL_ENSURE(lambda->IsLambda()); + YQL_ENSURE(lambda->ChildPtr(0)->ChildrenSize() == 1); + + auto identity = MakeIdentityLambda(lambda->Pos(), ctx); + return ctx.Builder(lambda->Pos()) + .Lambda() + .Param("arg") + .Callable("MatchType") + .Apply(0, lambda) + .With(0, "arg") + .Seal() + .Atom(1, "Optional", TNodeFlags::Default) + .Add(2, identity) + .Atom(3, "Null", TNodeFlags::Default) + .Add(4, identity) + .Lambda(5) + .Param("result") + .Callable("Just") + .Arg(0, "result") + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +struct TRawTrait { + TPositionHandle Pos; + + // Init/Update/Default are set only for aggregations + TExprNode::TPtr InitLambda; + TExprNode::TPtr UpdateLambda; + TExprNode::TPtr DefaultValue; + + TExprNode::TPtr CalculateLambda; + TMaybe<i64> CalculateLambdaLead; // lead/lag for input to CalculateLambda; + + + const TTypeAnnotationNode* OutputType = nullptr; + + TWindowFrameSettings FrameSettings; +}; + + +struct TCalcOverWindowTraits { + TMap<TStringBuf, TRawTrait> RawTraits; + ui64 MaxDataOutpace = 0; + ui64 MaxDataLag = 0; + ui64 MaxUnboundedPrecedingLag = 0; + const TTypeAnnotationNode* LagQueueItemType = nullptr; +}; + + +TCalcOverWindowTraits ExtractCalcOverWindowTraits(const TExprNode::TPtr& frames, TExprContext& ctx) { + TCalcOverWindowTraits result; + + auto& maxDataOutpace = result.MaxDataOutpace; + auto& maxDataLag = result.MaxDataLag; + auto& maxUnboundedPrecedingLag = result.MaxUnboundedPrecedingLag; + + TVector<const TItemExprType*> lagQueueStructItems; + for (auto& winOn : frames->ChildrenList()) { + YQL_ENSURE(winOn->IsCallable("WinOnRows")); + + TWindowFrameSettings frameSettings; + YQL_ENSURE(ParseWindowFrameSettings(*winOn, frameSettings, ctx)); + + ui64 frameOutpace = 0; + ui64 frameLag = 0; + const EFrameType frameType = FrameType(frameSettings); + const auto frameFirst = frameSettings.First; + const auto frameLast = frameSettings.Last; + + if (frameType != EFrameType::EMPTY) { + if (!frameLast.Defined() || *frameLast > 0) { + frameOutpace = frameLast.Defined() ? ui64(*frameLast) : Max<ui64>(); + } + + if (frameFirst.Defined() && *frameFirst < 0) { + frameLag = ui64(0 - *frameFirst); + } + } + + const auto& winOnChildren = winOn->ChildrenList(); + YQL_ENSURE(winOnChildren.size() > 1); + for (size_t i = 1; i < winOnChildren.size(); ++i) { + auto item = winOnChildren[i]; + YQL_ENSURE(item->IsList()); + + auto nameNode = item->Child(0); + YQL_ENSURE(nameNode->IsAtom()); + + TStringBuf name = nameNode->Content(); + + YQL_ENSURE(!result.RawTraits.contains(name)); + + auto traits = item->Child(1); + + auto& rawTraits = result.RawTraits[name]; + rawTraits.FrameSettings = frameSettings; + rawTraits.Pos = traits->Pos(); + + if (traits->IsCallable("WindowTraits")) { + maxDataOutpace = Max(maxDataOutpace, frameOutpace); + maxDataLag = Max(maxDataLag, frameLag); + + auto initLambda = traits->ChildPtr(1); + auto updateLambda = traits->ChildPtr(2); + auto calculateLambda = traits->ChildPtr(4); + + rawTraits.OutputType = calculateLambda->GetTypeAnn(); + YQL_ENSURE(rawTraits.OutputType); + + if (initLambda->Child(0)->ChildrenSize() == 2) { + initLambda = ReplaceLastLambdaArgWithStringLiteral(*initLambda, name, ctx); + } + + if (updateLambda->Child(0)->ChildrenSize() == 3) { + updateLambda = ReplaceLastLambdaArgWithStringLiteral(*updateLambda, name, ctx); + } + + auto lambdaInputType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + initLambda = ReplaceFirstLambdaArgWithCastStruct(*initLambda, *lambdaInputType, ctx); + updateLambda = ReplaceFirstLambdaArgWithCastStruct(*updateLambda, *lambdaInputType, ctx); + + rawTraits.InitLambda = initLambda; + rawTraits.UpdateLambda = updateLambda; + rawTraits.CalculateLambda = calculateLambda; + rawTraits.DefaultValue = traits->ChildPtr(5); + + if (frameType == EFrameType::LAGGING) { + maxUnboundedPrecedingLag = Max(maxUnboundedPrecedingLag, ui64(abs(*frameLast))); + lagQueueStructItems.push_back(ctx.MakeType<TItemExprType>(name, rawTraits.OutputType)); + } + } else if (traits->IsCallable({"Lead", "Lag"})) { + i64 lead = 1; + if (traits->ChildrenSize() == 3) { + YQL_ENSURE(traits->Child(2)->IsCallable("Int64")); + lead = FromString<i64>(traits->Child(2)->Child(0)->Content()); + } + + if (traits->IsCallable("Lag")) { + lead = -lead; + } + + if (lead < 0) { + maxDataLag = Max(maxDataLag, ui64(abs(lead))); + } else { + maxDataOutpace = Max<ui64>(maxDataOutpace, lead); + } + + auto lambdaInputType = + traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TListExprType>()->GetItemType(); + + rawTraits.CalculateLambda = ReplaceFirstLambdaArgWithCastStruct(*traits->Child(1), *lambdaInputType, ctx); + rawTraits.CalculateLambdaLead = lead; + rawTraits.OutputType = traits->Child(1)->GetTypeAnn(); + YQL_ENSURE(rawTraits.OutputType); + } else { + YQL_ENSURE(traits->IsCallable({"RowNumber", "Rank", "DenseRank"})); + + rawTraits.CalculateLambda = traits; + rawTraits.OutputType = traits->IsCallable("RowNumber") ? + ctx.MakeType<TDataExprType>(EDataSlot::Uint64) : traits->Child(1)->GetTypeAnn(); + } + } + } + + result.LagQueueItemType = ctx.MakeType<TStructExprType>(lagQueueStructItems); + + return result; +} + +TExprNode::TPtr BuildUint64(TPositionHandle pos, ui64 value, TExprContext& ctx) { + return ctx.Builder(pos) + .Callable("Uint64") + .Atom(0, ToString(value)) + .Seal() + .Build(); +} + +TExprNode::TPtr BuildQueuePeek(TPositionHandle pos, const TExprNode::TPtr& queue, ui64 index, const TExprNode::TPtr& dependsOn, + TExprContext& ctx) +{ + return ctx.Builder(pos) + .Callable("QueuePeek") + .Add(0, queue) + .Add(1, BuildUint64(pos, index, ctx)) + .Callable(2, "DependsOn") + .Add(0, dependsOn) + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildQueueRange(TPositionHandle pos, const TExprNode::TPtr& queue, ui64 begin, ui64 end, + const TExprNode::TPtr& dependsOn, TExprContext& ctx) +{ + return ctx.Builder(pos) + .Callable("FlatMap") + .Callable(0, "QueueRange") + .Add(0, queue) + .Add(1, BuildUint64(pos, begin, ctx)) + .Add(2, BuildUint64(pos, end, ctx)) + .Callable(3, "DependsOn") + .Add(0, dependsOn) + .Seal() + .Seal() + .Lambda(1) + .Param("item") + .Arg("item") + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr BuildQueue(TPositionHandle pos, const TTypeAnnotationNode& itemType, ui64 queueSize, ui64 initSize, + const TExprNode::TPtr& dependsOn, TExprContext& ctx) +{ + TExprNode::TPtr size; + if (queueSize == Max<ui64>()) { + size = ctx.NewCallable(pos, "Void", {}); + } else { + size = BuildUint64(pos, queueSize, ctx); + } + + return ctx.Builder(pos) + .Callable("QueueCreate") + .Add(0, ExpandType(pos, itemType, ctx)) + .Add(1, size) + .Add(2, BuildUint64(pos, initSize, ctx)) + .Callable(3, "DependsOn") + .Add(0, dependsOn) + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr CoalesceQueueOutput(TPositionHandle pos, const TExprNode::TPtr& output, bool rawOutputIsOptional, + const TExprNode::TPtr& defaultValue, TExprContext& ctx) +{ + if (defaultValue->IsCallable("Null")) { + if (!rawOutputIsOptional) { + return output; + } + + return ctx.Builder(pos) + .Callable("FlatMap") + .Add(0, output) + .Lambda(1) + .Param("item") + .Arg("item") + .Seal() + .Seal() + .Build(); + } + if (defaultValue->IsCallable("EmptyList")) { return ctx.Builder(pos) .Callable("FlatMap") @@ -369,844 +369,844 @@ TExprNode::TPtr CoalesceQueueOutput(TPositionHandle pos, const TExprNode::TPtr& .Build(); } - return ctx.Builder(pos) - .Callable("Coalesce") - .Add(0, output) - .Add(1, defaultValue) - .Seal() - .Build(); -} - + return ctx.Builder(pos) + .Callable("Coalesce") + .Add(0, output) + .Add(1, defaultValue) + .Seal() + .Build(); +} + TExprNode::TPtr BuildInitLambdaForChain1Map(TPositionHandle pos, const TExprNode::TPtr& initStateLambda, - const TExprNode::TPtr& calculateLambda, TExprContext& ctx) -{ - return ctx.Builder(pos) - .Lambda() - .Param("row") - .List() - .Apply(0, calculateLambda) - .With(0) - .Apply(initStateLambda) - .With(0, "row") - .Seal() - .Done() - .Seal() - .Apply(1, initStateLambda) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Build(); -} - + const TExprNode::TPtr& calculateLambda, TExprContext& ctx) +{ + return ctx.Builder(pos) + .Lambda() + .Param("row") + .List() + .Apply(0, calculateLambda) + .With(0) + .Apply(initStateLambda) + .With(0, "row") + .Seal() + .Done() + .Seal() + .Apply(1, initStateLambda) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Build(); +} + TExprNode::TPtr BuildUpdateLambdaForChain1Map(TPositionHandle pos, const TExprNode::TPtr& updateStateLambda, - const TExprNode::TPtr& calculateLambda, TExprContext& ctx) -{ - return ctx.Builder(pos) - .Lambda() - .Param("row") - .Param("state") - .List() - .Apply(0, calculateLambda) - .With(0) - .Apply(updateStateLambda) - .With(0, "row") - .With(1, "state") - .Seal() - .Done() - .Seal() - .Apply(1, updateStateLambda) - .With(0, "row") - .With(1, "state") - .Seal() - .Seal() - .Seal() - .Build(); -} - - - + const TExprNode::TPtr& calculateLambda, TExprContext& ctx) +{ + return ctx.Builder(pos) + .Lambda() + .Param("row") + .Param("state") + .List() + .Apply(0, calculateLambda) + .With(0) + .Apply(updateStateLambda) + .With(0, "row") + .With(1, "state") + .Seal() + .Done() + .Seal() + .Apply(1, updateStateLambda) + .With(0, "row") + .With(1, "state") + .Seal() + .Seal() + .Seal() + .Build(); +} + + + class TChain1MapTraits : public TThrRefBase, public TNonCopyable { -public: +public: using TPtr = TIntrusivePtr<TChain1MapTraits>; - + TChain1MapTraits(TStringBuf name, TPositionHandle pos) - : Name(name) - , Pos(pos) - { - } - - TStringBuf GetName() const { - return Name; - } - - TPositionHandle GetPos() const { - return Pos; - } - - // Lambda(row) -> AsTuple(output, state) - virtual TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const = 0; - - // Lambda(row, state) -> AsTuple(output, state) - virtual TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const = 0; - - virtual TExprNode::TPtr ExtractLaggingOutput(const TExprNode::TPtr& lagQueue, - const TExprNode::TPtr& dependsOn, TExprContext& ctx) const - { - Y_UNUSED(lagQueue); - Y_UNUSED(dependsOn); - Y_UNUSED(ctx); - return {}; - } - + : Name(name) + , Pos(pos) + { + } + + TStringBuf GetName() const { + return Name; + } + + TPositionHandle GetPos() const { + return Pos; + } + + // Lambda(row) -> AsTuple(output, state) + virtual TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const = 0; + + // Lambda(row, state) -> AsTuple(output, state) + virtual TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const = 0; + + virtual TExprNode::TPtr ExtractLaggingOutput(const TExprNode::TPtr& lagQueue, + const TExprNode::TPtr& dependsOn, TExprContext& ctx) const + { + Y_UNUSED(lagQueue); + Y_UNUSED(dependsOn); + Y_UNUSED(ctx); + return {}; + } + virtual ~TChain1MapTraits() = default; -private: - const TStringBuf Name; - const TPositionHandle Pos; -}; - +private: + const TStringBuf Name; + const TPositionHandle Pos; +}; + class TChain1MapTraitsLagLead : public TChain1MapTraits { -public: +public: TChain1MapTraitsLagLead(TStringBuf name, const TRawTrait& raw, TMaybe<ui64> queueOffset) : TChain1MapTraits(name, raw.Pos) - , QueueOffset(queueOffset) - , LeadLagLambda(raw.CalculateLambda) - { - } - - // Lambda(row) -> AsTuple(output, state) + , QueueOffset(queueOffset) + , LeadLagLambda(raw.CalculateLambda) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .List() - .Apply(0, CalculateOutputLambda(dataQueue, ctx)) - .With(0, "row") - .Seal() - .Callable(1, "Void") - .Seal() - .Seal() - .Seal() - .Build(); - } - - // Lambda(row, state) -> AsTuple(output, state) + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .List() + .Apply(0, CalculateOutputLambda(dataQueue, ctx)) + .With(0, "row") + .Seal() + .Callable(1, "Void") + .Seal() + .Seal() + .Seal() + .Build(); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Param("state") - .List() - .Apply(0, CalculateOutputLambda(dataQueue, ctx)) - .With(0, "row") - .Seal() - .Arg(1, "state") - .Seal() - .Seal() - .Build(); - } - -private: - TExprNode::TPtr CalculateOutputLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const { - if (!QueueOffset.Defined()) { - return AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx); - } - - YQL_ENSURE(dataQueue); - - auto rowArg = ctx.NewArgument(GetPos(), "row"); - - auto body = ctx.Builder(GetPos()) - .Callable("FlatMap") - .Add(0, BuildQueuePeek(GetPos(), dataQueue, *QueueOffset, rowArg, ctx)) - .Add(1, AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx)) - .Seal() - .Build(); - - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(body)); - } - - const TMaybe<ui64> QueueOffset; - const TExprNode::TPtr LeadLagLambda; -}; - - + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Param("state") + .List() + .Apply(0, CalculateOutputLambda(dataQueue, ctx)) + .With(0, "row") + .Seal() + .Arg(1, "state") + .Seal() + .Seal() + .Build(); + } + +private: + TExprNode::TPtr CalculateOutputLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const { + if (!QueueOffset.Defined()) { + return AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx); + } + + YQL_ENSURE(dataQueue); + + auto rowArg = ctx.NewArgument(GetPos(), "row"); + + auto body = ctx.Builder(GetPos()) + .Callable("FlatMap") + .Add(0, BuildQueuePeek(GetPos(), dataQueue, *QueueOffset, rowArg, ctx)) + .Add(1, AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx)) + .Seal() + .Build(); + + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(body)); + } + + const TMaybe<ui64> QueueOffset; + const TExprNode::TPtr LeadLagLambda; +}; + + class TChain1MapTraitsRowNumber : public TChain1MapTraits { -public: +public: TChain1MapTraitsRowNumber(TStringBuf name, const TRawTrait& raw) : TChain1MapTraits(name, raw.Pos) - { - } - - // Lambda(row) -> AsTuple(output, state) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .List() - .Add(0, BuildUint64(GetPos(), 1, ctx)) - .Add(1, BuildUint64(GetPos(), 1, ctx)) - .Seal() - .Seal() - .Build(); - } - - // Lambda(row, state) -> AsTuple(output, state) + Y_UNUSED(dataQueue); + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .List() + .Add(0, BuildUint64(GetPos(), 1, ctx)) + .Add(1, BuildUint64(GetPos(), 1, ctx)) + .Seal() + .Seal() + .Build(); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Param("state") - .List() - .Callable(0, "Inc") - .Arg(0, "state") - .Seal() - .Callable(1, "Inc") - .Arg(0, "state") - .Seal() - .Seal() - .Seal() - .Build(); - } -}; - + Y_UNUSED(dataQueue); + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Param("state") + .List() + .Callable(0, "Inc") + .Arg(0, "state") + .Seal() + .Callable(1, "Inc") + .Arg(0, "state") + .Seal() + .Seal() + .Seal() + .Build(); + } +}; + class TChain1MapTraitsRankBase : public TChain1MapTraits { -public: +public: TChain1MapTraitsRankBase(TStringBuf name, const TRawTrait& raw) : TChain1MapTraits(name, raw.Pos) - , ExtractForCompareLambda(raw.CalculateLambda->ChildPtr(1)) - , Ansi(HasSetting(*raw.CalculateLambda->Child(2), "ansi")) - , KeyType(raw.OutputType) - { - } - - virtual TExprNode::TPtr BuildCalculateLambda(TExprContext& ctx) const { - return ctx.Builder(GetPos()) - .Lambda() - .Param("state") - .Callable("Nth") - .Arg(0, "state") - .Atom(1, "0") - .Seal() - .Seal() - .Build(); - } - - // Lambda(row) -> AsTuple(output, state) + , ExtractForCompareLambda(raw.CalculateLambda->ChildPtr(1)) + , Ansi(HasSetting(*raw.CalculateLambda->Child(2), "ansi")) + , KeyType(raw.OutputType) + { + } + + virtual TExprNode::TPtr BuildCalculateLambda(TExprContext& ctx) const { + return ctx.Builder(GetPos()) + .Lambda() + .Param("state") + .Callable("Nth") + .Arg(0, "state") + .Atom(1, "0") + .Seal() + .Seal() + .Build(); + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const final { - Y_UNUSED(dataQueue); - - - auto initKeyLambda = BuildRawInitLambda(ctx); - if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { - auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); - initKeyLambda = BuildOptKeyInitLambda(initKeyLambda, stateType, ctx); - } - - auto initRowLambda = ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Apply(initKeyLambda) - .With(0) - .Apply(ExtractForCompareLambda) - .With(0, "row") - .Seal() - .Done() - .Seal() - .Seal() - .Build(); - + Y_UNUSED(dataQueue); + + + auto initKeyLambda = BuildRawInitLambda(ctx); + if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { + auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); + initKeyLambda = BuildOptKeyInitLambda(initKeyLambda, stateType, ctx); + } + + auto initRowLambda = ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Apply(initKeyLambda) + .With(0) + .Apply(ExtractForCompareLambda) + .With(0, "row") + .Seal() + .Done() + .Seal() + .Seal() + .Build(); + return BuildInitLambdaForChain1Map(GetPos(), initRowLambda, BuildCalculateLambda(ctx), ctx); - } - - // Lambda(row, state) -> AsTuple(output, state) + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const final { - Y_UNUSED(dataQueue); - - bool useAggrEquals = Ansi; - auto updateKeyLambda = BuildRawUpdateLambda(useAggrEquals, ctx); - - if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { - auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); - updateKeyLambda = ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .Param("state") - .Callable("IfPresent") - .Arg(0, "state") - .Lambda(1) - .Param("unwrappedState") - .Callable("IfPresent") - .Arg(0, "key") - .Lambda(1) - .Param("unwrappedKey") - .Callable("Just") - .Apply(0, updateKeyLambda) - .With(0, "unwrappedKey") - .With(1, "unwrappedState") - .Seal() - .Seal() - .Seal() - .Callable(2, "Just") - .Arg(0, "unwrappedState") - .Seal() - .Seal() - .Seal() - .Apply(2, BuildOptKeyInitLambda(BuildRawInitLambda(ctx), stateType, ctx)) - .With(0, "key") - .Seal() - .Seal() - .Seal() - .Build(); - } - - auto updateRowLambda = ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Param("state") - .Apply(updateKeyLambda) - .With(0) - .Apply(ExtractForCompareLambda) - .With(0, "row") - .Seal() - .Done() - .With(1, "state") - .Seal() - .Seal() - .Build(); - + Y_UNUSED(dataQueue); + + bool useAggrEquals = Ansi; + auto updateKeyLambda = BuildRawUpdateLambda(useAggrEquals, ctx); + + if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { + auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); + updateKeyLambda = ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .Param("state") + .Callable("IfPresent") + .Arg(0, "state") + .Lambda(1) + .Param("unwrappedState") + .Callable("IfPresent") + .Arg(0, "key") + .Lambda(1) + .Param("unwrappedKey") + .Callable("Just") + .Apply(0, updateKeyLambda) + .With(0, "unwrappedKey") + .With(1, "unwrappedState") + .Seal() + .Seal() + .Seal() + .Callable(2, "Just") + .Arg(0, "unwrappedState") + .Seal() + .Seal() + .Seal() + .Apply(2, BuildOptKeyInitLambda(BuildRawInitLambda(ctx), stateType, ctx)) + .With(0, "key") + .Seal() + .Seal() + .Seal() + .Build(); + } + + auto updateRowLambda = ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Param("state") + .Apply(updateKeyLambda) + .With(0) + .Apply(ExtractForCompareLambda) + .With(0, "row") + .Seal() + .Done() + .With(1, "state") + .Seal() + .Seal() + .Build(); + return BuildUpdateLambdaForChain1Map(GetPos(), updateRowLambda, BuildCalculateLambda(ctx), ctx); - } - - virtual TExprNode::TPtr BuildRawInitLambda(TExprContext& ctx) const = 0; - virtual TExprNode::TPtr BuildRawUpdateLambda(bool useAggrEquals, TExprContext& ctx) const = 0; - virtual const TTypeAnnotationNode* GetStateType(const TTypeAnnotationNode* keyType, TExprContext& ctx) const = 0; - -private: - TExprNode::TPtr BuildOptKeyInitLambda(const TExprNode::TPtr& rawInitKeyLambda, - const TTypeAnnotationNode* stateType, TExprContext& ctx) const - { - auto optStateType = ctx.MakeType<TOptionalExprType>(stateType); - return ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .Callable("IfPresent") - .Arg(0, "key") - .Lambda(1) - .Param("unwrapped") - .Callable("Just") - .Apply(0, rawInitKeyLambda) - .With(0, "unwrapped") - .Seal() - .Seal() - .Seal() - .Callable(2, "Nothing") - .Add(0, ExpandType(GetPos(), *optStateType, ctx)) - .Seal() - .Seal() - .Seal() - .Build(); - } - - const TExprNode::TPtr ExtractForCompareLambda; - const bool Ansi; - const TTypeAnnotationNode* const KeyType; -}; - + } + + virtual TExprNode::TPtr BuildRawInitLambda(TExprContext& ctx) const = 0; + virtual TExprNode::TPtr BuildRawUpdateLambda(bool useAggrEquals, TExprContext& ctx) const = 0; + virtual const TTypeAnnotationNode* GetStateType(const TTypeAnnotationNode* keyType, TExprContext& ctx) const = 0; + +private: + TExprNode::TPtr BuildOptKeyInitLambda(const TExprNode::TPtr& rawInitKeyLambda, + const TTypeAnnotationNode* stateType, TExprContext& ctx) const + { + auto optStateType = ctx.MakeType<TOptionalExprType>(stateType); + return ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .Callable("IfPresent") + .Arg(0, "key") + .Lambda(1) + .Param("unwrapped") + .Callable("Just") + .Apply(0, rawInitKeyLambda) + .With(0, "unwrapped") + .Seal() + .Seal() + .Seal() + .Callable(2, "Nothing") + .Add(0, ExpandType(GetPos(), *optStateType, ctx)) + .Seal() + .Seal() + .Seal() + .Build(); + } + + const TExprNode::TPtr ExtractForCompareLambda; + const bool Ansi; + const TTypeAnnotationNode* const KeyType; +}; + class TChain1MapTraitsRank : public TChain1MapTraitsRankBase { -public: +public: TChain1MapTraitsRank(TStringBuf name, const TRawTrait& raw) : TChain1MapTraitsRankBase(name, raw) - { - } - + { + } + TExprNode::TPtr BuildRawInitLambda(TExprContext& ctx) const final { - auto one = BuildUint64(GetPos(), 1, ctx); - return ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .List() - .Add(0, one) - .Add(1, one) - .Arg(2, "key") - .Seal() - .Seal() - .Build(); - } - + auto one = BuildUint64(GetPos(), 1, ctx); + return ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .List() + .Add(0, one) + .Add(1, one) + .Arg(2, "key") + .Seal() + .Seal() + .Build(); + } + TExprNode::TPtr BuildRawUpdateLambda(bool useAggrEquals, TExprContext& ctx) const final { - return ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .Param("state") - .List() - .Callable(0, "If") - .Callable(0, useAggrEquals ? "AggrEquals" : "==") - .Arg(0, "key") - .Callable(1, "Nth") - .Arg(0, "state") - .Atom(1, "2") - .Seal() - .Seal() - .Callable(1, "Nth") - .Arg(0, "state") - .Atom(1, "0") - .Seal() - .Callable(2, "Inc") - .Callable(0, "Nth") - .Arg(0, "state") - .Atom(1, "1") - .Seal() - .Seal() - .Seal() - .Callable(1, "Inc") - .Callable(0, "Nth") - .Arg(0, "state") - .Atom(1, "1") - .Seal() - .Seal() - .Arg(2, "key") - .Seal() - .Seal() - .Build(); - } - + return ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .Param("state") + .List() + .Callable(0, "If") + .Callable(0, useAggrEquals ? "AggrEquals" : "==") + .Arg(0, "key") + .Callable(1, "Nth") + .Arg(0, "state") + .Atom(1, "2") + .Seal() + .Seal() + .Callable(1, "Nth") + .Arg(0, "state") + .Atom(1, "0") + .Seal() + .Callable(2, "Inc") + .Callable(0, "Nth") + .Arg(0, "state") + .Atom(1, "1") + .Seal() + .Seal() + .Seal() + .Callable(1, "Inc") + .Callable(0, "Nth") + .Arg(0, "state") + .Atom(1, "1") + .Seal() + .Seal() + .Arg(2, "key") + .Seal() + .Seal() + .Build(); + } + const TTypeAnnotationNode* GetStateType(const TTypeAnnotationNode* keyType, TExprContext& ctx) const final { - return ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{ - ctx.MakeType<TDataExprType>(EDataSlot::Uint64), - ctx.MakeType<TDataExprType>(EDataSlot::Uint64), - keyType - }); - } -}; - + return ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{ + ctx.MakeType<TDataExprType>(EDataSlot::Uint64), + ctx.MakeType<TDataExprType>(EDataSlot::Uint64), + keyType + }); + } +}; + class TChain1MapTraitsDenseRank : public TChain1MapTraitsRankBase { -public: +public: TChain1MapTraitsDenseRank(TStringBuf name, const TRawTrait& raw) : TChain1MapTraitsRankBase(name, raw) - { - } - + { + } + TExprNode::TPtr BuildRawInitLambda(TExprContext& ctx) const final { - return ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .List() - .Add(0, BuildUint64(GetPos(), 1, ctx)) - .Arg(1, "key") - .Seal() - .Seal() - .Build(); - } - + return ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .List() + .Add(0, BuildUint64(GetPos(), 1, ctx)) + .Arg(1, "key") + .Seal() + .Seal() + .Build(); + } + TExprNode::TPtr BuildRawUpdateLambda(bool useAggrEquals, TExprContext& ctx) const final { - return ctx.Builder(GetPos()) - .Lambda() - .Param("key") - .Param("state") - .List() - .Callable(0, "If") - .Callable(0, useAggrEquals ? "AggrEquals" : "==") - .Arg(0, "key") - .Callable(1, "Nth") - .Arg(0, "state") - .Atom(1, "1") - .Seal() - .Seal() - .Callable(1, "Nth") - .Arg(0, "state") - .Atom(1, "0") - .Seal() - .Callable(2, "Inc") - .Callable(0, "Nth") - .Arg(0, "state") - .Atom(1, "0") - .Seal() - .Seal() - .Seal() - .Arg(1, "key") - .Seal() - .Seal() - .Build(); - } - + return ctx.Builder(GetPos()) + .Lambda() + .Param("key") + .Param("state") + .List() + .Callable(0, "If") + .Callable(0, useAggrEquals ? "AggrEquals" : "==") + .Arg(0, "key") + .Callable(1, "Nth") + .Arg(0, "state") + .Atom(1, "1") + .Seal() + .Seal() + .Callable(1, "Nth") + .Arg(0, "state") + .Atom(1, "0") + .Seal() + .Callable(2, "Inc") + .Callable(0, "Nth") + .Arg(0, "state") + .Atom(1, "0") + .Seal() + .Seal() + .Seal() + .Arg(1, "key") + .Seal() + .Seal() + .Build(); + } + const TTypeAnnotationNode* GetStateType(const TTypeAnnotationNode* keyType, TExprContext& ctx) const final { - return ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{ - ctx.MakeType<TDataExprType>(EDataSlot::Uint64), - keyType - }); - } -}; - + return ctx.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{ + ctx.MakeType<TDataExprType>(EDataSlot::Uint64), + keyType + }); + } +}; + class TChain1MapTraitsStateBase : public TChain1MapTraits { -public: +public: TChain1MapTraitsStateBase(TStringBuf name, const TRawTrait& raw) : TChain1MapTraits(name, raw.Pos) - , InitLambda(raw.InitLambda) - , UpdateLambda(raw.UpdateLambda) - , CalculateLambda(raw.CalculateLambda) - , DefaultValue(raw.DefaultValue) - { - } - -protected: - TExprNode::TPtr GetInitLambda() const { - return InitLambda; - } - - TExprNode::TPtr GetUpdateLambda() const { - return UpdateLambda; - } - - TExprNode::TPtr GetCalculateLambda() const { - return CalculateLambda; - } - - TExprNode::TPtr GetDefaultValue() const { - return DefaultValue; - } - -private: - const TExprNode::TPtr InitLambda; - const TExprNode::TPtr UpdateLambda; - const TExprNode::TPtr CalculateLambda; - const TExprNode::TPtr DefaultValue; -}; - - + , InitLambda(raw.InitLambda) + , UpdateLambda(raw.UpdateLambda) + , CalculateLambda(raw.CalculateLambda) + , DefaultValue(raw.DefaultValue) + { + } + +protected: + TExprNode::TPtr GetInitLambda() const { + return InitLambda; + } + + TExprNode::TPtr GetUpdateLambda() const { + return UpdateLambda; + } + + TExprNode::TPtr GetCalculateLambda() const { + return CalculateLambda; + } + + TExprNode::TPtr GetDefaultValue() const { + return DefaultValue; + } + +private: + const TExprNode::TPtr InitLambda; + const TExprNode::TPtr UpdateLambda; + const TExprNode::TPtr CalculateLambda; + const TExprNode::TPtr DefaultValue; +}; + + class TChain1MapTraitsCurrentOrLagging : public TChain1MapTraitsStateBase { -public: +public: TChain1MapTraitsCurrentOrLagging(TStringBuf name, const TRawTrait& raw, TMaybe<ui64> lagQueueIndex) : TChain1MapTraitsStateBase(name, raw) - , LaggingQueueIndex(lagQueueIndex) - , OutputIsOptional(raw.OutputType->GetKind() == ETypeAnnotationKind::Optional) - { - } - - // Lambda(row) -> AsTuple(output, state) + , LaggingQueueIndex(lagQueueIndex) + , OutputIsOptional(raw.OutputType->GetKind() == ETypeAnnotationKind::Optional) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); + Y_UNUSED(dataQueue); return BuildInitLambdaForChain1Map(GetPos(), GetInitLambda(), GetCalculateLambda(), ctx); - } - - // Lambda(row, state) -> AsTuple(output, state) + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); + Y_UNUSED(dataQueue); return BuildUpdateLambdaForChain1Map(GetPos(), GetUpdateLambda(), GetCalculateLambda(), ctx); - } - + } + TExprNode::TPtr ExtractLaggingOutput(const TExprNode::TPtr& lagQueue, const TExprNode::TPtr& dependsOn, TExprContext& ctx) const override - { - if (!LaggingQueueIndex.Defined()) { - return {}; - } - - auto output = ctx.Builder(GetPos()) - .Callable("Map") - .Add(0, BuildQueuePeek(GetPos(), lagQueue, *LaggingQueueIndex, dependsOn, ctx)) - .Lambda(1) - .Param("struct") - .Callable("Member") - .Arg(0, "struct") - .Atom(1, GetName()) - .Seal() - .Seal() - .Seal() - .Build(); - return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); - } - -private: - const TMaybe<ui64> LaggingQueueIndex; - const bool OutputIsOptional; -}; - + { + if (!LaggingQueueIndex.Defined()) { + return {}; + } + + auto output = ctx.Builder(GetPos()) + .Callable("Map") + .Add(0, BuildQueuePeek(GetPos(), lagQueue, *LaggingQueueIndex, dependsOn, ctx)) + .Lambda(1) + .Param("struct") + .Callable("Member") + .Arg(0, "struct") + .Atom(1, GetName()) + .Seal() + .Seal() + .Seal() + .Build(); + return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); + } + +private: + const TMaybe<ui64> LaggingQueueIndex; + const bool OutputIsOptional; +}; + class TChain1MapTraitsLeading : public TChain1MapTraitsStateBase { -public: +public: TChain1MapTraitsLeading(TStringBuf name, const TRawTrait& raw, ui64 currentRowIndex, ui64 lastRowIndex) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(currentRowIndex + 1) - , QueueEnd(lastRowIndex + 1) - { - } - - // Lambda(row) -> AsTuple(output, state) + , QueueBegin(currentRowIndex + 1) + , QueueEnd(lastRowIndex + 1) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - YQL_ENSURE(dataQueue); - auto originalInit = GetInitLambda(); - auto originalUpdate = GetUpdateLambda(); - auto calculate = GetCalculateLambda(); - - auto rowArg = ctx.NewArgument(GetPos(), "row"); - auto state = ctx.Builder(GetPos()) - .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) - .Apply(1, originalInit) - .With(0, rowArg) - .Seal() - .Add(2, ctx.DeepCopyLambda(*originalUpdate)) - .Seal() - .Build(); - - auto initBody = ctx.Builder(GetPos()) - .List() - .Apply(0, calculate) - .With(0, state) - .Seal() - .Apply(1, originalInit) - .With(0, rowArg) - .Seal() - .Seal() - .Build(); - - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(initBody)); - } - - // Lambda(row, state) -> AsTuple(output, state) + YQL_ENSURE(dataQueue); + auto originalInit = GetInitLambda(); + auto originalUpdate = GetUpdateLambda(); + auto calculate = GetCalculateLambda(); + + auto rowArg = ctx.NewArgument(GetPos(), "row"); + auto state = ctx.Builder(GetPos()) + .Callable("Fold") + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Apply(1, originalInit) + .With(0, rowArg) + .Seal() + .Add(2, ctx.DeepCopyLambda(*originalUpdate)) + .Seal() + .Build(); + + auto initBody = ctx.Builder(GetPos()) + .List() + .Apply(0, calculate) + .With(0, state) + .Seal() + .Apply(1, originalInit) + .With(0, rowArg) + .Seal() + .Seal() + .Build(); + + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(initBody)); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - YQL_ENSURE(dataQueue); - auto originalInit = GetInitLambda(); - auto originalUpdate = GetUpdateLambda(); - auto calculate = GetCalculateLambda(); - - auto rowArg = ctx.NewArgument(GetPos(), "row"); - auto stateArg = ctx.NewArgument(GetPos(), "state"); - - auto state = ctx.Builder(GetPos()) - .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) - .Apply(1, originalUpdate) - .With(0, rowArg) - .With(1, stateArg) - .Seal() - .Add(2, ctx.DeepCopyLambda(*originalUpdate)) - .Seal() - .Build(); - - auto updateBody = ctx.Builder(GetPos()) - .List() - .Apply(0, calculate) - .With(0, state) - .Seal() - .Apply(1, originalUpdate) - .With(0, rowArg) - .With(1, stateArg) - .Seal() - .Seal() - .Build(); - - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg, stateArg}), std::move(updateBody)); - } - -private: - const ui64 QueueBegin; - const ui64 QueueEnd; -}; - - + YQL_ENSURE(dataQueue); + auto originalInit = GetInitLambda(); + auto originalUpdate = GetUpdateLambda(); + auto calculate = GetCalculateLambda(); + + auto rowArg = ctx.NewArgument(GetPos(), "row"); + auto stateArg = ctx.NewArgument(GetPos(), "state"); + + auto state = ctx.Builder(GetPos()) + .Callable("Fold") + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Apply(1, originalUpdate) + .With(0, rowArg) + .With(1, stateArg) + .Seal() + .Add(2, ctx.DeepCopyLambda(*originalUpdate)) + .Seal() + .Build(); + + auto updateBody = ctx.Builder(GetPos()) + .List() + .Apply(0, calculate) + .With(0, state) + .Seal() + .Apply(1, originalUpdate) + .With(0, rowArg) + .With(1, stateArg) + .Seal() + .Seal() + .Build(); + + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg, stateArg}), std::move(updateBody)); + } + +private: + const ui64 QueueBegin; + const ui64 QueueEnd; +}; + + class TChain1MapTraitsFull : public TChain1MapTraitsStateBase { -public: +public: TChain1MapTraitsFull(TStringBuf name, const TRawTrait& raw, ui64 currentRowIndex) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(currentRowIndex + 1) - { - } - - // Lambda(row) -> AsTuple(output, state) - // state == output + , QueueBegin(currentRowIndex + 1) + { + } + + // Lambda(row) -> AsTuple(output, state) + // state == output TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - auto originalInit = GetInitLambda(); - auto originalUpdate = GetUpdateLambda(); - auto calculate = GetCalculateLambda(); - - auto rowArg = ctx.NewArgument(GetPos(), "row"); - auto state = ctx.Builder(GetPos()) - .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, Max<ui64>(), rowArg, ctx)) - .Apply(1, originalInit) - .With(0, rowArg) - .Seal() - .Add(2, ctx.DeepCopyLambda(*originalUpdate)) - .Seal() - .Build(); - - auto initBody = ctx.Builder(GetPos()) - .List() - .Apply(0, calculate) - .With(0, state) - .Seal() - .Apply(1, calculate) - .With(0, state) - .Seal() - .Seal() - .Build(); - - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(initBody)); - } - - // Lambda(row, state) -> AsTuple(output, state) + auto originalInit = GetInitLambda(); + auto originalUpdate = GetUpdateLambda(); + auto calculate = GetCalculateLambda(); + + auto rowArg = ctx.NewArgument(GetPos(), "row"); + auto state = ctx.Builder(GetPos()) + .Callable("Fold") + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, Max<ui64>(), rowArg, ctx)) + .Apply(1, originalInit) + .With(0, rowArg) + .Seal() + .Add(2, ctx.DeepCopyLambda(*originalUpdate)) + .Seal() + .Build(); + + auto initBody = ctx.Builder(GetPos()) + .List() + .Apply(0, calculate) + .With(0, state) + .Seal() + .Apply(1, calculate) + .With(0, state) + .Seal() + .Seal() + .Build(); + + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(initBody)); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Param("state") - .List() - .Arg(0, "state") - .Arg(1, "state") - .Seal() - .Seal() - .Build(); - } - -private: - const ui64 QueueBegin; -}; - - + Y_UNUSED(dataQueue); + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Param("state") + .List() + .Arg(0, "state") + .Arg(1, "state") + .Seal() + .Seal() + .Build(); + } + +private: + const ui64 QueueBegin; +}; + + class TChain1MapTraitsGeneric : public TChain1MapTraitsStateBase { -public: +public: TChain1MapTraitsGeneric(TStringBuf name, const TRawTrait& raw, ui64 queueBegin, ui64 queueEnd) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(queueBegin) - , QueueEnd(queueEnd) - , FrameNeverEmpty(raw.FrameSettings.NeverEmpty) - , OutputIsOptional(raw.OutputType->GetKind() == ETypeAnnotationKind::Optional) - { - } - - // Lambda(row) -> AsTuple(output, state) + , QueueBegin(queueBegin) + , QueueEnd(queueEnd) + , FrameNeverEmpty(raw.FrameSettings.NeverEmpty) + , OutputIsOptional(raw.OutputType->GetKind() == ETypeAnnotationKind::Optional) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - auto rowArg = ctx.NewArgument(GetPos(), "row"); - auto body = ctx.Builder(GetPos()) - .List() - .Add(0, BuildFinalOutput(rowArg, dataQueue, ctx)) - .Callable(1, "Void") - .Seal() - .Seal() - .Build(); - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(body)); - } - - // Lambda(row, state) -> AsTuple(output, state) + auto rowArg = ctx.NewArgument(GetPos(), "row"); + auto body = ctx.Builder(GetPos()) + .List() + .Add(0, BuildFinalOutput(rowArg, dataQueue, ctx)) + .Callable(1, "Void") + .Seal() + .Seal() + .Build(); + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(body)); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - auto rowArg = ctx.NewArgument(GetPos(), "row"); - auto stateArg = ctx.NewArgument(GetPos(), "state"); - auto body = ctx.Builder(GetPos()) - .List() - .Add(0, BuildFinalOutput(rowArg, dataQueue, ctx)) - .Add(1, stateArg) - .Seal() - .Build(); - return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg, stateArg}), std::move(body)); - } - -private: - TExprNode::TPtr BuildFinalOutput(const TExprNode::TPtr& rowArg, const TExprNode::TPtr& dataQueue, TExprContext& ctx) const { - YQL_ENSURE(dataQueue); - auto originalInit = GetInitLambda(); - auto originalUpdate = GetUpdateLambda(); - auto calculate = GetCalculateLambda(); - - auto output = ctx.Builder(GetPos()) - .Callable("Map") - .Callable(0, "Fold1") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) - .Add(1, ctx.DeepCopyLambda(*originalInit)) - .Add(2, ctx.DeepCopyLambda(*originalUpdate)) - .Seal() - .Add(1, ctx.DeepCopyLambda(*calculate)) - .Seal() - .Build(); - - if (FrameNeverEmpty) { - // output is always non-empty optional in this case - // we do IfPresent with some fake output value to remove optional - // this will have exactly the same result as Unwrap(output) - return ctx.Builder(GetPos()) - .Callable("IfPresent") - .Add(0, output) - .Lambda(1) - .Param("unwrapped") - .Arg("unwrapped") - .Seal() - .Apply(2, calculate) - .With(0) - .Apply(originalInit) - .With(0, rowArg) - .Seal() - .Done() - .Seal() - .Seal() - .Build(); - } - - return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); - } - - const ui64 QueueBegin; - const ui64 QueueEnd; - const bool FrameNeverEmpty; - const bool OutputIsOptional; -}; - + auto rowArg = ctx.NewArgument(GetPos(), "row"); + auto stateArg = ctx.NewArgument(GetPos(), "state"); + auto body = ctx.Builder(GetPos()) + .List() + .Add(0, BuildFinalOutput(rowArg, dataQueue, ctx)) + .Add(1, stateArg) + .Seal() + .Build(); + return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg, stateArg}), std::move(body)); + } + +private: + TExprNode::TPtr BuildFinalOutput(const TExprNode::TPtr& rowArg, const TExprNode::TPtr& dataQueue, TExprContext& ctx) const { + YQL_ENSURE(dataQueue); + auto originalInit = GetInitLambda(); + auto originalUpdate = GetUpdateLambda(); + auto calculate = GetCalculateLambda(); + + auto output = ctx.Builder(GetPos()) + .Callable("Map") + .Callable(0, "Fold1") + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Add(1, ctx.DeepCopyLambda(*originalInit)) + .Add(2, ctx.DeepCopyLambda(*originalUpdate)) + .Seal() + .Add(1, ctx.DeepCopyLambda(*calculate)) + .Seal() + .Build(); + + if (FrameNeverEmpty) { + // output is always non-empty optional in this case + // we do IfPresent with some fake output value to remove optional + // this will have exactly the same result as Unwrap(output) + return ctx.Builder(GetPos()) + .Callable("IfPresent") + .Add(0, output) + .Lambda(1) + .Param("unwrapped") + .Arg("unwrapped") + .Seal() + .Apply(2, calculate) + .With(0) + .Apply(originalInit) + .With(0, rowArg) + .Seal() + .Done() + .Seal() + .Seal() + .Build(); + } + + return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); + } + + const ui64 QueueBegin; + const ui64 QueueEnd; + const bool FrameNeverEmpty; + const bool OutputIsOptional; +}; + class TChain1MapTraitsEmpty : public TChain1MapTraitsStateBase { -public: +public: TChain1MapTraitsEmpty(TStringBuf name, const TRawTrait& raw) : TChain1MapTraitsStateBase(name, raw) - , RawOutputType(raw.OutputType) - { - } - - // Lambda(row) -> AsTuple(output, state) + , RawOutputType(raw.OutputType) + { + } + + // Lambda(row) -> AsTuple(output, state) TExprNode::TPtr BuildInitLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .List() - .Add(0, BuildFinalOutput(ctx)) - .Callable(1, "Void") - .Seal() - .Seal() - .Seal() - .Build(); - } - - // Lambda(row, state) -> AsTuple(output, state) + Y_UNUSED(dataQueue); + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .List() + .Add(0, BuildFinalOutput(ctx)) + .Callable(1, "Void") + .Seal() + .Seal() + .Seal() + .Build(); + } + + // Lambda(row, state) -> AsTuple(output, state) TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const override { - Y_UNUSED(dataQueue); - return ctx.Builder(GetPos()) - .Lambda() - .Param("row") - .Param("state") - .List() - .Add(0, BuildFinalOutput(ctx)) - .Arg(1, "state") - .Seal() - .Seal() - .Build(); - } - -private: - TExprNode::TPtr BuildFinalOutput(TExprContext& ctx) const { + Y_UNUSED(dataQueue); + return ctx.Builder(GetPos()) + .Lambda() + .Param("row") + .Param("state") + .List() + .Add(0, BuildFinalOutput(ctx)) + .Arg(1, "state") + .Seal() + .Seal() + .Build(); + } + +private: + TExprNode::TPtr BuildFinalOutput(TExprContext& ctx) const { const auto defaultValue = GetDefaultValue(); - - if (defaultValue->IsCallable("Null")) { - auto resultingType = RawOutputType; - if (resultingType->GetKind() != ETypeAnnotationKind::Optional) { - resultingType = ctx.MakeType<TOptionalExprType>(resultingType); - } - - return ctx.Builder(GetPos()) - .Callable("Nothing") - .Add(0, ExpandType(GetPos(), *resultingType, ctx)) - .Seal() - .Build(); - } + + if (defaultValue->IsCallable("Null")) { + auto resultingType = RawOutputType; + if (resultingType->GetKind() != ETypeAnnotationKind::Optional) { + resultingType = ctx.MakeType<TOptionalExprType>(resultingType); + } + + return ctx.Builder(GetPos()) + .Callable("Nothing") + .Add(0, ExpandType(GetPos(), *resultingType, ctx)) + .Seal() + .Build(); + } if (defaultValue->IsCallable("EmptyList")) { auto resultingType = RawOutputType; if (resultingType->GetKind() != ETypeAnnotationKind::List) { @@ -1219,290 +1219,290 @@ private: .Seal() .Build(); } - return defaultValue; - } - - const TTypeAnnotationNode* const RawOutputType; -}; - + return defaultValue; + } + + const TTypeAnnotationNode* const RawOutputType; +}; + TVector<TChain1MapTraits::TPtr> BuildFoldMapTraits(ui64& dataOutpace, ui64& dataLag, ui64& lagQueueSize, - const TTypeAnnotationNode*& lagQueueItemType, const TExprNode::TPtr& frames, TExprContext& ctx) -{ - dataOutpace = dataLag = lagQueueSize = 0; - lagQueueItemType = nullptr; - + const TTypeAnnotationNode*& lagQueueItemType, const TExprNode::TPtr& frames, TExprContext& ctx) +{ + dataOutpace = dataLag = lagQueueSize = 0; + lagQueueItemType = nullptr; + TVector<TChain1MapTraits::TPtr> result; - - TCalcOverWindowTraits traits = ExtractCalcOverWindowTraits(frames, ctx); - - if (traits.LagQueueItemType->Cast<TStructExprType>()->GetSize()) { - YQL_ENSURE(traits.MaxUnboundedPrecedingLag > 0); - lagQueueSize = traits.MaxUnboundedPrecedingLag; - lagQueueItemType = traits.LagQueueItemType; - } - - ui64 currentRowIndex = 0; - if (traits.MaxDataOutpace || traits.MaxDataLag) { - dataOutpace = traits.MaxDataOutpace; - dataLag = traits.MaxDataLag; - currentRowIndex = dataLag; - } - - for (const auto& item : traits.RawTraits) { - TStringBuf name = item.first; - const TRawTrait& trait = item.second; - - if (!trait.InitLambda) { - YQL_ENSURE(!trait.UpdateLambda); - YQL_ENSURE(!trait.DefaultValue); - - if (trait.CalculateLambdaLead.Defined()) { - TMaybe<ui64> queueOffset; - if (*trait.CalculateLambdaLead) { - queueOffset = currentRowIndex + *trait.CalculateLambdaLead; - } - + + TCalcOverWindowTraits traits = ExtractCalcOverWindowTraits(frames, ctx); + + if (traits.LagQueueItemType->Cast<TStructExprType>()->GetSize()) { + YQL_ENSURE(traits.MaxUnboundedPrecedingLag > 0); + lagQueueSize = traits.MaxUnboundedPrecedingLag; + lagQueueItemType = traits.LagQueueItemType; + } + + ui64 currentRowIndex = 0; + if (traits.MaxDataOutpace || traits.MaxDataLag) { + dataOutpace = traits.MaxDataOutpace; + dataLag = traits.MaxDataLag; + currentRowIndex = dataLag; + } + + for (const auto& item : traits.RawTraits) { + TStringBuf name = item.first; + const TRawTrait& trait = item.second; + + if (!trait.InitLambda) { + YQL_ENSURE(!trait.UpdateLambda); + YQL_ENSURE(!trait.DefaultValue); + + if (trait.CalculateLambdaLead.Defined()) { + TMaybe<ui64> queueOffset; + if (*trait.CalculateLambdaLead) { + queueOffset = currentRowIndex + *trait.CalculateLambdaLead; + } + result.push_back(new TChain1MapTraitsLagLead(name, trait, queueOffset)); - } else if (trait.CalculateLambda->IsCallable("RowNumber")) { + } else if (trait.CalculateLambda->IsCallable("RowNumber")) { result.push_back(new TChain1MapTraitsRowNumber(name, trait)); - } else if (trait.CalculateLambda->IsCallable("Rank")) { + } else if (trait.CalculateLambda->IsCallable("Rank")) { result.push_back(new TChain1MapTraitsRank(name, trait)); - } else { - YQL_ENSURE(trait.CalculateLambda->IsCallable("DenseRank")); + } else { + YQL_ENSURE(trait.CalculateLambda->IsCallable("DenseRank")); result.push_back(new TChain1MapTraitsDenseRank(name, trait)); - } - - continue; - } - - switch(FrameType(trait.FrameSettings)) { - case EFrameType::CURRENT: - case EFrameType::LAGGING: { - TMaybe<ui64> lagQueueIndex; - auto end = *trait.FrameSettings.Last; - YQL_ENSURE(end <= 0); - if (end < 0) { - YQL_ENSURE(lagQueueSize >= ui64(0 - end)); - lagQueueIndex = lagQueueSize + end; - } - + } + + continue; + } + + switch(FrameType(trait.FrameSettings)) { + case EFrameType::CURRENT: + case EFrameType::LAGGING: { + TMaybe<ui64> lagQueueIndex; + auto end = *trait.FrameSettings.Last; + YQL_ENSURE(end <= 0); + if (end < 0) { + YQL_ENSURE(lagQueueSize >= ui64(0 - end)); + lagQueueIndex = lagQueueSize + end; + } + result.push_back(new TChain1MapTraitsCurrentOrLagging(name, trait, lagQueueIndex)); - break; - } - case EFrameType::LEADING: { - auto end = *trait.FrameSettings.Last; - YQL_ENSURE(end > 0); - ui64 lastRowIndex = currentRowIndex + ui64(end); + break; + } + case EFrameType::LEADING: { + auto end = *trait.FrameSettings.Last; + YQL_ENSURE(end > 0); + ui64 lastRowIndex = currentRowIndex + ui64(end); result.push_back(new TChain1MapTraitsLeading(name, trait, currentRowIndex, lastRowIndex)); - break; - } - case EFrameType::FULL: { + break; + } + case EFrameType::FULL: { result.push_back(new TChain1MapTraitsFull(name, trait, currentRowIndex)); - break; - } - case EFrameType::GENERIC: { - auto first = trait.FrameSettings.First; - auto last = trait.FrameSettings.Last; - YQL_ENSURE(first.Defined()); - ui64 beginIndex = currentRowIndex + *first; - ui64 endIndex = last.Defined() ? (currentRowIndex + *last + 1) : Max<ui64>(); + break; + } + case EFrameType::GENERIC: { + auto first = trait.FrameSettings.First; + auto last = trait.FrameSettings.Last; + YQL_ENSURE(first.Defined()); + ui64 beginIndex = currentRowIndex + *first; + ui64 endIndex = last.Defined() ? (currentRowIndex + *last + 1) : Max<ui64>(); result.push_back(new TChain1MapTraitsGeneric(name, trait, beginIndex, endIndex)); - break; - } - case EFrameType::EMPTY: { + break; + } + case EFrameType::EMPTY: { result.push_back(new TChain1MapTraitsEmpty(name, trait)); - break; - } - } - } - - return result; -} - -TExprNode::TPtr ConvertStructOfTuplesToTupleOfStructs(TPositionHandle pos, const TExprNode::TPtr& input, TExprContext& ctx) { - return ctx.Builder(pos) - .List() - .Callable(0, "StaticMap") - .Add(0, input) - .Lambda(1) - .Param("tuple") - .Callable("Nth") - .Arg(0, "tuple") - .Atom(1, "0") - .Seal() - .Seal() - .Seal() - .Callable(1, "StaticMap") - .Add(0, input) - .Lambda(1) - .Param("tuple") - .Callable("Nth") - .Arg(0, "tuple") - .Atom(1, "1") - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr AddInputMembersToOutput(TPositionHandle pos, const TExprNode::TPtr& tupleOfOutputStructAndStateStruct, - const TExprNode::TPtr& rowArg, TExprContext& ctx) -{ - return ctx.Builder(pos) - .List() - .Callable(0, "FlattenMembers") - .List(0) - .Atom(0, "") - .Callable(1, "Nth") - .Add(0, tupleOfOutputStructAndStateStruct) - .Atom(1, "0") - .Seal() - .Seal() - .List(1) - .Atom(0, "") - .Add(1, rowArg) - .Seal() - .Seal() - .Callable(1, "Nth") - .Add(0, tupleOfOutputStructAndStateStruct) - .Atom(1, "1") - .Seal() - .Seal() - .Build(); -} - -template<typename T> -TExprNode::TPtr SelectMembers(TPositionHandle pos, const T& members, const TExprNode::TPtr& structNode, TExprContext& ctx) { - TExprNodeList structItems; - for (auto& name : members) { - structItems.push_back( - ctx.Builder(pos) - .List() - .Atom(0, name) - .Callable(1, "Member") - .Add(0, structNode) - .Atom(1, name) - .Seal() - .Seal() - .Build() - ); - } - return ctx.NewCallable(pos, "AsStruct", std::move(structItems)); -} - -TExprNode::TPtr HandleLaggingItems(TPositionHandle pos, const TExprNode::TPtr& rowArg, + break; + } + } + } + + return result; +} + +TExprNode::TPtr ConvertStructOfTuplesToTupleOfStructs(TPositionHandle pos, const TExprNode::TPtr& input, TExprContext& ctx) { + return ctx.Builder(pos) + .List() + .Callable(0, "StaticMap") + .Add(0, input) + .Lambda(1) + .Param("tuple") + .Callable("Nth") + .Arg(0, "tuple") + .Atom(1, "0") + .Seal() + .Seal() + .Seal() + .Callable(1, "StaticMap") + .Add(0, input) + .Lambda(1) + .Param("tuple") + .Callable("Nth") + .Arg(0, "tuple") + .Atom(1, "1") + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr AddInputMembersToOutput(TPositionHandle pos, const TExprNode::TPtr& tupleOfOutputStructAndStateStruct, + const TExprNode::TPtr& rowArg, TExprContext& ctx) +{ + return ctx.Builder(pos) + .List() + .Callable(0, "FlattenMembers") + .List(0) + .Atom(0, "") + .Callable(1, "Nth") + .Add(0, tupleOfOutputStructAndStateStruct) + .Atom(1, "0") + .Seal() + .Seal() + .List(1) + .Atom(0, "") + .Add(1, rowArg) + .Seal() + .Seal() + .Callable(1, "Nth") + .Add(0, tupleOfOutputStructAndStateStruct) + .Atom(1, "1") + .Seal() + .Seal() + .Build(); +} + +template<typename T> +TExprNode::TPtr SelectMembers(TPositionHandle pos, const T& members, const TExprNode::TPtr& structNode, TExprContext& ctx) { + TExprNodeList structItems; + for (auto& name : members) { + structItems.push_back( + ctx.Builder(pos) + .List() + .Atom(0, name) + .Callable(1, "Member") + .Add(0, structNode) + .Atom(1, name) + .Seal() + .Seal() + .Build() + ); + } + return ctx.NewCallable(pos, "AsStruct", std::move(structItems)); +} + +TExprNode::TPtr HandleLaggingItems(TPositionHandle pos, const TExprNode::TPtr& rowArg, const TExprNode::TPtr& tupleOfOutputAndState, const TVector<TChain1MapTraits::TPtr>& traits, - const TExprNode::TPtr& lagQueue, TExprContext& ctx) -{ - - TExprNodeList laggingStructItems; - TSet<TStringBuf> laggingNames; - TSet<TStringBuf> otherNames; - for (auto& trait : traits) { - auto name = trait->GetName(); - auto laggingOutput = trait->ExtractLaggingOutput(lagQueue, rowArg, ctx); - if (laggingOutput) { - laggingNames.insert(name); - laggingStructItems.push_back( - ctx.Builder(pos) - .List() - .Atom(0, name) - .Add(1, laggingOutput) - .Seal() - .Build() - ); - } else { - otherNames.insert(trait->GetName()); - } - } - - if (laggingStructItems.empty()) { - return tupleOfOutputAndState; - } - - YQL_ENSURE(lagQueue); - - auto output = ctx.NewCallable(pos, "Nth", { tupleOfOutputAndState, ctx.NewAtom(pos, "0")}); - auto state = ctx.NewCallable(pos, "Nth", { tupleOfOutputAndState, ctx.NewAtom(pos, "1")}); - - auto leadingOutput = SelectMembers(pos, laggingNames, output, ctx); - auto otherOutput = SelectMembers(pos, otherNames, output, ctx); - auto laggingOutput = ctx.NewCallable(pos, "AsStruct", std::move(laggingStructItems)); - - output = ctx.Builder(pos) - .Callable("FlattenMembers") - .List(0) - .Atom(0, "") - .Add(1, laggingOutput) - .Seal() - .List(1) - .Atom(0, "") - .Add(1, otherOutput) - .Seal() - .Seal() - .Build(); - - - return ctx.Builder(pos) - .List() - .Add(0, output) - .Callable(1, "Seq") - .Add(0, output) - .Add(1, state) - .Add(2, leadingOutput) - .List(3) - .Add(0, state) - .Callable(1, "QueuePush") - .Callable(0, "QueuePop") - .Add(0, lagQueue) - .Seal() - .Add(1, leadingOutput) - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - - + const TExprNode::TPtr& lagQueue, TExprContext& ctx) +{ + + TExprNodeList laggingStructItems; + TSet<TStringBuf> laggingNames; + TSet<TStringBuf> otherNames; + for (auto& trait : traits) { + auto name = trait->GetName(); + auto laggingOutput = trait->ExtractLaggingOutput(lagQueue, rowArg, ctx); + if (laggingOutput) { + laggingNames.insert(name); + laggingStructItems.push_back( + ctx.Builder(pos) + .List() + .Atom(0, name) + .Add(1, laggingOutput) + .Seal() + .Build() + ); + } else { + otherNames.insert(trait->GetName()); + } + } + + if (laggingStructItems.empty()) { + return tupleOfOutputAndState; + } + + YQL_ENSURE(lagQueue); + + auto output = ctx.NewCallable(pos, "Nth", { tupleOfOutputAndState, ctx.NewAtom(pos, "0")}); + auto state = ctx.NewCallable(pos, "Nth", { tupleOfOutputAndState, ctx.NewAtom(pos, "1")}); + + auto leadingOutput = SelectMembers(pos, laggingNames, output, ctx); + auto otherOutput = SelectMembers(pos, otherNames, output, ctx); + auto laggingOutput = ctx.NewCallable(pos, "AsStruct", std::move(laggingStructItems)); + + output = ctx.Builder(pos) + .Callable("FlattenMembers") + .List(0) + .Atom(0, "") + .Add(1, laggingOutput) + .Seal() + .List(1) + .Atom(0, "") + .Add(1, otherOutput) + .Seal() + .Seal() + .Build(); + + + return ctx.Builder(pos) + .List() + .Add(0, output) + .Callable(1, "Seq") + .Add(0, output) + .Add(1, state) + .Add(2, leadingOutput) + .List(3) + .Add(0, state) + .Callable(1, "QueuePush") + .Callable(0, "QueuePop") + .Add(0, lagQueue) + .Seal() + .Add(1, leadingOutput) + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + + TExprNode::TPtr BuildChain1MapInitLambda(TPositionHandle pos, const TVector<TChain1MapTraits::TPtr>& traits, - const TExprNode::TPtr& dataQueue, ui64 lagQueueSize, const TTypeAnnotationNode* lagQueueItemType, TExprContext& ctx) -{ - auto rowArg = ctx.NewArgument(pos, "row"); - - TExprNode::TPtr lagQueue; - if (lagQueueSize) { - YQL_ENSURE(lagQueueItemType); - lagQueue = BuildQueue(pos, *lagQueueItemType, lagQueueSize, lagQueueSize, rowArg, ctx); - } - - TExprNodeList structItems; - for (auto& trait : traits) { - structItems.push_back( - ctx.Builder(pos) - .List() - .Atom(0, trait->GetName()) - .Apply(1, trait->BuildInitLambda(dataQueue, ctx)) - .With(0, rowArg) - .Seal() - .Seal() - .Build() - ); - } - - auto asStruct = ctx.NewCallable(pos, "AsStruct", std::move(structItems)); - auto tupleOfOutputAndState = ConvertStructOfTuplesToTupleOfStructs(pos, asStruct, ctx); - - tupleOfOutputAndState = HandleLaggingItems(pos, rowArg, tupleOfOutputAndState, traits, lagQueue, ctx); - - auto finalBody = AddInputMembersToOutput(pos, tupleOfOutputAndState, rowArg, ctx); - return ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg}), std::move(finalBody)); -} - + const TExprNode::TPtr& dataQueue, ui64 lagQueueSize, const TTypeAnnotationNode* lagQueueItemType, TExprContext& ctx) +{ + auto rowArg = ctx.NewArgument(pos, "row"); + + TExprNode::TPtr lagQueue; + if (lagQueueSize) { + YQL_ENSURE(lagQueueItemType); + lagQueue = BuildQueue(pos, *lagQueueItemType, lagQueueSize, lagQueueSize, rowArg, ctx); + } + + TExprNodeList structItems; + for (auto& trait : traits) { + structItems.push_back( + ctx.Builder(pos) + .List() + .Atom(0, trait->GetName()) + .Apply(1, trait->BuildInitLambda(dataQueue, ctx)) + .With(0, rowArg) + .Seal() + .Seal() + .Build() + ); + } + + auto asStruct = ctx.NewCallable(pos, "AsStruct", std::move(structItems)); + auto tupleOfOutputAndState = ConvertStructOfTuplesToTupleOfStructs(pos, asStruct, ctx); + + tupleOfOutputAndState = HandleLaggingItems(pos, rowArg, tupleOfOutputAndState, traits, lagQueue, ctx); + + auto finalBody = AddInputMembersToOutput(pos, tupleOfOutputAndState, rowArg, ctx); + return ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg}), std::move(finalBody)); +} + TExprNode::TPtr BuildChain1MapUpdateLambda(TPositionHandle pos, const TVector<TChain1MapTraits::TPtr>& traits, - const TExprNode::TPtr& dataQueue, bool haveLagQueue, TExprContext& ctx) -{ + const TExprNode::TPtr& dataQueue, bool haveLagQueue, TExprContext& ctx) +{ const auto rowArg = ctx.NewArgument(pos, "row"); const auto stateArg = ctx.NewArgument(pos, "state"); auto state = ctx.Builder(pos) @@ -1511,489 +1511,489 @@ TExprNode::TPtr BuildChain1MapUpdateLambda(TPositionHandle pos, const TVector<TC .Atom(1, "1", TNodeFlags::Default) .Seal() .Build(); - - TExprNode::TPtr lagQueue; - if (haveLagQueue) { + + TExprNode::TPtr lagQueue; + if (haveLagQueue) { lagQueue = ctx.Builder(pos) - .Callable("Nth") + .Callable("Nth") .Add(0, state) .Atom(1, "1", TNodeFlags::Default) - .Seal() - .Build(); + .Seal() + .Build(); state = ctx.Builder(pos) - .Callable("Nth") + .Callable("Nth") .Add(0, std::move(state)) .Atom(1, "0", TNodeFlags::Default) - .Seal() - .Build(); - } - - TExprNodeList structItems; - for (auto& trait : traits) { - structItems.push_back( - ctx.Builder(pos) - .List() - .Atom(0, trait->GetName()) - .Apply(1, trait->BuildUpdateLambda(dataQueue, ctx)) - .With(0, rowArg) - .With(1) - .Callable("Member") - .Add(0, state) - .Atom(1, trait->GetName()) - .Seal() - .Done() - .Seal() - .Seal() - .Build() - ); - } - - auto asStruct = ctx.NewCallable(pos, "AsStruct", std::move(structItems)); - auto tupleOfOutputAndState = ConvertStructOfTuplesToTupleOfStructs(pos, asStruct, ctx); - - tupleOfOutputAndState = HandleLaggingItems(pos, rowArg, tupleOfOutputAndState, traits, lagQueue, ctx); - - auto finalBody = AddInputMembersToOutput(pos, tupleOfOutputAndState, rowArg, ctx); - return ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg, stateArg}), std::move(finalBody)); -} - -bool IsNonCompactFullFrame(const TExprNode& winOnRows, TExprContext& ctx) { - YQL_ENSURE(winOnRows.IsCallable("WinOnRows")); - - TWindowFrameSettings frameSettings; - YQL_ENSURE(ParseWindowFrameSettings(winOnRows, frameSettings, ctx)); - - return !frameSettings.IsCompact && !frameSettings.First.Defined() && !frameSettings.Last.Defined(); -} - -TExprNode::TPtr DeduceCompatibleSort(const TExprNode::TPtr& traitsOne, const TExprNode::TPtr& traitsTwo) { - YQL_ENSURE(traitsOne->IsCallable({"Void", "SortTraits"})); - YQL_ENSURE(traitsTwo->IsCallable({"Void", "SortTraits"})); - - if (traitsOne->IsCallable("Void")) { - return traitsTwo; - } - - if (traitsTwo->IsCallable("Void")) { - return traitsOne; - } - - // TODO: need more advanced logic here - if (traitsOne == traitsTwo) { - return traitsOne; - } - - return {}; -} - -TExprNode::TPtr BuildPartitionsByKeys(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNode::TPtr& keySelector, - const TExprNode::TPtr& sortOrder, const TExprNode::TPtr& sortKey, const TExprNode::TPtr& streamProcessingLambda, - const TExprNode::TPtr& sessionKey, const TExprNode::TPtr& sessionInit, const TExprNode::TPtr& sessionUpdate, - const TExprNode::TPtr& sessionColumns, TExprContext& ctx) -{ - TExprNode::TPtr preprocessLambda; - TExprNode::TPtr chopperKeySelector; - const TExprNode::TPtr addSessionColumnsArg = ctx.NewArgument(pos, "row"); - TExprNode::TPtr addSessionColumnsBody = addSessionColumnsArg; - if (sessionUpdate) { - YQL_ENSURE(sessionKey); - YQL_ENSURE(sessionInit); - preprocessLambda = - AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx); - - chopperKeySelector = ctx.Builder(pos) - .Lambda() - .Param("item") - .List() - .Apply(0, keySelector) - .With(0, "item") - .Seal() - .Callable(1, "Member") - .Arg(0, "item") - .Atom(1, SessionStartMemberName) - .Seal() - .Seal() - .Seal() - .Build(); - } else { - YQL_ENSURE(!sessionKey); - preprocessLambda = MakeIdentityLambda(pos, ctx); - chopperKeySelector = keySelector; - } - - for (auto& column : sessionColumns->ChildrenList()) { - addSessionColumnsBody = ctx.Builder(pos) - .Callable("AddMember") - .Add(0, addSessionColumnsBody) - .Add(1, column) - .Callable(2, "Member") - .Add(0, addSessionColumnsArg) - .Atom(1, SessionParamsMemberName) - .Seal() - .Seal() - .Build(); - } - - addSessionColumnsBody = ctx.Builder(pos) - .Callable("ForceRemoveMember") - .Callable(0, "ForceRemoveMember") - .Add(0, addSessionColumnsBody) - .Atom(1, SessionStartMemberName) - .Seal() - .Atom(1, SessionParamsMemberName) - .Seal() - .Build(); - - auto addSessionColumnsLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { addSessionColumnsArg }), std::move(addSessionColumnsBody)); - - auto groupSwitchLambda = ctx.Builder(pos) - .Lambda() - .Param("prevKey") - .Param("item") - .Callable("AggrNotEquals") - .Arg(0, "prevKey") - .Apply(1, chopperKeySelector) - .With(0, "item") - .Seal() - .Seal() - .Seal() - .Build(); - - return ctx.Builder(pos) - .Callable("PartitionsByKeys") - .Add(0, input) - .Add(1, keySelector) - .Add(2, sortOrder) - .Add(3, sortKey) - .Lambda(4) - .Param("partitionedStream") - .Callable("ForwardList") - .Callable(0, "Chopper") - .Callable(0, "ToStream") - .Apply(0, preprocessLambda) - .With(0, "partitionedStream") - .Seal() - .Seal() - .Add(1, chopperKeySelector) - .Add(2, groupSwitchLambda) - .Lambda(3) - .Param("key") - .Param("singlePartition") - .Callable("Map") - .Apply(0, streamProcessingLambda) - .With(0, "singlePartition") - .Seal() - .Add(1, addSessionColumnsLambda) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -enum EFold1LambdaKind { - INIT, - UPDATE, - CALCULATE, -}; - -TExprNode::TPtr BuildFold1Lambda(TPositionHandle pos, const TExprNode::TPtr& frames, EFold1LambdaKind kind, - const TExprNodeList& keyColumns, TExprContext& ctx) -{ - TExprNode::TPtr arg1 = ctx.NewArgument(pos, "arg1"); - TExprNodeList args = { arg1 }; - - TExprNode::TPtr arg2; - if (kind == EFold1LambdaKind::UPDATE) { - arg2 = ctx.NewArgument(pos, "arg2"); - args.push_back(arg2); - } - - TExprNodeList structItems; - for (auto& winOn : frames->ChildrenList()) { - YQL_ENSURE(IsNonCompactFullFrame(*winOn, ctx)); - for (ui32 i = 1; i < winOn->ChildrenSize(); ++i) { - YQL_ENSURE(winOn->Child(i)->IsList()); - YQL_ENSURE(winOn->Child(i)->Child(0)->IsAtom()); - YQL_ENSURE(winOn->Child(i)->Child(1)->IsCallable("WindowTraits")); - - auto column = winOn->Child(i)->ChildPtr(0); - auto traits = winOn->Child(i)->ChildPtr(1); - auto rowInputType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - - TExprNode::TPtr applied; - - switch (kind) { - case EFold1LambdaKind::INIT: { - auto lambda = traits->ChildPtr(1); - if (lambda->Child(0)->ChildrenSize() == 2) { - lambda = ReplaceLastLambdaArgWithStringLiteral(*lambda, column->Content(), ctx); - } - lambda = ReplaceFirstLambdaArgWithCastStruct(*lambda, *rowInputType, ctx); - YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 1); - applied = ctx.Builder(pos) - .Apply(lambda) - .With(0, arg1) - .Seal() - .Build(); - break; - } - case EFold1LambdaKind::CALCULATE: { - auto lambda = traits->ChildPtr(4); - YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 1); - applied = ctx.Builder(pos) - .Apply(lambda) - .With(0) - .Callable("Member") - .Add(0, arg1) - .Add(1, column) - .Seal() - .Done() - .Seal() - .Build(); - break; - } - case EFold1LambdaKind::UPDATE: { - auto lambda = traits->ChildPtr(2); - if (lambda->Child(0)->ChildrenSize() == 3) { - lambda = ReplaceLastLambdaArgWithStringLiteral(*lambda, column->Content(), ctx); - } - lambda = ReplaceFirstLambdaArgWithCastStruct(*lambda, *rowInputType, ctx); - YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 2); - applied = ctx.Builder(pos) - .Apply(lambda) - .With(0, arg1) - .With(1) - .Callable("Member") - .Add(0, arg2) - .Add(1, column) - .Seal() - .Done() - .Seal() - .Build(); - break; - } - } - - structItems.push_back(ctx.NewList(pos, {column, applied})); - } - } - - // pass key columns as-is - for (auto& keyColumn : keyColumns) { - YQL_ENSURE(keyColumn->IsAtom()); - structItems.push_back( - ctx.Builder(pos) - .List() - .Add(0, keyColumn) - .Callable(1, "Member") - .Add(0, arg1) - .Add(1, keyColumn) - .Seal() - .Seal() - .Build() - ); - } - return ctx.NewLambda(pos, ctx.NewArguments(pos, std::move(args)), ctx.NewCallable(pos, "AsStruct", std::move(structItems))); -} - - -TExprNode::TPtr ExpandNonCompactFullFrames(TPositionHandle pos, const TExprNode::TPtr& inputList, - const TExprNode::TPtr& originalKeyColumns, const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, - const TExprNode::TPtr& sessionTraits, const TExprNode::TPtr& sessionColumns, TExprContext& ctx) -{ - TExprNode::TPtr sessionKey; - TExprNode::TPtr sessionInit; - TExprNode::TPtr sessionUpdate; - TExprNode::TPtr sessionSortTraits; - const TTypeAnnotationNode* sessionKeyType = nullptr; - const TTypeAnnotationNode* sessionParamsType = nullptr; - ExtractSessionWindowParams(pos, sessionTraits, sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, sessionInit, sessionUpdate, ctx); - - TExprNode::TPtr sortKey; - TExprNode::TPtr sortOrder; - TExprNode::TPtr input = inputList; - if (input->IsCallable("ForwardList")) { - // full frame strategy uses input 2 times (for grouping and join) - // TODO: better way to detect "single use input" - input = ctx.NewCallable(pos, "Collect", { input }); - } - - const auto rowType = inputList->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - TVector<const TItemExprType*> rowItems = rowType->GetItems(); - TExprNodeList originalKeysWithSession = originalKeyColumns->ChildrenList(); - - TExprNodeList addedColumns; - const auto commonSortTraits = DeduceCompatibleSort(sortTraits, sessionSortTraits); - ExtractSortKeyAndOrder(pos, commonSortTraits ? commonSortTraits : sortTraits, sortKey, sortOrder, ctx); - if (!commonSortTraits) { - YQL_ENSURE(sessionKey); - YQL_ENSURE(sessionInit); - YQL_ENSURE(sessionUpdate); - TExprNode::TPtr sessionSortKey; - TExprNode::TPtr sessionSortOrder; - ExtractSortKeyAndOrder(pos, sessionSortTraits, sessionSortKey, sessionSortOrder, ctx); - const auto keySelector = BuildKeySelector(pos, *rowType, originalKeyColumns, ctx); - input = ctx.Builder(pos) - .Callable("PartitionsByKeys") - .Add(0, input) - .Add(1, keySelector) - .Add(2, sessionSortOrder) - .Add(3, sessionSortKey) - .Lambda(4) - .Param("partitionedStream") - .Apply(AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx)) - .With(0, "partitionedStream") - .Seal() - .Seal() - .Seal() - .Build(); - - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); - addedColumns.push_back(ctx.NewAtom(pos, SessionParamsMemberName)); - - originalKeysWithSession.push_back(ctx.NewAtom(pos, SessionStartMemberName)); - addedColumns.push_back(originalKeysWithSession.back()); - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); - sessionKey = sessionInit = sessionUpdate = {}; - } - - TExprNodeList keyColumns; - - auto rowArg = ctx.NewArgument(pos, "row"); - auto addMembersBody = rowArg; - - static const TStringBuf keyColumnNamePrefix = "_yql_CalcOverWindowJoinKey"; - - const TStructExprType* rowTypeWithSession = ctx.MakeType<TStructExprType>(rowItems); - for (auto& keyColumn : originalKeysWithSession) { - YQL_ENSURE(keyColumn->IsAtom()); - auto columnName = keyColumn->Content(); - const TTypeAnnotationNode* columnType = - rowTypeWithSession->GetItems()[*rowTypeWithSession->FindItem(columnName)]->GetItemType(); - if (columnType->HasOptionalOrNull()) { - addedColumns.push_back(ctx.NewAtom(pos, TStringBuilder() << keyColumnNamePrefix << addedColumns.size())); - keyColumns.push_back(addedColumns.back()); - - TStringBuf newName = addedColumns.back()->Content(); - const TTypeAnnotationNode* newType = ctx.MakeType<TDataExprType>(EDataSlot::String); - rowItems.push_back(ctx.MakeType<TItemExprType>(newName, newType)); - - addMembersBody = ctx.Builder(pos) - .Callable("AddMember") - .Add(0, addMembersBody) - .Atom(1, newName) - .Callable(2, "StablePickle") - .Callable(0, "Member") - .Add(0, rowArg) - .Add(1, keyColumn) - .Seal() - .Seal() - .Seal() - .Build(); - } else { - keyColumns.push_back(keyColumn); - } - } - - input = ctx.Builder(pos) - .Callable("Map") - .Add(0, input) - .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, { rowArg }), std::move(addMembersBody))) - .Seal() - .Build(); - - auto keySelector = BuildKeySelector(pos, *ctx.MakeType<TStructExprType>(rowItems), - ctx.NewList(pos, TExprNodeList{keyColumns}), ctx); - - TExprNode::TPtr preprocessLambda; - TExprNode::TPtr groupKeySelector; - TExprNode::TPtr condenseSwitch; - if (sessionUpdate) { - YQL_ENSURE(sessionKey); - YQL_ENSURE(sessionInit); - YQL_ENSURE(sessionKeyType); - YQL_ENSURE(commonSortTraits); - - preprocessLambda = - AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx); - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); - - addedColumns.push_back(ctx.NewAtom(pos, SessionStartMemberName)); - addedColumns.push_back(ctx.NewAtom(pos, SessionParamsMemberName)); - - if (sessionKeyType->HasOptionalOrNull()) { - addedColumns.push_back(ctx.NewAtom(pos, TStringBuilder() << keyColumnNamePrefix << addedColumns.size())); - preprocessLambda = ctx.Builder(pos) - .Lambda() - .Param("stream") - .Callable("OrderedMap") - .Apply(0, preprocessLambda) - .With(0, "stream") - .Seal() - .Lambda(1) - .Param("item") - .Callable("AddMember") - .Arg(0, "item") - .Add(1, addedColumns.back()) - .Callable(2, "StablePickle") - .Callable(0, "Member") - .Arg(0, "item") - .Atom(1, SessionStartMemberName) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - - TStringBuf newName = addedColumns.back()->Content(); - const TTypeAnnotationNode* newType = ctx.MakeType<TDataExprType>(EDataSlot::String); - rowItems.push_back(ctx.MakeType<TItemExprType>(newName, newType)); - } - - keyColumns.push_back(addedColumns.back()); - - auto groupKeySelector = BuildKeySelector(pos, *ctx.MakeType<TStructExprType>(rowItems), - ctx.NewList(pos, TExprNodeList{keyColumns}), ctx); - - condenseSwitch = ctx.Builder(pos) - .Lambda() - .Param("row") - .Param("state") - .Callable("AggrNotEquals") - .Apply(0, groupKeySelector) - .With(0, "row") - .Seal() - .Apply(1, groupKeySelector) - .With(0, "state") - .Seal() - .Seal() - .Seal() - .Build(); - } else { - YQL_ENSURE(!sessionKey); - preprocessLambda = MakeIdentityLambda(pos, ctx); - auto groupKeySelector = keySelector; - - condenseSwitch = ctx.Builder(pos) - .Lambda() - .Param("row") - .Param("state") - .Callable("IsKeySwitch") - .Arg(0, "row") - .Arg(1, "state") - .Add(2, groupKeySelector) - .Add(3, groupKeySelector) - .Seal() - .Seal() - .Build(); - } - + .Seal() + .Build(); + } + + TExprNodeList structItems; + for (auto& trait : traits) { + structItems.push_back( + ctx.Builder(pos) + .List() + .Atom(0, trait->GetName()) + .Apply(1, trait->BuildUpdateLambda(dataQueue, ctx)) + .With(0, rowArg) + .With(1) + .Callable("Member") + .Add(0, state) + .Atom(1, trait->GetName()) + .Seal() + .Done() + .Seal() + .Seal() + .Build() + ); + } + + auto asStruct = ctx.NewCallable(pos, "AsStruct", std::move(structItems)); + auto tupleOfOutputAndState = ConvertStructOfTuplesToTupleOfStructs(pos, asStruct, ctx); + + tupleOfOutputAndState = HandleLaggingItems(pos, rowArg, tupleOfOutputAndState, traits, lagQueue, ctx); + + auto finalBody = AddInputMembersToOutput(pos, tupleOfOutputAndState, rowArg, ctx); + return ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg, stateArg}), std::move(finalBody)); +} + +bool IsNonCompactFullFrame(const TExprNode& winOnRows, TExprContext& ctx) { + YQL_ENSURE(winOnRows.IsCallable("WinOnRows")); + + TWindowFrameSettings frameSettings; + YQL_ENSURE(ParseWindowFrameSettings(winOnRows, frameSettings, ctx)); + + return !frameSettings.IsCompact && !frameSettings.First.Defined() && !frameSettings.Last.Defined(); +} + +TExprNode::TPtr DeduceCompatibleSort(const TExprNode::TPtr& traitsOne, const TExprNode::TPtr& traitsTwo) { + YQL_ENSURE(traitsOne->IsCallable({"Void", "SortTraits"})); + YQL_ENSURE(traitsTwo->IsCallable({"Void", "SortTraits"})); + + if (traitsOne->IsCallable("Void")) { + return traitsTwo; + } + + if (traitsTwo->IsCallable("Void")) { + return traitsOne; + } + + // TODO: need more advanced logic here + if (traitsOne == traitsTwo) { + return traitsOne; + } + + return {}; +} + +TExprNode::TPtr BuildPartitionsByKeys(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNode::TPtr& keySelector, + const TExprNode::TPtr& sortOrder, const TExprNode::TPtr& sortKey, const TExprNode::TPtr& streamProcessingLambda, + const TExprNode::TPtr& sessionKey, const TExprNode::TPtr& sessionInit, const TExprNode::TPtr& sessionUpdate, + const TExprNode::TPtr& sessionColumns, TExprContext& ctx) +{ + TExprNode::TPtr preprocessLambda; + TExprNode::TPtr chopperKeySelector; + const TExprNode::TPtr addSessionColumnsArg = ctx.NewArgument(pos, "row"); + TExprNode::TPtr addSessionColumnsBody = addSessionColumnsArg; + if (sessionUpdate) { + YQL_ENSURE(sessionKey); + YQL_ENSURE(sessionInit); + preprocessLambda = + AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx); + + chopperKeySelector = ctx.Builder(pos) + .Lambda() + .Param("item") + .List() + .Apply(0, keySelector) + .With(0, "item") + .Seal() + .Callable(1, "Member") + .Arg(0, "item") + .Atom(1, SessionStartMemberName) + .Seal() + .Seal() + .Seal() + .Build(); + } else { + YQL_ENSURE(!sessionKey); + preprocessLambda = MakeIdentityLambda(pos, ctx); + chopperKeySelector = keySelector; + } + + for (auto& column : sessionColumns->ChildrenList()) { + addSessionColumnsBody = ctx.Builder(pos) + .Callable("AddMember") + .Add(0, addSessionColumnsBody) + .Add(1, column) + .Callable(2, "Member") + .Add(0, addSessionColumnsArg) + .Atom(1, SessionParamsMemberName) + .Seal() + .Seal() + .Build(); + } + + addSessionColumnsBody = ctx.Builder(pos) + .Callable("ForceRemoveMember") + .Callable(0, "ForceRemoveMember") + .Add(0, addSessionColumnsBody) + .Atom(1, SessionStartMemberName) + .Seal() + .Atom(1, SessionParamsMemberName) + .Seal() + .Build(); + + auto addSessionColumnsLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, { addSessionColumnsArg }), std::move(addSessionColumnsBody)); + + auto groupSwitchLambda = ctx.Builder(pos) + .Lambda() + .Param("prevKey") + .Param("item") + .Callable("AggrNotEquals") + .Arg(0, "prevKey") + .Apply(1, chopperKeySelector) + .With(0, "item") + .Seal() + .Seal() + .Seal() + .Build(); + + return ctx.Builder(pos) + .Callable("PartitionsByKeys") + .Add(0, input) + .Add(1, keySelector) + .Add(2, sortOrder) + .Add(3, sortKey) + .Lambda(4) + .Param("partitionedStream") + .Callable("ForwardList") + .Callable(0, "Chopper") + .Callable(0, "ToStream") + .Apply(0, preprocessLambda) + .With(0, "partitionedStream") + .Seal() + .Seal() + .Add(1, chopperKeySelector) + .Add(2, groupSwitchLambda) + .Lambda(3) + .Param("key") + .Param("singlePartition") + .Callable("Map") + .Apply(0, streamProcessingLambda) + .With(0, "singlePartition") + .Seal() + .Add(1, addSessionColumnsLambda) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +enum EFold1LambdaKind { + INIT, + UPDATE, + CALCULATE, +}; + +TExprNode::TPtr BuildFold1Lambda(TPositionHandle pos, const TExprNode::TPtr& frames, EFold1LambdaKind kind, + const TExprNodeList& keyColumns, TExprContext& ctx) +{ + TExprNode::TPtr arg1 = ctx.NewArgument(pos, "arg1"); + TExprNodeList args = { arg1 }; + + TExprNode::TPtr arg2; + if (kind == EFold1LambdaKind::UPDATE) { + arg2 = ctx.NewArgument(pos, "arg2"); + args.push_back(arg2); + } + + TExprNodeList structItems; + for (auto& winOn : frames->ChildrenList()) { + YQL_ENSURE(IsNonCompactFullFrame(*winOn, ctx)); + for (ui32 i = 1; i < winOn->ChildrenSize(); ++i) { + YQL_ENSURE(winOn->Child(i)->IsList()); + YQL_ENSURE(winOn->Child(i)->Child(0)->IsAtom()); + YQL_ENSURE(winOn->Child(i)->Child(1)->IsCallable("WindowTraits")); + + auto column = winOn->Child(i)->ChildPtr(0); + auto traits = winOn->Child(i)->ChildPtr(1); + auto rowInputType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); + + TExprNode::TPtr applied; + + switch (kind) { + case EFold1LambdaKind::INIT: { + auto lambda = traits->ChildPtr(1); + if (lambda->Child(0)->ChildrenSize() == 2) { + lambda = ReplaceLastLambdaArgWithStringLiteral(*lambda, column->Content(), ctx); + } + lambda = ReplaceFirstLambdaArgWithCastStruct(*lambda, *rowInputType, ctx); + YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 1); + applied = ctx.Builder(pos) + .Apply(lambda) + .With(0, arg1) + .Seal() + .Build(); + break; + } + case EFold1LambdaKind::CALCULATE: { + auto lambda = traits->ChildPtr(4); + YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 1); + applied = ctx.Builder(pos) + .Apply(lambda) + .With(0) + .Callable("Member") + .Add(0, arg1) + .Add(1, column) + .Seal() + .Done() + .Seal() + .Build(); + break; + } + case EFold1LambdaKind::UPDATE: { + auto lambda = traits->ChildPtr(2); + if (lambda->Child(0)->ChildrenSize() == 3) { + lambda = ReplaceLastLambdaArgWithStringLiteral(*lambda, column->Content(), ctx); + } + lambda = ReplaceFirstLambdaArgWithCastStruct(*lambda, *rowInputType, ctx); + YQL_ENSURE(lambda->Child(0)->ChildrenSize() == 2); + applied = ctx.Builder(pos) + .Apply(lambda) + .With(0, arg1) + .With(1) + .Callable("Member") + .Add(0, arg2) + .Add(1, column) + .Seal() + .Done() + .Seal() + .Build(); + break; + } + } + + structItems.push_back(ctx.NewList(pos, {column, applied})); + } + } + + // pass key columns as-is + for (auto& keyColumn : keyColumns) { + YQL_ENSURE(keyColumn->IsAtom()); + structItems.push_back( + ctx.Builder(pos) + .List() + .Add(0, keyColumn) + .Callable(1, "Member") + .Add(0, arg1) + .Add(1, keyColumn) + .Seal() + .Seal() + .Build() + ); + } + return ctx.NewLambda(pos, ctx.NewArguments(pos, std::move(args)), ctx.NewCallable(pos, "AsStruct", std::move(structItems))); +} + + +TExprNode::TPtr ExpandNonCompactFullFrames(TPositionHandle pos, const TExprNode::TPtr& inputList, + const TExprNode::TPtr& originalKeyColumns, const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, + const TExprNode::TPtr& sessionTraits, const TExprNode::TPtr& sessionColumns, TExprContext& ctx) +{ + TExprNode::TPtr sessionKey; + TExprNode::TPtr sessionInit; + TExprNode::TPtr sessionUpdate; + TExprNode::TPtr sessionSortTraits; + const TTypeAnnotationNode* sessionKeyType = nullptr; + const TTypeAnnotationNode* sessionParamsType = nullptr; + ExtractSessionWindowParams(pos, sessionTraits, sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, sessionInit, sessionUpdate, ctx); + + TExprNode::TPtr sortKey; + TExprNode::TPtr sortOrder; + TExprNode::TPtr input = inputList; + if (input->IsCallable("ForwardList")) { + // full frame strategy uses input 2 times (for grouping and join) + // TODO: better way to detect "single use input" + input = ctx.NewCallable(pos, "Collect", { input }); + } + + const auto rowType = inputList->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + TVector<const TItemExprType*> rowItems = rowType->GetItems(); + TExprNodeList originalKeysWithSession = originalKeyColumns->ChildrenList(); + + TExprNodeList addedColumns; + const auto commonSortTraits = DeduceCompatibleSort(sortTraits, sessionSortTraits); + ExtractSortKeyAndOrder(pos, commonSortTraits ? commonSortTraits : sortTraits, sortKey, sortOrder, ctx); + if (!commonSortTraits) { + YQL_ENSURE(sessionKey); + YQL_ENSURE(sessionInit); + YQL_ENSURE(sessionUpdate); + TExprNode::TPtr sessionSortKey; + TExprNode::TPtr sessionSortOrder; + ExtractSortKeyAndOrder(pos, sessionSortTraits, sessionSortKey, sessionSortOrder, ctx); + const auto keySelector = BuildKeySelector(pos, *rowType, originalKeyColumns, ctx); + input = ctx.Builder(pos) + .Callable("PartitionsByKeys") + .Add(0, input) + .Add(1, keySelector) + .Add(2, sessionSortOrder) + .Add(3, sessionSortKey) + .Lambda(4) + .Param("partitionedStream") + .Apply(AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx)) + .With(0, "partitionedStream") + .Seal() + .Seal() + .Seal() + .Build(); + + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); + addedColumns.push_back(ctx.NewAtom(pos, SessionParamsMemberName)); + + originalKeysWithSession.push_back(ctx.NewAtom(pos, SessionStartMemberName)); + addedColumns.push_back(originalKeysWithSession.back()); + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); + sessionKey = sessionInit = sessionUpdate = {}; + } + + TExprNodeList keyColumns; + + auto rowArg = ctx.NewArgument(pos, "row"); + auto addMembersBody = rowArg; + + static const TStringBuf keyColumnNamePrefix = "_yql_CalcOverWindowJoinKey"; + + const TStructExprType* rowTypeWithSession = ctx.MakeType<TStructExprType>(rowItems); + for (auto& keyColumn : originalKeysWithSession) { + YQL_ENSURE(keyColumn->IsAtom()); + auto columnName = keyColumn->Content(); + const TTypeAnnotationNode* columnType = + rowTypeWithSession->GetItems()[*rowTypeWithSession->FindItem(columnName)]->GetItemType(); + if (columnType->HasOptionalOrNull()) { + addedColumns.push_back(ctx.NewAtom(pos, TStringBuilder() << keyColumnNamePrefix << addedColumns.size())); + keyColumns.push_back(addedColumns.back()); + + TStringBuf newName = addedColumns.back()->Content(); + const TTypeAnnotationNode* newType = ctx.MakeType<TDataExprType>(EDataSlot::String); + rowItems.push_back(ctx.MakeType<TItemExprType>(newName, newType)); + + addMembersBody = ctx.Builder(pos) + .Callable("AddMember") + .Add(0, addMembersBody) + .Atom(1, newName) + .Callable(2, "StablePickle") + .Callable(0, "Member") + .Add(0, rowArg) + .Add(1, keyColumn) + .Seal() + .Seal() + .Seal() + .Build(); + } else { + keyColumns.push_back(keyColumn); + } + } + + input = ctx.Builder(pos) + .Callable("Map") + .Add(0, input) + .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, { rowArg }), std::move(addMembersBody))) + .Seal() + .Build(); + + auto keySelector = BuildKeySelector(pos, *ctx.MakeType<TStructExprType>(rowItems), + ctx.NewList(pos, TExprNodeList{keyColumns}), ctx); + + TExprNode::TPtr preprocessLambda; + TExprNode::TPtr groupKeySelector; + TExprNode::TPtr condenseSwitch; + if (sessionUpdate) { + YQL_ENSURE(sessionKey); + YQL_ENSURE(sessionInit); + YQL_ENSURE(sessionKeyType); + YQL_ENSURE(commonSortTraits); + + preprocessLambda = + AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx); + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); + + addedColumns.push_back(ctx.NewAtom(pos, SessionStartMemberName)); + addedColumns.push_back(ctx.NewAtom(pos, SessionParamsMemberName)); + + if (sessionKeyType->HasOptionalOrNull()) { + addedColumns.push_back(ctx.NewAtom(pos, TStringBuilder() << keyColumnNamePrefix << addedColumns.size())); + preprocessLambda = ctx.Builder(pos) + .Lambda() + .Param("stream") + .Callable("OrderedMap") + .Apply(0, preprocessLambda) + .With(0, "stream") + .Seal() + .Lambda(1) + .Param("item") + .Callable("AddMember") + .Arg(0, "item") + .Add(1, addedColumns.back()) + .Callable(2, "StablePickle") + .Callable(0, "Member") + .Arg(0, "item") + .Atom(1, SessionStartMemberName) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + + TStringBuf newName = addedColumns.back()->Content(); + const TTypeAnnotationNode* newType = ctx.MakeType<TDataExprType>(EDataSlot::String); + rowItems.push_back(ctx.MakeType<TItemExprType>(newName, newType)); + } + + keyColumns.push_back(addedColumns.back()); + + auto groupKeySelector = BuildKeySelector(pos, *ctx.MakeType<TStructExprType>(rowItems), + ctx.NewList(pos, TExprNodeList{keyColumns}), ctx); + + condenseSwitch = ctx.Builder(pos) + .Lambda() + .Param("row") + .Param("state") + .Callable("AggrNotEquals") + .Apply(0, groupKeySelector) + .With(0, "row") + .Seal() + .Apply(1, groupKeySelector) + .With(0, "state") + .Seal() + .Seal() + .Seal() + .Build(); + } else { + YQL_ENSURE(!sessionKey); + preprocessLambda = MakeIdentityLambda(pos, ctx); + auto groupKeySelector = keySelector; + + condenseSwitch = ctx.Builder(pos) + .Lambda() + .Param("row") + .Param("state") + .Callable("IsKeySwitch") + .Arg(0, "row") + .Arg(1, "state") + .Add(2, groupKeySelector) + .Add(3, groupKeySelector) + .Seal() + .Seal() + .Build(); + } + auto aggregated = ctx.Builder(pos) .Callable("PartitionsByKeys") .Add(0, input) @@ -2004,320 +2004,320 @@ TExprNode::TPtr ExpandNonCompactFullFrames(TPositionHandle pos, const TExprNode: .Param("stream") .Callable("Map") .Callable(0, "Condense1") - .Apply(0, preprocessLambda) - .With(0, "stream") - .Seal() + .Apply(0, preprocessLambda) + .With(0, "stream") + .Seal() .Add(1, BuildFold1Lambda(pos, frames, EFold1LambdaKind::INIT, keyColumns, ctx)) - .Add(2, condenseSwitch) + .Add(2, condenseSwitch) .Add(3, BuildFold1Lambda(pos, frames, EFold1LambdaKind::UPDATE, keyColumns, ctx)) - .Seal() + .Seal() .Add(1, BuildFold1Lambda(pos, frames, EFold1LambdaKind::CALCULATE, keyColumns, ctx)) - .Seal() - .Seal() + .Seal() + .Seal() .Seal().Build(); - - if (sessionUpdate) { - // preprocess input without aggregation - input = ctx.Builder(pos) - .Callable("PartitionsByKeys") - .Add(0, input) - .Add(1, ctx.DeepCopyLambda(*keySelector)) - .Add(2, sortOrder) - .Add(3, ctx.DeepCopyLambda(*sortKey)) - .Lambda(4) - .Param("stream") - .Apply(preprocessLambda) - .With(0, "stream") - .Seal() - .Seal() - .Seal() - .Build(); - } - - TExprNode::TPtr joined; - if (!keyColumns.empty()) { - // SELECT * FROM input AS a JOIN aggregated AS b USING(keyColumns) - auto buildJoinKeysTuple = [&](TStringBuf side) { - TExprNodeList items; - for (const auto& keyColumn : keyColumns) { - items.push_back(ctx.NewAtom(pos, side)); - items.push_back(keyColumn); - } - return ctx.NewList(pos, std::move(items)); - }; - - joined = ctx.Builder(pos) - .Callable("EquiJoin") - .List(0) - .Add(0, input) - .Atom(1, "a") - .Seal() - .List(1) - .Add(0, aggregated) - .Atom(1, "b") - .Seal() - .List(2) - .Atom(0, "Inner") - .Atom(1, "a") - .Atom(2, "b") - .Add(3, buildJoinKeysTuple("a")) - .Add(4, buildJoinKeysTuple("b")) - .List(5) - .List(0) - .Atom(0, "right") - .Atom(1, "any") - .Seal() - .Seal() - .Seal() - .List(3) - .List(0) - .Atom(0, "keep_sys", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Build(); - - // remove b.keys* - auto rowArg = ctx.NewArgument(pos, "row"); - - TExprNode::TPtr removed = rowArg; - - auto removeSide = [&](const TString& side, const TExprNodeList& keys) { - for (const auto& keyColumn : keys) { - YQL_ENSURE(keyColumn->IsAtom()); - TString toRemove = side + keyColumn->Content(); - removed = ctx.Builder(pos) - .Callable("RemoveMember") - .Add(0, removed) - .Atom(1, toRemove) - .Seal() - .Build(); - } - }; - - removeSide("b.", keyColumns); - - // add session columns - for (auto column : sessionColumns->ChildrenList()) { - removed = ctx.Builder(pos) - .Callable("AddMember") - .Add(0, removed) - .Add(1, column) - .Callable(2, "Member") - .Add(0, rowArg) - .Atom(1, TString("a.") + SessionParamsMemberName) - .Seal() - .Seal() - .Build(); - } - - removeSide("a.", addedColumns); - - joined = ctx.Builder(pos) - .Callable("Map") - .Add(0, joined) - .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg}), std::move(removed))) - .Seal() - .Build(); - } else { - // SELECT * FROM input AS a CROSS JOIN aggregated AS b - joined = ctx.Builder(pos) - .Callable("EquiJoin") - .List(0) - .Add(0, input) - .Atom(1, "a") - .Seal() - .List(1) - .Add(0, aggregated) - .Atom(1, "b") - .Seal() - .List(2) - .Atom(0, "Cross") - .Atom(1, "a") - .Atom(2, "b") - .List(3).Seal() - .List(4).Seal() - .List(5).Seal() - .Seal() - .List(3) - .List(0) - .Atom(0, "keep_sys", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Build(); - } - - return ctx.Builder(pos) - .Callable("Map") - .Add(0, joined) - .Lambda(1) - .Param("row") - .Callable("DivePrefixMembers") - .Arg(0, "row") - .List(1) - .Atom(0, "a.") - .Atom(1, "b.") - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - - -TExprNode::TPtr TryExpandNonCompactFullFrames(TPositionHandle pos, const TExprNode::TPtr& inputList, const TExprNode::TPtr& keyColumns, - const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, const TExprNode::TPtr& sessionTraits, - const TExprNode::TPtr& sessionColumns, TExprContext& ctx) -{ - TExprNodeList nonCompactAggregatingFullFrames; - TExprNodeList otherFrames; - - for (auto& winOn : frames->ChildrenList()) { - if (!IsNonCompactFullFrame(*winOn, ctx)) { - otherFrames.push_back(winOn); - continue; - } - - TExprNodeList nonAggregates = { winOn->ChildPtr(0) }; - TExprNodeList aggregates = { winOn->ChildPtr(0) }; - - for (ui32 i = 1; i < winOn->ChildrenSize(); ++i) { - auto item = winOn->Child(i)->Child(1); - if (item->IsCallable("WindowTraits")) { - aggregates.push_back(winOn->ChildPtr(i)); - } else { - nonAggregates.push_back(winOn->ChildPtr(i)); - } - } - - if (aggregates.size() == 1) { - otherFrames.push_back(winOn); - continue; - } - - nonCompactAggregatingFullFrames.push_back(ctx.NewCallable(winOn->Pos(), "WinOnRows", std::move(aggregates))); - if (nonAggregates.size() > 1) { - otherFrames.push_back(ctx.NewCallable(winOn->Pos(), "WinOnRows", std::move(nonAggregates))); - } - } - - if (nonCompactAggregatingFullFrames.empty()) { - return {}; - } - - auto fullFrames = ctx.NewList(pos, std::move(nonCompactAggregatingFullFrames)); - auto nonFullFrames = ctx.NewList(pos, std::move(otherFrames)); - auto expanded = ExpandNonCompactFullFrames(pos, inputList, keyColumns, sortTraits, fullFrames, sessionTraits, sessionColumns, ctx); - - if (sessionTraits && !sessionTraits->IsCallable("Void")) { - return Build<TCoCalcOverSessionWindow>(ctx, pos) - .Input(expanded) - .Keys(keyColumns) - .SortSpec(sortTraits) - .Frames(nonFullFrames) - .SessionSpec(sessionTraits) - .SessionColumns(sessionColumns) - .Done().Ptr(); - } - YQL_ENSURE(sessionColumns->ChildrenSize() == 0); - return Build<TCoCalcOverWindow>(ctx, pos) - .Input(expanded) - .Keys(keyColumns) - .SortSpec(sortTraits) - .Frames(nonFullFrames) - .Done().Ptr(); -} - -TExprNode::TPtr ExpandSingleCalcOverWindow(TPositionHandle pos, const TExprNode::TPtr& inputList, const TExprNode::TPtr& keyColumns, - const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, const TExprNode::TPtr& sessionTraits, - const TExprNode::TPtr& sessionColumns, TExprContext& ctx) -{ - if (auto expanded = TryExpandNonCompactFullFrames(pos, inputList, keyColumns, sortTraits, frames, sessionTraits, sessionColumns, ctx)) { - YQL_CLOG(INFO, Core) << "Expanded non-compact CalcOverWindow"; - return expanded; - } - - ui64 dataOutpace; - ui64 dataLag; - ui64 lagQueueSize; - const TTypeAnnotationNode* lagQueueItemType; + + if (sessionUpdate) { + // preprocess input without aggregation + input = ctx.Builder(pos) + .Callable("PartitionsByKeys") + .Add(0, input) + .Add(1, ctx.DeepCopyLambda(*keySelector)) + .Add(2, sortOrder) + .Add(3, ctx.DeepCopyLambda(*sortKey)) + .Lambda(4) + .Param("stream") + .Apply(preprocessLambda) + .With(0, "stream") + .Seal() + .Seal() + .Seal() + .Build(); + } + + TExprNode::TPtr joined; + if (!keyColumns.empty()) { + // SELECT * FROM input AS a JOIN aggregated AS b USING(keyColumns) + auto buildJoinKeysTuple = [&](TStringBuf side) { + TExprNodeList items; + for (const auto& keyColumn : keyColumns) { + items.push_back(ctx.NewAtom(pos, side)); + items.push_back(keyColumn); + } + return ctx.NewList(pos, std::move(items)); + }; + + joined = ctx.Builder(pos) + .Callable("EquiJoin") + .List(0) + .Add(0, input) + .Atom(1, "a") + .Seal() + .List(1) + .Add(0, aggregated) + .Atom(1, "b") + .Seal() + .List(2) + .Atom(0, "Inner") + .Atom(1, "a") + .Atom(2, "b") + .Add(3, buildJoinKeysTuple("a")) + .Add(4, buildJoinKeysTuple("b")) + .List(5) + .List(0) + .Atom(0, "right") + .Atom(1, "any") + .Seal() + .Seal() + .Seal() + .List(3) + .List(0) + .Atom(0, "keep_sys", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Build(); + + // remove b.keys* + auto rowArg = ctx.NewArgument(pos, "row"); + + TExprNode::TPtr removed = rowArg; + + auto removeSide = [&](const TString& side, const TExprNodeList& keys) { + for (const auto& keyColumn : keys) { + YQL_ENSURE(keyColumn->IsAtom()); + TString toRemove = side + keyColumn->Content(); + removed = ctx.Builder(pos) + .Callable("RemoveMember") + .Add(0, removed) + .Atom(1, toRemove) + .Seal() + .Build(); + } + }; + + removeSide("b.", keyColumns); + + // add session columns + for (auto column : sessionColumns->ChildrenList()) { + removed = ctx.Builder(pos) + .Callable("AddMember") + .Add(0, removed) + .Add(1, column) + .Callable(2, "Member") + .Add(0, rowArg) + .Atom(1, TString("a.") + SessionParamsMemberName) + .Seal() + .Seal() + .Build(); + } + + removeSide("a.", addedColumns); + + joined = ctx.Builder(pos) + .Callable("Map") + .Add(0, joined) + .Add(1, ctx.NewLambda(pos, ctx.NewArguments(pos, {rowArg}), std::move(removed))) + .Seal() + .Build(); + } else { + // SELECT * FROM input AS a CROSS JOIN aggregated AS b + joined = ctx.Builder(pos) + .Callable("EquiJoin") + .List(0) + .Add(0, input) + .Atom(1, "a") + .Seal() + .List(1) + .Add(0, aggregated) + .Atom(1, "b") + .Seal() + .List(2) + .Atom(0, "Cross") + .Atom(1, "a") + .Atom(2, "b") + .List(3).Seal() + .List(4).Seal() + .List(5).Seal() + .Seal() + .List(3) + .List(0) + .Atom(0, "keep_sys", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Build(); + } + + return ctx.Builder(pos) + .Callable("Map") + .Add(0, joined) + .Lambda(1) + .Param("row") + .Callable("DivePrefixMembers") + .Arg(0, "row") + .List(1) + .Atom(0, "a.") + .Atom(1, "b.") + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + + +TExprNode::TPtr TryExpandNonCompactFullFrames(TPositionHandle pos, const TExprNode::TPtr& inputList, const TExprNode::TPtr& keyColumns, + const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, const TExprNode::TPtr& sessionTraits, + const TExprNode::TPtr& sessionColumns, TExprContext& ctx) +{ + TExprNodeList nonCompactAggregatingFullFrames; + TExprNodeList otherFrames; + + for (auto& winOn : frames->ChildrenList()) { + if (!IsNonCompactFullFrame(*winOn, ctx)) { + otherFrames.push_back(winOn); + continue; + } + + TExprNodeList nonAggregates = { winOn->ChildPtr(0) }; + TExprNodeList aggregates = { winOn->ChildPtr(0) }; + + for (ui32 i = 1; i < winOn->ChildrenSize(); ++i) { + auto item = winOn->Child(i)->Child(1); + if (item->IsCallable("WindowTraits")) { + aggregates.push_back(winOn->ChildPtr(i)); + } else { + nonAggregates.push_back(winOn->ChildPtr(i)); + } + } + + if (aggregates.size() == 1) { + otherFrames.push_back(winOn); + continue; + } + + nonCompactAggregatingFullFrames.push_back(ctx.NewCallable(winOn->Pos(), "WinOnRows", std::move(aggregates))); + if (nonAggregates.size() > 1) { + otherFrames.push_back(ctx.NewCallable(winOn->Pos(), "WinOnRows", std::move(nonAggregates))); + } + } + + if (nonCompactAggregatingFullFrames.empty()) { + return {}; + } + + auto fullFrames = ctx.NewList(pos, std::move(nonCompactAggregatingFullFrames)); + auto nonFullFrames = ctx.NewList(pos, std::move(otherFrames)); + auto expanded = ExpandNonCompactFullFrames(pos, inputList, keyColumns, sortTraits, fullFrames, sessionTraits, sessionColumns, ctx); + + if (sessionTraits && !sessionTraits->IsCallable("Void")) { + return Build<TCoCalcOverSessionWindow>(ctx, pos) + .Input(expanded) + .Keys(keyColumns) + .SortSpec(sortTraits) + .Frames(nonFullFrames) + .SessionSpec(sessionTraits) + .SessionColumns(sessionColumns) + .Done().Ptr(); + } + YQL_ENSURE(sessionColumns->ChildrenSize() == 0); + return Build<TCoCalcOverWindow>(ctx, pos) + .Input(expanded) + .Keys(keyColumns) + .SortSpec(sortTraits) + .Frames(nonFullFrames) + .Done().Ptr(); +} + +TExprNode::TPtr ExpandSingleCalcOverWindow(TPositionHandle pos, const TExprNode::TPtr& inputList, const TExprNode::TPtr& keyColumns, + const TExprNode::TPtr& sortTraits, const TExprNode::TPtr& frames, const TExprNode::TPtr& sessionTraits, + const TExprNode::TPtr& sessionColumns, TExprContext& ctx) +{ + if (auto expanded = TryExpandNonCompactFullFrames(pos, inputList, keyColumns, sortTraits, frames, sessionTraits, sessionColumns, ctx)) { + YQL_CLOG(INFO, Core) << "Expanded non-compact CalcOverWindow"; + return expanded; + } + + ui64 dataOutpace; + ui64 dataLag; + ui64 lagQueueSize; + const TTypeAnnotationNode* lagQueueItemType; TVector<TChain1MapTraits::TPtr> traits = BuildFoldMapTraits(dataOutpace, dataLag, lagQueueSize, lagQueueItemType, frames, ctx); - - TExprNode::TPtr sessionKey; - TExprNode::TPtr sessionSortTraits; - const TTypeAnnotationNode* sessionKeyType = nullptr; - const TTypeAnnotationNode* sessionParamsType = nullptr; - TExprNode::TPtr sessionInit; - TExprNode::TPtr sessionUpdate; - ExtractSessionWindowParams(pos, sessionTraits, sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, sessionInit, sessionUpdate, ctx); - - const auto originalRowType = inputList->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); - TVector<const TItemExprType*> rowItems = originalRowType->GetItems(); - if (sessionKeyType) { - YQL_ENSURE(sessionParamsType); - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); - rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); - } - const auto rowType = ctx.MakeType<TStructExprType>(rowItems); - - auto keySelector = BuildKeySelector(pos, *rowType->Cast<TStructExprType>(), keyColumns, ctx); - - TExprNode::TPtr sortKey; - TExprNode::TPtr sortOrder; - TExprNode::TPtr input = inputList; - - const auto commonSortTraits = DeduceCompatibleSort(sortTraits, sessionSortTraits); - ExtractSortKeyAndOrder(pos, commonSortTraits ? commonSortTraits : sortTraits, sortKey, sortOrder, ctx); - if (!commonSortTraits) { - YQL_ENSURE(sessionKey); - YQL_ENSURE(sessionInit); - YQL_ENSURE(sessionUpdate); - TExprNode::TPtr sessionSortKey; - TExprNode::TPtr sessionSortOrder; - ExtractSortKeyAndOrder(pos, sessionSortTraits, sessionSortKey, sessionSortOrder, ctx); - input = ctx.Builder(pos) - .Callable("PartitionsByKeys") - .Add(0, input) - .Add(1, keySelector) - .Add(2, sessionSortOrder) - .Add(3, sessionSortKey) - .Lambda(4) - .Param("partitionedStream") - .Apply(AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx)) - .With(0, "partitionedStream") - .Seal() - .Seal() - .Seal() - .Build(); - - TExprNodeList keyColumnsList = keyColumns->ChildrenList(); - keyColumnsList.push_back(ctx.NewAtom(pos, SessionStartMemberName)); - - auto keyColumnsWithSessionStart = ctx.NewList(pos, std::move(keyColumnsList)); - - keySelector = BuildKeySelector(pos, *rowType, keyColumnsWithSessionStart, ctx); - sessionKey = sessionInit = sessionUpdate = {}; - } - - auto topLevelStreamArg = ctx.NewArgument(pos, "stream"); - - TExprNode::TPtr processed = topLevelStreamArg; - TExprNode::TPtr dataQueue; - if (dataOutpace || dataLag) { - ui64 queueSize = (dataOutpace == Max<ui64>()) ? Max<ui64>() : (dataOutpace + dataLag + 2); - dataQueue = BuildQueue(pos, *rowType, queueSize, dataLag, topLevelStreamArg, ctx); - processed = ctx.Builder(pos) - .Callable("PreserveStream") - .Add(0, topLevelStreamArg) - .Add(1, dataQueue) - .Add(2, BuildUint64(pos, dataOutpace, ctx)) - .Seal() - .Build(); - } - - - processed = ctx.Builder(pos) + + TExprNode::TPtr sessionKey; + TExprNode::TPtr sessionSortTraits; + const TTypeAnnotationNode* sessionKeyType = nullptr; + const TTypeAnnotationNode* sessionParamsType = nullptr; + TExprNode::TPtr sessionInit; + TExprNode::TPtr sessionUpdate; + ExtractSessionWindowParams(pos, sessionTraits, sessionKey, sessionKeyType, sessionParamsType, sessionSortTraits, sessionInit, sessionUpdate, ctx); + + const auto originalRowType = inputList->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + TVector<const TItemExprType*> rowItems = originalRowType->GetItems(); + if (sessionKeyType) { + YQL_ENSURE(sessionParamsType); + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionStartMemberName, sessionKeyType)); + rowItems.push_back(ctx.MakeType<TItemExprType>(SessionParamsMemberName, sessionParamsType)); + } + const auto rowType = ctx.MakeType<TStructExprType>(rowItems); + + auto keySelector = BuildKeySelector(pos, *rowType->Cast<TStructExprType>(), keyColumns, ctx); + + TExprNode::TPtr sortKey; + TExprNode::TPtr sortOrder; + TExprNode::TPtr input = inputList; + + const auto commonSortTraits = DeduceCompatibleSort(sortTraits, sessionSortTraits); + ExtractSortKeyAndOrder(pos, commonSortTraits ? commonSortTraits : sortTraits, sortKey, sortOrder, ctx); + if (!commonSortTraits) { + YQL_ENSURE(sessionKey); + YQL_ENSURE(sessionInit); + YQL_ENSURE(sessionUpdate); + TExprNode::TPtr sessionSortKey; + TExprNode::TPtr sessionSortOrder; + ExtractSortKeyAndOrder(pos, sessionSortTraits, sessionSortKey, sessionSortOrder, ctx); + input = ctx.Builder(pos) + .Callable("PartitionsByKeys") + .Add(0, input) + .Add(1, keySelector) + .Add(2, sessionSortOrder) + .Add(3, sessionSortKey) + .Lambda(4) + .Param("partitionedStream") + .Apply(AddSessionParamsMemberLambda(pos, SessionStartMemberName, SessionParamsMemberName, keySelector, sessionKey, sessionInit, sessionUpdate, ctx)) + .With(0, "partitionedStream") + .Seal() + .Seal() + .Seal() + .Build(); + + TExprNodeList keyColumnsList = keyColumns->ChildrenList(); + keyColumnsList.push_back(ctx.NewAtom(pos, SessionStartMemberName)); + + auto keyColumnsWithSessionStart = ctx.NewList(pos, std::move(keyColumnsList)); + + keySelector = BuildKeySelector(pos, *rowType, keyColumnsWithSessionStart, ctx); + sessionKey = sessionInit = sessionUpdate = {}; + } + + auto topLevelStreamArg = ctx.NewArgument(pos, "stream"); + + TExprNode::TPtr processed = topLevelStreamArg; + TExprNode::TPtr dataQueue; + if (dataOutpace || dataLag) { + ui64 queueSize = (dataOutpace == Max<ui64>()) ? Max<ui64>() : (dataOutpace + dataLag + 2); + dataQueue = BuildQueue(pos, *rowType, queueSize, dataLag, topLevelStreamArg, ctx); + processed = ctx.Builder(pos) + .Callable("PreserveStream") + .Add(0, topLevelStreamArg) + .Add(1, dataQueue) + .Add(2, BuildUint64(pos, dataOutpace, ctx)) + .Seal() + .Build(); + } + + + processed = ctx.Builder(pos) .Callable("OrderedMap") .Callable(0, "Chain1Map") .Add(0, std::move(processed)) @@ -2331,469 +2331,469 @@ TExprNode::TPtr ExpandSingleCalcOverWindow(TPositionHandle pos, const TExprNode: .Atom(1, "0", TNodeFlags::Default) .Seal() .Seal() - .Seal() - .Build(); - - auto topLevelStreamProcessingLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, {topLevelStreamArg}), std::move(processed)); - - YQL_CLOG(INFO, Core) << "Expanded compact CalcOverWindow"; - return BuildPartitionsByKeys(pos, input, keySelector, sortOrder, sortKey, topLevelStreamProcessingLambda, sessionKey, - sessionInit, sessionUpdate, sessionColumns, ctx); -} - -} // namespace - -TExprNode::TPtr ExpandCalcOverWindow(const TExprNode::TPtr& node, TExprContext& ctx) { - YQL_ENSURE(node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})); - - auto input = node->ChildPtr(0); - auto calcs = ExtractCalcsOverWindow(node, ctx); - if (calcs.empty()) { - return input; - } - - TCoCalcOverWindowTuple calc(calcs.front()); - if (calc.Frames().Size() != 0 || calc.SessionColumns().Size() != 0) { - input = ExpandSingleCalcOverWindow(node->Pos(), input, calc.Keys().Ptr(), calc.SortSpec().Ptr(), calc.Frames().Ptr(), - calc.SessionSpec().Ptr(), calc.SessionColumns().Ptr(), ctx); - } - - calcs.erase(calcs.begin()); - return RebuildCalcOverWindowGroup(node->Pos(), input, calcs, ctx); -} - - -TExprNodeList ExtractCalcsOverWindow(const TExprNodePtr& node, TExprContext& ctx) { - TExprNodeList result; - if (auto maybeBase = TMaybeNode<TCoCalcOverWindowBase>(node)) { - TCoCalcOverWindowBase self(maybeBase.Cast()); - TExprNode::TPtr sessionSpec; - TExprNode::TPtr sessionColumns; - if (auto session = TMaybeNode<TCoCalcOverSessionWindow>(node)) { - sessionSpec = session.Cast().SessionSpec().Ptr(); - sessionColumns = session.Cast().SessionColumns().Ptr(); - } else { - sessionSpec = ctx.NewCallable(node->Pos(), "Void", {}); - sessionColumns = ctx.NewList(node->Pos(), {}); - } - result.emplace_back( - Build<TCoCalcOverWindowTuple>(ctx, node->Pos()) - .Keys(self.Keys()) - .SortSpec(self.SortSpec()) - .Frames(self.Frames()) - .SessionSpec(sessionSpec) - .SessionColumns(sessionColumns) - .Done().Ptr() - ); - } else { - result = TMaybeNode<TCoCalcOverWindowGroup>(node).Cast().Calcs().Ref().ChildrenList(); - } - return result; -} - -TExprNode::TPtr RebuildCalcOverWindowGroup(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNodeList& calcs, TExprContext& ctx) { - auto inputType = ctx.Builder(input->Pos()) - .Callable("TypeOf") - .Add(0, input) - .Seal() - .Build(); - - auto inputItemType = ctx.Builder(input->Pos()) - .Callable("ListItemType") - .Add(0, inputType) - .Seal() - .Build(); - - TExprNodeList fixedCalcs; - for (auto calcNode : calcs) { - TCoCalcOverWindowTuple calc(calcNode); - auto sortSpec = calc.SortSpec().Ptr(); - if (sortSpec->IsCallable("SortTraits")) { - sortSpec = ctx.Builder(sortSpec->Pos()) - .Callable("SortTraits") - .Add(0, inputType) - .Add(1, sortSpec->ChildPtr(1)) - .Add(2, ctx.DeepCopyLambda(*sortSpec->Child(2))) - .Seal() - .Build(); - } else { - YQL_ENSURE(sortSpec->IsCallable("Void")); - } - - auto sessionSpec = calc.SessionSpec().Ptr(); - if (sessionSpec->IsCallable("SessionWindowTraits")) { - TCoSessionWindowTraits traits(sessionSpec); - auto sessionSortSpec = traits.SortSpec().Ptr(); - if (auto maybeSort = TMaybeNode<TCoSortTraits>(sessionSortSpec)) { - sessionSortSpec = Build<TCoSortTraits>(ctx, sessionSortSpec->Pos()) - .ListType(inputType) - .SortDirections(maybeSort.Cast().SortDirections()) - .SortKeySelectorLambda(ctx.DeepCopyLambda(maybeSort.Cast().SortKeySelectorLambda().Ref())) - .Done().Ptr(); - } else { - YQL_ENSURE(sessionSortSpec->IsCallable("Void")); - } - - sessionSpec = Build<TCoSessionWindowTraits>(ctx, traits.Pos()) - .ListType(inputType) - .SortSpec(sessionSortSpec) - .InitState(ctx.DeepCopyLambda(traits.InitState().Ref())) - .UpdateState(ctx.DeepCopyLambda(traits.UpdateState().Ref())) - .Calculate(ctx.DeepCopyLambda(traits.Calculate().Ref())) - .Done().Ptr(); - } else { - YQL_ENSURE(sessionSpec->IsCallable("Void")); - } - - auto sessionColumns = calc.SessionColumns().Ptr(); - - TExprNodeList newFrames; - for (auto frameNode : calc.Frames().Ref().Children()) { - YQL_ENSURE(frameNode->IsCallable("WinOnRows")); - TExprNodeList winOnRowsArgs = { frameNode->ChildPtr(0) }; - for (ui32 i = 1; i < frameNode->ChildrenSize(); ++i) { - auto kvTuple = frameNode->ChildPtr(i); - YQL_ENSURE(kvTuple->IsList()); - YQL_ENSURE(kvTuple->ChildrenSize() == 2); - - auto columnName = kvTuple->ChildPtr(0); - - auto traits = kvTuple->ChildPtr(1); - YQL_ENSURE(traits->IsCallable({"Lag", "Lead", "RowNumber", "Rank", "DenseRank", "WindowTraits"})); - if (traits->IsCallable("WindowTraits")) { - traits = ctx.Builder(traits->Pos()) - .Callable(traits->Content()) - .Add(0, inputItemType) - .Add(1, ctx.DeepCopyLambda(*traits->Child(1))) - .Add(2, ctx.DeepCopyLambda(*traits->Child(2))) - .Add(3, ctx.DeepCopyLambda(*traits->Child(3))) - .Add(4, ctx.DeepCopyLambda(*traits->Child(4))) - .Add(5, traits->Child(5)->IsLambda() ? ctx.DeepCopyLambda(*traits->Child(5)) : traits->ChildPtr(5)) - .Seal() - .Build(); - } else { - TExprNodeList args; - args.push_back(inputType); - if (traits->ChildrenSize() > 1) { - args.push_back(ctx.DeepCopyLambda(*traits->Child(1))); - } - if (traits->ChildrenSize() > 2) { - args.push_back(traits->ChildPtr(2)); - } - traits = ctx.NewCallable(traits->Pos(), traits->Content(), std::move(args)); - } - - winOnRowsArgs.push_back(ctx.NewList(kvTuple->Pos(), {columnName, traits})); - } - newFrames.push_back(ctx.NewCallable(frameNode->Pos(), "WinOnRows", std::move(winOnRowsArgs))); - } - - fixedCalcs.push_back( - Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) - .Keys(calc.Keys()) - .SortSpec(sortSpec) - .Frames(ctx.NewList(calc.Frames().Pos(), std::move(newFrames))) - .SessionSpec(sessionSpec) - .SessionColumns(sessionColumns) - .Done().Ptr() - ); - } - - return Build<TCoCalcOverWindowGroup>(ctx, pos) - .Input(input) - .Calcs(ctx.NewList(pos, std::move(fixedCalcs))) - .Done().Ptr(); -} - -bool ParseWindowFrameSettings(const TExprNode& node, TWindowFrameSettings& settings, TExprContext& ctx) { - settings = {}; - - YQL_ENSURE(node.IsCallable("WinOnRows")); - auto frameSpec = node.Child(0); - if (frameSpec->Type() == TExprNode::List) { - bool hasBegin = false; - bool hasEnd = false; - - for (const auto& setting : frameSpec->Children()) { - if (!EnsureTupleMinSize(*setting, 1, ctx)) { - return false; - } - - if (!EnsureAtom(setting->Head(), ctx)) { - return false; - } - - const auto settingName = setting->Head().Content(); - if (settingName != "begin" && settingName != "end" && settingName != "compact") { - ctx.AddError( - TIssue(ctx.GetPosition(setting->Pos()), TStringBuilder() << "Invalid frame bound '" << settingName << "'")); - return false; - } - - if (settingName == "compact") { - settings.IsCompact = true; - continue; - } - - if (!EnsureTupleSize(*setting, 2, ctx)) { - return false; - } - - bool& hasBound = (settingName == "begin") ? hasBegin : hasEnd; - if (hasBound) { - ctx.AddError( - TIssue(ctx.GetPosition(setting->Pos()), TStringBuilder() << "Duplicate " << settingName << " frame bound detected")); - return false; - } - - TMaybe<i32>& boundValue = (settingName == "begin") ? settings.First : settings.Last; - if (setting->Tail().IsCallable("Int32")) { - auto& valNode = setting->Tail().Head(); - YQL_ENSURE(valNode.IsAtom()); - i32 value; - YQL_ENSURE(TryFromString(valNode.Content(), value)); - boundValue = value; - } else if (!setting->Tail().IsCallable("Void")) { - const TTypeAnnotationNode* type = setting->Tail().GetTypeAnn(); - TStringBuilder errMsg; - if (!type) { - errMsg << "lambda"; - } else if (setting->Tail().IsCallable()) { - errMsg << setting->Tail().Content() << " with type " << *type; - } else { - errMsg << *type; - } - - ctx.AddError(TIssue(ctx.GetPosition(setting->Tail().Pos()), - TStringBuilder() << "Invalid " << settingName << " frame bound - expecting Void or Int32 callable, but got: " << errMsg)); - return false; - } - hasBound = true; - } - - if (!hasBegin || !hasEnd) { - ctx.AddError(TIssue(ctx.GetPosition(frameSpec->Pos()), - TStringBuilder() << "Missing " << (!hasBegin ? "begin" : "end") << " bound in frame definition")); - return false; - } - } else if (frameSpec->IsCallable("Void")) { - settings.First = {}; - settings.Last = 0; - } else { - const TTypeAnnotationNode* type = frameSpec->GetTypeAnn(); - ctx.AddError(TIssue(ctx.GetPosition(frameSpec->Pos()), - TStringBuilder() << "Invalid window frame - expecting Tuple or Void, but got: " << (type ? FormatType(type) : "lambda"))); - return false; - } - - // frame will always contain rows if it includes current row - if (!settings.First.Defined()) { - settings.NeverEmpty = !settings.Last.Defined() || *settings.Last >= 0; - } else if (!settings.Last.Defined()) { - settings.NeverEmpty = !settings.First.Defined() || *settings.First <= 0; - } else { - settings.NeverEmpty = *settings.First <= *settings.Last && *settings.First <= 0 && *settings.Last >= 0; - } - - return true; -} - -TExprNode::TPtr ZipWithSessionParamsLambda(TPositionHandle pos, const TExprNode::TPtr& partitionKeySelector, - const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, - const TExprNode::TPtr& sessionUpdate, TExprContext& ctx) -{ - auto extractTupleItem = [&](ui32 idx) { - return ctx.Builder(pos) - .Lambda() - .Param("tuple") - .Callable("Nth") - .Arg(0, "tuple") - .Atom(1, ToString(idx), TNodeFlags::Default) - .Seal() - .Seal() - .Build(); - }; - - auto initLambda = ctx.Builder(pos) - .Lambda() - .Param("row") - .List() // row, sessionKey, sessionState, partitionKey - .Arg(0, "row") - .Apply(1, sessionKeySelector) - .With(0, "row") - .With(1) - .Apply(sessionInit) - .With(0, "row") - .Seal() - .Done() - .Seal() - .Apply(2, sessionInit) - .With(0, "row") - .Seal() - .Apply(3, partitionKeySelector) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Build(); - - auto newPartitionLambda = ctx.Builder(pos) - .Lambda() - .Param("row") - .Param("prevBigState") - .Callable("AggrNotEquals") - .Apply(0, partitionKeySelector) - .With(0, "row") - .Seal() - .Apply(1, partitionKeySelector) - .With(0) - .Apply(extractTupleItem(0)) - .With(0, "prevBigState") - .Seal() - .Done() - .Seal() - .Seal() - .Seal() - .Build(); - - auto newSessionOrUpdatedStateLambda = [&](bool newSession) { - return ctx.Builder(pos) - .Lambda() - .Param("row") - .Param("prevBigState") - .Apply(extractTupleItem(newSession ? 0 : 1)) - .With(0) - .Apply(sessionUpdate) - .With(0, "row") - .With(1) - .Apply(extractTupleItem(2)) - .With(0, "prevBigState") - .Seal() - .Done() - .Seal() - .Done() - .Seal() - .Seal() - .Build(); - }; - - return ctx.Builder(pos) - .Lambda() - .Param("input") - .Callable("Chain1Map") - .Arg(0, "input") - .Add(1, initLambda) - .Lambda(2) - .Param("row") - .Param("prevBigState") - .Callable("If") - .Apply(0, newPartitionLambda) - .With(0, "row") - .With(1, "prevBigState") - .Seal() - .Apply(1, initLambda) - .With(0, "row") - .Seal() - .List(2) - .Arg(0, "row") - .Callable(1, "If") - .Apply(0, newSessionOrUpdatedStateLambda(/* newSession = */ true)) - .With(0, "row") - .With(1, "prevBigState") - .Seal() - .Apply(1, sessionKeySelector) - .With(0, "row") - .With(1) - .Apply(newSessionOrUpdatedStateLambda(/* newSession = */ false)) - .With(0, "row") - .With(1, "prevBigState") - .Seal() - .Done() - .Seal() - .Apply(2, extractTupleItem(1)) - .With(0, "prevBigState") - .Seal() - .Seal() - .Apply(2, newSessionOrUpdatedStateLambda(/* newSession = */ false)) - .With(0, "row") - .With(1, "prevBigState") - .Seal() - .Apply(3, partitionKeySelector) - .With(0, "row") - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); -} - -TExprNode::TPtr AddSessionParamsMemberLambda(TPositionHandle pos, - TStringBuf sessionStartMemberName, TStringBuf sessionParamsMemberName, - const TExprNode::TPtr& partitionKeySelector, - const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, - const TExprNode::TPtr& sessionUpdate, TExprContext& ctx) -{ - YQL_ENSURE(sessionStartMemberName); - TExprNode::TPtr addLambda = ctx.Builder(pos) - .Lambda() - .Param("tupleOfItemAndSessionParams") - .Callable("AddMember") - .Callable(0, "Nth") - .Arg(0, "tupleOfItemAndSessionParams") - .Atom(1, "0", TNodeFlags::Default) - .Seal() - .Atom(1, sessionStartMemberName) - .Callable(2, "Nth") - .Arg(0, "tupleOfItemAndSessionParams") - .Atom(1, "1", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Build(); - - if (sessionParamsMemberName) { - addLambda = ctx.Builder(pos) - .Lambda() - .Param("tupleOfItemAndSessionParams") - .Callable("AddMember") - .Apply(0, addLambda) - .With(0, "tupleOfItemAndSessionParams") - .Seal() - .Atom(1, sessionParamsMemberName) - .Callable(2, "AsStruct") - .List(0) - .Atom(0, "start", TNodeFlags::Default) - .Callable(1, "Nth") - .Arg(0, "tupleOfItemAndSessionParams") - .Atom(1, "1", TNodeFlags::Default) - .Seal() - .Seal() - .List(1) - .Atom(0, "state", TNodeFlags::Default) - .Callable(1, "Nth") - .Arg(0, "tupleOfItemAndSessionParams") - .Atom(1, "2", TNodeFlags::Default) - .Seal() - .Seal() - .Seal() - .Seal() - .Seal() - .Build(); - } - - return ctx.Builder(pos) - .Lambda() - .Param("input") - .Callable("OrderedMap") - .Apply(0, ZipWithSessionParamsLambda(pos, partitionKeySelector, sessionKeySelector, sessionInit, sessionUpdate, ctx)) - .With(0, "input") - .Seal() - .Add(1, addLambda) - .Seal() - .Seal() - .Build(); -} - -} + .Seal() + .Build(); + + auto topLevelStreamProcessingLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, {topLevelStreamArg}), std::move(processed)); + + YQL_CLOG(INFO, Core) << "Expanded compact CalcOverWindow"; + return BuildPartitionsByKeys(pos, input, keySelector, sortOrder, sortKey, topLevelStreamProcessingLambda, sessionKey, + sessionInit, sessionUpdate, sessionColumns, ctx); +} + +} // namespace + +TExprNode::TPtr ExpandCalcOverWindow(const TExprNode::TPtr& node, TExprContext& ctx) { + YQL_ENSURE(node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"})); + + auto input = node->ChildPtr(0); + auto calcs = ExtractCalcsOverWindow(node, ctx); + if (calcs.empty()) { + return input; + } + + TCoCalcOverWindowTuple calc(calcs.front()); + if (calc.Frames().Size() != 0 || calc.SessionColumns().Size() != 0) { + input = ExpandSingleCalcOverWindow(node->Pos(), input, calc.Keys().Ptr(), calc.SortSpec().Ptr(), calc.Frames().Ptr(), + calc.SessionSpec().Ptr(), calc.SessionColumns().Ptr(), ctx); + } + + calcs.erase(calcs.begin()); + return RebuildCalcOverWindowGroup(node->Pos(), input, calcs, ctx); +} + + +TExprNodeList ExtractCalcsOverWindow(const TExprNodePtr& node, TExprContext& ctx) { + TExprNodeList result; + if (auto maybeBase = TMaybeNode<TCoCalcOverWindowBase>(node)) { + TCoCalcOverWindowBase self(maybeBase.Cast()); + TExprNode::TPtr sessionSpec; + TExprNode::TPtr sessionColumns; + if (auto session = TMaybeNode<TCoCalcOverSessionWindow>(node)) { + sessionSpec = session.Cast().SessionSpec().Ptr(); + sessionColumns = session.Cast().SessionColumns().Ptr(); + } else { + sessionSpec = ctx.NewCallable(node->Pos(), "Void", {}); + sessionColumns = ctx.NewList(node->Pos(), {}); + } + result.emplace_back( + Build<TCoCalcOverWindowTuple>(ctx, node->Pos()) + .Keys(self.Keys()) + .SortSpec(self.SortSpec()) + .Frames(self.Frames()) + .SessionSpec(sessionSpec) + .SessionColumns(sessionColumns) + .Done().Ptr() + ); + } else { + result = TMaybeNode<TCoCalcOverWindowGroup>(node).Cast().Calcs().Ref().ChildrenList(); + } + return result; +} + +TExprNode::TPtr RebuildCalcOverWindowGroup(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNodeList& calcs, TExprContext& ctx) { + auto inputType = ctx.Builder(input->Pos()) + .Callable("TypeOf") + .Add(0, input) + .Seal() + .Build(); + + auto inputItemType = ctx.Builder(input->Pos()) + .Callable("ListItemType") + .Add(0, inputType) + .Seal() + .Build(); + + TExprNodeList fixedCalcs; + for (auto calcNode : calcs) { + TCoCalcOverWindowTuple calc(calcNode); + auto sortSpec = calc.SortSpec().Ptr(); + if (sortSpec->IsCallable("SortTraits")) { + sortSpec = ctx.Builder(sortSpec->Pos()) + .Callable("SortTraits") + .Add(0, inputType) + .Add(1, sortSpec->ChildPtr(1)) + .Add(2, ctx.DeepCopyLambda(*sortSpec->Child(2))) + .Seal() + .Build(); + } else { + YQL_ENSURE(sortSpec->IsCallable("Void")); + } + + auto sessionSpec = calc.SessionSpec().Ptr(); + if (sessionSpec->IsCallable("SessionWindowTraits")) { + TCoSessionWindowTraits traits(sessionSpec); + auto sessionSortSpec = traits.SortSpec().Ptr(); + if (auto maybeSort = TMaybeNode<TCoSortTraits>(sessionSortSpec)) { + sessionSortSpec = Build<TCoSortTraits>(ctx, sessionSortSpec->Pos()) + .ListType(inputType) + .SortDirections(maybeSort.Cast().SortDirections()) + .SortKeySelectorLambda(ctx.DeepCopyLambda(maybeSort.Cast().SortKeySelectorLambda().Ref())) + .Done().Ptr(); + } else { + YQL_ENSURE(sessionSortSpec->IsCallable("Void")); + } + + sessionSpec = Build<TCoSessionWindowTraits>(ctx, traits.Pos()) + .ListType(inputType) + .SortSpec(sessionSortSpec) + .InitState(ctx.DeepCopyLambda(traits.InitState().Ref())) + .UpdateState(ctx.DeepCopyLambda(traits.UpdateState().Ref())) + .Calculate(ctx.DeepCopyLambda(traits.Calculate().Ref())) + .Done().Ptr(); + } else { + YQL_ENSURE(sessionSpec->IsCallable("Void")); + } + + auto sessionColumns = calc.SessionColumns().Ptr(); + + TExprNodeList newFrames; + for (auto frameNode : calc.Frames().Ref().Children()) { + YQL_ENSURE(frameNode->IsCallable("WinOnRows")); + TExprNodeList winOnRowsArgs = { frameNode->ChildPtr(0) }; + for (ui32 i = 1; i < frameNode->ChildrenSize(); ++i) { + auto kvTuple = frameNode->ChildPtr(i); + YQL_ENSURE(kvTuple->IsList()); + YQL_ENSURE(kvTuple->ChildrenSize() == 2); + + auto columnName = kvTuple->ChildPtr(0); + + auto traits = kvTuple->ChildPtr(1); + YQL_ENSURE(traits->IsCallable({"Lag", "Lead", "RowNumber", "Rank", "DenseRank", "WindowTraits"})); + if (traits->IsCallable("WindowTraits")) { + traits = ctx.Builder(traits->Pos()) + .Callable(traits->Content()) + .Add(0, inputItemType) + .Add(1, ctx.DeepCopyLambda(*traits->Child(1))) + .Add(2, ctx.DeepCopyLambda(*traits->Child(2))) + .Add(3, ctx.DeepCopyLambda(*traits->Child(3))) + .Add(4, ctx.DeepCopyLambda(*traits->Child(4))) + .Add(5, traits->Child(5)->IsLambda() ? ctx.DeepCopyLambda(*traits->Child(5)) : traits->ChildPtr(5)) + .Seal() + .Build(); + } else { + TExprNodeList args; + args.push_back(inputType); + if (traits->ChildrenSize() > 1) { + args.push_back(ctx.DeepCopyLambda(*traits->Child(1))); + } + if (traits->ChildrenSize() > 2) { + args.push_back(traits->ChildPtr(2)); + } + traits = ctx.NewCallable(traits->Pos(), traits->Content(), std::move(args)); + } + + winOnRowsArgs.push_back(ctx.NewList(kvTuple->Pos(), {columnName, traits})); + } + newFrames.push_back(ctx.NewCallable(frameNode->Pos(), "WinOnRows", std::move(winOnRowsArgs))); + } + + fixedCalcs.push_back( + Build<TCoCalcOverWindowTuple>(ctx, calc.Pos()) + .Keys(calc.Keys()) + .SortSpec(sortSpec) + .Frames(ctx.NewList(calc.Frames().Pos(), std::move(newFrames))) + .SessionSpec(sessionSpec) + .SessionColumns(sessionColumns) + .Done().Ptr() + ); + } + + return Build<TCoCalcOverWindowGroup>(ctx, pos) + .Input(input) + .Calcs(ctx.NewList(pos, std::move(fixedCalcs))) + .Done().Ptr(); +} + +bool ParseWindowFrameSettings(const TExprNode& node, TWindowFrameSettings& settings, TExprContext& ctx) { + settings = {}; + + YQL_ENSURE(node.IsCallable("WinOnRows")); + auto frameSpec = node.Child(0); + if (frameSpec->Type() == TExprNode::List) { + bool hasBegin = false; + bool hasEnd = false; + + for (const auto& setting : frameSpec->Children()) { + if (!EnsureTupleMinSize(*setting, 1, ctx)) { + return false; + } + + if (!EnsureAtom(setting->Head(), ctx)) { + return false; + } + + const auto settingName = setting->Head().Content(); + if (settingName != "begin" && settingName != "end" && settingName != "compact") { + ctx.AddError( + TIssue(ctx.GetPosition(setting->Pos()), TStringBuilder() << "Invalid frame bound '" << settingName << "'")); + return false; + } + + if (settingName == "compact") { + settings.IsCompact = true; + continue; + } + + if (!EnsureTupleSize(*setting, 2, ctx)) { + return false; + } + + bool& hasBound = (settingName == "begin") ? hasBegin : hasEnd; + if (hasBound) { + ctx.AddError( + TIssue(ctx.GetPosition(setting->Pos()), TStringBuilder() << "Duplicate " << settingName << " frame bound detected")); + return false; + } + + TMaybe<i32>& boundValue = (settingName == "begin") ? settings.First : settings.Last; + if (setting->Tail().IsCallable("Int32")) { + auto& valNode = setting->Tail().Head(); + YQL_ENSURE(valNode.IsAtom()); + i32 value; + YQL_ENSURE(TryFromString(valNode.Content(), value)); + boundValue = value; + } else if (!setting->Tail().IsCallable("Void")) { + const TTypeAnnotationNode* type = setting->Tail().GetTypeAnn(); + TStringBuilder errMsg; + if (!type) { + errMsg << "lambda"; + } else if (setting->Tail().IsCallable()) { + errMsg << setting->Tail().Content() << " with type " << *type; + } else { + errMsg << *type; + } + + ctx.AddError(TIssue(ctx.GetPosition(setting->Tail().Pos()), + TStringBuilder() << "Invalid " << settingName << " frame bound - expecting Void or Int32 callable, but got: " << errMsg)); + return false; + } + hasBound = true; + } + + if (!hasBegin || !hasEnd) { + ctx.AddError(TIssue(ctx.GetPosition(frameSpec->Pos()), + TStringBuilder() << "Missing " << (!hasBegin ? "begin" : "end") << " bound in frame definition")); + return false; + } + } else if (frameSpec->IsCallable("Void")) { + settings.First = {}; + settings.Last = 0; + } else { + const TTypeAnnotationNode* type = frameSpec->GetTypeAnn(); + ctx.AddError(TIssue(ctx.GetPosition(frameSpec->Pos()), + TStringBuilder() << "Invalid window frame - expecting Tuple or Void, but got: " << (type ? FormatType(type) : "lambda"))); + return false; + } + + // frame will always contain rows if it includes current row + if (!settings.First.Defined()) { + settings.NeverEmpty = !settings.Last.Defined() || *settings.Last >= 0; + } else if (!settings.Last.Defined()) { + settings.NeverEmpty = !settings.First.Defined() || *settings.First <= 0; + } else { + settings.NeverEmpty = *settings.First <= *settings.Last && *settings.First <= 0 && *settings.Last >= 0; + } + + return true; +} + +TExprNode::TPtr ZipWithSessionParamsLambda(TPositionHandle pos, const TExprNode::TPtr& partitionKeySelector, + const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, + const TExprNode::TPtr& sessionUpdate, TExprContext& ctx) +{ + auto extractTupleItem = [&](ui32 idx) { + return ctx.Builder(pos) + .Lambda() + .Param("tuple") + .Callable("Nth") + .Arg(0, "tuple") + .Atom(1, ToString(idx), TNodeFlags::Default) + .Seal() + .Seal() + .Build(); + }; + + auto initLambda = ctx.Builder(pos) + .Lambda() + .Param("row") + .List() // row, sessionKey, sessionState, partitionKey + .Arg(0, "row") + .Apply(1, sessionKeySelector) + .With(0, "row") + .With(1) + .Apply(sessionInit) + .With(0, "row") + .Seal() + .Done() + .Seal() + .Apply(2, sessionInit) + .With(0, "row") + .Seal() + .Apply(3, partitionKeySelector) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Build(); + + auto newPartitionLambda = ctx.Builder(pos) + .Lambda() + .Param("row") + .Param("prevBigState") + .Callable("AggrNotEquals") + .Apply(0, partitionKeySelector) + .With(0, "row") + .Seal() + .Apply(1, partitionKeySelector) + .With(0) + .Apply(extractTupleItem(0)) + .With(0, "prevBigState") + .Seal() + .Done() + .Seal() + .Seal() + .Seal() + .Build(); + + auto newSessionOrUpdatedStateLambda = [&](bool newSession) { + return ctx.Builder(pos) + .Lambda() + .Param("row") + .Param("prevBigState") + .Apply(extractTupleItem(newSession ? 0 : 1)) + .With(0) + .Apply(sessionUpdate) + .With(0, "row") + .With(1) + .Apply(extractTupleItem(2)) + .With(0, "prevBigState") + .Seal() + .Done() + .Seal() + .Done() + .Seal() + .Seal() + .Build(); + }; + + return ctx.Builder(pos) + .Lambda() + .Param("input") + .Callable("Chain1Map") + .Arg(0, "input") + .Add(1, initLambda) + .Lambda(2) + .Param("row") + .Param("prevBigState") + .Callable("If") + .Apply(0, newPartitionLambda) + .With(0, "row") + .With(1, "prevBigState") + .Seal() + .Apply(1, initLambda) + .With(0, "row") + .Seal() + .List(2) + .Arg(0, "row") + .Callable(1, "If") + .Apply(0, newSessionOrUpdatedStateLambda(/* newSession = */ true)) + .With(0, "row") + .With(1, "prevBigState") + .Seal() + .Apply(1, sessionKeySelector) + .With(0, "row") + .With(1) + .Apply(newSessionOrUpdatedStateLambda(/* newSession = */ false)) + .With(0, "row") + .With(1, "prevBigState") + .Seal() + .Done() + .Seal() + .Apply(2, extractTupleItem(1)) + .With(0, "prevBigState") + .Seal() + .Seal() + .Apply(2, newSessionOrUpdatedStateLambda(/* newSession = */ false)) + .With(0, "row") + .With(1, "prevBigState") + .Seal() + .Apply(3, partitionKeySelector) + .With(0, "row") + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); +} + +TExprNode::TPtr AddSessionParamsMemberLambda(TPositionHandle pos, + TStringBuf sessionStartMemberName, TStringBuf sessionParamsMemberName, + const TExprNode::TPtr& partitionKeySelector, + const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, + const TExprNode::TPtr& sessionUpdate, TExprContext& ctx) +{ + YQL_ENSURE(sessionStartMemberName); + TExprNode::TPtr addLambda = ctx.Builder(pos) + .Lambda() + .Param("tupleOfItemAndSessionParams") + .Callable("AddMember") + .Callable(0, "Nth") + .Arg(0, "tupleOfItemAndSessionParams") + .Atom(1, "0", TNodeFlags::Default) + .Seal() + .Atom(1, sessionStartMemberName) + .Callable(2, "Nth") + .Arg(0, "tupleOfItemAndSessionParams") + .Atom(1, "1", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Build(); + + if (sessionParamsMemberName) { + addLambda = ctx.Builder(pos) + .Lambda() + .Param("tupleOfItemAndSessionParams") + .Callable("AddMember") + .Apply(0, addLambda) + .With(0, "tupleOfItemAndSessionParams") + .Seal() + .Atom(1, sessionParamsMemberName) + .Callable(2, "AsStruct") + .List(0) + .Atom(0, "start", TNodeFlags::Default) + .Callable(1, "Nth") + .Arg(0, "tupleOfItemAndSessionParams") + .Atom(1, "1", TNodeFlags::Default) + .Seal() + .Seal() + .List(1) + .Atom(0, "state", TNodeFlags::Default) + .Callable(1, "Nth") + .Arg(0, "tupleOfItemAndSessionParams") + .Atom(1, "2", TNodeFlags::Default) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Build(); + } + + return ctx.Builder(pos) + .Lambda() + .Param("input") + .Callable("OrderedMap") + .Apply(0, ZipWithSessionParamsLambda(pos, partitionKeySelector, sessionKeySelector, sessionInit, sessionUpdate, ctx)) + .With(0, "input") + .Seal() + .Add(1, addLambda) + .Seal() + .Seal() + .Build(); +} + +} diff --git a/ydb/library/yql/core/yql_opt_window.h b/ydb/library/yql/core/yql_opt_window.h index a043cdada4..9664840581 100644 --- a/ydb/library/yql/core/yql_opt_window.h +++ b/ydb/library/yql/core/yql_opt_window.h @@ -5,28 +5,28 @@ namespace NYql { TExprNode::TPtr ExpandCalcOverWindow(const TExprNode::TPtr& node, TExprContext& ctx); -TExprNodeList ExtractCalcsOverWindow(const TExprNode::TPtr& node, TExprContext& ctx); -TExprNode::TPtr RebuildCalcOverWindowGroup(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNodeList& calcs, TExprContext& ctx); - -struct TWindowFrameSettings { - TMaybe<i32> First; - TMaybe<i32> Last; - bool NeverEmpty = true; - bool IsCompact = false; -}; - -bool ParseWindowFrameSettings(const TExprNode& node, TWindowFrameSettings& settings, TExprContext& ctx); - -// Lambda(input: Stream/List<T>) -> Stream/List<Tuple<T, SessionKey, SessionState, ....>> -// input is assumed to be partitioned by partitionKeySelector -TExprNode::TPtr ZipWithSessionParamsLambda(TPositionHandle pos, const TExprNode::TPtr& partitionKeySelector, - const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, - const TExprNode::TPtr& sessionUpdate, TExprContext& ctx); - -// input should be List/Stream of structs + see above -TExprNode::TPtr AddSessionParamsMemberLambda(TPositionHandle pos, - TStringBuf sessionStartMemberName, TStringBuf sessionParamsMemberName, - const TExprNode::TPtr& partitionKeySelector, - const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, - const TExprNode::TPtr& sessionUpdate, TExprContext& ctx); +TExprNodeList ExtractCalcsOverWindow(const TExprNode::TPtr& node, TExprContext& ctx); +TExprNode::TPtr RebuildCalcOverWindowGroup(TPositionHandle pos, const TExprNode::TPtr& input, const TExprNodeList& calcs, TExprContext& ctx); + +struct TWindowFrameSettings { + TMaybe<i32> First; + TMaybe<i32> Last; + bool NeverEmpty = true; + bool IsCompact = false; +}; + +bool ParseWindowFrameSettings(const TExprNode& node, TWindowFrameSettings& settings, TExprContext& ctx); + +// Lambda(input: Stream/List<T>) -> Stream/List<Tuple<T, SessionKey, SessionState, ....>> +// input is assumed to be partitioned by partitionKeySelector +TExprNode::TPtr ZipWithSessionParamsLambda(TPositionHandle pos, const TExprNode::TPtr& partitionKeySelector, + const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, + const TExprNode::TPtr& sessionUpdate, TExprContext& ctx); + +// input should be List/Stream of structs + see above +TExprNode::TPtr AddSessionParamsMemberLambda(TPositionHandle pos, + TStringBuf sessionStartMemberName, TStringBuf sessionParamsMemberName, + const TExprNode::TPtr& partitionKeySelector, + const TExprNode::TPtr& sessionKeySelector, const TExprNode::TPtr& sessionInit, + const TExprNode::TPtr& sessionUpdate, TExprContext& ctx); } diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp index 2aae3815e0..9f50af3b63 100644 --- a/ydb/library/yql/core/yql_type_annotation.cpp +++ b/ydb/library/yql/core/yql_type_annotation.cpp @@ -1,8 +1,8 @@ #include "yql_type_annotation.h" - -#include "yql_expr_type_annotation.h" + +#include "yql_expr_type_annotation.h" #include "yql_library_compiler.h" -#include "yql_type_helpers.h" +#include "yql_type_helpers.h" #include <ydb/library/yql/sql/sql.h> #include <ydb/library/yql/sql/settings/translation_settings.h> @@ -10,7 +10,7 @@ #include <ydb/library/yql/utils/log/log.h> #include <util/stream/file.h> -#include <util/string/join.h> +#include <util/string/join.h> namespace NYql { @@ -50,93 +50,93 @@ bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) { return true; } -TString FormatColumnOrder(const TMaybe<TColumnOrder>& columnOrder) { - TStringStream ss; - if (columnOrder) { - ss << "[" << JoinSeq(", ", *columnOrder) << "]"; - } else { - ss << "default"; - } - return ss.Str(); -} - -ui64 AddColumnOrderHash(const TMaybe<TColumnOrder>& columnOrder, ui64 hash) { - if (!columnOrder) { - return hash; - } - - hash = CombineHashes(hash, NumericHash(columnOrder->size())); - for (auto& col : *columnOrder) { - hash = CombineHashes(hash, THash<TString>()(col)); - } - - return hash; -} - - -TMaybe<TColumnOrder> TTypeAnnotationContext::LookupColumnOrder(const TExprNode& node) const { - return ColumnOrderStorage->Lookup(node.UniqueId()); -} - -IGraphTransformer::TStatus TTypeAnnotationContext::SetColumnOrder(const TExprNode& node, - const TColumnOrder& columnOrder, TExprContext& ctx) -{ - if (!OrderedColumns) { - return IGraphTransformer::TStatus::Ok; - } - - YQL_ENSURE(node.GetTypeAnn()); - YQL_ENSURE(node.IsCallable()); - - if (auto existing = ColumnOrderStorage->Lookup(node.UniqueId())) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), - TStringBuilder() << "Column order " << FormatColumnOrder(existing) << " is already set for node " << node.Content())); - return IGraphTransformer::TStatus::Error; - } - - auto nodeType = node.GetTypeAnn(); - // allow Tuple(world, sequence-of-struct) - if (nodeType->GetKind() == ETypeAnnotationKind::Tuple) { - if (!EnsureTupleTypeSize(node, 2, ctx)) { - return IGraphTransformer::TStatus::Error; - } - - auto worldType = nodeType->Cast<TTupleExprType>()->GetItems()[0]; - if (worldType->GetKind() != ETypeAnnotationKind::World) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), - TStringBuilder() << "Expected world type as type of first tuple element, but got: " << *worldType)); - return IGraphTransformer::TStatus::Error; - } - - nodeType = nodeType->Cast<TTupleExprType>()->GetItems()[1]; - } - - TSet<TStringBuf> allColumns = GetColumnsOfStructOrSequenceOfStruct(*nodeType); - - for (auto& col : columnOrder) { - auto it = allColumns.find(col); - if (it == allColumns.end()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), - TStringBuilder() << "Unable to set column order " << FormatColumnOrder(columnOrder) << " for node " - << node.Content() << " with type: " << *node.GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - allColumns.erase(it); - } - - if (!allColumns.empty()) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), - TStringBuilder() << "Some columns are left unordered with column order " << FormatColumnOrder(columnOrder) << " for node " - << node.Content() << " with type: " << *node.GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - YQL_CLOG(DEBUG, Core) << "Setting column order " << FormatColumnOrder(columnOrder) << " for " << node.Content() << "#" << node.UniqueId(); - - ColumnOrderStorage->Set(node.UniqueId(), columnOrder); - return IGraphTransformer::TStatus::Ok; -} - +TString FormatColumnOrder(const TMaybe<TColumnOrder>& columnOrder) { + TStringStream ss; + if (columnOrder) { + ss << "[" << JoinSeq(", ", *columnOrder) << "]"; + } else { + ss << "default"; + } + return ss.Str(); +} + +ui64 AddColumnOrderHash(const TMaybe<TColumnOrder>& columnOrder, ui64 hash) { + if (!columnOrder) { + return hash; + } + + hash = CombineHashes(hash, NumericHash(columnOrder->size())); + for (auto& col : *columnOrder) { + hash = CombineHashes(hash, THash<TString>()(col)); + } + + return hash; +} + + +TMaybe<TColumnOrder> TTypeAnnotationContext::LookupColumnOrder(const TExprNode& node) const { + return ColumnOrderStorage->Lookup(node.UniqueId()); +} + +IGraphTransformer::TStatus TTypeAnnotationContext::SetColumnOrder(const TExprNode& node, + const TColumnOrder& columnOrder, TExprContext& ctx) +{ + if (!OrderedColumns) { + return IGraphTransformer::TStatus::Ok; + } + + YQL_ENSURE(node.GetTypeAnn()); + YQL_ENSURE(node.IsCallable()); + + if (auto existing = ColumnOrderStorage->Lookup(node.UniqueId())) { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), + TStringBuilder() << "Column order " << FormatColumnOrder(existing) << " is already set for node " << node.Content())); + return IGraphTransformer::TStatus::Error; + } + + auto nodeType = node.GetTypeAnn(); + // allow Tuple(world, sequence-of-struct) + if (nodeType->GetKind() == ETypeAnnotationKind::Tuple) { + if (!EnsureTupleTypeSize(node, 2, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + auto worldType = nodeType->Cast<TTupleExprType>()->GetItems()[0]; + if (worldType->GetKind() != ETypeAnnotationKind::World) { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), + TStringBuilder() << "Expected world type as type of first tuple element, but got: " << *worldType)); + return IGraphTransformer::TStatus::Error; + } + + nodeType = nodeType->Cast<TTupleExprType>()->GetItems()[1]; + } + + TSet<TStringBuf> allColumns = GetColumnsOfStructOrSequenceOfStruct(*nodeType); + + for (auto& col : columnOrder) { + auto it = allColumns.find(col); + if (it == allColumns.end()) { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), + TStringBuilder() << "Unable to set column order " << FormatColumnOrder(columnOrder) << " for node " + << node.Content() << " with type: " << *node.GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + allColumns.erase(it); + } + + if (!allColumns.empty()) { + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), + TStringBuilder() << "Some columns are left unordered with column order " << FormatColumnOrder(columnOrder) << " for node " + << node.Content() << " with type: " << *node.GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + YQL_CLOG(DEBUG, Core) << "Setting column order " << FormatColumnOrder(columnOrder) << " for " << node.Content() << "#" << node.UniqueId(); + + ColumnOrderStorage->Set(node.UniqueId(), columnOrder); + return IGraphTransformer::TStatus::Ok; +} + const TCredential* TTypeAnnotationContext::FindCredential(const TStringBuf& name) const { for (auto& x : Credentials) { auto data = x->FindPtr(name); @@ -226,7 +226,7 @@ bool TModuleResolver::AddFromUrl(const TStringBuf& file, const TStringBuf& url, bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) { if (!UserData) { - ctx.AddError(TIssue(TPosition(), "Loading libraries is prohibited")); + ctx.AddError(TIssue(TPosition(), "Loading libraries is prohibited")); return false; } @@ -348,7 +348,7 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu if (exports) { exports->clear(); - for (auto p : cohesion.Exports.Symbols()) { + for (auto p : cohesion.Exports.Symbols()) { exports->push_back(p.first); } } diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h index daac7175ea..1a1f1fb476 100644 --- a/ydb/library/yql/core/yql_type_annotation.h +++ b/ydb/library/yql/core/yql_type_annotation.h @@ -125,7 +125,7 @@ struct TCredential { }; using TCredentialTable = THashMap<TString, TCredential>; -using TCredentialTablePtr = std::shared_ptr<TCredentialTable>; +using TCredentialTablePtr = std::shared_ptr<TCredentialTable>; struct TUserCredentials { TString OauthToken; @@ -148,30 +148,30 @@ struct TYqlOperationOptions { TMaybe<NYT::TNode> ParametersYson; }; -using TColumnOrder = TVector<TString>; -TString FormatColumnOrder(const TMaybe<TColumnOrder>& columnOrder); -ui64 AddColumnOrderHash(const TMaybe<TColumnOrder>& columnOrder, ui64 hash); - -class TColumnOrderStorage: public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TColumnOrderStorage>; - TColumnOrderStorage() = default; - - TMaybe<TColumnOrder> Lookup(ui64 uniqueId) const { - auto it = Storage.find(uniqueId); - if (it == Storage.end()) { - return {}; - } - return it->second; - } - - void Set(ui64 uniqueId, const TColumnOrder& order) { - Storage[uniqueId] = order; - } -private: - THashMap<ui64, TColumnOrder> Storage; -}; - +using TColumnOrder = TVector<TString>; +TString FormatColumnOrder(const TMaybe<TColumnOrder>& columnOrder); +ui64 AddColumnOrderHash(const TMaybe<TColumnOrder>& columnOrder, ui64 hash); + +class TColumnOrderStorage: public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TColumnOrderStorage>; + TColumnOrderStorage() = default; + + TMaybe<TColumnOrder> Lookup(ui64 uniqueId) const { + auto it = Storage.find(uniqueId); + if (it == Storage.end()) { + return {}; + } + return it->second; + } + + void Set(ui64 uniqueId, const TColumnOrder& order) { + Storage[uniqueId] = order; + } +private: + THashMap<ui64, TColumnOrder> Storage; +}; + struct TTypeAnnotationContext: public TThrRefBase { TIntrusivePtr<ITimeProvider> TimeProvider; TIntrusivePtr<IRandomProvider> RandomProvider; @@ -203,12 +203,12 @@ struct TTypeAnnotationContext: public TThrRefBase { bool EvaluationInProgress = false; THashMap<ui64, const TTypeAnnotationNode*> ExpectedTypes; THashMap<ui64, std::vector<const TConstraintNode*>> ExpectedConstraints; - THashMap<ui64, TColumnOrder> ExpectedColumnOrders; + THashMap<ui64, TColumnOrder> ExpectedColumnOrders; THashSet<TString> DisableConstraintCheck; bool UdfSupportsYield = false; ui32 EvaluateForLimit = 500; ui32 EvaluateOrderByColumnLimit = 100; - bool PullUpFlatMapOverJoin = true; + bool PullUpFlatMapOverJoin = true; bool DeprecatedSQL = false; THashMap<std::tuple<TString, TString, const TTypeAnnotationNode*>, std::tuple<const TTypeAnnotationNode*, const TTypeAnnotationNode*, const TTypeAnnotationNode*>> @@ -222,13 +222,13 @@ struct TTypeAnnotationContext: public TThrRefBase { bool JsonQueryReturnsJsonDocument = false; ui32 FolderSubDirsLimit = 1000; - // compatibility with v0 or raw s-expression code - bool OrderedColumns = false; - TColumnOrderStorage::TPtr ColumnOrderStorage = new TColumnOrderStorage; - - TMaybe<TColumnOrder> LookupColumnOrder(const TExprNode& node) const; - IGraphTransformer::TStatus SetColumnOrder(const TExprNode& node, const TColumnOrder& columnOrder, TExprContext& ctx); - + // compatibility with v0 or raw s-expression code + bool OrderedColumns = false; + TColumnOrderStorage::TPtr ColumnOrderStorage = new TColumnOrderStorage; + + TMaybe<TColumnOrder> LookupColumnOrder(const TExprNode& node) const; + IGraphTransformer::TStatus SetColumnOrder(const TExprNode& node, const TColumnOrder& columnOrder, TExprContext& ctx); + // cached constants std::optional<ui64> CachedNow; std::tuple<std::optional<ui64>, std::optional<double>, std::optional<TGUID>> CachedRandom; diff --git a/ydb/library/yql/core/yql_type_helpers.cpp b/ydb/library/yql/core/yql_type_helpers.cpp index 7dc2fb98b3..9ae7cab56f 100644 --- a/ydb/library/yql/core/yql_type_helpers.cpp +++ b/ydb/library/yql/core/yql_type_helpers.cpp @@ -21,40 +21,40 @@ const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type) { return nullptr; } -TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type) { - const TTypeAnnotationNode* itemType = nullptr; - if (type.GetKind() != ETypeAnnotationKind::Struct) { - itemType = GetItemType(type); - YQL_ENSURE(itemType); - } else { - itemType = &type; - } - - TSet<TStringBuf> result; - for (auto& item : itemType->Cast<TStructExprType>()->GetItems()) { - result.insert(item->GetName()); - } - - return result; -} - -namespace { - -bool SilentGetSequenceItemType(TPosition pos, const TTypeAnnotationNode& inputType, bool allowMultiIO, - const TTypeAnnotationNode*& result, TIssue& error) -{ - result = nullptr; +TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type) { + const TTypeAnnotationNode* itemType = nullptr; + if (type.GetKind() != ETypeAnnotationKind::Struct) { + itemType = GetItemType(type); + YQL_ENSURE(itemType); + } else { + itemType = &type; + } + + TSet<TStringBuf> result; + for (auto& item : itemType->Cast<TStructExprType>()->GetItems()) { + result.insert(item->GetName()); + } + + return result; +} + +namespace { + +bool SilentGetSequenceItemType(TPosition pos, const TTypeAnnotationNode& inputType, bool allowMultiIO, + const TTypeAnnotationNode*& result, TIssue& error) +{ + result = nullptr; const TTypeAnnotationNode* itemType = GetItemType(inputType); if (!itemType) { error = TIssue(pos, TStringBuilder() << "Expected list, stream, flow or optional, but got: " << inputType); - return false; + return false; } if (allowMultiIO && itemType->GetKind() != ETypeAnnotationKind::Struct && itemType->GetKind() != ETypeAnnotationKind::Multi) { - if (itemType->GetKind() != ETypeAnnotationKind::Variant) { + if (itemType->GetKind() != ETypeAnnotationKind::Variant) { error = TIssue(pos, TStringBuilder() << "Expected Struct or Variant as row type, but got: " << *itemType); - return false; - } + return false; + } auto varType = itemType->Cast<TVariantExprType>()->GetUnderlyingType(); TTypeAnnotationNode::TListType varItemTypes; if (varType->GetKind() == ETypeAnnotationKind::Struct) { @@ -64,68 +64,68 @@ bool SilentGetSequenceItemType(TPosition pos, const TTypeAnnotationNode& inputTy } else { varItemTypes = varType->Cast<TTupleExprType>()->GetItems(); } - if (varItemTypes.size() < 2) { - error = TIssue(pos, TStringBuilder() << "Expected at least two items in Variant row type, but got: " << varItemTypes.size()); - return false; - } + if (varItemTypes.size() < 2) { + error = TIssue(pos, TStringBuilder() << "Expected at least two items in Variant row type, but got: " << varItemTypes.size()); + return false; + } for (auto varItemType: varItemTypes) { - if (varItemType->GetKind() != ETypeAnnotationKind::Struct) { - error = TIssue(pos, TStringBuilder() << "Expected Struct in Variant item type, but got: " << *varItemType); - return false; - } + if (varItemType->GetKind() != ETypeAnnotationKind::Struct) { + error = TIssue(pos, TStringBuilder() << "Expected Struct in Variant item type, but got: " << *varItemType); + return false; + } } - } else { + } else { if (itemType->GetKind() != ETypeAnnotationKind::Struct && itemType->GetKind() != ETypeAnnotationKind::Multi) { error = TIssue(pos, TStringBuilder() << "Expected Struct or Multi as row type, but got: " << *itemType); - return false; - } + return false; + } } - - result = itemType; - YQL_ENSURE(result); - return true; -} - -} - -const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO) { - const TTypeAnnotationNode* itemType = nullptr; - TIssue error; - if (!SilentGetSequenceItemType({}, *listNode.Ref().GetTypeAnn(), allowMultiIO, itemType, error)) { - // position is not used - YQL_ENSURE(false, "" << error.Message); + + result = itemType; + YQL_ENSURE(result); + return true; +} + +} + +const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO) { + const TTypeAnnotationNode* itemType = nullptr; + TIssue error; + if (!SilentGetSequenceItemType({}, *listNode.Ref().GetTypeAnn(), allowMultiIO, itemType, error)) { + // position is not used + YQL_ENSURE(false, "" << error.Message); } return itemType; } -const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx) { - return GetSequenceItemType(listNode.Pos(), listNode.Ref().GetTypeAnn(), allowMultiIO, ctx); +const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx) { + return GetSequenceItemType(listNode.Pos(), listNode.Ref().GetTypeAnn(), allowMultiIO, ctx); } -const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType, bool allowMultiIO, - TExprContext& ctx) -{ +const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType, bool allowMultiIO, + TExprContext& ctx) +{ const TTypeAnnotationNode* itemType = nullptr; - TIssue error; - if (!SilentGetSequenceItemType(ctx.GetPosition(pos), *inputType, allowMultiIO, itemType, error)) { - ctx.AddError(error); + TIssue error; + if (!SilentGetSequenceItemType(ctx.GetPosition(pos), *inputType, allowMultiIO, itemType, error)) { + ctx.AddError(error); return nullptr; } return itemType; } -bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx) { - itemType = GetSequenceItemType(list.Pos(), list.GetTypeAnn(), false, ctx); +bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx) { + itemType = GetSequenceItemType(list.Pos(), list.GetTypeAnn(), false, ctx); return itemType != nullptr; } -const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO) { - TIssue error; - const TTypeAnnotationNode* itemType = nullptr; - if (!SilentGetSequenceItemType({}, *list.GetTypeAnn(), allowMultiIO, itemType, error)) { - return nullptr; - } - return itemType; -} - +const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO) { + TIssue error; + const TTypeAnnotationNode* itemType = nullptr; + if (!SilentGetSequenceItemType({}, *list.GetTypeAnn(), allowMultiIO, itemType, error)) { + return nullptr; + } + return itemType; +} + } // namespace NYql diff --git a/ydb/library/yql/core/yql_type_helpers.h b/ydb/library/yql/core/yql_type_helpers.h index 9d4a5f9d05..593c77592b 100644 --- a/ydb/library/yql/core/yql_type_helpers.h +++ b/ydb/library/yql/core/yql_type_helpers.h @@ -3,18 +3,18 @@ #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> -#include <util/generic/set.h> +#include <util/generic/set.h> namespace NYql { const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type); -TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type); +TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type); -const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO); -const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx); -const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType, - bool allowMultiIO, TExprContext& ctx); -bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx); -const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO); +const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO); +const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx); +const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType, + bool allowMultiIO, TExprContext& ctx); +bool GetSequenceItemType(const TExprNode& list, const TTypeAnnotationNode*& itemType, TExprContext& ctx); +const TTypeAnnotationNode* SilentGetSequenceItemType(const TExprNode& list, bool allowMultiIO); } // namespace NYql diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp index 63ce0fdbc7..fed0f09f0f 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp @@ -127,16 +127,16 @@ TExprBase DqEnforceCompactPartition(TExprBase node, TExprList frames, TExprConte } TExprBase DqExpandWindowFunctions(TExprBase node, TExprContext& ctx, bool enforceCompact) { - if (node.Maybe<TCoCalcOverWindowBase>() || node.Maybe<TCoCalcOverWindowGroup>()) { + if (node.Maybe<TCoCalcOverWindowBase>() || node.Maybe<TCoCalcOverWindowGroup>()) { if (enforceCompact) { - auto calcs = ExtractCalcsOverWindow(node.Ptr(), ctx); - for (auto& c : calcs) { - TCoCalcOverWindowTuple win(c); + auto calcs = ExtractCalcsOverWindow(node.Ptr(), ctx); + for (auto& c : calcs) { + TCoCalcOverWindowTuple win(c); node = DqEnforceCompactPartition(node, win.Frames(), ctx); } } - - return TExprBase(ExpandCalcOverWindow(node.Ptr(), ctx)); + + return TExprBase(ExpandCalcOverWindow(node.Ptr(), ctx)); } else { return node; } diff --git a/ydb/library/yql/dq/runtime/dq_input_producer.cpp b/ydb/library/yql/dq/runtime/dq_input_producer.cpp index 41f2ba6d7d..3fe4830a08 100644 --- a/ydb/library/yql/dq/runtime/dq_input_producer.cpp +++ b/ydb/library/yql/dq/runtime/dq_input_producer.cpp @@ -73,7 +73,7 @@ public: for (auto& sortCol : SortCols) { TMaybe<EDataSlot> maybeDataSlot = FindDataSlot(sortCol.TypeId); YQL_ENSURE(maybeDataSlot, "Trying to compare columns with unknown type id: " << sortCol.TypeId); - YQL_ENSURE(IsTypeSupportedInMergeCn(*maybeDataSlot), "Column '" << sortCol.Name << + YQL_ENSURE(IsTypeSupportedInMergeCn(*maybeDataSlot), "Column '" << sortCol.Name << "' has unsupported type for Merge connection: " << *maybeDataSlot); SortColTypes[sortCol.Index] = *maybeDataSlot; } @@ -107,8 +107,8 @@ private: chosenIndex = i; } } - YQL_ENSURE(chosenIndex < InputsSize); - YQL_ENSURE(res); + YQL_ENSURE(chosenIndex < InputsSize); + YQL_ENSURE(res); ++CurrentItemIndexes[chosenIndex]; return *res; } @@ -118,7 +118,7 @@ private: for (auto sortCol = SortCols.begin(); sortCol != SortCols.end() && compRes == 0; ++sortCol) { auto lhsColValue = lhs.GetElement(sortCol->Index); auto rhsColValue = rhs.GetElement(sortCol->Index); - compRes = NKikimr::NMiniKQL::CompareValues(SortColTypes[sortCol->Index], + compRes = NKikimr::NMiniKQL::CompareValues(SortColTypes[sortCol->Index], sortCol->Ascending, /* isOptional */ true, lhsColValue, rhsColValue); } return compRes; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp index 1d879e0da3..06010bd4e5 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp @@ -131,7 +131,7 @@ private: for (const auto node : ArgNodes) { const auto arg = ExtractValueInst::Create(arguments, {i++}, "arg", block); const auto codegenArgNode = dynamic_cast<ICodegeneratorExternalNode*>(node); - MKQL_ENSURE(codegenArgNode, "Argument must be codegenerator node."); + MKQL_ENSURE(codegenArgNode, "Argument must be codegenerator node."); codegenArgNode->CreateSetValue(ctx, block, arg); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp index 41d33d8d21..48c931dac6 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp @@ -109,9 +109,9 @@ private: const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(ItemArg); const auto codegenKeyArg = dynamic_cast<ICodegeneratorExternalNode*>(KeyArg); - MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); - MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); - + MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); + MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); + const auto valueType = Type::getInt128Ty(context); const auto funcType = FunctionType::get(valueType, {PointerType::getUnqual(GetCompContextType(context))}, false); @@ -178,10 +178,10 @@ public: const auto codegenKeyArg = dynamic_cast<ICodegeneratorExternalNode*>(KeyArg); const auto codegenInput = dynamic_cast<ICodegeneratorExternalNode*>(Input); - MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); - MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); - MKQL_ENSURE(codegenInput, "Input arg must be codegenerator node."); - + MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); + MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input arg must be codegenerator node."); + codegenInput->SetValueGetter(GenerateHandler(ctx.Codegen)); auto& context = ctx.Codegen->GetContext(); @@ -404,7 +404,7 @@ private: case NUdf::EFetchStatus::Finish: return status; } while (!Chop->GetValue(Ctx).Get<bool>()); - [[fallthrough]]; + [[fallthrough]]; case EState::Chop: state = EState::Next; KeyArg->SetValue(Ctx, Key->GetValue(Ctx)); @@ -544,9 +544,9 @@ private: const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(ItemArg); const auto codegenKeyArg = dynamic_cast<ICodegeneratorExternalNode*>(KeyArg); - MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); - MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); - + MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); + MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); + const auto valueType = Type::getInt128Ty(context); const auto containerType = codegen->GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows ? static_cast<Type*>(PointerType::getUnqual(valueType)) : static_cast<Type*>(valueType); const auto contextType = GetCompContextType(context); @@ -637,10 +637,10 @@ private: const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(ItemArg); const auto codegenKeyArg = dynamic_cast<ICodegeneratorExternalNode*>(KeyArg); - MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); - MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); - MKQL_ENSURE(codegenInput, "Input arg must be codegenerator node."); - + MKQL_ENSURE(codegenItemArg, "Item arg must be codegenerator node."); + MKQL_ENSURE(codegenKeyArg, "Key arg must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input arg must be codegenerator node."); + const auto valueType = Type::getInt128Ty(context); const auto containerType = codegen->GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows ? static_cast<Type*>(PointerType::getUnqual(valueType)) : static_cast<Type*>(valueType); const auto contextType = GetCompContextType(context); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp index 1aed8e722b..4e13f63a14 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp @@ -177,7 +177,7 @@ public: } } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } NUdf::TUnboxedValuePod DoCalculate(NUdf::TUnboxedValue& state, NUdf::TUnboxedValue& current, TComputationContext& ctx) const { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp index e3352c5d21..57599acb64 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp @@ -53,20 +53,20 @@ #include "mkql_map_join.h" #include "mkql_multihopping.h" #include "mkql_multimap.h" -#include "mkql_next_value.h" +#include "mkql_next_value.h" #include "mkql_now.h" #include "mkql_null.h" #include "mkql_pickle.h" #include "mkql_prepend.h" #include "mkql_queue.h" #include "mkql_random.h" -#include "mkql_range.h" +#include "mkql_range.h" #include "mkql_reduce.h" #include "mkql_removemember.h" #include "mkql_replicate.h" #include "mkql_reverse.h" -#include "mkql_round.h" -#include "mkql_seq.h" +#include "mkql_round.h" +#include "mkql_seq.h" #include "mkql_size.h" #include "mkql_skip.h" #include "mkql_sort.h" @@ -244,8 +244,8 @@ struct TCallableComputationNodeBuilderFuncMapFiller { {"QueuePush", &WrapQueuePush}, {"QueuePop", &WrapQueuePop}, {"QueuePeek", &WrapQueuePeek}, - {"QueueRange", &WrapQueueRange}, - {"Seq", &WrapSeq}, + {"QueueRange", &WrapQueueRange}, + {"Seq", &WrapSeq}, {"PreserveStream", &WrapPreserveStream}, {"FromYsonSimpleType", &WrapFromYsonSimpleType}, {"TryWeakMemberFromDict", &WrapTryWeakMemberFromDict}, @@ -292,14 +292,14 @@ struct TCallableComputationNodeBuilderFuncMapFiller { {"WideChopper", &WrapWideChopper}, {"WideFlowArg", &WrapWideFlowArg}, {"Source", &WrapSource}, - {"RangeCreate", &WrapRangeCreate}, - {"RangeUnion", &WrapRangeUnion}, - {"RangeIntersect", &WrapRangeIntersect}, - {"RangeMultiply", &WrapRangeMultiply}, - {"RangeFinalize", &WrapRangeFinalize}, - {"RoundUp", &WrapRound}, - {"RoundDown", &WrapRound}, - {"NextValue", &WrapNextValue}, + {"RangeCreate", &WrapRangeCreate}, + {"RangeUnion", &WrapRangeUnion}, + {"RangeIntersect", &WrapRangeIntersect}, + {"RangeMultiply", &WrapRangeMultiply}, + {"RangeFinalize", &WrapRangeFinalize}, + {"RoundUp", &WrapRound}, + {"RoundDown", &WrapRound}, + {"NextValue", &WrapNextValue}, }; }; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp index dcece8eda4..a9d49e109d 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp @@ -7,8 +7,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class TFlowFlatMapFlowWrapper : public TStatefulFlowCodegeneratorNode<TFlowFlatMapFlowWrapper> { @@ -45,7 +45,7 @@ public: #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { const auto codegenInput = dynamic_cast<ICodegeneratorExternalNode*>(Input); - MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); auto& context = ctx.Codegen->GetContext(); @@ -138,7 +138,7 @@ public: #ifndef MKQL_DISABLE_CODEGEN TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { const auto codegenInput = dynamic_cast<ICodegeneratorExternalNode*>(Input); - MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); auto& context = ctx.Codegen->GetContext(); @@ -236,7 +236,7 @@ public: #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { const auto codegenInput = dynamic_cast<ICodegeneratorExternalNode*>(Input); - MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); auto& context = ctx.Codegen->GetContext(); @@ -335,7 +335,7 @@ public: #ifndef MKQL_DISABLE_CODEGEN TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { const auto codegenInput = dynamic_cast<ICodegeneratorExternalNode*>(Input); - MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); + MKQL_ENSURE(codegenInput, "Input must be codegenerator node."); auto& context = ctx.Codegen->GetContext(); @@ -480,7 +480,7 @@ public: for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); BranchInst::Create(work, block); @@ -781,7 +781,7 @@ public: } else { for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); value = GetNodeValue(NewItem, ctx, block); } @@ -1402,8 +1402,8 @@ public: this->Item->SetValue(ctx, std::move(item)); *it = this->NewItem->GetValue(ctx); if (IsMultiRowPerItem || *it) { - auto value = it->GetOptionalValueIf<!IsMultiRowPerItem && ResultContainerOpt>(); - *it++ = value; + auto value = it->GetOptionalValueIf<!IsMultiRowPerItem && ResultContainerOpt>(); + *it++ = value; } }); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp index 4b2167e3c7..7090aff8cd 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp @@ -125,7 +125,7 @@ public: block = good; - if (IsOptional || !IsStrict) { + if (IsOptional || !IsStrict) { phi->addIncoming(SetterForInt128(decimal, block), block); BranchInst::Create(last, block); @@ -228,7 +228,7 @@ public: if (Data->IsTemporaryValue()) ValueCleanup(Data->GetRepresentation(), value, ctx, block); - if (IsOptional) { + if (IsOptional) { phi->addIncoming(data, block); } @@ -241,15 +241,15 @@ public: const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(FunctionType::get(Type::getVoidTy(context), {valType, slotType}, false)), "thrower", block); CallInst::Create(doFuncPtr, { value, slot }, "", block); new UnreachableInst(context, block); - } else if (IsOptional) { + } else if (IsOptional) { BranchInst::Create(last, block); } - if (IsOptional || IsStrict) { + if (IsOptional || IsStrict) { block = last; } - return IsOptional ? phi : data; + return IsOptional ? phi : data; } #endif private: diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp index d8aaab9c55..27d1e000ac 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp @@ -213,7 +213,7 @@ public: std::vector<TBucket> Buckets; // circular buffer std::deque<NUdf::TUnboxedValue> Ready; // buffer for fetching results - ui64 HopIndex = 0; + ui64 HopIndex = 0; bool Started = false; bool Finished = false; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp index 554637eda4..e017c7ea88 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp @@ -36,7 +36,7 @@ public: auto& context = ctx.Codegen->GetContext(); const auto codegenItem = dynamic_cast<ICodegeneratorExternalNode*>(Item); - MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); + MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); const auto previous = codegenItem->CreateGetValue(ctx, block); const auto fast = BasicBlock::Create(context, "fast", ctx.Func); @@ -118,7 +118,7 @@ public: auto& context = ctx.Codegen->GetContext(); const auto codegenItem = dynamic_cast<ICodegeneratorExternalNode*>(Item); - MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); + MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); const auto previous = codegenItem->CreateGetValue(ctx, block); const auto fast = BasicBlock::Create(context, "fast", ctx.Func); @@ -200,7 +200,7 @@ public: auto& context = ctx.Codegen->GetContext(); const auto codegenItem = dynamic_cast<ICodegeneratorExternalNode*>(Item); - MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); + MKQL_ENSURE(codegenItem, "Item must be codegenerator node."); const auto previous = codegenItem->CreateGetValue(ctx, block); const auto init = BasicBlock::Create(context, "init", ctx.Func); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp index a0ddcbc2ed..d2182bbc23 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp @@ -60,7 +60,7 @@ public: , IsSealed(false) #endif , Index(ui64(-1)) - , SingleShot(singleShot) + , SingleShot(singleShot) {} TSpillList(TSpillList&& rhs) = delete; @@ -112,11 +112,11 @@ public: #ifndef NDEBUG Y_VERIFY_DEBUG(!IsSealed); #endif - if (SingleShot && Count > 0) { - MKQL_ENSURE(Count == 1, "Counter inconsistent"); - return; - } - + if (SingleShot && Count > 0) { + MKQL_ENSURE(Count == 1, "Counter inconsistent"); + return; + } + if (FileState) { Write(std::move(value)); } else { @@ -175,12 +175,12 @@ public: return std::move(LiveValue); } - auto value = LiveFlow->GetValue(ctx); - while (SingleShot && !value.IsSpecial()) { - // skip all remaining values - value = LiveFlow->GetValue(ctx); - } - + auto value = LiveFlow->GetValue(ctx); + while (SingleShot && !value.IsSpecial()) { + // skip all remaining values + value = LiveFlow->GetValue(ctx); + } + if (!value.IsSpecial()) { ++Index; } @@ -247,7 +247,7 @@ public: } void Rewind() { - Y_VERIFY_DEBUG(!IsLive()); + Y_VERIFY_DEBUG(!IsLive()); #ifndef NDEBUG Y_VERIFY_DEBUG(IsSealed); #endif @@ -309,7 +309,7 @@ private: bool IsSealed; #endif ui64 Index; - const bool SingleShot; + const bool SingleShot; struct TFileState { TFileState() : File(TTempFileHandle::InCurrentDir()) @@ -324,7 +324,7 @@ private: }; std::unique_ptr<TFileState> FileState; - IComputationNode* LiveFlow = nullptr; + IComputationNode* LiveFlow = nullptr; TLiveFetcher Fetcher; NUdf::TUnboxedValue LiveValue; NUdf::TUnboxedValue* LiveValues = nullptr; @@ -344,8 +344,8 @@ public: TValue(TMemoryUsageInfo* memInfo, TComputationContext& ctx, const TSelf* self) : TBase(memInfo) , Self(self) - , List1(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinLeft(Self->AnyJoinSettings)) - , List2(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinRight(Self->AnyJoinSettings)) + , List1(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinLeft(Self->AnyJoinSettings)) + , List2(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinRight(Self->AnyJoinSettings)) { Init(); } @@ -683,7 +683,7 @@ public: TCommonJoinCoreWrapper(TComputationMutables& mutables, IComputationNode* flow, const TType* inputStructType, ui32 inputWidth, ui32 tableIndexPos, std::vector<ui32>&& leftInputColumns, std::vector<ui32>&& rightInputColumns, std::vector<ui32>&& requiredColumns, std::vector<ui32>&& leftOutputColumns, std::vector<ui32>&& rightOutputColumns, ui64 memLimit, - std::optional<ui32> sortedTableOrder, std::vector<ui32>&& keyColumns, EAnyJoinSettings anyJoinSettings) + std::optional<ui32> sortedTableOrder, std::vector<ui32>&& keyColumns, EAnyJoinSettings anyJoinSettings) : TBaseComputation(mutables, flow, EValueRepresentation::Boxed, EValueRepresentation::Any) , Flow(flow) , InputStructType(inputStructType) @@ -700,7 +700,7 @@ public: , IsRequiredColumn(FillRequiredStructColumn(inputWidth, RequiredColumns)) , ResStruct(mutables) , ResStreamIndex(mutables.CurValueIndex++) - , AnyJoinSettings(anyJoinSettings) + , AnyJoinSettings(anyJoinSettings) { } @@ -733,7 +733,7 @@ private: const TContainerCacheOnContext ResStruct; const ui32 ResStreamIndex; - const EAnyJoinSettings AnyJoinSettings; + const EAnyJoinSettings AnyJoinSettings; }; template <EJoinKind Kind, bool TTrackRss> @@ -1364,7 +1364,7 @@ namespace NStream { class TSpillList { public: - TSpillList(TValuePacker& itemPacker, bool singleShot) + TSpillList(TValuePacker& itemPacker, bool singleShot) : ItemPacker(itemPacker) , Ctx(nullptr) , Count(0) @@ -1372,7 +1372,7 @@ public: , IsSealed(false) #endif , Index(ui64(-1)) - , SingleShot(singleShot) + , SingleShot(singleShot) {} TSpillList(TSpillList&& rhs) = delete; @@ -1422,11 +1422,11 @@ public: #ifndef NDEBUG Y_VERIFY_DEBUG(!IsSealed); #endif - if (SingleShot && Count > 0) { - MKQL_ENSURE(Count == 1, "Counter inconsistent"); - return; - } - + if (SingleShot && Count > 0) { + MKQL_ENSURE(Count == 1, "Counter inconsistent"); + return; + } + if (FileState) { Write(std::move(value)); } else { @@ -1480,23 +1480,23 @@ public: Y_VERIFY_DEBUG(IsSealed); #endif if (IsLive()) { - auto status = NUdf::EFetchStatus::Ok; + auto status = NUdf::EFetchStatus::Ok; NUdf::TUnboxedValue value; if ((Index + 1) == 0) { value = std::move(LiveValue); } else { - status = LiveStream.Fetch(value); - while (SingleShot && status == NUdf::EFetchStatus::Ok) { - // skip all remaining values - status = LiveStream.Fetch(value); + status = LiveStream.Fetch(value); + while (SingleShot && status == NUdf::EFetchStatus::Ok) { + // skip all remaining values + status = LiveStream.Fetch(value); } } - if (status == NUdf::EFetchStatus::Ok) { - result = std::move(value); - ++Index; - } - return status; + if (status == NUdf::EFetchStatus::Ok) { + result = std::move(value); + ++Index; + } + return status; } if ((Index + 1) == Count) { @@ -1518,7 +1518,7 @@ public: } void Rewind() { - Y_VERIFY_DEBUG(!IsLive()); + Y_VERIFY_DEBUG(!IsLive()); #ifndef NDEBUG Y_VERIFY_DEBUG(IsSealed); #endif @@ -1579,7 +1579,7 @@ private: bool IsSealed; #endif ui64 Index; - const bool SingleShot; + const bool SingleShot; struct TFileState { TFileState() : File(TTempFileHandle::InCurrentDir()) @@ -1614,8 +1614,8 @@ public: , Stream(std::move(stream)) , Ctx(ctx) , Self(self) - , List1(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinLeft(Self->AnyJoinSettings)) - , List2(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinRight(Self->AnyJoinSettings)) + , List1(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinLeft(Self->AnyJoinSettings)) + , List2(Self->Packer.RefMutableObject(ctx, false, Self->InputStructType), IsAnyJoinRight(Self->AnyJoinSettings)) { Init(); } @@ -1978,7 +1978,7 @@ public: TCommonJoinCoreWrapper(TComputationMutables& mutables, IComputationNode* stream, const TType* inputStructType, ui32 inputWidth, ui32 tableIndexPos, std::vector<ui32>&& leftInputColumns, std::vector<ui32>&& rightInputColumns, std::vector<ui32>&& requiredColumns, std::vector<ui32>&& leftOutputColumns, std::vector<ui32>&& rightOutputColumns, ui64 memLimit, - std::optional<ui32> sortedTableOrder, std::vector<ui32>&& keyColumns, EAnyJoinSettings anyJoinSettings) + std::optional<ui32> sortedTableOrder, std::vector<ui32>&& keyColumns, EAnyJoinSettings anyJoinSettings) : TBaseComputation(mutables) , Stream(stream) , InputStructType(inputStructType) @@ -1995,7 +1995,7 @@ public: , IsRequiredColumn(FillRequiredStructColumn(inputWidth, RequiredColumns)) , ResStruct(mutables) , ResStreamIndex(mutables.CurValueIndex++) - , AnyJoinSettings(anyJoinSettings) + , AnyJoinSettings(anyJoinSettings) { } @@ -2031,7 +2031,7 @@ private: const TContainerCacheOnContext ResStruct; const ui32 ResStreamIndex; - const EAnyJoinSettings AnyJoinSettings; + const EAnyJoinSettings AnyJoinSettings; }; } @@ -2143,8 +2143,8 @@ IComputationNode* WrapCommonJoinCore(TCallable& callable, const TComputationNode MKQL_ENSURE(*sortedTableOrder < 2, "Bad sorted table order"); } - const EAnyJoinSettings anyJoinSettings = GetAnyJoinSettings(AS_VALUE(TDataLiteral, callable.GetInput(10))->AsValue().Get<ui32>()); - + const EAnyJoinSettings anyJoinSettings = GetAnyJoinSettings(AS_VALUE(TDataLiteral, callable.GetInput(10))->AsValue().Get<ui32>()); + const auto tableIndexPos = 12U == callable.GetInputsCount() ? AS_VALUE(TDataLiteral, callable.GetInput(11U))->AsValue().Get<ui32>(): AS_TYPE(TStructType, inputRowType)->GetMemberIndex("_yql_table_index"); @@ -2180,11 +2180,11 @@ IComputationNode* WrapCommonJoinCore(TCallable& callable, const TComputationNode if (trackRss) \ return new NStream::TCommonJoinCoreWrapper<EJoinKind::KIND, true>(ctx.Mutables, flow, inputRowType, inputRepresentations.size(), tableIndexPos, \ std::move(leftInputColumns), std::move(rightInputColumns), std::move(requiredColumns), \ - std::move(leftOutputColumns), std::move(rightOutputColumns), memLimit, sortedTableOrder, std::move(keyColumns), anyJoinSettings); \ + std::move(leftOutputColumns), std::move(rightOutputColumns), memLimit, sortedTableOrder, std::move(keyColumns), anyJoinSettings); \ else \ return new NStream::TCommonJoinCoreWrapper<EJoinKind::KIND, false>(ctx.Mutables, flow, inputRowType, inputRepresentations.size(), tableIndexPos, \ std::move(leftInputColumns), std::move(rightInputColumns), std::move(requiredColumns), \ - std::move(leftOutputColumns), std::move(rightOutputColumns), memLimit, sortedTableOrder, std::move(keyColumns), anyJoinSettings); \ + std::move(leftOutputColumns), std::move(rightOutputColumns), memLimit, sortedTableOrder, std::move(keyColumns), anyJoinSettings); \ } switch (kind) { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp index cdc380fc9f..5eeac1ea86 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp @@ -8,45 +8,45 @@ namespace NMiniKQL { namespace { -template<typename T> -ui64 ShiftByMaxNegative(T value) { - if (std::is_signed<T>()) { - if (value < 0) { - return ui64(value + std::numeric_limits<T>::max() + T(1)); - } - return ui64(value) + ui64(std::numeric_limits<T>::max() + 1ul); - } - return ui64(value); -} - -ui64 GetElementsCount(ui64 start, ui64 end, ui64 step) { - if (step == 0 || start >= end) { - return 0; - } - - ui64 diff = end - start; - ui64 div = diff / step; - ui64 rem = diff % step; - - return rem ? (div + 1) : div; -} - +template<typename T> +ui64 ShiftByMaxNegative(T value) { + if (std::is_signed<T>()) { + if (value < 0) { + return ui64(value + std::numeric_limits<T>::max() + T(1)); + } + return ui64(value) + ui64(std::numeric_limits<T>::max() + 1ul); + } + return ui64(value); +} + +ui64 GetElementsCount(ui64 start, ui64 end, ui64 step) { + if (step == 0 || start >= end) { + return 0; + } + + ui64 diff = end - start; + ui64 div = diff / step; + ui64 rem = diff % step; + + return rem ? (div + 1) : div; +} + template<typename T, typename TStep> ui64 GetElementsCount(T start, T end, TStep step) { - ui64 newStart = ShiftByMaxNegative(start); - ui64 newEnd = ShiftByMaxNegative(end); - ui64 newStep; - - if (step < 0) { + ui64 newStart = ShiftByMaxNegative(start); + ui64 newEnd = ShiftByMaxNegative(end); + ui64 newStep; + + if (step < 0) { newStep = (step == std::numeric_limits<TStep>::min()) ? (ui64(std::numeric_limits<TStep>::max()) + 1ul) : ui64(TStep(0) - step); - std::swap(newStart, newEnd); - } else { - newStep = ui64(step); - } - - return GetElementsCount(newStart, newEnd, newStep); -} - + std::swap(newStart, newEnd); + } else { + newStep = ui64(step); + } + + return GetElementsCount(newStart, newEnd, newStep); +} + template <typename T, typename TStep = std::make_signed_t<T>, std::conditional_t<std::is_floating_point_v<TStep>, i8, TStep> TConstFactor = 1, ui64 TConstLimit = std::numeric_limits<ui64>::max(), bool TzDate = false> class TListFromRangeWrapper : public TMutableCodegeneratorNode<TListFromRangeWrapper<T, TStep, TConstFactor, TConstLimit, TzDate>> { private: @@ -62,35 +62,35 @@ private: public: TIterator(TMemoryUsageInfo* memInfo, T start, T end, TStep step) : TComputationValue<TIterator>(memInfo) - , Current(start) + , Current(start) , Step(step) , Count(GetElementsCount<T, TStep>(start, end, step)) {} protected: bool Skip() final { - if (!Count) { - return false; + if (!Count) { + return false; } - Current += Step; - --Count; - return true; + Current += Step; + --Count; + return true; } bool Next(NUdf::TUnboxedValue& value) override { - if (!Count) { + if (!Count) { return false; } value = NUdf::TUnboxedValuePod(Current); - Current += Step; - --Count; + Current += Step; + --Count; return true; } T Current; const TStep Step; - ui64 Count; + ui64 Count; }; template <bool Asc> @@ -150,10 +150,10 @@ private: } ui64 GetListLength() const final { - if (std::is_integral<T>()) { + if (std::is_integral<T>()) { return GetElementsCount<T, TStep>(Start, End, Step); - } - + } + if (Step > T(0) && Start < End) { ui64 len = 0ULL; for (T i = 0; i * Step < End - Start; i += T(1)) { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp index 168716246d..613958f74b 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp @@ -169,14 +169,14 @@ protected: if constexpr (IsTuple) { NUdf::TUnboxedValue* items = nullptr; const auto keys = KeyTuple.NewArray(ctx, LeftKeyColumns.size(), items); - if (!LeftKeyColumns.empty()) { - Y_VERIFY(items); - for (auto i = 0U; i < LeftKeyColumns.size(); ++i) { - const auto value = Fields[LeftKeyColumns[i]]; - const auto converter = LeftKeyConverters[i].Function; - if (!(*items++ = converter ? converter(value) : *value)) - return NUdf::TUnboxedValuePod(); - } + if (!LeftKeyColumns.empty()) { + Y_VERIFY(items); + for (auto i = 0U; i < LeftKeyColumns.size(); ++i) { + const auto value = Fields[LeftKeyColumns[i]]; + const auto converter = LeftKeyConverters[i].Function; + if (!(*items++ = converter ? converter(value) : *value)) + return NUdf::TUnboxedValuePod(); + } } return keys; @@ -602,10 +602,10 @@ protected: {} static void FillStruct(const NUdf::TUnboxedValue& structObj, NUdf::TUnboxedValue* items, const std::vector<ui32>& renames) { - if (renames.empty()) { - return; - } - Y_VERIFY(items); + if (renames.empty()) { + return; + } + Y_VERIFY(items); if (const auto ptr = structObj.GetElements()) { for (auto i = 0U; i < renames.size();) { const auto prevIndex = renames[i++]; @@ -633,15 +633,15 @@ protected: if (IsTuple) { NUdf::TUnboxedValue* items = nullptr; const auto keys = KeyTuple.NewArray(ctx, LeftKeyColumns.size(), items); - if (!LeftKeyColumns.empty()) { - Y_VERIFY(items); - const auto ptr = structObj.GetElements(); - for (auto i = 0U; i < LeftKeyColumns.size(); ++i) { - auto value = ptr ? ptr[LeftKeyColumns[i]] : structObj.GetElement(LeftKeyColumns[i]); - const auto converter = LeftKeyConverters[i].Function; - if (!(*items++ = converter ? NUdf::TUnboxedValue(converter(&value)) : std::move(value))) - return NUdf::TUnboxedValuePod(); - } + if (!LeftKeyColumns.empty()) { + Y_VERIFY(items); + const auto ptr = structObj.GetElements(); + for (auto i = 0U; i < LeftKeyColumns.size(); ++i) { + auto value = ptr ? ptr[LeftKeyColumns[i]] : structObj.GetElement(LeftKeyColumns[i]); + const auto converter = LeftKeyConverters[i].Function; + if (!(*items++ = converter ? NUdf::TUnboxedValue(converter(&value)) : std::move(value))) + return NUdf::TUnboxedValuePod(); + } } return keys; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp index f213d48b5a..e9e93aca67 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp @@ -9,8 +9,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class TFlowMultiMapWrapper : public TStatefulFlowCodegeneratorNode<TFlowMultiMapWrapper> { @@ -485,7 +485,7 @@ public: Value* head = nullptr; for (auto i = 0U; i < Items.size(); ++i) { if (Items[i]->GetDependencesCount() > 0U || PasstroughtMap[i]) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, NewItems.front() == Items[i] ? (head = getres.second[i](ctx, block)) : getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, NewItems.front() == Items[i] ? (head = getres.second[i](ctx, block)) : getres.second[i](ctx, block)); } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp index f6f07fc5ef..43320a3af3 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp @@ -1,78 +1,78 @@ -#include "mkql_next_value.h" +#include "mkql_next_value.h" #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/computation/presort.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <ydb/library/yql/public/udf/udf_data_type.h> #include <ydb/library/yql/utils/utf8.h> - -#include <algorithm> - -namespace NKikimr { -namespace NMiniKQL { - -using namespace NYql::NUdf; - -namespace { - -class TNextValueStringWrapper : public TMutableComputationNode<TNextValueStringWrapper> { - using TSelf = TNextValueStringWrapper; - using TBase = TMutableComputationNode<TSelf>; - typedef TBase TBaseComputation; -public: - TNextValueStringWrapper(TComputationMutables& mutables, IComputationNode* source, EDataSlot slot) - : TBaseComputation(mutables) - , Source(source) - , Slot(slot) - { - } - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - TUnboxedValue input = Source->GetValue(ctx); - const auto& inputStr = input.AsStringRef(); - auto output = (Slot == EDataSlot::Utf8) ? NYql::NextValidUtf8(inputStr) : NYql::NextLexicographicString(inputStr); - if (!output) { - return {}; - } - return MakeString(*output); - } - -private: - void RegisterDependencies() const final { - this->DependsOn(Source); - } - - IComputationNode* const Source; - const EDataSlot Slot; -}; - -} // namespace - -IComputationNode* WrapNextValue(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); - - auto type = callable.GetInput(0).GetStaticType(); - MKQL_ENSURE(type->IsData(), "Expecting Data as argument"); - - auto returnType = callable.GetType()->GetReturnType(); - MKQL_ENSURE(returnType->IsOptional(), "Expecting Optional as return type"); - - auto targetType = static_cast<TOptionalType*>(returnType)->GetItemType(); - MKQL_ENSURE(targetType->IsData(), "Expecting Optional of Data as return type"); - - auto from = GetDataSlot(static_cast<TDataType*>(type)->GetSchemeType()); - auto to = GetDataSlot(static_cast<TDataType*>(targetType)->GetSchemeType()); - - MKQL_ENSURE(from == to, "Input/output should have the same Data slot"); - - MKQL_ENSURE(from == EDataSlot::String || from == EDataSlot::Utf8, "Only String or Utf8 is supported"); - - auto source = LocateNode(ctx.NodeLocator, callable, 0); - return new TNextValueStringWrapper(ctx.Mutables, source, from); -} - -} -} + +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +using namespace NYql::NUdf; + +namespace { + +class TNextValueStringWrapper : public TMutableComputationNode<TNextValueStringWrapper> { + using TSelf = TNextValueStringWrapper; + using TBase = TMutableComputationNode<TSelf>; + typedef TBase TBaseComputation; +public: + TNextValueStringWrapper(TComputationMutables& mutables, IComputationNode* source, EDataSlot slot) + : TBaseComputation(mutables) + , Source(source) + , Slot(slot) + { + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + TUnboxedValue input = Source->GetValue(ctx); + const auto& inputStr = input.AsStringRef(); + auto output = (Slot == EDataSlot::Utf8) ? NYql::NextValidUtf8(inputStr) : NYql::NextLexicographicString(inputStr); + if (!output) { + return {}; + } + return MakeString(*output); + } + +private: + void RegisterDependencies() const final { + this->DependsOn(Source); + } + + IComputationNode* const Source; + const EDataSlot Slot; +}; + +} // namespace + +IComputationNode* WrapNextValue(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); + + auto type = callable.GetInput(0).GetStaticType(); + MKQL_ENSURE(type->IsData(), "Expecting Data as argument"); + + auto returnType = callable.GetType()->GetReturnType(); + MKQL_ENSURE(returnType->IsOptional(), "Expecting Optional as return type"); + + auto targetType = static_cast<TOptionalType*>(returnType)->GetItemType(); + MKQL_ENSURE(targetType->IsData(), "Expecting Optional of Data as return type"); + + auto from = GetDataSlot(static_cast<TDataType*>(type)->GetSchemeType()); + auto to = GetDataSlot(static_cast<TDataType*>(targetType)->GetSchemeType()); + + MKQL_ENSURE(from == to, "Input/output should have the same Data slot"); + + MKQL_ENSURE(from == EDataSlot::String || from == EDataSlot::Utf8, "Only String or Utf8 is supported"); + + auto source = LocateNode(ctx.NodeLocator, callable, 0); + return new TNextValueStringWrapper(ctx.Mutables, source, from); +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_next_value.h b/ydb/library/yql/minikql/comp_nodes/mkql_next_value.h index 8ea184da81..ca1dd0821c 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_next_value.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_next_value.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapNextValue(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapNextValue(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp index d7d85cee2b..49cc897f0e 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp @@ -13,25 +13,25 @@ namespace { class TQueueResource : public TComputationValue<TQueueResource> { public: - TQueueResource(TMemoryUsageInfo* memInfo, const TStringBuf& tag, TMaybe<ui64> capacity, ui64 initSize) + TQueueResource(TMemoryUsageInfo* memInfo, const TStringBuf& tag, TMaybe<ui64> capacity, ui64 initSize) : TComputationValue(memInfo) , ResourceTag(tag) , Buffer(capacity, TUnboxedValue(), initSize) - , BufferBytes(CurrentMemUsage()) + , BufferBytes(CurrentMemUsage()) { - MKQL_MEM_TAKE(memInfo, &Buffer, BufferBytes); + MKQL_MEM_TAKE(memInfo, &Buffer, BufferBytes); } ~TQueueResource() { - Y_VERIFY_DEBUG(BufferBytes == CurrentMemUsage()); - MKQL_MEM_RETURN(GetMemInfo(), &Buffer, CurrentMemUsage()); - Buffer.Clear(); + Y_VERIFY_DEBUG(BufferBytes == CurrentMemUsage()); + MKQL_MEM_RETURN(GetMemInfo(), &Buffer, CurrentMemUsage()); + Buffer.Clear(); } - void UpdateBufferStats() { - MKQL_MEM_RETURN(GetMemInfo(), &Buffer, BufferBytes); - BufferBytes = CurrentMemUsage(); - MKQL_MEM_TAKE(GetMemInfo(), &Buffer, BufferBytes); + void UpdateBufferStats() { + MKQL_MEM_RETURN(GetMemInfo(), &Buffer, BufferBytes); + BufferBytes = CurrentMemUsage(); + MKQL_MEM_TAKE(GetMemInfo(), &Buffer, BufferBytes); } TSafeCircularBuffer<TUnboxedValue>& GetBuffer() { @@ -47,13 +47,13 @@ private: return this; } - size_t CurrentMemUsage() const { - return Buffer.Size() * sizeof(TUnboxedValue); - } - + size_t CurrentMemUsage() const { + return Buffer.Size() * sizeof(TUnboxedValue); + } + const TStringBuf ResourceTag; TSafeCircularBuffer<TUnboxedValue> Buffer; - size_t BufferBytes; + size_t BufferBytes; }; class TQueueResource; @@ -61,7 +61,7 @@ class TQueueResourceUser { public: TQueueResourceUser(TStringBuf&& tag, IComputationNode* resource); TSafeCircularBuffer<NUdf::TUnboxedValue>& CheckAndGetBuffer(const NUdf::TUnboxedValuePod& resource) const; - void UpdateBufferStats(const NUdf::TUnboxedValuePod& resource) const; + void UpdateBufferStats(const NUdf::TUnboxedValuePod& resource) const; protected: const TStringBuf Tag; @@ -79,8 +79,8 @@ TSafeCircularBuffer<TUnboxedValue>& TQueueResourceUser::CheckAndGetBuffer(const return GetResource(resource).GetBuffer(); } -void TQueueResourceUser::UpdateBufferStats(const TUnboxedValuePod& resource) const { - GetResource(resource).UpdateBufferStats(); +void TQueueResourceUser::UpdateBufferStats(const TUnboxedValuePod& resource) const { + GetResource(resource).UpdateBufferStats(); } TQueueResource& TQueueResourceUser::GetResource(const TUnboxedValuePod& resource) const { @@ -92,7 +92,7 @@ TQueueResource& TQueueResourceUser::GetResource(const TUnboxedValuePod& resource class TQueueCreateWrapper : public TMutableComputationNode<TQueueCreateWrapper> { typedef TMutableComputationNode<TQueueCreateWrapper> TBaseComputation; public: - TQueueCreateWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& dependentNodes, const TString& name, TMaybe<ui64> capacity, ui64 initSize) + TQueueCreateWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& dependentNodes, const TString& name, TMaybe<ui64> capacity, ui64 initSize) : TBaseComputation(mutables) , DependentNodes(std::move(dependentNodes)) , Name(name) @@ -111,7 +111,7 @@ private: const TComputationNodePtrVector DependentNodes; const TString Name; - const TMaybe<ui64> Capacity; + const TMaybe<ui64> Capacity; const ui64 InitSize; }; @@ -127,9 +127,9 @@ public: NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { auto resource = Resource->GetValue(ctx); auto& buffer = CheckAndGetBuffer(resource); - buffer.PushBack(Value->GetValue(ctx)); - if (buffer.IsUnbounded()) { - UpdateBufferStats(resource); + buffer.PushBack(Value->GetValue(ctx)); + if (buffer.IsUnbounded()) { + UpdateBufferStats(resource); } return resource.Release(); } @@ -190,130 +190,130 @@ private: const TComputationNodePtrVector DependentNodes; }; -class TQueueRangeWrapper : public TMutableComputationNode<TQueueRangeWrapper>, public TQueueResourceUser { - typedef TMutableComputationNode<TQueueRangeWrapper> TBaseComputation; -public: - class TValue : public TComputationValue<TValue>, public TQueueResourceUser { - public: - class TIterator : public TComputationValue<TIterator>, public TQueueResourceUser { - public: - TIterator(TMemoryUsageInfo* memInfo, TUnboxedValue queue, size_t begin, size_t end, ui64 generation, TStringBuf tag, IComputationNode* resource) - : TComputationValue<TIterator>(memInfo) - , TQueueResourceUser(std::move(tag), resource) - , Queue(queue) - , Buffer(CheckAndGetBuffer(queue)) - , Current(begin) - , End(end) - , Generation(generation) - { - } - - private: - bool Next(NUdf::TUnboxedValue& value) override { - MKQL_ENSURE(Generation == Buffer.Generation(), - "Queue generation changed while doing QueueRange: expected " << Generation << ", got: " << Buffer.Generation()); - if (Current >= End) { - return false; - } - - const auto& valRef = Buffer.Get(Current++); - value = !valRef ? NUdf::TUnboxedValuePod() : valRef.MakeOptional(); - return true; - } - - bool Skip() override { - if (Current >= End) { - return false; - } - Current++; - return true; - } - - const TUnboxedValue Queue; - const TSafeCircularBuffer<TUnboxedValue>& Buffer; - size_t Current; - const size_t End; - const ui64 Generation; - }; - - TValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, TUnboxedValue queue, size_t begin, size_t end, TStringBuf tag, IComputationNode* resource) - : TComputationValue<TValue>(memInfo) - , TQueueResourceUser(std::move(tag), resource) - , CompCtx(compCtx) - , Queue(queue) - , Begin(begin) - , End(std::min(end, CheckAndGetBuffer(Queue).UsedSize())) - , Generation(CheckAndGetBuffer(Queue).Generation()) - { - } - - private: - ui64 GetListLength() const final { - return Begin < End ? (End - Begin) : 0; - } - - bool HasListItems() const final { - return GetListLength() != 0; - } - - bool HasFastListLength() const final { - return true; - } - - NUdf::TUnboxedValue GetListIterator() const final { - return CompCtx.HolderFactory.Create<TIterator>(Queue, Begin, End, Generation, Tag, Resource); - } - - TComputationContext& CompCtx; - const TUnboxedValue Queue; - const size_t Begin; - const size_t End; - const ui64 Generation; - }; - - TQueueRangeWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& dependentNodes, const TResourceType* resourceType, IComputationNode* resource, - IComputationNode* begin, IComputationNode* end) - : TBaseComputation(mutables) - , TQueueResourceUser(resourceType->GetTag(), resource) - , Begin(begin) - , End(end) - , DependentNodes(std::move(dependentNodes)) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - auto queue = Resource->GetValue(ctx); - - auto begin = Begin->GetValue(ctx).Get<ui64>(); - auto end = End->GetValue(ctx).Get<ui64>(); - - return ctx.HolderFactory.Create<TValue>(ctx, queue, begin, end, Tag, Resource); - } - -private: - void RegisterDependencies() const final { - DependsOn(Resource); - DependsOn(Begin); - DependsOn(End); - std::for_each(DependentNodes.cbegin(), DependentNodes.cend(), std::bind(&TQueueRangeWrapper::DependsOn, this, std::placeholders::_1)); - } - - IComputationNode* const Begin; - IComputationNode* const End; - const TComputationNodePtrVector DependentNodes; -}; - -class TPreserveStreamValue : public TComputationValue<TPreserveStreamValue>, public TQueueResourceUser { +class TQueueRangeWrapper : public TMutableComputationNode<TQueueRangeWrapper>, public TQueueResourceUser { + typedef TMutableComputationNode<TQueueRangeWrapper> TBaseComputation; +public: + class TValue : public TComputationValue<TValue>, public TQueueResourceUser { + public: + class TIterator : public TComputationValue<TIterator>, public TQueueResourceUser { + public: + TIterator(TMemoryUsageInfo* memInfo, TUnboxedValue queue, size_t begin, size_t end, ui64 generation, TStringBuf tag, IComputationNode* resource) + : TComputationValue<TIterator>(memInfo) + , TQueueResourceUser(std::move(tag), resource) + , Queue(queue) + , Buffer(CheckAndGetBuffer(queue)) + , Current(begin) + , End(end) + , Generation(generation) + { + } + + private: + bool Next(NUdf::TUnboxedValue& value) override { + MKQL_ENSURE(Generation == Buffer.Generation(), + "Queue generation changed while doing QueueRange: expected " << Generation << ", got: " << Buffer.Generation()); + if (Current >= End) { + return false; + } + + const auto& valRef = Buffer.Get(Current++); + value = !valRef ? NUdf::TUnboxedValuePod() : valRef.MakeOptional(); + return true; + } + + bool Skip() override { + if (Current >= End) { + return false; + } + Current++; + return true; + } + + const TUnboxedValue Queue; + const TSafeCircularBuffer<TUnboxedValue>& Buffer; + size_t Current; + const size_t End; + const ui64 Generation; + }; + + TValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, TUnboxedValue queue, size_t begin, size_t end, TStringBuf tag, IComputationNode* resource) + : TComputationValue<TValue>(memInfo) + , TQueueResourceUser(std::move(tag), resource) + , CompCtx(compCtx) + , Queue(queue) + , Begin(begin) + , End(std::min(end, CheckAndGetBuffer(Queue).UsedSize())) + , Generation(CheckAndGetBuffer(Queue).Generation()) + { + } + + private: + ui64 GetListLength() const final { + return Begin < End ? (End - Begin) : 0; + } + + bool HasListItems() const final { + return GetListLength() != 0; + } + + bool HasFastListLength() const final { + return true; + } + + NUdf::TUnboxedValue GetListIterator() const final { + return CompCtx.HolderFactory.Create<TIterator>(Queue, Begin, End, Generation, Tag, Resource); + } + + TComputationContext& CompCtx; + const TUnboxedValue Queue; + const size_t Begin; + const size_t End; + const ui64 Generation; + }; + + TQueueRangeWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& dependentNodes, const TResourceType* resourceType, IComputationNode* resource, + IComputationNode* begin, IComputationNode* end) + : TBaseComputation(mutables) + , TQueueResourceUser(resourceType->GetTag(), resource) + , Begin(begin) + , End(end) + , DependentNodes(std::move(dependentNodes)) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + auto queue = Resource->GetValue(ctx); + + auto begin = Begin->GetValue(ctx).Get<ui64>(); + auto end = End->GetValue(ctx).Get<ui64>(); + + return ctx.HolderFactory.Create<TValue>(ctx, queue, begin, end, Tag, Resource); + } + +private: + void RegisterDependencies() const final { + DependsOn(Resource); + DependsOn(Begin); + DependsOn(End); + std::for_each(DependentNodes.cbegin(), DependentNodes.cend(), std::bind(&TQueueRangeWrapper::DependsOn, this, std::placeholders::_1)); + } + + IComputationNode* const Begin; + IComputationNode* const End; + const TComputationNodePtrVector DependentNodes; +}; + +class TPreserveStreamValue : public TComputationValue<TPreserveStreamValue>, public TQueueResourceUser { public: using TBase = TComputationValue<TPreserveStreamValue>; TPreserveStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& stream - , NUdf::TUnboxedValue&& queue, TStringBuf tag, IComputationNode* resource, ui64 outpace) + , NUdf::TUnboxedValue&& queue, TStringBuf tag, IComputationNode* resource, ui64 outpace) : TBase(memInfo) - , TQueueResourceUser(std::move(tag), resource) + , TQueueResourceUser(std::move(tag), resource) , Stream(std::move(stream)) - , Queue(std::move(queue)) + , Queue(std::move(queue)) , OutpaceGoal(outpace) - , Buffer(CheckAndGetBuffer(Queue)) + , Buffer(CheckAndGetBuffer(Queue)) , FrontIndex(Buffer.UsedSize()) {} @@ -324,7 +324,7 @@ private: Buffer.PopFront(); --Outpace; } - while (State != EPreserveState::Emit && Outpace <= OutpaceGoal) { + while (State != EPreserveState::Emit && Outpace <= OutpaceGoal) { switch (Stream.Fetch(item)) { case NUdf::EFetchStatus::Yield: return NUdf::EFetchStatus::Yield; @@ -333,11 +333,11 @@ private: break; case NUdf::EFetchStatus::Ok: Buffer.PushBack(std::move(item)); - if (Buffer.IsUnbounded()) { - UpdateBufferStats(Queue); - } + if (Buffer.IsUnbounded()) { + UpdateBufferStats(Queue); + } ++Outpace; - if (Outpace > OutpaceGoal) { + if (Outpace > OutpaceGoal) { State = EPreserveState::GoOn; } } @@ -356,8 +356,8 @@ private: Emit, }; const NUdf::TUnboxedValue Stream; - const NUdf::TUnboxedValue Queue; - const ui64 OutpaceGoal; + const NUdf::TUnboxedValue Queue; + const ui64 OutpaceGoal; TSafeCircularBuffer<TUnboxedValue>& Buffer; const size_t FrontIndex; @@ -368,7 +368,7 @@ private: class TPreserveStreamWrapper : public TMutableComputationNode<TPreserveStreamWrapper>, public TQueueResourceUser { typedef TMutableComputationNode<TPreserveStreamWrapper> TBaseComputation; public: - TPreserveStreamWrapper(TComputationMutables& mutables, IComputationNode* stream, const TResourceType* resourceType, IComputationNode* resource, ui64 outpace) + TPreserveStreamWrapper(TComputationMutables& mutables, IComputationNode* stream, const TResourceType* resourceType, IComputationNode* resource, ui64 outpace) : TBaseComputation(mutables) , TQueueResourceUser(resourceType->GetTag(), resource) , Stream(stream) @@ -376,7 +376,7 @@ public: {} NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - return ctx.HolderFactory.Create<TPreserveStreamValue>(Stream->GetValue(ctx), Resource->GetValue(ctx), Tag, Resource, Outpace); + return ctx.HolderFactory.Create<TPreserveStreamValue>(Stream->GetValue(ctx), Resource->GetValue(ctx), Tag, Resource, Outpace); } private: @@ -386,7 +386,7 @@ private: } IComputationNode* const Stream; - const ui64 Outpace; + const ui64 Outpace; }; template<class T, class...Args> @@ -404,11 +404,11 @@ IComputationNode* WrapQueueCreate(TCallable& callable, const TComputationNodeFac const unsigned reqArgs = 3; MKQL_ENSURE(callable.GetInputsCount() >= reqArgs, "QueueCreate: Expected at least " << reqArgs << " arg"); auto queueNameValue = AS_VALUE(TDataLiteral, callable.GetInput(0)); - TMaybe<ui64> capacity; - if (!callable.GetInput(1).GetStaticType()->IsVoid()) { - auto queueCapacityValue = AS_VALUE(TDataLiteral, callable.GetInput(1)); - capacity = queueCapacityValue->AsValue().Get<ui64>(); - } + TMaybe<ui64> capacity; + if (!callable.GetInput(1).GetStaticType()->IsVoid()) { + auto queueCapacityValue = AS_VALUE(TDataLiteral, callable.GetInput(1)); + capacity = queueCapacityValue->AsValue().Get<ui64>(); + } auto queueInitSizeValue = AS_VALUE(TDataLiteral, callable.GetInput(2)); const TString name(queueNameValue->AsValue().AsStringRef()); const auto initSize = queueInitSizeValue->AsValue().Get<ui64>(); @@ -441,31 +441,31 @@ IComputationNode* WrapQueuePeek(TCallable& callable, const TComputationNodeFacto return MakeNodeWithDeps<TQueuePeekWrapper>(callable, ctx, reqArgs, resourceType, resource, index); } -IComputationNode* WrapQueueRange(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - const unsigned reqArgs = 3; - MKQL_ENSURE(callable.GetInputsCount() >= reqArgs, "QueueRange: Expected at least " << reqArgs << " arg"); - auto resourceType = AS_TYPE(TResourceType, callable.GetInput(0)); - - TDataType* beginIndexType = AS_TYPE(TDataType, callable.GetInput(1)); - MKQL_ENSURE(beginIndexType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "Expected ui64 as queue begin index"); - - TDataType* endIndexType = AS_TYPE(TDataType, callable.GetInput(2)); - MKQL_ENSURE(endIndexType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "Expected ui64 as queue end index"); - - auto resource = LocateNode(ctx.NodeLocator, callable, 0); - auto beginIndex = LocateNode(ctx.NodeLocator, callable, 1); - auto endIndex = LocateNode(ctx.NodeLocator, callable, 2); - return MakeNodeWithDeps<TQueueRangeWrapper>(callable, ctx, reqArgs, resourceType, resource, beginIndex, endIndex); -} - +IComputationNode* WrapQueueRange(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + const unsigned reqArgs = 3; + MKQL_ENSURE(callable.GetInputsCount() >= reqArgs, "QueueRange: Expected at least " << reqArgs << " arg"); + auto resourceType = AS_TYPE(TResourceType, callable.GetInput(0)); + + TDataType* beginIndexType = AS_TYPE(TDataType, callable.GetInput(1)); + MKQL_ENSURE(beginIndexType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "Expected ui64 as queue begin index"); + + TDataType* endIndexType = AS_TYPE(TDataType, callable.GetInput(2)); + MKQL_ENSURE(endIndexType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "Expected ui64 as queue end index"); + + auto resource = LocateNode(ctx.NodeLocator, callable, 0); + auto beginIndex = LocateNode(ctx.NodeLocator, callable, 1); + auto endIndex = LocateNode(ctx.NodeLocator, callable, 2); + return MakeNodeWithDeps<TQueueRangeWrapper>(callable, ctx, reqArgs, resourceType, resource, beginIndex, endIndex); +} + IComputationNode* WrapPreserveStream(TCallable& callable, const TComputationNodeFactoryContext& ctx) { Y_UNUSED(ctx); - MKQL_ENSURE(callable.GetInputsCount() == 3, "PreserveStream: Expected 3 arg"); + MKQL_ENSURE(callable.GetInputsCount() == 3, "PreserveStream: Expected 3 arg"); auto stream = LocateNode(ctx.NodeLocator, callable, 0); auto resource = LocateNode(ctx.NodeLocator, callable, 1); auto resourceType = AS_TYPE(TResourceType, callable.GetInput(1)); - auto outpaceValue = AS_VALUE(TDataLiteral, callable.GetInput(2)); - const auto outpace = outpaceValue->AsValue().Get<ui64>(); + auto outpaceValue = AS_VALUE(TDataLiteral, callable.GetInput(2)); + const auto outpace = outpaceValue->AsValue().Get<ui64>(); return new TPreserveStreamWrapper(ctx.Mutables, stream, resourceType, resource, outpace); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_queue.h b/ydb/library/yql/minikql/comp_nodes/mkql_queue.h index 210850ee0f..8b59f441f7 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_queue.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_queue.h @@ -11,7 +11,7 @@ IComputationNode* WrapQueueCreate(TCallable& callable, const TComputationNodeFac IComputationNode* WrapQueuePush(TCallable& callable, const TComputationNodeFactoryContext& ctx); IComputationNode* WrapQueuePop(TCallable& callable, const TComputationNodeFactoryContext& ctx); IComputationNode* WrapQueuePeek(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapQueueRange(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapQueueRange(TCallable& callable, const TComputationNodeFactoryContext& ctx); IComputationNode* WrapPreserveStream(TCallable& callable, const TComputationNodeFactoryContext& ctx); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp index c3818e14f7..ebd1b6fafd 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp @@ -1,840 +1,840 @@ -#include "mkql_range.h" +#include "mkql_range.h" #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/computation/presort.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/minikql/mkql_string_util.h> - -#include <algorithm> - -namespace NKikimr { -namespace NMiniKQL { - -using namespace NYql::NUdf; - -namespace { - -using TTypeList = std::vector<TType*>; -using TUnboxedValueQueue = std::deque<NUdf::TUnboxedValue, TMKQLAllocator<NUdf::TUnboxedValue>>; - -struct TRangeTypeInfo { - TType* RangeType = nullptr; - ICompare::TPtr RangeCompare; - - TType* BoundaryType = nullptr; - ICompare::TPtr BoundaryCompare; - - TTypeList Components; - std::vector<ICompare::TPtr> ComponentsCompare; -}; - -TType* RemoveAllOptionals(TType* type) { - Y_ENSURE(type); - while (type->IsOptional()) { - type = static_cast<TOptionalType*>(type)->GetItemType(); - } - return type; -} - -TRangeTypeInfo ExtractTypes(TType* rangeType) { - TRangeTypeInfo result; - result.RangeType = rangeType; - result.RangeCompare = MakeCompareImpl(result.RangeType); - - MKQL_ENSURE(result.RangeType->IsTuple(), "Expecting range to be of tuple type"); - auto rangeTupleType = static_cast<TTupleType*>(result.RangeType); - MKQL_ENSURE(rangeTupleType->GetElementsCount() == 2, "Expecting range to be of tuple type with 2 elements"); - MKQL_ENSURE(rangeTupleType->GetElementType(0)->IsSameType(*rangeTupleType->GetElementType(1)), - "Expecting range to be of tuple type with 2 elements of same type"); - - result.BoundaryType = rangeTupleType->GetElementType(0); - result.BoundaryCompare = MakeCompareImpl(result.BoundaryType); - - MKQL_ENSURE(result.BoundaryType->IsTuple(), "Expecting range boundary to be of tuple type"); - auto rangeBoundaryTupleType = static_cast<TTupleType*>(result.BoundaryType); - MKQL_ENSURE(rangeBoundaryTupleType->GetElementsCount() >= 3, - "Expecting range boundary to be of tuple type with at least 3 elements"); - - MKQL_ENSURE(rangeBoundaryTupleType->GetElementsCount() % 2 == 1, - "Expecting range boundary to be of tuple type with odd element count"); - - for (ui32 i = 0; i < rangeBoundaryTupleType->GetElementsCount(); ++i) { - auto type = rangeBoundaryTupleType->GetElementType(i); - if (i % 2 == 1) { - auto baseType = RemoveAllOptionals(type); - MKQL_ENSURE(type->IsOptional() && baseType->IsData(), - "Expecting (multiple) optional of Data at odd positions of range boundary tuple"); - } else { - MKQL_ENSURE(type->IsData() && static_cast<TDataType*>(type)->GetSchemeType() == NUdf::TDataType<i32>::Id, - "Expected i32 at even positions of range boundary tuple"); - } - result.Components.push_back(type); - result.ComponentsCompare.push_back(MakeCompareImpl(type)); - } - return result; -} - -struct TExpandedRangeBoundary { - int Included = 0; // -1 = [; 0 = (); +1 = ] - TUnboxedValue Value; // AsTuple(Inf, x, Inf, y, Inf, z, ..., Included), where -1 = -inf, +1 = +inf, 0 - finite value - TUnboxedValueVector Components; -}; - -struct TExpandedRange { - TExpandedRangeBoundary Left; - TExpandedRangeBoundary Right; -}; - -TExpandedRangeBoundary Max(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { - return cmp->Less(a.Value, b.Value) ? b : a; -} - -TExpandedRangeBoundary Min(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { - return cmp->Less(a.Value, b.Value) ? a : b; -} - -i32 GetInfSign(bool hasPrefix, bool isIncluded, bool isLeft) { - if (!hasPrefix || isIncluded) { - return (isLeft ? -1 : 1); - } - return (isLeft ? 1 : -1); -} - -TExpandedRangeBoundary ExpandRangeBoundary(TUnboxedValue value, bool left) { - auto elements = value.GetElements(); - auto elementsCount = value.GetListLength(); - - Y_ENSURE(elements); - Y_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1); - - TExpandedRangeBoundary result; - result.Value = value; - const bool hasPrefix = elements[0].Get<i32>() == 0; - const bool isIncluded = elements[elementsCount - 1].Get<i32>() != 0; - for (size_t i = 0; i < elementsCount - 1; i += 2) { - i32 inf = elements[i].Get<i32>(); - MKQL_ENSURE(inf == 0 || inf == GetInfSign(hasPrefix, isIncluded, left), - "Invalid value for range boundary inf marker: " << inf << " at position " << i); - MKQL_ENSURE((inf != 0) ^ bool(elements[i + 1]), - "Value does not match inf marker: " << inf << " at position " << i); - } - result.Components.assign(elements, elements + elementsCount); - result.Included = result.Components.back().Get<i32>(); - MKQL_ENSURE(!result.Included || result.Included == (left ? -1 : 1), - "Invalid value for range boundary last element: " << result.Included); - return result; -} - -TExpandedRange ExpandRange(TUnboxedValue value) { - auto elements = value.GetElements(); - auto elementsCount = value.GetListLength(); - - Y_ENSURE(elements); - Y_ENSURE(elementsCount == 2); - - - TExpandedRange result; - result.Left = ExpandRangeBoundary(elements[0], true); - result.Right = ExpandRangeBoundary(elements[1], false); - - Y_ENSURE(result.Left.Components.size() == result.Right.Components.size()); - bool seenInfRange = false; - for (size_t i = 0; i < result.Left.Components.size() - 1; i += 2) { - if (result.Left.Components[i].Get<i32>() && result.Right.Components[i].Get<i32>()) { - seenInfRange = true; - } else { - MKQL_ENSURE(!seenInfRange, "Non inf component follows inf component at position " << i); - } - } - return result; -} - -size_t GetFiniteComponentsCount(const TExpandedRangeBoundary& boundary) { - size_t result = 0; - for (size_t i = 0; i < boundary.Components.size() - 1; i += 2) { - if (boundary.Components[i].Get<i32>() != 0) { - break; - } - result += 1; - } - return result; -} - -template<typename T> -bool IsAdjacentNumericValues(TUnboxedValue left, TUnboxedValue right) { - T l = left.Get<T>(); - T r = right.Get<T>(); - Y_ENSURE(l < r); - return l + 1 == r; -} - -bool CanConvertToPointRange(const TExpandedRange& range, const TRangeTypeInfo& typeInfo) { - if (!(range.Left.Included && !range.Right.Included || - !range.Left.Included && range.Right.Included)) - { - return false; - } - const size_t compsCount = GetFiniteComponentsCount(range.Left); - if (compsCount == 0 || GetFiniteComponentsCount(range.Right) != compsCount) { - return false; - } - - const size_t lastCompIdx = 2 * (compsCount - 1) + 1; - - // check for suitable type - TType* baseType = RemoveAllOptionals(static_cast<TTupleType*>(typeInfo.BoundaryType)->GetElementType(lastCompIdx)); - auto slot = static_cast<TDataType*>(baseType)->GetDataSlot(); - Y_ENSURE(slot); - if (!(GetDataTypeInfo(*slot).Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType))) { - return false; - } - - // all components before last should be equal - for (size_t i = 1; i < lastCompIdx; i += 2) { - if (typeInfo.ComponentsCompare[i]->Compare(range.Left.Components[i], range.Right.Components[i])) { - return false; - } - } - - auto left = range.Left.Components[lastCompIdx]; - auto right = range.Right.Components[lastCompIdx]; - if (!left.HasValue() || !right.HasValue()) { - return false; - } - - switch (*slot) { - case EDataSlot::Int8: return IsAdjacentNumericValues<i8>(left, right); - case EDataSlot::Uint8: return IsAdjacentNumericValues<ui8>(left, right); - case EDataSlot::Int16: return IsAdjacentNumericValues<i16>(left, right); - case EDataSlot::Uint16: return IsAdjacentNumericValues<ui16>(left, right); - case EDataSlot::Int32: return IsAdjacentNumericValues<i32>(left, right); - case EDataSlot::Uint32: return IsAdjacentNumericValues<ui32>(left, right); - case EDataSlot::Int64: return IsAdjacentNumericValues<i64>(left, right); - case EDataSlot::Uint64: return IsAdjacentNumericValues<ui64>(left, right); - - case EDataSlot::Date: return IsAdjacentNumericValues<ui16>(left, right); - case EDataSlot::Datetime: return IsAdjacentNumericValues<ui32>(left, right); - default: - Y_ENSURE(*slot == EDataSlot::Timestamp); - } - return IsAdjacentNumericValues<ui64>(left, right); -} - -bool RangeIsEmpty(const TExpandedRange& range, const TRangeTypeInfo& typeInfo) { - if (typeInfo.BoundaryCompare->Compare(range.Left.Value, range.Right.Value) >= 0) { - // left >= right - return true; - } - - Y_ENSURE(typeInfo.ComponentsCompare.size() == range.Left.Components.size()); - // range is not empty if components are not equal - for (size_t i = 0; i < typeInfo.ComponentsCompare.size() - 1; ++i) { - if (typeInfo.ComponentsCompare[i]->Compare(range.Left.Components[i], range.Right.Components[i])) { - return false; - } - } - - // all component are equal, and range is empty if any side is excluded - return range.Left.Components.back().Get<i32>() == 0 || range.Right.Components.back().Get<i32>() == 0; -} - -bool RangeCanMerge(const TExpandedRange& a, const TExpandedRange& b, const TRangeTypeInfo& typeInfo) { - // It is assumed that a <= b here - // < { > } - // a.Left b.Left a.Right b.Right - TExpandedRange intersection = { b.Left, a.Right }; - int cmp = typeInfo.BoundaryCompare->Compare(intersection.Left.Value, intersection.Right.Value); - if (cmp > 0) { - return false; - } - - const auto& lefts = intersection.Left.Components; - const auto& rights = intersection.Right.Components; - - bool leftIncluded = lefts.back().Get<i32>() != 0; - bool rightIncluded = rights.back().Get<i32>() != 0; - - for (size_t i = 0; i < lefts.size() - 1; i += 2) { - auto infCmp = typeInfo.ComponentsCompare[i]; - auto compCmp = typeInfo.ComponentsCompare[i + 1]; - - auto infCompareRes = infCmp->Compare(lefts[i], rights[i]); - Y_ENSURE(infCompareRes <= 0); - if (infCompareRes < 0) { - return true; - } - - auto componentCompareRes = compCmp->Compare(lefts[i + 1], rights[i + 1]); - Y_ENSURE(componentCompareRes <= 0); - if (componentCompareRes < 0) { - return true; - } - } - - return leftIncluded || rightIncluded; -} - -class TRangeComputeBase { -public: - TRangeComputeBase(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) - : TypeInfos(std::move(typeInfos)) - , Lists(std::move(lists)) - { - Y_ENSURE(Lists.size() == TypeInfos.size()); - Y_ENSURE(!Lists.empty()); - } -protected: - std::vector<TUnboxedValueQueue> ExpandLists(TComputationContext& ctx) const { - TUnboxedValueVector lists; - lists.reserve(Lists.size()); - for (auto& list : Lists) { - lists.emplace_back(list->GetValue(ctx)); - } - - std::vector<TUnboxedValueQueue> expandedLists; - for (size_t i = 0; i < lists.size(); ++i) { - expandedLists.emplace_back(); - TThresher<false>::DoForEachItem(lists[i], - [&] (NUdf::TUnboxedValue&& item) { - expandedLists.back().emplace_back(std::move(item)); - } - ); - NormalizeRanges(expandedLists.back(), TypeInfos[i]); - } - - return expandedLists; - } - -private: - template<typename TContainer> - static void NormalizeRanges(TContainer& ranges, const TRangeTypeInfo& typeInfo) { - auto rangeLess = [&](const TUnboxedValuePod& a, const TUnboxedValuePod& b) { - return typeInfo.RangeCompare->Less(a, b); - }; - - auto rangeEqual = [&](const TUnboxedValuePod& a, const TUnboxedValuePod& b) { - return typeInfo.RangeCompare->Compare(a, b) == 0; - }; - - for (size_t i = 1; i < ranges.size(); ++i) { - if (rangeLess(ranges[i], ranges[i - 1])) { - std::sort(ranges.begin(), ranges.end(), rangeLess); - break; - } - } - - ranges.erase( - std::remove_if(ranges.begin(), ranges.end(), - [&](const TUnboxedValue& range) { return RangeIsEmpty(ExpandRange(range), typeInfo); }), - ranges.end()); - ranges.erase(std::unique(ranges.begin(), ranges.end(), rangeEqual), ranges.end()); - } - -protected: - const TComputationNodePtrVector Lists; - const std::vector<TRangeTypeInfo> TypeInfos; -}; - -class TRangeUnionWrapper : public TMutableComputationNode<TRangeUnionWrapper>, public TRangeComputeBase { - typedef TMutableComputationNode<TRangeUnionWrapper> TBaseComputation; -public: - TRangeUnionWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) - : TBaseComputation(mutables) - , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - TUnboxedValueVector mergedLists; - auto expandedLists = ExpandLists(ctx); - while (!expandedLists.empty()) { - TMaybe<size_t> argMin; - bool haveEmpty = false; - for (size_t i = 0; i < expandedLists.size(); ++i) { - if (expandedLists[i].empty()) { - haveEmpty = true; - continue; - } - auto& current = expandedLists[i].front(); - if (!argMin || TypeInfos.front().RangeCompare->Less(current, expandedLists[*argMin].front())) { - argMin = i; - } - } - if (argMin) { - auto& from = expandedLists[*argMin]; - if (!RangeIsEmpty(ExpandRange(from.front()), TypeInfos.front())) { - mergedLists.emplace_back(std::move(from.front())); - } - from.pop_front(); - haveEmpty = haveEmpty || from.empty(); - } - - if (haveEmpty) { - expandedLists.erase( - std::remove_if(expandedLists.begin(), expandedLists.end(), [](const auto& list) { return list.empty(); }), - expandedLists.end()); - } - } - - TUnboxedValueVector unionList; - if (!mergedLists.empty()) { - unionList.push_back(mergedLists.front()); - auto current = ExpandRange(unionList.back()); - for (size_t i = 1; i < mergedLists.size(); ++i) { - auto toUnion = ExpandRange(mergedLists[i]); - if (RangeCanMerge(current, toUnion, TypeInfos.front())) { - current = { current.Left, Max(current.Right, toUnion.Right, TypeInfos.front().BoundaryCompare) }; - TUnboxedValueVector newValue = { current.Left.Value, current.Right.Value }; - unionList.back() = ctx.HolderFactory.VectorAsArray(newValue); - } else { - unionList.emplace_back(std::move(mergedLists[i])); - current = ExpandRange(unionList.back()); - } - } - } - - TDefaultListRepresentation res; - for (auto& item : unionList) { - res = res.Append(std::move(item)); - } - return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); - } - -private: - void RegisterDependencies() const final { - std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeUnionWrapper::DependsOn, this, std::placeholders::_1)); - } -}; - -class TRangeIntersectWrapper : public TMutableComputationNode<TRangeIntersectWrapper>, public TRangeComputeBase { - typedef TMutableComputationNode<TRangeIntersectWrapper> TBaseComputation; -public: - TRangeIntersectWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) - : TBaseComputation(mutables) - , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - TUnboxedValueVector mergedLists; - auto expandedLists = ExpandLists(ctx); - Y_ENSURE(!expandedLists.empty()); - TUnboxedValueQueue intersected = std::move(expandedLists.front()); - for (size_t i = 1; i < expandedLists.size(); ++i) { - DoIntersect(ctx, intersected, std::move(expandedLists[i])); - } - - TDefaultListRepresentation res; - for (auto& item : intersected) { - res = res.Append(std::move(item)); - } - return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); - } - -private: - void RegisterDependencies() const final { - std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeIntersectWrapper::DependsOn, this, std::placeholders::_1)); - } - - void DoIntersect(TComputationContext& ctx, TUnboxedValueQueue& current, TUnboxedValueQueue&& next) const { - TUnboxedValueQueue result; - auto cmp = TypeInfos.front().RangeCompare; - auto boundaryCmp = TypeInfos.front().BoundaryCompare; - while (!current.empty() && !next.empty()) { - TUnboxedValueQueue* minInput; - TUnboxedValueQueue* maxInput; - if (cmp->Less(current.front(), next.front())) { - minInput = ¤t; - maxInput = &next; - } else { - minInput = &next; - maxInput = ¤t; - } - - auto minRange = ExpandRange(minInput->front()); - auto maxRange = ExpandRange(maxInput->front()); - - TExpandedRange intersected; - intersected.Left = maxRange.Left; - intersected.Right = Min(minRange.Right, maxRange.Right, TypeInfos.front().BoundaryCompare); - if (!RangeIsEmpty(intersected, TypeInfos.front())) { - TUnboxedValueVector newValue = { intersected.Left.Value, intersected.Right.Value }; - result.push_back(ctx.HolderFactory.VectorAsArray(newValue)); - - if (boundaryCmp->Less(minRange.Right.Value, maxRange.Right.Value)) { - minInput->pop_front(); - } else { - maxInput->pop_front(); - } - } else { - minInput->pop_front(); - } - } - std::swap(current, result); - } -}; - -class TRangeMultiplyWrapper : public TMutableComputationNode<TRangeMultiplyWrapper>, public TRangeComputeBase { - typedef TMutableComputationNode<TRangeMultiplyWrapper> TBaseComputation; -public: - TRangeMultiplyWrapper(TComputationMutables& mutables, IComputationNode* limit, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) - : TBaseComputation(mutables) - , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) - , Limit(limit) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - const ui64 limit = Limit->GetValue(ctx).Get<ui64>(); - TUnboxedValueVector mergedLists; - auto expandedLists = ExpandLists(ctx); - Y_ENSURE(!expandedLists.empty()); - if (expandedLists.size() == 1 && expandedLists.front().size() > limit) { - return FullRange(ctx); - } - - TUnboxedValueQueue current = std::move(expandedLists.front()); - std::vector<ICompare::TPtr> currentComponentsCompare = TypeInfos.front().ComponentsCompare; - for (size_t i = 1; i < expandedLists.size(); ++i) { - if (expandedLists[i].empty()) { - return ctx.HolderFactory.GetEmptyContainer(); - } - if (!DoMultiply(ctx, limit, current, expandedLists[i], currentComponentsCompare, TypeInfos[i])) { - return FullRange(ctx); - } - } - - TDefaultListRepresentation res; - for (auto& item : current) { - res = res.Append(std::move(item)); - } - return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); - } - -private: - void RegisterDependencies() const final { - DependsOn(Limit); - std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeMultiplyWrapper::DependsOn, this, std::placeholders::_1)); - } - - bool DoMultiply(TComputationContext& ctx, ui64 limit, TUnboxedValueQueue& current, const TUnboxedValueQueue& next, - std::vector<ICompare::TPtr>& currentCmps, const TRangeTypeInfo& nextTypeInfo) const - { - TUnboxedValueQueue result; - Y_ENSURE(currentCmps.size() >= 3 && currentCmps.size() % 2 == 1); - size_t extraColumns = (nextTypeInfo.ComponentsCompare.size() - 1) / 2; - for (const auto& c : current) { - auto curr = ExpandRange(c); - if (RangeIsPoint(curr, currentCmps)) { - if (result.size() + next.size() > limit) { - return false; - } - for (const auto& n : next) { - result.push_back(Append(ctx, curr, ExpandRange(n))); - } - } else { - if (result.size() + 1 > limit) { - return false; - } - result.push_back(AppendInfs(ctx, curr, extraColumns)); - } - } - - ICompare::TPtr currentLast = currentCmps.back(); - currentCmps.pop_back(); - currentCmps.insert(currentCmps.end(), nextTypeInfo.ComponentsCompare.begin(), nextTypeInfo.ComponentsCompare.end()); - - std::swap(current, result); - return true; - } - - static bool RangeIsPoint(const TExpandedRange& range, const std::vector<ICompare::TPtr>& cmps) { - Y_ENSURE(range.Left.Components.size() == cmps.size()); - TUnboxedValue leftIncluded = range.Left.Components.back(); - TUnboxedValue rightIncluded = range.Right.Components.back(); - if (!leftIncluded.Get<i32>() || !rightIncluded.Get<i32>()) { - return false; - } - - bool allEqual = true; - for (size_t i = 0; allEqual && i < cmps.size() - 1; ++i) { - allEqual = allEqual && - cmps[i]->Compare(range.Left.Components[i], range.Right.Components[i]) == 0; - } - - return allEqual; - } - - static TUnboxedValuePod Append(TComputationContext& ctx, const TExpandedRange& first, const TExpandedRange& second) { - auto left = Append(ctx, first.Left, second.Left); - auto right = Append(ctx, first.Right, second.Right); - TUnboxedValueVector range = { left, right }; - return ctx.HolderFactory.VectorAsArray(range); - } - - static TUnboxedValuePod Append(TComputationContext& ctx, const TExpandedRangeBoundary& first, - const TExpandedRangeBoundary& second) - { - TUnboxedValueVector components(first.Components.begin(), first.Components.end() - 1); - components.insert(components.end(), second.Components.begin(), second.Components.end()); - if (second.Components.front().Get<i32>() != 0) { - // preserve original include/exclude flag when appending nulls (+-inf) - components.back() = first.Components.back(); - } - return ctx.HolderFactory.VectorAsArray(components); - } - - static TUnboxedValuePod AppendInfs(TComputationContext& ctx, const TExpandedRange& range, size_t count) { - auto left = AppendInfs(ctx, true, range.Left, count); - auto right = AppendInfs(ctx, false, range.Right, count); - TUnboxedValueVector newRange = { left, right }; - return ctx.HolderFactory.VectorAsArray(newRange); - } - - static TUnboxedValuePod AppendInfs(TComputationContext& ctx, bool isLeft, const TExpandedRangeBoundary& boundary, size_t count) { - Y_ENSURE(!boundary.Components.empty()); - TUnboxedValueVector components(boundary.Components.begin(), boundary.Components.end() - 1); - const bool hasPrefix = boundary.Components.size() > 1 && boundary.Components.front().Get<i32>() == 0; - const bool isIncluded = boundary.Components.back().Get<i32>() != 0; - for (size_t i = 0; i < count; ++i) { - components.push_back(TUnboxedValuePod(GetInfSign(hasPrefix, isIncluded, isLeft))); - components.emplace_back(); - } - components.push_back(boundary.Components.back()); - return ctx.HolderFactory.VectorAsArray(components); - } - - TUnboxedValuePod FullRange(TComputationContext& ctx) const { - size_t columnCount = 0; - for (const auto& ti : TypeInfos) { - Y_ENSURE(ti.Components.size() % 2 == 1); - columnCount += (ti.Components.size() - 1) / 2; - } - - TExpandedRange range; - range.Left.Components.push_back(TUnboxedValuePod(0)); - range.Right.Components.push_back(TUnboxedValuePod(0)); - TUnboxedValueVector result = { AppendInfs(ctx, range, columnCount) }; - return ctx.HolderFactory.VectorAsArray(result); - } - - IComputationNode* const Limit; -}; - - -class TRangeFinalizeWrapper : public TMutableComputationNode<TRangeFinalizeWrapper>, public TRangeComputeBase { - typedef TMutableComputationNode<TRangeFinalizeWrapper> TBaseComputation; -public: - TRangeFinalizeWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) - : TBaseComputation(mutables) - , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - auto expandedLists = ExpandLists(ctx); - Y_ENSURE(expandedLists.size() == 1); - - TDefaultListRepresentation res; - for (auto& item : expandedLists.front()) { - auto range = ExpandRange(item); - if (CanConvertToPointRange(range, TypeInfos.front())) { - if (range.Left.Included) { - range.Right = range.Left; - } else { - range.Left = range.Right; - } - } - - auto left = ConvertFromInternal(range.Left.Components, ctx); - auto right = ConvertFromInternal(range.Right.Components, ctx); - - TUnboxedValueVector rangeVector = { left, right }; - res = res.Append(ctx.HolderFactory.VectorAsArray(rangeVector)); - } - return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); - } - -private: - void RegisterDependencies() const final { - std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeFinalizeWrapper::DependsOn, this, std::placeholders::_1)); - } - - TUnboxedValue ConvertFromInternal(const TUnboxedValueVector& boundaryComponents, TComputationContext& ctx) const { - size_t compsSize = boundaryComponents.size(); - Y_ENSURE(compsSize >= 3); - Y_ENSURE(compsSize % 2 == 1); - - TUnboxedValueVector converted; - for (size_t i = 0; i < compsSize - 1; ++i) { - if (i % 2 == 1) { - converted.push_back(boundaryComponents[i]); - } - } - i32 included = boundaryComponents.back().Get<i32>(); - if (included != 0) { - included = 1; - } - converted.push_back(TUnboxedValuePod(included)); - return ctx.HolderFactory.VectorAsArray(converted); - } -}; - -enum ERangeOp { - RANGE_UNION, - RANGE_INTERSECT, - RANGE_MULTIPLY, - RANGE_FINALIZE, -}; - -IComputationNode* WrapRange(ERangeOp func, TCallable& callable, const TComputationNodeFactoryContext& ctx) { - TComputationNodePtrVector lists; - std::vector<TRangeTypeInfo> typeInfos; - size_t listsStart = 0; - if (func == RANGE_FINALIZE) { - MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting single argument"); - } else if (func == RANGE_MULTIPLY) { - MKQL_ENSURE(callable.GetInputsCount() > 1, "Expecting at least two arguments"); - listsStart = 1; - auto limitType = callable.GetInput(0).GetStaticType(); - MKQL_ENSURE(limitType->IsData() && static_cast<TDataType*>(limitType)->GetSchemeType() == NUdf::TDataType<ui64>::Id, - "Expecting Uint64 as first argument"); - } else { - MKQL_ENSURE(callable.GetInputsCount() > 0, "Expecting at least one argument"); - } - - lists.reserve(callable.GetInputsCount()); - typeInfos.reserve(callable.GetInputsCount()); - for (ui32 i = listsStart; i < callable.GetInputsCount(); ++i) { - auto type = callable.GetInput(i).GetStaticType(); - MKQL_ENSURE(type->IsList(), "Expecting list as argument"); - auto rangeType = static_cast<TListType*>(type)->GetItemType(); - - if (func != RANGE_MULTIPLY) { - MKQL_ENSURE(type->IsSameType(*callable.GetInput(listsStart).GetStaticType()), "All arguments must be of same type"); - } - - lists.push_back(LocateNode(ctx.NodeLocator, callable, i)); - typeInfos.push_back(ExtractTypes(rangeType)); - } - - switch (func) { - case RANGE_UNION: - return new TRangeUnionWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); - case RANGE_INTERSECT: - return new TRangeIntersectWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); - case RANGE_MULTIPLY: { - auto limit = LocateNode(ctx.NodeLocator, callable, 0); - return new TRangeMultiplyWrapper(ctx.Mutables, limit, std::move(lists), std::move(typeInfos)); - } - case RANGE_FINALIZE: - return new TRangeFinalizeWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); - default: - Y_ENSURE(!"Unknown callable"); - } -} - -class TRangeCreateWrapper : public TMutableComputationNode<TRangeCreateWrapper> { - typedef TMutableComputationNode<TRangeCreateWrapper> TBaseComputation; -public: - TRangeCreateWrapper(TComputationMutables& mutables, IComputationNode* list) - : TBaseComputation(mutables) - , List(list) - {} - - TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - TUnboxedValue list = List->GetValue(ctx); - - TDefaultListRepresentation res; - TThresher<false>::DoForEachItem(list, - [&] (NUdf::TUnboxedValue&& item) { - auto left = ConvertToInternal(item.GetElement(0), true, ctx); - auto right = ConvertToInternal(item.GetElement(1), false, ctx); - - TUnboxedValueVector rangeVector = { left, right }; - auto range = ctx.HolderFactory.VectorAsArray(rangeVector); - res = res.Append(std::move(range)); - } - ); - return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); - } - -private: - void RegisterDependencies() const final { - DependsOn(List); - } - - TUnboxedValue ConvertToInternal(TUnboxedValue boundary, bool isLeft, TComputationContext& ctx) const { - auto elements = boundary.GetElements(); - auto elementsCount = boundary.GetListLength(); - - Y_ENSURE(elements); - Y_ENSURE(elementsCount >= 2); - - TUnboxedValueVector converted; - i32 included = elements[elementsCount - 1].Get<i32>(); - const auto hasPrefix = bool(elements[0]); - bool tail = false; - for (size_t i = 0; i < elementsCount - 1; ++i) { - i32 infValue; - tail = tail || !elements[i]; - if (elements[i]) { - MKQL_ENSURE(!tail, "Invalid boundary value - non null element follows null"); - infValue = 0; - } else { - infValue = GetInfSign(hasPrefix, included, isLeft); - } - converted.push_back(TUnboxedValuePod(infValue)); - converted.push_back(elements[i]); - } - included = included ? (isLeft ? -1 : 1) : 0; - converted.push_back(TUnboxedValuePod(included)); - return ctx.HolderFactory.VectorAsArray(converted); - } - - IComputationNode* const List; -}; - - -} // namespace - -IComputationNode* WrapRangeCreate(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); - - auto list = callable.GetInput(0); - - auto itemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); - MKQL_ENSURE(itemType->IsTuple(), "Expecting list of tuples"); - - auto tupleType = static_cast<TTupleType*>(itemType); - MKQL_ENSURE(tupleType->GetElementsCount() == 2, - "Expecting list ot 2-element tuples, got: " << tupleType->GetElementsCount() << " elements"); - - MKQL_ENSURE(tupleType->GetElementType(0)->IsSameType(*tupleType->GetElementType(1)), - "Expecting list ot 2-element tuples of same type"); - - MKQL_ENSURE(tupleType->GetElementType(0)->IsTuple(), - "Expecting range boundary to be tuple"); - - auto boundaryType = static_cast<TTupleType*>(tupleType->GetElementType(0)); - MKQL_ENSURE(boundaryType->GetElementsCount() >= 2, - "Range boundary should have at least 2 components, got: " << boundaryType->GetElementsCount()); - - return new TRangeCreateWrapper(ctx.Mutables, LocateNode(ctx.NodeLocator, callable, 0)); -} - -IComputationNode* WrapRangeUnion(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - return WrapRange(RANGE_UNION, callable, ctx); -} - -IComputationNode* WrapRangeIntersect(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - return WrapRange(RANGE_INTERSECT, callable, ctx); -} - -IComputationNode* WrapRangeMultiply(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - return WrapRange(RANGE_MULTIPLY, callable, ctx); -} - -IComputationNode* WrapRangeFinalize(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - return WrapRange(RANGE_FINALIZE, callable, ctx); -} - -} -} + +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +using namespace NYql::NUdf; + +namespace { + +using TTypeList = std::vector<TType*>; +using TUnboxedValueQueue = std::deque<NUdf::TUnboxedValue, TMKQLAllocator<NUdf::TUnboxedValue>>; + +struct TRangeTypeInfo { + TType* RangeType = nullptr; + ICompare::TPtr RangeCompare; + + TType* BoundaryType = nullptr; + ICompare::TPtr BoundaryCompare; + + TTypeList Components; + std::vector<ICompare::TPtr> ComponentsCompare; +}; + +TType* RemoveAllOptionals(TType* type) { + Y_ENSURE(type); + while (type->IsOptional()) { + type = static_cast<TOptionalType*>(type)->GetItemType(); + } + return type; +} + +TRangeTypeInfo ExtractTypes(TType* rangeType) { + TRangeTypeInfo result; + result.RangeType = rangeType; + result.RangeCompare = MakeCompareImpl(result.RangeType); + + MKQL_ENSURE(result.RangeType->IsTuple(), "Expecting range to be of tuple type"); + auto rangeTupleType = static_cast<TTupleType*>(result.RangeType); + MKQL_ENSURE(rangeTupleType->GetElementsCount() == 2, "Expecting range to be of tuple type with 2 elements"); + MKQL_ENSURE(rangeTupleType->GetElementType(0)->IsSameType(*rangeTupleType->GetElementType(1)), + "Expecting range to be of tuple type with 2 elements of same type"); + + result.BoundaryType = rangeTupleType->GetElementType(0); + result.BoundaryCompare = MakeCompareImpl(result.BoundaryType); + + MKQL_ENSURE(result.BoundaryType->IsTuple(), "Expecting range boundary to be of tuple type"); + auto rangeBoundaryTupleType = static_cast<TTupleType*>(result.BoundaryType); + MKQL_ENSURE(rangeBoundaryTupleType->GetElementsCount() >= 3, + "Expecting range boundary to be of tuple type with at least 3 elements"); + + MKQL_ENSURE(rangeBoundaryTupleType->GetElementsCount() % 2 == 1, + "Expecting range boundary to be of tuple type with odd element count"); + + for (ui32 i = 0; i < rangeBoundaryTupleType->GetElementsCount(); ++i) { + auto type = rangeBoundaryTupleType->GetElementType(i); + if (i % 2 == 1) { + auto baseType = RemoveAllOptionals(type); + MKQL_ENSURE(type->IsOptional() && baseType->IsData(), + "Expecting (multiple) optional of Data at odd positions of range boundary tuple"); + } else { + MKQL_ENSURE(type->IsData() && static_cast<TDataType*>(type)->GetSchemeType() == NUdf::TDataType<i32>::Id, + "Expected i32 at even positions of range boundary tuple"); + } + result.Components.push_back(type); + result.ComponentsCompare.push_back(MakeCompareImpl(type)); + } + return result; +} + +struct TExpandedRangeBoundary { + int Included = 0; // -1 = [; 0 = (); +1 = ] + TUnboxedValue Value; // AsTuple(Inf, x, Inf, y, Inf, z, ..., Included), where -1 = -inf, +1 = +inf, 0 - finite value + TUnboxedValueVector Components; +}; + +struct TExpandedRange { + TExpandedRangeBoundary Left; + TExpandedRangeBoundary Right; +}; + +TExpandedRangeBoundary Max(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { + return cmp->Less(a.Value, b.Value) ? b : a; +} + +TExpandedRangeBoundary Min(TExpandedRangeBoundary a, TExpandedRangeBoundary b, ICompare::TPtr cmp) { + return cmp->Less(a.Value, b.Value) ? a : b; +} + +i32 GetInfSign(bool hasPrefix, bool isIncluded, bool isLeft) { + if (!hasPrefix || isIncluded) { + return (isLeft ? -1 : 1); + } + return (isLeft ? 1 : -1); +} + +TExpandedRangeBoundary ExpandRangeBoundary(TUnboxedValue value, bool left) { + auto elements = value.GetElements(); + auto elementsCount = value.GetListLength(); + + Y_ENSURE(elements); + Y_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1); + + TExpandedRangeBoundary result; + result.Value = value; + const bool hasPrefix = elements[0].Get<i32>() == 0; + const bool isIncluded = elements[elementsCount - 1].Get<i32>() != 0; + for (size_t i = 0; i < elementsCount - 1; i += 2) { + i32 inf = elements[i].Get<i32>(); + MKQL_ENSURE(inf == 0 || inf == GetInfSign(hasPrefix, isIncluded, left), + "Invalid value for range boundary inf marker: " << inf << " at position " << i); + MKQL_ENSURE((inf != 0) ^ bool(elements[i + 1]), + "Value does not match inf marker: " << inf << " at position " << i); + } + result.Components.assign(elements, elements + elementsCount); + result.Included = result.Components.back().Get<i32>(); + MKQL_ENSURE(!result.Included || result.Included == (left ? -1 : 1), + "Invalid value for range boundary last element: " << result.Included); + return result; +} + +TExpandedRange ExpandRange(TUnboxedValue value) { + auto elements = value.GetElements(); + auto elementsCount = value.GetListLength(); + + Y_ENSURE(elements); + Y_ENSURE(elementsCount == 2); + + + TExpandedRange result; + result.Left = ExpandRangeBoundary(elements[0], true); + result.Right = ExpandRangeBoundary(elements[1], false); + + Y_ENSURE(result.Left.Components.size() == result.Right.Components.size()); + bool seenInfRange = false; + for (size_t i = 0; i < result.Left.Components.size() - 1; i += 2) { + if (result.Left.Components[i].Get<i32>() && result.Right.Components[i].Get<i32>()) { + seenInfRange = true; + } else { + MKQL_ENSURE(!seenInfRange, "Non inf component follows inf component at position " << i); + } + } + return result; +} + +size_t GetFiniteComponentsCount(const TExpandedRangeBoundary& boundary) { + size_t result = 0; + for (size_t i = 0; i < boundary.Components.size() - 1; i += 2) { + if (boundary.Components[i].Get<i32>() != 0) { + break; + } + result += 1; + } + return result; +} + +template<typename T> +bool IsAdjacentNumericValues(TUnboxedValue left, TUnboxedValue right) { + T l = left.Get<T>(); + T r = right.Get<T>(); + Y_ENSURE(l < r); + return l + 1 == r; +} + +bool CanConvertToPointRange(const TExpandedRange& range, const TRangeTypeInfo& typeInfo) { + if (!(range.Left.Included && !range.Right.Included || + !range.Left.Included && range.Right.Included)) + { + return false; + } + const size_t compsCount = GetFiniteComponentsCount(range.Left); + if (compsCount == 0 || GetFiniteComponentsCount(range.Right) != compsCount) { + return false; + } + + const size_t lastCompIdx = 2 * (compsCount - 1) + 1; + + // check for suitable type + TType* baseType = RemoveAllOptionals(static_cast<TTupleType*>(typeInfo.BoundaryType)->GetElementType(lastCompIdx)); + auto slot = static_cast<TDataType*>(baseType)->GetDataSlot(); + Y_ENSURE(slot); + if (!(GetDataTypeInfo(*slot).Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType))) { + return false; + } + + // all components before last should be equal + for (size_t i = 1; i < lastCompIdx; i += 2) { + if (typeInfo.ComponentsCompare[i]->Compare(range.Left.Components[i], range.Right.Components[i])) { + return false; + } + } + + auto left = range.Left.Components[lastCompIdx]; + auto right = range.Right.Components[lastCompIdx]; + if (!left.HasValue() || !right.HasValue()) { + return false; + } + + switch (*slot) { + case EDataSlot::Int8: return IsAdjacentNumericValues<i8>(left, right); + case EDataSlot::Uint8: return IsAdjacentNumericValues<ui8>(left, right); + case EDataSlot::Int16: return IsAdjacentNumericValues<i16>(left, right); + case EDataSlot::Uint16: return IsAdjacentNumericValues<ui16>(left, right); + case EDataSlot::Int32: return IsAdjacentNumericValues<i32>(left, right); + case EDataSlot::Uint32: return IsAdjacentNumericValues<ui32>(left, right); + case EDataSlot::Int64: return IsAdjacentNumericValues<i64>(left, right); + case EDataSlot::Uint64: return IsAdjacentNumericValues<ui64>(left, right); + + case EDataSlot::Date: return IsAdjacentNumericValues<ui16>(left, right); + case EDataSlot::Datetime: return IsAdjacentNumericValues<ui32>(left, right); + default: + Y_ENSURE(*slot == EDataSlot::Timestamp); + } + return IsAdjacentNumericValues<ui64>(left, right); +} + +bool RangeIsEmpty(const TExpandedRange& range, const TRangeTypeInfo& typeInfo) { + if (typeInfo.BoundaryCompare->Compare(range.Left.Value, range.Right.Value) >= 0) { + // left >= right + return true; + } + + Y_ENSURE(typeInfo.ComponentsCompare.size() == range.Left.Components.size()); + // range is not empty if components are not equal + for (size_t i = 0; i < typeInfo.ComponentsCompare.size() - 1; ++i) { + if (typeInfo.ComponentsCompare[i]->Compare(range.Left.Components[i], range.Right.Components[i])) { + return false; + } + } + + // all component are equal, and range is empty if any side is excluded + return range.Left.Components.back().Get<i32>() == 0 || range.Right.Components.back().Get<i32>() == 0; +} + +bool RangeCanMerge(const TExpandedRange& a, const TExpandedRange& b, const TRangeTypeInfo& typeInfo) { + // It is assumed that a <= b here + // < { > } + // a.Left b.Left a.Right b.Right + TExpandedRange intersection = { b.Left, a.Right }; + int cmp = typeInfo.BoundaryCompare->Compare(intersection.Left.Value, intersection.Right.Value); + if (cmp > 0) { + return false; + } + + const auto& lefts = intersection.Left.Components; + const auto& rights = intersection.Right.Components; + + bool leftIncluded = lefts.back().Get<i32>() != 0; + bool rightIncluded = rights.back().Get<i32>() != 0; + + for (size_t i = 0; i < lefts.size() - 1; i += 2) { + auto infCmp = typeInfo.ComponentsCompare[i]; + auto compCmp = typeInfo.ComponentsCompare[i + 1]; + + auto infCompareRes = infCmp->Compare(lefts[i], rights[i]); + Y_ENSURE(infCompareRes <= 0); + if (infCompareRes < 0) { + return true; + } + + auto componentCompareRes = compCmp->Compare(lefts[i + 1], rights[i + 1]); + Y_ENSURE(componentCompareRes <= 0); + if (componentCompareRes < 0) { + return true; + } + } + + return leftIncluded || rightIncluded; +} + +class TRangeComputeBase { +public: + TRangeComputeBase(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) + : TypeInfos(std::move(typeInfos)) + , Lists(std::move(lists)) + { + Y_ENSURE(Lists.size() == TypeInfos.size()); + Y_ENSURE(!Lists.empty()); + } +protected: + std::vector<TUnboxedValueQueue> ExpandLists(TComputationContext& ctx) const { + TUnboxedValueVector lists; + lists.reserve(Lists.size()); + for (auto& list : Lists) { + lists.emplace_back(list->GetValue(ctx)); + } + + std::vector<TUnboxedValueQueue> expandedLists; + for (size_t i = 0; i < lists.size(); ++i) { + expandedLists.emplace_back(); + TThresher<false>::DoForEachItem(lists[i], + [&] (NUdf::TUnboxedValue&& item) { + expandedLists.back().emplace_back(std::move(item)); + } + ); + NormalizeRanges(expandedLists.back(), TypeInfos[i]); + } + + return expandedLists; + } + +private: + template<typename TContainer> + static void NormalizeRanges(TContainer& ranges, const TRangeTypeInfo& typeInfo) { + auto rangeLess = [&](const TUnboxedValuePod& a, const TUnboxedValuePod& b) { + return typeInfo.RangeCompare->Less(a, b); + }; + + auto rangeEqual = [&](const TUnboxedValuePod& a, const TUnboxedValuePod& b) { + return typeInfo.RangeCompare->Compare(a, b) == 0; + }; + + for (size_t i = 1; i < ranges.size(); ++i) { + if (rangeLess(ranges[i], ranges[i - 1])) { + std::sort(ranges.begin(), ranges.end(), rangeLess); + break; + } + } + + ranges.erase( + std::remove_if(ranges.begin(), ranges.end(), + [&](const TUnboxedValue& range) { return RangeIsEmpty(ExpandRange(range), typeInfo); }), + ranges.end()); + ranges.erase(std::unique(ranges.begin(), ranges.end(), rangeEqual), ranges.end()); + } + +protected: + const TComputationNodePtrVector Lists; + const std::vector<TRangeTypeInfo> TypeInfos; +}; + +class TRangeUnionWrapper : public TMutableComputationNode<TRangeUnionWrapper>, public TRangeComputeBase { + typedef TMutableComputationNode<TRangeUnionWrapper> TBaseComputation; +public: + TRangeUnionWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) + : TBaseComputation(mutables) + , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + TUnboxedValueVector mergedLists; + auto expandedLists = ExpandLists(ctx); + while (!expandedLists.empty()) { + TMaybe<size_t> argMin; + bool haveEmpty = false; + for (size_t i = 0; i < expandedLists.size(); ++i) { + if (expandedLists[i].empty()) { + haveEmpty = true; + continue; + } + auto& current = expandedLists[i].front(); + if (!argMin || TypeInfos.front().RangeCompare->Less(current, expandedLists[*argMin].front())) { + argMin = i; + } + } + if (argMin) { + auto& from = expandedLists[*argMin]; + if (!RangeIsEmpty(ExpandRange(from.front()), TypeInfos.front())) { + mergedLists.emplace_back(std::move(from.front())); + } + from.pop_front(); + haveEmpty = haveEmpty || from.empty(); + } + + if (haveEmpty) { + expandedLists.erase( + std::remove_if(expandedLists.begin(), expandedLists.end(), [](const auto& list) { return list.empty(); }), + expandedLists.end()); + } + } + + TUnboxedValueVector unionList; + if (!mergedLists.empty()) { + unionList.push_back(mergedLists.front()); + auto current = ExpandRange(unionList.back()); + for (size_t i = 1; i < mergedLists.size(); ++i) { + auto toUnion = ExpandRange(mergedLists[i]); + if (RangeCanMerge(current, toUnion, TypeInfos.front())) { + current = { current.Left, Max(current.Right, toUnion.Right, TypeInfos.front().BoundaryCompare) }; + TUnboxedValueVector newValue = { current.Left.Value, current.Right.Value }; + unionList.back() = ctx.HolderFactory.VectorAsArray(newValue); + } else { + unionList.emplace_back(std::move(mergedLists[i])); + current = ExpandRange(unionList.back()); + } + } + } + + TDefaultListRepresentation res; + for (auto& item : unionList) { + res = res.Append(std::move(item)); + } + return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); + } + +private: + void RegisterDependencies() const final { + std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeUnionWrapper::DependsOn, this, std::placeholders::_1)); + } +}; + +class TRangeIntersectWrapper : public TMutableComputationNode<TRangeIntersectWrapper>, public TRangeComputeBase { + typedef TMutableComputationNode<TRangeIntersectWrapper> TBaseComputation; +public: + TRangeIntersectWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) + : TBaseComputation(mutables) + , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + TUnboxedValueVector mergedLists; + auto expandedLists = ExpandLists(ctx); + Y_ENSURE(!expandedLists.empty()); + TUnboxedValueQueue intersected = std::move(expandedLists.front()); + for (size_t i = 1; i < expandedLists.size(); ++i) { + DoIntersect(ctx, intersected, std::move(expandedLists[i])); + } + + TDefaultListRepresentation res; + for (auto& item : intersected) { + res = res.Append(std::move(item)); + } + return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); + } + +private: + void RegisterDependencies() const final { + std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeIntersectWrapper::DependsOn, this, std::placeholders::_1)); + } + + void DoIntersect(TComputationContext& ctx, TUnboxedValueQueue& current, TUnboxedValueQueue&& next) const { + TUnboxedValueQueue result; + auto cmp = TypeInfos.front().RangeCompare; + auto boundaryCmp = TypeInfos.front().BoundaryCompare; + while (!current.empty() && !next.empty()) { + TUnboxedValueQueue* minInput; + TUnboxedValueQueue* maxInput; + if (cmp->Less(current.front(), next.front())) { + minInput = ¤t; + maxInput = &next; + } else { + minInput = &next; + maxInput = ¤t; + } + + auto minRange = ExpandRange(minInput->front()); + auto maxRange = ExpandRange(maxInput->front()); + + TExpandedRange intersected; + intersected.Left = maxRange.Left; + intersected.Right = Min(minRange.Right, maxRange.Right, TypeInfos.front().BoundaryCompare); + if (!RangeIsEmpty(intersected, TypeInfos.front())) { + TUnboxedValueVector newValue = { intersected.Left.Value, intersected.Right.Value }; + result.push_back(ctx.HolderFactory.VectorAsArray(newValue)); + + if (boundaryCmp->Less(minRange.Right.Value, maxRange.Right.Value)) { + minInput->pop_front(); + } else { + maxInput->pop_front(); + } + } else { + minInput->pop_front(); + } + } + std::swap(current, result); + } +}; + +class TRangeMultiplyWrapper : public TMutableComputationNode<TRangeMultiplyWrapper>, public TRangeComputeBase { + typedef TMutableComputationNode<TRangeMultiplyWrapper> TBaseComputation; +public: + TRangeMultiplyWrapper(TComputationMutables& mutables, IComputationNode* limit, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) + : TBaseComputation(mutables) + , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) + , Limit(limit) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + const ui64 limit = Limit->GetValue(ctx).Get<ui64>(); + TUnboxedValueVector mergedLists; + auto expandedLists = ExpandLists(ctx); + Y_ENSURE(!expandedLists.empty()); + if (expandedLists.size() == 1 && expandedLists.front().size() > limit) { + return FullRange(ctx); + } + + TUnboxedValueQueue current = std::move(expandedLists.front()); + std::vector<ICompare::TPtr> currentComponentsCompare = TypeInfos.front().ComponentsCompare; + for (size_t i = 1; i < expandedLists.size(); ++i) { + if (expandedLists[i].empty()) { + return ctx.HolderFactory.GetEmptyContainer(); + } + if (!DoMultiply(ctx, limit, current, expandedLists[i], currentComponentsCompare, TypeInfos[i])) { + return FullRange(ctx); + } + } + + TDefaultListRepresentation res; + for (auto& item : current) { + res = res.Append(std::move(item)); + } + return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); + } + +private: + void RegisterDependencies() const final { + DependsOn(Limit); + std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeMultiplyWrapper::DependsOn, this, std::placeholders::_1)); + } + + bool DoMultiply(TComputationContext& ctx, ui64 limit, TUnboxedValueQueue& current, const TUnboxedValueQueue& next, + std::vector<ICompare::TPtr>& currentCmps, const TRangeTypeInfo& nextTypeInfo) const + { + TUnboxedValueQueue result; + Y_ENSURE(currentCmps.size() >= 3 && currentCmps.size() % 2 == 1); + size_t extraColumns = (nextTypeInfo.ComponentsCompare.size() - 1) / 2; + for (const auto& c : current) { + auto curr = ExpandRange(c); + if (RangeIsPoint(curr, currentCmps)) { + if (result.size() + next.size() > limit) { + return false; + } + for (const auto& n : next) { + result.push_back(Append(ctx, curr, ExpandRange(n))); + } + } else { + if (result.size() + 1 > limit) { + return false; + } + result.push_back(AppendInfs(ctx, curr, extraColumns)); + } + } + + ICompare::TPtr currentLast = currentCmps.back(); + currentCmps.pop_back(); + currentCmps.insert(currentCmps.end(), nextTypeInfo.ComponentsCompare.begin(), nextTypeInfo.ComponentsCompare.end()); + + std::swap(current, result); + return true; + } + + static bool RangeIsPoint(const TExpandedRange& range, const std::vector<ICompare::TPtr>& cmps) { + Y_ENSURE(range.Left.Components.size() == cmps.size()); + TUnboxedValue leftIncluded = range.Left.Components.back(); + TUnboxedValue rightIncluded = range.Right.Components.back(); + if (!leftIncluded.Get<i32>() || !rightIncluded.Get<i32>()) { + return false; + } + + bool allEqual = true; + for (size_t i = 0; allEqual && i < cmps.size() - 1; ++i) { + allEqual = allEqual && + cmps[i]->Compare(range.Left.Components[i], range.Right.Components[i]) == 0; + } + + return allEqual; + } + + static TUnboxedValuePod Append(TComputationContext& ctx, const TExpandedRange& first, const TExpandedRange& second) { + auto left = Append(ctx, first.Left, second.Left); + auto right = Append(ctx, first.Right, second.Right); + TUnboxedValueVector range = { left, right }; + return ctx.HolderFactory.VectorAsArray(range); + } + + static TUnboxedValuePod Append(TComputationContext& ctx, const TExpandedRangeBoundary& first, + const TExpandedRangeBoundary& second) + { + TUnboxedValueVector components(first.Components.begin(), first.Components.end() - 1); + components.insert(components.end(), second.Components.begin(), second.Components.end()); + if (second.Components.front().Get<i32>() != 0) { + // preserve original include/exclude flag when appending nulls (+-inf) + components.back() = first.Components.back(); + } + return ctx.HolderFactory.VectorAsArray(components); + } + + static TUnboxedValuePod AppendInfs(TComputationContext& ctx, const TExpandedRange& range, size_t count) { + auto left = AppendInfs(ctx, true, range.Left, count); + auto right = AppendInfs(ctx, false, range.Right, count); + TUnboxedValueVector newRange = { left, right }; + return ctx.HolderFactory.VectorAsArray(newRange); + } + + static TUnboxedValuePod AppendInfs(TComputationContext& ctx, bool isLeft, const TExpandedRangeBoundary& boundary, size_t count) { + Y_ENSURE(!boundary.Components.empty()); + TUnboxedValueVector components(boundary.Components.begin(), boundary.Components.end() - 1); + const bool hasPrefix = boundary.Components.size() > 1 && boundary.Components.front().Get<i32>() == 0; + const bool isIncluded = boundary.Components.back().Get<i32>() != 0; + for (size_t i = 0; i < count; ++i) { + components.push_back(TUnboxedValuePod(GetInfSign(hasPrefix, isIncluded, isLeft))); + components.emplace_back(); + } + components.push_back(boundary.Components.back()); + return ctx.HolderFactory.VectorAsArray(components); + } + + TUnboxedValuePod FullRange(TComputationContext& ctx) const { + size_t columnCount = 0; + for (const auto& ti : TypeInfos) { + Y_ENSURE(ti.Components.size() % 2 == 1); + columnCount += (ti.Components.size() - 1) / 2; + } + + TExpandedRange range; + range.Left.Components.push_back(TUnboxedValuePod(0)); + range.Right.Components.push_back(TUnboxedValuePod(0)); + TUnboxedValueVector result = { AppendInfs(ctx, range, columnCount) }; + return ctx.HolderFactory.VectorAsArray(result); + } + + IComputationNode* const Limit; +}; + + +class TRangeFinalizeWrapper : public TMutableComputationNode<TRangeFinalizeWrapper>, public TRangeComputeBase { + typedef TMutableComputationNode<TRangeFinalizeWrapper> TBaseComputation; +public: + TRangeFinalizeWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& lists, std::vector<TRangeTypeInfo>&& typeInfos) + : TBaseComputation(mutables) + , TRangeComputeBase(mutables, std::move(lists), std::move(typeInfos)) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + auto expandedLists = ExpandLists(ctx); + Y_ENSURE(expandedLists.size() == 1); + + TDefaultListRepresentation res; + for (auto& item : expandedLists.front()) { + auto range = ExpandRange(item); + if (CanConvertToPointRange(range, TypeInfos.front())) { + if (range.Left.Included) { + range.Right = range.Left; + } else { + range.Left = range.Right; + } + } + + auto left = ConvertFromInternal(range.Left.Components, ctx); + auto right = ConvertFromInternal(range.Right.Components, ctx); + + TUnboxedValueVector rangeVector = { left, right }; + res = res.Append(ctx.HolderFactory.VectorAsArray(rangeVector)); + } + return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); + } + +private: + void RegisterDependencies() const final { + std::for_each(Lists.cbegin(), Lists.cend(), std::bind(&TRangeFinalizeWrapper::DependsOn, this, std::placeholders::_1)); + } + + TUnboxedValue ConvertFromInternal(const TUnboxedValueVector& boundaryComponents, TComputationContext& ctx) const { + size_t compsSize = boundaryComponents.size(); + Y_ENSURE(compsSize >= 3); + Y_ENSURE(compsSize % 2 == 1); + + TUnboxedValueVector converted; + for (size_t i = 0; i < compsSize - 1; ++i) { + if (i % 2 == 1) { + converted.push_back(boundaryComponents[i]); + } + } + i32 included = boundaryComponents.back().Get<i32>(); + if (included != 0) { + included = 1; + } + converted.push_back(TUnboxedValuePod(included)); + return ctx.HolderFactory.VectorAsArray(converted); + } +}; + +enum ERangeOp { + RANGE_UNION, + RANGE_INTERSECT, + RANGE_MULTIPLY, + RANGE_FINALIZE, +}; + +IComputationNode* WrapRange(ERangeOp func, TCallable& callable, const TComputationNodeFactoryContext& ctx) { + TComputationNodePtrVector lists; + std::vector<TRangeTypeInfo> typeInfos; + size_t listsStart = 0; + if (func == RANGE_FINALIZE) { + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting single argument"); + } else if (func == RANGE_MULTIPLY) { + MKQL_ENSURE(callable.GetInputsCount() > 1, "Expecting at least two arguments"); + listsStart = 1; + auto limitType = callable.GetInput(0).GetStaticType(); + MKQL_ENSURE(limitType->IsData() && static_cast<TDataType*>(limitType)->GetSchemeType() == NUdf::TDataType<ui64>::Id, + "Expecting Uint64 as first argument"); + } else { + MKQL_ENSURE(callable.GetInputsCount() > 0, "Expecting at least one argument"); + } + + lists.reserve(callable.GetInputsCount()); + typeInfos.reserve(callable.GetInputsCount()); + for (ui32 i = listsStart; i < callable.GetInputsCount(); ++i) { + auto type = callable.GetInput(i).GetStaticType(); + MKQL_ENSURE(type->IsList(), "Expecting list as argument"); + auto rangeType = static_cast<TListType*>(type)->GetItemType(); + + if (func != RANGE_MULTIPLY) { + MKQL_ENSURE(type->IsSameType(*callable.GetInput(listsStart).GetStaticType()), "All arguments must be of same type"); + } + + lists.push_back(LocateNode(ctx.NodeLocator, callable, i)); + typeInfos.push_back(ExtractTypes(rangeType)); + } + + switch (func) { + case RANGE_UNION: + return new TRangeUnionWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); + case RANGE_INTERSECT: + return new TRangeIntersectWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); + case RANGE_MULTIPLY: { + auto limit = LocateNode(ctx.NodeLocator, callable, 0); + return new TRangeMultiplyWrapper(ctx.Mutables, limit, std::move(lists), std::move(typeInfos)); + } + case RANGE_FINALIZE: + return new TRangeFinalizeWrapper(ctx.Mutables, std::move(lists), std::move(typeInfos)); + default: + Y_ENSURE(!"Unknown callable"); + } +} + +class TRangeCreateWrapper : public TMutableComputationNode<TRangeCreateWrapper> { + typedef TMutableComputationNode<TRangeCreateWrapper> TBaseComputation; +public: + TRangeCreateWrapper(TComputationMutables& mutables, IComputationNode* list) + : TBaseComputation(mutables) + , List(list) + {} + + TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + TUnboxedValue list = List->GetValue(ctx); + + TDefaultListRepresentation res; + TThresher<false>::DoForEachItem(list, + [&] (NUdf::TUnboxedValue&& item) { + auto left = ConvertToInternal(item.GetElement(0), true, ctx); + auto right = ConvertToInternal(item.GetElement(1), false, ctx); + + TUnboxedValueVector rangeVector = { left, right }; + auto range = ctx.HolderFactory.VectorAsArray(rangeVector); + res = res.Append(std::move(range)); + } + ); + return ctx.HolderFactory.CreateDirectListHolder(std::move(res)); + } + +private: + void RegisterDependencies() const final { + DependsOn(List); + } + + TUnboxedValue ConvertToInternal(TUnboxedValue boundary, bool isLeft, TComputationContext& ctx) const { + auto elements = boundary.GetElements(); + auto elementsCount = boundary.GetListLength(); + + Y_ENSURE(elements); + Y_ENSURE(elementsCount >= 2); + + TUnboxedValueVector converted; + i32 included = elements[elementsCount - 1].Get<i32>(); + const auto hasPrefix = bool(elements[0]); + bool tail = false; + for (size_t i = 0; i < elementsCount - 1; ++i) { + i32 infValue; + tail = tail || !elements[i]; + if (elements[i]) { + MKQL_ENSURE(!tail, "Invalid boundary value - non null element follows null"); + infValue = 0; + } else { + infValue = GetInfSign(hasPrefix, included, isLeft); + } + converted.push_back(TUnboxedValuePod(infValue)); + converted.push_back(elements[i]); + } + included = included ? (isLeft ? -1 : 1) : 0; + converted.push_back(TUnboxedValuePod(included)); + return ctx.HolderFactory.VectorAsArray(converted); + } + + IComputationNode* const List; +}; + + +} // namespace + +IComputationNode* WrapRangeCreate(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); + + auto list = callable.GetInput(0); + + auto itemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); + MKQL_ENSURE(itemType->IsTuple(), "Expecting list of tuples"); + + auto tupleType = static_cast<TTupleType*>(itemType); + MKQL_ENSURE(tupleType->GetElementsCount() == 2, + "Expecting list ot 2-element tuples, got: " << tupleType->GetElementsCount() << " elements"); + + MKQL_ENSURE(tupleType->GetElementType(0)->IsSameType(*tupleType->GetElementType(1)), + "Expecting list ot 2-element tuples of same type"); + + MKQL_ENSURE(tupleType->GetElementType(0)->IsTuple(), + "Expecting range boundary to be tuple"); + + auto boundaryType = static_cast<TTupleType*>(tupleType->GetElementType(0)); + MKQL_ENSURE(boundaryType->GetElementsCount() >= 2, + "Range boundary should have at least 2 components, got: " << boundaryType->GetElementsCount()); + + return new TRangeCreateWrapper(ctx.Mutables, LocateNode(ctx.NodeLocator, callable, 0)); +} + +IComputationNode* WrapRangeUnion(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapRange(RANGE_UNION, callable, ctx); +} + +IComputationNode* WrapRangeIntersect(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapRange(RANGE_INTERSECT, callable, ctx); +} + +IComputationNode* WrapRangeMultiply(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapRange(RANGE_MULTIPLY, callable, ctx); +} + +IComputationNode* WrapRangeFinalize(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapRange(RANGE_FINALIZE, callable, ctx); +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_range.h b/ydb/library/yql/minikql/comp_nodes/mkql_range.h index 086b663afb..4603100b19 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_range.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_range.h @@ -1,14 +1,14 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapRangeCreate(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapRangeUnion(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapRangeIntersect(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapRangeMultiply(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapRangeFinalize(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapRangeCreate(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapRangeUnion(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapRangeIntersect(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapRangeMultiply(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapRangeFinalize(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp index 75ad9fe77b..bd5f29452c 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp @@ -24,24 +24,24 @@ public: NUdf::TUnboxedValue* itemsPtr = nullptr; const auto result = Cache.NewArray(ctx, Representations.size() - 1U, itemsPtr); - if (Representations.size() > 1) { - Y_VERIFY(itemsPtr); - if (const auto ptr = baseStruct.GetElements()) { - for (ui32 i = 0; i < Index; ++i) { - *itemsPtr++ = ptr[i]; - } - - for (ui32 i = Index + 1; i < Representations.size(); ++i) { - *itemsPtr++ = ptr[i]; - } - } else { - for (ui32 i = 0; i < Index; ++i) { - *itemsPtr++ = baseStruct.GetElement(i); - } - - for (ui32 i = Index + 1; i < Representations.size(); ++i) { - *itemsPtr++ = baseStruct.GetElement(i); - } + if (Representations.size() > 1) { + Y_VERIFY(itemsPtr); + if (const auto ptr = baseStruct.GetElements()) { + for (ui32 i = 0; i < Index; ++i) { + *itemsPtr++ = ptr[i]; + } + + for (ui32 i = Index + 1; i < Representations.size(); ++i) { + *itemsPtr++ = ptr[i]; + } + } else { + for (ui32 i = 0; i < Index; ++i) { + *itemsPtr++ = baseStruct.GetElement(i); + } + + for (ui32 i = Index + 1; i < Representations.size(); ++i) { + *itemsPtr++ = baseStruct.GetElement(i); + } } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp index fa10b988ef..77c334f282 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp @@ -1,227 +1,227 @@ -#include "mkql_round.h" +#include "mkql_round.h" #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/computation/presort.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <ydb/library/yql/public/udf/udf_data_type.h> #include <ydb/library/yql/utils/utf8.h> - -#include <algorithm> - -namespace NKikimr { -namespace NMiniKQL { - -using namespace NYql::NUdf; - -namespace { - -template<typename From, typename To> -class TRoundIntegralWrapper : public TMutableComputationNode<TRoundIntegralWrapper<From, To>> { - using TSelf = TRoundIntegralWrapper<From, To>; - using TBase = TMutableComputationNode<TSelf>; - typedef TBase TBaseComputation; -public: - TRoundIntegralWrapper(TComputationMutables& mutables, IComputationNode* source, bool down) - : TBaseComputation(mutables) - , Source(source) - , Down(down) - { - } - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - auto value = Source->GetValue(ctx).Get<From>(); - auto toMin = std::numeric_limits<To>::min(); - auto toMax = std::numeric_limits<To>::max(); - - if constexpr (std::is_signed<From>::value && std::is_unsigned<To>::value) { - if (value < 0) { - return Down ? TUnboxedValuePod() : TUnboxedValuePod(toMin); - } - - if (value > toMax) { - return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); - } - - return TUnboxedValuePod(static_cast<To>(value)); - } - - if constexpr (std::is_unsigned<From>::value && std::is_signed<To>::value) { - if (value > toMax) { - return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); - } - - return TUnboxedValuePod(static_cast<To>(value)); - } - - if (value < toMin) { - return Down ? TUnboxedValuePod() : TUnboxedValuePod(toMin); - } - - if (value > toMax) { - return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); - } - - return TUnboxedValuePod(static_cast<To>(value)); - } - -private: - void RegisterDependencies() const final { - this->DependsOn(Source); - } - - IComputationNode* const Source; - const bool Down; -}; - -class TRoundDateTypeWrapper : public TMutableComputationNode<TRoundDateTypeWrapper> { - using TSelf = TRoundDateTypeWrapper; - using TBase = TMutableComputationNode<TSelf>; - typedef TBase TBaseComputation; -public: - TRoundDateTypeWrapper(TComputationMutables& mutables, IComputationNode* source, bool down, EDataSlot from, EDataSlot to) - : TBaseComputation(mutables) - , Source(source) - , Down(down) - , From(from) - , To(to) - { - } - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - constexpr ui64 usInDay = 86400000000ull; - constexpr ui32 usInSec = 1000000u; - - ui64 us; - if (From == EDataSlot::Timestamp) { - us = Source->GetValue(ctx).Get<NUdf::TDataType<TTimestamp>::TLayout>(); - } else { - Y_ENSURE(From == EDataSlot::Datetime); - us = Source->GetValue(ctx).Get<NUdf::TDataType<TDatetime>::TLayout>(); - us *= usInSec; - } - - TUnboxedValuePod result; - if (To == EDataSlot::Date) { - NUdf::TDataType<TTimestamp>::TLayout rounded = (us + (Down ? 0 : (usInDay - 1u))) / usInDay; - if (rounded >= MAX_DATE) { - return {}; - } - result = TUnboxedValuePod(rounded); - } else { - Y_ENSURE(To == EDataSlot::Datetime); - NUdf::TDataType<TDatetime>::TLayout rounded = (us + (Down ? 0 : (usInSec - 1u))) / usInSec; - if (rounded >= MAX_DATETIME) { - return {}; - } - result = TUnboxedValuePod(rounded); - } - - return result; - } - -private: - void RegisterDependencies() const final { - this->DependsOn(Source); - } - - IComputationNode* const Source; - const bool Down; - const EDataSlot From; - const EDataSlot To; -}; - -class TRoundStringWrapper : public TMutableComputationNode<TRoundStringWrapper> { - using TSelf = TRoundStringWrapper; - using TBase = TMutableComputationNode<TSelf>; - typedef TBase TBaseComputation; -public: - TRoundStringWrapper(TComputationMutables& mutables, IComputationNode* source, bool down) - : TBaseComputation(mutables) - , Source(source) - , Down(down) - { - } - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - TUnboxedValue input = Source->GetValue(ctx); - auto output = NYql::RoundToNearestValidUtf8(input.AsStringRef(), Down); - if (!output) { - return {}; - } - return MakeString(*output); - } - -private: - void RegisterDependencies() const final { - this->DependsOn(Source); - } - - IComputationNode* const Source; - const bool Down; -}; - -template<typename From> -IComputationNode* FromIntegral(TComputationMutables& mutables, IComputationNode* source, bool down, EDataSlot target) { - switch (target) { - case EDataSlot::Int8: return new TRoundIntegralWrapper<From, i8>(mutables, source, down); - case EDataSlot::Uint8: return new TRoundIntegralWrapper<From, ui8>(mutables, source, down); - case EDataSlot::Int16: return new TRoundIntegralWrapper<From, i16>(mutables, source, down); - case EDataSlot::Uint16: return new TRoundIntegralWrapper<From, ui16>(mutables, source, down); - case EDataSlot::Int32: return new TRoundIntegralWrapper<From, i32>(mutables, source, down); - case EDataSlot::Uint32: return new TRoundIntegralWrapper<From, ui32>(mutables, source, down); - case EDataSlot::Int64: return new TRoundIntegralWrapper<From, i64>(mutables, source, down); - case EDataSlot::Uint64: return new TRoundIntegralWrapper<From, ui64>(mutables, source, down); - default: Y_ENSURE(false, "Unsupported integral rounding"); - } - return nullptr; -} - - -} // namespace - -IComputationNode* WrapRound(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); - - auto type = callable.GetInput(0).GetStaticType(); - MKQL_ENSURE(type->IsData(), "Expecting data as argument"); - - auto returnType = callable.GetType()->GetReturnType(); - MKQL_ENSURE(returnType->IsOptional(), "Expecting optional as return type"); - - auto targetType = static_cast<TOptionalType*>(returnType)->GetItemType(); - MKQL_ENSURE(targetType->IsData(), "Expecting Data as target type"); - - auto from = GetDataSlot(static_cast<TDataType*>(type)->GetSchemeType()); - auto to = GetDataSlot(static_cast<TDataType*>(targetType)->GetSchemeType()); - - bool down = callable.GetType()->GetName() == "RoundDown"; - auto source = LocateNode(ctx.NodeLocator, callable, 0); - - switch (from) { - case EDataSlot::Int8: return FromIntegral<i8>(ctx.Mutables, source, down, to); - case EDataSlot::Uint8: return FromIntegral<ui8>(ctx.Mutables, source, down, to); - case EDataSlot::Int16: return FromIntegral<i16>(ctx.Mutables, source, down, to); - case EDataSlot::Uint16: return FromIntegral<ui16>(ctx.Mutables, source, down, to); - case EDataSlot::Int32: return FromIntegral<i32>(ctx.Mutables, source, down, to); - case EDataSlot::Uint32: return FromIntegral<ui32>(ctx.Mutables, source, down, to); - case EDataSlot::Int64: return FromIntegral<i64>(ctx.Mutables, source, down, to); - case EDataSlot::Uint64: return FromIntegral<ui64>(ctx.Mutables, source, down, to); - case EDataSlot::Datetime: - case EDataSlot::Timestamp: - Y_ENSURE(GetDataTypeInfo(to).Features & DateType); - return new TRoundDateTypeWrapper(ctx.Mutables, source, down, from, to); - case EDataSlot::String: - Y_ENSURE(to == EDataSlot::Utf8); - return new TRoundStringWrapper(ctx.Mutables, source, down); - default: - Y_ENSURE(false, - "Unsupported rounding from " << GetDataTypeInfo(from).Name << " to " << GetDataTypeInfo(to).Name); - } - return nullptr; -} - -} -} + +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +using namespace NYql::NUdf; + +namespace { + +template<typename From, typename To> +class TRoundIntegralWrapper : public TMutableComputationNode<TRoundIntegralWrapper<From, To>> { + using TSelf = TRoundIntegralWrapper<From, To>; + using TBase = TMutableComputationNode<TSelf>; + typedef TBase TBaseComputation; +public: + TRoundIntegralWrapper(TComputationMutables& mutables, IComputationNode* source, bool down) + : TBaseComputation(mutables) + , Source(source) + , Down(down) + { + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + auto value = Source->GetValue(ctx).Get<From>(); + auto toMin = std::numeric_limits<To>::min(); + auto toMax = std::numeric_limits<To>::max(); + + if constexpr (std::is_signed<From>::value && std::is_unsigned<To>::value) { + if (value < 0) { + return Down ? TUnboxedValuePod() : TUnboxedValuePod(toMin); + } + + if (value > toMax) { + return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); + } + + return TUnboxedValuePod(static_cast<To>(value)); + } + + if constexpr (std::is_unsigned<From>::value && std::is_signed<To>::value) { + if (value > toMax) { + return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); + } + + return TUnboxedValuePod(static_cast<To>(value)); + } + + if (value < toMin) { + return Down ? TUnboxedValuePod() : TUnboxedValuePod(toMin); + } + + if (value > toMax) { + return Down ? TUnboxedValuePod(toMax) : TUnboxedValuePod(); + } + + return TUnboxedValuePod(static_cast<To>(value)); + } + +private: + void RegisterDependencies() const final { + this->DependsOn(Source); + } + + IComputationNode* const Source; + const bool Down; +}; + +class TRoundDateTypeWrapper : public TMutableComputationNode<TRoundDateTypeWrapper> { + using TSelf = TRoundDateTypeWrapper; + using TBase = TMutableComputationNode<TSelf>; + typedef TBase TBaseComputation; +public: + TRoundDateTypeWrapper(TComputationMutables& mutables, IComputationNode* source, bool down, EDataSlot from, EDataSlot to) + : TBaseComputation(mutables) + , Source(source) + , Down(down) + , From(from) + , To(to) + { + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + constexpr ui64 usInDay = 86400000000ull; + constexpr ui32 usInSec = 1000000u; + + ui64 us; + if (From == EDataSlot::Timestamp) { + us = Source->GetValue(ctx).Get<NUdf::TDataType<TTimestamp>::TLayout>(); + } else { + Y_ENSURE(From == EDataSlot::Datetime); + us = Source->GetValue(ctx).Get<NUdf::TDataType<TDatetime>::TLayout>(); + us *= usInSec; + } + + TUnboxedValuePod result; + if (To == EDataSlot::Date) { + NUdf::TDataType<TTimestamp>::TLayout rounded = (us + (Down ? 0 : (usInDay - 1u))) / usInDay; + if (rounded >= MAX_DATE) { + return {}; + } + result = TUnboxedValuePod(rounded); + } else { + Y_ENSURE(To == EDataSlot::Datetime); + NUdf::TDataType<TDatetime>::TLayout rounded = (us + (Down ? 0 : (usInSec - 1u))) / usInSec; + if (rounded >= MAX_DATETIME) { + return {}; + } + result = TUnboxedValuePod(rounded); + } + + return result; + } + +private: + void RegisterDependencies() const final { + this->DependsOn(Source); + } + + IComputationNode* const Source; + const bool Down; + const EDataSlot From; + const EDataSlot To; +}; + +class TRoundStringWrapper : public TMutableComputationNode<TRoundStringWrapper> { + using TSelf = TRoundStringWrapper; + using TBase = TMutableComputationNode<TSelf>; + typedef TBase TBaseComputation; +public: + TRoundStringWrapper(TComputationMutables& mutables, IComputationNode* source, bool down) + : TBaseComputation(mutables) + , Source(source) + , Down(down) + { + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + TUnboxedValue input = Source->GetValue(ctx); + auto output = NYql::RoundToNearestValidUtf8(input.AsStringRef(), Down); + if (!output) { + return {}; + } + return MakeString(*output); + } + +private: + void RegisterDependencies() const final { + this->DependsOn(Source); + } + + IComputationNode* const Source; + const bool Down; +}; + +template<typename From> +IComputationNode* FromIntegral(TComputationMutables& mutables, IComputationNode* source, bool down, EDataSlot target) { + switch (target) { + case EDataSlot::Int8: return new TRoundIntegralWrapper<From, i8>(mutables, source, down); + case EDataSlot::Uint8: return new TRoundIntegralWrapper<From, ui8>(mutables, source, down); + case EDataSlot::Int16: return new TRoundIntegralWrapper<From, i16>(mutables, source, down); + case EDataSlot::Uint16: return new TRoundIntegralWrapper<From, ui16>(mutables, source, down); + case EDataSlot::Int32: return new TRoundIntegralWrapper<From, i32>(mutables, source, down); + case EDataSlot::Uint32: return new TRoundIntegralWrapper<From, ui32>(mutables, source, down); + case EDataSlot::Int64: return new TRoundIntegralWrapper<From, i64>(mutables, source, down); + case EDataSlot::Uint64: return new TRoundIntegralWrapper<From, ui64>(mutables, source, down); + default: Y_ENSURE(false, "Unsupported integral rounding"); + } + return nullptr; +} + + +} // namespace + +IComputationNode* WrapRound(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expecting exactly one argument"); + + auto type = callable.GetInput(0).GetStaticType(); + MKQL_ENSURE(type->IsData(), "Expecting data as argument"); + + auto returnType = callable.GetType()->GetReturnType(); + MKQL_ENSURE(returnType->IsOptional(), "Expecting optional as return type"); + + auto targetType = static_cast<TOptionalType*>(returnType)->GetItemType(); + MKQL_ENSURE(targetType->IsData(), "Expecting Data as target type"); + + auto from = GetDataSlot(static_cast<TDataType*>(type)->GetSchemeType()); + auto to = GetDataSlot(static_cast<TDataType*>(targetType)->GetSchemeType()); + + bool down = callable.GetType()->GetName() == "RoundDown"; + auto source = LocateNode(ctx.NodeLocator, callable, 0); + + switch (from) { + case EDataSlot::Int8: return FromIntegral<i8>(ctx.Mutables, source, down, to); + case EDataSlot::Uint8: return FromIntegral<ui8>(ctx.Mutables, source, down, to); + case EDataSlot::Int16: return FromIntegral<i16>(ctx.Mutables, source, down, to); + case EDataSlot::Uint16: return FromIntegral<ui16>(ctx.Mutables, source, down, to); + case EDataSlot::Int32: return FromIntegral<i32>(ctx.Mutables, source, down, to); + case EDataSlot::Uint32: return FromIntegral<ui32>(ctx.Mutables, source, down, to); + case EDataSlot::Int64: return FromIntegral<i64>(ctx.Mutables, source, down, to); + case EDataSlot::Uint64: return FromIntegral<ui64>(ctx.Mutables, source, down, to); + case EDataSlot::Datetime: + case EDataSlot::Timestamp: + Y_ENSURE(GetDataTypeInfo(to).Features & DateType); + return new TRoundDateTypeWrapper(ctx.Mutables, source, down, from, to); + case EDataSlot::String: + Y_ENSURE(to == EDataSlot::Utf8); + return new TRoundStringWrapper(ctx.Mutables, source, down); + default: + Y_ENSURE(false, + "Unsupported rounding from " << GetDataTypeInfo(from).Name << " to " << GetDataTypeInfo(to).Name); + } + return nullptr; +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_round.h b/ydb/library/yql/minikql/comp_nodes/mkql_round.h index aeab011feb..e704077ba2 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_round.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_round.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapRound(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapRound(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.h b/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.h index ca9b1030d9..282349756b 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.h @@ -1,8 +1,8 @@ #pragma once #include <ydb/library/yql/utils/yql_panic.h> - + #include <util/system/yassert.h> -#include <util/generic/maybe.h> +#include <util/generic/maybe.h> #include <vector> namespace NKikimr { @@ -11,35 +11,35 @@ namespace NMiniKQL { template<class T> class TSafeCircularBuffer { public: - TSafeCircularBuffer(TMaybe<size_t> size, T emptyValue, size_t initSize = 0) - : Buffer(size ? *size : initSize, emptyValue) + TSafeCircularBuffer(TMaybe<size_t> size, T emptyValue, size_t initSize = 0) + : Buffer(size ? *size : initSize, emptyValue) , EmptyValue(emptyValue) - , Unbounded(!size.Defined()) - , Count(initSize) + , Unbounded(!size.Defined()) + , Count(initSize) { - if (!Unbounded) { - Y_VERIFY(initSize <= *size); - } + if (!Unbounded) { + Y_VERIFY(initSize <= *size); + } } - bool IsUnbounded() const { - return Unbounded; + bool IsUnbounded() const { + return Unbounded; } void PushBack(T&& data) { - if (Unbounded) { - Y_VERIFY(Head + Count == Size()); - Buffer.emplace_back(std::move(data)); - } else { - YQL_ENSURE(!IsFull()); - Buffer[RealIndex(Head + Count)] = std::move(data); - } - Count++; - MutationCount++; + if (Unbounded) { + Y_VERIFY(Head + Count == Size()); + Buffer.emplace_back(std::move(data)); + } else { + YQL_ENSURE(!IsFull()); + Buffer[RealIndex(Head + Count)] = std::move(data); + } + Count++; + MutationCount++; } const T& Get(size_t index) const { - if (index < Count) { + if (index < Count) { return Buffer[RealIndex(Head + index)]; } else { // Circular buffer out of bounds @@ -48,28 +48,28 @@ public: } void PopFront() { - if (!Count) { + if (!Count) { // Circular buffer not have elements for pop, no elements, no problem } else { Buffer[Head] = EmptyValue; - Head = RealIndex(Head+1); - Count--; + Head = RealIndex(Head+1); + Count--; } - MutationCount++; + MutationCount++; } - size_t Size() const { + size_t Size() const { return Buffer.size(); } size_t UsedSize() const { - return Count; + return Count; + } + + ui64 Generation() const { + return MutationCount; } - ui64 Generation() const { - return MutationCount; - } - void Clean() { const auto usedSize = UsedSize(); for (size_t index = 0; index < usedSize; ++index) { @@ -78,31 +78,31 @@ public: } void Clear() { - Head = Count = 0; + Head = Count = 0; Buffer.clear(); Buffer.shrink_to_fit(); } -private: - bool IsFull() const { - if (Unbounded) { - return false; - } - return UsedSize() == Size(); +private: + bool IsFull() const { + if (Unbounded) { + return false; + } + return UsedSize() == Size(); + } + + size_t RealIndex(size_t index) const { + auto size = Size(); + Y_VERIFY(size); + return index % size; } - size_t RealIndex(size_t index) const { - auto size = Size(); - Y_VERIFY(size); - return index % size; - } - std::vector<T> Buffer; const T EmptyValue; - const bool Unbounded; + const bool Unbounded; size_t Head = 0; - size_t Count; - ui64 MutationCount = 0; + size_t Count; + ui64 MutationCount = 0; }; } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp index 916cc86fc2..b34947b3c5 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp @@ -1,50 +1,50 @@ -#include "mkql_seq.h" - +#include "mkql_seq.h" + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - -class TSeqWrapper : public TMutableComputationNode<TSeqWrapper> { - typedef TMutableComputationNode<TSeqWrapper> TBaseComputation; -public: - TSeqWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& args) - : TBaseComputation(mutables) - , Args(std::move(args)) - {} - - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - for (size_t i = 0; i + 1 < Args.size(); ++i) { - Args[i]->GetValue(ctx); - } - - auto value = Args.back()->GetValue(ctx); - return value.Release(); - } - -private: - void RegisterDependencies() const final { - std::for_each(Args.cbegin(), Args.cend(), std::bind(&TSeqWrapper::DependsOn, this, std::placeholders::_1)); - } - - const TComputationNodePtrVector Args; -}; - -} - -IComputationNode* WrapSeq(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() >= 1, "Seq: Expected at least one argument"); - - TComputationNodePtrVector args; - args.reserve(callable.GetInputsCount()); - for (ui32 i = 0; i < callable.GetInputsCount(); ++i) { - args.push_back(LocateNode(ctx.NodeLocator, callable, i)); - } - - return new TSeqWrapper(ctx.Mutables, std::move(args)); -} - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +class TSeqWrapper : public TMutableComputationNode<TSeqWrapper> { + typedef TMutableComputationNode<TSeqWrapper> TBaseComputation; +public: + TSeqWrapper(TComputationMutables& mutables, TComputationNodePtrVector&& args) + : TBaseComputation(mutables) + , Args(std::move(args)) + {} + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + for (size_t i = 0; i + 1 < Args.size(); ++i) { + Args[i]->GetValue(ctx); + } + + auto value = Args.back()->GetValue(ctx); + return value.Release(); + } + +private: + void RegisterDependencies() const final { + std::for_each(Args.cbegin(), Args.cend(), std::bind(&TSeqWrapper::DependsOn, this, std::placeholders::_1)); + } + + const TComputationNodePtrVector Args; +}; + +} + +IComputationNode* WrapSeq(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() >= 1, "Seq: Expected at least one argument"); + + TComputationNodePtrVector args; + args.reserve(callable.GetInputsCount()); + for (ui32 i = 0; i < callable.GetInputsCount(); ++i) { + args.push_back(LocateNode(ctx.NodeLocator, callable, i)); + } + + return new TSeqWrapper(ctx.Mutables, std::move(args)); +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_seq.h b/ydb/library/yql/minikql/comp_nodes/mkql_seq.h index 1395f391d6..cdbf70524a 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_seq.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_seq.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapSeq(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapSeq(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp index f46a750000..facb20ba54 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp @@ -61,7 +61,7 @@ public: return list; } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp index 2d31f48d64..4571141d80 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp @@ -9,8 +9,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { static const TStatKey Switch_FlushesCount("Switch_FlushesCount", true); @@ -169,7 +169,7 @@ public: return childRes.Release(); } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN private: @@ -283,7 +283,7 @@ private: public: Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { for (ui32 i = 0U; i < Handlers.size(); ++i) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(Handlers[i].Item)->SetValueGetter(GenerateHandler(i, ctx.Codegen)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Handlers[i].Item)->SetValueGetter(GenerateHandler(i, ctx.Codegen)); } auto& context = ctx.Codegen->GetContext(); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp index 0c0ff12969..2e299e487d 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp @@ -17,8 +17,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class THashedMultiMapAccumulator { @@ -72,10 +72,10 @@ public: return value; }; - ui64 start = 0; - ui64 finish = pair.second.size(); + ui64 start = 0; + ui64 finish = pair.second.size(); auto payloadList = CreateOwningVectorListAdapter(std::move(pair.second), itemFactory, - start, finish, false, Ctx.HolderFactory.GetMemInfo()); + start, finish, false, Ctx.HolderFactory.GetMemInfo()); targetMap.emplace(pair.first, std::move(payloadList)); } @@ -170,10 +170,10 @@ public: return value; }; - ui64 start = 0; - ui64 finish = pair.second.size(); + ui64 start = 0; + ui64 finish = pair.second.size(); auto payloadList = CreateOwningVectorListAdapter(std::move(pair.second), itemFactory, - start, finish, false, + start, finish, false, Ctx.HolderFactory.GetMemInfo()); targetMap.emplace(NUdf::TUnboxedValuePod(pair.first), std::move(payloadList)); @@ -788,14 +788,14 @@ public: statePtr->Insert(Key->GetValue(ctx).Release()); } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { auto& context = ctx.Codegen->GetContext(); const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(Item); - MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node."); + MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node."); const auto valueType = Type::getInt128Ty(context); const auto structPtrType = PointerType::getUnqual(StructType::get(context)); @@ -976,7 +976,7 @@ public: } } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { @@ -1042,7 +1042,7 @@ public: if (!PasstroughKey) { for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); } const auto key = PasstroughKey ? getres.second[*PasstroughKey](ctx, block) : GetNodeValue(Key, ctx, block); @@ -1279,14 +1279,14 @@ public: statePtr->Insert(Key->GetValue(ctx).Release(), Payload->GetValue(ctx).Release()); } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { auto& context = ctx.Codegen->GetContext(); const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(Item); - MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node."); + MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node."); const auto valueType = Type::getInt128Ty(context); const auto structPtrType = PointerType::getUnqual(StructType::get(context)); @@ -1476,7 +1476,7 @@ public: } } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { @@ -1542,7 +1542,7 @@ public: if (!(PasstroughKey && PasstroughPayload)) { for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); } const auto key = PasstroughKey ? getres.second[*PasstroughKey](ctx, block) : GetNodeValue(Key, ctx, block); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp index 094e4c6d7a..76adbc982b 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp @@ -46,7 +46,7 @@ public: block = var; const auto codegenArg = dynamic_cast<ICodegeneratorExternalNode*>(Args[i]); - MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); + MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); codegenArg->CreateSetValue(ctx, block, unpack.second); const auto item = GetNodeValue(NewNodes[i], ctx, block); @@ -121,7 +121,7 @@ public: block = var; const auto codegenArg = dynamic_cast<ICodegeneratorExternalNode*>(Args[i]); - MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); + MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); codegenArg->CreateSetValue(ctx, block, unpack.second); BranchInst::Create(work, block); } @@ -216,7 +216,7 @@ public: block = var; const auto codegenArg = dynamic_cast<ICodegeneratorExternalNode*>(Args[i]); - MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); + MKQL_ENSURE(codegenArg, "Arg must be codegenerator node."); codegenArg->CreateSetValue(ctx, block, unpack.second); BranchInst::Create(work, block); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp index 417f48e800..00ef832c57 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp @@ -8,8 +8,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class TWideChain1MapWrapper : public TStatefulWideFlowCodegeneratorNode<TWideChain1MapWrapper> { @@ -53,7 +53,7 @@ public: block = good; for (auto i = 0U; i < Inputs.size(); ++i) if (Inputs[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Inputs[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Inputs[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); const auto init = BasicBlock::Create(context, "init", ctx.Func); const auto next = BasicBlock::Create(context, "next", ctx.Func); @@ -67,7 +67,7 @@ public: if (Outputs[i]->GetDependencesCount() > 0U || OutputsOnUpdate[i]) { const auto& map = InitOnInputs[i]; const auto value = map ? getres.second[*map](ctx, block) : GetNodeValue(InitItems[i], ctx, block); - EnsureDynamicCast<ICodegeneratorExternalNode*>(Outputs[i])->CreateSetValue(ctx, block, value); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Outputs[i])->CreateSetValue(ctx, block, value); } } @@ -86,7 +86,7 @@ public: for (auto i = 0U; i < outputs.size(); ++i) if (const auto out = outputs[i]) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Outputs[i])->CreateSetValue(ctx, block, out); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Outputs[i])->CreateSetValue(ctx, block, out); BranchInst::Create(done, block); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp index 316eac89db..adb3a65dc4 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp @@ -7,8 +7,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { using namespace std::placeholders; @@ -161,7 +161,7 @@ private: std::vector<Value*> items(ItemArgs.size(), nullptr); for (ui32 i = 0U; i < items.size(); ++i) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); } const auto chop = SwitchItem ? items[*SwitchItem] : GetNodeValue(Chop, ctx, block); @@ -186,7 +186,7 @@ private: } public: TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { - EnsureDynamicCast<IWideFlowProxyCodegeneratorNode*>(Input)->SetGenerator(std::bind(&TWideChopperWrapper::DoGenGetValuesInput, this, _1, _2)); + EnsureDynamicCast<IWideFlowProxyCodegeneratorNode*>(Input)->SetGenerator(std::bind(&TWideChopperWrapper::DoGenGetValuesInput, this, _1, _2)); auto& context = ctx.Codegen->GetContext(); @@ -220,14 +220,14 @@ public: std::vector<Value*> items(ItemArgs.size(), nullptr); for (ui32 i = 0U; i < items.size(); ++i) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getfirst.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getfirst.second[i](ctx, block)); } for (ui32 i = 0U; i < Keys.size(); ++i) { if (KeyArgs[i]->GetDependencesCount() > 0U) { const auto map = KeysOnItems[i]; const auto key = map ? items[*map] : GetNodeValue(Keys[i], ctx, block); - EnsureDynamicCast<ICodegeneratorExternalNode*>(KeyArgs[i])->CreateSetValue(ctx, block, key); + EnsureDynamicCast<ICodegeneratorExternalNode*>(KeyArgs[i])->CreateSetValue(ctx, block, key); } } @@ -249,7 +249,7 @@ public: block = part; - EnsureDynamicCast<IWideFlowProxyCodegeneratorNode*>(Input)->CreateInvalidate(ctx, block); + EnsureDynamicCast<IWideFlowProxyCodegeneratorNode*>(Input)->CreateInvalidate(ctx, block); result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::Finish)), block); @@ -272,7 +272,7 @@ public: std::vector<Value*> items(ItemArgs.size(), nullptr); for (ui32 i = 0U; i < items.size(); ++i) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getnext.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(ItemArgs[i])->CreateSetValue(ctx, block, items[i] = getnext.second[i](ctx, block)); } const auto chop = SwitchItem ? items[*SwitchItem] : GetNodeValue(Chop, ctx, block); @@ -288,7 +288,7 @@ public: if (KeyArgs[i]->GetDependencesCount() > 0U) { const auto map = KeysOnItems[i]; const auto key = GetNodeValue(Keys[i], ctx, block); - EnsureDynamicCast<ICodegeneratorExternalNode*>(KeyArgs[i])->CreateSetValue(ctx, block, key); + EnsureDynamicCast<ICodegeneratorExternalNode*>(KeyArgs[i])->CreateSetValue(ctx, block, key); } } @@ -362,9 +362,9 @@ IComputationNode* WrapWideChopper(TCallable& callable, const TComputationNodeFac std::generate_n(std::back_inserter(keyArgs), keysSize, [&](){ return LocateExternalNode(ctx.NodeLocator, callable, ++index); } ); if (const auto wide = dynamic_cast<IComputationWideFlowNode*>(flow)) { - return new TWideChopperWrapper(ctx.Mutables, wide, std::move(itemArgs), std::move(keys), std::move(keyArgs), switchResult, - EnsureDynamicCast<IComputationWideFlowProxyNode*>(input), - EnsureDynamicCast<IComputationWideFlowNode*>(output)); + return new TWideChopperWrapper(ctx.Mutables, wide, std::move(itemArgs), std::move(keys), std::move(keyArgs), switchResult, + EnsureDynamicCast<IComputationWideFlowProxyNode*>(input), + EnsureDynamicCast<IComputationWideFlowNode*>(output)); } THROW yexception() << "Expected wide flow."; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp index f2467a14d8..1fe1734a15 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp @@ -12,8 +12,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + extern TStatKey Combine_FlushesCount; extern TStatKey Combine_MaxRowsCount; @@ -257,7 +257,7 @@ public: return EFetchResult::One; } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } #ifndef MKQL_DISABLE_CODEGEN ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const { @@ -368,7 +368,7 @@ public: std::vector<Value*> items(Nodes.ItemNodes.size(), nullptr); for (ui32 i = 0U; i < items.size(); ++i) { if (Nodes.ItemNodes[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.ItemNodes[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.ItemNodes[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); else if (Nodes.PasstroughtItems[i]) items[i] = getres.second[i](ctx, block); } @@ -390,7 +390,7 @@ public: } if (Nodes.KeyNodes[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.KeyNodes[i])->CreateSetValue(ctx, block, key); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.KeyNodes[i])->CreateSetValue(ctx, block, key); new StoreInst(key, keyPtr, block); } @@ -448,10 +448,10 @@ public: if (hasDependency || i != *map) { stored[i] = new LoadInst(pointers[i], (TString("state_") += ToString(i)).c_str(), block); if (hasDependency) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.StateNodes[i])->CreateSetValue(ctx, block, stored[i]); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.StateNodes[i])->CreateSetValue(ctx, block, stored[i]); } } else if (hasDependency) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.StateNodes[i])->CreateSetValue(ctx, block, pointers[i]); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.StateNodes[i])->CreateSetValue(ctx, block, pointers[i]); } else { ValueUnRef(Nodes.StateNodes[i]->GetRepresentation(), pointers[i], ctx, block); } @@ -520,7 +520,7 @@ public: for (ui32 i = 0U; i < Nodes.FinishNodes.size(); ++i) { const auto ptr = GetElementPtrInst::CreateInBounds(out, {ConstantInt::get(Type::getInt32Ty(context), i)}, (TString("out_key_") += ToString(i)).c_str(), block); if (Nodes.FinishNodes[i]->GetDependencesCount() > 0 || Nodes.ItemsOnResult[i]) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.FinishNodes[i])->CreateSetValue(ctx, block, ptr); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Nodes.FinishNodes[i])->CreateSetValue(ctx, block, ptr); else ValueUnRef(Nodes.FinishNodes[i]->GetRepresentation(), ptr, ctx, block); } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp index 11b82715e8..6473ad4b44 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp @@ -9,8 +9,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { template <bool Interruptable> @@ -121,7 +121,7 @@ public: new StoreInst(GetFalse(context), statePtr, block); for (ui32 i = 0U; i < State.size(); ++i) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(State[i])->CreateSetValue(ctx, block, GetNodeValue(InitState[i], ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(State[i])->CreateSetValue(ctx, block, GetNodeValue(InitState[i], ctx, block)); } empty->addIncoming(ConstantInt::getFalse(context), block); @@ -140,7 +140,7 @@ public: std::vector<Value*> items(Items.size(), nullptr); for (ui32 i = 0U; i < items.size(); ++i) { if (Items[i]->GetDependencesCount() > 0U || ItemsOnInit[i]) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, items[i] = getres.second[i](ctx, block)); else if (ItemsOnUpdate[i] || SwitchItem && i == *SwitchItem) items[i] = getres.second[i](ctx, block); } @@ -181,7 +181,7 @@ public: for (ui32 i = 0U; i < updates.size(); ++i) { if (const auto s = updates[i]) - EnsureDynamicCast<ICodegeneratorExternalNode*>(State[i])->CreateSetValue(ctx, block, s); + EnsureDynamicCast<ICodegeneratorExternalNode*>(State[i])->CreateSetValue(ctx, block, s); } empty->addIncoming(ConstantInt::getFalse(context), block); diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp index 9421613971..7a6ad7967a 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp @@ -4,12 +4,12 @@ #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/utils/cast.h> - + namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class TBaseWideFilterWrapper { @@ -42,7 +42,7 @@ protected: for (auto i = 0U; i < Items.size(); ++i) if (Predicate == Items[i] || Items[i]->GetDependencesCount() > 0U) { - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getters[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getters[i](ctx, block)); if constexpr (ReplaceOriginalGetter) getters[i] = [node=Items[i]](const TCodegenContext& ctx, BasicBlock*& block){ return GetNodeValue(node, ctx, block); }; } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp index 89413853f3..a4e3e3ad04 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp @@ -7,8 +7,8 @@ namespace NKikimr { namespace NMiniKQL { -using NYql::EnsureDynamicCast; - +using NYql::EnsureDynamicCast; + namespace { class TExpandMapWrapper : public TStatelessWideFlowCodegeneratorNode<TExpandMapWrapper> { @@ -133,7 +133,7 @@ public: for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, result.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, result.second[i](ctx, block)); BranchInst::Create(pass, block); @@ -219,7 +219,7 @@ public: } else { for (auto i = 0U; i < Items.size(); ++i) if (Items[i]->GetDependencesCount() > 0U) - EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); + EnsureDynamicCast<ICodegeneratorExternalNode*>(Items[i])->CreateSetValue(ctx, block, getres.second[i](ctx, block)); result->addIncoming(GetNodeValue(NewItem, ctx, block), block); } diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp index 05cfc1e8e6..9df6d01d4e 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp @@ -55,61 +55,61 @@ Y_UNIT_TEST_SUITE(TMiniKQLSafeCircularBuffer) { } } - Y_UNIT_TEST(TestOverflowNoInitSize) { - TBufUnboxed buffer(3, TUnboxedValuePod(), 0); - buffer.PushBack(TUnboxedValue::Embedded("1")); - buffer.PushBack(TUnboxedValue::Embedded("2")); - buffer.PushBack(TUnboxedValue::Embedded("3")); - UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("4")), yexception); - } - - Y_UNIT_TEST(TestOverflowWithInitSize) { - TBufUnboxed buffer(3, TUnboxedValuePod(), 3); - buffer.PopFront(); - buffer.PushBack(TUnboxedValue::Embedded("1")); - buffer.PopFront(); - buffer.PushBack(TUnboxedValue::Embedded("2")); - UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); - UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "1"); - UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("3")), yexception); - } - - Y_UNIT_TEST(TestOverflowOnEmpty) { - TBufUnboxed buffer(0, TUnboxedValuePod()); - buffer.PopFront(); - UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("1")), yexception); - } - - Y_UNIT_TEST(TestUnbounded) { - TBufUnboxed buffer({}, TUnboxedValuePod(), 3); - for (size_t i = 0; i < 100; ++i) { - buffer.PushBack(TUnboxedValue::Embedded(ToString(i))); - } - - UNIT_ASSERT(!buffer.Get(0)); - UNIT_ASSERT(!buffer.Get(1)); - UNIT_ASSERT(!buffer.Get(2)); - - for (size_t i = 0; i < 100; ++i) { + Y_UNIT_TEST(TestOverflowNoInitSize) { + TBufUnboxed buffer(3, TUnboxedValuePod(), 0); + buffer.PushBack(TUnboxedValue::Embedded("1")); + buffer.PushBack(TUnboxedValue::Embedded("2")); + buffer.PushBack(TUnboxedValue::Embedded("3")); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("4")), yexception); + } + + Y_UNIT_TEST(TestOverflowWithInitSize) { + TBufUnboxed buffer(3, TUnboxedValuePod(), 3); + buffer.PopFront(); + buffer.PushBack(TUnboxedValue::Embedded("1")); + buffer.PopFront(); + buffer.PushBack(TUnboxedValue::Embedded("2")); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); + UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "1"); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("3")), yexception); + } + + Y_UNIT_TEST(TestOverflowOnEmpty) { + TBufUnboxed buffer(0, TUnboxedValuePod()); + buffer.PopFront(); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("1")), yexception); + } + + Y_UNIT_TEST(TestUnbounded) { + TBufUnboxed buffer({}, TUnboxedValuePod(), 3); + for (size_t i = 0; i < 100; ++i) { + buffer.PushBack(TUnboxedValue::Embedded(ToString(i))); + } + + UNIT_ASSERT(!buffer.Get(0)); + UNIT_ASSERT(!buffer.Get(1)); + UNIT_ASSERT(!buffer.Get(2)); + + for (size_t i = 0; i < 100; ++i) { UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(i + 3).AsStringRef()), ToString(i)); - } - - for (size_t i = 0; i < 100; ++i) { - buffer.PopFront(); - } - - UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); + } + + for (size_t i = 0; i < 100; ++i) { + buffer.PopFront(); + } + + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(0).AsStringRef()), ToString(97)); UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(1).AsStringRef()), ToString(98)); UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(2).AsStringRef()), ToString(99)); - - buffer.PopFront(); - buffer.PopFront(); - buffer.PopFront(); - UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0); - UNIT_ASSERT_EQUAL(buffer.Size(), 103); - } - + + buffer.PopFront(); + buffer.PopFront(); + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0); + UNIT_ASSERT_EQUAL(buffer.Size(), 103); + } + Y_UNIT_TEST(TestUnboxedFewCycles) { const auto multFillLevel = 0.6; const auto multPopLevel = 0.5; diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index ee6e66ddae..55838c4012 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -116,8 +116,8 @@ SRCS( mkql_multihopping.h mkql_multimap.cpp mkql_multimap.h - mkql_next_value.cpp - mkql_next_value.h + mkql_next_value.cpp + mkql_next_value.h mkql_now.cpp mkql_now.h mkql_null.cpp @@ -130,8 +130,8 @@ SRCS( mkql_queue.h mkql_random.cpp mkql_random.h - mkql_range.cpp - mkql_range.h + mkql_range.cpp + mkql_range.h mkql_reduce.cpp mkql_reduce.h mkql_removemember.cpp @@ -140,13 +140,13 @@ SRCS( mkql_replicate.h mkql_reverse.cpp mkql_reverse.h - mkql_round.cpp - mkql_round.h + mkql_round.cpp + mkql_round.h mkql_safe_circular_buffer.cpp mkql_safe_circular_buffer.h mkql_saveload.h - mkql_seq.cpp - mkql_seq.h + mkql_seq.cpp + mkql_seq.h mkql_size.cpp mkql_size.h mkql_skip.cpp diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node.h b/ydb/library/yql/minikql/computation/mkql_computation_node.h index eba87d64d0..a5d11b180b 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node.h @@ -255,7 +255,7 @@ using TComputationNodeFactory = std::function<IComputationNode* (TCallable&, con using TStreamEmitter = std::function<void(NUdf::TUnboxedValue&&)>; struct TComputationPatternOpts { - TComputationPatternOpts(const std::shared_ptr<TInjectedAlloc>& cacheAlloc, const std::shared_ptr<TTypeEnvironment>& cacheEnv) + TComputationPatternOpts(const std::shared_ptr<TInjectedAlloc>& cacheAlloc, const std::shared_ptr<TTypeEnvironment>& cacheEnv) : CacheAlloc(std::move(cacheAlloc)) , CacheEnv(std::move(cacheEnv)) , AllocState(CacheAlloc->InjectedState()) @@ -305,8 +305,8 @@ struct TComputationPatternOpts { SecureParamsProvider = secureParamsProvider; } - mutable std::shared_ptr<TInjectedAlloc> CacheAlloc; - mutable std::shared_ptr<TTypeEnvironment> CacheEnv; + mutable std::shared_ptr<TInjectedAlloc> CacheAlloc; + mutable std::shared_ptr<TTypeEnvironment> CacheEnv; TAllocState& AllocState; const TTypeEnvironment& Env; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index fa1620b77e..f2b188ae47 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -200,7 +200,7 @@ private: THolder<TDefaultValueBuilder> ValueBuilder; TComputationMutables Mutables; TComputationNodePtrDeque ComputationNodesList; - IComputationNode* RootNode = nullptr; + IComputationNode* RootNode = nullptr; TComputationExternalNodePtrVector Runtime2Computation; TComputationNodeOnNodeMap ElementsCache; }; @@ -771,13 +771,13 @@ public: Alloc->Acquire(); } PatternNodes.Reset(); - TypeEnv.reset(); + TypeEnv.reset(); if (Alloc) { Alloc->Release(); } } - void HoldInternals(const std::shared_ptr<TScopedAlloc>& alloc, const std::shared_ptr<TTypeEnvironment>& typeEnv) { + void HoldInternals(const std::shared_ptr<TScopedAlloc>& alloc, const std::shared_ptr<TTypeEnvironment>& typeEnv) { Alloc = alloc; TypeEnv = typeEnv; } @@ -799,8 +799,8 @@ public: return MakeHolder<TComputationGraph>(PatternNodes, compOpts); } private: - std::shared_ptr<TScopedAlloc> Alloc; - std::shared_ptr<TTypeEnvironment> TypeEnv; + std::shared_ptr<TScopedAlloc> Alloc; + std::shared_ptr<TTypeEnvironment> TypeEnv; TPatternNodes::TPtr PatternNodes; NYql::NCodegen::ICodegen::TPtr Codegen; }; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp index 8d9261ca62..67a7cfb302 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp @@ -565,11 +565,11 @@ public: NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { NUdf::TUnboxedValue *items = nullptr; const auto result = Cache.NewArray(ctx, ValueNodes.size(), items); - if (!ValueNodes.empty()) { - Y_VERIFY(items); - for (const auto& node : ValueNodes) { - *items++ = node->GetValue(ctx); - } + if (!ValueNodes.empty()) { + Y_VERIFY(items); + for (const auto& node : ValueNodes) { + *items++ = node->GetValue(ctx); + } } return result; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp index 4d95f11a1b..625a269eaf 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp @@ -303,7 +303,7 @@ namespace { return exit; } } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } Function* CreatePackFunction(const TType* type, bool useTopLength, Module &module, LLVMContext &context) { diff --git a/ydb/library/yql/minikql/computation/presort.cpp b/ydb/library/yql/minikql/computation/presort.cpp index 964ac61abe..26b96a60aa 100644 --- a/ydb/library/yql/minikql/computation/presort.cpp +++ b/ydb/library/yql/minikql/computation/presort.cpp @@ -536,7 +536,7 @@ void EncodeValue(TType* type, const NUdf::TUnboxedValue& value, TVector<ui8>& ou auto hasValue = (bool)value; EncodeBool<false>(output, hasValue); if (hasValue) { - EncodeValue(itemType, value.GetOptionalValue(), output); + EncodeValue(itemType, value.GetOptionalValue(), output); } break; diff --git a/ydb/library/yql/minikql/computation/presort_ut.cpp b/ydb/library/yql/minikql/computation/presort_ut.cpp index 20ea2f4094..4adf957ca1 100644 --- a/ydb/library/yql/minikql/computation/presort_ut.cpp +++ b/ydb/library/yql/minikql/computation/presort_ut.cpp @@ -487,33 +487,33 @@ Y_UNIT_TEST(GenericOptional) { UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); } -Y_UNIT_TEST(NestedOptional) { - TScopedAlloc alloc; - TTypeEnvironment env(alloc); - // Int32??? - auto type = - TOptionalType::Create(TOptionalType::Create(TOptionalType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, env), env), env), env); - TGenericPresortEncoder encoder(type); - - NUdf::TUnboxedValue null = {}; - auto buf = encoder.Encode(null, false); - UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); - - auto justNull = null.MakeOptional(); - buf = encoder.Encode(justNull, false); - UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00"sv)); - - auto justJustNull = justNull.MakeOptional(); - buf = encoder.Encode(justJustNull, false); - UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv)); - - - auto zero = NUdf::TUnboxedValuePod(0).MakeOptional().MakeOptional().MakeOptional(); - buf = encoder.Encode(zero, false); - UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x80\x00\x00\x00"sv)); -} - - +Y_UNIT_TEST(NestedOptional) { + TScopedAlloc alloc; + TTypeEnvironment env(alloc); + // Int32??? + auto type = + TOptionalType::Create(TOptionalType::Create(TOptionalType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, env), env), env), env); + TGenericPresortEncoder encoder(type); + + NUdf::TUnboxedValue null = {}; + auto buf = encoder.Encode(null, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); + + auto justNull = null.MakeOptional(); + buf = encoder.Encode(justNull, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00"sv)); + + auto justJustNull = justNull.MakeOptional(); + buf = encoder.Encode(justJustNull, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv)); + + + auto zero = NUdf::TUnboxedValuePod(0).MakeOptional().MakeOptional().MakeOptional(); + buf = encoder.Encode(zero, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x80\x00\x00\x00"sv)); +} + + Y_UNIT_TEST(GenericList) { TScopedAlloc alloc; TTypeEnvironment env(alloc); diff --git a/ydb/library/yql/minikql/dom/make.cpp b/ydb/library/yql/minikql/dom/make.cpp index f6950dac02..ae3c698092 100644 --- a/ydb/library/yql/minikql/dom/make.cpp +++ b/ydb/library/yql/minikql/dom/make.cpp @@ -161,9 +161,9 @@ TUnboxedValuePod MakeDom(const ITypeInfoHelper::TPtr typeHelper, const TType* sh case ETypeKind::Resource: if (const auto inspector = TResourceTypeInspector(*typeHelper, shape); TStringBuf(inspector.GetTag()) == NodeResourceName) return value; - [[fallthrough]]; + [[fallthrough]]; default: - Y_FAIL("Unsupported data kind: %s", ToCString(kind)); + Y_FAIL("Unsupported data kind: %s", ToCString(kind)); } } diff --git a/ydb/library/yql/minikql/dom/peel.cpp b/ydb/library/yql/minikql/dom/peel.cpp index 28973e7e05..33f22011dd 100644 --- a/ydb/library/yql/minikql/dom/peel.cpp +++ b/ydb/library/yql/minikql/dom/peel.cpp @@ -184,7 +184,7 @@ TUnboxedValuePod MakeStub(const ITypeInfoHelper::TPtr typeHelper, const TType* s case ETypeKind::Resource: if (const auto inspector = TResourceTypeInspector(*typeHelper, shape); TStringBuf(inspector.GetTag()) == NodeResourceName) return MakeEntity(); - [[fallthrough]]; + [[fallthrough]]; default: UdfTerminate((::TStringBuilder() << "Unsupported data kind: " << kind).c_str()); } diff --git a/ydb/library/yql/minikql/jsonpath/JsonPath.g b/ydb/library/yql/minikql/jsonpath/JsonPath.g index d91ce36e29..f32f98d185 100644 --- a/ydb/library/yql/minikql/jsonpath/JsonPath.g +++ b/ydb/library/yql/minikql/jsonpath/JsonPath.g @@ -174,8 +174,8 @@ UNKNOWN: 'unknown'; WITH: 'with'; // String literal -fragment STRING_CORE_SINGLE: ( ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) )*; -fragment STRING_CORE_DOUBLE: ( ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) )*; +fragment STRING_CORE_SINGLE: ( ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) )*; +fragment STRING_CORE_DOUBLE: ( ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) )*; fragment STRING_SINGLE: (QUOTE_SINGLE STRING_CORE_SINGLE QUOTE_SINGLE); fragment STRING_DOUBLE: (QUOTE_DOUBLE STRING_CORE_DOUBLE QUOTE_DOUBLE); @@ -198,7 +198,7 @@ IDENTIFIER: ID_START (ID_CORE)*; // Jsonpath variable VARIABLE: DOLLAR (ID_CORE)*; -WS: (' '|'\r'|'\t'|'\n') {$channel=HIDDEN;}; -// FIXME: WS and COMMENT tokens are currently required. -// FIXME: Since there are no comments in JSONPATH, we split whitespace characters between WS and COMMENT -COMMENT: ('\u000C') {$channel=HIDDEN;}; +WS: (' '|'\r'|'\t'|'\n') {$channel=HIDDEN;}; +// FIXME: WS and COMMENT tokens are currently required. +// FIXME: Since there are no comments in JSONPATH, we split whitespace characters between WS and COMMENT +COMMENT: ('\u000C') {$channel=HIDDEN;}; diff --git a/ydb/library/yql/minikql/jsonpath/ast_builder.cpp b/ydb/library/yql/minikql/jsonpath/ast_builder.cpp index 66e097b802..c13a2264c1 100644 --- a/ydb/library/yql/minikql/jsonpath/ast_builder.cpp +++ b/ydb/library/yql/minikql/jsonpath/ast_builder.cpp @@ -315,7 +315,7 @@ TAstNodePtr TAstBuilder::BuildPredicateExpr(const TRule_predicate_expr& node) { case TRule_predicate_expr_TAlt1_TBlock1_TBlock2::ALT_NOT_SET: Y_FAIL("Alternative for inner block of 'predicate_expr' rule is not set"); } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } case TRule_predicate_expr::kAltPredicateExpr2: { const auto& predicate = node.GetAlt_predicate_expr2().GetBlock1(); @@ -325,7 +325,7 @@ TAstNodePtr TAstBuilder::BuildPredicateExpr(const TRule_predicate_expr& node) { case TRule_predicate_expr::ALT_NOT_SET: Y_FAIL("Alternative for 'predicate' rule is not set"); } - Y_UNREACHABLE(); + Y_UNREACHABLE(); } TAstNodePtr TAstBuilder::BuildUnaryExpr(const TRule_unary_expr& node) { diff --git a/ydb/library/yql/minikql/jsonpath/executor.h b/ydb/library/yql/minikql/jsonpath/executor.h index 0dc0f697c6..782ab1818d 100644 --- a/ydb/library/yql/minikql/jsonpath/executor.h +++ b/ydb/library/yql/minikql/jsonpath/executor.h @@ -83,9 +83,9 @@ public: } private: - i64 From = 0; + i64 From = 0; TPosition FromPos; - i64 To = 0; + i64 To = 0; TPosition ToPos; bool HasTo; }; diff --git a/ydb/library/yql/minikql/jsonpath/jsonpath.cpp b/ydb/library/yql/minikql/jsonpath/jsonpath.cpp index 727ea90e7f..985ea72c36 100644 --- a/ydb/library/yql/minikql/jsonpath/jsonpath.cpp +++ b/ydb/library/yql/minikql/jsonpath/jsonpath.cpp @@ -64,7 +64,7 @@ const TAstNodePtr ParseJsonPathAst(const TStringBuf path, TIssues& issues, size_ #if defined(_tsan_enabled_) TGuard<TMutex> guard(SanitizerJsonPathTranslationMutex); #endif - NProtoAST::TProtoASTBuilder<NALP::JsonPathParser, NALP::JsonPathLexer> builder(path, "JsonPath", &arena); + NProtoAST::TProtoASTBuilder<NALP::JsonPathParser, NALP::JsonPathLexer> builder(path, "JsonPath", &arena); TParseErrorsCollector collector(issues, maxParseErrors); rawAst = builder.BuildAST(collector); } diff --git a/ydb/library/yql/minikql/mkql_alloc.h b/ydb/library/yql/minikql/mkql_alloc.h index 1bae7135c0..5e864cdc4c 100644 --- a/ydb/library/yql/minikql/mkql_alloc.h +++ b/ydb/library/yql/minikql/mkql_alloc.h @@ -133,7 +133,7 @@ public: : OldTlsAllocState(TlsAllocState) { TlsAllocState = nullptr; - InjectedAlloc = std::make_shared<TScopedAlloc>(counters, supportsSizedAllocators); + InjectedAlloc = std::make_shared<TScopedAlloc>(counters, supportsSizedAllocators); AcquireOriginal(); } @@ -149,7 +149,7 @@ public: return InjectedAlloc->Ref(); } - std::shared_ptr<TScopedAlloc> InjectedScopeAlloc() { + std::shared_ptr<TScopedAlloc> InjectedScopeAlloc() { return InjectedAlloc; } @@ -164,7 +164,7 @@ public: } private: TAllocState* OldTlsAllocState; - std::shared_ptr<TScopedAlloc> InjectedAlloc; + std::shared_ptr<TScopedAlloc> InjectedAlloc; }; class TInjectFreeGuard: TNonCopyable { diff --git a/ydb/library/yql/minikql/mkql_function_registry.cpp b/ydb/library/yql/minikql/mkql_function_registry.cpp index c10a6d44d6..d3545ad138 100644 --- a/ydb/library/yql/minikql/mkql_function_registry.cpp +++ b/ydb/library/yql/minikql/mkql_function_registry.cpp @@ -31,13 +31,13 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry { struct TUdfModule { TString LibraryPath; - std::shared_ptr<NUdf::IUdfModule> Impl; + std::shared_ptr<NUdf::IUdfModule> Impl; }; using TUdfModulesMap = THashMap<TString, TUdfModule>; struct TUdfLibrary: public TThrRefBase { - ui32 AbiVersion = 0; + ui32 AbiVersion = 0; TDynamicLibrary Lib; TUdfLibrary() { @@ -68,7 +68,7 @@ class TMutableFunctionRegistry: public IMutableFunctionRegistry if (!HasError()) { TUdfModule m; m.LibraryPath = LibraryPath; - m.Impl.reset(module.Release()); + m.Impl.reset(module.Release()); auto it = Remappings.find(name); const TString& newName = (it == Remappings.end()) @@ -232,8 +232,8 @@ public: if (it != UdfModules_.end()) { TFunctionTypeInfoBuilder typeInfoBuilder(env, typeInfoHelper, moduleName, (flags & NUdf::IUdfModule::TFlags::TypesOnly) ? nullptr : countersProvider, pos, secureParamsProvider); - const auto& module = *it->second.Impl; - module.BuildFunctionTypeInfo( + const auto& module = *it->second.Impl; + module.BuildFunctionTypeInfo( funcName, userType, typeConfig, flags, typeInfoBuilder); if (typeInfoBuilder.HasError()) { diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index ea5298905a..47ae6e4c8f 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -2443,22 +2443,22 @@ TRuntimeNode TProgramBuilder::DataCompare(const std::string_view& callableName, return Invoke(callableName, resultType, args); } -TRuntimeNode TProgramBuilder::BuildRangeLogical(const std::string_view& callableName, const TArrayRef<const TRuntimeNode>& lists) { - MKQL_ENSURE(!lists.empty(), "Expecting at least one argument"); - - for (auto& list : lists) { - MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting lists"); - MKQL_ENSURE(list.GetStaticType()->IsSameType(*lists.front().GetStaticType()), "Expecting arguments of same type"); - } - - TCallableBuilder callableBuilder(Env, callableName, lists.front().GetStaticType()); - for (auto& list : lists) { - callableBuilder.Add(list); - } - - return TRuntimeNode(callableBuilder.Build(), false); -} - +TRuntimeNode TProgramBuilder::BuildRangeLogical(const std::string_view& callableName, const TArrayRef<const TRuntimeNode>& lists) { + MKQL_ENSURE(!lists.empty(), "Expecting at least one argument"); + + for (auto& list : lists) { + MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting lists"); + MKQL_ENSURE(list.GetStaticType()->IsSameType(*lists.front().GetStaticType()), "Expecting arguments of same type"); + } + + TCallableBuilder callableBuilder(Env, callableName, lists.front().GetStaticType()); + for (auto& list : lists) { + callableBuilder.Add(list); + } + + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::AggrEquals(TRuntimeNode data1, TRuntimeNode data2) { return AggrCompare(__func__, data1, data2); } @@ -2724,17 +2724,17 @@ TRuntimeNode TProgramBuilder::AggrAdd(TRuntimeNode data1, TRuntimeNode data2) { TRuntimeNode TProgramBuilder::QueueCreate(TRuntimeNode initCapacity, TRuntimeNode initSize, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType) { auto resType = AS_TYPE(TResourceType, returnType); const auto tag = resType->GetTag(); - - if (initCapacity.GetStaticType()->IsVoid()) { + + if (initCapacity.GetStaticType()->IsVoid()) { MKQL_ENSURE(RuntimeVersion >= 13, "Unbounded queue is not supported in runtime version " << RuntimeVersion); - } else { - auto initCapacityType = AS_TYPE(TDataType, initCapacity); - MKQL_ENSURE(initCapacityType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "init capcity must be ui64"); - } - + } else { + auto initCapacityType = AS_TYPE(TDataType, initCapacity); + MKQL_ENSURE(initCapacityType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "init capcity must be ui64"); + } + auto initSizeType = AS_TYPE(TDataType, initSize); MKQL_ENSURE(initSizeType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "init size must be ui64"); - + TCallableBuilder callableBuilder(Env, __func__, returnType, true); callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(tag)); callableBuilder.Add(initCapacity); @@ -2780,30 +2780,30 @@ TRuntimeNode TProgramBuilder::QueuePeek(TRuntimeNode resource, TRuntimeNode inde return TRuntimeNode(callableBuilder.Build(), false); } -TRuntimeNode TProgramBuilder::QueueRange(TRuntimeNode resource, TRuntimeNode begin, TRuntimeNode end, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType) { +TRuntimeNode TProgramBuilder::QueueRange(TRuntimeNode resource, TRuntimeNode begin, TRuntimeNode end, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType) { MKQL_ENSURE(RuntimeVersion >= 14, "QueueRange is not supported in runtime version " << RuntimeVersion); - - MKQL_ENSURE(returnType->IsList(), "Expected list type as result of QueueRange"); - auto resType = AS_TYPE(TResourceType, resource); - - auto beginType = AS_TYPE(TDataType, begin); - MKQL_ENSURE(beginType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "begin index must be ui64"); - - auto endType = AS_TYPE(TDataType, end); - MKQL_ENSURE(endType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "end index must be ui64"); - - const auto tag = resType->GetTag(); - MKQL_ENSURE(tag.StartsWith(ResourceQueuePrefix), "Expected Queue resource"); - TCallableBuilder callableBuilder(Env, __func__, returnType); - callableBuilder.Add(resource); - callableBuilder.Add(begin); - callableBuilder.Add(end); - for (auto node : dependentNodes) { - callableBuilder.Add(node); - } - return TRuntimeNode(callableBuilder.Build(), false); -} - + + MKQL_ENSURE(returnType->IsList(), "Expected list type as result of QueueRange"); + auto resType = AS_TYPE(TResourceType, resource); + + auto beginType = AS_TYPE(TDataType, begin); + MKQL_ENSURE(beginType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "begin index must be ui64"); + + auto endType = AS_TYPE(TDataType, end); + MKQL_ENSURE(endType->GetSchemeType() == NUdf::TDataType<ui64>::Id, "end index must be ui64"); + + const auto tag = resType->GetTag(); + MKQL_ENSURE(tag.StartsWith(ResourceQueuePrefix), "Expected Queue resource"); + TCallableBuilder callableBuilder(Env, __func__, returnType); + callableBuilder.Add(resource); + callableBuilder.Add(begin); + callableBuilder.Add(end); + for (auto node : dependentNodes) { + callableBuilder.Add(node); + } + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::PreserveStream(TRuntimeNode stream, TRuntimeNode queue, TRuntimeNode outpace) { auto streamType = AS_TYPE(TStreamType, stream); auto resType = AS_TYPE(TResourceType, queue); @@ -2818,16 +2818,16 @@ TRuntimeNode TProgramBuilder::PreserveStream(TRuntimeNode stream, TRuntimeNode q return TRuntimeNode(callableBuilder.Build(), false); } -TRuntimeNode TProgramBuilder::Seq(const TArrayRef<const TRuntimeNode>& args, TType* returnType) { - MKQL_ENSURE(RuntimeVersion >= 15, "Seq is not supported in runtime version " << RuntimeVersion); - - TCallableBuilder callableBuilder(Env, __func__, returnType); - for (auto node : args) { - callableBuilder.Add(node); - } - return TRuntimeNode(callableBuilder.Build(), false); -} - +TRuntimeNode TProgramBuilder::Seq(const TArrayRef<const TRuntimeNode>& args, TType* returnType) { + MKQL_ENSURE(RuntimeVersion >= 15, "Seq is not supported in runtime version " << RuntimeVersion); + + TCallableBuilder callableBuilder(Env, __func__, returnType); + for (auto node : args) { + callableBuilder.Add(node); + } + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::FromYsonSimpleType(TRuntimeNode input, NUdf::TDataTypeId schemeType) { auto type = input.GetStaticType(); if (type->IsOptional()) { @@ -4806,164 +4806,164 @@ TRuntimeNode TProgramBuilder::Cast(TRuntimeNode arg, TType* type) { return useToIntegral ? ToIntegral(arg, type) : Convert(arg, type); } -TRuntimeNode TProgramBuilder::RangeCreate(TRuntimeNode list) { - MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); - auto itemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); - MKQL_ENSURE(itemType->IsTuple(), "Expecting list of tuples"); - - auto tupleType = static_cast<TTupleType*>(itemType); - MKQL_ENSURE(tupleType->GetElementsCount() == 2, - "Expecting list ot 2-element tuples, got: " << tupleType->GetElementsCount() << " elements"); - - MKQL_ENSURE(tupleType->GetElementType(0)->IsSameType(*tupleType->GetElementType(1)), - "Expecting list ot 2-element tuples of same type"); - - MKQL_ENSURE(tupleType->GetElementType(0)->IsTuple(), - "Expecting range boundary to be tuple"); - - auto boundaryType = static_cast<TTupleType*>(tupleType->GetElementType(0)); - MKQL_ENSURE(boundaryType->GetElementsCount() >= 2, - "Range boundary should have at least 2 components, got: " << boundaryType->GetElementsCount()); - - auto lastComp = boundaryType->GetElementType(boundaryType->GetElementsCount() - 1); - std::vector<TType*> outputComponents; - for (ui32 i = 0; i < boundaryType->GetElementsCount() - 1; ++i) { - outputComponents.push_back(lastComp); - outputComponents.push_back(boundaryType->GetElementType(i)); - } - outputComponents.push_back(lastComp); - - auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); - std::vector<TType*> outputRangeComps(2, outputBoundary); - auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); - - TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); - callableBuilder.Add(list); - - return TRuntimeNode(callableBuilder.Build(), false); -} - -TRuntimeNode TProgramBuilder::RangeUnion(const TArrayRef<const TRuntimeNode>& lists) { - return BuildRangeLogical(__func__, lists); -} - -TRuntimeNode TProgramBuilder::RangeIntersect(const TArrayRef<const TRuntimeNode>& lists) { - return BuildRangeLogical(__func__, lists); -} - -TRuntimeNode TProgramBuilder::RangeMultiply(const TArrayRef<const TRuntimeNode>& args) { - MKQL_ENSURE(args.size() >= 2, "Expecting at least two arguments"); - - MKQL_ENSURE(args.front().GetStaticType()->IsData() && - static_cast<TDataType*>(args.front().GetStaticType())->GetSchemeType() == NUdf::TDataType<ui64>::Id, - "Expected ui64 as first argument"); - - std::vector<TType*> outputComponents; - for (size_t i = 1; i < args.size(); ++i) { - const auto& list = args[i]; - - MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); - - auto listItemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); - MKQL_ENSURE(listItemType->IsTuple(), "Expecting list of tuples"); - - auto rangeType = static_cast<TTupleType*>(listItemType); - MKQL_ENSURE(rangeType->GetElementsCount() == 2, "Expecting list of 2-element tuples"); - MKQL_ENSURE(rangeType->GetElementType(0)->IsTuple(), "Range boundary should be tuple"); - - auto boundaryType = static_cast<TTupleType*>(rangeType->GetElementType(0)); - - ui32 elementsCount = boundaryType->GetElementsCount(); - MKQL_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1, "Range boundary should have odd number components (at least 3)"); - - for (size_t j = 0; j < elementsCount - 1; ++j) { - outputComponents.push_back(boundaryType->GetElementType(j)); - } - } - outputComponents.push_back(TDataType::Create(NUdf::TDataType<i32>::Id, Env)); - - auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); - std::vector<TType*> outputRangeComps(2, outputBoundary); - auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); - - TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); - for (auto& arg : args) { - callableBuilder.Add(arg); - } - - return TRuntimeNode(callableBuilder.Build(), false); -} - -TRuntimeNode TProgramBuilder::RangeFinalize(TRuntimeNode list) { - MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); - - auto listItemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); - MKQL_ENSURE(listItemType->IsTuple(), "Expecting list of tuples"); - - auto rangeType = static_cast<TTupleType*>(listItemType); - MKQL_ENSURE(rangeType->GetElementsCount() == 2, "Expecting list of 2-element tuples"); - MKQL_ENSURE(rangeType->GetElementType(0)->IsTuple(), "Range boundary should be tuple"); - - auto boundaryType = static_cast<TTupleType*>(rangeType->GetElementType(0)); - - ui32 elementsCount = boundaryType->GetElementsCount(); - MKQL_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1, "Range boundary should have odd number components (at least 3)"); - - std::vector<TType*> outputComponents; - for (ui32 i = 0; i < elementsCount; ++i) { - if (i % 2 == 1 || i + 1 == elementsCount) { - outputComponents.push_back(boundaryType->GetElementType(i)); - } - } - - auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); - std::vector<TType*> outputRangeComps(2, outputBoundary); - auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); - - TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); - callableBuilder.Add(list); - - return TRuntimeNode(callableBuilder.Build(), false); -} - -TRuntimeNode TProgramBuilder::Round(const std::string_view& callableName, TRuntimeNode source, TType* targetType) { - const auto sourceType = source.GetStaticType(); - MKQL_ENSURE(sourceType->IsData(), "Expecting first arg to be of Data type"); - MKQL_ENSURE(targetType->IsData(), "Expecting second arg to be Data type"); - - - const auto ss = *static_cast<TDataType*>(sourceType)->GetDataSlot(); - const auto ts = *static_cast<TDataType*>(targetType)->GetDataSlot(); - - const auto options = NKikimr::NUdf::GetCastResult(ss, ts); - MKQL_ENSURE(options && !(*options & NKikimr::NUdf::ECastOptions::Impossible), - "Impossible to cast " << *sourceType << " into " << *targetType); - - MKQL_ENSURE(*options & (NKikimr::NUdf::ECastOptions::MayFail | - NKikimr::NUdf::ECastOptions::MayLoseData | - NKikimr::NUdf::ECastOptions::AnywayLoseData), - "Rounding from " << *sourceType << " to " << *targetType << " is trivial"); - - TCallableBuilder callableBuilder(Env, callableName, TOptionalType::Create(targetType, Env)); - callableBuilder.Add(source); - - return TRuntimeNode(callableBuilder.Build(), false); -} - -TRuntimeNode TProgramBuilder::NextValue(TRuntimeNode value) { - const auto valueType = value.GetStaticType(); - MKQL_ENSURE(valueType->IsData(), "Expecting argument of Data type"); - - const auto slot = *static_cast<TDataType*>(valueType)->GetDataSlot(); - MKQL_ENSURE(slot == NUdf::EDataSlot::String || slot == NUdf::EDataSlot::Utf8, - "Unsupported type: " << *valueType); - - TCallableBuilder callableBuilder(Env, __func__, TOptionalType::Create(valueType, Env)); - callableBuilder.Add(value); - - return TRuntimeNode(callableBuilder.Build(), false); -} - +TRuntimeNode TProgramBuilder::RangeCreate(TRuntimeNode list) { + MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); + auto itemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); + MKQL_ENSURE(itemType->IsTuple(), "Expecting list of tuples"); + + auto tupleType = static_cast<TTupleType*>(itemType); + MKQL_ENSURE(tupleType->GetElementsCount() == 2, + "Expecting list ot 2-element tuples, got: " << tupleType->GetElementsCount() << " elements"); + + MKQL_ENSURE(tupleType->GetElementType(0)->IsSameType(*tupleType->GetElementType(1)), + "Expecting list ot 2-element tuples of same type"); + + MKQL_ENSURE(tupleType->GetElementType(0)->IsTuple(), + "Expecting range boundary to be tuple"); + + auto boundaryType = static_cast<TTupleType*>(tupleType->GetElementType(0)); + MKQL_ENSURE(boundaryType->GetElementsCount() >= 2, + "Range boundary should have at least 2 components, got: " << boundaryType->GetElementsCount()); + + auto lastComp = boundaryType->GetElementType(boundaryType->GetElementsCount() - 1); + std::vector<TType*> outputComponents; + for (ui32 i = 0; i < boundaryType->GetElementsCount() - 1; ++i) { + outputComponents.push_back(lastComp); + outputComponents.push_back(boundaryType->GetElementType(i)); + } + outputComponents.push_back(lastComp); + + auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); + std::vector<TType*> outputRangeComps(2, outputBoundary); + auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); + + TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); + callableBuilder.Add(list); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +TRuntimeNode TProgramBuilder::RangeUnion(const TArrayRef<const TRuntimeNode>& lists) { + return BuildRangeLogical(__func__, lists); +} + +TRuntimeNode TProgramBuilder::RangeIntersect(const TArrayRef<const TRuntimeNode>& lists) { + return BuildRangeLogical(__func__, lists); +} + +TRuntimeNode TProgramBuilder::RangeMultiply(const TArrayRef<const TRuntimeNode>& args) { + MKQL_ENSURE(args.size() >= 2, "Expecting at least two arguments"); + + MKQL_ENSURE(args.front().GetStaticType()->IsData() && + static_cast<TDataType*>(args.front().GetStaticType())->GetSchemeType() == NUdf::TDataType<ui64>::Id, + "Expected ui64 as first argument"); + + std::vector<TType*> outputComponents; + for (size_t i = 1; i < args.size(); ++i) { + const auto& list = args[i]; + + MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); + + auto listItemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); + MKQL_ENSURE(listItemType->IsTuple(), "Expecting list of tuples"); + + auto rangeType = static_cast<TTupleType*>(listItemType); + MKQL_ENSURE(rangeType->GetElementsCount() == 2, "Expecting list of 2-element tuples"); + MKQL_ENSURE(rangeType->GetElementType(0)->IsTuple(), "Range boundary should be tuple"); + + auto boundaryType = static_cast<TTupleType*>(rangeType->GetElementType(0)); + + ui32 elementsCount = boundaryType->GetElementsCount(); + MKQL_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1, "Range boundary should have odd number components (at least 3)"); + + for (size_t j = 0; j < elementsCount - 1; ++j) { + outputComponents.push_back(boundaryType->GetElementType(j)); + } + } + outputComponents.push_back(TDataType::Create(NUdf::TDataType<i32>::Id, Env)); + + auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); + std::vector<TType*> outputRangeComps(2, outputBoundary); + auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); + + TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); + for (auto& arg : args) { + callableBuilder.Add(arg); + } + + return TRuntimeNode(callableBuilder.Build(), false); +} + +TRuntimeNode TProgramBuilder::RangeFinalize(TRuntimeNode list) { + MKQL_ENSURE(list.GetStaticType()->IsList(), "Expecting list"); + + auto listItemType = static_cast<TListType*>(list.GetStaticType())->GetItemType(); + MKQL_ENSURE(listItemType->IsTuple(), "Expecting list of tuples"); + + auto rangeType = static_cast<TTupleType*>(listItemType); + MKQL_ENSURE(rangeType->GetElementsCount() == 2, "Expecting list of 2-element tuples"); + MKQL_ENSURE(rangeType->GetElementType(0)->IsTuple(), "Range boundary should be tuple"); + + auto boundaryType = static_cast<TTupleType*>(rangeType->GetElementType(0)); + + ui32 elementsCount = boundaryType->GetElementsCount(); + MKQL_ENSURE(elementsCount >= 3 && elementsCount % 2 == 1, "Range boundary should have odd number components (at least 3)"); + + std::vector<TType*> outputComponents; + for (ui32 i = 0; i < elementsCount; ++i) { + if (i % 2 == 1 || i + 1 == elementsCount) { + outputComponents.push_back(boundaryType->GetElementType(i)); + } + } + + auto outputBoundary = TTupleType::Create(outputComponents.size(), &outputComponents.front(), Env); + std::vector<TType*> outputRangeComps(2, outputBoundary); + auto outputRange = TTupleType::Create(outputRangeComps.size(), &outputRangeComps.front(), Env); + + TCallableBuilder callableBuilder(Env, __func__, TListType::Create(outputRange, Env)); + callableBuilder.Add(list); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +TRuntimeNode TProgramBuilder::Round(const std::string_view& callableName, TRuntimeNode source, TType* targetType) { + const auto sourceType = source.GetStaticType(); + MKQL_ENSURE(sourceType->IsData(), "Expecting first arg to be of Data type"); + MKQL_ENSURE(targetType->IsData(), "Expecting second arg to be Data type"); + + + const auto ss = *static_cast<TDataType*>(sourceType)->GetDataSlot(); + const auto ts = *static_cast<TDataType*>(targetType)->GetDataSlot(); + + const auto options = NKikimr::NUdf::GetCastResult(ss, ts); + MKQL_ENSURE(options && !(*options & NKikimr::NUdf::ECastOptions::Impossible), + "Impossible to cast " << *sourceType << " into " << *targetType); + + MKQL_ENSURE(*options & (NKikimr::NUdf::ECastOptions::MayFail | + NKikimr::NUdf::ECastOptions::MayLoseData | + NKikimr::NUdf::ECastOptions::AnywayLoseData), + "Rounding from " << *sourceType << " to " << *targetType << " is trivial"); + + TCallableBuilder callableBuilder(Env, callableName, TOptionalType::Create(targetType, Env)); + callableBuilder.Add(source); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +TRuntimeNode TProgramBuilder::NextValue(TRuntimeNode value) { + const auto valueType = value.GetStaticType(); + MKQL_ENSURE(valueType->IsData(), "Expecting argument of Data type"); + + const auto slot = *static_cast<TDataType*>(valueType)->GetDataSlot(); + MKQL_ENSURE(slot == NUdf::EDataSlot::String || slot == NUdf::EDataSlot::Utf8, + "Unsupported type: " << *valueType); + + TCallableBuilder callableBuilder(Env, __func__, TOptionalType::Create(valueType, Env)); + callableBuilder.Add(value); + + return TRuntimeNode(callableBuilder.Build(), false); +} + bool TProgramBuilder::IsNull(TRuntimeNode arg) { return arg.GetStaticType()->IsSameType(*NewNull().GetStaticType()); // TODO ->IsNull(); } diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 51d6afe17d..10f1ad7ccf 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -39,13 +39,13 @@ enum class EDictItems { Payloads = 2 }; -enum class EAnyJoinSettings { - None = 0, - Left = 1, - Right = 2, - Both = 1 | 2, -}; - +enum class EAnyJoinSettings { + None = 0, + Left = 1, + Right = 2, + Both = 1 | 2, +}; + inline EJoinKind GetJoinKind(ui32 kind) { MKQL_ENSURE(kind >= (ui32)EJoinKind::Min && kind <= (ui32)EJoinKind::Max || kind == (ui32)EJoinKind::LeftSemi || kind == (ui32)EJoinKind::RightSemi || @@ -65,25 +65,25 @@ constexpr bool IsSemiJoin(EJoinKind kind) { return ((ui32)EJoinKind::SemiMask & (ui32)kind) != 0; } -inline EAnyJoinSettings GetAnyJoinSettings(ui32 settings) { - MKQL_ENSURE(settings <= (ui32)EAnyJoinSettings::Both, "Bad AnyJoin settings: " << settings); - return (EAnyJoinSettings)settings; -} - -inline bool IsAnyJoinLeft(EAnyJoinSettings settings) { - return ((ui32)settings & (ui32)EAnyJoinSettings::Left) != 0; -} - -inline bool IsAnyJoinRight(EAnyJoinSettings settings) { - return ((ui32)settings & (ui32)EAnyJoinSettings::Right) != 0; -} - -inline void AddAnyJoinSide(EAnyJoinSettings& combined, EAnyJoinSettings value) { - ui32 combinedVal = (ui32)combined; - combinedVal |= (ui32)value; - combined = (EAnyJoinSettings)combinedVal; -} - +inline EAnyJoinSettings GetAnyJoinSettings(ui32 settings) { + MKQL_ENSURE(settings <= (ui32)EAnyJoinSettings::Both, "Bad AnyJoin settings: " << settings); + return (EAnyJoinSettings)settings; +} + +inline bool IsAnyJoinLeft(EAnyJoinSettings settings) { + return ((ui32)settings & (ui32)EAnyJoinSettings::Left) != 0; +} + +inline bool IsAnyJoinRight(EAnyJoinSettings settings) { + return ((ui32)settings & (ui32)EAnyJoinSettings::Right) != 0; +} + +inline void AddAnyJoinSide(EAnyJoinSettings& combined, EAnyJoinSettings value) { + ui32 combinedVal = (ui32)combined; + combinedVal |= (ui32)value; + combined = (EAnyJoinSettings)combinedVal; +} + #define MKQL_SCRIPT_TYPES(xx) \ xx(Unknown, 0, unknown) \ xx(Python, 1, python) \ @@ -584,12 +584,12 @@ public: TRuntimeNode QueuePush(TRuntimeNode resource, TRuntimeNode value); TRuntimeNode QueuePop(TRuntimeNode resource); TRuntimeNode QueuePeek(TRuntimeNode resource, TRuntimeNode index, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType); - TRuntimeNode QueueRange(TRuntimeNode resource, TRuntimeNode begin, TRuntimeNode end, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType); + TRuntimeNode QueueRange(TRuntimeNode resource, TRuntimeNode begin, TRuntimeNode end, const TArrayRef<const TRuntimeNode>& dependentNodes, TType* returnType); TRuntimeNode PreserveStream(TRuntimeNode stream, TRuntimeNode preserve, TRuntimeNode outpace); - TRuntimeNode Seq(const TArrayRef<const TRuntimeNode>& items, TType* returnType); - + TRuntimeNode Seq(const TArrayRef<const TRuntimeNode>& items, TType* returnType); + TRuntimeNode FromYsonSimpleType(TRuntimeNode input, NUdf::TDataTypeId schemeType); TRuntimeNode TryWeakMemberFromDict(TRuntimeNode other, TRuntimeNode rest, NUdf::TDataTypeId schemeType, const std::string_view& memberName); @@ -604,16 +604,16 @@ public: TRuntimeNode Cast(TRuntimeNode data, TType* type); TRuntimeNode Default(TType* type); - TRuntimeNode RangeCreate(TRuntimeNode list); - TRuntimeNode RangeUnion(const TArrayRef<const TRuntimeNode>& lists); - TRuntimeNode RangeIntersect(const TArrayRef<const TRuntimeNode>& lists); - TRuntimeNode RangeMultiply(const TArrayRef<const TRuntimeNode>& args); - TRuntimeNode RangeFinalize(TRuntimeNode list); - - TRuntimeNode Round(const std::string_view& callableName, TRuntimeNode source, TType* targetType); - - TRuntimeNode NextValue(TRuntimeNode value); - + TRuntimeNode RangeCreate(TRuntimeNode list); + TRuntimeNode RangeUnion(const TArrayRef<const TRuntimeNode>& lists); + TRuntimeNode RangeIntersect(const TArrayRef<const TRuntimeNode>& lists); + TRuntimeNode RangeMultiply(const TArrayRef<const TRuntimeNode>& args); + TRuntimeNode RangeFinalize(TRuntimeNode list); + + TRuntimeNode Round(const std::string_view& callableName, TRuntimeNode source, TType* targetType); + + TRuntimeNode NextValue(TRuntimeNode value); + typedef TRuntimeNode (TProgramBuilder::*UnaryFunctionMethod)(TRuntimeNode); typedef TRuntimeNode (TProgramBuilder::*BinaryFunctionMethod)(TRuntimeNode, TRuntimeNode); typedef TRuntimeNode (TProgramBuilder::*TernaryFunctionMethod)(TRuntimeNode, TRuntimeNode, TRuntimeNode); @@ -675,8 +675,8 @@ private: TRuntimeNode AggrCompare(const std::string_view& callableName, TRuntimeNode data1, TRuntimeNode data2); TRuntimeNode DataCompare(const std::string_view& callableName, TRuntimeNode data1, TRuntimeNode data2); - TRuntimeNode BuildRangeLogical(const std::string_view& callableName, const TArrayRef<const TRuntimeNode>& lists); - + TRuntimeNode BuildRangeLogical(const std::string_view& callableName, const TArrayRef<const TRuntimeNode>& lists); + template<bool Default> TRuntimeNode DefaultResult(TRuntimeNode data); diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp index 2963a1f380..4aecb564bf 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.cpp +++ b/ydb/library/yql/minikql/mkql_type_builder.cpp @@ -635,7 +635,7 @@ public: if (!value) { return 0; } - return CombineHashes(ui64(1), Hash_->Hash(value.GetOptionalValue())); + return CombineHashes(ui64(1), Hash_->Hash(value.GetOptionalValue())); } private: @@ -834,7 +834,7 @@ public: if (!rhs) { return false; } - return Equate_->Equals(lhs.GetOptionalValue(), rhs.GetOptionalValue()); + return Equate_->Equals(lhs.GetOptionalValue(), rhs.GetOptionalValue()); } } @@ -1063,8 +1063,8 @@ public: template <> class TCompare<NMiniKQL::TType::EKind::Optional> final : public NUdf::ICompare { public: - explicit TCompare(const NMiniKQL::TType* type) - : Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType())) + explicit TCompare(const NMiniKQL::TType* type) + : Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType())) {} bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { @@ -1077,7 +1077,7 @@ public: if (!rhs) { return false; } - return Compare_->Less(lhs.GetOptionalValue(), rhs.GetOptionalValue()); + return Compare_->Less(lhs.GetOptionalValue(), rhs.GetOptionalValue()); } } @@ -1091,7 +1091,7 @@ public: if (!rhs) { return 1; } - return Compare_->Compare(lhs.GetOptionalValue(), rhs.GetOptionalValue()); + return Compare_->Compare(lhs.GetOptionalValue(), rhs.GetOptionalValue()); } } @@ -1102,12 +1102,12 @@ private: template <> class TCompare<NMiniKQL::TType::EKind::Tuple> final : public NUdf::ICompare { public: - explicit TCompare(const NMiniKQL::TType* type) { + explicit TCompare(const NMiniKQL::TType* type) { auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type); auto count = tupleType->GetElementsCount(); Compare_.reserve(count); for (ui32 i = 0; i < count; ++i) { - Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i))); + Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i))); } } @@ -1132,18 +1132,18 @@ private: }; template <> -class TCompare<NMiniKQL::TType::EKind::Variant> final : public NUdf::ICompare { +class TCompare<NMiniKQL::TType::EKind::Variant> final : public NUdf::ICompare { public: - explicit TCompare(const NMiniKQL::TType* type) { - auto variantType = static_cast<const NMiniKQL::TVariantType*>(type); - if (variantType->GetUnderlyingType()->IsStruct()) { - throw TTypeNotSupported() << "Variant over struct is unordered"; + explicit TCompare(const NMiniKQL::TType* type) { + auto variantType = static_cast<const NMiniKQL::TVariantType*>(type); + if (variantType->GetUnderlyingType()->IsStruct()) { + throw TTypeNotSupported() << "Variant over struct is unordered"; } - auto tupleType = static_cast<const NMiniKQL::TTupleType*>(variantType->GetUnderlyingType()); - ui32 count = tupleType->GetElementsCount(); + auto tupleType = static_cast<const NMiniKQL::TTupleType*>(variantType->GetUnderlyingType()); + ui32 count = tupleType->GetElementsCount(); Compare_.reserve(count); for (ui32 i = 0; i < count; ++i) { - Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i))); + Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i))); } } @@ -1175,8 +1175,8 @@ private: template <> class TCompare<NMiniKQL::TType::EKind::List> final : public NUdf::ICompare { public: - explicit TCompare(const NMiniKQL::TType* type) - : Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TListType*>(type)->GetItemType())) + explicit TCompare(const NMiniKQL::TType* type) + : Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TListType*>(type)->GetItemType())) {} bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { @@ -1540,7 +1540,7 @@ NUdf::IEquate::TPtr TFunctionTypeInfoBuilder::MakeEquate(const NUdf::TType* type NUdf::ICompare::TPtr TFunctionTypeInfoBuilder::MakeCompare(const NUdf::TType* type) { try { auto mkqlType = static_cast<const NMiniKQL::TType*>(type); - return MakeCompareImpl(mkqlType); + return MakeCompareImpl(mkqlType); } catch (const TTypeNotSupported& ex) { SetError(TStringBuf(ex.what())); return nullptr; @@ -1713,131 +1713,131 @@ void TTypeInfoHelper::DoTagged(const NMiniKQL::TTaggedType* tt, NUdf::ITypeVisit } } -NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type) { - switch (type->GetKind()) { - case NMiniKQL::TType::EKind::Data: { - -#define MAKE_HASH(slot, ...) \ - case NUdf::EDataSlot::slot: \ - return new THash<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; - - auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); - if (!slot) { - throw TTypeNotSupported() << "Invalid data slot"; - } - if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanHash)) { - throw TTypeNotSupported() << "Data type is not hashable"; - } - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_HASH) - } - -#undef MAKE_HASH - } - case NMiniKQL::TType::EKind::Optional: - return new THash<NMiniKQL::TType::EKind::Optional>(type); - case NMiniKQL::TType::EKind::Tuple: - return new THash<NMiniKQL::TType::EKind::Tuple>(type); - case NMiniKQL::TType::EKind::Struct: - return new THash<NMiniKQL::TType::EKind::Struct>(type); - case NMiniKQL::TType::EKind::List: - return new THash<NMiniKQL::TType::EKind::List>(type); - case NMiniKQL::TType::EKind::Variant: - return new THash<NMiniKQL::TType::EKind::Variant>(type); - case NMiniKQL::TType::EKind::Dict: - return new THash<NMiniKQL::TType::EKind::Dict>(type); - case NMiniKQL::TType::EKind::Void: - case NMiniKQL::TType::EKind::Null: - case NMiniKQL::TType::EKind::EmptyList: - case NMiniKQL::TType::EKind::EmptyDict: - return new TEmptyHash(); - default: - throw TTypeNotSupported() << "Data, Optional, Tuple, Struct, List, Variant or Dict is expected for hashing"; - } -} - -NUdf::ICompare::TPtr MakeCompareImpl(const NMiniKQL::TType* type) { - switch (type->GetKind()) { - case NMiniKQL::TType::EKind::Data: { - -#define MAKE_COMPARE(slot, ...) \ - case NUdf::EDataSlot::slot: \ - return new TCompare<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; - - auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); - if (!slot) { - throw TTypeNotSupported() << "Invalid data slot"; - } - if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) { - throw TTypeNotSupported() << "Data type is not comparable"; - } - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_COMPARE) - } - -#undef MAKE_COMPARE - } - case NMiniKQL::TType::EKind::Optional: - return new TCompare<NMiniKQL::TType::EKind::Optional>(type); - case NMiniKQL::TType::EKind::Tuple: - return new TCompare<NMiniKQL::TType::EKind::Tuple>(type); - case NMiniKQL::TType::EKind::Void: - case NMiniKQL::TType::EKind::Null: - case NMiniKQL::TType::EKind::EmptyList: - case NMiniKQL::TType::EKind::EmptyDict: - return new TEmptyCompare(); - case NMiniKQL::TType::EKind::Variant: { - return new TCompare<NMiniKQL::TType::EKind::Variant>(type); - } - case NMiniKQL::TType::EKind::List: - return new TCompare<NMiniKQL::TType::EKind::List>(type); - default: - throw TTypeNotSupported() << "Data, Optional, Variant over Tuple, Tuple or List is expected for comparing"; - } -} - -NUdf::IEquate::TPtr MakeEquateImpl(const NMiniKQL::TType* type) { - switch (type->GetKind()) { - case NMiniKQL::TType::EKind::Data: { - -#define MAKE_EQUATE(slot, ...) \ - case NUdf::EDataSlot::slot: \ - return new TEquate<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; - - auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); - if (!slot) { - throw TTypeNotSupported() << "Invalid data slot"; - } - if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanEquate)) { - throw TTypeNotSupported() << "Data type is not equatable"; - } - switch (*slot) { - UDF_TYPE_ID_MAP(MAKE_EQUATE) - } - -#undef MAKE_EQUATE - } - case NMiniKQL::TType::EKind::Optional: - return new TEquate<NMiniKQL::TType::EKind::Optional>(type); - case NMiniKQL::TType::EKind::Tuple: - return new TEquate<NMiniKQL::TType::EKind::Tuple>(type); - case NMiniKQL::TType::EKind::Struct: - return new TEquate<NMiniKQL::TType::EKind::Struct>(type); - case NMiniKQL::TType::EKind::List: - return new TEquate<NMiniKQL::TType::EKind::List>(type); - case NMiniKQL::TType::EKind::Void: - case NMiniKQL::TType::EKind::Null: - case NMiniKQL::TType::EKind::EmptyList: - case NMiniKQL::TType::EKind::EmptyDict: - return new TEmptyEquate(); - case NMiniKQL::TType::EKind::Variant: - return new TEquate<NMiniKQL::TType::EKind::Variant>(type); - case NMiniKQL::TType::EKind::Dict: - return new TEquate<NMiniKQL::TType::EKind::Dict>(type); - default: - throw TTypeNotSupported() << "Data, Optional, Tuple, Struct, List, Variant or Dict is expected for equating"; - } -} - +NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type) { + switch (type->GetKind()) { + case NMiniKQL::TType::EKind::Data: { + +#define MAKE_HASH(slot, ...) \ + case NUdf::EDataSlot::slot: \ + return new THash<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; + + auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); + if (!slot) { + throw TTypeNotSupported() << "Invalid data slot"; + } + if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanHash)) { + throw TTypeNotSupported() << "Data type is not hashable"; + } + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_HASH) + } + +#undef MAKE_HASH + } + case NMiniKQL::TType::EKind::Optional: + return new THash<NMiniKQL::TType::EKind::Optional>(type); + case NMiniKQL::TType::EKind::Tuple: + return new THash<NMiniKQL::TType::EKind::Tuple>(type); + case NMiniKQL::TType::EKind::Struct: + return new THash<NMiniKQL::TType::EKind::Struct>(type); + case NMiniKQL::TType::EKind::List: + return new THash<NMiniKQL::TType::EKind::List>(type); + case NMiniKQL::TType::EKind::Variant: + return new THash<NMiniKQL::TType::EKind::Variant>(type); + case NMiniKQL::TType::EKind::Dict: + return new THash<NMiniKQL::TType::EKind::Dict>(type); + case NMiniKQL::TType::EKind::Void: + case NMiniKQL::TType::EKind::Null: + case NMiniKQL::TType::EKind::EmptyList: + case NMiniKQL::TType::EKind::EmptyDict: + return new TEmptyHash(); + default: + throw TTypeNotSupported() << "Data, Optional, Tuple, Struct, List, Variant or Dict is expected for hashing"; + } +} + +NUdf::ICompare::TPtr MakeCompareImpl(const NMiniKQL::TType* type) { + switch (type->GetKind()) { + case NMiniKQL::TType::EKind::Data: { + +#define MAKE_COMPARE(slot, ...) \ + case NUdf::EDataSlot::slot: \ + return new TCompare<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; + + auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); + if (!slot) { + throw TTypeNotSupported() << "Invalid data slot"; + } + if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) { + throw TTypeNotSupported() << "Data type is not comparable"; + } + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_COMPARE) + } + +#undef MAKE_COMPARE + } + case NMiniKQL::TType::EKind::Optional: + return new TCompare<NMiniKQL::TType::EKind::Optional>(type); + case NMiniKQL::TType::EKind::Tuple: + return new TCompare<NMiniKQL::TType::EKind::Tuple>(type); + case NMiniKQL::TType::EKind::Void: + case NMiniKQL::TType::EKind::Null: + case NMiniKQL::TType::EKind::EmptyList: + case NMiniKQL::TType::EKind::EmptyDict: + return new TEmptyCompare(); + case NMiniKQL::TType::EKind::Variant: { + return new TCompare<NMiniKQL::TType::EKind::Variant>(type); + } + case NMiniKQL::TType::EKind::List: + return new TCompare<NMiniKQL::TType::EKind::List>(type); + default: + throw TTypeNotSupported() << "Data, Optional, Variant over Tuple, Tuple or List is expected for comparing"; + } +} + +NUdf::IEquate::TPtr MakeEquateImpl(const NMiniKQL::TType* type) { + switch (type->GetKind()) { + case NMiniKQL::TType::EKind::Data: { + +#define MAKE_EQUATE(slot, ...) \ + case NUdf::EDataSlot::slot: \ + return new TEquate<NMiniKQL::TType::EKind::Data, NUdf::EDataSlot::slot>; + + auto slot = static_cast<const NMiniKQL::TDataType*>(type)->GetDataSlot(); + if (!slot) { + throw TTypeNotSupported() << "Invalid data slot"; + } + if (!(NUdf::GetDataTypeInfo(*slot).Features & NUdf::CanEquate)) { + throw TTypeNotSupported() << "Data type is not equatable"; + } + switch (*slot) { + UDF_TYPE_ID_MAP(MAKE_EQUATE) + } + +#undef MAKE_EQUATE + } + case NMiniKQL::TType::EKind::Optional: + return new TEquate<NMiniKQL::TType::EKind::Optional>(type); + case NMiniKQL::TType::EKind::Tuple: + return new TEquate<NMiniKQL::TType::EKind::Tuple>(type); + case NMiniKQL::TType::EKind::Struct: + return new TEquate<NMiniKQL::TType::EKind::Struct>(type); + case NMiniKQL::TType::EKind::List: + return new TEquate<NMiniKQL::TType::EKind::List>(type); + case NMiniKQL::TType::EKind::Void: + case NMiniKQL::TType::EKind::Null: + case NMiniKQL::TType::EKind::EmptyList: + case NMiniKQL::TType::EKind::EmptyDict: + return new TEmptyEquate(); + case NMiniKQL::TType::EKind::Variant: + return new TEquate<NMiniKQL::TType::EKind::Variant>(type); + case NMiniKQL::TType::EKind::Dict: + return new TEquate<NMiniKQL::TType::EKind::Dict>(type); + default: + throw TTypeNotSupported() << "Data, Optional, Tuple, Struct, List, Variant or Dict is expected for equating"; + } +} + } // namespace NMiniKQL } // namespace Nkikimr diff --git a/ydb/library/yql/minikql/mkql_type_builder.h b/ydb/library/yql/minikql/mkql_type_builder.h index c0a52e47e8..a93fb41cee 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.h +++ b/ydb/library/yql/minikql/mkql_type_builder.h @@ -161,9 +161,9 @@ private: static void DoTagged(const NMiniKQL::TTaggedType* tt, NUdf::ITypeVisitor* v); }; -NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type); -NUdf::ICompare::TPtr MakeCompareImpl(const NMiniKQL::TType* type); -NUdf::IEquate::TPtr MakeEquateImpl(const NMiniKQL::TType* type); - +NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type); +NUdf::ICompare::TPtr MakeCompareImpl(const NMiniKQL::TType* type); +NUdf::IEquate::TPtr MakeEquateImpl(const NMiniKQL::TType* type); + } // namespace NMiniKQL } // namespace Nkikimr diff --git a/ydb/library/yql/mount/lib/yql/aggregate.yql b/ydb/library/yql/mount/lib/yql/aggregate.yql index a408a63541..a1c231f339 100644 --- a/ydb/library/yql/mount/lib/yql/aggregate.yql +++ b/ydb/library/yql/mount/lib/yql/aggregate.yql @@ -65,7 +65,7 @@ # list_type:type # support optional values -(let count_if_traits_factory_opt (lambda '(list_type) (Apply count_traits_factory list_type (lambda '(value) (If (Coalesce value (Bool '0)) (Uint64 '1) (Uint64 '0)))))) +(let count_if_traits_factory_opt (lambda '(list_type) (Apply count_traits_factory list_type (lambda '(value) (If (Coalesce value (Bool '0)) (Uint64 '1) (Uint64 '0)))))) (let count_all_traits_factory_opt (lambda '(list_type) (Apply count_traits_factory list_type (lambda '(value) (Uint64 '1))))) # list_type:type diff --git a/ydb/library/yql/mount/lib/yql/window.yql b/ydb/library/yql/mount/lib/yql/window.yql index bd8f59531d..371f4c687c 100644 --- a/ydb/library/yql/mount/lib/yql/window.yql +++ b/ydb/library/yql/mount/lib/yql/window.yql @@ -2,7 +2,7 @@ ( (let empty_list (lambda '(list_type) (MatchType list_type 'Null (lambda '() (EmptyList)) (lambda '() (List list_type))))) - + # list_type:type function:lambda # doesn't support optional values # simple function not support shift and recalculated on whole window @@ -11,7 +11,7 @@ (let update (lambda '(value state) (Apply function value state))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) state)) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type function:lambda @@ -22,7 +22,7 @@ (let update (lambda '(value state) (Apply function_update value state))) (let shift (lambda '(value state) (Apply function_shift value state))) (let current (lambda '(state) state)) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type function:lambda @@ -33,7 +33,7 @@ (let update (lambda '(value state) (Apply function_update (Apply function_map value) state))) (let shift (lambda '(value state) (Apply function_shift (Apply function_map value) state))) (let current (lambda '(state) state)) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type, factory:lambda, extractor:lambda @@ -43,10 +43,10 @@ (let update (NthArg '2 traits)) (let shift (lambda '(value state) (Void))) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_ext (lambda '(row) (Apply init (Apply extractor row)))) (let update_ext (lambda '(row state) (Apply update (Apply extractor row) state))) - (return (WindowTraits (ListItemType list_type) init_ext update_ext shift current defval)) + (return (WindowTraits (ListItemType list_type) init_ext update_ext shift current defval)) )))) (let rank_traits_factory_raw (lambda '(list_type) (block '( @@ -54,7 +54,7 @@ (let update (lambda '(value state) '((If (== value (Nth state '2)) (Nth state '0) (Inc (Nth state '1))) (Inc (Nth state '1)) value))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Nth state '0))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) (let dense_rank_traits_factory_raw (lambda '(list_type) (block '( @@ -62,7 +62,7 @@ (let update (lambda '(value state) '((If (== value (Nth state '1)) (Nth state '0) (Inc (Nth state '0))) value))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Nth state '0))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type @@ -90,7 +90,7 @@ (let update (lambda '(value state) (+ state (Apply init value)))) (let shift (lambda '(value state) (+ state (Minus (Apply init value))))) (let current (lambda '(state) state)) - (return (WindowTraits (ListItemType list_type) init update shift current (Uint64 '0))) + (return (WindowTraits (ListItemType list_type) init update shift current (Uint64 '0))) )))) # list_type:type @@ -100,7 +100,7 @@ (let update (lambda '(value state) (AggrCountUpdate value state))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) state)) - (return (WindowTraits (ListItemType list_type) init update shift current (Uint64 '0))) + (return (WindowTraits (ListItemType list_type) init update shift current (Uint64 '0))) )))) # list_type:type @@ -120,12 +120,12 @@ 'Decimal (lambda '() (Cast (Div (Nth state '0) (Nth state '1)) (ListItemType list_type))) 'Interval (lambda '() (Apply convert_decimal_to_interval (Div (Nth state '0) (Nth state '1)) ) ) (lambda '() (Div (Nth state '0) (Nth state '1)))))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type # support optional values -(let list_traits_factory_opt_gen (lambda '(limit defval) (block '( +(let list_traits_factory_opt_gen (lambda '(limit defval) (block '( (return (lambda '(list_type) (block '( (let item_type (ListItemType list_type)) (let init (lambda '(value) (MatchType item_type @@ -140,7 +140,7 @@ (return (If (== limit (Uint64 '0)) x (Take x limit))))))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) state)) - (return (WindowTraits item_type init update shift current defval)) + (return (WindowTraits item_type init update shift current defval)) )))))))) # list_type:type stddev:bool sample:bool @@ -163,7 +163,7 @@ (let count (Nth state '1)) (let result (/ (Nth state '2) (If sample (- count (Double '1)) count))) (return (If stddev (Apply (Udf 'Math.Sqrt) result) result)))))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type @@ -190,7 +190,7 @@ (let divisor2 (- divisor2 (* (Nth state '2) (Nth state '2)))) (return (/ dividend (Apply (Udf 'Math.Sqrt) (* divisor1 divisor2)))) )))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type sample:bool @@ -204,7 +204,7 @@ (let covariance_result (/ covariance_result (If sample (- (Nth state '0) (Uint64 '1)) (Nth state '0)))) (return covariance_result) )))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type @@ -219,7 +219,7 @@ (let update (lambda '(row state parent) (NamedApply (Udf (Combine histogram '_AddValue)) '(state (Apply value row) (Apply weight row)) (AsStruct) (DependsOn parent)))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Apply (Udf (Combine histogram '_GetResult)) state))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type n:ui32 buffer:ui32 @@ -330,7 +330,7 @@ 'Tuple (lambda '() (StaticMap n (lambda '(n) (Apply get_convert_percentile state n)))) (lambda '() (Apply get_convert_percentile state n)) ))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type @@ -340,7 +340,7 @@ (let update (lambda '(value state parent) (NamedApply (Udf 'HyperLogLog.AddValue) '(state (Apply (Udf 'Digest.MurMurHash) (Pickle value))) (AsStruct) (DependsOn parent)))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Apply (Udf 'HyperLogLog.GetResult) state))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type value:lambda weight:lambda intervals:integer @@ -358,7 +358,7 @@ (let update (lambda '(row state parent) (NamedApply (Udf (Combine histogram '_AddValue)) '(state (Apply value row) (Double '"1.0") (AsStruct) (DependsOn parent))))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Apply (Udf (Combine histogram '_GetResult)) state))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type value:lambda binsize:double minimum:double maximum:double @@ -375,7 +375,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_optional (lambda '(value) @@ -416,7 +416,7 @@ (let shift_optional (lambda '(value state) (Void))) (let current_optional (lambda '(state) (MatchType item_type 'Optional (lambda '() (Map state current)) 'Null (lambda '() defval) (lambda '() (Apply current state))))) - (return (WindowTraits item_type init_optional update_optional shift_optional current_optional defval)) + (return (WindowTraits item_type init_optional update_optional shift_optional current_optional defval)) )))) # list_type:type factory:lambda @@ -428,7 +428,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_optional (lambda '(value parent) @@ -468,7 +468,7 @@ ) (let current_optional (lambda '(state) (MatchType item_type 'Optional (lambda '() (Map state current)) 'Null (lambda '() defval) (lambda '() (Apply current state))))) - (return (WindowTraits item_type init_optional update_optional shift_optional current_optional defval)) + (return (WindowTraits item_type init_optional update_optional shift_optional current_optional defval)) )))) # list_type:type factory:lambda @@ -564,7 +564,7 @@ # list_type:type init:lambda update:lambda shift:lambda current:lambda # support optional values -(let udwf_traits_factory_opt (lambda '(list_type init update shift current defval) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (WindowTraits (ListItemType list_type) init update shift current defval))))) +(let udwf_traits_factory_opt (lambda '(list_type init update shift current defval) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (WindowTraits (ListItemType list_type) init update shift current defval))))) # list_type:type, factory:lambda, extractor:lambda (let extractor_traits_factory (lambda '(list_type extractor factory) (block '( @@ -573,7 +573,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_ext (lambda '(row) (Apply init (Apply extractor row)))) (let update_ext (lambda '(row state) (Apply update (Apply extractor row) state))) (let shift_ext (lambda '(row state) (Apply shift (Apply extractor row) state))) @@ -587,7 +587,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_ext (lambda '(row parent) (Apply init (Apply extractor row) parent))) (let update_ext (lambda '(row state parent) (Apply update (Apply extractor row) state parent))) (let shift_ext (lambda '(row state) (Apply shift (Apply extractor row) state))) @@ -610,17 +610,17 @@ (let or_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type (lambda '(v) (SafeCast (Apply extractor v) (DataType 'Bool))) bool_or_traits_factory_opt))) (let xor_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type (lambda '(v) (SafeCast (Apply extractor v) (DataType 'Bool))) bool_xor_traits_factory_opt))) (let avg_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor avg_traits_factory_opt))) -(let list_traits_factory (lambda '(list_type extractor limit) (Apply extractor_traits_factory list_type extractor (Apply list_traits_factory_opt_gen limit (Null))))) -(let list2_traits_factory (lambda '(list_type extractor limit) (Apply extractor_traits_factory list_type extractor (Apply list_traits_factory_opt_gen limit empty_list)))) +(let list_traits_factory (lambda '(list_type extractor limit) (Apply extractor_traits_factory list_type extractor (Apply list_traits_factory_opt_gen limit (Null))))) +(let list2_traits_factory (lambda '(list_type extractor limit) (Apply extractor_traits_factory list_type extractor (Apply list_traits_factory_opt_gen limit empty_list)))) (let variance_0_0_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor variance_0_0_traits_factory_opt))) (let variance_1_0_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor variance_1_0_traits_factory_opt))) (let variance_0_1_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor variance_0_1_traits_factory_opt))) (let variance_1_1_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor variance_1_1_traits_factory_opt))) -# TODO: rank_traits_factory and dense_rank_traits_factory are only used in v0 syntax +# TODO: rank_traits_factory and dense_rank_traits_factory are only used in v0 syntax (let rank_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor rank_traits_factory_opt))) (let dense_rank_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor dense_rank_traits_factory_opt))) - + (let first_value_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor first_value_traits_factory_opt))) (let last_value_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor last_value_traits_factory_opt))) (let first_value_ignore_nulls_traits_factory (lambda '(list_type extractor) (Apply extractor_traits_factory list_type extractor first_value_ignore_nulls_traits_factory_opt))) @@ -644,7 +644,7 @@ # list_type:type extractor:lambda init:lambda update:lambda shift:lambda current:lambda # support optional values -(let udwf_traits_factory (lambda '(list_type extractor init update shift current defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udwf_traits_factory_opt list_type init update shift current defval))))) +(let udwf_traits_factory (lambda '(list_type extractor init update shift current defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udwf_traits_factory_opt list_type init update shift current defval))))) # list_type:type compare:lambda first:lambda second:lambda stub # doesn't support optional values @@ -654,7 +654,7 @@ (let update (lambda '(row state) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state))) (let shift (lambda '(row state) (lambda '(value state) (Void)))) (let current (lambda '(state) (Nth state '0))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # list_type:type compare:lambda first:lambda second:lambda limit:Uint64 @@ -665,7 +665,7 @@ (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state)))) (let shift (lambda '(value state) (Void))) (let current (lambda '(state) (Nth state '0))) - (return (WindowTraits (ListItemType list_type) init update shift current (Null))) + (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) # factory:lambda list_type:type first:lambda second:lambda @@ -685,7 +685,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_opt (lambda '(row) (MatchType test_type 'Optional (lambda '() (Map (Apply first row) (lambda '(key) (Apply init row)))) @@ -705,7 +705,7 @@ )) (lambda '() (Apply current state)) ))) - (return (WindowTraits item_type init_opt update_opt shift_opt current_opt defval)) + (return (WindowTraits item_type init_opt update_opt shift_opt current_opt defval)) )))) # factory:lambda list_type:type first:lambda second:lambda @@ -725,7 +725,7 @@ (let update (NthArg '2 traits)) (let shift (NthArg '3 traits)) (let current (NthArg '4 traits)) - (let defval (NthArg '5 traits)) + (let defval (NthArg '5 traits)) (let init_opt (lambda '(row parent) (MatchType test_type 'Optional (lambda '() (Map (Apply first row) (lambda '(key) (Apply init row parent)))) @@ -739,7 +739,7 @@ )) (let shift_opt (lambda '(value state) (Void))) (let current_opt (lambda '(state) (MatchType test_type 'Optional (lambda '() (Map state current)) (lambda '() (Apply current state))))) - (return (WindowTraits item_type init_opt update_opt shift_opt current_opt defval)) + (return (WindowTraits item_type init_opt update_opt shift_opt current_opt defval)) )))) # factory:lambda list_type:type first:lambda second:lambda @@ -828,9 +828,9 @@ # list_type:type init:lambda update:lambda finish:lambda # support optional values -(let udaf_traits_factory_opt (lambda '(list_type init update finish defval) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (WindowTraits (ListItemType list_type) init update (lambda '(value state) (Void)) finish defval))))) +(let udaf_traits_factory_opt (lambda '(list_type init update finish defval) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (WindowTraits (ListItemType list_type) init update (lambda '(value state) (Void)) finish defval))))) -(let udaf_traits_factory (lambda '(list_type extractor init update merge finish save load defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udaf_traits_factory_opt list_type init update finish defval))))) +(let udaf_traits_factory (lambda '(list_type extractor init update merge finish save load defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udaf_traits_factory_opt list_type init update finish defval))))) (let top_bottom_traits_factory (lambda '(list_type extractor count is_top) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) diff --git a/ydb/library/yql/parser/lexer_common/lexer.h b/ydb/library/yql/parser/lexer_common/lexer.h index d5622f9b48..ad19f09b8f 100644 --- a/ydb/library/yql/parser/lexer_common/lexer.h +++ b/ydb/library/yql/parser/lexer_common/lexer.h @@ -1,37 +1,37 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/stream/output.h> - -namespace NYql { - -class TIssues; - -} - -namespace NSQLTranslation { - -struct TParsedToken { - // TODO: TStringBuf for Name & Content - TString Name; - TString Content; - // Position of first token symbol - ui32 Line = 0; // starts from 1 - ui32 LinePos = 0; // starts from 0 -}; - -using TParsedTokenList = TVector<TParsedToken>; - -IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end); - -class ILexer { -public: - using TPtr = THolder<ILexer>; - - virtual bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) = 0; - virtual ~ILexer() = default; -}; - -} - +#pragma once + +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/stream/output.h> + +namespace NYql { + +class TIssues; + +} + +namespace NSQLTranslation { + +struct TParsedToken { + // TODO: TStringBuf for Name & Content + TString Name; + TString Content; + // Position of first token symbol + ui32 Line = 0; // starts from 1 + ui32 LinePos = 0; // starts from 0 +}; + +using TParsedTokenList = TVector<TParsedToken>; + +IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end); + +class ILexer { +public: + using TPtr = THolder<ILexer>; + + virtual bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) = 0; + virtual ~ILexer() = default; +}; + +} + diff --git a/ydb/library/yql/parser/lexer_common/tokens.cpp b/ydb/library/yql/parser/lexer_common/tokens.cpp index 2f519d6600..87a5dd3088 100644 --- a/ydb/library/yql/parser/lexer_common/tokens.cpp +++ b/ydb/library/yql/parser/lexer_common/tokens.cpp @@ -1,13 +1,13 @@ -#include "lexer.h" - - -namespace NSQLTranslation { - -IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end) { - for (auto it = begin; it != end; ++it) { - out << it->Content; - } - return out; -} - -} +#include "lexer.h" + + +namespace NSQLTranslation { + +IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end) { + for (auto it = begin; it != end; ++it) { + out << it->Content; + } + return out; +} + +} diff --git a/ydb/library/yql/parser/lexer_common/ya.make b/ydb/library/yql/parser/lexer_common/ya.make index c6fd662c46..238c36ab3f 100644 --- a/ydb/library/yql/parser/lexer_common/ya.make +++ b/ydb/library/yql/parser/lexer_common/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - tokens.cpp - lexer.h -) - -END() +LIBRARY() + +OWNER(g:yql) + +SRCS( + tokens.cpp + lexer.h +) + +END() diff --git a/ydb/library/yql/parser/pg_query_wrapper/apply.sh b/ydb/library/yql/parser/pg_query_wrapper/apply.sh index 1194e86017..8d28bfe287 100755 --- a/ydb/library/yql/parser/pg_query_wrapper/apply.sh +++ b/ydb/library/yql/parser/pg_query_wrapper/apply.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eux -mkdir -p contrib - +mkdir -p contrib + cp original/pg_query.h contrib/ cp original/.gitignore contrib/ cp original/CHANGELOG.md contrib/ @@ -23,7 +23,7 @@ cp -R original/vendor contrib/ cp -R original/src contrib/ cp patches/src/postgres/include/pg_config.h contrib/src/postgres/include cp patches/src/postgres/include/pg_config_manual.h contrib/src/postgres/include - -for i in $(find patches -name "*.patch" | sort); do - cat $i | patch -p0 -done + +for i in $(find patches -name "*.patch" | sort); do + cat $i | patch -p0 +done diff --git a/ydb/library/yql/parser/pg_query_wrapper/contrib/src/postgres/src_backend_utils_mb_mbutils.c b/ydb/library/yql/parser/pg_query_wrapper/contrib/src/postgres/src_backend_utils_mb_mbutils.c index f90c901402..debf461a3c 100644 --- a/ydb/library/yql/parser/pg_query_wrapper/contrib/src/postgres/src_backend_utils_mb_mbutils.c +++ b/ydb/library/yql/parser/pg_query_wrapper/contrib/src/postgres/src_backend_utils_mb_mbutils.c @@ -619,11 +619,11 @@ GetDatabaseEncodingName(void) int pg_database_encoding_max_length(void) { - int encoding = GetDatabaseEncoding(); - if (!PG_VALID_ENCODING(encoding)) { - pg_unreachable(); - } - return pg_wchar_table[encoding].maxmblen; + int encoding = GetDatabaseEncoding(); + if (!PG_VALID_ENCODING(encoding)) { + pg_unreachable(); + } + return pg_wchar_table[encoding].maxmblen; } /* @@ -660,9 +660,9 @@ pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError) mbverifier mbverify; int mb_len; - if (!PG_VALID_ENCODING(encoding)) { - pg_unreachable(); - } + if (!PG_VALID_ENCODING(encoding)) { + pg_unreachable(); + } /* * In single-byte encodings, we need only reject nulls (\0). diff --git a/ydb/library/yql/parser/pg_query_wrapper/patches/010_build.patch b/ydb/library/yql/parser/pg_query_wrapper/patches/010_build.patch index e1cec211c9..c369b4d88b 100644 --- a/ydb/library/yql/parser/pg_query_wrapper/patches/010_build.patch +++ b/ydb/library/yql/parser/pg_query_wrapper/patches/010_build.patch @@ -1,1152 +1,1152 @@ -diff -ruN original/pg_query.h my/pg_query.h ---- original/pg_query.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/pg_query.h 2021-11-14 20:05:00.000000000 +0300 -@@ -57,29 +57,29 @@ - PgQueryError* error; - } PgQueryPlpgsqlParseResult; - --typedef struct { -+/*typedef struct { - uint64_t fingerprint; - char* fingerprint_str; - char* stderr_buffer; - PgQueryError* error; --} PgQueryFingerprintResult; -+} PgQueryFingerprintResult;*/ - --typedef struct { -+/*typedef struct { - char* normalized_query; - PgQueryError* error; --} PgQueryNormalizeResult; -+} PgQueryNormalizeResult;*/ - - #ifdef __cplusplus - extern "C" { - #endif - --PgQueryNormalizeResult pg_query_normalize(const char* input); -+//PgQueryNormalizeResult pg_query_normalize(const char* input); - PgQueryScanResult pg_query_scan(const char* input); - PgQueryParseResult pg_query_parse(const char* input); - PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); --PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); -+//PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); - --PgQueryFingerprintResult pg_query_fingerprint(const char* input); -+//PgQueryFingerprintResult pg_query_fingerprint(const char* input); - - // Use pg_query_split_with_scanner when you need to split statements that may - // contain parse errors, otherwise pg_query_split_with_parser is recommended -@@ -93,14 +93,14 @@ - - PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree); - --void pg_query_free_normalize_result(PgQueryNormalizeResult result); -+//void pg_query_free_normalize_result(PgQueryNormalizeResult result); - void pg_query_free_scan_result(PgQueryScanResult result); - void pg_query_free_parse_result(PgQueryParseResult result); - void pg_query_free_split_result(PgQuerySplitResult result); - void pg_query_free_deparse_result(PgQueryDeparseResult result); - void pg_query_free_protobuf_parse_result(PgQueryProtobufParseResult result); --void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); --void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); -+//void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); -+//void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); - - // Optional, cleans up the top-level memory context (automatically done for threads that exit) - void pg_query_exit(void); -diff -ruN original/src/pg_query.c my/src/pg_query.c ---- original/src/pg_query.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/pg_query.c 2021-11-14 20:05:00.000000000 +0300 -@@ -5,15 +5,19 @@ - #include <utils/memutils.h> - #include <utils/memdebug.h> - -+#ifndef WIN32 - #include <pthread.h> -+#endif - #include <signal.h> - - const char* progname = "pg_query"; - --__thread sig_atomic_t pg_query_initialized = 0; -+__thread volatile sig_atomic_t pg_query_initialized = 0; - -+#ifndef WIN32 - static pthread_key_t pg_query_thread_exit_key; - static void pg_query_thread_exit(void *key); -+#endif - - void pg_query_init(void) - { -@@ -23,8 +27,10 @@ - MemoryContextInit(); - SetDatabaseEncoding(PG_UTF8); - -+#ifndef WIN32 - pthread_key_create(&pg_query_thread_exit_key, pg_query_thread_exit); - pthread_setspecific(pg_query_thread_exit_key, TopMemoryContext); -+#endif - } - - void pg_query_free_top_memory_context(MemoryContext context) -@@ -55,11 +61,13 @@ - ErrorContext = NULL; - } - -+#ifndef WIN32 - static void pg_query_thread_exit(void *key) - { - MemoryContext context = (MemoryContext) key; - pg_query_free_top_memory_context(context); - } -+#endif - - void pg_query_exit(void) - { -diff -ruN original/src/pg_query_parse.c my/src/pg_query_parse.c ---- original/src/pg_query_parse.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/pg_query_parse.c 2021-11-14 20:05:00.000000000 +0300 -@@ -6,7 +6,7 @@ - #include "parser/scanner.h" - #include "parser/scansup.h" - --#include <unistd.h> -+//#include <unistd.h> - #include <fcntl.h> - - PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input) -@@ -111,8 +111,10 @@ - { - MemoryContext ctx = NULL; - PgQueryInternalParsetreeAndError parsetree_and_error; -- PgQueryProtobufParseResult result = {}; -+ PgQueryProtobufParseResult result; -+ memset(&result, 0, sizeof(result)); - -+ - ctx = pg_query_enter_memory_context(); - - parsetree_and_error = pg_query_raw_parse(input); -diff -ruN original/src/postgres/include/access/xlog.h my/src/postgres/include/access/xlog.h ---- original/src/postgres/include/access/xlog.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/access/xlog.h 2021-11-14 20:05:00.000000000 +0300 -@@ -18,7 +18,7 @@ - #include "datatype/timestamp.h" - #include "lib/stringinfo.h" - #include "nodes/pg_list.h" --#include "storage/fd.h" -+//#include "storage/fd.h" - - - /* Sync methods */ -diff -ruN original/src/postgres/include/c.h my/src/postgres/include/c.h ---- original/src/postgres/include/c.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/c.h 2021-11-14 20:05:00.000000000 +0300 -@@ -62,7 +62,7 @@ - #include <stddef.h> - #include <stdarg.h> - #ifdef HAVE_STRINGS_H --#include <strings.h> -+//#include <strings.h> - #endif - #include <stdint.h> - #include <sys/types.h> -@@ -135,8 +135,10 @@ - - /* GCC and XLC support format attributes */ - #if defined(__GNUC__) || defined(__IBMC__) --#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) --#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) -+#define pg_attribute_format_arg(a) -+// __attribute__((format_arg(a))) -+#define pg_attribute_printf(f,a) -+// __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) - #else - #define pg_attribute_format_arg(a) - #define pg_attribute_printf(f,a) -diff -ruN original/src/postgres/include/getaddrinfo.h my/src/postgres/include/getaddrinfo.h ---- original/src/postgres/include/getaddrinfo.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/getaddrinfo.h 2021-11-14 20:05:00.000000000 +0300 -@@ -22,8 +22,8 @@ - #ifndef GETADDRINFO_H - #define GETADDRINFO_H - --#include <sys/socket.h> --#include <netdb.h> -+//#include <sys/socket.h> -+//#include <netdb.h> - - - /* Various macros that ought to be in <netdb.h>, but might not be */ -diff -ruN original/src/postgres/include/libpq/libpq-be.h my/src/postgres/include/libpq/libpq-be.h ---- original/src/postgres/include/libpq/libpq-be.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/libpq/libpq-be.h 2021-11-14 20:05:00.000000000 +0300 -@@ -18,10 +18,10 @@ - #ifndef LIBPQ_BE_H - #define LIBPQ_BE_H - --#include <sys/time.h> -+//#include <sys/time.h> - #ifdef USE_OPENSSL --#include <openssl/ssl.h> --#include <openssl/err.h> -+//#include <openssl/ssl.h> -+//#include <openssl/err.h> - #endif - #ifdef HAVE_NETINET_TCP_H - #include <netinet/tcp.h> -diff -ruN original/src/postgres/include/libpq/libpq.h my/src/postgres/include/libpq/libpq.h ---- original/src/postgres/include/libpq/libpq.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/libpq/libpq.h 2021-11-14 20:05:00.000000000 +0300 -@@ -14,7 +14,7 @@ - #ifndef LIBPQ_H - #define LIBPQ_H - --#include <netinet/in.h> -+//#include <netinet/in.h> - - #include "lib/stringinfo.h" - #include "libpq/libpq-be.h" -diff -ruN original/src/postgres/include/libpq/pqcomm.h my/src/postgres/include/libpq/pqcomm.h ---- original/src/postgres/include/libpq/pqcomm.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/libpq/pqcomm.h 2021-11-14 20:05:00.000000000 +0300 -@@ -16,12 +16,12 @@ - #ifndef PQCOMM_H - #define PQCOMM_H - --#include <sys/socket.h> --#include <netdb.h> -+//#include <sys/socket.h> -+//#include <netdb.h> - #ifdef HAVE_SYS_UN_H - #include <sys/un.h> - #endif --#include <netinet/in.h> -+//#include <netinet/in.h> - - #ifdef HAVE_STRUCT_SOCKADDR_STORAGE - -diff -ruN original/src/postgres/include/miscadmin.h my/src/postgres/include/miscadmin.h ---- original/src/postgres/include/miscadmin.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/miscadmin.h 2021-11-14 20:05:00.000000000 +0300 -@@ -94,7 +94,7 @@ - /* in tcop/postgres.c */ - extern void ProcessInterrupts(void); - --#ifndef WIN32 -+#if 1 - - #define CHECK_FOR_INTERRUPTS() \ - do { \ -diff -ruN original/src/postgres/include/nodes/execnodes.h my/src/postgres/include/nodes/execnodes.h ---- original/src/postgres/include/nodes/execnodes.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/nodes/execnodes.h 2021-11-14 20:05:00.000000000 +0300 -@@ -26,7 +26,7 @@ - #include "utils/hsearch.h" - #include "utils/queryenvironment.h" - #include "utils/reltrigger.h" --#include "utils/sharedtuplestore.h" -+//#include "utils/sharedtuplestore.h" - #include "utils/snapshot.h" - #include "utils/sortsupport.h" - #include "utils/tuplesort.h" -diff -ruN original/src/postgres/include/pg_getopt.h my/src/postgres/include/pg_getopt.h ---- original/src/postgres/include/pg_getopt.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/pg_getopt.h 2021-11-14 20:05:00.000000000 +0300 -@@ -19,7 +19,7 @@ - #define PG_GETOPT_H - - /* POSIX says getopt() is provided by unistd.h */ --#include <unistd.h> -+//#include <unistd.h> - - /* rely on the system's getopt.h if present */ - #ifdef HAVE_GETOPT_H -diff -ruN original/src/postgres/include/port/atomics.h my/src/postgres/include/port/atomics.h ---- original/src/postgres/include/port/atomics.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/port/atomics.h 2021-11-14 20:05:00.000000000 +0300 -@@ -65,15 +65,15 @@ - */ - #if defined(__arm__) || defined(__arm) || \ - defined(__aarch64__) || defined(__aarch64) --#include "port/atomics/arch-arm.h" -+//#include "port/atomics/arch-arm.h" - #elif defined(__i386__) || defined(__i386) || defined(__x86_64__) --#include "port/atomics/arch-x86.h" -+//#include "port/atomics/arch-x86.h" - #elif defined(__ia64__) || defined(__ia64) --#include "port/atomics/arch-ia64.h" -+//#include "port/atomics/arch-ia64.h" - #elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) --#include "port/atomics/arch-ppc.h" -+//#include "port/atomics/arch-ppc.h" - #elif defined(__hppa) || defined(__hppa__) --#include "port/atomics/arch-hppa.h" -+//#include "port/atomics/arch-hppa.h" - #endif - - /* -@@ -94,11 +94,11 @@ - #if (defined(__GNUC__) || defined(__INTEL_COMPILER)) && !(defined(__IBMC__) || defined(__IBMCPP__)) - #include "port/atomics/generic-gcc.h" - #elif defined(_MSC_VER) --#include "port/atomics/generic-msvc.h" -+//#include "port/atomics/generic-msvc.h" - #elif defined(__hpux) && defined(__ia64) && !defined(__GNUC__) --#include "port/atomics/generic-acc.h" -+//#include "port/atomics/generic-acc.h" - #elif defined(__SUNPRO_C) && !defined(__GNUC__) --#include "port/atomics/generic-sunpro.h" -+//#include "port/atomics/generic-sunpro.h" - #else - /* - * Unsupported compiler, we'll likely use slower fallbacks... At least -diff -ruN original/src/postgres/include/port/pg_bitutils.h my/src/postgres/include/port/pg_bitutils.h ---- original/src/postgres/include/port/pg_bitutils.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/port/pg_bitutils.h 2021-11-14 20:05:00.000000000 +0300 -@@ -207,10 +207,18 @@ - return pg_leftmost_one_pos64(num - 1) + 1; - } - --/* Count the number of one-bits in a uint32 or uint64 */ -+#ifdef TRY_POPCNT_FAST -+/* Attempt to use the POPCNT instruction, but perform a runtime check first */ - extern int (*pg_popcount32) (uint32 word); - extern int (*pg_popcount64) (uint64 word); - -+#else -+/* Use a portable implementation -- no need for a function pointer. */ -+extern int pg_popcount32(uint32 word); -+extern int pg_popcount64(uint64 word); -+ -+#endif -+ - /* Count the number of one-bits in a byte array */ - extern uint64 pg_popcount(const char *buf, int bytes); - -diff -ruN original/src/postgres/include/port.h my/src/postgres/include/port.h ---- original/src/postgres/include/port.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/port.h 2021-11-14 20:05:00.000000000 +0300 -@@ -14,8 +14,8 @@ - #define PG_PORT_H - - #include <ctype.h> --#include <netdb.h> --#include <pwd.h> -+//#include <netdb.h> -+//#include <pwd.h> - - /* - * Windows has enough specialized port stuff that we push most of it off -@@ -23,7 +23,7 @@ - * Note: Some CYGWIN includes might #define WIN32. - */ - #if defined(WIN32) && !defined(__CYGWIN__) --#include "port/win32_port.h" -+//#include "port/win32_port.h" - #endif - - /* socket has a different definition on WIN32 */ -@@ -32,6 +32,7 @@ - - #define PGINVALID_SOCKET (-1) - #else -+#include <winsock2.h> - typedef SOCKET pgsocket; - - #define PGINVALID_SOCKET INVALID_SOCKET -@@ -462,15 +463,15 @@ - - /* thread.h */ - #ifndef WIN32 --extern int pqGetpwuid(uid_t uid, struct passwd *resultbuf, char *buffer, -- size_t buflen, struct passwd **result); -+//extern int pqGetpwuid(uid_t uid, struct passwd *resultbuf, char *buffer, -+// size_t buflen, struct passwd **result); - #endif - --extern int pqGethostbyname(const char *name, -+/*extern int pqGethostbyname(const char *name, - struct hostent *resultbuf, - char *buffer, size_t buflen, - struct hostent **result, -- int *herrno); -+ int *herrno);*/ - - extern void pg_qsort(void *base, size_t nel, size_t elsize, - int (*cmp) (const void *, const void *)); -diff -ruN original/src/postgres/include/storage/pmsignal.h my/src/postgres/include/storage/pmsignal.h ---- original/src/postgres/include/storage/pmsignal.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/storage/pmsignal.h 2021-11-14 20:05:00.000000000 +0300 -@@ -21,7 +21,7 @@ - #endif - - #ifdef HAVE_SYS_PROCCTL_H --#include "sys/procctl.h" -+//#include "sys/procctl.h" - #endif - - /* -diff -ruN original/src/postgres/include/storage/s_lock.h my/src/postgres/include/storage/s_lock.h ---- original/src/postgres/include/storage/s_lock.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/storage/s_lock.h 2021-11-14 20:05:00.000000000 +0300 -@@ -676,7 +676,7 @@ - #if defined(__m32r__) && defined(HAVE_SYS_TAS_H) /* Renesas' M32R */ - #define HAS_TEST_AND_SET - --#include <sys/tas.h> -+//#include <sys/tas.h> - - typedef int slock_t; - -@@ -837,7 +837,7 @@ - - typedef unsigned int slock_t; - --#include <ia64/sys/inline.h> -+//#include <ia64/sys/inline.h> - #define TAS(lock) _Asm_xchg(_SZ_W, lock, 1, _LDHINT_NONE) - /* On IA64, it's a win to use a non-locking test before the xchg proper */ - #define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) -diff -ruN original/src/postgres/include/utils/elog.h my/src/postgres/include/utils/elog.h ---- original/src/postgres/include/utils/elog.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/utils/elog.h 2021-11-14 20:05:00.000000000 +0300 -@@ -40,6 +40,9 @@ - #define WARNING 19 /* Warnings. NOTICE is for expected messages - * like implicit sequence creation by SERIAL. - * WARNING is for unexpected messages. */ -+#ifdef ERROR -+#undef ERROR -+#endif - #define ERROR 20 /* user error - abort transaction; return to - * known state */ - /* Save ERROR value in PGERROR so it can be restored when Win32 includes -diff -ruN original/src/postgres/include/utils/pg_locale.h my/src/postgres/include/utils/pg_locale.h ---- original/src/postgres/include/utils/pg_locale.h 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/include/utils/pg_locale.h 2021-11-14 20:05:00.000000000 +0300 -@@ -16,7 +16,7 @@ - #include <xlocale.h> - #endif - #ifdef USE_ICU --#include <unicode/ucol.h> -+//#include <unicode/ucol.h> - #endif - - #include "utils/guc.h" -diff -ruN original/src/postgres/src_backend_nodes_copyfuncs.c my/src/postgres/src_backend_nodes_copyfuncs.c ---- original/src/postgres/src_backend_nodes_copyfuncs.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_nodes_copyfuncs.c 2021-11-14 20:05:00.000000000 +0300 -@@ -6011,3 +6011,5 @@ - - return retval; - } -+ -+PGDLLIMPORT Node *newNodeMacroHolder; -diff -ruN original/src/postgres/src_backend_parser_scan.c my/src/postgres/src_backend_parser_scan.c ---- original/src/postgres/src_backend_parser_scan.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_parser_scan.c 2021-11-14 20:05:00.000000000 +0300 -@@ -79,7 +79,7 @@ - #include "postgres.h" - - #include <ctype.h> --#include <unistd.h> -+//#include <unistd.h> - - #include "common/string.h" - #include "parser/gramparse.h" -@@ -4603,7 +4603,7 @@ - * down here because we want the user's section 1 to have been scanned first. - * The user has a chance to override it with an option. - */ --#include <unistd.h> -+//#include <unistd.h> - #endif - - #ifndef YY_EXTRA_TYPE -diff -ruN original/src/postgres/src_backend_storage_ipc_ipc.c my/src/postgres/src_backend_storage_ipc_ipc.c ---- original/src/postgres/src_backend_storage_ipc_ipc.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_storage_ipc_ipc.c 2021-11-14 20:05:00.000000000 +0300 -@@ -27,7 +27,7 @@ - #include "postgres.h" - - #include <signal.h> --#include <unistd.h> -+//#include <unistd.h> - #include <sys/stat.h> - - #include "miscadmin.h" -diff -ruN original/src/postgres/src_backend_storage_lmgr_s_lock.c my/src/postgres/src_backend_storage_lmgr_s_lock.c ---- original/src/postgres/src_backend_storage_lmgr_s_lock.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_storage_lmgr_s_lock.c 2021-11-14 20:05:00.000000000 +0300 -@@ -58,7 +58,7 @@ - #include "postgres.h" - - #include <time.h> --#include <unistd.h> -+//#include <unistd.h> - - #include "port/atomics.h" - #include "storage/s_lock.h" -diff -ruN original/src/postgres/src_backend_tcop_postgres.c my/src/postgres/src_backend_tcop_postgres.c ---- original/src/postgres/src_backend_tcop_postgres.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_tcop_postgres.c 2021-11-14 20:05:00.000000000 +0300 -@@ -35,18 +35,18 @@ - #include <fcntl.h> - #include <limits.h> - #include <signal.h> --#include <unistd.h> --#include <sys/socket.h> -+//#include <unistd.h> -+//#include <sys/socket.h> - #ifdef HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif - #ifdef HAVE_SYS_RESOURCE_H --#include <sys/time.h> -+//#include <sys/time.h> - #include <sys/resource.h> - #endif - - #ifndef HAVE_GETRUSAGE --#include "rusagestub.h" -+//#include "rusagestub.h" - #endif - - #include "access/parallel.h" -@@ -57,7 +57,7 @@ - #include "commands/prepare.h" - #include "executor/spi.h" - #include "jit/jit.h" --#include "libpq/libpq.h" -+//#include "libpq/libpq.h" - #include "libpq/pqformat.h" - #include "libpq/pqsignal.h" - #include "mb/pg_wchar.h" -@@ -69,7 +69,7 @@ - #include "parser/parser.h" - #include "pg_getopt.h" - #include "pg_trace.h" --#include "pgstat.h" -+//#include "pgstat.h" - #include "postmaster/autovacuum.h" - #include "postmaster/interrupt.h" - #include "postmaster/postmaster.h" -@@ -538,11 +538,11 @@ - - #if defined(__hpux) && !defined(__GNUC__) && !defined(__INTEL_COMPILER) - /* Assume it's HP-UX native compiler */ --#include <ia64/sys/inline.h> -+//#include <ia64/sys/inline.h> - #define ia64_get_bsp() ((char *) (_Asm_mov_from_ar(_AREG_BSP, _NO_FENCE))) - #elif defined(__INTEL_COMPILER) - /* icc */ --#include <asm/ia64regs.h> -+//#include <asm/ia64regs.h> - #define ia64_get_bsp() ((char *) __getReg(_IA64_REG_AR_BSP)) - #else - /* gcc */ -diff -ruN original/src/postgres/src_backend_utils_adt_ruleutils.c my/src/postgres/src_backend_utils_adt_ruleutils.c ---- original/src/postgres/src_backend_utils_adt_ruleutils.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_adt_ruleutils.c 2021-11-14 20:05:00.000000000 +0300 -@@ -23,7 +23,7 @@ - #include "postgres.h" - - #include <ctype.h> --#include <unistd.h> -+//#include <unistd.h> - #include <fcntl.h> - - #include "access/amapi.h" -diff -ruN original/src/postgres/src_backend_utils_error_assert.c my/src/postgres/src_backend_utils_error_assert.c ---- original/src/postgres/src_backend_utils_error_assert.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_error_assert.c 2021-11-14 20:05:00.000000000 +0300 -@@ -23,7 +23,7 @@ - */ - #include "postgres.h" - --#include <unistd.h> -+//#include <unistd.h> - #ifdef HAVE_EXECINFO_H - #include <execinfo.h> - #endif -diff -ruN original/src/postgres/src_backend_utils_error_elog.c my/src/postgres/src_backend_utils_error_elog.c ---- original/src/postgres/src_backend_utils_error_elog.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_error_elog.c 2021-11-14 20:05:00.000000000 +0300 -@@ -94,7 +94,7 @@ - - #include <fcntl.h> - #include <time.h> --#include <unistd.h> -+//#include <unistd.h> - #include <signal.h> - #include <ctype.h> - #ifdef HAVE_SYSLOG -@@ -106,11 +106,11 @@ - - #include "access/transam.h" - #include "access/xact.h" --#include "libpq/libpq.h" -+//#include "libpq/libpq.h" - #include "libpq/pqformat.h" - #include "mb/pg_wchar.h" - #include "miscadmin.h" --#include "postmaster/bgworker.h" -+//#include "postmaster/bgworker.h" - #include "postmaster/postmaster.h" - #include "postmaster/syslogger.h" - #include "storage/ipc.h" -@@ -132,6 +132,7 @@ - - __thread sigjmp_buf *PG_exception_stack = NULL; - -+__thread bool ClientAuthInProgress = false; - - extern bool redirection_done; - -@@ -553,8 +554,8 @@ - * what we want for NOTICE messages, but not for fatal exits.) This hack - * is necessary because of poor design of old-style copy protocol. - */ -- if (elevel >= FATAL && whereToSendOutput == DestRemote) -- pq_endcopyout(true); -+ //if (elevel >= FATAL && whereToSendOutput == DestRemote) -+ // pq_endcopyout(true); - - /* Emit the message to the right places */ - EmitErrorReport(); -@@ -1505,8 +1506,8 @@ - * to error messages thrown deep inside pgwin32_message_to_UTF16(). - */ - if (!in_error_recursion_trouble() && -- CurrentMemoryContext != NULL && -- GetMessageEncoding() != GetACPEncoding()) -+ CurrentMemoryContext != NULL /*&& -+ GetMessageEncoding() != GetACPEncoding()*/) - { - utf16 = pgwin32_message_to_UTF16(line, len, NULL); - if (utf16) -@@ -1675,7 +1676,7 @@ - fmt = _(fmt); - - va_start(ap, fmt); --#ifndef WIN32 -+#if 1 - /* On Unix, we just fprintf to stderr */ - vfprintf(stderr, fmt, ap); - fflush(stderr); -@@ -1686,7 +1687,7 @@ - * On Win32, we print to stderr if running on a console, or write to - * eventlog if running as a service - */ -- if (pgwin32_is_service()) /* Running as a service */ -+ if (false /*pgwin32_is_service()*/) /* Running as a service */ - { - write_eventlog(ERROR, errbuf, strlen(errbuf)); - } -diff -ruN original/src/postgres/src_backend_utils_fmgr_fmgr.c my/src/postgres/src_backend_utils_fmgr_fmgr.c ---- original/src/postgres/src_backend_utils_fmgr_fmgr.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_fmgr_fmgr.c 2021-11-14 20:05:00.000000000 +0300 -@@ -30,7 +30,7 @@ - #include "miscadmin.h" - #include "nodes/makefuncs.h" - #include "nodes/nodeFuncs.h" --#include "pgstat.h" -+//#include "pgstat.h" - #include "utils/acl.h" - #include "utils/builtins.h" - #include "utils/fmgrtab.h" -diff -ruN original/src/postgres/src_backend_utils_init_globals.c my/src/postgres/src_backend_utils_init_globals.c ---- original/src/postgres/src_backend_utils_init_globals.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_init_globals.c 2021-11-14 20:05:00.000000000 +0300 -@@ -30,14 +30,13 @@ - #include "postgres.h" - - #include "common/file_perm.h" --#include "libpq/libpq-be.h" --#include "libpq/pqcomm.h" -+//#include "libpq/libpq-be.h" -+//#include "libpq/pqcomm.h" - #include "miscadmin.h" - #include "storage/backendid.h" - - - -- - __thread volatile sig_atomic_t InterruptPending = false; - - -diff -ruN original/src/postgres/src_backend_utils_mb_mbutils.c my/src/postgres/src_backend_utils_mb_mbutils.c ---- original/src/postgres/src_backend_utils_mb_mbutils.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_mb_mbutils.c 2021-11-14 20:05:00.000000000 +0300 -@@ -516,7 +516,7 @@ - #ifdef WIN32 - if (!raw_pg_bind_textdomain_codeset(domainname, new_msgenc)) - /* On failure, the old message encoding remains valid. */ -- return GetMessageEncoding(); -+ return PG_SQL_ASCII;//GetMessageEncoding(); - #endif - - return new_msgenc; -@@ -775,7 +775,7 @@ - WCHAR * - pgwin32_message_to_UTF16(const char *str, int len, int *utf16len) - { -- int msgenc = GetMessageEncoding(); -+ int msgenc = PG_SQL_ASCII;//GetMessageEncoding(); - WCHAR *utf16; - int dstlen; - UINT codepage; -@@ -805,14 +805,14 @@ - * XXX pg_do_encoding_conversion() requires a transaction. In the - * absence of one, hope for the input to be valid UTF8. - */ -- if (IsTransactionState()) -+ if (false /*IsTransactionState()*/) - { -- utf8 = (char *) pg_do_encoding_conversion((unsigned char *) str, -+ /*utf8 = (char *) pg_do_encoding_conversion((unsigned char *) str, - len, - msgenc, - PG_UTF8); - if (utf8 != str) -- len = strlen(utf8); -+ len = strlen(utf8);*/ - } - else - utf8 = (char *) str; -diff -ruN original/src/postgres/src_backend_utils_misc_guc.c my/src/postgres/src_backend_utils_misc_guc.c ---- original/src/postgres/src_backend_utils_misc_guc.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_backend_utils_misc_guc.c 2021-11-14 20:05:00.000000000 +0300 -@@ -30,7 +30,7 @@ - #include <float.h> - #include <math.h> - #include <limits.h> --#include <unistd.h> -+//#include <unistd.h> - #include <sys/stat.h> - #ifdef HAVE_SYSLOG - #include <syslog.h> -@@ -56,8 +56,8 @@ - #include "common/string.h" - #include "funcapi.h" - #include "jit/jit.h" --#include "libpq/auth.h" --#include "libpq/libpq.h" -+//#include "libpq/auth.h" -+//#include "libpq/libpq.h" - #include "libpq/pqformat.h" - #include "miscadmin.h" - #include "optimizer/cost.h" -@@ -69,7 +69,7 @@ - #include "parser/parse_type.h" - #include "parser/parser.h" - #include "parser/scansup.h" --#include "pgstat.h" -+//#include "pgstat.h" - #include "postmaster/autovacuum.h" - #include "postmaster/bgworker_internals.h" - #include "postmaster/bgwriter.h" -@@ -84,7 +84,7 @@ - #include "replication/walsender.h" - #include "storage/bufmgr.h" - #include "storage/dsm_impl.h" --#include "storage/fd.h" -+//#include "storage/fd.h" - #include "storage/large_object.h" - #include "storage/pg_shmem.h" - #include "storage/predicate.h" -@@ -1359,7 +1359,7 @@ - /* - * Open file - */ -- fp = AllocateFile(CONFIG_EXEC_PARAMS_NEW, "w"); -+ fp = 0;//AllocateFile(CONFIG_EXEC_PARAMS_NEW, "w"); - if (!fp) - { - ereport(elevel, -@@ -1374,7 +1374,7 @@ - write_one_nondefault_variable(fp, guc_variables[i]); - } - -- if (FreeFile(fp)) -+ if (false/*FreeFile(fp)*/) - { - ereport(elevel, - (errcode_for_file_access(), -@@ -1448,7 +1448,7 @@ - /* - * Open file - */ -- fp = AllocateFile(CONFIG_EXEC_PARAMS, "r"); -+ fp = 0;//AllocateFile(CONFIG_EXEC_PARAMS, "r"); - if (!fp) - { - /* File not found is fine */ -@@ -1492,7 +1492,7 @@ - free(varsourcefile); - } - -- FreeFile(fp); -+ //FreeFile(fp); - } - #endif /* EXEC_BACKEND */ - -diff -ruN original/src/postgres/src_common_encnames.c my/src/postgres/src_common_encnames.c ---- original/src/postgres/src_common_encnames.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_common_encnames.c 2021-11-14 20:05:00.000000000 +0300 -@@ -19,7 +19,7 @@ - #include "c.h" - - #include <ctype.h> --#include <unistd.h> -+//#include <unistd.h> - - #include "mb/pg_wchar.h" - -diff -ruN original/src/postgres/src_common_psprintf.c my/src/postgres/src_common_psprintf.c ---- original/src/postgres/src_common_psprintf.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_common_psprintf.c 2021-11-14 20:05:00.000000000 +0300 -@@ -29,7 +29,7 @@ - - #else - --#include "postgres_fe.h" -+//#include "postgres_fe.h" - - /* It's possible we could use a different value for this in frontend code */ - #define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */ -diff -ruN original/src/postgres/src_common_string.c my/src/postgres/src_common_string.c ---- original/src/postgres/src_common_string.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_common_string.c 2021-11-14 20:05:00.000000000 +0300 -@@ -24,7 +24,7 @@ - #ifndef FRONTEND - #include "postgres.h" - #else --#include "postgres_fe.h" -+//#include "postgres_fe.h" - #endif - - #include "common/string.h" -diff -ruN original/src/postgres/src_common_stringinfo.c my/src/postgres/src_common_stringinfo.c ---- original/src/postgres/src_common_stringinfo.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_common_stringinfo.c 2021-11-14 20:05:00.000000000 +0300 -@@ -36,7 +36,7 @@ - - #else - --#include "postgres_fe.h" -+//#include "postgres_fe.h" - - /* It's possible we could use a different value for this in frontend code */ - #define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */ -diff -ruN original/src/postgres/src_port_pg_bitutils.c my/src/postgres/src_port_pg_bitutils.c ---- original/src/postgres/src_port_pg_bitutils.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_port_pg_bitutils.c 2021-11-14 20:05:00.000000000 +0300 -@@ -1,23 +1,9 @@ --/*-------------------------------------------------------------------- -- * Symbols referenced in this file: -- * - pg_popcount64 -- * - pg_popcount64_choose -- * - pg_popcount32 -- * - pg_popcount32_choose -- * - pg_popcount_available -- * - pg_popcount32_asm -- * - pg_popcount64_asm -- * - pg_popcount32_slow -- * - pg_popcount64_slow -- *-------------------------------------------------------------------- -- */ -- - /*------------------------------------------------------------------------- - * - * pg_bitutils.c - * Miscellaneous functions for bit-wise operations. - * -- * Copyright (c) 2019-2020, PostgreSQL Global Development Group -+ * Copyright (c) 2019-2021, PostgreSQL Global Development Group - * - * IDENTIFICATION - * src/port/pg_bitutils.c -@@ -45,7 +31,24 @@ - * HAVE__BUILTIN_CLZ is defined, but we provide it anyway, so that - * extensions possibly compiled with a different compiler can use it. - */ -- -+const uint8 pg_leftmost_one_pos[256] = { -+ 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, -+ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, -+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, -+ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -+ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -+ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -+ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, -+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 -+}; - - /* - * Array giving the position of the right-most set bit for each possible -@@ -56,7 +59,24 @@ - * HAVE__BUILTIN_CTZ is defined, but we provide it anyway, so that - * extensions possibly compiled with a different compiler can use it. - */ -- -+const uint8 pg_rightmost_one_pos[256] = { -+ 0, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, -+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 -+}; - - /* - * Array giving the number of 1-bits in each possible byte value. -@@ -64,39 +84,40 @@ - * Note: we export this for use by functions in which explicit use - * of the popcount functions seems unlikely to be a win. - */ -- -- --/* -- * On x86_64, we can use the hardware popcount instruction, but only if -- * we can verify that the CPU supports it via the cpuid instruction. -- * -- * Otherwise, we fall back to __builtin_popcount if the compiler has that, -- * or a hand-rolled implementation if not. -- */ --#ifdef HAVE_X86_64_POPCNTQ --#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID) --#define USE_POPCNT_ASM 1 --#endif --#endif -+const uint8 pg_number_of_ones[256] = { -+ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, -+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, -+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, -+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, -+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, -+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, -+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, -+ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 -+}; - - static int pg_popcount32_slow(uint32 word); - static int pg_popcount64_slow(uint64 word); - --#ifdef USE_POPCNT_ASM -+#ifdef TRY_POPCNT_FAST - static bool pg_popcount_available(void); - static int pg_popcount32_choose(uint32 word); - static int pg_popcount64_choose(uint64 word); --static int pg_popcount32_asm(uint32 word); --static int pg_popcount64_asm(uint64 word); -+static int pg_popcount32_fast(uint32 word); -+static int pg_popcount64_fast(uint64 word); - - int (*pg_popcount32) (uint32 word) = pg_popcount32_choose; - int (*pg_popcount64) (uint64 word) = pg_popcount64_choose; --#else --int (*pg_popcount32) (uint32 word) = pg_popcount32_slow; --int (*pg_popcount64) (uint64 word) = pg_popcount64_slow; --#endif /* USE_POPCNT_ASM */ -+#endif /* TRY_POPCNT_FAST */ - --#ifdef USE_POPCNT_ASM -+#ifdef TRY_POPCNT_FAST - - /* - * Return true if CPUID indicates that the POPCNT instruction is available. -@@ -128,8 +149,8 @@ - { - if (pg_popcount_available()) - { -- pg_popcount32 = pg_popcount32_asm; -- pg_popcount64 = pg_popcount64_asm; -+ pg_popcount32 = pg_popcount32_fast; -+ pg_popcount64 = pg_popcount64_fast; - } - else - { -@@ -145,8 +166,8 @@ - { - if (pg_popcount_available()) - { -- pg_popcount32 = pg_popcount32_asm; -- pg_popcount64 = pg_popcount64_asm; -+ pg_popcount32 = pg_popcount32_fast; -+ pg_popcount64 = pg_popcount64_fast; - } - else - { -@@ -158,32 +179,40 @@ - } - - /* -- * pg_popcount32_asm -+ * pg_popcount32_fast - * Return the number of 1 bits set in word - */ - static int --pg_popcount32_asm(uint32 word) -+pg_popcount32_fast(uint32 word) - { -+#ifdef _MSC_VER -+ return __popcnt(word); -+#else - uint32 res; - - __asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); - return (int) res; -+#endif - } - - /* -- * pg_popcount64_asm -+ * pg_popcount64_fast - * Return the number of 1 bits set in word - */ - static int --pg_popcount64_asm(uint64 word) -+pg_popcount64_fast(uint64 word) - { -+#ifdef _MSC_VER -+ return __popcnt64(word); -+#else - uint64 res; - - __asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); - return (int) res; -+#endif - } - --#endif /* USE_POPCNT_ASM */ -+#endif /* TRY_POPCNT_FAST */ - - - /* -@@ -236,11 +265,71 @@ - #endif /* HAVE__BUILTIN_POPCOUNT */ - } - -+#ifndef TRY_POPCNT_FAST -+ -+/* -+ * When the POPCNT instruction is not available, there's no point in using -+ * function pointers to vary the implementation between the fast and slow -+ * method. We instead just make these actual external functions when -+ * TRY_POPCNT_FAST is not defined. The compiler should be able to inline -+ * the slow versions here. -+ */ -+int -+pg_popcount32(uint32 word) -+{ -+ return pg_popcount32_slow(word); -+} -+ -+int -+pg_popcount64(uint64 word) -+{ -+ return pg_popcount64_slow(word); -+} -+ -+#endif /* !TRY_POPCNT_FAST */ - - /* - * pg_popcount - * Returns the number of 1-bits in buf - */ -+uint64 -+pg_popcount(const char *buf, int bytes) -+{ -+ uint64 popcnt = 0; -+ - #if SIZEOF_VOID_P >= 8 -+ /* Process in 64-bit chunks if the buffer is aligned. */ -+ if (buf == (const char *) TYPEALIGN(8, buf)) -+ { -+ const uint64 *words = (const uint64 *) buf; -+ -+ while (bytes >= 8) -+ { -+ popcnt += pg_popcount64(*words++); -+ bytes -= 8; -+ } -+ -+ buf = (const char *) words; -+ } - #else -+ /* Process in 32-bit chunks if the buffer is aligned. */ -+ if (buf == (const char *) TYPEALIGN(4, buf)) -+ { -+ const uint32 *words = (const uint32 *) buf; -+ -+ while (bytes >= 4) -+ { -+ popcnt += pg_popcount32(*words++); -+ bytes -= 4; -+ } -+ -+ buf = (const char *) words; -+ } - #endif -+ -+ /* Process any remaining bytes */ -+ while (bytes--) -+ popcnt += pg_number_of_ones[(unsigned char) *buf++]; -+ -+ return popcnt; -+} -\ No newline at end of file -diff -ruN original/src/postgres/src_port_pgsleep.c my/src/postgres/src_port_pgsleep.c ---- original/src/postgres/src_port_pgsleep.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/postgres/src_port_pgsleep.c 2021-11-14 20:05:00.000000000 +0300 -@@ -18,8 +18,8 @@ - */ - #include "c.h" - --#include <unistd.h> --#include <sys/time.h> -+//#include <unistd.h> -+//#include <sys/time.h> - #ifdef HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif -@@ -28,7 +28,8 @@ - * In a Windows backend, we don't use this implementation, but rather - * the signal-aware version in src/backend/port/win32/signal.c. - */ --#if defined(FRONTEND) || !defined(WIN32) -+#if 1 -+//defined(FRONTEND) || !defined(WIN32) - - /* - * pg_usleep --- delay the specified number of microseconds. +diff -ruN original/pg_query.h my/pg_query.h +--- original/pg_query.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/pg_query.h 2021-11-14 20:05:00.000000000 +0300 +@@ -57,29 +57,29 @@ + PgQueryError* error; + } PgQueryPlpgsqlParseResult; +-typedef struct { ++/*typedef struct { + uint64_t fingerprint; + char* fingerprint_str; + char* stderr_buffer; + PgQueryError* error; +-} PgQueryFingerprintResult; ++} PgQueryFingerprintResult;*/ + +-typedef struct { ++/*typedef struct { + char* normalized_query; + PgQueryError* error; +-} PgQueryNormalizeResult; ++} PgQueryNormalizeResult;*/ + + #ifdef __cplusplus + extern "C" { + #endif + +-PgQueryNormalizeResult pg_query_normalize(const char* input); ++//PgQueryNormalizeResult pg_query_normalize(const char* input); + PgQueryScanResult pg_query_scan(const char* input); + PgQueryParseResult pg_query_parse(const char* input); + PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); +-PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); ++//PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); + +-PgQueryFingerprintResult pg_query_fingerprint(const char* input); ++//PgQueryFingerprintResult pg_query_fingerprint(const char* input); + + // Use pg_query_split_with_scanner when you need to split statements that may + // contain parse errors, otherwise pg_query_split_with_parser is recommended +@@ -93,14 +93,14 @@ + + PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree); + +-void pg_query_free_normalize_result(PgQueryNormalizeResult result); ++//void pg_query_free_normalize_result(PgQueryNormalizeResult result); + void pg_query_free_scan_result(PgQueryScanResult result); + void pg_query_free_parse_result(PgQueryParseResult result); + void pg_query_free_split_result(PgQuerySplitResult result); + void pg_query_free_deparse_result(PgQueryDeparseResult result); + void pg_query_free_protobuf_parse_result(PgQueryProtobufParseResult result); +-void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); +-void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); ++//void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); ++//void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); + + // Optional, cleans up the top-level memory context (automatically done for threads that exit) + void pg_query_exit(void); +diff -ruN original/src/pg_query.c my/src/pg_query.c +--- original/src/pg_query.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/pg_query.c 2021-11-14 20:05:00.000000000 +0300 +@@ -5,15 +5,19 @@ + #include <utils/memutils.h> + #include <utils/memdebug.h> + ++#ifndef WIN32 + #include <pthread.h> ++#endif + #include <signal.h> + + const char* progname = "pg_query"; + +-__thread sig_atomic_t pg_query_initialized = 0; ++__thread volatile sig_atomic_t pg_query_initialized = 0; + ++#ifndef WIN32 + static pthread_key_t pg_query_thread_exit_key; + static void pg_query_thread_exit(void *key); ++#endif + + void pg_query_init(void) + { +@@ -23,8 +27,10 @@ + MemoryContextInit(); + SetDatabaseEncoding(PG_UTF8); + ++#ifndef WIN32 + pthread_key_create(&pg_query_thread_exit_key, pg_query_thread_exit); + pthread_setspecific(pg_query_thread_exit_key, TopMemoryContext); ++#endif + } + + void pg_query_free_top_memory_context(MemoryContext context) +@@ -55,11 +61,13 @@ + ErrorContext = NULL; + } + ++#ifndef WIN32 + static void pg_query_thread_exit(void *key) + { + MemoryContext context = (MemoryContext) key; + pg_query_free_top_memory_context(context); + } ++#endif + + void pg_query_exit(void) + { +diff -ruN original/src/pg_query_parse.c my/src/pg_query_parse.c +--- original/src/pg_query_parse.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/pg_query_parse.c 2021-11-14 20:05:00.000000000 +0300 +@@ -6,7 +6,7 @@ + #include "parser/scanner.h" + #include "parser/scansup.h" + +-#include <unistd.h> ++//#include <unistd.h> + #include <fcntl.h> + + PgQueryInternalParsetreeAndError pg_query_raw_parse(const char* input) +@@ -111,8 +111,10 @@ + { + MemoryContext ctx = NULL; + PgQueryInternalParsetreeAndError parsetree_and_error; +- PgQueryProtobufParseResult result = {}; ++ PgQueryProtobufParseResult result; ++ memset(&result, 0, sizeof(result)); + ++ + ctx = pg_query_enter_memory_context(); + + parsetree_and_error = pg_query_raw_parse(input); +diff -ruN original/src/postgres/include/access/xlog.h my/src/postgres/include/access/xlog.h +--- original/src/postgres/include/access/xlog.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/access/xlog.h 2021-11-14 20:05:00.000000000 +0300 +@@ -18,7 +18,7 @@ + #include "datatype/timestamp.h" + #include "lib/stringinfo.h" + #include "nodes/pg_list.h" +-#include "storage/fd.h" ++//#include "storage/fd.h" + + + /* Sync methods */ +diff -ruN original/src/postgres/include/c.h my/src/postgres/include/c.h +--- original/src/postgres/include/c.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/c.h 2021-11-14 20:05:00.000000000 +0300 +@@ -62,7 +62,7 @@ + #include <stddef.h> + #include <stdarg.h> + #ifdef HAVE_STRINGS_H +-#include <strings.h> ++//#include <strings.h> + #endif + #include <stdint.h> + #include <sys/types.h> +@@ -135,8 +135,10 @@ + + /* GCC and XLC support format attributes */ + #if defined(__GNUC__) || defined(__IBMC__) +-#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) +-#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) ++#define pg_attribute_format_arg(a) ++// __attribute__((format_arg(a))) ++#define pg_attribute_printf(f,a) ++// __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) + #else + #define pg_attribute_format_arg(a) + #define pg_attribute_printf(f,a) +diff -ruN original/src/postgres/include/getaddrinfo.h my/src/postgres/include/getaddrinfo.h +--- original/src/postgres/include/getaddrinfo.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/getaddrinfo.h 2021-11-14 20:05:00.000000000 +0300 +@@ -22,8 +22,8 @@ + #ifndef GETADDRINFO_H + #define GETADDRINFO_H + +-#include <sys/socket.h> +-#include <netdb.h> ++//#include <sys/socket.h> ++//#include <netdb.h> + + + /* Various macros that ought to be in <netdb.h>, but might not be */ +diff -ruN original/src/postgres/include/libpq/libpq-be.h my/src/postgres/include/libpq/libpq-be.h +--- original/src/postgres/include/libpq/libpq-be.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/libpq/libpq-be.h 2021-11-14 20:05:00.000000000 +0300 +@@ -18,10 +18,10 @@ + #ifndef LIBPQ_BE_H + #define LIBPQ_BE_H + +-#include <sys/time.h> ++//#include <sys/time.h> + #ifdef USE_OPENSSL +-#include <openssl/ssl.h> +-#include <openssl/err.h> ++//#include <openssl/ssl.h> ++//#include <openssl/err.h> + #endif + #ifdef HAVE_NETINET_TCP_H + #include <netinet/tcp.h> +diff -ruN original/src/postgres/include/libpq/libpq.h my/src/postgres/include/libpq/libpq.h +--- original/src/postgres/include/libpq/libpq.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/libpq/libpq.h 2021-11-14 20:05:00.000000000 +0300 +@@ -14,7 +14,7 @@ + #ifndef LIBPQ_H + #define LIBPQ_H + +-#include <netinet/in.h> ++//#include <netinet/in.h> + + #include "lib/stringinfo.h" + #include "libpq/libpq-be.h" +diff -ruN original/src/postgres/include/libpq/pqcomm.h my/src/postgres/include/libpq/pqcomm.h +--- original/src/postgres/include/libpq/pqcomm.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/libpq/pqcomm.h 2021-11-14 20:05:00.000000000 +0300 +@@ -16,12 +16,12 @@ + #ifndef PQCOMM_H + #define PQCOMM_H + +-#include <sys/socket.h> +-#include <netdb.h> ++//#include <sys/socket.h> ++//#include <netdb.h> + #ifdef HAVE_SYS_UN_H + #include <sys/un.h> + #endif +-#include <netinet/in.h> ++//#include <netinet/in.h> + + #ifdef HAVE_STRUCT_SOCKADDR_STORAGE + +diff -ruN original/src/postgres/include/miscadmin.h my/src/postgres/include/miscadmin.h +--- original/src/postgres/include/miscadmin.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/miscadmin.h 2021-11-14 20:05:00.000000000 +0300 +@@ -94,7 +94,7 @@ + /* in tcop/postgres.c */ + extern void ProcessInterrupts(void); + +-#ifndef WIN32 ++#if 1 + + #define CHECK_FOR_INTERRUPTS() \ + do { \ +diff -ruN original/src/postgres/include/nodes/execnodes.h my/src/postgres/include/nodes/execnodes.h +--- original/src/postgres/include/nodes/execnodes.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/nodes/execnodes.h 2021-11-14 20:05:00.000000000 +0300 +@@ -26,7 +26,7 @@ + #include "utils/hsearch.h" + #include "utils/queryenvironment.h" + #include "utils/reltrigger.h" +-#include "utils/sharedtuplestore.h" ++//#include "utils/sharedtuplestore.h" + #include "utils/snapshot.h" + #include "utils/sortsupport.h" + #include "utils/tuplesort.h" +diff -ruN original/src/postgres/include/pg_getopt.h my/src/postgres/include/pg_getopt.h +--- original/src/postgres/include/pg_getopt.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/pg_getopt.h 2021-11-14 20:05:00.000000000 +0300 +@@ -19,7 +19,7 @@ + #define PG_GETOPT_H + + /* POSIX says getopt() is provided by unistd.h */ +-#include <unistd.h> ++//#include <unistd.h> + + /* rely on the system's getopt.h if present */ + #ifdef HAVE_GETOPT_H +diff -ruN original/src/postgres/include/port/atomics.h my/src/postgres/include/port/atomics.h +--- original/src/postgres/include/port/atomics.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/port/atomics.h 2021-11-14 20:05:00.000000000 +0300 +@@ -65,15 +65,15 @@ + */ + #if defined(__arm__) || defined(__arm) || \ + defined(__aarch64__) || defined(__aarch64) +-#include "port/atomics/arch-arm.h" ++//#include "port/atomics/arch-arm.h" + #elif defined(__i386__) || defined(__i386) || defined(__x86_64__) +-#include "port/atomics/arch-x86.h" ++//#include "port/atomics/arch-x86.h" + #elif defined(__ia64__) || defined(__ia64) +-#include "port/atomics/arch-ia64.h" ++//#include "port/atomics/arch-ia64.h" + #elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) +-#include "port/atomics/arch-ppc.h" ++//#include "port/atomics/arch-ppc.h" + #elif defined(__hppa) || defined(__hppa__) +-#include "port/atomics/arch-hppa.h" ++//#include "port/atomics/arch-hppa.h" + #endif + + /* +@@ -94,11 +94,11 @@ + #if (defined(__GNUC__) || defined(__INTEL_COMPILER)) && !(defined(__IBMC__) || defined(__IBMCPP__)) + #include "port/atomics/generic-gcc.h" + #elif defined(_MSC_VER) +-#include "port/atomics/generic-msvc.h" ++//#include "port/atomics/generic-msvc.h" + #elif defined(__hpux) && defined(__ia64) && !defined(__GNUC__) +-#include "port/atomics/generic-acc.h" ++//#include "port/atomics/generic-acc.h" + #elif defined(__SUNPRO_C) && !defined(__GNUC__) +-#include "port/atomics/generic-sunpro.h" ++//#include "port/atomics/generic-sunpro.h" + #else + /* + * Unsupported compiler, we'll likely use slower fallbacks... At least +diff -ruN original/src/postgres/include/port/pg_bitutils.h my/src/postgres/include/port/pg_bitutils.h +--- original/src/postgres/include/port/pg_bitutils.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/port/pg_bitutils.h 2021-11-14 20:05:00.000000000 +0300 +@@ -207,10 +207,18 @@ + return pg_leftmost_one_pos64(num - 1) + 1; + } + +-/* Count the number of one-bits in a uint32 or uint64 */ ++#ifdef TRY_POPCNT_FAST ++/* Attempt to use the POPCNT instruction, but perform a runtime check first */ + extern int (*pg_popcount32) (uint32 word); + extern int (*pg_popcount64) (uint64 word); + ++#else ++/* Use a portable implementation -- no need for a function pointer. */ ++extern int pg_popcount32(uint32 word); ++extern int pg_popcount64(uint64 word); ++ ++#endif ++ + /* Count the number of one-bits in a byte array */ + extern uint64 pg_popcount(const char *buf, int bytes); + +diff -ruN original/src/postgres/include/port.h my/src/postgres/include/port.h +--- original/src/postgres/include/port.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/port.h 2021-11-14 20:05:00.000000000 +0300 +@@ -14,8 +14,8 @@ + #define PG_PORT_H + + #include <ctype.h> +-#include <netdb.h> +-#include <pwd.h> ++//#include <netdb.h> ++//#include <pwd.h> + + /* + * Windows has enough specialized port stuff that we push most of it off +@@ -23,7 +23,7 @@ + * Note: Some CYGWIN includes might #define WIN32. + */ + #if defined(WIN32) && !defined(__CYGWIN__) +-#include "port/win32_port.h" ++//#include "port/win32_port.h" + #endif + + /* socket has a different definition on WIN32 */ +@@ -32,6 +32,7 @@ + + #define PGINVALID_SOCKET (-1) + #else ++#include <winsock2.h> + typedef SOCKET pgsocket; + + #define PGINVALID_SOCKET INVALID_SOCKET +@@ -462,15 +463,15 @@ + + /* thread.h */ + #ifndef WIN32 +-extern int pqGetpwuid(uid_t uid, struct passwd *resultbuf, char *buffer, +- size_t buflen, struct passwd **result); ++//extern int pqGetpwuid(uid_t uid, struct passwd *resultbuf, char *buffer, ++// size_t buflen, struct passwd **result); + #endif + +-extern int pqGethostbyname(const char *name, ++/*extern int pqGethostbyname(const char *name, + struct hostent *resultbuf, + char *buffer, size_t buflen, + struct hostent **result, +- int *herrno); ++ int *herrno);*/ + + extern void pg_qsort(void *base, size_t nel, size_t elsize, + int (*cmp) (const void *, const void *)); +diff -ruN original/src/postgres/include/storage/pmsignal.h my/src/postgres/include/storage/pmsignal.h +--- original/src/postgres/include/storage/pmsignal.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/storage/pmsignal.h 2021-11-14 20:05:00.000000000 +0300 +@@ -21,7 +21,7 @@ + #endif + + #ifdef HAVE_SYS_PROCCTL_H +-#include "sys/procctl.h" ++//#include "sys/procctl.h" + #endif + + /* +diff -ruN original/src/postgres/include/storage/s_lock.h my/src/postgres/include/storage/s_lock.h +--- original/src/postgres/include/storage/s_lock.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/storage/s_lock.h 2021-11-14 20:05:00.000000000 +0300 +@@ -676,7 +676,7 @@ + #if defined(__m32r__) && defined(HAVE_SYS_TAS_H) /* Renesas' M32R */ + #define HAS_TEST_AND_SET + +-#include <sys/tas.h> ++//#include <sys/tas.h> + + typedef int slock_t; + +@@ -837,7 +837,7 @@ + + typedef unsigned int slock_t; + +-#include <ia64/sys/inline.h> ++//#include <ia64/sys/inline.h> + #define TAS(lock) _Asm_xchg(_SZ_W, lock, 1, _LDHINT_NONE) + /* On IA64, it's a win to use a non-locking test before the xchg proper */ + #define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) +diff -ruN original/src/postgres/include/utils/elog.h my/src/postgres/include/utils/elog.h +--- original/src/postgres/include/utils/elog.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/utils/elog.h 2021-11-14 20:05:00.000000000 +0300 +@@ -40,6 +40,9 @@ + #define WARNING 19 /* Warnings. NOTICE is for expected messages + * like implicit sequence creation by SERIAL. + * WARNING is for unexpected messages. */ ++#ifdef ERROR ++#undef ERROR ++#endif + #define ERROR 20 /* user error - abort transaction; return to + * known state */ + /* Save ERROR value in PGERROR so it can be restored when Win32 includes +diff -ruN original/src/postgres/include/utils/pg_locale.h my/src/postgres/include/utils/pg_locale.h +--- original/src/postgres/include/utils/pg_locale.h 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/include/utils/pg_locale.h 2021-11-14 20:05:00.000000000 +0300 +@@ -16,7 +16,7 @@ + #include <xlocale.h> + #endif + #ifdef USE_ICU +-#include <unicode/ucol.h> ++//#include <unicode/ucol.h> + #endif + + #include "utils/guc.h" +diff -ruN original/src/postgres/src_backend_nodes_copyfuncs.c my/src/postgres/src_backend_nodes_copyfuncs.c +--- original/src/postgres/src_backend_nodes_copyfuncs.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_nodes_copyfuncs.c 2021-11-14 20:05:00.000000000 +0300 +@@ -6011,3 +6011,5 @@ + + return retval; + } ++ ++PGDLLIMPORT Node *newNodeMacroHolder; +diff -ruN original/src/postgres/src_backend_parser_scan.c my/src/postgres/src_backend_parser_scan.c +--- original/src/postgres/src_backend_parser_scan.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_parser_scan.c 2021-11-14 20:05:00.000000000 +0300 +@@ -79,7 +79,7 @@ + #include "postgres.h" + + #include <ctype.h> +-#include <unistd.h> ++//#include <unistd.h> + + #include "common/string.h" + #include "parser/gramparse.h" +@@ -4603,7 +4603,7 @@ + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +-#include <unistd.h> ++//#include <unistd.h> + #endif + + #ifndef YY_EXTRA_TYPE +diff -ruN original/src/postgres/src_backend_storage_ipc_ipc.c my/src/postgres/src_backend_storage_ipc_ipc.c +--- original/src/postgres/src_backend_storage_ipc_ipc.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_storage_ipc_ipc.c 2021-11-14 20:05:00.000000000 +0300 +@@ -27,7 +27,7 @@ + #include "postgres.h" + + #include <signal.h> +-#include <unistd.h> ++//#include <unistd.h> + #include <sys/stat.h> + + #include "miscadmin.h" +diff -ruN original/src/postgres/src_backend_storage_lmgr_s_lock.c my/src/postgres/src_backend_storage_lmgr_s_lock.c +--- original/src/postgres/src_backend_storage_lmgr_s_lock.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_storage_lmgr_s_lock.c 2021-11-14 20:05:00.000000000 +0300 +@@ -58,7 +58,7 @@ + #include "postgres.h" + + #include <time.h> +-#include <unistd.h> ++//#include <unistd.h> + + #include "port/atomics.h" + #include "storage/s_lock.h" +diff -ruN original/src/postgres/src_backend_tcop_postgres.c my/src/postgres/src_backend_tcop_postgres.c +--- original/src/postgres/src_backend_tcop_postgres.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_tcop_postgres.c 2021-11-14 20:05:00.000000000 +0300 +@@ -35,18 +35,18 @@ + #include <fcntl.h> + #include <limits.h> + #include <signal.h> +-#include <unistd.h> +-#include <sys/socket.h> ++//#include <unistd.h> ++//#include <sys/socket.h> + #ifdef HAVE_SYS_SELECT_H + #include <sys/select.h> + #endif + #ifdef HAVE_SYS_RESOURCE_H +-#include <sys/time.h> ++//#include <sys/time.h> + #include <sys/resource.h> + #endif + + #ifndef HAVE_GETRUSAGE +-#include "rusagestub.h" ++//#include "rusagestub.h" + #endif + + #include "access/parallel.h" +@@ -57,7 +57,7 @@ + #include "commands/prepare.h" + #include "executor/spi.h" + #include "jit/jit.h" +-#include "libpq/libpq.h" ++//#include "libpq/libpq.h" + #include "libpq/pqformat.h" + #include "libpq/pqsignal.h" + #include "mb/pg_wchar.h" +@@ -69,7 +69,7 @@ + #include "parser/parser.h" + #include "pg_getopt.h" + #include "pg_trace.h" +-#include "pgstat.h" ++//#include "pgstat.h" + #include "postmaster/autovacuum.h" + #include "postmaster/interrupt.h" + #include "postmaster/postmaster.h" +@@ -538,11 +538,11 @@ + + #if defined(__hpux) && !defined(__GNUC__) && !defined(__INTEL_COMPILER) + /* Assume it's HP-UX native compiler */ +-#include <ia64/sys/inline.h> ++//#include <ia64/sys/inline.h> + #define ia64_get_bsp() ((char *) (_Asm_mov_from_ar(_AREG_BSP, _NO_FENCE))) + #elif defined(__INTEL_COMPILER) + /* icc */ +-#include <asm/ia64regs.h> ++//#include <asm/ia64regs.h> + #define ia64_get_bsp() ((char *) __getReg(_IA64_REG_AR_BSP)) + #else + /* gcc */ +diff -ruN original/src/postgres/src_backend_utils_adt_ruleutils.c my/src/postgres/src_backend_utils_adt_ruleutils.c +--- original/src/postgres/src_backend_utils_adt_ruleutils.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_adt_ruleutils.c 2021-11-14 20:05:00.000000000 +0300 +@@ -23,7 +23,7 @@ + #include "postgres.h" + + #include <ctype.h> +-#include <unistd.h> ++//#include <unistd.h> + #include <fcntl.h> + + #include "access/amapi.h" +diff -ruN original/src/postgres/src_backend_utils_error_assert.c my/src/postgres/src_backend_utils_error_assert.c +--- original/src/postgres/src_backend_utils_error_assert.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_error_assert.c 2021-11-14 20:05:00.000000000 +0300 +@@ -23,7 +23,7 @@ + */ + #include "postgres.h" + +-#include <unistd.h> ++//#include <unistd.h> + #ifdef HAVE_EXECINFO_H + #include <execinfo.h> + #endif +diff -ruN original/src/postgres/src_backend_utils_error_elog.c my/src/postgres/src_backend_utils_error_elog.c +--- original/src/postgres/src_backend_utils_error_elog.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_error_elog.c 2021-11-14 20:05:00.000000000 +0300 +@@ -94,7 +94,7 @@ + + #include <fcntl.h> + #include <time.h> +-#include <unistd.h> ++//#include <unistd.h> + #include <signal.h> + #include <ctype.h> + #ifdef HAVE_SYSLOG +@@ -106,11 +106,11 @@ + + #include "access/transam.h" + #include "access/xact.h" +-#include "libpq/libpq.h" ++//#include "libpq/libpq.h" + #include "libpq/pqformat.h" + #include "mb/pg_wchar.h" + #include "miscadmin.h" +-#include "postmaster/bgworker.h" ++//#include "postmaster/bgworker.h" + #include "postmaster/postmaster.h" + #include "postmaster/syslogger.h" + #include "storage/ipc.h" +@@ -132,6 +132,7 @@ + + __thread sigjmp_buf *PG_exception_stack = NULL; + ++__thread bool ClientAuthInProgress = false; + + extern bool redirection_done; + +@@ -553,8 +554,8 @@ + * what we want for NOTICE messages, but not for fatal exits.) This hack + * is necessary because of poor design of old-style copy protocol. + */ +- if (elevel >= FATAL && whereToSendOutput == DestRemote) +- pq_endcopyout(true); ++ //if (elevel >= FATAL && whereToSendOutput == DestRemote) ++ // pq_endcopyout(true); + + /* Emit the message to the right places */ + EmitErrorReport(); +@@ -1505,8 +1506,8 @@ + * to error messages thrown deep inside pgwin32_message_to_UTF16(). + */ + if (!in_error_recursion_trouble() && +- CurrentMemoryContext != NULL && +- GetMessageEncoding() != GetACPEncoding()) ++ CurrentMemoryContext != NULL /*&& ++ GetMessageEncoding() != GetACPEncoding()*/) + { + utf16 = pgwin32_message_to_UTF16(line, len, NULL); + if (utf16) +@@ -1675,7 +1676,7 @@ + fmt = _(fmt); + + va_start(ap, fmt); +-#ifndef WIN32 ++#if 1 + /* On Unix, we just fprintf to stderr */ + vfprintf(stderr, fmt, ap); + fflush(stderr); +@@ -1686,7 +1687,7 @@ + * On Win32, we print to stderr if running on a console, or write to + * eventlog if running as a service + */ +- if (pgwin32_is_service()) /* Running as a service */ ++ if (false /*pgwin32_is_service()*/) /* Running as a service */ + { + write_eventlog(ERROR, errbuf, strlen(errbuf)); + } +diff -ruN original/src/postgres/src_backend_utils_fmgr_fmgr.c my/src/postgres/src_backend_utils_fmgr_fmgr.c +--- original/src/postgres/src_backend_utils_fmgr_fmgr.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_fmgr_fmgr.c 2021-11-14 20:05:00.000000000 +0300 +@@ -30,7 +30,7 @@ + #include "miscadmin.h" + #include "nodes/makefuncs.h" + #include "nodes/nodeFuncs.h" +-#include "pgstat.h" ++//#include "pgstat.h" + #include "utils/acl.h" + #include "utils/builtins.h" + #include "utils/fmgrtab.h" +diff -ruN original/src/postgres/src_backend_utils_init_globals.c my/src/postgres/src_backend_utils_init_globals.c +--- original/src/postgres/src_backend_utils_init_globals.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_init_globals.c 2021-11-14 20:05:00.000000000 +0300 +@@ -30,14 +30,13 @@ + #include "postgres.h" + + #include "common/file_perm.h" +-#include "libpq/libpq-be.h" +-#include "libpq/pqcomm.h" ++//#include "libpq/libpq-be.h" ++//#include "libpq/pqcomm.h" + #include "miscadmin.h" + #include "storage/backendid.h" + + + +- + __thread volatile sig_atomic_t InterruptPending = false; + + +diff -ruN original/src/postgres/src_backend_utils_mb_mbutils.c my/src/postgres/src_backend_utils_mb_mbutils.c +--- original/src/postgres/src_backend_utils_mb_mbutils.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_mb_mbutils.c 2021-11-14 20:05:00.000000000 +0300 +@@ -516,7 +516,7 @@ + #ifdef WIN32 + if (!raw_pg_bind_textdomain_codeset(domainname, new_msgenc)) + /* On failure, the old message encoding remains valid. */ +- return GetMessageEncoding(); ++ return PG_SQL_ASCII;//GetMessageEncoding(); + #endif + + return new_msgenc; +@@ -775,7 +775,7 @@ + WCHAR * + pgwin32_message_to_UTF16(const char *str, int len, int *utf16len) + { +- int msgenc = GetMessageEncoding(); ++ int msgenc = PG_SQL_ASCII;//GetMessageEncoding(); + WCHAR *utf16; + int dstlen; + UINT codepage; +@@ -805,14 +805,14 @@ + * XXX pg_do_encoding_conversion() requires a transaction. In the + * absence of one, hope for the input to be valid UTF8. + */ +- if (IsTransactionState()) ++ if (false /*IsTransactionState()*/) + { +- utf8 = (char *) pg_do_encoding_conversion((unsigned char *) str, ++ /*utf8 = (char *) pg_do_encoding_conversion((unsigned char *) str, + len, + msgenc, + PG_UTF8); + if (utf8 != str) +- len = strlen(utf8); ++ len = strlen(utf8);*/ + } + else + utf8 = (char *) str; +diff -ruN original/src/postgres/src_backend_utils_misc_guc.c my/src/postgres/src_backend_utils_misc_guc.c +--- original/src/postgres/src_backend_utils_misc_guc.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_backend_utils_misc_guc.c 2021-11-14 20:05:00.000000000 +0300 +@@ -30,7 +30,7 @@ + #include <float.h> + #include <math.h> + #include <limits.h> +-#include <unistd.h> ++//#include <unistd.h> + #include <sys/stat.h> + #ifdef HAVE_SYSLOG + #include <syslog.h> +@@ -56,8 +56,8 @@ + #include "common/string.h" + #include "funcapi.h" + #include "jit/jit.h" +-#include "libpq/auth.h" +-#include "libpq/libpq.h" ++//#include "libpq/auth.h" ++//#include "libpq/libpq.h" + #include "libpq/pqformat.h" + #include "miscadmin.h" + #include "optimizer/cost.h" +@@ -69,7 +69,7 @@ + #include "parser/parse_type.h" + #include "parser/parser.h" + #include "parser/scansup.h" +-#include "pgstat.h" ++//#include "pgstat.h" + #include "postmaster/autovacuum.h" + #include "postmaster/bgworker_internals.h" + #include "postmaster/bgwriter.h" +@@ -84,7 +84,7 @@ + #include "replication/walsender.h" + #include "storage/bufmgr.h" + #include "storage/dsm_impl.h" +-#include "storage/fd.h" ++//#include "storage/fd.h" + #include "storage/large_object.h" + #include "storage/pg_shmem.h" + #include "storage/predicate.h" +@@ -1359,7 +1359,7 @@ + /* + * Open file + */ +- fp = AllocateFile(CONFIG_EXEC_PARAMS_NEW, "w"); ++ fp = 0;//AllocateFile(CONFIG_EXEC_PARAMS_NEW, "w"); + if (!fp) + { + ereport(elevel, +@@ -1374,7 +1374,7 @@ + write_one_nondefault_variable(fp, guc_variables[i]); + } + +- if (FreeFile(fp)) ++ if (false/*FreeFile(fp)*/) + { + ereport(elevel, + (errcode_for_file_access(), +@@ -1448,7 +1448,7 @@ + /* + * Open file + */ +- fp = AllocateFile(CONFIG_EXEC_PARAMS, "r"); ++ fp = 0;//AllocateFile(CONFIG_EXEC_PARAMS, "r"); + if (!fp) + { + /* File not found is fine */ +@@ -1492,7 +1492,7 @@ + free(varsourcefile); + } + +- FreeFile(fp); ++ //FreeFile(fp); + } + #endif /* EXEC_BACKEND */ + +diff -ruN original/src/postgres/src_common_encnames.c my/src/postgres/src_common_encnames.c +--- original/src/postgres/src_common_encnames.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_common_encnames.c 2021-11-14 20:05:00.000000000 +0300 +@@ -19,7 +19,7 @@ + #include "c.h" + + #include <ctype.h> +-#include <unistd.h> ++//#include <unistd.h> + + #include "mb/pg_wchar.h" + +diff -ruN original/src/postgres/src_common_psprintf.c my/src/postgres/src_common_psprintf.c +--- original/src/postgres/src_common_psprintf.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_common_psprintf.c 2021-11-14 20:05:00.000000000 +0300 +@@ -29,7 +29,7 @@ + + #else + +-#include "postgres_fe.h" ++//#include "postgres_fe.h" + + /* It's possible we could use a different value for this in frontend code */ + #define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */ +diff -ruN original/src/postgres/src_common_string.c my/src/postgres/src_common_string.c +--- original/src/postgres/src_common_string.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_common_string.c 2021-11-14 20:05:00.000000000 +0300 +@@ -24,7 +24,7 @@ + #ifndef FRONTEND + #include "postgres.h" + #else +-#include "postgres_fe.h" ++//#include "postgres_fe.h" + #endif + + #include "common/string.h" +diff -ruN original/src/postgres/src_common_stringinfo.c my/src/postgres/src_common_stringinfo.c +--- original/src/postgres/src_common_stringinfo.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_common_stringinfo.c 2021-11-14 20:05:00.000000000 +0300 +@@ -36,7 +36,7 @@ + + #else + +-#include "postgres_fe.h" ++//#include "postgres_fe.h" + + /* It's possible we could use a different value for this in frontend code */ + #define MaxAllocSize ((Size) 0x3fffffff) /* 1 gigabyte - 1 */ +diff -ruN original/src/postgres/src_port_pg_bitutils.c my/src/postgres/src_port_pg_bitutils.c +--- original/src/postgres/src_port_pg_bitutils.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_port_pg_bitutils.c 2021-11-14 20:05:00.000000000 +0300 +@@ -1,23 +1,9 @@ +-/*-------------------------------------------------------------------- +- * Symbols referenced in this file: +- * - pg_popcount64 +- * - pg_popcount64_choose +- * - pg_popcount32 +- * - pg_popcount32_choose +- * - pg_popcount_available +- * - pg_popcount32_asm +- * - pg_popcount64_asm +- * - pg_popcount32_slow +- * - pg_popcount64_slow +- *-------------------------------------------------------------------- +- */ +- + /*------------------------------------------------------------------------- + * + * pg_bitutils.c + * Miscellaneous functions for bit-wise operations. + * +- * Copyright (c) 2019-2020, PostgreSQL Global Development Group ++ * Copyright (c) 2019-2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_bitutils.c +@@ -45,7 +31,24 @@ + * HAVE__BUILTIN_CLZ is defined, but we provide it anyway, so that + * extensions possibly compiled with a different compiler can use it. + */ +- ++const uint8 pg_leftmost_one_pos[256] = { ++ 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, ++ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, ++ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ++ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ++ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ++ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ++ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ++ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ++ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ++}; + + /* + * Array giving the position of the right-most set bit for each possible +@@ -56,7 +59,24 @@ + * HAVE__BUILTIN_CTZ is defined, but we provide it anyway, so that + * extensions possibly compiled with a different compiler can use it. + */ +- ++const uint8 pg_rightmost_one_pos[256] = { ++ 0, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, ++ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 ++}; + + /* + * Array giving the number of 1-bits in each possible byte value. +@@ -64,39 +84,40 @@ + * Note: we export this for use by functions in which explicit use + * of the popcount functions seems unlikely to be a win. + */ +- +- +-/* +- * On x86_64, we can use the hardware popcount instruction, but only if +- * we can verify that the CPU supports it via the cpuid instruction. +- * +- * Otherwise, we fall back to __builtin_popcount if the compiler has that, +- * or a hand-rolled implementation if not. +- */ +-#ifdef HAVE_X86_64_POPCNTQ +-#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID) +-#define USE_POPCNT_ASM 1 +-#endif +-#endif ++const uint8 pg_number_of_ones[256] = { ++ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, ++ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, ++ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, ++ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, ++ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, ++ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, ++ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, ++ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 ++}; + + static int pg_popcount32_slow(uint32 word); + static int pg_popcount64_slow(uint64 word); + +-#ifdef USE_POPCNT_ASM ++#ifdef TRY_POPCNT_FAST + static bool pg_popcount_available(void); + static int pg_popcount32_choose(uint32 word); + static int pg_popcount64_choose(uint64 word); +-static int pg_popcount32_asm(uint32 word); +-static int pg_popcount64_asm(uint64 word); ++static int pg_popcount32_fast(uint32 word); ++static int pg_popcount64_fast(uint64 word); + + int (*pg_popcount32) (uint32 word) = pg_popcount32_choose; + int (*pg_popcount64) (uint64 word) = pg_popcount64_choose; +-#else +-int (*pg_popcount32) (uint32 word) = pg_popcount32_slow; +-int (*pg_popcount64) (uint64 word) = pg_popcount64_slow; +-#endif /* USE_POPCNT_ASM */ ++#endif /* TRY_POPCNT_FAST */ + +-#ifdef USE_POPCNT_ASM ++#ifdef TRY_POPCNT_FAST + + /* + * Return true if CPUID indicates that the POPCNT instruction is available. +@@ -128,8 +149,8 @@ + { + if (pg_popcount_available()) + { +- pg_popcount32 = pg_popcount32_asm; +- pg_popcount64 = pg_popcount64_asm; ++ pg_popcount32 = pg_popcount32_fast; ++ pg_popcount64 = pg_popcount64_fast; + } + else + { +@@ -145,8 +166,8 @@ + { + if (pg_popcount_available()) + { +- pg_popcount32 = pg_popcount32_asm; +- pg_popcount64 = pg_popcount64_asm; ++ pg_popcount32 = pg_popcount32_fast; ++ pg_popcount64 = pg_popcount64_fast; + } + else + { +@@ -158,32 +179,40 @@ + } + + /* +- * pg_popcount32_asm ++ * pg_popcount32_fast + * Return the number of 1 bits set in word + */ + static int +-pg_popcount32_asm(uint32 word) ++pg_popcount32_fast(uint32 word) + { ++#ifdef _MSC_VER ++ return __popcnt(word); ++#else + uint32 res; + + __asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); + return (int) res; ++#endif + } + + /* +- * pg_popcount64_asm ++ * pg_popcount64_fast + * Return the number of 1 bits set in word + */ + static int +-pg_popcount64_asm(uint64 word) ++pg_popcount64_fast(uint64 word) + { ++#ifdef _MSC_VER ++ return __popcnt64(word); ++#else + uint64 res; + + __asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); + return (int) res; ++#endif + } + +-#endif /* USE_POPCNT_ASM */ ++#endif /* TRY_POPCNT_FAST */ + + + /* +@@ -236,11 +265,71 @@ + #endif /* HAVE__BUILTIN_POPCOUNT */ + } + ++#ifndef TRY_POPCNT_FAST ++ ++/* ++ * When the POPCNT instruction is not available, there's no point in using ++ * function pointers to vary the implementation between the fast and slow ++ * method. We instead just make these actual external functions when ++ * TRY_POPCNT_FAST is not defined. The compiler should be able to inline ++ * the slow versions here. ++ */ ++int ++pg_popcount32(uint32 word) ++{ ++ return pg_popcount32_slow(word); ++} ++ ++int ++pg_popcount64(uint64 word) ++{ ++ return pg_popcount64_slow(word); ++} ++ ++#endif /* !TRY_POPCNT_FAST */ + + /* + * pg_popcount + * Returns the number of 1-bits in buf + */ ++uint64 ++pg_popcount(const char *buf, int bytes) ++{ ++ uint64 popcnt = 0; ++ + #if SIZEOF_VOID_P >= 8 ++ /* Process in 64-bit chunks if the buffer is aligned. */ ++ if (buf == (const char *) TYPEALIGN(8, buf)) ++ { ++ const uint64 *words = (const uint64 *) buf; ++ ++ while (bytes >= 8) ++ { ++ popcnt += pg_popcount64(*words++); ++ bytes -= 8; ++ } ++ ++ buf = (const char *) words; ++ } + #else ++ /* Process in 32-bit chunks if the buffer is aligned. */ ++ if (buf == (const char *) TYPEALIGN(4, buf)) ++ { ++ const uint32 *words = (const uint32 *) buf; ++ ++ while (bytes >= 4) ++ { ++ popcnt += pg_popcount32(*words++); ++ bytes -= 4; ++ } ++ ++ buf = (const char *) words; ++ } + #endif ++ ++ /* Process any remaining bytes */ ++ while (bytes--) ++ popcnt += pg_number_of_ones[(unsigned char) *buf++]; ++ ++ return popcnt; ++} +\ No newline at end of file +diff -ruN original/src/postgres/src_port_pgsleep.c my/src/postgres/src_port_pgsleep.c +--- original/src/postgres/src_port_pgsleep.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/postgres/src_port_pgsleep.c 2021-11-14 20:05:00.000000000 +0300 +@@ -18,8 +18,8 @@ + */ + #include "c.h" + +-#include <unistd.h> +-#include <sys/time.h> ++//#include <unistd.h> ++//#include <sys/time.h> + #ifdef HAVE_SYS_SELECT_H + #include <sys/select.h> + #endif +@@ -28,7 +28,8 @@ + * In a Windows backend, we don't use this implementation, but rather + * the signal-aware version in src/backend/port/win32/signal.c. + */ +-#if defined(FRONTEND) || !defined(WIN32) ++#if 1 ++//defined(FRONTEND) || !defined(WIN32) + + /* + * pg_usleep --- delay the specified number of microseconds. + diff --git a/ydb/library/yql/parser/pg_query_wrapper/patches/020_expose_internal.patch b/ydb/library/yql/parser/pg_query_wrapper/patches/020_expose_internal.patch index 8805f44ef8..80d8499ba5 100644 --- a/ydb/library/yql/parser/pg_query_wrapper/patches/020_expose_internal.patch +++ b/ydb/library/yql/parser/pg_query_wrapper/patches/020_expose_internal.patch @@ -1,57 +1,57 @@ -Index: contrib/pg_query.h -=================================================================== ---- contrib/pg_query.h (revision 8784387) -+++ contrib/pg_query.h (revision 8784388) -@@ -75,7 +75,7 @@ - - //PgQueryNormalizeResult pg_query_normalize(const char* input); - PgQueryScanResult pg_query_scan(const char* input); --PgQueryParseResult pg_query_parse(const char* input); -+//PgQueryParseResult pg_query_parse(const char* input); - PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); - //PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); - -Index: contrib/src/pg_query_outfuncs_defs.c -=================================================================== ---- contrib/src/pg_query_outfuncs_defs.c (revision 8784387) -+++ contrib/src/pg_query_outfuncs_defs.c (revision 8784388) -@@ -51,7 +51,7 @@ - static void _outFromExpr(OUT_TYPE(FromExpr, FromExpr) out_node, const FromExpr *node); - static void _outOnConflictExpr(OUT_TYPE(OnConflictExpr, OnConflictExpr) out_node, const OnConflictExpr *node); - static void _outIntoClause(OUT_TYPE(IntoClause, IntoClause) out_node, const IntoClause *node); --static void _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out_node, const RawStmt *node); -+/*static*/ void _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out_node, const RawStmt *node); - static void _outQuery(OUT_TYPE(Query, Query) out_node, const Query *node); - static void _outInsertStmt(OUT_TYPE(InsertStmt, InsertStmt) out_node, const InsertStmt *node); - static void _outDeleteStmt(OUT_TYPE(DeleteStmt, DeleteStmt) out_node, const DeleteStmt *node); -@@ -772,7 +772,7 @@ - WRITE_BOOL_FIELD(skip_data, skipData, skipData); - } - --static void -+/*static*/ void - _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out, const RawStmt *node) - { - WRITE_NODE_PTR_FIELD(stmt, stmt, stmt); -Index: contrib/src/pg_query_parse.c -=================================================================== ---- contrib/src/pg_query_parse.c (revision 8784387) -+++ contrib/src/pg_query_parse.c (revision 8784388) -@@ -83,7 +83,7 @@ - return result; - } - --PgQueryParseResult pg_query_parse(const char* input) -+/*PgQueryParseResult pg_query_parse(const char* input) - { - MemoryContext ctx = NULL; - PgQueryInternalParsetreeAndError parsetree_and_error; -@@ -105,7 +105,7 @@ - pg_query_exit_memory_context(ctx); - - return result; --} -+}*/ - - PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input) - { +Index: contrib/pg_query.h +=================================================================== +--- contrib/pg_query.h (revision 8784387) ++++ contrib/pg_query.h (revision 8784388) +@@ -75,7 +75,7 @@ + + //PgQueryNormalizeResult pg_query_normalize(const char* input); + PgQueryScanResult pg_query_scan(const char* input); +-PgQueryParseResult pg_query_parse(const char* input); ++//PgQueryParseResult pg_query_parse(const char* input); + PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); + //PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); + +Index: contrib/src/pg_query_outfuncs_defs.c +=================================================================== +--- contrib/src/pg_query_outfuncs_defs.c (revision 8784387) ++++ contrib/src/pg_query_outfuncs_defs.c (revision 8784388) +@@ -51,7 +51,7 @@ + static void _outFromExpr(OUT_TYPE(FromExpr, FromExpr) out_node, const FromExpr *node); + static void _outOnConflictExpr(OUT_TYPE(OnConflictExpr, OnConflictExpr) out_node, const OnConflictExpr *node); + static void _outIntoClause(OUT_TYPE(IntoClause, IntoClause) out_node, const IntoClause *node); +-static void _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out_node, const RawStmt *node); ++/*static*/ void _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out_node, const RawStmt *node); + static void _outQuery(OUT_TYPE(Query, Query) out_node, const Query *node); + static void _outInsertStmt(OUT_TYPE(InsertStmt, InsertStmt) out_node, const InsertStmt *node); + static void _outDeleteStmt(OUT_TYPE(DeleteStmt, DeleteStmt) out_node, const DeleteStmt *node); +@@ -772,7 +772,7 @@ + WRITE_BOOL_FIELD(skip_data, skipData, skipData); + } + +-static void ++/*static*/ void + _outRawStmt(OUT_TYPE(RawStmt, RawStmt) out, const RawStmt *node) + { + WRITE_NODE_PTR_FIELD(stmt, stmt, stmt); +Index: contrib/src/pg_query_parse.c +=================================================================== +--- contrib/src/pg_query_parse.c (revision 8784387) ++++ contrib/src/pg_query_parse.c (revision 8784388) +@@ -83,7 +83,7 @@ + return result; + } + +-PgQueryParseResult pg_query_parse(const char* input) ++/*PgQueryParseResult pg_query_parse(const char* input) + { + MemoryContext ctx = NULL; + PgQueryInternalParsetreeAndError parsetree_and_error; +@@ -105,7 +105,7 @@ + pg_query_exit_memory_context(ctx); + + return result; +-} ++}*/ + + PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input) + { diff --git a/ydb/library/yql/parser/pg_query_wrapper/patches/030_no_recursion.patch b/ydb/library/yql/parser/pg_query_wrapper/patches/030_no_recursion.patch index ac11c58418..91517ed087 100644 --- a/ydb/library/yql/parser/pg_query_wrapper/patches/030_no_recursion.patch +++ b/ydb/library/yql/parser/pg_query_wrapper/patches/030_no_recursion.patch @@ -1,85 +1,85 @@ ---- original/src/pg_query_outfuncs_protobuf.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/src/pg_query_outfuncs_protobuf.c 2021-11-14 21:33:02.000000000 +0300 -@@ -1,3 +1,5 @@ -+#include "protobuf/pg_query.pb-c.h" -+ - #include "pg_query_outfuncs.h" - - #include "postgres.h" -@@ -8,7 +10,8 @@ - #include "nodes/value.h" - #include "utils/datum.h" - --#include "protobuf/pg_query.pb-c.h" -+typedef void (*out_func)(void*, void*); -+int register_specific_node(void* out, void* node, out_func func); - - #define OUT_TYPE(typename, typename_c) PgQuery__##typename_c* - -@@ -85,20 +88,24 @@ - { \ - PgQuery__##typename *__node = palloc(sizeof(PgQuery__##typename)); \ - pg_query__##typename_underscore##__init(__node); \ -- _out##typename(__node, &node->fldname); \ - out->outname = __node; \ -+ if (!register_specific_node(__node, &node->fldname, (out_func)&_out##typename)) \ -+ _out##typename(__node, &node->fldname); \ - } - - #define WRITE_SPECIFIC_NODE_PTR_FIELD(typename, typename_underscore, outname, outname_json, fldname) \ - if (node->fldname != NULL) { \ - PgQuery__##typename *__node = palloc(sizeof(PgQuery__##typename)); \ - pg_query__##typename_underscore##__init(__node); \ -- _out##typename(__node, node->fldname); \ - out->outname = __node; \ -+ if (!register_specific_node(__node, node->fldname, (out_func)&_out##typename)) \ -+ _out##typename(__node, node->fldname); \ - } - - static void _outNode(PgQuery__Node* out, const void *obj); - -+void _outNodeImpl(PgQuery__Node* out, const void *obj); -+ - static void - _outList(PgQuery__List* out, const List *node) - { -@@ -182,8 +189,18 @@ - #include "pg_query_enum_defs.c" - #include "pg_query_outfuncs_defs.c" - -+int register_out_node(PgQuery__Node* out, const void *obj); -+ - static void --_outNode(PgQuery__Node* out, const void *obj) -+_outNode(PgQuery__Node* out, const void *obj) { -+ if (register_out_node(out, obj)) -+ return; -+ -+ _outNodeImpl(out, obj); -+} -+ -+void -+_outNodeImpl(PgQuery__Node* out, const void *obj) - { - if (obj == NULL) - return; // Keep out as NULL ---- original/vendor/protobuf-c/protobuf-c.c 2021-11-14 19:42:26.000000000 +0300 -+++ contrib/vendor/protobuf-c/protobuf-c.c 2021-11-14 21:33:02.000000000 +0300 -@@ -526,7 +526,7 @@ - return required_field_get_packed_size(field, member); - } - --static protobuf_c_boolean -+protobuf_c_boolean - field_is_zeroish(const ProtobufCFieldDescriptor *field, - const void *member) - { -@@ -1234,7 +1234,7 @@ - * \return - * Size of the field. - */ --static inline size_t -+size_t - sizeof_elt_in_repeated_array(ProtobufCType type) - { - switch (type) { +--- original/src/pg_query_outfuncs_protobuf.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/src/pg_query_outfuncs_protobuf.c 2021-11-14 21:33:02.000000000 +0300 +@@ -1,3 +1,5 @@ ++#include "protobuf/pg_query.pb-c.h" ++ + #include "pg_query_outfuncs.h" + + #include "postgres.h" +@@ -8,7 +10,8 @@ + #include "nodes/value.h" + #include "utils/datum.h" + +-#include "protobuf/pg_query.pb-c.h" ++typedef void (*out_func)(void*, void*); ++int register_specific_node(void* out, void* node, out_func func); + + #define OUT_TYPE(typename, typename_c) PgQuery__##typename_c* + +@@ -85,20 +88,24 @@ + { \ + PgQuery__##typename *__node = palloc(sizeof(PgQuery__##typename)); \ + pg_query__##typename_underscore##__init(__node); \ +- _out##typename(__node, &node->fldname); \ + out->outname = __node; \ ++ if (!register_specific_node(__node, &node->fldname, (out_func)&_out##typename)) \ ++ _out##typename(__node, &node->fldname); \ + } + + #define WRITE_SPECIFIC_NODE_PTR_FIELD(typename, typename_underscore, outname, outname_json, fldname) \ + if (node->fldname != NULL) { \ + PgQuery__##typename *__node = palloc(sizeof(PgQuery__##typename)); \ + pg_query__##typename_underscore##__init(__node); \ +- _out##typename(__node, node->fldname); \ + out->outname = __node; \ ++ if (!register_specific_node(__node, node->fldname, (out_func)&_out##typename)) \ ++ _out##typename(__node, node->fldname); \ + } + + static void _outNode(PgQuery__Node* out, const void *obj); + ++void _outNodeImpl(PgQuery__Node* out, const void *obj); ++ + static void + _outList(PgQuery__List* out, const List *node) + { +@@ -182,8 +189,18 @@ + #include "pg_query_enum_defs.c" + #include "pg_query_outfuncs_defs.c" + ++int register_out_node(PgQuery__Node* out, const void *obj); ++ + static void +-_outNode(PgQuery__Node* out, const void *obj) ++_outNode(PgQuery__Node* out, const void *obj) { ++ if (register_out_node(out, obj)) ++ return; ++ ++ _outNodeImpl(out, obj); ++} ++ ++void ++_outNodeImpl(PgQuery__Node* out, const void *obj) + { + if (obj == NULL) + return; // Keep out as NULL +--- original/vendor/protobuf-c/protobuf-c.c 2021-11-14 19:42:26.000000000 +0300 ++++ contrib/vendor/protobuf-c/protobuf-c.c 2021-11-14 21:33:02.000000000 +0300 +@@ -526,7 +526,7 @@ + return required_field_get_packed_size(field, member); + } + +-static protobuf_c_boolean ++protobuf_c_boolean + field_is_zeroish(const ProtobufCFieldDescriptor *field, + const void *member) + { +@@ -1234,7 +1234,7 @@ + * \return + * Size of the field. + */ +-static inline size_t ++size_t + sizeof_elt_in_repeated_array(ProtobufCType type) + { + switch (type) { diff --git a/ydb/library/yql/parser/pg_query_wrapper/patches/040_coverity_yq-571.patch b/ydb/library/yql/parser/pg_query_wrapper/patches/040_coverity_yq-571.patch index 076792e8bc..29d3f5138e 100644 --- a/ydb/library/yql/parser/pg_query_wrapper/patches/040_coverity_yq-571.patch +++ b/ydb/library/yql/parser/pg_query_wrapper/patches/040_coverity_yq-571.patch @@ -1,28 +1,28 @@ -Index: contrib/src/postgres/src_backend_utils_mb_mbutils.c -=================================================================== ---- contrib/src/postgres/src_backend_utils_mb_mbutils.c (revision 8880671) -+++ contrib/src/postgres/src_backend_utils_mb_mbutils.c (working copy) -@@ -619,7 +619,11 @@ - int - pg_database_encoding_max_length(void) - { -- return pg_wchar_table[GetDatabaseEncoding()].maxmblen; -+ int encoding = GetDatabaseEncoding(); -+ if (!PG_VALID_ENCODING(encoding)) { -+ pg_unreachable(); -+ } -+ return pg_wchar_table[encoding].maxmblen; - } - - /* -@@ -656,7 +660,9 @@ - mbverifier mbverify; - int mb_len; - -- Assert(PG_VALID_ENCODING(encoding)); -+ if (!PG_VALID_ENCODING(encoding)) { -+ pg_unreachable(); -+ } - - /* - * In single-byte encodings, we need only reject nulls (\0). +Index: contrib/src/postgres/src_backend_utils_mb_mbutils.c +=================================================================== +--- contrib/src/postgres/src_backend_utils_mb_mbutils.c (revision 8880671) ++++ contrib/src/postgres/src_backend_utils_mb_mbutils.c (working copy) +@@ -619,7 +619,11 @@ + int + pg_database_encoding_max_length(void) + { +- return pg_wchar_table[GetDatabaseEncoding()].maxmblen; ++ int encoding = GetDatabaseEncoding(); ++ if (!PG_VALID_ENCODING(encoding)) { ++ pg_unreachable(); ++ } ++ return pg_wchar_table[encoding].maxmblen; + } + + /* +@@ -656,7 +660,9 @@ + mbverifier mbverify; + int mb_len; + +- Assert(PG_VALID_ENCODING(encoding)); ++ if (!PG_VALID_ENCODING(encoding)) { ++ pg_unreachable(); ++ } + + /* + * In single-byte encodings, we need only reject nulls (\0). diff --git a/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h b/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h index 62082ec869..e9da86f7e7 100644 --- a/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h +++ b/ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h @@ -3,7 +3,7 @@ #include <ydb/library/yql/parser/proto_ast/proto_ast.h> #include <ydb/library/yql/public/issue/yql_issue.h> -namespace NSQLTranslation { +namespace NSQLTranslation { class TErrorCollectorOverIssues : public NProtoAST::IErrorCollector { public: @@ -24,4 +24,4 @@ private: const TString File_; }; -} // namespace NSQLTranslation +} // namespace NSQLTranslation diff --git a/ydb/library/yql/parser/proto_ast/collect_issues/ya.make b/ydb/library/yql/parser/proto_ast/collect_issues/ya.make index 9d09a3b6fe..14a878884d 100644 --- a/ydb/library/yql/parser/proto_ast/collect_issues/ya.make +++ b/ydb/library/yql/parser/proto_ast/collect_issues/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - -OWNER(g:yql g:yql_ydb_core) - -PEERDIR( +LIBRARY() + +OWNER(g:yql g:yql_ydb_core) + +PEERDIR( ydb/library/yql/public/issue ydb/library/yql/parser/proto_ast -) - -SRCS( - collect_issues.h -) - -END() +) + +SRCS( + collect_issues.h +) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make b/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make index 488fa6bf99..deeb75f177 100644 --- a/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make @@ -11,7 +11,7 @@ IF (CPP_PROTO) SET(PROTOBUF_HEADER_PATH ${MODDIR}) SET(LEXER_PARSER_NAMESPACE NALP) - + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg) diff --git a/ydb/library/yql/parser/proto_ast/gen/v0/ya.make b/ydb/library/yql/parser/proto_ast/gen/v0/ya.make index c796690df8..7aa382db01 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v0/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v0/ya.make @@ -9,7 +9,7 @@ IF (CPP_PROTO) SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates) SET(sql_grammar ${ARCADIA_ROOT}/ydb/library/yql/sql/v0/SQL.g) - + SET(ANTLR_PACKAGE_NAME NSQLGenerated) SET(PROTOBUF_HEADER_PATH ${MODDIR}) SET(LEXER_PARSER_NAMESPACE NALP) diff --git a/ydb/library/yql/parser/proto_ast/gen/v1/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1/ya.make index 1996db33f2..2a3c9560d0 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v1/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v1/ya.make @@ -1,24 +1,24 @@ -LIBRARY() +LIBRARY() -PEERDIR ( +PEERDIR ( ydb/library/yql/parser/proto_ast/gen/v1_proto -) - +) + OWNER(g:yql g:yql_ydb_core) -SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) +SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates) -SET(sql_grammar ${antlr_output}/SQLv1.g) +SET(sql_grammar ${antlr_output}/SQLv1.g) SET(ANTLR_PACKAGE_NAME NSQLv1Generated) SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto) -SET(LEXER_PARSER_NAMESPACE NALPDefault) - -SET(GRAMMAR_STRING_CORE_SINGLE "\"~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .)\"") -SET(GRAMMAR_STRING_CORE_DOUBLE "\"~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .)\"") -SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") - +SET(LEXER_PARSER_NAMESPACE NALPDefault) + +SET(GRAMMAR_STRING_CORE_SINGLE "\"~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .)\"") +SET(GRAMMAR_STRING_CORE_DOUBLE "\"~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .)\"") +SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1.g.in ${sql_grammar}) @@ -33,7 +33,7 @@ RUN_ANTLR( IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg OUT SQLv1Parser.cpp SQLv1Lexer.cpp SQLv1Parser.h SQLv1Lexer.h OUTPUT_INCLUDES - ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h + ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h ${STG_INCLUDES} CWD ${antlr_output} ) diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make index 940a0a004b..d9ac8c257b 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make @@ -1,41 +1,41 @@ -LIBRARY() - -PEERDIR ( +LIBRARY() + +PEERDIR ( ydb/library/yql/parser/proto_ast/gen/v1_proto -) - -OWNER(g:yql g:yql_ydb_core) - -SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) -SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates) -SET(sql_grammar ${antlr_output}/SQLv1.g) - -SET(ANTLR_PACKAGE_NAME NSQLv1Generated) +) + +OWNER(g:yql g:yql_ydb_core) + +SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) +SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates) +SET(sql_grammar ${antlr_output}/SQLv1.g) + +SET(ANTLR_PACKAGE_NAME NSQLv1Generated) SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto) - -SET(LEXER_PARSER_NAMESPACE NALPAnsi) - -SET(GRAMMAR_STRING_CORE_SINGLE "\"~QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE)\"") -SET(GRAMMAR_STRING_CORE_DOUBLE "\"~QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE)\"") -SET(GRAMMAR_MULTILINE_COMMENT_CORE "\"MULTILINE_COMMENT | .\"") - + +SET(LEXER_PARSER_NAMESPACE NALPAnsi) + +SET(GRAMMAR_STRING_CORE_SINGLE "\"~QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE)\"") +SET(GRAMMAR_STRING_CORE_DOUBLE "\"~QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE)\"") +SET(GRAMMAR_MULTILINE_COMMENT_CORE "\"MULTILINE_COMMENT | .\"") + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1.g.in ${sql_grammar}) - -NO_COMPILER_WARNINGS() - + +NO_COMPILER_WARNINGS() + INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) - -RUN_ANTLR( - ${sql_grammar} - -lib . - -fo ${antlr_output} - IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg - OUT SQLv1Parser.cpp SQLv1Lexer.cpp SQLv1Parser.h SQLv1Lexer.h - OUTPUT_INCLUDES - ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h - ${STG_INCLUDES} - CWD ${antlr_output} -) - -END() + +RUN_ANTLR( + ${sql_grammar} + -lib . + -fo ${antlr_output} + IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg + OUT SQLv1Parser.cpp SQLv1Lexer.cpp SQLv1Parser.h SQLv1Lexer.h + OUTPUT_INCLUDES + ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.h + ${STG_INCLUDES} + CWD ${antlr_output} +) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_proto/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_proto/ya.make index dacba25864..6432aaacad 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v1_proto/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v1_proto/ya.make @@ -1,21 +1,21 @@ -PROTO_LIBRARY() - -OWNER(g:yql g:yql_ydb_core) - +PROTO_LIBRARY() + +OWNER(g:yql g:yql_ydb_core) + IF (GEN_PROTO) SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) SET(antlr_templates ${antlr_output}/org/antlr/codegen/templates) SET(sql_grammar ${antlr_output}/SQLv1.g) - + SET(ANTLR_PACKAGE_NAME NSQLv1Generated) - + SET(GRAMMAR_STRING_CORE_SINGLE "\"~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .)\"") SET(GRAMMAR_STRING_CORE_DOUBLE "\"~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .)\"") SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") - + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg) CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1.g.in ${sql_grammar}) - + RUN_ANTLR( ${sql_grammar} -lib . @@ -26,9 +26,9 @@ IF (GEN_PROTO) CWD ${antlr_output} ) ENDIF() - + SRCS(SQLv1Parser.proto) EXCLUDE_TAGS(GO_PROTO JAVA_PROTO) - -END() + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/ya.make b/ydb/library/yql/parser/proto_ast/gen/ya.make index 429c22b673..72c7ac4da7 100644 --- a/ydb/library/yql/parser/proto_ast/gen/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/ya.make @@ -1,6 +1,6 @@ RECURSE( v0 v1 - v1_proto + v1_proto jsonpath ) diff --git a/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in b/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in index 621d51db71..e18b3ad7f6 100755 --- a/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in +++ b/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in @@ -133,7 +133,7 @@ outputFile( LEXER, * Include the ANTLR3 generated header file. */ <if(PARSER)> -#include \<@PROTOBUF_HEADER_PATH@/<name>.pb.h> +#include \<@PROTOBUF_HEADER_PATH@/<name>.pb.h> <endif> #include "<recognizer.grammar.name>Lexer.h" @@ -148,8 +148,8 @@ outputFile( LEXER, <recognizer.grammar.delegators: {g|#include "<g.recognizerName>.hpp" }; separator="\n"> <endif> -namespace @LEXER_PARSER_NAMESPACE@ { - +namespace @LEXER_PARSER_NAMESPACE@ { + /* ----------------------------------------- */ <docComment> @@ -178,8 +178,8 @@ namespace @LEXER_PARSER_NAMESPACE@ { <recognizer> -} // namespace @LEXER_PARSER_NAMESPACE@ - +} // namespace @LEXER_PARSER_NAMESPACE@ + /* End of code * ============================================================================= */ @@ -340,8 +340,8 @@ headerFile( LEXER, #include \<ydb/library/yql/parser/proto_ast/proto_ast.h> -namespace @LEXER_PARSER_NAMESPACE@ { - +namespace @LEXER_PARSER_NAMESPACE@ { + <if(LEXER)> template \<class ImplTraits> class <name>Traits : public antlr3::CustomTraitsBase\<ImplTraits> { @@ -359,14 +359,14 @@ typedef antlr3::Traits\<<recognizer.grammar.name>Lexer, <recognizer.grammar.name #undef TOKEN_QUERY #endif -namespace { - -inline bool IsBetween(ANTLR_UINT32 value, ANTLR_UINT32 lower, ANTLR_UINT32 upper) { - return value >= lower && value \<= upper; -} - -} - +namespace { + +inline bool IsBetween(ANTLR_UINT32 value, ANTLR_UINT32 lower, ANTLR_UINT32 upper) { + return value >= lower && value \<= upper; +} + +} + class <name>Tokens { public: /** Symbolic definitions of all the tokens that the <grammarType()> will work with. @@ -433,9 +433,9 @@ private: NProtoAST::IErrorCollector* Errors = nullptr; ui32 FollowDepth_ = 0; google::protobuf::Arena* Arena = nullptr; - static inline bool IsHiddenToken(ANTLR_UINT32 token) { - return token == TOKEN_WS || token == TOKEN_COMMENT; - } + static inline bool IsHiddenToken(ANTLR_UINT32 token) { + return token == TOKEN_WS || token == TOKEN_COMMENT; + } public: <name>(InputType* instream<recognizer.grammar.delegators:{g|, <g.recognizerName>* <g:delegateName()>}>, google::protobuf::Arena* arena); @@ -514,8 +514,8 @@ extern ANTLR_UINT8* <recognizer.grammar.composite.rootGrammar.recognizerName>T * ============================================================================= */ -} // namespace @LEXER_PARSER_NAMESPACE@ - +} // namespace @LEXER_PARSER_NAMESPACE@ + <endNamespace(actions)> /* END - Note:Keep extra line feed to satisfy UNIX systems */ @@ -1457,7 +1457,7 @@ void <name>::m<ruleName>(<ruleDescriptor.parameterScope:parameterScope()>) <block> // TODO: FIX - if (IsHiddenToken(_type)) { + if (IsHiddenToken(_type)) { this->get_lexstate()->set_channel(HIDDEN); } this->get_lexstate()->set_type(_type); @@ -1597,7 +1597,7 @@ positiveClosureBlock(alts,decls,decision,enclosingBlockLevel,blockLevel,decision } loop<decisionNumber>: ; /* Jump to here if this rule does not match */ <if(PARSER)> - Y_VERIFY(parentBlock<enclosingBlockLevel>, "parentBlock is null"); + Y_VERIFY(parentBlock<enclosingBlockLevel>, "parentBlock is null"); if (!parentBlock<enclosingBlockLevel>->Block<elemId>Size()) { <ruleBacktrackFailure()> <earlyExitEx()> @@ -1711,7 +1711,7 @@ alt(elements,altNum,description,autoAST,outerAlt,treeLevel,rew) ::= << <@initializations()> <if(PARSER)> <if(!ruleDescriptor.isSynPred)> - Y_VERIFY(parent, "Parent is null"); + Y_VERIFY(parent, "Parent is null"); parent->SetDescr("<description>"); <endif> <endif> @@ -1739,7 +1739,7 @@ tokenRef(token,label,elementIndex,terminalOptions) ::= << auto token = matchToken(TOKEN_<token>, &FOLLOW_<token>_in_<ruleName><elementIndex>); if (token) { <if(!ruleDescriptor.isSynPred)> - Y_VERIFY(parent, "Parent is null"); + Y_VERIFY(parent, "Parent is null"); parent->MutableToken<elemId>()->SetId(token->get_type()); parent->MutableToken<elemId>()->SetLine(token->get_line()); parent->MutableToken<elemId>()->SetColumn(token->get_charPositionInLine()); @@ -1797,7 +1797,7 @@ matchSet(s,label,elementIndex,terminalOptions,postmatchCode="") ::= << <if(PARSER)> if (token) { <if(!ruleDescriptor.isSynPred)> - Y_VERIFY(parent, "Parent is null"); + Y_VERIFY(parent, "Parent is null"); parent->MutableToken<elemId>()->SetId(token->get_type()); parent->MutableToken<elemId>()->SetLine(token->get_line()); parent->MutableToken<elemId>()->SetColumn(token->get_charPositionInLine()); @@ -1919,7 +1919,7 @@ Y_ASSERT(dynamic_cast\<@ANTLR_PACKAGE_NAME@::TRule_<rule.name>* >(res)); ruleResult() ::= << <if(!ruleDescriptor.isSynPred)> -Y_VERIFY(parent, "Parent is null"); +Y_VERIFY(parent, "Parent is null"); parent->unsafe_arena_set_allocated_rule_<rule.name><elemId>(static_cast\<@ANTLR_PACKAGE_NAME@::TRule_<rule.name>* >(res)); <endif> >> @@ -2347,9 +2347,9 @@ cyclicDFAState(decisionNumber,stateNumber,edges,needErrorClause,semPredState) :: ctx->rewindLast();<\n> <endif> s = -1; - [&]() -> void { - <edges; separator="\nelse "> - }(); + [&]() -> void { + <edges; separator="\nelse "> + }(); <if(semPredState)> <! return input cursor to state before we rewound !> ctx->seek(index<decisionNumber>_<stateNumber>);<\n> <endif> @@ -2399,10 +2399,10 @@ lookaheadTest(atom,k,atomAsInt) ::= "LA<decisionNumber>_<stateNumber> == <if(PAR isolatedLookaheadTest(atom,k,atomAsInt) ::= "this->LA(<k>) == <if(PARSER)>TOKEN_<endif><atom>" lookaheadRangeTest(lower,upper,k,rangeNumber,lowerAsInt,upperAsInt) ::= <% -IsBetween(LA<decisionNumber>_<stateNumber>, <if(PARSER)>TOKEN_<endif><lower>, <if(PARSER)>TOKEN_<endif><upper>) +IsBetween(LA<decisionNumber>_<stateNumber>, <if(PARSER)>TOKEN_<endif><lower>, <if(PARSER)>TOKEN_<endif><upper>) %> -isolatedLookaheadRangeTest(lower,upper,k,rangeNumber,lowerAsInt,upperAsInt) ::= "IsBetween(this->LA(<k>), <if(PARSER)>TOKEN_<endif><lower>, <if(PARSER)>TOKEN_<endif><upper>)" +isolatedLookaheadRangeTest(lower,upper,k,rangeNumber,lowerAsInt,upperAsInt) ::= "IsBetween(this->LA(<k>), <if(PARSER)>TOKEN_<endif><lower>, <if(PARSER)>TOKEN_<endif><upper>)" setTest(ranges) ::= "<ranges; separator=\" || \">" diff --git a/ydb/library/yql/parser/proto_ast/proto_ast.h b/ydb/library/yql/parser/proto_ast/proto_ast.h index 9f5d188ca2..5bb8a9f5f9 100644 --- a/ydb/library/yql/parser/proto_ast/proto_ast.h +++ b/ydb/library/yql/parser/proto_ast/proto_ast.h @@ -1,12 +1,12 @@ #pragma once #include <ydb/library/yql/parser/lexer_common/lexer.h> - + #include <contrib/libs/antlr3_cpp_runtime/include/antlr3.hpp> #include <google/protobuf/message.h> #include <util/generic/ptr.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/charset/utf8.h> namespace NProtoAST { @@ -100,52 +100,52 @@ namespace NProtoAST { typename TParser::TokenStreamType TokenStream; TParser Parser; }; - - template <typename TLexer> - class TLexerTokensCollector { - typedef ANTLR_UINT8 TChar; - - public: - TLexerTokensCollector(TStringBuf data, const char** tokenNames, const TString& queryName = "query") - : TokenNames(tokenNames) - , QueryName(queryName) - , InputStream((const TChar*)data.data(), antlr3::ENC_8BIT, data.length(), (TChar*)QueryName.begin()) - , Lexer(&InputStream, static_cast<google::protobuf::Arena*>(nullptr)) - { - } - - NSQLTranslation::TParsedTokenList CollectTokens(IErrorCollector& errors) { - NSQLTranslation::TParsedTokenList result; - try { - Lexer.ReportErrors(&errors); - auto src = Lexer.get_tokSource(); - for (;;) { - auto token = src->nextToken(); - auto type = token->getType(); - const bool isEOF = type == TLexer::CommonTokenType::TOKEN_EOF; - result.emplace_back(); - NSQLTranslation::TParsedToken& last = result.back(); - last.Name = isEOF ? "EOF" : TokenNames[type]; - last.Content = token->getText(); - last.Line = token->get_line(); - last.LinePos = token->get_charPositionInLine(); - if (isEOF) { - break; - } - } - return result; - } catch (const TTooManyErrors&) { - return result; - } catch (const yexception& e) { - errors.Error(0, 0, e.what()); - return result; - } - } - - private: - const char** TokenNames; - TString QueryName; - typename TLexer::InputStreamType InputStream; - TLexer Lexer; - }; + + template <typename TLexer> + class TLexerTokensCollector { + typedef ANTLR_UINT8 TChar; + + public: + TLexerTokensCollector(TStringBuf data, const char** tokenNames, const TString& queryName = "query") + : TokenNames(tokenNames) + , QueryName(queryName) + , InputStream((const TChar*)data.data(), antlr3::ENC_8BIT, data.length(), (TChar*)QueryName.begin()) + , Lexer(&InputStream, static_cast<google::protobuf::Arena*>(nullptr)) + { + } + + NSQLTranslation::TParsedTokenList CollectTokens(IErrorCollector& errors) { + NSQLTranslation::TParsedTokenList result; + try { + Lexer.ReportErrors(&errors); + auto src = Lexer.get_tokSource(); + for (;;) { + auto token = src->nextToken(); + auto type = token->getType(); + const bool isEOF = type == TLexer::CommonTokenType::TOKEN_EOF; + result.emplace_back(); + NSQLTranslation::TParsedToken& last = result.back(); + last.Name = isEOF ? "EOF" : TokenNames[type]; + last.Content = token->getText(); + last.Line = token->get_line(); + last.LinePos = token->get_charPositionInLine(); + if (isEOF) { + break; + } + } + return result; + } catch (const TTooManyErrors&) { + return result; + } catch (const yexception& e) { + errors.Error(0, 0, e.what()); + return result; + } + } + + private: + const char** TokenNames; + TString QueryName; + typename TLexer::InputStreamType InputStream; + TLexer Lexer; + }; } // namespace NProtoAST diff --git a/ydb/library/yql/parser/proto_ast/ya.make b/ydb/library/yql/parser/proto_ast/ya.make index 47d9838612..536d1c27f8 100644 --- a/ydb/library/yql/parser/proto_ast/ya.make +++ b/ydb/library/yql/parser/proto_ast/ya.make @@ -18,5 +18,5 @@ END() RECURSE( gen - collect_issues + collect_issues ) diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp index 7e8da10497..fd18aa1286 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp @@ -95,11 +95,11 @@ public: } auto itemType = found->ItemType; - auto columnOrder = found->ColumnOrder; + auto columnOrder = found->ColumnOrder; if (columnsSet) { - TVector<const TItemExprType*> items = itemType->GetItems(); - EraseIf(items, [&](const TItemExprType* item) { return !columnsSet->contains(item->GetName()); }); - EraseIf(columnOrder, [&](const TString& col) { return !columnsSet->contains(col); }); + TVector<const TItemExprType*> items = itemType->GetItems(); + EraseIf(items, [&](const TItemExprType* item) { return !columnsSet->contains(item->GetName()); }); + EraseIf(columnOrder, [&](const TString& col) { return !columnsSet->contains(col); }); itemType = ctx.MakeType<TStructExprType>(items); } @@ -108,7 +108,7 @@ public: ctx.MakeType<TListExprType>(itemType) })); - return State_->Types->SetColumnOrder(*input, columnOrder, ctx); + return State_->Types->SetColumnOrder(*input, columnOrder, ctx); } private: diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp index 5ae5bcc1a1..eeaeffcffc 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp @@ -162,7 +162,7 @@ public: .Columns<TCoVoid>().Build() .Timezone().Value(parse.second).Build() .Done().Ptr(); - State_->Tables.emplace(it->first, meta); + State_->Tables.emplace(it->first, meta); } else bad = true; break; diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h index 0c352d3bde..7184fe1ba3 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h @@ -17,7 +17,7 @@ struct TClickHouseState : public TThrRefBase struct TTableMeta { const TStructExprType* ItemType = nullptr; - TVector<TString> ColumnOrder; + TVector<TString> ColumnOrder; }; THashMap<std::pair<TString, TString>, TTableMeta> Tables; diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.cpp index 304d8c423b..c13d5a7d90 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.cpp @@ -9,7 +9,7 @@ TClickHouseConfiguration::TClickHouseConfiguration() } TClickHouseSettings::TConstPtr TClickHouseConfiguration::Snapshot() const { - return std::make_shared<const TClickHouseSettings>(*this); + return std::make_shared<const TClickHouseSettings>(*this); } bool TClickHouseConfiguration::HasCluster(TStringBuf cluster) const { diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h index 5583b6219b..8d0b094af4 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h @@ -11,7 +11,7 @@ namespace NYql { struct TClickHouseSettings { - using TConstPtr = std::shared_ptr<const TClickHouseSettings>; + using TConstPtr = std::shared_ptr<const TClickHouseSettings>; }; struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TSettingDispatcher { diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 2787e6858c..7587e188fa 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -290,7 +290,7 @@ TMaybe<TVector<ui32>> CreateStructPositions(TType* inputType, const TVector<TStr return structPositions; } -namespace { +namespace { NYT::TNode DataValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type) { YQL_ENSURE(type->GetKind() == TType::EKind::Data); @@ -357,53 +357,53 @@ NYT::TNode DataValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr return NYT::TNode(ToString(TStringBuf(value.AsStringRef()))); } } - YQL_ENSURE(false, "Unsupported type: " << static_cast<int>(dataType->GetSchemeType())); + YQL_ENSURE(false, "Unsupported type: " << static_cast<int>(dataType->GetSchemeType())); } -TExprNode::TPtr DataNodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) { - YQL_ENSURE(type.GetKind() == ETypeAnnotationKind::Data, "Expecting data type, got: " << type); - - TString strData; - if (type.Cast<TDataExprType>()->GetSlot() == EDataSlot::Yson) { - strData = NYT::NodeToYsonString(node); - } else { - switch (node.GetType()) { - case NYT::TNode::String: - strData = node.AsString(); - break; - case NYT::TNode::Int64: - strData = ToString(node.AsInt64()); - break; - case NYT::TNode::Uint64: - strData = ToString(node.AsUint64()); - break; - case NYT::TNode::Double: - strData = FloatToString(node.AsDouble()); - break; - case NYT::TNode::Bool: - strData = ToString(node.AsBool()); - break; - default: - YQL_ENSURE(false, "Unexpected Yson type: " << node.GetType() << " while deserializing literal of type " << type); - } - } - - return ctx.Builder(pos) - .Callable(type.Cast<TDataExprType>()->GetName()) - .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - parent.Atom(0, strData); - if (IsDataTypeDecimal(type.Cast<TDataExprType>()->GetSlot())) { - auto decimalType = type.Cast<TDataExprParamsType>(); - parent.Atom(1, decimalType->GetParamOne()); - parent.Atom(2, decimalType->GetParamTwo()); - } - return parent; - }) - .Seal() - .Build(); -} - - +TExprNode::TPtr DataNodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) { + YQL_ENSURE(type.GetKind() == ETypeAnnotationKind::Data, "Expecting data type, got: " << type); + + TString strData; + if (type.Cast<TDataExprType>()->GetSlot() == EDataSlot::Yson) { + strData = NYT::NodeToYsonString(node); + } else { + switch (node.GetType()) { + case NYT::TNode::String: + strData = node.AsString(); + break; + case NYT::TNode::Int64: + strData = ToString(node.AsInt64()); + break; + case NYT::TNode::Uint64: + strData = ToString(node.AsUint64()); + break; + case NYT::TNode::Double: + strData = FloatToString(node.AsDouble()); + break; + case NYT::TNode::Bool: + strData = ToString(node.AsBool()); + break; + default: + YQL_ENSURE(false, "Unexpected Yson type: " << node.GetType() << " while deserializing literal of type " << type); + } + } + + return ctx.Builder(pos) + .Callable(type.Cast<TDataExprType>()->GetName()) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + parent.Atom(0, strData); + if (IsDataTypeDecimal(type.Cast<TDataExprType>()->GetSlot())) { + auto decimalType = type.Cast<TDataExprParamsType>(); + parent.Atom(1, decimalType->GetParamOne()); + parent.Atom(2, decimalType->GetParamTwo()); + } + return parent; + }) + .Seal() + .Build(); +} + + TString DataValueToString(const NKikimr::NUdf::TUnboxedValuePod& value, const TDataExprType* type) { switch (type->GetSlot()) { case NUdf::EDataSlot::Int32: @@ -467,90 +467,90 @@ TString DataValueToString(const NKikimr::NUdf::TUnboxedValuePod& value, const TD Y_FAIL("Unexpected"); } -} //namespace - -NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type) { - NYT::TNode result; - switch (type->GetKind()) { - case TType::EKind::Optional: { - result = NYT::TNode::CreateList(); - if (value) { - result.Add(ValueToNode(value.GetOptionalValue(), AS_TYPE(TOptionalType, type)->GetItemType())); - } - break; - } - case TType::EKind::Tuple: { - auto tupleType = AS_TYPE(TTupleType, type); - result = NYT::TNode::CreateList(); - for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { - result.Add(ValueToNode(value.GetElement(i), tupleType->GetElementType(i))); - } - break; - } - case TType::EKind::List: { - auto listType = AS_TYPE(TListType, type); - result = NYT::TNode::CreateList(); - const auto iter = value.GetListIterator(); - for (NUdf::TUnboxedValue item; iter.Next(item); ) { - result.Add(ValueToNode(item, listType->GetItemType())); - } - break; - } - default: { - result = DataValueToNode(value, type); - } - } - return result; +} //namespace + +NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type) { + NYT::TNode result; + switch (type->GetKind()) { + case TType::EKind::Optional: { + result = NYT::TNode::CreateList(); + if (value) { + result.Add(ValueToNode(value.GetOptionalValue(), AS_TYPE(TOptionalType, type)->GetItemType())); + } + break; + } + case TType::EKind::Tuple: { + auto tupleType = AS_TYPE(TTupleType, type); + result = NYT::TNode::CreateList(); + for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { + result.Add(ValueToNode(value.GetElement(i), tupleType->GetElementType(i))); + } + break; + } + case TType::EKind::List: { + auto listType = AS_TYPE(TListType, type); + result = NYT::TNode::CreateList(); + const auto iter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item); ) { + result.Add(ValueToNode(item, listType->GetItemType())); + } + break; + } + default: { + result = DataValueToNode(value, type); + } + } + return result; +} + +TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) { + TExprNode::TPtr result; + switch(type.GetKind()) { + case ETypeAnnotationKind::Optional: { + YQL_ENSURE(node.IsList() || node.IsNull()); + if (node.IsNull() || node.AsList().empty()) { + return ctx.NewCallable(pos, "Nothing", { ExpandType(pos, type, ctx) }); + } + YQL_ENSURE(node.AsList().size() == 1); + result = ctx.NewCallable(pos, "Just", { + NodeToExprLiteral(pos, *type.Cast<TOptionalExprType>()->GetItemType(), node.AsList().front(), ctx) + }); + break; + } + case ETypeAnnotationKind::Tuple: { + YQL_ENSURE(node.IsList()); + const TTypeAnnotationNode::TListType& itemTypes = type.Cast<TTupleExprType>()->GetItems(); + const auto& items = node.AsList(); + YQL_ENSURE(itemTypes.size() == items.size()); + TExprNodeList resultNodes; + for (size_t i = 0; i < items.size(); ++i) { + resultNodes.push_back(NodeToExprLiteral(pos, *itemTypes[i], items[i], ctx)); + } + result = ctx.NewList(pos, std::move(resultNodes)); + break; + } + case ETypeAnnotationKind::List: { + YQL_ENSURE(node.IsList()); + const TTypeAnnotationNode& itemType = *type.Cast<TListExprType>()->GetItemType(); + if (node.AsList().empty()) { + return ctx.NewCallable(pos, "List", { ExpandType(pos, *ctx.MakeType<TListExprType>(&itemType), ctx) }); + } + + TExprNodeList children; + for (auto& child : node.AsList()) { + children.push_back(NodeToExprLiteral(pos, itemType, child, ctx)); + } + + result = ctx.NewCallable(pos, "AsList", std::move(children)); + break; + } + default: { + result = DataNodeToExprLiteral(pos, type, node, ctx); + } + } + return result; } -TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) { - TExprNode::TPtr result; - switch(type.GetKind()) { - case ETypeAnnotationKind::Optional: { - YQL_ENSURE(node.IsList() || node.IsNull()); - if (node.IsNull() || node.AsList().empty()) { - return ctx.NewCallable(pos, "Nothing", { ExpandType(pos, type, ctx) }); - } - YQL_ENSURE(node.AsList().size() == 1); - result = ctx.NewCallable(pos, "Just", { - NodeToExprLiteral(pos, *type.Cast<TOptionalExprType>()->GetItemType(), node.AsList().front(), ctx) - }); - break; - } - case ETypeAnnotationKind::Tuple: { - YQL_ENSURE(node.IsList()); - const TTypeAnnotationNode::TListType& itemTypes = type.Cast<TTupleExprType>()->GetItems(); - const auto& items = node.AsList(); - YQL_ENSURE(itemTypes.size() == items.size()); - TExprNodeList resultNodes; - for (size_t i = 0; i < items.size(); ++i) { - resultNodes.push_back(NodeToExprLiteral(pos, *itemTypes[i], items[i], ctx)); - } - result = ctx.NewList(pos, std::move(resultNodes)); - break; - } - case ETypeAnnotationKind::List: { - YQL_ENSURE(node.IsList()); - const TTypeAnnotationNode& itemType = *type.Cast<TListExprType>()->GetItemType(); - if (node.AsList().empty()) { - return ctx.NewCallable(pos, "List", { ExpandType(pos, *ctx.MakeType<TListExprType>(&itemType), ctx) }); - } - - TExprNodeList children; - for (auto& child : node.AsList()) { - children.push_back(NodeToExprLiteral(pos, itemType, child, ctx)); - } - - result = ctx.NewCallable(pos, "AsList", std::move(children)); - break; - } - default: { - result = DataNodeToExprLiteral(pos, type, node, ctx); - } - } - return result; -} - void CopyYsonWithAttrs(char cmd, TInputBuf& buf, TVector<char>& yson) { if (cmd == BeginAttributesSymbol) { yson.push_back(cmd); @@ -704,15 +704,15 @@ TStringBuf ReadNextString(char cmd, TInputBuf& buf) { template <typename T> T ReadNextSerializedNumber(char cmd, TInputBuf& buf) { auto nextString = ReadNextString(cmd, buf); - if constexpr (!std::numeric_limits<T>::is_integer) { - if (nextString == "inf" || nextString == "+inf") { - return std::numeric_limits<T>::infinity(); - } else if (nextString == "-inf") { - return -std::numeric_limits<T>::infinity(); - } else if (nextString == "nan") { - return std::numeric_limits<T>::quiet_NaN(); - } - } + if constexpr (!std::numeric_limits<T>::is_integer) { + if (nextString == "inf" || nextString == "+inf") { + return std::numeric_limits<T>::infinity(); + } else if (nextString == "-inf") { + return -std::numeric_limits<T>::infinity(); + } else if (nextString == "nan") { + return std::numeric_limits<T>::quiet_NaN(); + } + } return FromString<T>(nextString); } @@ -2252,7 +2252,7 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl } TExprNode::TPtr ValueToExprLiteral(const TTypeAnnotationNode* type, const NKikimr::NUdf::TUnboxedValuePod& value, TExprContext& ctx, - TPositionHandle pos) { + TPositionHandle pos) { switch (type->GetKind()) { case ETypeAnnotationKind::Variant: { auto variantType = type->Cast<TVariantExprType>(); diff --git a/ydb/library/yql/providers/common/codec/yql_codec.h b/ydb/library/yql/providers/common/codec/yql_codec.h index 7336455e33..512e01224c 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.h +++ b/ydb/library/yql/providers/common/codec/yql_codec.h @@ -43,8 +43,8 @@ TMaybe<TVector<ui32>> CreateStructPositions( const TVector<TString>* columns = nullptr ); -NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type); -TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx); +NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type); +TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx); struct TCodecContext { const NKikimr::NMiniKQL::TTypeEnvironment& Env; @@ -98,7 +98,7 @@ extern "C" void WriteContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 void WriteYsonValueInTableFormat(TOutputBuf& buf, NKikimr::NMiniKQL::TType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel); TExprNode::TPtr ValueToExprLiteral(const TTypeAnnotationNode* type, const NKikimr::NUdf::TUnboxedValuePod& value, TExprContext& ctx, - TPositionHandle pos = {}); + TPositionHandle pos = {}); } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_formatcode.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_formatcode.cpp index 6f382fa71f..c6ec91c0d5 100644 --- a/ydb/library/yql/providers/common/comp_nodes/yql_formatcode.cpp +++ b/ydb/library/yql/providers/common/comp_nodes/yql_formatcode.cpp @@ -10,18 +10,18 @@ namespace NMiniKQL { class TFormatCodeWrapper : public TMutableComputationNode<TFormatCodeWrapper> { typedef TMutableComputationNode<TFormatCodeWrapper> TBaseComputation; public: - TFormatCodeWrapper(TComputationMutables& mutables, IComputationNode* code, bool annotatePosition, ui32 exprCtxMutableIndex) + TFormatCodeWrapper(TComputationMutables& mutables, IComputationNode* code, bool annotatePosition, ui32 exprCtxMutableIndex) : TBaseComputation(mutables) , Code_(code) , AnnotatePosition_(annotatePosition) - , ExprCtxMutableIndex_(exprCtxMutableIndex) + , ExprCtxMutableIndex_(exprCtxMutableIndex) {} NUdf::TUnboxedValue DoCalculate(TComputationContext& ctx) const { auto codeValue = Code_->GetValue(ctx); auto code = GetYqlCode(codeValue); - NYql::TExprContext& exprCtx = GetExprContext(ctx, ExprCtxMutableIndex_); - auto ast = NYql::ConvertToAst(*code, exprCtx, AnnotatePosition_ ? + NYql::TExprContext& exprCtx = GetExprContext(ctx, ExprCtxMutableIndex_); + auto ast = NYql::ConvertToAst(*code, exprCtx, AnnotatePosition_ ? NYql::TExprAnnotationFlags::Position : NYql::TExprAnnotationFlags::None, true); auto str = ast.Root->ToString(NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); @@ -35,23 +35,23 @@ public: private: IComputationNode* Code_; bool AnnotatePosition_; - const ui32 ExprCtxMutableIndex_; + const ui32 ExprCtxMutableIndex_; }; class TSerializeCodeWrapper : public TMutableComputationNode<TSerializeCodeWrapper> { typedef TMutableComputationNode<TSerializeCodeWrapper> TBaseComputation; public: - TSerializeCodeWrapper(TComputationMutables& mutables, IComputationNode* code, ui32 exprCtxMutableIndex) + TSerializeCodeWrapper(TComputationMutables& mutables, IComputationNode* code, ui32 exprCtxMutableIndex) : TBaseComputation(mutables) , Code_(code) - , ExprCtxMutableIndex_(exprCtxMutableIndex) + , ExprCtxMutableIndex_(exprCtxMutableIndex) {} NUdf::TUnboxedValue DoCalculate(TComputationContext& ctx) const { auto codeValue = Code_->GetValue(ctx); auto code = GetYqlCode(codeValue); - NYql::TExprContext& exprCtx = GetExprContext(ctx, ExprCtxMutableIndex_); - auto str = NYql::SerializeGraph(*code, exprCtx, + NYql::TExprContext& exprCtx = GetExprContext(ctx, ExprCtxMutableIndex_); + auto str = NYql::SerializeGraph(*code, exprCtx, NYql::TSerializedExprGraphComponents::Graph | NYql::TSerializedExprGraphComponents::Positions); return MakeString(str); @@ -63,14 +63,14 @@ public: private: IComputationNode* Code_; - const ui32 ExprCtxMutableIndex_; + const ui32 ExprCtxMutableIndex_; }; template <bool AnnotatePosition> IComputationNode* WrapFormatCode(TCallable& callable, const TComputationNodeFactoryContext& ctx, ui32 exprCtxMutableIndex) { MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 arg"); auto code = LocateNode(ctx.NodeLocator, callable, 0); - return new TFormatCodeWrapper(ctx.Mutables, code, AnnotatePosition, exprCtxMutableIndex); + return new TFormatCodeWrapper(ctx.Mutables, code, AnnotatePosition, exprCtxMutableIndex); } template IComputationNode* WrapFormatCode<false> @@ -82,7 +82,7 @@ template IComputationNode* WrapFormatCode<true> IComputationNode* WrapSerializeCode(TCallable& callable, const TComputationNodeFactoryContext& ctx, ui32 exprCtxMutableIndex) { MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 arg"); auto code = LocateNode(ctx.NodeLocator, callable, 0); - return new TSerializeCodeWrapper(ctx.Mutables, code, exprCtxMutableIndex); + return new TSerializeCodeWrapper(ctx.Mutables, code, exprCtxMutableIndex); } } diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.cpp index 7c95193dc5..9d693ab95a 100644 --- a/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.cpp +++ b/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.cpp @@ -10,10 +10,10 @@ extern const char YqlTypeResourceTag[] = "_Type"; extern const char YqlCodeResourceTag[] = "_Expr"; extern const char YqlExprContextResourceTag[] = "_ExprContext"; -std::shared_ptr<NYql::TExprContext> GetExprContextPtr(TComputationContext& ctx, ui32 index) { +std::shared_ptr<NYql::TExprContext> GetExprContextPtr(TComputationContext& ctx, ui32 index) { auto& value = ctx.MutableValues[index]; if (value.IsInvalid()) { - value = NUdf::TUnboxedValuePod(new TYqlExprContextResource(std::make_shared<NYql::TExprContext>())); + value = NUdf::TUnboxedValuePod(new TYqlExprContextResource(std::make_shared<NYql::TExprContext>())); } return *dynamic_cast<TYqlExprContextResource*>(value.AsBoxed().Get())->Get(); diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.h b/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.h index f4c3df0b1d..14b045154a 100644 --- a/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.h +++ b/ydb/library/yql/providers/common/comp_nodes/yql_type_resource.h @@ -11,14 +11,14 @@ extern const char YqlTypeResourceTag[6]; extern const char YqlCodeResourceTag[6]; extern const char YqlExprContextResourceTag[13]; -using TYqlTypeResource = NUdf::TBoxedResource<std::pair<std::shared_ptr<NYql::TExprContext>, +using TYqlTypeResource = NUdf::TBoxedResource<std::pair<std::shared_ptr<NYql::TExprContext>, const NYql::TTypeAnnotationNode*>, YqlTypeResourceTag>; -using TYqlCodeResource = NUdf::TBoxedResource<std::pair<std::shared_ptr<NYql::TExprContext>, +using TYqlCodeResource = NUdf::TBoxedResource<std::pair<std::shared_ptr<NYql::TExprContext>, NYql::TExprNode::TPtr>, YqlCodeResourceTag>; -using TYqlExprContextResource = NUdf::TBoxedResource<std::shared_ptr<NYql::TExprContext>, YqlExprContextResourceTag>; +using TYqlExprContextResource = NUdf::TBoxedResource<std::shared_ptr<NYql::TExprContext>, YqlExprContextResourceTag>; NYql::TExprContext& GetExprContext(TComputationContext& ctx, ui32 index); -std::shared_ptr<NYql::TExprContext> GetExprContextPtr(TComputationContext& ctx, ui32 index); +std::shared_ptr<NYql::TExprContext> GetExprContextPtr(TComputationContext& ctx, ui32 index); const NYql::TTypeAnnotationNode* GetYqlType(const NUdf::TUnboxedValue& value); NYql::TExprNode::TPtr GetYqlCode(const NUdf::TUnboxedValue& value); diff --git a/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp b/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp index 5a7d32d5af..9fbd44acb3 100644 --- a/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp +++ b/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp @@ -78,7 +78,7 @@ IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprN auto name = TString(node->Child(3)->Content()); if (name.StartsWith('_')) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(3)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(node->Child(3)->Pos()), TStringBuilder() << "Failed to override system setting: " << name)); return nullptr; } @@ -113,7 +113,7 @@ IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprN return nullptr; } } else { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(2)->Pos()), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(node->Child(2)->Pos()), TStringBuilder() << "Unsupported configuration option: " << atom)); return nullptr; } @@ -125,7 +125,7 @@ IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprN return status; } -bool TProviderConfigurationTransformer::HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, +bool TProviderConfigurationTransformer::HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, const TMaybe<TString>& value, TExprContext& ctx) { Y_UNUSED(pos); @@ -134,17 +134,17 @@ bool TProviderConfigurationTransformer::HandleAttr(TPositionHandle pos, const TS return true; } -bool TProviderConfigurationTransformer::HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, +bool TProviderConfigurationTransformer::HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx) { auto cred = Types.FindCredential(alias); if (!cred) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Unknown credential: " << alias)); + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Unknown credential: " << alias)); return false; } if (cred->Category != Provider) { - ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Mismatch credential category, expected: " << Provider << ", but found: " << cred->Category)); return false; diff --git a/ydb/library/yql/providers/common/config/yql_configuration_transformer.h b/ydb/library/yql/providers/common/config/yql_configuration_transformer.h index 9ce584fa8d..38997103ff 100644 --- a/ydb/library/yql/providers/common/config/yql_configuration_transformer.h +++ b/ydb/library/yql/providers/common/config/yql_configuration_transformer.h @@ -20,9 +20,9 @@ public: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final; protected: - virtual bool HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, + virtual bool HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, const TMaybe<TString>& value, TExprContext& ctx); - virtual bool HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx); + virtual bool HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx); protected: TSettingDispatcher::TPtr Dispatcher; diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index 02bfccf167..dd1f162d78 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -593,27 +593,27 @@ TMkqlCommonCallableCompiler::TShared::TShared() { {"WideTakeWhileInclusive", &TProgramBuilder::WideTakeWhileInclusive}, }); - AddSimpleCallables({ - {"RangeUnion", &TProgramBuilder::RangeUnion}, - {"RangeIntersect", &TProgramBuilder::RangeIntersect}, - {"RangeMultiply", &TProgramBuilder::RangeMultiply}, - }); - - AddSimpleCallables({ - {"RangeCreate", &TProgramBuilder::RangeCreate}, - {"RangeFinalize", &TProgramBuilder::RangeFinalize}, - }); - - AddCallable({"RoundUp", "RoundDown"}, [](const TExprNode& node, TMkqlBuildContext& ctx) { - const auto arg = MkqlBuildExpr(node.Head(), ctx); - const auto dstType = BuildType(node.Tail(), *node.Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); - return ctx.ProgramBuilder.Round(node.Content(), arg, dstType); - }); - - AddSimpleCallables({ - {"NextValue", &TProgramBuilder::NextValue}, - }); - + AddSimpleCallables({ + {"RangeUnion", &TProgramBuilder::RangeUnion}, + {"RangeIntersect", &TProgramBuilder::RangeIntersect}, + {"RangeMultiply", &TProgramBuilder::RangeMultiply}, + }); + + AddSimpleCallables({ + {"RangeCreate", &TProgramBuilder::RangeCreate}, + {"RangeFinalize", &TProgramBuilder::RangeFinalize}, + }); + + AddCallable({"RoundUp", "RoundDown"}, [](const TExprNode& node, TMkqlBuildContext& ctx) { + const auto arg = MkqlBuildExpr(node.Head(), ctx); + const auto dstType = BuildType(node.Tail(), *node.Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); + return ctx.ProgramBuilder.Round(node.Content(), arg, dstType); + }); + + AddSimpleCallables({ + {"NextValue", &TProgramBuilder::NextValue}, + }); + AddCallable({"MultiMap", "OrderedMultiMap"}, [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto arg = MkqlBuildExpr(node.Head(), ctx); const auto lambda = [&](TRuntimeNode item) { return MkqlBuildWideLambda(node.Tail(), ctx, {item}); }; @@ -1193,8 +1193,8 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("Unwrap", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto opt = MkqlBuildExpr(node.Head(), ctx); const auto message = node.ChildrenSize() > 1 ? MkqlBuildExpr(node.Tail(), ctx) : ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(""); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); - return ctx.ProgramBuilder.Unwrap(opt, message, pos.File, pos.Row, pos.Column); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + return ctx.ProgramBuilder.Unwrap(opt, message, pos.File, pos.Row, pos.Column); }); AddCallable("Nothing", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -1491,15 +1491,15 @@ TMkqlCommonCallableCompiler::TShared::TShared() { sortedTableOrder = sortSetting->Child(1)->Content() == "left" ? 0 : 1; } - - EAnyJoinSettings anyJoinSettings = EAnyJoinSettings::None; - if (const auto anyNode = GetSetting(*node.Child(6), "any")) { - for (auto sideNode : anyNode->Child(1)->Children()) { - YQL_ENSURE(sideNode->IsAtom()); - AddAnyJoinSide(anyJoinSettings, sideNode->Content() == "left" ? EAnyJoinSettings::Left : EAnyJoinSettings::Right); - } - } - + + EAnyJoinSettings anyJoinSettings = EAnyJoinSettings::None; + if (const auto anyNode = GetSetting(*node.Child(6), "any")) { + for (auto sideNode : anyNode->Child(1)->Children()) { + YQL_ENSURE(sideNode->IsAtom()); + AddAnyJoinSide(anyJoinSettings, sideNode->Content() == "left" ? EAnyJoinSettings::Left : EAnyJoinSettings::Right); + } + } + const auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); return ctx.ProgramBuilder.CommonJoinCore(list, joinKind, leftColumns, rightColumns, requiredColumns, keyColumns, memLimit, sortedTableOrder, anyJoinSettings, tableIndexFieldPos, returnType); @@ -1644,7 +1644,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() { TMaybe<bool> isHashed; TMaybe<ui64> itemsCount; bool isCompact; - if (const auto error = ParseToDictSettings(node, ctx.ExprCtx, isMany, isHashed, itemsCount, isCompact)) { + if (const auto error = ParseToDictSettings(node, ctx.ExprCtx, isMany, isHashed, itemsCount, isCompact)) { ythrow TNodeException(node) << error->Message; } @@ -1821,8 +1821,8 @@ TMkqlCommonCallableCompiler::TShared::TShared() { const auto value = MkqlBuildExpr(node.Head(), ctx); const auto predicate = MkqlBuildExpr(*node.Child(1), ctx); const auto message = node.ChildrenSize() > 2 ? MkqlBuildExpr(node.Tail(), ctx) : ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(""); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); - return ctx.ProgramBuilder.Ensure(value, predicate, message, pos.File, pos.Row, pos.Column); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + return ctx.ProgramBuilder.Ensure(value, predicate, message, pos.File, pos.Row, pos.Column); }); AddCallable("Replicate", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -1916,11 +1916,11 @@ TMkqlCommonCallableCompiler::TShared::TShared() { const auto yson = WriteTypeToYson(type); const auto& args = GetAllArguments(node, ctx); const auto retType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); TCallableBuilder call(ctx.ProgramBuilder.GetTypeEnvironment(), node.Content(), retType); - call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); + call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); for (auto arg : args) { call.Add(arg); @@ -1985,11 +1985,11 @@ TMkqlCommonCallableCompiler::TShared::TShared() { }, [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto& args = GetAllArguments(node, ctx); const auto retType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); TCallableBuilder call(ctx.ProgramBuilder.GetTypeEnvironment(), node.Content(), retType); - call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); + call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); for (auto arg : args) { call.Add(arg); @@ -2001,11 +2001,11 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("LambdaCode", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto lambda = node.Child(node.ChildrenSize() - 1); const auto retType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); TCallableBuilder call(ctx.ProgramBuilder.GetTypeEnvironment(), node.Content(), retType); - call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); - call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); + call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(pos.File)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Row)); + call.Add(ctx.ProgramBuilder.NewDataLiteral(pos.Column)); if (node.ChildrenSize() == 2) { auto count = MkqlBuildExpr(node.Head(), ctx); call.Add(count); @@ -2058,14 +2058,14 @@ TMkqlCommonCallableCompiler::TShared::TShared() { return MkqlBuildExpr(node.Head(), ctx); }); - AddCallable("Error", [](const TExprNode& node, TMkqlBuildContext& ctx)->NKikimr::NMiniKQL::TRuntimeNode { + AddCallable("Error", [](const TExprNode& node, TMkqlBuildContext& ctx)->NKikimr::NMiniKQL::TRuntimeNode { const auto err = node.GetTypeAnn()->Cast<TErrorExprType>()->GetError(); - ythrow TNodeException(ctx.ExprCtx.AppendPosition(err.Position)) << err.Message; + ythrow TNodeException(ctx.ExprCtx.AppendPosition(err.Position)) << err.Message; }); - AddCallable("ErrorType", [](const TExprNode& node, TMkqlBuildContext& ctx)->NKikimr::NMiniKQL::TRuntimeNode { + AddCallable("ErrorType", [](const TExprNode& node, TMkqlBuildContext& ctx)->NKikimr::NMiniKQL::TRuntimeNode { const auto err = node.GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TErrorExprType>()->GetError(); - ythrow TNodeException(ctx.ExprCtx.AppendPosition(err.Position)) << err.Message; + ythrow TNodeException(ctx.ExprCtx.AppendPosition(err.Position)) << err.Message; }); AddCallable("Join", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2126,9 +2126,9 @@ TMkqlCommonCallableCompiler::TShared::TShared() { const auto userType = BuildType(*node.Child(2), *node.Child(2)->GetTypeAnn(), ctx.ProgramBuilder, true); const auto typeConfig = node.Child(3)->Content(); const auto callableType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); return ctx.ProgramBuilder.TypedUdf(function, callableType, runConfig, userType, typeConfig, - pos.File, pos.Row, pos.Column); + pos.File, pos.Row, pos.Column); }); AddCallable("ScriptUdf", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2143,9 +2143,9 @@ TMkqlCommonCallableCompiler::TShared::TShared() { const auto typeNode = node.Child(2); const auto funcType = BuildType(*typeNode, *typeNode->GetTypeAnn(), ctx.ProgramBuilder); const auto script = MkqlBuildExpr(*node.Child(3), ctx); - const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); + const auto pos = ctx.ExprCtx.GetPosition(node.Pos()); return ctx.ProgramBuilder.ScriptUdf(scriptType, funcName, funcType, script, - pos.File, pos.Row, pos.Column); + pos.File, pos.Row, pos.Column); }); AddCallable("Apply", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2245,21 +2245,21 @@ TMkqlCommonCallableCompiler::TShared::TShared() { return ctx.ProgramBuilder.QueuePeek(resource, index, args, returnType); }); - AddCallable("QueueRange", [](const TExprNode& node, TMkqlBuildContext& ctx) { - const auto resource = MkqlBuildExpr(node.Head(), ctx); - const auto begin = MkqlBuildExpr(*node.Child(1), ctx); - const auto end = MkqlBuildExpr(*node.Child(2), ctx); - const auto& args = GetArgumentsFrom<3U>(node, ctx); - const auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - return ctx.ProgramBuilder.QueueRange(resource, begin, end, args, returnType); - }); - - AddCallable("Seq", [](const TExprNode& node, TMkqlBuildContext& ctx) { - const auto& args = GetArgumentsFrom<0U>(node, ctx); - const auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - return ctx.ProgramBuilder.Seq(args, returnType); - }); - + AddCallable("QueueRange", [](const TExprNode& node, TMkqlBuildContext& ctx) { + const auto resource = MkqlBuildExpr(node.Head(), ctx); + const auto begin = MkqlBuildExpr(*node.Child(1), ctx); + const auto end = MkqlBuildExpr(*node.Child(2), ctx); + const auto& args = GetArgumentsFrom<3U>(node, ctx); + const auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); + return ctx.ProgramBuilder.QueueRange(resource, begin, end, args, returnType); + }); + + AddCallable("Seq", [](const TExprNode& node, TMkqlBuildContext& ctx) { + const auto& args = GetArgumentsFrom<0U>(node, ctx); + const auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); + return ctx.ProgramBuilder.Seq(args, returnType); + }); + AddCallable("FromYsonSimpleType", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto input = MkqlBuildExpr(node.Head(), ctx); const auto schemeType = ParseDataType(node, node.Child(1)->Content()); diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h index 880672ee82..a25c03b488 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h @@ -15,7 +15,7 @@ struct TMkqlBuildContext { const IMkqlCallableCompiler& MkqlCompiler; NKikimr::NMiniKQL::TProgramBuilder& ProgramBuilder; - TExprContext& ExprCtx; + TExprContext& ExprCtx; TMemoizedNodesMap Memoization; TMkqlBuildContext *const ParentCtx = nullptr; const size_t Level = 0ULL; @@ -25,7 +25,7 @@ struct TMkqlBuildContext { TMkqlBuildContext(const IMkqlCallableCompiler& mkqlCompiler, NKikimr::NMiniKQL::TProgramBuilder& builder, TExprContext& exprCtx, ui64 lambdaId = 0ULL, TArgumentsMap&& args = {}) : MkqlCompiler(mkqlCompiler) , ProgramBuilder(builder) - , ExprCtx(exprCtx) + , ExprCtx(exprCtx) , Memoization(std::move(args)) , LambdaId(lambdaId) {} @@ -33,7 +33,7 @@ struct TMkqlBuildContext { TMkqlBuildContext(TMkqlBuildContext& parent, TArgumentsMap&& args, ui64 lambdaId) : MkqlCompiler(parent.MkqlCompiler) , ProgramBuilder(parent.ProgramBuilder) - , ExprCtx(parent.ExprCtx) + , ExprCtx(parent.ExprCtx) , Memoization(std::move(args)) , ParentCtx(&parent) , Level(parent.Level + 1U) diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index c7565067c0..8561fe2536 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -418,21 +418,21 @@ message TYqlCoreConfig { repeated TCoreAttr Flags = 1; } -/////////////////////////////// Sql Core /////////////////////////////// -message TWarnAsErrorByHourPercentage { - required uint32 Hour = 1; - required uint32 Percentage = 2; -} - -message TSqlCoreConfig { - optional uint32 V0SyntaxWarnAsErrorPercentage = 1 [default = 0]; // Probability of 'deprecated syntax' - // warning to become an error - repeated TWarnAsErrorByHourPercentage V0SyntaxWarnAsErrorByHour = 2; - repeated string NoV0SyntaxErrorForUsers = 3; +/////////////////////////////// Sql Core /////////////////////////////// +message TWarnAsErrorByHourPercentage { + required uint32 Hour = 1; + required uint32 Percentage = 2; +} + +message TSqlCoreConfig { + optional uint32 V0SyntaxWarnAsErrorPercentage = 1 [default = 0]; // Probability of 'deprecated syntax' + // warning to become an error + repeated TWarnAsErrorByHourPercentage V0SyntaxWarnAsErrorByHour = 2; + repeated string NoV0SyntaxErrorForUsers = 3; repeated string TranslationFlags = 4; -} - +} + /////////////////////////////// Root /////////////////////////////// message TGatewaysConfig { @@ -447,7 +447,7 @@ message TGatewaysConfig { optional TFileStorageAdditionalConfig Fs = 9; optional TYqlCoreConfig YqlCore = 10; optional TPostgresqlGatewayConfig Postgresql = 11; - optional TSqlCoreConfig SqlCore = 12; + optional TSqlCoreConfig SqlCore = 12; optional TDqGatewayConfig Dq = 13; optional TMysqlGatewayConfig Mysql = 14; optional TYdbGatewayConfig Ydb = 15; diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp index a8f793274a..8de3d57fd3 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp @@ -58,21 +58,21 @@ TString TPlanFormatterBase::GetOperationDisplayName(const TExprNode& node) { return TString(node.Content()); } -void TTrackableNodeProcessorBase::GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) { - Y_UNUSED(node); - usedNodeIds.clear(); -} - -void TTrackableNodeProcessorBase::GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) { - Y_UNUSED(node); - Y_UNUSED(ctx); - createdNodes.clear(); -} - -IGraphTransformer& TTrackableNodeProcessorBase::GetCleanupTransformer() { - return NullTransformer_; -} - +void TTrackableNodeProcessorBase::GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) { + Y_UNUSED(node); + usedNodeIds.clear(); +} + +void TTrackableNodeProcessorBase::GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) { + Y_UNUSED(node); + Y_UNUSED(ctx); + createdNodes.clear(); +} + +IGraphTransformer& TTrackableNodeProcessorBase::GetCleanupTransformer() { + return NullTransformer_; +} + bool TDataProviderBase::Initialize(TExprContext& ctx) { Y_UNUSED(ctx); return true; @@ -278,10 +278,10 @@ IPlanFormatter& TDataProviderBase::GetPlanFormatter() { return *this; } -ITrackableNodeProcessor& TDataProviderBase::GetTrackableNodeProcessor() { - return NullTrackableNodeProcessor_; -} - +ITrackableNodeProcessor& TDataProviderBase::GetTrackableNodeProcessor() { + return NullTrackableNodeProcessor_; +} + IGraphTransformer& TDataProviderBase::GetPlanInfoTransformer() { return NullTransformer_; } diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h index b2b41b991b..242a39877e 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h @@ -24,18 +24,18 @@ public: TString GetOperationDisplayName(const TExprNode& node) override; }; -class TTrackableNodeProcessorBase : public ITrackableNodeProcessor { -public: - TTrackableNodeProcessorBase() = default; - - void GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) override; - void GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) override; - IGraphTransformer& GetCleanupTransformer() override; - -protected: - TNullTransformer NullTransformer_; -}; - +class TTrackableNodeProcessorBase : public ITrackableNodeProcessor { +public: + TTrackableNodeProcessorBase() = default; + + void GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) override; + void GetCreatedNodes(const TExprNode& node, TVector<TExprNodeAndId>& createdNodes, TExprContext& ctx) override; + IGraphTransformer& GetCleanupTransformer() override; + +protected: + TNullTransformer NullTransformer_; +}; + class TDataProviderBase : public IDataProvider, public TPlanFormatterBase { public: TDataProviderBase() = default; @@ -79,14 +79,14 @@ public: bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) override; bool CollectDiscoveredData(NYson::TYsonWriter& writer) override; IPlanFormatter& GetPlanFormatter() override; - ITrackableNodeProcessor& GetTrackableNodeProcessor() override; + ITrackableNodeProcessor& GetTrackableNodeProcessor() override; IGraphTransformer& GetPlanInfoTransformer() override; IDqIntegration* GetDqIntegration() override; protected: THolder<IGraphTransformer> DefConstraintTransformer_; TNullTransformer NullTransformer_; - TTrackableNodeProcessorBase NullTrackableNodeProcessor_; + TTrackableNodeProcessorBase NullTrackableNodeProcessor_; }; } // namespace NYql diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 7a68a05ba0..aa782ee93f 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -20,7 +20,7 @@ using namespace NNodes; bool TCommitSettings::EnsureModeEmpty(TExprContext& ctx) { if (Mode) { - ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() << "Unsupported mode:" << Mode.Cast().Value())); return false; } @@ -30,7 +30,7 @@ bool TCommitSettings::EnsureModeEmpty(TExprContext& ctx) { bool TCommitSettings::EnsureEpochEmpty(TExprContext& ctx) { if (Epoch) { - ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() << "Epochs are unsupported.")); return false; } @@ -40,7 +40,7 @@ bool TCommitSettings::EnsureEpochEmpty(TExprContext& ctx) { bool TCommitSettings::EnsureOtherEmpty(TExprContext& ctx) { if (!Other.Empty()) { - ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() + ctx.AddError(TIssue(ctx.GetPosition(Pos), TStringBuilder() << "Unsupported setting:" << Other.Item(0).Name().Value())); return false; } @@ -90,7 +90,7 @@ const TStructExprType* BuildCommonTableListType(TExprContext& ctx) { return ctx.MakeType<TStructExprType>(items); } -TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& ann, TExprContext& ctx) { +TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& ann, TExprContext& ctx) { return ExpandType(pos, ann, ctx); } @@ -434,7 +434,7 @@ bool FillUsedFilesImpl( const auto& name = node.Head().Content(); const auto block = types.UserDataStorage->FindUserDataBlock(name); if (!block) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << name)); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << name)); return false; } else { @@ -447,7 +447,7 @@ bool FillUsedFilesImpl( const auto& name = node.Head().Content(); auto blocks = types.UserDataStorage->FindUserDataFolder(name); if (!blocks) { - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Folder not found: " << name)); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Folder not found: " << name)); return false; } else { for (const auto& x : *blocks) { @@ -643,25 +643,25 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedF completedFuture.GetValue()(); } catch (const std::exception& e) { - auto inputPos = ctx.GetPosition(input->Pos()); + auto inputPos = ctx.GetPosition(input->Pos()); TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED)); + return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED)); }); - ctx.AddError(ExceptionToIssue(e, inputPos)); - input->SetState(TExprNode::EState::Error); + ctx.AddError(ExceptionToIssue(e, inputPos)); + input->SetState(TExprNode::EState::Error); return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); } catch (...) { - auto inputPos = ctx.GetPosition(input->Pos()); + auto inputPos = ctx.GetPosition(input->Pos()); TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED)); + return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED)); }); - ctx.AddError(YqlIssue(inputPos, TIssuesIds::UNEXPECTED, CurrentExceptionMessage())); - input->SetState(TExprNode::EState::Error); + ctx.AddError(YqlIssue(inputPos, TIssuesIds::UNEXPECTED, CurrentExceptionMessage())); + input->SetState(TExprNode::EState::Error); return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); } - input->SetState(TExprNode::EState::ExecutionRequired); + input->SetState(TExprNode::EState::ExecutionRequired); return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat); }); })); @@ -693,19 +693,19 @@ void WriteColumns(NYson::TYsonWriter& writer, const TExprBase& columns) { } } -TString SerializeExpr(TExprContext& ctx, const TExprNode& expr, bool withTypes) { +TString SerializeExpr(TExprContext& ctx, const TExprNode& expr, bool withTypes) { ui32 typeFlags = TExprAnnotationFlags::None; if (withTypes) { typeFlags |= TExprAnnotationFlags::Types; } - auto ast = ConvertToAst(expr, ctx, typeFlags, true); + auto ast = ConvertToAst(expr, ctx, typeFlags, true); YQL_ENSURE(ast.Root); return ast.Root->ToString(); } -TString ExprToPrettyString(TExprContext& ctx, const TExprNode& expr) { - auto ast = ConvertToAst(expr, ctx, TExprAnnotationFlags::None, true); +TString ExprToPrettyString(TExprContext& ctx, const TExprNode& expr) { + auto ast = ConvertToAst(expr, ctx, TExprAnnotationFlags::None, true); TStringStream exprStream; YQL_ENSURE(ast.Root); ast.Root->PrettyPrintTo(exprStream, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); @@ -834,7 +834,7 @@ void WriteStreams(NYson::TYsonWriter& writer, TStringBuf name, const NNodes::TCo writer.OnEndList(); } -double GetDataReplicationFactor(double factor, const TExprNode* node, const TExprNode* stream, TExprContext& ctx) { +double GetDataReplicationFactor(double factor, const TExprNode* node, const TExprNode* stream, TExprContext& ctx) { if (node == stream) { return factor; } @@ -851,12 +851,12 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp double applyFactor = 0.0; for (size_t i = 1; i < node->ChildrenSize(); ++i) { if (IsFlowOrStream(node->Child(i))) { - applyFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx); + applyFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx); } else if (node->Child(i)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) { if (node->Child(i)->IsCallable("ForwardList")) { - applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx); + applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx); } else if (node->Child(i)->IsCallable("Collect") && IsFlowOrStream(node->Child(i)->HeadPtr().Get())) { - applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx); + applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx); } } } @@ -870,7 +870,7 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp } if (!TCoExtendBase::Match(node) && node->ChildrenSize() > 0) { - factor = GetDataReplicationFactor(factor, node->Child(0), stream, ctx); + factor = GetDataReplicationFactor(factor, node->Child(0), stream, ctx); } if (TCoFlatMapBase::Match(node)) { @@ -908,7 +908,7 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp TMaybe<bool> isHashed; bool isCompact = false; TMaybe<ui64> itemsCount; - ParseToDictSettings(*node->Child(1), ctx, isMany, isHashed, itemsCount, isCompact); + ParseToDictSettings(*node->Child(1), ctx, isMany, isHashed, itemsCount, isCompact); if (isMany.GetOrElse(true)) { factor *= 5.0; } @@ -924,7 +924,7 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp else if (TCoExtendBase::Match(node) && node->ChildrenSize() > 0) { double extendFactor = 0.0; for (size_t i = 0; i < node->ChildrenSize(); ++i) { - extendFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx); + extendFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx); } factor = Max(1.0, extendFactor); } @@ -935,7 +935,7 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp return factor; } -double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx) { +double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx) { return GetDataReplicationFactor(1.0, lambda.Child(1), lambda.Head().ChildrenSize() > 0 ? lambda.Head().Child(0) : nullptr, ctx); } diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index ccdd4861ca..9e80cfd135 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -62,7 +62,7 @@ struct TWriteRoleSettings { struct TCommitSettings { - TPositionHandle Pos; + TPositionHandle Pos; NNodes::TMaybeNode<NNodes::TCoAtom> Mode; NNodes::TMaybeNode<NNodes::TCoAtom> Epoch; NNodes::TCoNameValueTupleList Other; @@ -79,7 +79,7 @@ struct TCommitSettings const TStructExprType* BuildCommonTableListType(TExprContext& ctx); -TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& ann, TExprContext& ctx); +TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& ann, TExprContext& ctx); bool HasResOrPullOption(const TExprNode& node, const TStringBuf& option); @@ -110,13 +110,13 @@ bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTy void WriteColumns(NYson::TYsonWriter& writer, const NNodes::TExprBase& columns); -TString SerializeExpr(TExprContext& ctx, const TExprNode& expr, bool withTypes = false); -TString ExprToPrettyString(TExprContext& ctx, const TExprNode& expr); +TString SerializeExpr(TExprContext& ctx, const TExprNode& expr, bool withTypes = false); +TString ExprToPrettyString(TExprContext& ctx, const TExprNode& expr); void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprNode* source); void WriteStreams(NYson::TYsonWriter& writer, TStringBuf name, const NNodes::TCoLambda& lambda); -double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx); +double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx); void WriteStatistics(NYson::TYsonWriter& writer, bool totalOnly, const THashMap<ui32, TOperationStatistics>& statistics); diff --git a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp b/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp index bffa253e24..93b84f6db0 100644 --- a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp +++ b/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp @@ -2,7 +2,7 @@ #include <ydb/library/yql/ast/yql_expr.h> #include <array> - + namespace NYql { namespace NCommon { @@ -289,7 +289,7 @@ public: TKeyRange BuildKeyRange(TExprBase row, TExprContext& ctx) const { if (IsFullScan()) { - return TKeyRange(ctx, KeyColumns.size(), ResidualPredicate); + return TKeyRange(ctx, KeyColumns.size(), ResidualPredicate); } auto newResidualPredicate = ResidualPredicate; @@ -311,7 +311,7 @@ public: } } - return TKeyRange(ctx, newColumnRanges, newResidualPredicate); + return TKeyRange(ctx, newColumnRanges, newResidualPredicate); } private: @@ -436,7 +436,7 @@ private: : predicate; if (i == keyRanges.size() - 1 || less(keyRanges[i], keyRanges[i + 1])) { - TKeyRange keyRange(ctx.ExprCtx, keyRanges[i].GetColumnRanges(), residualPredicate); + TKeyRange keyRange(ctx.ExprCtx, keyRanges[i].GetColumnRanges(), residualPredicate); keyRanges[curIndex] = keyRange; ++curIndex; @@ -487,7 +487,7 @@ private: }); // TODO: Compute actual bounding range. - return TKeyRange(ctx, KeyColumns.size(), residualPredicate); + return TKeyRange(ctx, KeyColumns.size(), residualPredicate); } private: @@ -747,46 +747,46 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate, } } - auto maybeLookup = TMaybeNode<TCoLookupBase>(); - if (auto maybeLiteral = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) { - if (maybeLiteral.Cast().Value() == "false") { - maybeLookup = predicate.Maybe<TCoCoalesce>().Predicate().Maybe<TCoLookupBase>(); + auto maybeLookup = TMaybeNode<TCoLookupBase>(); + if (auto maybeLiteral = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) { + if (maybeLiteral.Cast().Value() == "false") { + maybeLookup = predicate.Maybe<TCoCoalesce>().Predicate().Maybe<TCoLookupBase>(); } - } else { - maybeLookup = predicate.Maybe<TCoLookupBase>(); - } - - if (maybeLookup) { - auto lookup = maybeLookup.Cast(); - - if (!lookup.Lookup().Maybe<TCoMember>()) { + } else { + maybeLookup = predicate.Maybe<TCoLookupBase>(); + } + + if (maybeLookup) { + auto lookup = maybeLookup.Cast(); + + if (!lookup.Lookup().Maybe<TCoMember>()) { return fullScan; } - auto member = lookup.Lookup().Cast<TCoMember>(); + auto member = lookup.Lookup().Cast<TCoMember>(); auto column = member.Name().Value(); if (member.Struct().Raw() != row.Raw()) { return fullScan; } - TExprNode::TPtr collection; - if (lookup.Collection().Ref().IsList()) { - collection = lookup.Collection().Ptr(); - } else if (auto maybeDictFromKeys = lookup.Collection().Maybe<TCoDictFromKeys>()) { - collection = maybeDictFromKeys.Cast().Keys().Ptr(); - } else { - return fullScan; - } - - auto size = collection->ChildrenSize(); - if (!size) { - return fullScan; - } - + TExprNode::TPtr collection; + if (lookup.Collection().Ref().IsList()) { + collection = lookup.Collection().Ptr(); + } else if (auto maybeDictFromKeys = lookup.Collection().Maybe<TCoDictFromKeys>()) { + collection = maybeDictFromKeys.Cast().Keys().Ptr(); + } else { + return fullScan; + } + + auto size = collection->ChildrenSize(); + if (!size) { + return fullScan; + } + TVector<TKeyRangeBuilder> keyRanges; - keyRanges.reserve(size); - for (const auto& key : collection->Children()) { - auto maybeValue = ctx.GetValueFunc(TExprBase(key), member.Ref().GetTypeAnn(), ctx.ExprCtx); + keyRanges.reserve(size); + for (const auto& key : collection->Children()) { + auto maybeValue = ctx.GetValueFunc(TExprBase(key), member.Ref().GetTypeAnn(), ctx.ExprCtx); if (!maybeValue) { return fullScan; } @@ -844,15 +844,15 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate, } // namespace TColumnRange TColumnRange::MakeNull(TExprContext& ctx) { - auto nullValue = Build<TCoNull>(ctx, TPositionHandle()).Done(); + auto nullValue = Build<TCoNull>(ctx, TPositionHandle()).Done(); TColumnRange range = MakePoint(nullValue); range.Null = true; return range; } -TKeyRange::TKeyRange(TExprContext& ctx, const TVector<TColumnRange>& columnRanges, TMaybeNode<NNodes::TExprBase> residualPredicate) - : Ctx(&ctx) - , ColumnRanges(columnRanges) +TKeyRange::TKeyRange(TExprContext& ctx, const TVector<TColumnRange>& columnRanges, TMaybeNode<NNodes::TExprBase> residualPredicate) + : Ctx(&ctx) + , ColumnRanges(columnRanges) , ResidualPredicate(residualPredicate) , NumDefined(0) { @@ -890,8 +890,8 @@ TKeyRange::TKeyRange(TExprContext& ctx, const TVector<TColumnRange>& columnRange } } - FromTuple = TKeyTuple(Ctx, fromValues, true, fromInclusive); - ToTuple = TKeyTuple(Ctx, toValues, false, toInclusive); + FromTuple = TKeyTuple(Ctx, fromValues, true, fromInclusive); + ToTuple = TKeyTuple(Ctx, toValues, false, toInclusive); } void TTableLookup::Print(IOutputStream& output) const { @@ -903,8 +903,8 @@ void TTableLookup::Print(IOutputStream& output) const { } void TKeyRange::Print(IOutputStream& output) const { - auto printExpr = [ctx = Ctx] (TExprBase node, IOutputStream& output) { - auto ast = ConvertToAst(node.Ref(), *ctx, TExprAnnotationFlags::None, true); + auto printExpr = [ctx = Ctx] (TExprBase node, IOutputStream& output) { + auto ast = ConvertToAst(node.Ref(), *ctx, TExprAnnotationFlags::None, true); ast.Root->PrintTo(output); }; @@ -960,8 +960,8 @@ void TKeyRange::Print(IOutputStream& output) const { } void TKeyTuple::Print(IOutputStream& output) const { - auto printExpr = [] (TExprContext& ctx, TExprBase node, IOutputStream& output) { - auto ast = ConvertToAst(node.Ref(), ctx, TExprAnnotationFlags::None, true); + auto printExpr = [] (TExprContext& ctx, TExprBase node, IOutputStream& output) { + auto ast = ConvertToAst(node.Ref(), ctx, TExprAnnotationFlags::None, true); ast.Root->PrintTo(output); }; @@ -974,8 +974,8 @@ void TKeyTuple::Print(IOutputStream& output) const { for (size_t i = 0; i < Size(); ++i) { auto value = GetValue(i); if (value) { - YQL_ENSURE(Ctx); - printExpr(*Ctx, value.Cast(), output); + YQL_ENSURE(Ctx); + printExpr(*Ctx, value.Cast(), output); } else { output << (IsFrom() ? IsInclusive() ? "-Inf" : "+Inf" diff --git a/ydb/library/yql/providers/common/provider/yql_table_lookup.h b/ydb/library/yql/providers/common/provider/yql_table_lookup.h index d88e736727..988fb5b4ba 100644 --- a/ydb/library/yql/providers/common/provider/yql_table_lookup.h +++ b/ydb/library/yql/providers/common/provider/yql_table_lookup.h @@ -97,14 +97,14 @@ private: class TKeyTuple { public: - TKeyTuple(TExprContext* ctx, const TVector<NNodes::TMaybeNode<NNodes::TExprBase>>& values, bool isFrom, bool isInclusive) - : Ctx(ctx) - , Values(values) + TKeyTuple(TExprContext* ctx, const TVector<NNodes::TMaybeNode<NNodes::TExprBase>>& values, bool isFrom, bool isInclusive) + : Ctx(ctx) + , Values(values) , From(isFrom) , Inclusive(isInclusive) {} TKeyTuple() - : TKeyTuple(nullptr, {}, false, false) {} + : TKeyTuple(nullptr, {}, false, false) {} size_t Size() const { return Values.size(); } bool IsFrom() const { return From; } @@ -114,7 +114,7 @@ public: void Print(IOutputStream& output) const; private: - TExprContext* Ctx; + TExprContext* Ctx; TVector<NNodes::TMaybeNode<NNodes::TExprBase>> Values; bool From; bool Inclusive; @@ -122,10 +122,10 @@ private: class TKeyRange { public: - TKeyRange(TExprContext& ctx, const TVector<TColumnRange>& columnRanges, NNodes::TMaybeNode<NNodes::TExprBase> residualPredicate); + TKeyRange(TExprContext& ctx, const TVector<TColumnRange>& columnRanges, NNodes::TMaybeNode<NNodes::TExprBase> residualPredicate); - TKeyRange(TExprContext& ctx, size_t keySize, NNodes::TMaybeNode<NNodes::TExprBase> residualPredicate) - : TKeyRange(ctx, TVector<TColumnRange>(keySize, TColumnRange()), residualPredicate) {} + TKeyRange(TExprContext& ctx, size_t keySize, NNodes::TMaybeNode<NNodes::TExprBase> residualPredicate) + : TKeyRange(ctx, TVector<TColumnRange>(keySize, TColumnRange()), residualPredicate) {} size_t GetColumnRangesCount() const { return ColumnRanges.size(); @@ -170,7 +170,7 @@ public: void Print(IOutputStream& output) const; private: - TExprContext* Ctx; + TExprContext* Ctx; TVector<TColumnRange> ColumnRanges; NNodes::TMaybeNode<NNodes::TExprBase> ResidualPredicate; bool EquiRange; diff --git a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp index a98ea9bf6b..0bbaf5c339 100644 --- a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp +++ b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp @@ -42,45 +42,45 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> { } }; - struct TMappingOrderedStructAdaptor { - TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members; - - TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TVector<TString>>& columns, const TStructExprType* type) - { - TMap<TStringBuf, const TTypeAnnotationNode*> members; - for (auto& item: type->GetItems()) { - TMaybe<TStringBuf> name = mapper ? mapper(item->GetName()) : item->GetName(); - if (!name) { - continue; - } - members[*name] = item->GetItemType(); - } - - if (columns) { - for (auto& column : *columns) { - auto it = members.find(column); - if (it != members.end()) { - Members.push_back(*it); - } - } - } else { - Members.insert(Members.end(), members.begin(), members.end()); - } - } - - ui32 GetMembersCount() const { - return Members.size(); - } - - const TStringBuf& GetMemberName(ui32 idx) const { - return Members[idx].first; - } - - const TTypeAnnotationNode* GetMemberType(ui32 idx) const { - return Members[idx].second; - } - }; - + struct TMappingOrderedStructAdaptor { + TVector<std::pair<TStringBuf, const TTypeAnnotationNode*>> Members; + + TMappingOrderedStructAdaptor(const TStructMemberMapper& mapper, const TMaybe<TVector<TString>>& columns, const TStructExprType* type) + { + TMap<TStringBuf, const TTypeAnnotationNode*> members; + for (auto& item: type->GetItems()) { + TMaybe<TStringBuf> name = mapper ? mapper(item->GetName()) : item->GetName(); + if (!name) { + continue; + } + members[*name] = item->GetItemType(); + } + + if (columns) { + for (auto& column : *columns) { + auto it = members.find(column); + if (it != members.end()) { + Members.push_back(*it); + } + } + } else { + Members.insert(Members.end(), members.begin(), members.end()); + } + } + + ui32 GetMembersCount() const { + return Members.size(); + } + + const TStringBuf& GetMemberName(ui32 idx) const { + return Members[idx].first; + } + + const TTypeAnnotationNode* GetMemberType(ui32 idx) const { + return Members[idx].second; + } + }; + struct TTupleAdaptor { const TTupleExprType* Type; @@ -224,9 +224,9 @@ public: } } - void SaveStructType(const TStructExprType* type, const TMaybe<TVector<TString>>& columns, const TStructMemberMapper& mapper) { - if (mapper || columns) { - TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type)); + void SaveStructType(const TStructExprType* type, const TMaybe<TVector<TString>>& columns, const TStructMemberMapper& mapper) { + if (mapper || columns) { + TBase::SaveStructType(TMappingOrderedStructAdaptor(mapper, columns, type)); } else { Save(type); } @@ -235,9 +235,9 @@ public: void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TVector<TString>>& columns, const TStructMemberMapper& mapper) { TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer); - saver.SaveStructType(type, columns, mapper); -} - + saver.SaveStructType(type, columns, mapper); +} + void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type) { TExprTypeSaver<TYqlTypeYsonSaverImpl> saver(writer); saver.Save(type); @@ -291,7 +291,7 @@ struct TExprTypeLoader { } TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& paramTwo, ui32 /*level*/) { auto ret = Ctx.MakeType<TDataExprParamsType>(NYql::NUdf::GetDataSlot(dataType), paramOne, paramTwo); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } TMaybe<TType> LoadResourceType(const TString& tag, ui32 /*level*/) { @@ -311,7 +311,7 @@ struct TExprTypeLoader { items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second)); } auto ret = Ctx.MakeType<TStructExprType>(items); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) { @@ -325,12 +325,12 @@ struct TExprTypeLoader { } TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) { auto ret = Ctx.MakeType<TTupleExprType>(elements); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) { auto ret = Ctx.MakeType<TDictExprType>(keyType, valType); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } TMaybe<TType> LoadCallableType(TType returnType, const TVector<TType>& argTypes, const TVector<TString>& argNames, @@ -344,12 +344,12 @@ struct TExprTypeLoader { args.back().Flags = argFlags[i]; } auto ret = Ctx.MakeType<TCallableExprType>(returnType, args, optionalCount, Ctx.AppendString(payload)); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) { auto ret = Ctx.MakeType<TVariantExprType>(underlyingType); - YQL_ENSURE(ret->Validate(TPosition(), Ctx)); + YQL_ENSURE(ret->Validate(TPosition(), Ctx)); return ret; } void Error(const TString& info) { @@ -368,49 +368,49 @@ const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext return ParseTypeFromYson(node, ctx, pos); } -const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos) { - NYT::TNode node; - TStringStream err; - if (!ParseYson(node, yson, err)) { - ctx.AddError(TIssue(pos, err.Str())); - return nullptr; - } - - return ParseOrderAwareTypeFromYson(node, topLevelColumns, ctx, pos); -} - +const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos) { + NYT::TNode node; + TStringStream err; + if (!ParseYson(node, yson, err)) { + ctx.AddError(TIssue(pos, err.Str())); + return nullptr; + } + + return ParseOrderAwareTypeFromYson(node, topLevelColumns, ctx, pos); +} + const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos) { TExprTypeLoader loader(ctx, pos); return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr); } -struct TOrderAwareExprTypeLoader: public TExprTypeLoader { - typedef const TTypeAnnotationNode* TType; - TVector<TString>& TopLevelColumns; - - TOrderAwareExprTypeLoader(TExprContext& ctx, const TPosition& pos, TVector<TString>& topLevelColumns) - : TExprTypeLoader(ctx, pos) - , TopLevelColumns(topLevelColumns) - { - TopLevelColumns.clear(); - } - - TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) { - if (level == 0) { - YQL_ENSURE(TopLevelColumns.empty()); - for (auto& [column, type] : members) { - TopLevelColumns.push_back(column); - } - } - return TExprTypeLoader::LoadStructType(members, level); - } -}; - -const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos) { - TOrderAwareExprTypeLoader loader(ctx, pos, topLevelColumns); - return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr); -} - +struct TOrderAwareExprTypeLoader: public TExprTypeLoader { + typedef const TTypeAnnotationNode* TType; + TVector<TString>& TopLevelColumns; + + TOrderAwareExprTypeLoader(TExprContext& ctx, const TPosition& pos, TVector<TString>& topLevelColumns) + : TExprTypeLoader(ctx, pos) + , TopLevelColumns(topLevelColumns) + { + TopLevelColumns.clear(); + } + + TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) { + if (level == 0) { + YQL_ENSURE(TopLevelColumns.empty()); + for (auto& [column, type] : members) { + TopLevelColumns.push_back(column); + } + } + return TExprTypeLoader::LoadStructType(members, level); + } +}; + +const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos) { + TOrderAwareExprTypeLoader loader(ctx, pos, topLevelColumns); + return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr); +} + void WriteResOrPullType(NYson::TYsonConsumerBase& writer,const TTypeAnnotationNode* type, const TVector<TString>& columns) { if (columns.empty() || type->GetKind() != ETypeAnnotationKind::List || @@ -422,7 +422,7 @@ void WriteResOrPullType(NYson::TYsonConsumerBase& writer,const TTypeAnnotationNo writer.OnStringScalar("ListType"); writer.OnListItem(); - SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns); + SaveStructTypeToYson(writer, type->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(), columns); writer.OnEndList(); } diff --git a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h index be86f8be56..397063052f 100644 --- a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h +++ b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h @@ -20,21 +20,21 @@ struct TExprContext; namespace NCommon { -// empty return value means "remove member" -using TStructMemberMapper = std::function<TMaybe<TStringBuf> (TStringBuf member)>; - +// empty return value means "remove member" +using TStructMemberMapper = std::function<TMaybe<TStringBuf> (TStringBuf member)>; + void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type); - -// saves in columns order + +// saves in columns order void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprType* type, const TMaybe<TVector<TString>>& columns = {}, const TStructMemberMapper& mapper = {}); - + NYT::TNode TypeToYsonNode(const TTypeAnnotationNode* type); TString WriteTypeToYson(const TTypeAnnotationNode* type, NYT::NYson::EYsonFormat format = NYT::NYson::EYsonFormat::Binary); const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext& ctx, const TPosition& pos = TPosition()); -const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition()); +const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition()); const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos = TPosition()); -const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition()); +const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition()); void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type, const TVector<TString>& columns); diff --git a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp index 298b8f0e87..ae3f281ed2 100644 --- a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp +++ b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp @@ -32,8 +32,8 @@ void TYqlTypeYsonSaverBase::SaveDataType(const TStringBuf& dataType) { Writer.OnListItem(); Writer.OnStringScalar(dataType); Writer.OnEndList(); -} - +} + void TYqlTypeYsonSaverBase::SaveDataTypeParams(const TStringBuf& dataType, const TStringBuf& paramOne, const TStringBuf& paramTwo) { SaveTypeHeader("DataType"); Writer.OnListItem(); diff --git a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h index 4ed2e9a3b1..a3116efb14 100644 --- a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h +++ b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h @@ -42,18 +42,18 @@ protected: template <typename TDerived> class TYqlTypeYsonSaverImpl: public TYqlTypeYsonSaverBase { typedef TYqlTypeYsonSaverImpl<TDerived> TSelf; - + public: TYqlTypeYsonSaverImpl(TConsumer& writer) : TYqlTypeYsonSaverBase(writer) { } - + template <typename TType> void Save(TType* type) { static_cast<TDerived*>(this)->Save(type); } - + protected: template <typename TTaggedType> void SaveTaggedType(const TTaggedType& taggedType) { diff --git a/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp b/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp index 30978eb368..c8372260ac 100644 --- a/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp +++ b/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp @@ -5,30 +5,30 @@ namespace NYql { namespace NCommon { -TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node) { - if (!node.IsList() || node.Size() < 1 || !node[0].IsString()) { - YQL_ENSURE(!"Invalid type scheme"); - } - - auto typeName = node[0].AsString(); - YQL_ENSURE(typeName == "StructType"); - - if (node.Size() != 2 || !node[1].IsList()) { - YQL_ENSURE(!"Invalid struct type scheme"); - } - - TVector<TString> columns; - for (auto& member : node[1].AsList()) { - if (!member.IsList() || member.Size() != 2 || !member[0].IsString()) { - YQL_ENSURE(!"Invalid struct type scheme"); - } - - columns.push_back(member[0].AsString()); - } - - return columns; -} - +TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node) { + if (!node.IsList() || node.Size() < 1 || !node[0].IsString()) { + YQL_ENSURE(!"Invalid type scheme"); + } + + auto typeName = node[0].AsString(); + YQL_ENSURE(typeName == "StructType"); + + if (node.Size() != 2 || !node[1].IsList()) { + YQL_ENSURE(!"Invalid struct type scheme"); + } + + TVector<TString> columns; + for (auto& member : node[1].AsList()) { + if (!member.IsList() || member.Size() != 2 || !member[0].IsString()) { + YQL_ENSURE(!"Invalid struct type scheme"); + } + + columns.push_back(member[0].AsString()); + } + + return columns; +} + bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right) { auto typeName = left[0].AsString(); if (typeName != right[0].AsString()) { diff --git a/ydb/library/yql/providers/common/schema/yql_schema_utils.h b/ydb/library/yql/providers/common/schema/yql_schema_utils.h index 53609e6dff..45061285a8 100644 --- a/ydb/library/yql/providers/common/schema/yql_schema_utils.h +++ b/ydb/library/yql/providers/common/schema/yql_schema_utils.h @@ -8,7 +8,7 @@ namespace NYql { namespace NCommon { -TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node); +TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node); bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right); } // namespace NCommon diff --git a/ydb/library/yql/providers/common/transform/yql_exec.cpp b/ydb/library/yql/providers/common/transform/yql_exec.cpp index a978751700..f771ae2b18 100644 --- a/ydb/library/yql/providers/common/transform/yql_exec.cpp +++ b/ydb/library/yql/providers/common/transform/yql_exec.cpp @@ -27,7 +27,7 @@ TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::CallbackTransfor return (handlerInfo->Handler)(input, output, ctx); } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Don't know how to execute node: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Don't know how to execute node: " << input->Content())); return SyncError(); } @@ -47,7 +47,7 @@ TExecTransformerBase::THandler TExecTransformerBase::Pass() { } TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::ExecPass(const TExprNode::TPtr& input, TExprContext& ctx) { - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return SyncOk(); } diff --git a/ydb/library/yql/providers/common/transform/yql_visit.cpp b/ydb/library/yql/providers/common/transform/yql_visit.cpp index a8459b433b..c6ec8a4a13 100644 --- a/ydb/library/yql/providers/common/transform/yql_visit.cpp +++ b/ydb/library/yql/providers/common/transform/yql_visit.cpp @@ -20,7 +20,7 @@ IGraphTransformer::TStatus TVisitorTransformerBase::DoTransform(TExprNode::TPtr return (*handler)(input, output, ctx); } if (FailOnUnknown) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported callable: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported callable: " << input->Content())); return TStatus::Error; } return TStatus::Ok; diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index a89b1822f3..6952e96cf0 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -75,15 +75,15 @@ namespace { writer.OnEndMap(); input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); return TStatus::Ok; } else { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported tag: " << tag)); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported tag: " << tag)); return TStatus::Error; } } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unknown node to pull, type: " + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unknown node to pull, type: " << nodeToPull->Type() << ", content: " << nodeToPull->Content())); return TStatus::Error; } @@ -94,7 +94,7 @@ namespace { return requireStatus; } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return TStatus::Ok; } @@ -105,12 +105,12 @@ namespace { return requireStatus; } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Failed to execute node: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Failed to execute node: " << input->Content())); return TStatus::Error; } @@ -128,10 +128,10 @@ namespace { } }; - TConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, const TAllowSettingPolicy& policy) + TConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, const TAllowSettingPolicy& policy) : Types(types) , CoreConfig(config && config->HasYqlCore() ? &config->GetYqlCore() : nullptr) - , Policy(policy) + , Policy(policy) {} TStringBuf GetName() const override { @@ -265,23 +265,23 @@ namespace { auto key = input->Child(2); if (!key->IsCallable("Key")) { - ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Expected key")); + ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Expected key")); return IGraphTransformer::TStatus::Error; } if (key->ChildrenSize() == 0) { - ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Empty key is not allowed")); + ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Empty key is not allowed")); return IGraphTransformer::TStatus::Error; } auto tag = key->Child(0)->Child(0)->Content(); if (key->Child(0)->ChildrenSize() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(key->Child(0)->Pos()), "Only tag must be specified")); + ctx.AddError(TIssue(ctx.GetPosition(key->Child(0)->Pos()), "Only tag must be specified")); return IGraphTransformer::TStatus::Error; } if (key->ChildrenSize() > 1) { - ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Too many tags")); + ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), "Too many tags")); return IGraphTransformer::TStatus::Error; } @@ -291,12 +291,12 @@ namespace { } if (fields->ChildrenSize() != 0) { - ctx.AddError(TIssue(ctx.GetPosition(fields->Pos()), "Fields tuple must be empty")); + ctx.AddError(TIssue(ctx.GetPosition(fields->Pos()), "Fields tuple must be empty")); return IGraphTransformer::TStatus::Error; } if (!input->Child(3)->GetTypeAnn() || !input->Child(3)->IsComposable()) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(3)->Pos()), "Expected composable data")); + ctx.AddError(TIssue(ctx.GetPosition(input->Child(3)->Pos()), "Expected composable data")); return IGraphTransformer::TStatus::Error; } @@ -306,7 +306,7 @@ namespace { } if (settings->ChildrenSize() != 0) { - ctx.AddError(TIssue(ctx.GetPosition(settings->Pos()), "Unsupported settings")); + ctx.AddError(TIssue(ctx.GetPosition(settings->Pos()), "Unsupported settings")); return IGraphTransformer::TStatus::Error; } @@ -317,7 +317,7 @@ namespace { if (tag == "data_sources" || tag == "data_sinks") { children.push_back(listOfString); } else { - ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), TStringBuilder() << "Unknown tag: " << tag)); + ctx.AddError(TIssue(ctx.GetPosition(key->Pos()), TStringBuilder() << "Unknown tag: " << tag)); return IGraphTransformer::TStatus::Error; } @@ -334,7 +334,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Config) Unsupported function: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Config) Unsupported function: " << input->Content())); return IGraphTransformer::TStatus::Error; }); } @@ -412,19 +412,19 @@ namespace { } private: - bool IsSettingAllowed(const TPosition& pos, TStringBuf name, TExprContext& ctx) { - if (Policy && !Policy(name)) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Changing setting " << name << " is not allowed")); - return false; - } - return true; - } - + bool IsSettingAllowed(const TPosition& pos, TStringBuf name, TExprContext& ctx) { + if (Policy && !Policy(name)) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Changing setting " << name << " is not allowed")); + return false; + } + return true; + } + bool ApplyFlag(const TPosition& pos, const TStringBuf name, const TVector<TStringBuf>& args, TExprContext& ctx) { - if (!IsSettingAllowed(pos, name, ctx)) { - return false; - } - + if (!IsSettingAllowed(pos, name, ctx)) { + return false; + } + if (name == "UnsecureCredential") { if (!AddCredential(pos, args, ctx)) { return false; @@ -727,14 +727,14 @@ namespace { Types.JsonQueryReturnsJsonDocument = (name == "DisableJsonQueryReturnsJsonDocument"); } - else if (name == "OrderedColumns" || name == "DisableOrderedColumns") { - if (args.size() != 0) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size())); - return false; - } - - Types.OrderedColumns = (name == "OrderedColumns"); - } + else if (name == "OrderedColumns" || name == "DisableOrderedColumns") { + if (args.size() != 0) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size())); + return false; + } + + Types.OrderedColumns = (name == "OrderedColumns"); + } else if (name == "FolderSubDirsLimit") { if (args.size() != 1) { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size())); @@ -855,11 +855,11 @@ namespace { } bool DoListSandboxFolder(const TStringBuf url, const TPosition& pos, TExprContext& ctx, NJson::TJsonValue& content) { - TString urlStr(url); - if (!url.empty() && url.back() != '/') { - urlStr += "/"; - } - const THttpURL& httpUrl = ParseURL(urlStr); + TString urlStr(url); + if (!url.empty() && url.back() != '/') { + urlStr += "/"; + } + const THttpURL& httpUrl = ParseURL(urlStr); if (httpUrl.GetHost() != "proxy.sandbox.yandex-team.ru") { ctx.AddError(TIssue(pos, TStringBuilder() << "Adding folder by URL is currently supported only for proxy.sandbox.yandex-team.ru. Host " << httpUrl.GetHost() << " is not supported")); return false; @@ -994,14 +994,14 @@ namespace { TAutoPtr<IGraphTransformer> ConfigurationTransformer; TAutoPtr<IGraphTransformer> CallableExecutionTransformer; const TYqlCoreConfig* CoreConfig; - const TAllowSettingPolicy Policy; + const TAllowSettingPolicy Policy; }; } -TIntrusivePtr<IDataProvider> CreateConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, - const TAllowSettingPolicy& policy) -{ - return new TConfigProvider(types, config, policy); +TIntrusivePtr<IDataProvider> CreateConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, + const TAllowSettingPolicy& policy) +{ + return new TConfigProvider(types, config, policy); } const THashSet<TStringBuf>& ConfigProviderFunctions() { diff --git a/ydb/library/yql/providers/config/yql_config_provider.h b/ydb/library/yql/providers/config/yql_config_provider.h index 44d570f27f..ea1f47508a 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.h +++ b/ydb/library/yql/providers/config/yql_config_provider.h @@ -2,19 +2,19 @@ #include <ydb/library/yql/core/yql_type_annotation.h> -#include <functional> - +#include <functional> + namespace NYql { class TGatewaysConfig; const TStringBuf ConfReadName = "ConfRead!"; -using TAllowSettingPolicy = std::function<bool(TStringBuf settingName)>; +using TAllowSettingPolicy = std::function<bool(TStringBuf settingName)>; + +TIntrusivePtr<IDataProvider> CreateConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, + const TAllowSettingPolicy& policy = TAllowSettingPolicy()); // allow all settings by default -TIntrusivePtr<IDataProvider> CreateConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, - const TAllowSettingPolicy& policy = TAllowSettingPolicy()); // allow all settings by default - const THashSet<TStringBuf>& ConfigProviderFunctions(); } // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp index f0a0951d4f..6acfb1277e 100644 --- a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp +++ b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp @@ -302,7 +302,7 @@ private: THashMap<ui64, TRequestInfo> RequestedNodes; ui32 RequestedCount; - ui64 ResourceId = 0; + ui64 ResourceId = 0; bool LocalMode = false; TVector<NDqProto::TWorkerGroup> AllocatedWorkers; diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp index 098b1cab42..f4503a60b2 100644 --- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp @@ -187,21 +187,21 @@ private: } else if (line.Contains("Container killed by OOM")) { // temporary workaround for YQL-12066 *fallbackFlag = true; - } else if (line.Contains("Expected data or optional of data, actual:")) { + } else if (line.Contains("Expected data or optional of data, actual:")) { // temporary workaround for YQL-12835 *fallbackFlag = true; - } else if (line.Contains("Cannot create Skiff writer for ")) { - // temporary workaround for YQL-12986 - *fallbackFlag = true; + } else if (line.Contains("Cannot create Skiff writer for ")) { + // temporary workaround for YQL-12986 + *fallbackFlag = true; } else if (line.Contains("Skiff format expected")) { // temporary workaround for YQL-12986 *fallbackFlag = true; - } else if (line.Contains("Pattern nodes can not get computation node by index:")) { - // temporary workaround for YQL-12987 - *fallbackFlag = true; - } else if (line.Contains("contrib/libs/protobuf/src/google/protobuf/messagext.cc") && line.Contains("Message size") && line.Contains("exceeds")) { - // temporary workaround for YQL-12988 - *fallbackFlag = true; + } else if (line.Contains("Pattern nodes can not get computation node by index:")) { + // temporary workaround for YQL-12987 + *fallbackFlag = true; + } else if (line.Contains("contrib/libs/protobuf/src/google/protobuf/messagext.cc") && line.Contains("Message size") && line.Contains("exceeds")) { + // temporary workaround for YQL-12988 + *fallbackFlag = true; } else if (line.Contains("Cannot start container")) { // temporary workaround for YQL-14221 *retriableFlag = true; @@ -799,7 +799,7 @@ private: TActorId TaskRunnerActor; NDqProto::TDqTask Task; - ui64 StageId = 0; + ui64 StageId = 0; bool TaskRunnerPrepared = false; THashMap<NActors::TActorId, TInputChannel> InputMap; diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h index 6ff980444d..3c37e4d48f 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h @@ -27,7 +27,7 @@ struct TDqSettings { static constexpr ui32 CloudFunctionConcurrency = 10; }; - using TPtr = std::shared_ptr<TDqSettings>; + using TPtr = std::shared_ptr<TDqSettings>; NCommon::TConfSetting<ui64, false> DataSizePerJob; NCommon::TConfSetting<ui64, false> MaxDataSizePerJob; @@ -109,7 +109,7 @@ struct TDqSettings { } TDqSettings::TPtr WithFillSettings(const IDataProvider::TFillSettings& fillSettings) const { - auto copy = std::make_shared<TDqSettings>(*this); + auto copy = std::make_shared<TDqSettings>(*this); if (fillSettings.RowsLimitPerWrite) { copy->_RowsLimitPerWrite = *fillSettings.RowsLimitPerWrite; } diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp index e5178fceba..9e87f81e7c 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp @@ -46,7 +46,7 @@ public: , Config(config) { #define HNDL(name) "DqsLogical-"#name, Hndl(&TDqsLogicalOptProposalTransformer::name) - AddHandler(0, &TCoUnorderedBase::Match, HNDL(SkipUnordered)); + AddHandler(0, &TCoUnorderedBase::Match, HNDL(SkipUnordered)); AddHandler(0, &TCoAggregate::Match, HNDL(RewriteAggregate)); AddHandler(0, &TCoTake::Match, HNDL(RewriteTakeSortToTopSort)); AddHandler(0, &TCoEquiJoin::Match, HNDL(RewriteEquiJoin)); @@ -61,7 +61,7 @@ public: protected: TMaybeNode<TExprBase> SkipUnordered(TExprBase node, TExprContext& ctx) { Y_UNUSED(ctx); - auto unordered = node.Cast<TCoUnorderedBase>(); + auto unordered = node.Cast<TCoUnorderedBase>(); if (unordered.Input().Maybe<TDqConnection>()) { return unordered.Input(); } diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 4e4c462569..8759f44221 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -458,7 +458,7 @@ private: } i64 dataLimit = static_cast<i64>(4_GB); - bool fallbackFlag = false; + bool fallbackFlag = false; if (sizeSum > dataLimit) { YQL_LOG(INFO) << "Too much data: " << sizeSum << " > " << dataLimit; fallbackFlag = true; @@ -922,7 +922,7 @@ private: executionPlanner->SetPublicIds(stage2publicId); - auto settings = std::make_shared<TDqSettings>(*State->Settings); + auto settings = std::make_shared<TDqSettings>(*State->Settings); auto tasksPerStage = settings->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage); const auto maxTasksPerOperation = State->Settings->MaxTasksPerOperation.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerOperation); diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp index 1a6e6a59a7..2341b28ad4 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp @@ -347,7 +347,7 @@ public: if (status.Ok()) { TGuard<TMutex> lock(ProgressMutex); TString stage; - TDqProgressWriter* dqProgressWriter = nullptr; + TDqProgressWriter* dqProgressWriter = nullptr; auto it = RunningQueries.find(sessionId); if (it != RunningQueries.end()) { dqProgressWriter = &it->second.first; diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp index ba54d0f960..cdcbd32ef5 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp @@ -255,7 +255,7 @@ protected: THolder<TPipedInput> Stdout; THolder<TPipedInput> Stderr; - int Pid = -1; + int Pid = -1; virtual void Exec() { char ** args; @@ -1465,7 +1465,7 @@ private: NKikimr::NMiniKQL::THolderFactory HolderFactory; std::atomic<bool> Running; - int Code = -1; + int Code = -1; TString Stderr; THolder<TChildProcess> Command; THolder<TThread> StderrReader; diff --git a/ydb/library/yql/providers/pq/cm_client/interface/client.h b/ydb/library/yql/providers/pq/cm_client/interface/client.h index e1e2449bf9..368d2f2039 100644 --- a/ydb/library/yql/providers/pq/cm_client/interface/client.h +++ b/ydb/library/yql/providers/pq/cm_client/interface/client.h @@ -184,7 +184,7 @@ struct TClientOptions { OPTION(TString, Endpoint, ); OPTION(std::shared_ptr<NYdb::ICredentialsProviderFactory>, CredentialsProviderFactory, ); OPTION(TDuration, RequestTimeout, = TDuration::Seconds(10)); - OPTION(bool, EnableSsl, = false); + OPTION(bool, EnableSsl, = false); }; #undef OPTION diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h index e0382656ad..29f05d1e95 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h @@ -17,7 +17,7 @@ class IFunctionRegistry; namespace NYql { class TPqGatewayConfig; -using TPqGatewayConfigPtr = std::shared_ptr<TPqGatewayConfig>; +using TPqGatewayConfigPtr = std::shared_ptr<TPqGatewayConfig>; struct TPqGatewayServices { const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp index 0b5c7f2270..278db980f4 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp @@ -11,7 +11,7 @@ TPqConfiguration::TPqConfiguration() { } TPqSettings::TConstPtr TPqConfiguration::Snapshot() const { - return std::make_shared<const TPqSettings>(*this); + return std::make_shared<const TPqSettings>(*this); } void TPqConfiguration::Init( diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h index 2d7075f33f..5ed40d07d9 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h @@ -13,7 +13,7 @@ namespace NYql { struct TPqSettings { - using TConstPtr = std::shared_ptr<const TPqSettings>; + using TConstPtr = std::shared_ptr<const TPqSettings>; NCommon::TConfSetting<TString, false> Consumer; NCommon::TConfSetting<TString, false> Database; // It is needed in case of Cloud.LB for external users, but can be taken from config for internal LB. diff --git a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h index 1799b2757c..a9770e379b 100644 --- a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h +++ b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h @@ -45,7 +45,7 @@ public: typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType; typedef TResultDataSink ResultType; - TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) + TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) , BuildFunc(buildFunc) {} diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp index 260614b5fe..13bfd5e0f7 100644 --- a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp +++ b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp @@ -102,7 +102,7 @@ namespace { } if (!child->IsAtom() && !child->IsList()) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), "either atom or tuple with prefix is expected")); + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), "either atom or tuple with prefix is expected")); return IGraphTransformer::TStatus::Error; } @@ -120,7 +120,7 @@ namespace { } if (child->Child(0)->Content() != "prefix") { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Expected 'prefix', but got: " << child->Child(0)->Content())); return IGraphTransformer::TStatus::Error; } @@ -134,13 +134,13 @@ namespace { } if (listType->GetKind() != ETypeAnnotationKind::List) { - ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), "columns requires list of struct")); + ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), "columns requires list of struct")); return IGraphTransformer::TStatus::Error; } auto itemType = listType->Cast<TListExprType>()->GetItemType(); if (itemType->GetKind() != ETypeAnnotationKind::Struct) { - ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), "columns requires list of struct")); + ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), "columns requires list of struct")); return IGraphTransformer::TStatus::Error; } @@ -152,13 +152,13 @@ namespace { if (child->IsAtom()) { orderedFields.push_back(child); if (!structType->FindItem(child->Content())) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Unknown field in hint: " << child->Content())); return IGraphTransformer::TStatus::Error; } if (!usedFields.insert(TString(child->Content())).second) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Duplicate field in hint: " << child->Content())); return IGraphTransformer::TStatus::Error; } @@ -168,7 +168,7 @@ namespace { if (x->GetName().StartsWith(prefix)) { orderedFields.push_back(ctx.NewAtom(child->Pos(), x->GetName())); if (!usedFields.insert(TString(x->GetName())).second) { - ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(child->Pos()), TStringBuilder() << "Duplicate field in hint: " << x->GetName())); return IGraphTransformer::TStatus::Error; } @@ -178,7 +178,7 @@ namespace { } if (usedFields.size() != structType->GetSize()) { - ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(columns->Pos()), TStringBuilder() << "Mismatch of fields in hint and in the struct, columns fields: " << usedFields.size() << ", struct fields:" << structType->GetSize())); return IGraphTransformer::TStatus::Error; @@ -220,13 +220,13 @@ namespace { if (input->Content() == CommitName) { if (ResultWriter) { - TExprBase commitChild(input->ChildPtr(0)); - - bool overflow = commitChild.Maybe<TResPull>() ? PullOverflow : FillOverflow; - ui64& committedSize = commitChild.Maybe<TResPull>() ? CommittedPullSize : CommittedFillSize; - + TExprBase commitChild(input->ChildPtr(0)); + + bool overflow = commitChild.Maybe<TResPull>() ? PullOverflow : FillOverflow; + ui64& committedSize = commitChild.Maybe<TResPull>() ? CommittedPullSize : CommittedFillSize; + if (!ResultWriter->IsDiscard()) { - ResultWriter->Commit(overflow); + ResultWriter->Commit(overflow); Config->CommittedResults.push_back(TString(ResultWriter->Str())); committedSize += Config->CommittedResults.back().size(); } @@ -234,18 +234,18 @@ namespace { ResultWriter.Reset(); } - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld (input->Pos())); return TStatus::Ok; } if (input->Content() == ConfigureName) { - input->SetState(TExprNode::EState::ExecutionComplete); + input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); return TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Failed to execute node: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Failed to execute node: " << input->Content())); return TStatus::Ok; } @@ -269,9 +269,9 @@ namespace { private: template <class TTarget> - bool& GetOverflowFlagAndCommitedSize(ui64& committed); - - template <class TTarget> + bool& GetOverflowFlagAndCommitedSize(ui64& committed); + + template <class TTarget> TStatus HandleFillOrPull(TExprBase input, TExprNode::TPtr& output, TExprContext& ctx, IDataProvider& provider) { auto requireWorld = RequireChild(input.Ref(), TResBase::idx_World); auto requireData = input.Maybe<TResPull>() ? RequireChild(input.Ref(), TResPull::idx_Data) : IGraphTransformer::TStatus::Ok; @@ -298,7 +298,7 @@ namespace { } input.Ptr()->SetResult(ctx.NewWorld(input.Pos())); - input.Ptr()->SetState(TExprNode::EState::ExecutionComplete); + input.Ptr()->SetState(TExprNode::EState::ExecutionComplete); return TStatus::Ok; } @@ -315,11 +315,11 @@ namespace { if ((current > total) || (total && current >= total)) { auto zero = ctx.NewAtom(TPositionHandle(), "0", TNodeFlags::Default); zero->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - zero->SetState(TExprNode::EState::ConstrComplete); + zero->SetState(TExprNode::EState::ConstrComplete); zero->SetDependencyScope(nullptr, nullptr); // HOTFIX for CSEE input.Ptr()->ChildRef(TResFor::idx_Current) = std::move(zero); //FIXME: Don't use ChilfRef input.Ptr()->SetResult(ctx.NewWorld(input.Pos())); - input.Ptr()->SetState(TExprNode::EState::ExecutionComplete); + input.Ptr()->SetState(TExprNode::EState::ExecutionComplete); return TStatus::Ok; } @@ -407,7 +407,7 @@ namespace { TProgramBuilder pgmBuilder(env, Config->FunctionRegistry); TType* mkqlType = NCommon::BuildType(*itemsNode.GetTypeAnn(), pgmBuilder, err); if (!mkqlType) { - ctx.AddError(TIssue(ctx.GetPosition(itemsNode.Pos()), TStringBuilder() << "Failed to process type: " << err.Str())); + ctx.AddError(TIssue(ctx.GetPosition(itemsNode.Pos()), TStringBuilder() << "Failed to process type: " << err.Str())); return TStatus::Error; } @@ -415,7 +415,7 @@ namespace { THolderFactory holderFactory(alloc.Ref(), memInfo); auto value = NCommon::ParseYsonNodeInResultFormat(holderFactory, resultDataNode, mkqlType, &err); if (!value) { - ctx.AddError(TIssue(ctx.GetPosition(itemsNode.Pos()), TStringBuilder() << "Failed to parse data: " << err.Str())); + ctx.AddError(TIssue(ctx.GetPosition(itemsNode.Pos()), TStringBuilder() << "Failed to parse data: " << err.Str())); return TStatus::Error; } @@ -466,16 +466,16 @@ namespace { auto fillSettings = Config->FillSettings; auto resultSize = ResultWriter ? ResultWriter->Size() : 0; - ui64 committedSize; - bool& overflow = GetOverflowFlagAndCommitedSize<TTarget>(committedSize); - - if (fillSettings.AllResultsBytesLimit && committedSize + resultSize >= *fillSettings.AllResultsBytesLimit) { - overflow = true; + ui64 committedSize; + bool& overflow = GetOverflowFlagAndCommitedSize<TTarget>(committedSize); + + if (fillSettings.AllResultsBytesLimit && committedSize + resultSize >= *fillSettings.AllResultsBytesLimit) { + overflow = true; } if (fillSettings.AllResultsBytesLimit) { - if (!overflow && committedSize <= *fillSettings.AllResultsBytesLimit) { - *fillSettings.AllResultsBytesLimit -= committedSize; + if (!overflow && committedSize <= *fillSettings.AllResultsBytesLimit) { + *fillSettings.AllResultsBytesLimit -= committedSize; } else { *fillSettings.AllResultsBytesLimit = 0; } @@ -509,7 +509,7 @@ namespace { YQL_ENSURE(Config->WriterFactory); ResultWriter = Config->WriterFactory(); ResultWriter->Init(discard, label, Config->SupportsResultPosition ? - TMaybe<TPosition>(ctx.GetPosition(input.Pos())) : Nothing()); + TMaybe<TPosition>(ctx.GetPosition(input.Pos())) : Nothing()); } if (input.Maybe<TResIf>() || input.Maybe<TResFor>()) { @@ -549,12 +549,12 @@ namespace { for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails, TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) { DelegatedNode->Child(idx)->SetTypeAnn(atomType); - DelegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); + DelegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); } DelegatedNode->SetTypeAnn(input.Ref().GetTypeAnn()); - DelegatedNode->SetState(TExprNode::EState::ConstrComplete); - input.Ptr()->SetState(TExprNode::EState::ExecutionInProgress); + DelegatedNode->SetState(TExprNode::EState::ConstrComplete); + input.Ptr()->SetState(TExprNode::EState::ExecutionInProgress); auto status = DelegatedProvider->GetCallableExecutionTransformer().Transform(DelegatedNode, DelegatedNodeOutput, ctx); if (status.Level != TStatus::Async) { FinishNode(*input.Ptr(), ctx, status); @@ -572,17 +572,17 @@ namespace { ResultWriter->Write(data); input.SetResult(ctx.NewAtom(input.Pos(), "")); - input.SetState(TExprNode::EState::ExecutionComplete); + input.SetState(TExprNode::EState::ExecutionComplete); } else { input.SetResult(ctx.NewAtom(input.Pos(), data)); - input.SetState(TExprNode::EState::ExecutionRequired); + input.SetState(TExprNode::EState::ExecutionRequired); } } else if (status.Level == TStatus::Error) { if (const auto issies = ctx.AssociativeIssues.extract(DelegatedNode.Get())) { ctx.IssueManager.RaiseIssues(issies.mapped()); } } else { - input.SetState(TExprNode::EState::ExecutionRequired); + input.SetState(TExprNode::EState::ExecutionRequired); } DelegatedProvider = nullptr; @@ -595,47 +595,47 @@ namespace { IDataProvider* DelegatedProvider = nullptr; TExprNode::TPtr DelegatedNode; TExprNode::TPtr DelegatedNodeOutput; - - ui64 CommittedPullSize = 0; - bool PullOverflow = false; - - ui64 CommittedFillSize = 0; - bool FillOverflow = false; - + + ui64 CommittedPullSize = 0; + bool PullOverflow = false; + + ui64 CommittedFillSize = 0; + bool FillOverflow = false; + TIntrusivePtr<IResultWriter> ResultWriter; }; - template <class TTarget> - bool& TResultCallableExecutionTransformer::GetOverflowFlagAndCommitedSize(ui64& committed) { - committed = CommittedFillSize; - return FillOverflow; - } - - template<> - bool& TResultCallableExecutionTransformer::GetOverflowFlagAndCommitedSize<TPull>(ui64& committed) { - committed = CommittedPullSize; - return PullOverflow; - } - - class TResultTrackableNodeProcessor : public TTrackableNodeProcessorBase { - public: - TResultTrackableNodeProcessor(const TIntrusivePtr<TResultProviderConfig>& config) - : Config(config) - {} - - void GetUsedNodes(const TExprNode& input, TVector<TString>& usedNodeIds) override { - usedNodeIds.clear(); + template <class TTarget> + bool& TResultCallableExecutionTransformer::GetOverflowFlagAndCommitedSize(ui64& committed) { + committed = CommittedFillSize; + return FillOverflow; + } + + template<> + bool& TResultCallableExecutionTransformer::GetOverflowFlagAndCommitedSize<TPull>(ui64& committed) { + committed = CommittedPullSize; + return PullOverflow; + } + + class TResultTrackableNodeProcessor : public TTrackableNodeProcessorBase { + public: + TResultTrackableNodeProcessor(const TIntrusivePtr<TResultProviderConfig>& config) + : Config(config) + {} + + void GetUsedNodes(const TExprNode& input, TVector<TString>& usedNodeIds) override { + usedNodeIds.clear(); if (TMaybeNode<TResFill>(&input) || TMaybeNode<TResPull>(&input) || TMaybeNode<TResIf>(&input) || TMaybeNode<TResFor>(&input)) { - auto provider = Config->Types.DataSourceMap.FindPtr(input.Child(5)->Content()); - Y_ENSURE(provider, "DataSource not exist: " << input.Child(5)->Content()); - (*provider)->GetTrackableNodeProcessor().GetUsedNodes(input, usedNodeIds); - } - } - private: - const TIntrusivePtr<TResultProviderConfig> Config; - }; - + auto provider = Config->Types.DataSourceMap.FindPtr(input.Child(5)->Content()); + Y_ENSURE(provider, "DataSource not exist: " << input.Child(5)->Content()); + (*provider)->GetTrackableNodeProcessor().GetUsedNodes(input, usedNodeIds); + } + } + private: + const TIntrusivePtr<TResultProviderConfig> Config; + }; + class TPhysicalFinalizingTransformer final : public TSyncTransformerBase { public: TPhysicalFinalizingTransformer(const TIntrusivePtr<TResultProviderConfig>& config) @@ -669,7 +669,7 @@ namespace { auto newInput = writeInput; if (isRef && !canRef) { - ctx.AddError(TIssue(ctx.GetPosition(writeInput.Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(writeInput.Pos()), TStringBuilder() << "RefSelect mode isn't supported by provider: " << source->GetName())); return nullptr; } @@ -713,7 +713,7 @@ namespace { if (!isRef) { auto data = resWrite.Data(); - if (auto unordered = data.Maybe<TCoUnorderedBase>()) { + if (auto unordered = data.Maybe<TCoUnorderedBase>()) { data = unordered.Cast().Input(); } @@ -875,7 +875,7 @@ namespace { TResultProvider(const TIntrusivePtr<TResultProviderConfig>& config) : Config(config) - , TrackableNodeProcessor(config) + , TrackableNodeProcessor(config) {} TStringBuf GetName() const override { @@ -921,7 +921,7 @@ namespace { } if (!res.Ref().Child(TResWriteBase::idx_Key)->IsCallable("Key") || res.Ref().Child(TResWriteBase::idx_Key)->ChildrenSize() > 0) { - ctx.AddError(TIssue(ctx.GetPosition(res.Ref().Child(TResWriteBase::idx_Key)->Pos()), "Expected empty key")); + ctx.AddError(TIssue(ctx.GetPosition(res.Ref().Child(TResWriteBase::idx_Key)->Pos()), "Expected empty key")); return IGraphTransformer::TStatus::Error; } @@ -934,7 +934,7 @@ namespace { return IGraphTransformer::TStatus::Error; } - TExprNode::TPtr columns; + TExprNode::TPtr columns; bool hasRef = false; bool hasAutoRef = false; ui32 settingPos = 0; @@ -973,12 +973,12 @@ namespace { ui64 limit = 0; if (!TryFromString(setting->Child(1)->Content(), limit)) { - ctx.AddError(TIssue(ctx.GetPosition(setting->Child(1)->Pos()), "Expected unsigned integer")); + ctx.AddError(TIssue(ctx.GetPosition(setting->Child(1)->Pos()), "Expected unsigned integer")); return IGraphTransformer::TStatus::Error; } } else if (content == "columns") { if (columns) { - ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "columns is already used")); + ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "columns is already used")); return IGraphTransformer::TStatus::Error; } @@ -986,7 +986,7 @@ namespace { return IGraphTransformer::TStatus::Error; } - columns = setting->ChildPtr(1); + columns = setting->ChildPtr(1); auto status = ValidateColumns(columns, res.Data().Ref().GetTypeAnn(), ctx); if (status.Level != IGraphTransformer::TStatus::Ok) { if (status.Level == IGraphTransformer::TStatus::Repeat) { @@ -997,10 +997,10 @@ namespace { return status; } - } else if (content == "freezeColumns") { - if (!EnsureTupleMaxSize(*setting, 1, ctx)) { - return IGraphTransformer::TStatus::Error; - } + } else if (content == "freezeColumns") { + if (!EnsureTupleMaxSize(*setting, 1, ctx)) { + return IGraphTransformer::TStatus::Error; + } } else if (content == "discard") { if (!EnsureTupleMaxSize(*setting, 1, ctx)) { return IGraphTransformer::TStatus::Error; @@ -1014,12 +1014,12 @@ namespace { return IGraphTransformer::TStatus::Error; } } else { - ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "Expected label,discard,ref,autoref,type,take or columns atom")); + ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "Expected label,discard,ref,autoref,type,take or columns atom")); return IGraphTransformer::TStatus::Error; } if (hasRef && hasAutoRef) { - ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "Either ref or autoref may be specified, not both")); + ctx.AddError(TIssue(ctx.GetPosition(setting->Pos()), "Either ref or autoref may be specified, not both")); return IGraphTransformer::TStatus::Error; } @@ -1045,7 +1045,7 @@ namespace { } if (!provider) { - ctx.AddError(TIssue(ctx.GetPosition(res.Data().Pos()), "Expected Right! over Datasource or Datasink")); + ctx.AddError(TIssue(ctx.GetPosition(res.Data().Pos()), "Expected Right! over Datasource or Datasink")); return IGraphTransformer::TStatus::Error; } } @@ -1057,31 +1057,31 @@ namespace { } if (!Config->Types.DataSourceMap.FindPtr(resTransient.DelegatedSource().Value())) { - ctx.AddError(TIssue(ctx.GetPosition(resTransient.DelegatedSource().Pos()), + ctx.AddError(TIssue(ctx.GetPosition(resTransient.DelegatedSource().Pos()), TStringBuilder() << "DataSource is not found: " << resTransient.DelegatedSource().Value())); return IGraphTransformer::TStatus::Error; } } - if (res.Data().Ref().IsCallable("AssumeColumnOrder")) { - if (!HasSetting(res.Settings().Ref(), "freezeColumns")) { - auto dataOrder = Config->Types.LookupColumnOrder(res.Data().Ref()); - YQL_ENSURE(dataOrder); - - YQL_CLOG(INFO, ProviderResult) << "Setting result column order: " << FormatColumnOrder(dataOrder); - auto settings = RemoveSetting(res.Settings().Ref(), "columns", ctx); - TExprNodeList columnsList; - for (auto& col : *dataOrder) { - columnsList.push_back(ctx.NewAtom(settings->Pos(), col)); - } - settings = AddSetting(*settings, settings->Pos(), "columns", ctx.NewList(settings->Pos(), std::move(columnsList)), ctx); - settings = AddSetting(*settings, settings->Pos(), "freezeColumns", nullptr, ctx); - output = ctx.ChangeChild(*input, TResWriteBase::idx_Settings, std::move(settings)); - } - output = ctx.ChangeChild(*output, TResWriteBase::idx_Data, res.Data().Ref().HeadPtr()); - return IGraphTransformer::TStatus::Repeat; - } - + if (res.Data().Ref().IsCallable("AssumeColumnOrder")) { + if (!HasSetting(res.Settings().Ref(), "freezeColumns")) { + auto dataOrder = Config->Types.LookupColumnOrder(res.Data().Ref()); + YQL_ENSURE(dataOrder); + + YQL_CLOG(INFO, ProviderResult) << "Setting result column order: " << FormatColumnOrder(dataOrder); + auto settings = RemoveSetting(res.Settings().Ref(), "columns", ctx); + TExprNodeList columnsList; + for (auto& col : *dataOrder) { + columnsList.push_back(ctx.NewAtom(settings->Pos(), col)); + } + settings = AddSetting(*settings, settings->Pos(), "columns", ctx.NewList(settings->Pos(), std::move(columnsList)), ctx); + settings = AddSetting(*settings, settings->Pos(), "freezeColumns", nullptr, ctx); + output = ctx.ChangeChild(*input, TResWriteBase::idx_Settings, std::move(settings)); + } + output = ctx.ChangeChild(*output, TResWriteBase::idx_Data, res.Data().Ref().HeadPtr()); + return IGraphTransformer::TStatus::Repeat; + } + input->SetTypeAnn(res.World().Ref().GetTypeAnn()); return IGraphTransformer::TStatus::Ok; } @@ -1115,7 +1115,7 @@ namespace { } if (!Config->Types.DataSourceMap.FindPtr(input->Child(TResIf::idx_DelegatedSource)->Content())) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResIf::idx_DelegatedSource)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResIf::idx_DelegatedSource)->Pos()), TStringBuilder() << "DataSource is not found: " << input->Child(TResIf::idx_DelegatedSource)->Content())); return IGraphTransformer::TStatus::Error; } @@ -1192,7 +1192,7 @@ namespace { } if (!Config->Types.DataSourceMap.FindPtr(input->Child(TResFor::idx_DelegatedSource)->Content())) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_DelegatedSource)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_DelegatedSource)->Pos()), TStringBuilder() << "DataSource is not found: " << input->Child(TResFor::idx_DelegatedSource)->Content())); return IGraphTransformer::TStatus::Error; } @@ -1209,7 +1209,7 @@ namespace { auto total = input->Child(TResFor::idx_Total)->Content(); ui32 totalValue = 0; if (total && !TryFromString(total, totalValue)) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Total)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Total)->Pos()), TStringBuilder() << "Expected number, but got: " << total)); return IGraphTransformer::TStatus::Error; } @@ -1221,13 +1221,13 @@ namespace { auto current = input->Child(TResFor::idx_Current)->Content(); ui32 currentValue = 0; if (current && !TryFromString(current, currentValue)) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Current)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Current)->Pos()), TStringBuilder() << "Expected number, but got: " << current)); return IGraphTransformer::TStatus::Error; } if (!total != !current) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Current)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(input->Child(TResFor::idx_Current)->Pos()), TStringBuilder() << "Current value should be set simultaneously with total value")); return IGraphTransformer::TStatus::Error; } @@ -1289,7 +1289,7 @@ namespace { const auto limitStr = input->Child(3)->Content(); ui64 limit; if (!TryFromString(limitStr, limit)) { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(3)->Pos()), TStringBuilder() << "expected integer, but got: " << limitStr)); + ctx.AddError(TIssue(ctx.GetPosition(input->Child(3)->Pos()), TStringBuilder() << "expected integer, but got: " << limitStr)); return IGraphTransformer::TStatus::Error; } @@ -1300,7 +1300,7 @@ namespace { } } else { - ctx.AddError(TIssue(ctx.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unsupported command: " << command)); + ctx.AddError(TIssue(ctx.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Unsupported command: " << command)); return IGraphTransformer::TStatus::Error; } @@ -1308,7 +1308,7 @@ namespace { return IGraphTransformer::TStatus::Ok; } - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Result) Unsupported function: " << input->Content())); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Result) Unsupported function: " << input->Content())); return IGraphTransformer::TStatus::Error; }); } @@ -1437,8 +1437,8 @@ namespace { TStringBuilder res; res << node.Content(); - if (resFor.Total().Value() && resFor.Total().Value() != "0") { - res << ", " << (node.GetState() == TExprNode::EState::ExecutionComplete ? + if (resFor.Total().Value() && resFor.Total().Value() != "0") { + res << ", " << (node.GetState() == TExprNode::EState::ExecutionComplete ? resFor.Total().Value() : resFor.Current().Value()) << "/" << resFor.Total().Value(); } @@ -1448,13 +1448,13 @@ namespace { return TString(node.Content()); } - ITrackableNodeProcessor& GetTrackableNodeProcessor() override { - return TrackableNodeProcessor; - } + ITrackableNodeProcessor& GetTrackableNodeProcessor() override { + return TrackableNodeProcessor; + } private: const TIntrusivePtr<TResultProviderConfig> Config; - TResultTrackableNodeProcessor TrackableNodeProcessor; + TResultTrackableNodeProcessor TrackableNodeProcessor; TAutoPtr<IGraphTransformer> TypeAnnotationTransformer; TAutoPtr<IGraphTransformer> PhysicalFinalizingTransformer; TAutoPtr<IGraphTransformer> CallableExecutionTransformer; diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp index de87cb7a0b..99c308eaf3 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp @@ -12,7 +12,7 @@ TS3Configuration::TS3Configuration() } TS3Settings::TConstPtr TS3Configuration::Snapshot() const { - return std::make_shared<const TS3Settings>(*this); + return std::make_shared<const TS3Settings>(*this); } bool TS3Configuration::HasCluster(TStringBuf cluster) const { diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h index 47d8f25270..9bbcb72c87 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h @@ -8,7 +8,7 @@ namespace NYql { struct TS3Settings { - using TConstPtr = std::shared_ptr<const TS3Settings>; + using TConstPtr = std::shared_ptr<const TS3Settings>; NCommon::TConfSetting<bool, false> SourceActor; }; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp index 9d8365657c..fe00223f7c 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp @@ -9,7 +9,7 @@ TSolomonConfiguration::TSolomonConfiguration() } TSolomonSettings::TConstPtr TSolomonConfiguration::Snapshot() const { - return std::make_shared<const TSolomonSettings>(*this); + return std::make_shared<const TSolomonSettings>(*this); } } // NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index 40e399b046..941b2adc30 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -8,7 +8,7 @@ namespace NYql { struct TSolomonSettings { - using TConstPtr = std::shared_ptr<const TSolomonSettings>; + using TConstPtr = std::shared_ptr<const TSolomonSettings>; }; struct TSolomonConfiguration diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp index 17302a2fea..b09fd381a0 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp @@ -33,7 +33,7 @@ public: return {}; } - TPositionHandle pos; + TPositionHandle pos; return Build<TCoAtom>(ctx, pos) .Value(config->GetCluster()) .Done().Ptr(); @@ -77,7 +77,7 @@ public: if (node.Child(0)->Content() == SolomonProviderName) { auto clusterName = node.Child(1)->Content(); if (!State_->Gateway->HasCluster(clusterName)) { - ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unknown cluster name: " << clusterName )); return false; @@ -86,7 +86,7 @@ public: return true; } } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSink parameters")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSink parameters")); return false; } @@ -109,14 +109,14 @@ public: auto tagName = key.Child(0)->Child(0)->Content(); if (tagName != TStringBuf("table")) { - ctx.AddError(TIssue(ctx.GetPosition(key.Child(0)->Pos()), + ctx.AddError(TIssue(ctx.GetPosition(key.Child(0)->Pos()), TStringBuilder() << "Unexpected tag: " << tagName)); return {}; } const TExprNode* nameNode = key.Child(0)->Child(1); if (!nameNode->IsCallable("String")) { - ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String as table key")); + ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String as table key")); return {}; } @@ -129,7 +129,7 @@ public: return {}; } if (tablePath->Content().empty()) { - ctx.AddError(TIssue(ctx.GetPosition(tablePath->Pos()), "Table name must not be empty")); + ctx.AddError(TIssue(ctx.GetPosition(tablePath->Pos()), "Table name must not be empty")); return {}; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp index 99e358c56a..31ceb1701c 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp @@ -57,7 +57,7 @@ public: if (node.Child(0)->Content() == SolomonProviderName) { auto clusterName = node.Child(1)->Content(); if (!State_->Gateway->HasCluster(clusterName)) { - ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << + ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unknown cluster name: " << clusterName)); return false; } @@ -65,7 +65,7 @@ public: return true; } } - ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSource parameters")); + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSource parameters")); return false; } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp index 44022719c7..06d1919027 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp @@ -10,7 +10,7 @@ TYdbConfiguration::TYdbConfiguration() } TYdbSettings::TConstPtr TYdbConfiguration::Snapshot() const { - return std::make_shared<const TYdbSettings>(*this); + return std::make_shared<const TYdbSettings>(*this); } bool TYdbConfiguration::HasCluster(TStringBuf cluster) const { diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h index 9beb4a8826..fc6fb03ced 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h @@ -13,7 +13,7 @@ namespace NYql { struct TYdbSettings { - using TConstPtr = std::shared_ptr<const TYdbSettings>; + using TConstPtr = std::shared_ptr<const TYdbSettings>; }; struct TClusterMainSettings { diff --git a/ydb/library/yql/public/issue/ut/ya.make b/ydb/library/yql/public/issue/ut/ya.make index 51f429f3bb..4fc5841a51 100644 --- a/ydb/library/yql/public/issue/ut/ya.make +++ b/ydb/library/yql/public/issue/ut/ya.make @@ -8,7 +8,7 @@ SRCS( yql_issue_ut.cpp yql_issue_manager_ut.cpp yql_issue_utils_ut.cpp - yql_warning_ut.cpp + yql_warning_ut.cpp ) PEERDIR( diff --git a/ydb/library/yql/public/issue/ya.make b/ydb/library/yql/public/issue/ya.make index de85a29574..04a84bee88 100644 --- a/ydb/library/yql/public/issue/ya.make +++ b/ydb/library/yql/public/issue/ya.make @@ -10,7 +10,7 @@ SRCS( yql_issue_message.cpp yql_issue_manager.cpp yql_issue_utils.cpp - yql_warning.cpp + yql_warning.cpp ) PEERDIR( @@ -22,7 +22,7 @@ PEERDIR( ydb/library/yql/utils ) -GENERATE_ENUM_SERIALIZATION(yql_warning.h) +GENERATE_ENUM_SERIALIZATION(yql_warning.h) END() diff --git a/ydb/library/yql/public/issue/yql_issue.cpp b/ydb/library/yql/public/issue/yql_issue.cpp index c0db9f24b6..3b89156874 100644 --- a/ydb/library/yql/public/issue/yql_issue.cpp +++ b/ydb/library/yql/public/issue/yql_issue.cpp @@ -27,31 +27,31 @@ void SanitizeNonAscii(TString& s) { } } -TTextWalker& TTextWalker::Advance(char c) { - if (c == '\n') { - HaveCr = false; - ++LfCount; - return *this; - } - - - if (c == '\r' && !HaveCr) { - HaveCr = true; - return *this; - } - - // either not '\r' or second '\r' - if (LfCount) { - Position.Row += LfCount; - Position.Column = 1; - LfCount = 0; - } else { - Position.Column += 1 + (HaveCr && c != '\r'); - } - HaveCr = (c == '\r'); - return *this; -} - +TTextWalker& TTextWalker::Advance(char c) { + if (c == '\n') { + HaveCr = false; + ++LfCount; + return *this; + } + + + if (c == '\r' && !HaveCr) { + HaveCr = true; + return *this; + } + + // either not '\r' or second '\r' + if (LfCount) { + Position.Row += LfCount; + Position.Column = 1; + LfCount = 0; + } else { + Position.Column += 1 + (HaveCr && c != '\r'); + } + HaveCr = (c == '\r'); + return *this; +} + void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { out << Range() << ": " << SeverityToString(GetSeverity()) << ": "; if (oneLine) { diff --git a/ydb/library/yql/public/issue/yql_issue.h b/ydb/library/yql/public/issue/yql_issue.h index 3bccc34bf5..7b97674c9d 100644 --- a/ydb/library/yql/public/issue/yql_issue.h +++ b/ydb/library/yql/public/issue/yql_issue.h @@ -53,31 +53,31 @@ struct TPosition { } }; -class TTextWalker { -public: - TTextWalker(TPosition& position) - : Position(position) - , HaveCr(false) - , LfCount(0) - { - } - - template<typename T> - TTextWalker& Advance(const T& buf) { - for (char c : buf) { - Advance(c); - } - return *this; - } - - TTextWalker& Advance(char c); - -private: - TPosition& Position; - bool HaveCr; - ui32 LfCount; -}; - +class TTextWalker { +public: + TTextWalker(TPosition& position) + : Position(position) + , HaveCr(false) + , LfCount(0) + { + } + + template<typename T> + TTextWalker& Advance(const T& buf) { + for (char c : buf) { + Advance(c); + } + return *this; + } + + TTextWalker& Advance(char c); + +private: + TPosition& Position; + bool HaveCr; + ui32 LfCount; +}; + struct TRange { TPosition Position; TPosition EndPosition; diff --git a/ydb/library/yql/public/issue/yql_issue_manager.cpp b/ydb/library/yql/public/issue/yql_issue_manager.cpp index c70ac7952c..7375d2d30f 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager.cpp +++ b/ydb/library/yql/public/issue/yql_issue_manager.cpp @@ -124,8 +124,8 @@ void TIssueManager::RaiseIssues(const TIssues& issues) { bool TIssueManager::RaiseWarning(TIssue issue) { bool isWarning = true; if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) { - const auto action = WarningPolicy_.GetAction(issue.GetCode()); - switch (action) { + const auto action = WarningPolicy_.GetAction(issue.GetCode()); + switch (action) { case EWarningAction::DISABLE: return isWarning; case EWarningAction::ERROR: @@ -139,8 +139,8 @@ bool TIssueManager::RaiseWarning(TIssue issue) { } isWarning = false; break; - case EWarningAction::DEFAULT: - break; + case EWarningAction::DEFAULT: + break; default: Y_ENSURE(false, "Unknown action"); } @@ -206,9 +206,9 @@ void TIssueManager::Reset() { Reset(TIssues()); } -void TIssueManager::AddWarningRule(const TWarningRule &rule) -{ - WarningPolicy_.AddRule(rule); +void TIssueManager::AddWarningRule(const TWarningRule &rule) +{ + WarningPolicy_.AddRule(rule); } void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) { diff --git a/ydb/library/yql/public/issue/yql_issue_manager.h b/ydb/library/yql/public/issue/yql_issue_manager.h index 4d66e492ea..9ad5ac7bb4 100644 --- a/ydb/library/yql/public/issue/yql_issue_manager.h +++ b/ydb/library/yql/public/issue/yql_issue_manager.h @@ -28,8 +28,8 @@ public: void Reset(const TIssues& issues); void Reset(); - - void AddWarningRule(const TWarningRule &rule); + + void AddWarningRule(const TWarningRule &rule); void SetWarningToErrorTreatMessage(const TString& msg); void SetIssueCountLimit(size_t limit) { @@ -52,7 +52,7 @@ private: TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_; TIssues CompletedIssues_; TMaybe<TString> WarningToErrorTreatMessage_; - TWarningPolicy WarningPolicy_; + TWarningPolicy WarningPolicy_; std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_; std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_; size_t IssueLimit_ = 0; diff --git a/ydb/library/yql/public/issue/yql_issue_ut.cpp b/ydb/library/yql/public/issue/yql_issue_ut.cpp index 50a13c908b..87b417da39 100644 --- a/ydb/library/yql/public/issue/yql_issue_ut.cpp +++ b/ydb/library/yql/public/issue/yql_issue_ut.cpp @@ -1,5 +1,5 @@ -#include "yql_issue.h" - +#include "yql_issue.h" + #include <library/cpp/testing/unittest/registar.h> #include <ydb/library/yql/public/issue/protos/issue_message.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> @@ -9,7 +9,7 @@ #include <google/protobuf/descriptor.h> using namespace google::protobuf; -using namespace NYql; +using namespace NYql; void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes); void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) { @@ -75,40 +75,40 @@ Y_UNIT_TEST_SUITE(IssueProtoTest) { UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); } } - - -Y_UNIT_TEST_SUITE(TextWalkerTest) { - Y_UNIT_TEST(BasicTest) { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos); + + +Y_UNIT_TEST_SUITE(TextWalkerTest) { + Y_UNIT_TEST(BasicTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos); walker.Advance(TStringBuf("a\r\taa")); - - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); walker.Advance(TStringBuf("\na")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); - } - - Y_UNIT_TEST(CrLfTest) { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); + } + + Y_UNIT_TEST(CrLfTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos); walker.Advance(TStringBuf("a\raa\r")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); walker.Advance(TStringBuf("\r\r\ra")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('\r'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('a'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); - } -} + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\r'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('a'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); + } +} Y_UNIT_TEST_SUITE(ToOneLineStringTest) { Y_UNIT_TEST(OneMessageTest) { diff --git a/ydb/library/yql/public/issue/yql_warning.cpp b/ydb/library/yql/public/issue/yql_warning.cpp index bfb3f9dd78..881e63e95f 100644 --- a/ydb/library/yql/public/issue/yql_warning.cpp +++ b/ydb/library/yql/public/issue/yql_warning.cpp @@ -1,73 +1,73 @@ -#include "yql_warning.h" - -#include <util/string/cast.h> -#include <util/string/join.h> - - -namespace NYql { - -TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action, - TWarningRule& result, TString& errorMessage) -{ - errorMessage.clear(); - result.IssueCodePattern.clear(); - result.Action = EWarningAction::DEFAULT; - - if (!TryFromString<EWarningAction>(to_upper(action), result.Action)) { - errorMessage = "unknown warning action '" + action + "', expecting one of " + - Join(", ", EWarningAction::DEFAULT, EWarningAction::ERROR, EWarningAction::DISABLE); - return EParseResult::PARSE_ACTION_FAIL; - } - - if (codePattern != "*") { - ui32 code; - if (!TryFromString(codePattern, code)) { - errorMessage = "unknown warning code '" + codePattern + "', expecting integer or '*'"; - return EParseResult::PARSE_PATTERN_FAIL; - } - } - - result.IssueCodePattern = codePattern; - return EParseResult::PARSE_OK; -} - -void TWarningPolicy::AddRule(const TWarningRule& rule) -{ - TString pattern = rule.GetPattern(); - if (pattern.empty()) { - return; - } - - Rules.push_back(rule); - - EWarningAction action = rule.GetAction(); - if (pattern == "*") { - BaseAction = action; - Overrides.clear(); - return; - } - - TIssueCode code; - Y_ENSURE(TryFromString(pattern, code)); - - if (action == BaseAction) { - Overrides.erase(Overrides.find(code)); - } else { - Overrides[code] = action; - } -} - -EWarningAction TWarningPolicy::GetAction(TIssueCode code) const -{ - auto it = Overrides.find(code); - return (it == Overrides.end()) ? BaseAction : it->second; -} - -void TWarningPolicy::Clear() -{ - BaseAction = EWarningAction::DEFAULT; - Overrides.clear(); - Rules.clear(); -} - -} +#include "yql_warning.h" + +#include <util/string/cast.h> +#include <util/string/join.h> + + +namespace NYql { + +TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage) +{ + errorMessage.clear(); + result.IssueCodePattern.clear(); + result.Action = EWarningAction::DEFAULT; + + if (!TryFromString<EWarningAction>(to_upper(action), result.Action)) { + errorMessage = "unknown warning action '" + action + "', expecting one of " + + Join(", ", EWarningAction::DEFAULT, EWarningAction::ERROR, EWarningAction::DISABLE); + return EParseResult::PARSE_ACTION_FAIL; + } + + if (codePattern != "*") { + ui32 code; + if (!TryFromString(codePattern, code)) { + errorMessage = "unknown warning code '" + codePattern + "', expecting integer or '*'"; + return EParseResult::PARSE_PATTERN_FAIL; + } + } + + result.IssueCodePattern = codePattern; + return EParseResult::PARSE_OK; +} + +void TWarningPolicy::AddRule(const TWarningRule& rule) +{ + TString pattern = rule.GetPattern(); + if (pattern.empty()) { + return; + } + + Rules.push_back(rule); + + EWarningAction action = rule.GetAction(); + if (pattern == "*") { + BaseAction = action; + Overrides.clear(); + return; + } + + TIssueCode code; + Y_ENSURE(TryFromString(pattern, code)); + + if (action == BaseAction) { + Overrides.erase(Overrides.find(code)); + } else { + Overrides[code] = action; + } +} + +EWarningAction TWarningPolicy::GetAction(TIssueCode code) const +{ + auto it = Overrides.find(code); + return (it == Overrides.end()) ? BaseAction : it->second; +} + +void TWarningPolicy::Clear() +{ + BaseAction = EWarningAction::DEFAULT; + Overrides.clear(); + Rules.clear(); +} + +} diff --git a/ydb/library/yql/public/issue/yql_warning.h b/ydb/library/yql/public/issue/yql_warning.h index 8769e164b8..d1d6a90922 100644 --- a/ydb/library/yql/public/issue/yql_warning.h +++ b/ydb/library/yql/public/issue/yql_warning.h @@ -1,51 +1,51 @@ -#pragma once - -#include "yql_issue_id.h" - -#include <util/generic/hash.h> -#include <util/generic/string.h> - -#if defined(_win_) -#undef ERROR -#endif - -namespace NYql { - -enum class EWarningAction { - DISABLE, - ERROR, - DEFAULT, -}; - -class TWarningRule { -public: - const TString& GetPattern() const { return IssueCodePattern; } - EWarningAction GetAction() const { return Action; } - - enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL }; - static EParseResult ParseFrom(const TString& codePattern, const TString& action, - TWarningRule& result, TString& errorMessage); -private: - TString IssueCodePattern; - EWarningAction Action = EWarningAction::DEFAULT; -}; - -using TWarningRules = TVector<TWarningRule>; - -class TWarningPolicy { -public: - void AddRule(const TWarningRule& rule); - - EWarningAction GetAction(TIssueCode code) const; - - const TWarningRules& GetRules() const { return Rules; } - - void Clear(); - -private: - TWarningRules Rules; - EWarningAction BaseAction = EWarningAction::DEFAULT; - THashMap<TIssueCode, EWarningAction> Overrides; -}; - -} +#pragma once + +#include "yql_issue_id.h" + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +#if defined(_win_) +#undef ERROR +#endif + +namespace NYql { + +enum class EWarningAction { + DISABLE, + ERROR, + DEFAULT, +}; + +class TWarningRule { +public: + const TString& GetPattern() const { return IssueCodePattern; } + EWarningAction GetAction() const { return Action; } + + enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL }; + static EParseResult ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage); +private: + TString IssueCodePattern; + EWarningAction Action = EWarningAction::DEFAULT; +}; + +using TWarningRules = TVector<TWarningRule>; + +class TWarningPolicy { +public: + void AddRule(const TWarningRule& rule); + + EWarningAction GetAction(TIssueCode code) const; + + const TWarningRules& GetRules() const { return Rules; } + + void Clear(); + +private: + TWarningRules Rules; + EWarningAction BaseAction = EWarningAction::DEFAULT; + THashMap<TIssueCode, EWarningAction> Overrides; +}; + +} diff --git a/ydb/library/yql/public/issue/yql_warning_ut.cpp b/ydb/library/yql/public/issue/yql_warning_ut.cpp index 43646bb918..5edf7ed363 100644 --- a/ydb/library/yql/public/issue/yql_warning_ut.cpp +++ b/ydb/library/yql/public/issue/yql_warning_ut.cpp @@ -1,94 +1,94 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "yql_warning.h" - -using namespace NYql; - -Y_UNIT_TEST_SUITE(TWarningRuleTest) { - - Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - } - - Y_UNIT_TEST(InvalidActionIsDetected) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); - } - - Y_UNIT_TEST(InvalidPatternIsDetected) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - } -} - - -void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action) -{ - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom(codePattern, action, rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - policy.AddRule(rule); -} - - -Y_UNIT_TEST_SUITE(TWarningPolicyTest) { - - Y_UNIT_TEST(BasicIntegerRules) { - TWarningPolicy policy; - AddRule(policy, "123", "error"); - AddRule(policy, "456", "error"); - AddRule(policy, "456", "disable"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); - } - - - Y_UNIT_TEST(AsteriskRules) { - TWarningPolicy policy; - AddRule(policy, "*", "error"); - AddRule(policy, "456", "disable"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); - - AddRule(policy, "*", "default"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); - - } - - Y_UNIT_TEST(ReactionOnPull) { - TWarningPolicy policy; - AddRule(policy, "*", "error"); - AddRule(policy, "456", "default"); - AddRule(policy, "999", "disable"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - policy.Clear(); - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); - } -} + +#include "yql_warning.h" + +using namespace NYql; + +Y_UNIT_TEST_SUITE(TWarningRuleTest) { + + Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + } + + Y_UNIT_TEST(InvalidActionIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + } + + Y_UNIT_TEST(InvalidPatternIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + } +} + + +void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action) +{ + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom(codePattern, action, rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + policy.AddRule(rule); +} + + +Y_UNIT_TEST_SUITE(TWarningPolicyTest) { + + Y_UNIT_TEST(BasicIntegerRules) { + TWarningPolicy policy; + AddRule(policy, "123", "error"); + AddRule(policy, "456", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); + } + + + Y_UNIT_TEST(AsteriskRules) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + + AddRule(policy, "*", "default"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + + } + + Y_UNIT_TEST(ReactionOnPull) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "default"); + AddRule(policy, "999", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + policy.Clear(); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + } +} diff --git a/ydb/library/yql/public/udf/udf_allocator.h b/ydb/library/yql/public/udf/udf_allocator.h index 3c5c17f38b..48a94ba6a1 100644 --- a/ydb/library/yql/public/udf/udf_allocator.h +++ b/ydb/library/yql/public/udf/udf_allocator.h @@ -2,7 +2,7 @@ #include "udf_version.h" #include <util/system/types.h> #include <new> -#include <cstddef> +#include <cstddef> #include <limits> #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) @@ -34,7 +34,7 @@ struct TStdAllocatorForUdf typedef const Type& const_reference; typedef size_t size_type; typedef ptrdiff_t difference_type; - + TStdAllocatorForUdf() noexcept = default; ~TStdAllocatorForUdf() noexcept = default; @@ -50,8 +50,8 @@ struct TStdAllocatorForUdf #else return static_cast<pointer>(UdfAllocate(n * sizeof(value_type))); #endif - } - + } + static void deallocate(const_pointer p, size_type n) noexcept { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) @@ -61,9 +61,9 @@ struct TStdAllocatorForUdf (void)(n); return UdfFree(static_cast<const void*>(p)); #endif - } -}; - + } +}; + struct TWithUdfAllocator { void* operator new(size_t sz) { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) diff --git a/ydb/library/yql/sql/settings/translation_settings.cpp b/ydb/library/yql/sql/settings/translation_settings.cpp index fbaacb59af..e09fd43aaa 100644 --- a/ydb/library/yql/sql/settings/translation_settings.cpp +++ b/ydb/library/yql/sql/settings/translation_settings.cpp @@ -2,31 +2,31 @@ #include <ydb/library/yql/core/issue/yql_issue.h> #include <ydb/library/yql/utils/utf8.h> - -#include <library/cpp/deprecated/split/split_iterator.h> - -#include <util/string/split.h> -#include <util/system/env.h> - -namespace { - bool InTestEnvironment() { - return GetEnv("YQL_LOCAL_ENVIRONMENT") || GetEnv("YA_TEST_RUNNER"); - } - - using namespace NSQLTranslation; + +#include <library/cpp/deprecated/split/split_iterator.h> + +#include <util/string/split.h> +#include <util/system/env.h> + +namespace { + bool InTestEnvironment() { + return GetEnv("YQL_LOCAL_ENVIRONMENT") || GetEnv("YA_TEST_RUNNER"); + } + + using namespace NSQLTranslation; class TAlwaysDiallowPolicy : public ISqlFeaturePolicy { - public: + public: TAlwaysDiallowPolicy() = default; bool Allow() const override { - return false; - } - }; -} - + return false; + } + }; +} + namespace NSQLTranslation { ISqlFeaturePolicy::TPtr ISqlFeaturePolicy::MakeAlwaysDisallow() { return new TAlwaysDiallowPolicy; - } + } TTranslationSettings::TTranslationSettings() : ModuleMapping({{"core", "/lib/yql/core.yql"}}) @@ -35,84 +35,84 @@ namespace NSQLTranslation { , EndOfQueryCommit(true) , EnableGenericUdfs(true) , SyntaxVersion(0) - , AnsiLexer(false) + , AnsiLexer(false) , PgParser(false) , InferSyntaxVersion(false) , V0Behavior(EV0Behavior::Silent) - , V0ForceDisable(InTestEnvironment()) - , WarnOnV0(true) + , V0ForceDisable(InTestEnvironment()) + , WarnOnV0(true) , V0WarnAsError(ISqlFeaturePolicy::MakeAlwaysDisallow()) , DqDefaultAuto(ISqlFeaturePolicy::MakeAlwaysDisallow()) - , AssumeYdbOnClusterWithSlash(false) + , AssumeYdbOnClusterWithSlash(false) {} - - bool ParseTranslationSettings(const TString& query, TTranslationSettings& settings, NYql::TIssues& issues) { - if (!NYql::IsUtf8(query)) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, 0), NYql::TIssuesIds::DEFAULT_ERROR, "Invalid UTF8 input")); - return false; - } - - TSplitDelimiters lineDelimiters("\n\r"); - TDelimitersSplit linesSplit(query, lineDelimiters); - auto lineIterator = linesSplit.Iterator(); - auto lineNumber = 0; - - while (!lineIterator.Eof()) { - const TString& line = lineIterator.NextString(); - ++lineNumber; - - TVector<TStringBuf> parts; - TSetDelimiter<const char> partsDelimiters(" \t"); - Split(line, partsDelimiters, parts); - - if (parts.empty()) { - continue; - } - - if (!parts[0].StartsWith("--!")) { - break; - } - - if (parts.size() > 1) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, - "Bad translation settings format")); - return false; - } - - auto value = TString(parts[0].SubString(3, TString::npos)); - if (value.empty()) { - continue; - } - - if ((value == "syntax_v0" || value == "syntax_v1") && !settings.InferSyntaxVersion) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, - "Explicit syntax version is not allowed")); - return false; - } - - if (value == "syntax_v0") { - if (settings.V0ForceDisable || settings.V0Behavior == NSQLTranslation::EV0Behavior::Disable) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, - "V0 syntax is disabled")); - return false; - } - - settings.SyntaxVersion = 0; - } else if (value == "syntax_v1") { - settings.SyntaxVersion = 1; - } else if (value == "ansi_lexer") { - settings.AnsiLexer = true; - } else if (value == "syntax_pg") { - settings.PgParser = true; - } else { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, - TStringBuilder() << "Unknown SQL translation setting: " << value)); - return false; - } - } - - return true; - } - + + bool ParseTranslationSettings(const TString& query, TTranslationSettings& settings, NYql::TIssues& issues) { + if (!NYql::IsUtf8(query)) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, 0), NYql::TIssuesIds::DEFAULT_ERROR, "Invalid UTF8 input")); + return false; + } + + TSplitDelimiters lineDelimiters("\n\r"); + TDelimitersSplit linesSplit(query, lineDelimiters); + auto lineIterator = linesSplit.Iterator(); + auto lineNumber = 0; + + while (!lineIterator.Eof()) { + const TString& line = lineIterator.NextString(); + ++lineNumber; + + TVector<TStringBuf> parts; + TSetDelimiter<const char> partsDelimiters(" \t"); + Split(line, partsDelimiters, parts); + + if (parts.empty()) { + continue; + } + + if (!parts[0].StartsWith("--!")) { + break; + } + + if (parts.size() > 1) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, + "Bad translation settings format")); + return false; + } + + auto value = TString(parts[0].SubString(3, TString::npos)); + if (value.empty()) { + continue; + } + + if ((value == "syntax_v0" || value == "syntax_v1") && !settings.InferSyntaxVersion) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, + "Explicit syntax version is not allowed")); + return false; + } + + if (value == "syntax_v0") { + if (settings.V0ForceDisable || settings.V0Behavior == NSQLTranslation::EV0Behavior::Disable) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, + "V0 syntax is disabled")); + return false; + } + + settings.SyntaxVersion = 0; + } else if (value == "syntax_v1") { + settings.SyntaxVersion = 1; + } else if (value == "ansi_lexer") { + settings.AnsiLexer = true; + } else if (value == "syntax_pg") { + settings.PgParser = true; + } else { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR, + TStringBuilder() << "Unknown SQL translation setting: " << value)); + return false; + } + } + + return true; + } + } // namespace NSQLTranslation diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h index 9ec0eea760..0b9194a249 100644 --- a/ydb/library/yql/sql/settings/translation_settings.h +++ b/ydb/library/yql/sql/settings/translation_settings.h @@ -7,10 +7,10 @@ namespace google::protobuf { class Arena; } -namespace NYql { - class TIssues; -} - +namespace NYql { + class TIssues; +} + namespace NSQLTranslation { constexpr const size_t SQL_MAX_PARSER_ERRORS = 100; @@ -35,20 +35,20 @@ namespace NSQLTranslation { }; class ISqlFeaturePolicy : public TThrRefBase { - public: + public: virtual ~ISqlFeaturePolicy() = default; virtual bool Allow() const = 0; - + using TPtr = TIntrusivePtr<ISqlFeaturePolicy>; - + static TPtr MakeAlwaysDisallow(); - }; - - struct TTableBindingSettings { - TString ClusterType; - THashMap<TString, TString> Settings; - }; - + }; + + struct TTableBindingSettings { + TString ClusterType; + THashMap<TString, TString> Settings; + }; + struct TTranslationSettings { TTranslationSettings(); @@ -62,9 +62,9 @@ namespace NSQLTranslation { THashSet<TString> Libraries; THashSet<TString> Flags; - THashMap<TString, TTableBindingSettings> PrivateBindings; - THashMap<TString, TTableBindingSettings> ScopedBindings; - + THashMap<TString, TTableBindingSettings> PrivateBindings; + THashMap<TString, TTableBindingSettings> ScopedBindings; + ESqlMode Mode; TString DefaultCluster; TIncrementMonCounterFunction IncrementCounter; @@ -73,17 +73,17 @@ namespace NSQLTranslation { TString File; bool EnableGenericUdfs; ui16 SyntaxVersion; - bool AnsiLexer; + bool AnsiLexer; bool PgParser; bool InferSyntaxVersion; EV0Behavior V0Behavior; - bool V0ForceDisable; - bool WarnOnV0; + bool V0ForceDisable; + bool WarnOnV0; ISqlFeaturePolicy::TPtr V0WarnAsError; ISqlFeaturePolicy::TPtr DqDefaultAuto; - bool AssumeYdbOnClusterWithSlash; + bool AssumeYdbOnClusterWithSlash; }; - bool ParseTranslationSettings(const TString& query, NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues); - + bool ParseTranslationSettings(const TString& query, NSQLTranslation::TTranslationSettings& settings, NYql::TIssues& issues); + } // namespace NSQLTranslation diff --git a/ydb/library/yql/sql/settings/ya.make b/ydb/library/yql/sql/settings/ya.make index c0da0fa8ed..d653383cb7 100644 --- a/ydb/library/yql/sql/settings/ya.make +++ b/ydb/library/yql/sql/settings/ya.make @@ -8,11 +8,11 @@ SRCS( translation_settings.cpp ) -PEERDIR( - library/cpp/deprecated/split +PEERDIR( + library/cpp/deprecated/split ydb/library/yql/core/issue ydb/library/yql/core/issue/protos ydb/library/yql/utils -) - +) + END() diff --git a/ydb/library/yql/sql/sql.cpp b/ydb/library/yql/sql/sql.cpp index 470da0e5ab..9b03f681f3 100644 --- a/ydb/library/yql/sql/sql.cpp +++ b/ydb/library/yql/sql/sql.cpp @@ -9,8 +9,8 @@ #include <google/protobuf/arena.h> -#include <util/string/builder.h> - +#include <util/string/builder.h> + namespace NSQLTranslation { NYql::TAstParseResult SqlToYql(const TString& query, const TTranslationSettings& settings, @@ -37,18 +37,18 @@ namespace NSQLTranslation { switch (parsedSettings.SyntaxVersion) { case 0: - if (settings.V0ForceDisable || parsedSettings.V0Behavior == EV0Behavior::Disable) { + if (settings.V0ForceDisable || parsedSettings.V0Behavior == EV0Behavior::Disable) { result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, "V0 syntax is disabled")); return result; } - if (parsedSettings.AnsiLexer) { - result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - "Ansi lexer is not supported in V0 syntax")); - return result; - } - + if (parsedSettings.AnsiLexer) { + result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "Ansi lexer is not supported in V0 syntax")); + return result; + } + return NSQLTranslationV0::SqlToYql(query, parsedSettings, warningRules); case 1: return NSQLTranslationV1::SqlToYql(query, parsedSettings, warningRules); @@ -73,21 +73,21 @@ namespace NSQLTranslation { switch (parsedSettings.SyntaxVersion) { case 0: - if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { + if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, "V0 syntax is disabled")); return nullptr; } - if (parsedSettings.AnsiLexer) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - "Ansi lexer is not supported in V0 syntax")); - return nullptr; - } - + if (parsedSettings.AnsiLexer) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "Ansi lexer is not supported in V0 syntax")); + return nullptr; + } + return NSQLTranslationV0::SqlAST(query, queryName, issues, maxErrors, settings.Arena); case 1: - return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, settings.Arena); + return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, settings.Arena); default: issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion)); @@ -95,57 +95,57 @@ namespace NSQLTranslation { } } - ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings, ui16* actualSyntaxVersion) - { - TTranslationSettings parsedSettings(settings); - if (!ParseTranslationSettings(query, parsedSettings, issues)) { - return {}; - } - - if (actualSyntaxVersion) { - *actualSyntaxVersion = parsedSettings.SyntaxVersion; - } - - switch (parsedSettings.SyntaxVersion) { - case 0: - if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - "V0 syntax is disabled")); - return {}; - } - - if (parsedSettings.AnsiLexer) { - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - "Ansi lexer is not supported in V0 syntax")); - return {}; - } - - return NSQLTranslationV0::MakeLexer(); - case 1: - return NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer); - default: - issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion)); - return {}; - } - } - + ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings, ui16* actualSyntaxVersion) + { + TTranslationSettings parsedSettings(settings); + if (!ParseTranslationSettings(query, parsedSettings, issues)) { + return {}; + } + + if (actualSyntaxVersion) { + *actualSyntaxVersion = parsedSettings.SyntaxVersion; + } + + switch (parsedSettings.SyntaxVersion) { + case 0: + if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "V0 syntax is disabled")); + return {}; + } + + if (parsedSettings.AnsiLexer) { + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "Ansi lexer is not supported in V0 syntax")); + return {}; + } + + return NSQLTranslationV0::MakeLexer(); + case 1: + return NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer); + default: + issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion)); + return {}; + } + } + NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TTranslationSettings& settings) { NYql::TAstParseResult result; switch (settings.SyntaxVersion) { case 0: - if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { + if (settings.V0ForceDisable || settings.V0Behavior == EV0Behavior::Disable) { result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, "V0 syntax is disabled")); return result; } - if (settings.AnsiLexer) { - result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, - "Ansi lexer is not supported in V0 syntax")); - return result; - } - + if (settings.AnsiLexer) { + result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, + "Ansi lexer is not supported in V0 syntax")); + return result; + } + return NSQLTranslationV0::SqlASTToYql(protoAst, settings); case 1: return NSQLTranslationV1::SqlASTToYql(protoAst, settings); diff --git a/ydb/library/yql/sql/sql.h b/ydb/library/yql/sql/sql.h index 6ca7dcd32c..a2d2d0d391 100644 --- a/ydb/library/yql/sql/sql.h +++ b/ydb/library/yql/sql/sql.h @@ -18,7 +18,7 @@ namespace NSQLTranslation { NYql::TWarningRules* warningRules = nullptr, ui16* actualSyntaxVersion = nullptr); google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); - ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); + ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TTranslationSettings& settings); } // namespace NSQLTranslationV0 diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g index 93b76f88ce..3b616b9c9f 100644 --- a/ydb/library/yql/sql/v0/SQL.g +++ b/ydb/library/yql/sql/v0/SQL.g @@ -49,8 +49,8 @@ xor_subexpr: eq_subexpr cond_expr?; cond_expr: NOT? match_op eq_subexpr (ESCAPE eq_subexpr)? - | NOT? IN COMPACT? in_expr - | (ISNULL | NOTNULL | IS NULL | (IS)? NOT NULL) + | NOT? IN COMPACT? in_expr + | (ISNULL | NOTNULL | IS NULL | (IS)? NOT NULL) | NOT? BETWEEN eq_subexpr AND eq_subexpr | ((EQUALS | EQUALS2 | NOT_EQUALS | NOT_EQUALS2) eq_subexpr)+ /* order of the eq subexpressions is reversed! */ ; @@ -74,41 +74,41 @@ unary_op: PLUS | MINUS | TILDA | NOT; unary_subexpr: (id_expr | atom_expr ) key_expr* (DOT (bind_parameter | DIGITS | id_or_string) key_expr*)* (COLLATE id)?; in_unary_subexpr: (in_id_expr | in_atom_expr) key_expr* (DOT (bind_parameter | DIGITS | id_or_string) key_expr*)* (COLLATE id)?; - + atom_expr: literal_value | bind_parameter | window_function | lambda - | cast_expr - | exists_expr - | case_expr - | id_or_string NAMESPACE id_or_string + | cast_expr + | exists_expr + | case_expr + | id_or_string NAMESPACE id_or_string | bitcast_expr ; -in_atom_expr: - literal_value - | bind_parameter - | in_window_function - | smart_parenthesis - | cast_expr - | case_expr - | LPAREN select_stmt RPAREN +in_atom_expr: + literal_value + | bind_parameter + | in_window_function + | smart_parenthesis + | cast_expr + | case_expr + | LPAREN select_stmt RPAREN | bitcast_expr -; - -cast_expr: CAST LPAREN expr AS type_name RPAREN; - +; + +cast_expr: CAST LPAREN expr AS type_name RPAREN; + bitcast_expr: BITCAST LPAREN expr AS type_name RPAREN; -exists_expr: EXISTS LPAREN select_stmt RPAREN; - -case_expr: CASE expr? when_expr+ (ELSE expr)? END; - +exists_expr: EXISTS LPAREN select_stmt RPAREN; + +case_expr: CASE expr? when_expr+ (ELSE expr)? END; + lambda: smart_parenthesis (ARROW LBRACE_CURLY lambda_body RBRACE_CURLY)?; -in_expr: in_unary_subexpr; +in_expr: in_unary_subexpr; // struct, tuple or named list smart_parenthesis: LPAREN named_expr_list? COMMA? RPAREN; @@ -134,8 +134,8 @@ named_column_list: named_column (COMMA named_column)*; call_expr: ((id_or_string NAMESPACE id_or_string) | id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; -in_call_expr: ((id_or_string NAMESPACE id_or_string) | in_id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; - +in_call_expr: ((id_or_string NAMESPACE id_or_string) | in_id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; + key_expr: LBRACE_CURLY expr RBRACE_CURLY; when_expr: WHEN expr THEN expr; @@ -345,8 +345,8 @@ set_target_list: LPAREN set_target (COMMA set_target)* RPAREN; // differ from 2003 for resolve conflict window_function: call_expr (null_treatment? OVER window_name_or_specification)?; -in_window_function: in_call_expr (null_treatment? OVER window_name_or_specification)?; - +in_window_function: in_call_expr (null_treatment? OVER window_name_or_specification)?; + null_treatment: RESPECT NULLS | IGNORE NULLS; window_name_or_specification: window_name | in_line_window_specification; @@ -412,8 +412,8 @@ rollback_stmt: ROLLBACK; id_or_string: IDENTIFIER | STRING | keyword; id: IDENTIFIER | keyword; id_schema: IDENTIFIER | keyword_restricted; -id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat | keyword_in_uncompat; -in_id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat; +id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat | keyword_in_uncompat; +in_id_expr: IDENTIFIER | keyword_compat | keyword_alter_uncompat; id_table: IDENTIFIER | keyword_restricted; id_table_or_at: AT? id_table; id_or_at: AT? id; @@ -421,7 +421,7 @@ id_or_at: AT? id; opt_id_prefix: (id_or_string DOT)?; keyword: keyword_restricted | keyword_alter_uncompat | keyword_table_uncompat; -keyword_restricted: keyword_compat | keyword_expr_uncompat | keyword_select_uncompat | keyword_in_uncompat; +keyword_restricted: keyword_compat | keyword_expr_uncompat | keyword_select_uncompat | keyword_in_uncompat; keyword_expr_uncompat: BITCAST @@ -463,10 +463,10 @@ keyword_alter_uncompat: COLUMN ; -keyword_in_uncompat: - COMPACT -; - +keyword_in_uncompat: + COMPACT +; + keyword_compat: ( ABORT | ACTION @@ -723,7 +723,7 @@ COLLATE: C O L L A T E; COLUMN: C O L U M N; COLUMNS: C O L U M N S; COMMIT: C O M M I T; -COMPACT: C O M P A C T; +COMPACT: C O M P A C T; CONFLICT: C O N F L I C T; CONSTRAINT: C O N S T R A I N T; CREATE: C R E A T E; @@ -879,13 +879,13 @@ XOR: X O R; TRUE: T R U E; FALSE: F A L S E; -fragment STRING_CORE_SINGLE: ( ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) )*; -fragment STRING_CORE_DOUBLE: ( ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) )*; +fragment STRING_CORE_SINGLE: ( ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) )*; +fragment STRING_CORE_DOUBLE: ( ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) )*; fragment STRING_SINGLE: (QUOTE_SINGLE STRING_CORE_SINGLE QUOTE_SINGLE); fragment STRING_DOUBLE: (QUOTE_DOUBLE STRING_CORE_DOUBLE QUOTE_DOUBLE); -fragment STRING_MULTILINE: (DOUBLE_AT .* DOUBLE_AT)+ AT?; +fragment STRING_MULTILINE: (DOUBLE_AT .* DOUBLE_AT)+ AT?; -STRING: (STRING_SINGLE | STRING_DOUBLE | STRING_MULTILINE); +STRING: (STRING_SINGLE | STRING_DOUBLE | STRING_MULTILINE); fragment ID_START: ('a'..'z' | 'A'..'Z' | UNDERSCORE); fragment ID_CORE: (ID_START | DIGIT | DOLLAR); @@ -910,9 +910,9 @@ fragment BINDIGITS: '0' B ('0' | '1')+; fragment DECDIGITS: DIGIT+; DIGITS: DECDIGITS | HEXDIGITS | OCTDIGITS | BINDIGITS; -INTEGER: DIGITS (U? (L | S | T)?); -LEGACY_TINY_INTEGER: DIGITS (U? B?); -integer: DIGITS | INTEGER | LEGACY_TINY_INTEGER; +INTEGER: DIGITS (U? (L | S | T)?); +LEGACY_TINY_INTEGER: DIGITS (U? B?); +integer: DIGITS | INTEGER | LEGACY_TINY_INTEGER; fragment FLOAT_EXP : E (PLUS | MINUS)? DECDIGITS ; REAL: @@ -925,7 +925,7 @@ real: REAL; BLOB: X QUOTE_SINGLE HEXDIGIT+ QUOTE_SINGLE; -fragment MULTILINE_COMMENT: '/*' ( options {greedy=false;} : . )* '*/'; +fragment MULTILINE_COMMENT: '/*' ( options {greedy=false;} : . )* '*/'; fragment LINE_COMMENT: '--' ~('\n'|'\r')* ('\n' | '\r' | EOF); -WS: (' '|'\r'|'\t'|'\u000C'|'\n') {$channel=HIDDEN;}; -COMMENT: (MULTILINE_COMMENT|LINE_COMMENT) {$channel=HIDDEN;}; +WS: (' '|'\r'|'\t'|'\u000C'|'\n') {$channel=HIDDEN;}; +COMMENT: (MULTILINE_COMMENT|LINE_COMMENT) {$channel=HIDDEN;}; diff --git a/ydb/library/yql/sql/v0/aggregation.cpp b/ydb/library/yql/sql/v0/aggregation.cpp index 54b7ab8274..6440604881 100644 --- a/ydb/library/yql/sql/v0/aggregation.cpp +++ b/ydb/library/yql/sql/v0/aggregation.cpp @@ -94,8 +94,8 @@ protected: return false; } DistinctKey = *column; - YQL_ENSURE(src); - if (src->GetJoin()) { + YQL_ENSURE(src); + if (src->GetJoin()) { const auto sourcePtr = Expr->GetSourceName(); if (!sourcePtr || !*sourcePtr) { if (!src->IsGroupByColumn(DistinctKey)) { @@ -131,8 +131,8 @@ protected: Y("NthArg", Q("1"), "x"), Y("NthArg", Q("2"), "x"), BuildLambda(Pos, Y("value", "state"), Y("Void")), - Y("NthArg", Q("6"), "x"), - Y("NthArg", Q("7"), "x") + Y("NthArg", Q("6"), "x"), + Y("NthArg", Q("7"), "x") )) )))); } @@ -729,7 +729,7 @@ private: TSourcePtr FakeSource; std::multimap<TString, TNodePtr> Percentiles; TNodePtr FactoryPercentile; - const TString* Column = nullptr; + const TString* Column = nullptr; }; TAggregationPtr BuildPercentileFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp index 62b3fba64e..a0945f9d05 100644 --- a/ydb/library/yql/sql/v0/builtin.cpp +++ b/ydb/library/yql/sql/v0/builtin.cpp @@ -1065,7 +1065,7 @@ private: class TYqlIn final: public TCallNode { public: TYqlIn(TPosition pos, const TVector<TNodePtr>& args) - : TCallNode(pos, "IN", 3, 3, args) + : TCallNode(pos, "IN", 3, 3, args) {} private: @@ -1077,53 +1077,53 @@ private: return false; } - auto key = Args[0]; + auto key = Args[0]; auto inNode = Args[1]; - auto hints = Args[2]; - - if (!key->Init(ctx, src)) { - return false; - } - - if (!inNode->Init(ctx, inNode->GetSource() ? nullptr : src)) { - return false; - } - + auto hints = Args[2]; + + if (!key->Init(ctx, src)) { + return false; + } + + if (!inNode->Init(ctx, inNode->GetSource() ? nullptr : src)) { + return false; + } + if (inNode->GetLiteral("String")) { - ctx.Error(inNode->GetPos()) << "Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source"; + ctx.Error(inNode->GetPos()) << "Unable to use IN predicate with string argument, it won't search substring - " + "expecting tuple, list, dict or single column table source"; return false; } - - if (inNode->GetTupleSize() == 1) { - auto singleElement = inNode->GetTupleElement(0); - // TODO: 'IN ((select ...))' is parsed exactly like 'IN (select ...)' instead of a single element tuple - if (singleElement->GetSource() || singleElement->IsSelect()) { - TStringBuf parenKind = singleElement->GetSource() ? "" : "external "; - ctx.Warning(inNode->GetPos(), - TIssuesIds::YQL_CONST_SUBREQUEST_IN_LIST) << "Using subrequest in scalar context after IN, " - << "perhaps you should remove " - << parenKind << "parenthesis here"; - } - } - - if (inNode->GetSource() || inNode->IsSelect()) { - TVector<TNodePtr> hintElements; - for (size_t i = 0; i < hints->GetTupleSize(); ++i) { - hintElements.push_back(hints->GetTupleElement(i)); - } - auto pos = inNode->GetPos(); - auto tableSourceHint = BuildTuple(pos, { BuildQuotedAtom(pos, "tableSource", NYql::TNodeFlags::Default) }); - hintElements.push_back(tableSourceHint); - hints = BuildTuple(pos, hintElements); - } - - OpName = "SqlIn"; - MinArgs = MaxArgs = 3; + + if (inNode->GetTupleSize() == 1) { + auto singleElement = inNode->GetTupleElement(0); + // TODO: 'IN ((select ...))' is parsed exactly like 'IN (select ...)' instead of a single element tuple + if (singleElement->GetSource() || singleElement->IsSelect()) { + TStringBuf parenKind = singleElement->GetSource() ? "" : "external "; + ctx.Warning(inNode->GetPos(), + TIssuesIds::YQL_CONST_SUBREQUEST_IN_LIST) << "Using subrequest in scalar context after IN, " + << "perhaps you should remove " + << parenKind << "parenthesis here"; + } + } + + if (inNode->GetSource() || inNode->IsSelect()) { + TVector<TNodePtr> hintElements; + for (size_t i = 0; i < hints->GetTupleSize(); ++i) { + hintElements.push_back(hints->GetTupleElement(i)); + } + auto pos = inNode->GetPos(); + auto tableSourceHint = BuildTuple(pos, { BuildQuotedAtom(pos, "tableSource", NYql::TNodeFlags::Default) }); + hintElements.push_back(tableSourceHint); + hints = BuildTuple(pos, hintElements); + } + + OpName = "SqlIn"; + MinArgs = MaxArgs = 3; Args = { - inNode->GetSource() ? inNode->GetSource() : inNode, + inNode->GetSource() ? inNode->GetSource() : inNode, key, - hints + hints }; return TCallNode::DoInit(ctx, src); @@ -1388,15 +1388,15 @@ public: bool DoInit(TContext& ctx, ISource* src) override { if (Module == "yql") { ui32 flags; - TString nameParseError; - TPosition pos = Pos; - TString parsedName; - if (!TryStringContent(Name, parsedName, flags, nameParseError, pos)) { - ctx.Error(pos) << "Failed to parse YQL: " << nameParseError; - return false; - } - - const TString yql("(" + parsedName + ")"); + TString nameParseError; + TPosition pos = Pos; + TString parsedName; + if (!TryStringContent(Name, parsedName, flags, nameParseError, pos)) { + ctx.Error(pos) << "Failed to parse YQL: " << nameParseError; + return false; + } + + const TString yql("(" + parsedName + ")"); TAstParseResult ast = ParseAst(yql, ctx.Pool.get()); /// TODO: do not drop warnings if (ast.IsOk()) { @@ -1436,7 +1436,7 @@ public: } void DoUpdateState() const override { - YQL_ENSURE(Node); + YQL_ENSURE(Node); State.Set(ENodeState::Const, Node->IsConstant()); State.Set(ENodeState::Aggregated, Node->IsAggregated()); } @@ -2236,32 +2236,32 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec positionalArgs = args[0]; namedArgs = args[1]; *mustUseNamed = false; - } - - TVector<TNodePtr> reuseArgs; - if (!namedArgs && args && funcPrepareNameNode) { - TString reusedBaseName = TStringBuilder() << "Arg" << to_title(nameSpace) << to_title(name); - reuseArgs.reserve(args.size()); - for (const auto& arg: args) { - reuseArgs.push_back(funcPrepareNameNode(reusedBaseName, arg)); - } - } - - auto usedArgs = reuseArgs ? reuseArgs : args; - - TVector<TNodePtr> positionalArgsElements; - - if (namedArgs) { - auto positionalArgsTuple = dynamic_cast<TTupleNode*>(positionalArgs.Get()); - Y_VERIFY_DEBUG(positionalArgsTuple, "unexpected value at String::SplitToList positional args"); - positionalArgsElements = positionalArgsTuple->Elements(); + } + + TVector<TNodePtr> reuseArgs; + if (!namedArgs && args && funcPrepareNameNode) { + TString reusedBaseName = TStringBuilder() << "Arg" << to_title(nameSpace) << to_title(name); + reuseArgs.reserve(args.size()); + for (const auto& arg: args) { + reuseArgs.push_back(funcPrepareNameNode(reusedBaseName, arg)); + } + } + + auto usedArgs = reuseArgs ? reuseArgs : args; + + TVector<TNodePtr> positionalArgsElements; + + if (namedArgs) { + auto positionalArgsTuple = dynamic_cast<TTupleNode*>(positionalArgs.Get()); + Y_VERIFY_DEBUG(positionalArgsTuple, "unexpected value at String::SplitToList positional args"); + positionalArgsElements = positionalArgsTuple->Elements(); } else { - positionalArgsElements = usedArgs; + positionalArgsElements = usedArgs; } - auto positionalArgsTupleSize = positionalArgsElements.size(); - auto argsSize = positionalArgsTupleSize; - + auto positionalArgsTupleSize = positionalArgsElements.size(); + auto argsSize = positionalArgsTupleSize; + TNodePtr trueLiteral = BuildLiteralBool(pos, "true"); TNodePtr falseLiteral = BuildLiteralBool(pos, "false"); TNodePtr namedDelimeterStringArg; @@ -2285,18 +2285,18 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec if (argsSize < 4 && !hasDelimeterString) { positionalArgsElements.push_back(trueLiteral); } - - if (namedArgs) { - positionalArgs = BuildTuple(pos, positionalArgsElements); - } else { - usedArgs = positionalArgsElements; + + if (namedArgs) { + positionalArgs = BuildTuple(pos, positionalArgsElements); + } else { + usedArgs = positionalArgsElements; } TNodePtr customUserType = nullptr; - const auto& udfArgs = BuildUdfArgs(ctx, pos, usedArgs, positionalArgs, namedArgs, customUserType); - TNodePtr udfNode = BuildUdf(ctx, pos, nameSpace, name, udfArgs); - TVector<TNodePtr> applyArgs = { udfNode }; - applyArgs.insert(applyArgs.end(), usedArgs.begin(), usedArgs.end()); + const auto& udfArgs = BuildUdfArgs(ctx, pos, usedArgs, positionalArgs, namedArgs, customUserType); + TNodePtr udfNode = BuildUdf(ctx, pos, nameSpace, name, udfArgs); + TVector<TNodePtr> applyArgs = { udfNode }; + applyArgs.insert(applyArgs.end(), usedArgs.begin(), usedArgs.end()); return new TCallNodeImpl(pos, namedArgs ? "NamedApply" : "Apply", applyArgs); } else if (moduleResource) { auto exportName = ns == "core" ? name : "$" + name; @@ -2427,11 +2427,11 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec (normalizedName == "asstruct" ? "AsStruct" : "StructType") << " requires all argument to be named"); } else if (normalizedName == "expandstruct") { - if (mustUseNamed) { - if (!*mustUseNamed) { - return new TInvalidBuiltin(pos, TStringBuilder() << "ExpandStruct requires at least one named argument"); - } - *mustUseNamed = false; + if (mustUseNamed) { + if (!*mustUseNamed) { + return new TInvalidBuiltin(pos, TStringBuilder() << "ExpandStruct requires at least one named argument"); + } + *mustUseNamed = false; } YQL_ENSURE(args.size() == 2); auto posArgs = static_cast<TTupleNode*>(args[0].Get()); diff --git a/ydb/library/yql/sql/v0/context.cpp b/ydb/library/yql/sql/v0/context.cpp index e2bc9157ad..a3a9013f2b 100644 --- a/ydb/library/yql/sql/v0/context.cpp +++ b/ydb/library/yql/sql/v0/context.cpp @@ -39,7 +39,7 @@ TNodePtr AddTablePathPrefix(TContext &ctx, TStringBuf prefixPath, const TDeferre typedef bool TContext::*TPragmaField; THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { - {"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin}, + {"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin}, }; } // namespace @@ -67,9 +67,9 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, value = false; ptr = CTX_PRAGMA_FIELDS.FindPtr(key); } - if (ptr) { - this->*(*ptr) = value; - } + if (ptr) { + this->*(*ptr) = value; + } } } @@ -111,12 +111,12 @@ IOutputStream& TContext::Info(NYql::TPosition pos) { IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TPosition pos) { if (severity == TSeverityIds::S_WARNING) { - auto action = WarningPolicy.GetAction(code); - if (action == EWarningAction::ERROR) { - severity = TSeverityIds::S_ERROR; - HasPendingErrors = true; - } else if (action == EWarningAction::DISABLE) { - return Cnull; + auto action = WarningPolicy.GetAction(code); + if (action == EWarningAction::ERROR) { + severity = TSeverityIds::S_ERROR; + HasPendingErrors = true; + } else if (action == EWarningAction::DISABLE) { + return Cnull; } } diff --git a/ydb/library/yql/sql/v0/context.h b/ydb/library/yql/sql/v0/context.h index 2c7b9b7ae8..45f40fae61 100644 --- a/ydb/library/yql/sql/v0/context.h +++ b/ydb/library/yql/sql/v0/context.h @@ -70,10 +70,10 @@ namespace NSQLTranslationV0 { TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); if (!provider) { - if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { + if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { normalizedClusterName = cluster; return TString(NYql::KikimrProviderName); - } + } return Nothing(); } @@ -148,14 +148,14 @@ namespace NSQLTranslationV0 { bool PragmaYsonAutoConvert = false; bool PragmaYsonStrict = false; bool PragmaClassicDivision = true; - bool PragmaPullUpFlatMapOverJoin = true; + bool PragmaPullUpFlatMapOverJoin = true; bool EnableSystemColumns = true; ui32 ResultRowsLimit = 0; ui64 ResultSizeLimit = 0; ui32 PragmaGroupByLimit = 1 << 5; ui32 PragmaGroupByCubeLimit = 5; THashSet<TString> Libraries; - NYql::TWarningPolicy WarningPolicy; + NYql::TWarningPolicy WarningPolicy; TVector<TString> AllResults; TSet<TString> UsedClusters; diff --git a/ydb/library/yql/sql/v0/join.cpp b/ydb/library/yql/sql/v0/join.cpp index 33ac954101..78f5affb49 100644 --- a/ydb/library/yql/sql/v0/join.cpp +++ b/ydb/library/yql/sql/v0/join.cpp @@ -176,9 +176,9 @@ public: expr = eq; } } - if (expr && Sources.size() > 2) { - ctx.Warning(ctx.Pos(), TIssuesIds::YQL_MULTIWAY_JOIN_WITH_USING) << "Multi-way JOINs should be connected with ON clause instead of USING clause"; - } + if (expr && Sources.size() > 2) { + ctx.Warning(ctx.Pos(), TIssuesIds::YQL_MULTIWAY_JOIN_WITH_USING) << "Multi-way JOINs should be connected with ON clause instead of USING clause"; + } return expr; } diff --git a/ydb/library/yql/sql/v0/lexer/lexer.cpp b/ydb/library/yql/sql/v0/lexer/lexer.cpp index e37c1423d1..698a1d4257 100644 --- a/ydb/library/yql/sql/v0/lexer/lexer.cpp +++ b/ydb/library/yql/sql/v0/lexer/lexer.cpp @@ -1,48 +1,48 @@ -#include "lexer.h" - +#include "lexer.h" + #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h> #include <ydb/library/yql/parser/proto_ast/gen/v0/SQLLexer.h> - -#if defined(_tsan_enabled_) -#include <util/system/mutex.h> -#endif - -namespace NALP { -extern ANTLR_UINT8* SQLParserTokenNames[]; -} - -namespace NSQLTranslationV0 { - -namespace { - -#if defined(_tsan_enabled_) -TMutex SanitizerSQLTranslationMutex; -#endif - -using NSQLTranslation::ILexer; -using NSQLTranslation::TParsedTokenList; - -class TV0Lexer : public ILexer { -public: - TV0Lexer() = default; - - bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override { - issues.Clear(); -#if defined(_tsan_enabled_) - TGuard<TMutex> grd(SanitizerSQLTranslationMutex); -#endif - NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, ""); - NProtoAST::TLexerTokensCollector<NALP::SQLLexer> tokensCollector(query, (const char**)NALP::SQLParserTokenNames, queryName); - tokens = tokensCollector.CollectTokens(collector); - return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); - } -}; - -} // namespace - -NSQLTranslation::ILexer::TPtr MakeLexer() { - return NSQLTranslation::ILexer::TPtr(new TV0Lexer); -} - -} // namespace NSQLTranslationV1 + +#if defined(_tsan_enabled_) +#include <util/system/mutex.h> +#endif + +namespace NALP { +extern ANTLR_UINT8* SQLParserTokenNames[]; +} + +namespace NSQLTranslationV0 { + +namespace { + +#if defined(_tsan_enabled_) +TMutex SanitizerSQLTranslationMutex; +#endif + +using NSQLTranslation::ILexer; +using NSQLTranslation::TParsedTokenList; + +class TV0Lexer : public ILexer { +public: + TV0Lexer() = default; + + bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override { + issues.Clear(); +#if defined(_tsan_enabled_) + TGuard<TMutex> grd(SanitizerSQLTranslationMutex); +#endif + NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, ""); + NProtoAST::TLexerTokensCollector<NALP::SQLLexer> tokensCollector(query, (const char**)NALP::SQLParserTokenNames, queryName); + tokens = tokensCollector.CollectTokens(collector); + return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } +}; + +} // namespace + +NSQLTranslation::ILexer::TPtr MakeLexer() { + return NSQLTranslation::ILexer::TPtr(new TV0Lexer); +} + +} // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v0/lexer/lexer.h b/ydb/library/yql/sql/v0/lexer/lexer.h index 0a9cca1623..b7f8f50906 100644 --- a/ydb/library/yql/sql/v0/lexer/lexer.h +++ b/ydb/library/yql/sql/v0/lexer/lexer.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/parser/lexer_common/lexer.h> - -namespace NSQLTranslationV0 { - -NSQLTranslation::ILexer::TPtr MakeLexer(); - -} + +namespace NSQLTranslationV0 { + +NSQLTranslation::ILexer::TPtr MakeLexer(); + +} diff --git a/ydb/library/yql/sql/v0/lexer/ya.make b/ydb/library/yql/sql/v0/lexer/ya.make index 99e3662c59..f1a45a10f5 100644 --- a/ydb/library/yql/sql/v0/lexer/ya.make +++ b/ydb/library/yql/sql/v0/lexer/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( g:yql g:yql_ydb_core -) - -PEERDIR( +) + +PEERDIR( ydb/library/yql/core/issue/protos ydb/library/yql/parser/proto_ast ydb/library/yql/parser/proto_ast/gen/v0 -) - -SRCS( - lexer.cpp -) - -END() +) + +SRCS( + lexer.cpp +) + +END() diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp index 1f5a01dd0a..57f0c6a561 100644 --- a/ydb/library/yql/sql/v0/node.cpp +++ b/ydb/library/yql/sql/v0/node.cpp @@ -1342,7 +1342,7 @@ bool ISource::SetSamplingOptions(TContext& ctx, Y_UNUSED(mode); Y_UNUSED(samplingRate); Y_UNUSED(samplingSeed); - ctx.Error() << "Sampling is only supported for table sources"; + ctx.Error() << "Sampling is only supported for table sources"; return false; } @@ -1484,7 +1484,7 @@ struct TWinFrame { struct TWinPartition { TString ParentLabel; - size_t Id = 0; + size_t Id = 0; TVector<size_t> FrameIds; TVector<TSortSpecificationPtr> OrderBy; TVector<TNodePtr> Partitions; @@ -1752,54 +1752,54 @@ IJoin* IJoin::GetJoin() { return this; } -bool TryStringContent(const TString& str, TString& result, ui32& flags, TString& error, TPosition& pos) { - error.clear(); - result.clear(); - +bool TryStringContent(const TString& str, TString& result, ui32& flags, TString& error, TPosition& pos) { + error.clear(); + result.clear(); + bool doubleQuoted = (str.StartsWith('"') && str.EndsWith('"')); bool singleQuoted = !doubleQuoted && (str.StartsWith('\'') && str.EndsWith('\'')); if (str.size() >= 2 && (doubleQuoted || singleQuoted)) { - flags = TNodeFlags::ArbitraryContent; + flags = TNodeFlags::ArbitraryContent; char quoteChar = doubleQuoted ? '"' : '\''; size_t readBytes = 0; TStringBuf atom(str); - TStringOutput sout(result); + TStringOutput sout(result); atom.Skip(1); - result.reserve(str.size()); - - auto unescapeResult = UnescapeArbitraryAtom(atom, quoteChar, &sout, &readBytes); - if (unescapeResult != EUnescapeResult::OK) { - TTextWalker walker(pos); - walker.Advance(atom.Trunc(readBytes)); - error = UnescapeResultToString(unescapeResult); - return false; - } + result.reserve(str.size()); + + auto unescapeResult = UnescapeArbitraryAtom(atom, quoteChar, &sout, &readBytes); + if (unescapeResult != EUnescapeResult::OK) { + TTextWalker walker(pos); + walker.Advance(atom.Trunc(readBytes)); + error = UnescapeResultToString(unescapeResult); + return false; + } } else if (str.size() >= 4 && str.StartsWith("@@") && str.EndsWith("@@")) { flags = TNodeFlags::MultilineContent; TString s = str.substr(2, str.length() - 4); SubstGlobal(s, "@@@@", "@@"); - result.swap(s); + result.swap(s); } else { flags = TNodeFlags::Default; - result = str; - } - return true; -} - -TString StringContent(TContext& ctx, const TString& str) { - ui32 flags = 0; - TString result; - TString error; - TPosition pos; - - if (!TryStringContent(str, result, flags, error, pos)) { - ctx.Error(pos) << "Failed to parse string literal: " << error; - return {}; - } - return result; -} - + result = str; + } + return true; +} + +TString StringContent(TContext& ctx, const TString& str) { + ui32 flags = 0; + TString result; + TString error; + TPosition pos; + + if (!TryStringContent(str, result, flags, error, pos)) { + ctx.Error(pos) << "Failed to parse string literal: " << error; + return {}; + } + return result; +} + TString IdContent(TContext& ctx, const TString& s) { YQL_ENSURE(!s.empty(), "Empty identifier not expected"); if (!s.StartsWith('[') && !s.StartsWith('`')) { @@ -1833,17 +1833,17 @@ TString IdContent(TContext& ctx, const TString& s) { unescapedStr.reserve(s.size()); size_t readBytes = 0; - TPosition pos = ctx.Pos(); - pos.Column += skipSymbols - 1; - - auto unescapeResult = UnescapeArbitraryAtom(atom, endSym, &sout, &readBytes); - if (unescapeResult != EUnescapeResult::OK) { - TTextWalker walker(pos); - walker.Advance(atom.Trunc(readBytes)); - ctx.Error(pos) << "Cannot parse broken identifier: " << UnescapeResultToString(unescapeResult); + TPosition pos = ctx.Pos(); + pos.Column += skipSymbols - 1; + + auto unescapeResult = UnescapeArbitraryAtom(atom, endSym, &sout, &readBytes); + if (unescapeResult != EUnescapeResult::OK) { + TTextWalker walker(pos); + walker.Advance(atom.Trunc(readBytes)); + ctx.Error(pos) << "Cannot parse broken identifier: " << UnescapeResultToString(unescapeResult); return {}; } - + if (readBytes != atom.size()) { ctx.Error() << "The identifier not parsed completely"; return {}; @@ -1852,32 +1852,32 @@ TString IdContent(TContext& ctx, const TString& s) { return unescapedStr; } -namespace { -class TInvalidLiteralNode final: public INode { -public: - TInvalidLiteralNode(TPosition pos) - : INode(pos) - { - } - - bool DoInit(TContext& ctx, ISource* source) override { - Y_UNUSED(ctx); - Y_UNUSED(source); - return false; - } - - TAstNode* Translate(TContext& ctx) const override { - Y_UNUSED(ctx); - return nullptr; - } - - TPtr DoClone() const override { - return {}; - } -}; - -} - +namespace { +class TInvalidLiteralNode final: public INode { +public: + TInvalidLiteralNode(TPosition pos) + : INode(pos) + { + } + + bool DoInit(TContext& ctx, ISource* source) override { + Y_UNUSED(ctx); + Y_UNUSED(source); + return false; + } + + TAstNode* Translate(TContext& ctx) const override { + Y_UNUSED(ctx); + return nullptr; + } + + TPtr DoClone() const override { + return {}; + } +}; + +} + TLiteralNode::TLiteralNode(TPosition pos, bool isNull) : TAstListNode(pos) , Null(isNull) @@ -1896,14 +1896,14 @@ TLiteralNode::TLiteralNode(TPosition pos, const TString& type, const TString& va Add(Type, BuildQuotedAtom(Pos, Value)); } -TLiteralNode::TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags) +TLiteralNode::TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags) : TAstListNode(pos) , Null(false) , Void(false) , Type("String") - , Value(value) + , Value(value) { - Add(Type, BuildQuotedAtom(pos, Value, nodeFlags)); + Add(Type, BuildQuotedAtom(pos, Value, nodeFlags)); } bool TLiteralNode::IsNull() const { @@ -1969,19 +1969,19 @@ TNodePtr BuildLiteralVoid(TPosition pos) { return new TLiteralNode(pos, false); } -TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value) { - - TString unescaped; - TString error; - TPosition pos = ctx.Pos(); - ui32 flags = 0; - - if (TryStringContent(value, unescaped, flags, error, pos)) { - return new TLiteralNode(ctx.Pos(), unescaped, flags); - } else { - ctx.Error(pos) << "Failed to parse string literal: " << error; - return new TInvalidLiteralNode(ctx.Pos()); - } +TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value) { + + TString unescaped; + TString error; + TPosition pos = ctx.Pos(); + ui32 flags = 0; + + if (TryStringContent(value, unescaped, flags, error, pos)) { + return new TLiteralNode(ctx.Pos(), unescaped, flags); + } else { + ctx.Error(pos) << "Failed to parse string literal: " << error; + return new TInvalidLiteralNode(ctx.Pos()); + } } TNodePtr BuildLiteralRawString(TPosition pos, const TString& value) { @@ -2086,7 +2086,7 @@ bool TStructNode::DoInit(TContext& ctx, ISource* src) { ctx.Error(expr->GetPos()) << "Structure does not allow anonymous members"; return false; } - Nodes.push_back(Q(Y(BuildQuotedAtom(expr->GetPos(), label), expr))); + Nodes.push_back(Q(Y(BuildQuotedAtom(expr->GetPos(), label), expr))); } return TAstListNode::DoInit(ctx, src); } diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h index 8680698a79..4c7cfc273b 100644 --- a/ydb/library/yql/sql/v0/node.h +++ b/ydb/library/yql/sql/v0/node.h @@ -833,7 +833,7 @@ namespace NSQLTranslationV0 { public: TLiteralNode(TPosition pos, bool isNull); TLiteralNode(TPosition pos, const TString& type, const TString& value); - TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags); + TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags); bool IsNull() const override; const TString* GetLiteral(const TString& type) const override; void DoUpdateState() const override; @@ -861,8 +861,8 @@ namespace NSQLTranslationV0 { TString View; }; - TString StringContent(TContext& ctx, const TString& str); - bool TryStringContent(const TString& input, TString& result, ui32& flags, TString& error, TPosition& pos); + TString StringContent(TContext& ctx, const TString& str); + bool TryStringContent(const TString& input, TString& result, ui32& flags, TString& error, TPosition& pos); TString IdContent(TContext& ctx, const TString& str); TVector<TString> GetContextHints(TContext& ctx); @@ -874,7 +874,7 @@ namespace NSQLTranslationV0 { TNodePtr BuildLiteralNull(TPosition pos); TNodePtr BuildLiteralVoid(TPosition pos); /// String is checked as quotable, support escaping and multiline - TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value); + TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value); TNodePtr BuildLiteralRawString(TPosition pos, const TString& value); TNodePtr BuildLiteralBool(TPosition pos, const TString& value); TNodePtr BuildEmptyAction(TPosition pos); diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 99449597e3..7752b28b01 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -985,10 +985,10 @@ public: auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName))); auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName))); - for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) { + for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) { Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), - BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); + BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), + BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); } if (ctx.ResultSizeLimit > 0) { @@ -996,10 +996,10 @@ public: BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit))))); } - if (!ctx.PragmaPullUpFlatMapOverJoin) { + if (!ctx.PragmaPullUpFlatMapOverJoin) { Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin")))); - } + BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin")))); + } } } diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index 5e2477b1f2..b028fc0439 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -34,8 +34,8 @@ using namespace NYql; namespace NSQLTranslationV0 { -using NALP::SQLLexerTokens; - +using NALP::SQLLexerTokens; + #if defined(_tsan_enabled_) TMutex SanitizerSQLTranslationMutex; #endif @@ -60,8 +60,8 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword_restricte return GetIdentifier(ctx, node.GetAlt_keyword_restricted2().GetRule_keyword_expr_uncompat1()); case TRule_keyword_restricted::kAltKeywordRestricted3: return GetIdentifier(ctx, node.GetAlt_keyword_restricted3().GetRule_keyword_select_uncompat1()); - case TRule_keyword_restricted::kAltKeywordRestricted4: - return GetIdentifier(ctx, node.GetAlt_keyword_restricted4().GetRule_keyword_in_uncompat1()); + case TRule_keyword_restricted::kAltKeywordRestricted4: + return GetIdentifier(ctx, node.GetAlt_keyword_restricted4().GetRule_keyword_in_uncompat1()); default: Y_FAIL("You should change implementation according grammar changes"); } @@ -142,45 +142,45 @@ static TString Id(const TRule_id_expr& node, TTranslation& ctx) { return ctx.Token(node.GetAlt_id_expr2().GetRule_keyword_compat1().GetToken1()); case TRule_id_expr::kAltIdExpr3: return ctx.Token(node.GetAlt_id_expr3().GetRule_keyword_alter_uncompat1().GetToken1()); - case TRule_id_expr::kAltIdExpr4: - return ctx.Token(node.GetAlt_id_expr4().GetRule_keyword_in_uncompat1().GetToken1()); + case TRule_id_expr::kAltIdExpr4: + return ctx.Token(node.GetAlt_id_expr4().GetRule_keyword_in_uncompat1().GetToken1()); + default: + Y_FAIL("You should change implementation according grammar changes"); + } +} + +static TString Id(const TRule_in_id_expr& node, TTranslation& ctx) { + switch (node.Alt_case()) { + case TRule_in_id_expr::kAltInIdExpr1: + return ctx.Identifier(node.GetAlt_in_id_expr1().GetToken1()); + case TRule_in_id_expr::kAltInIdExpr2: + return ctx.Token(node.GetAlt_in_id_expr2().GetRule_keyword_compat1().GetToken1()); + case TRule_in_id_expr::kAltInIdExpr3: + return ctx.Token(node.GetAlt_in_id_expr3().GetRule_keyword_alter_uncompat1().GetToken1()); + default: + Y_FAIL("You should change implementation according grammar changes"); + } +} + +static TIdentifier IdEx(const TRule_id& node, TTranslation& ctx) { + switch (node.Alt_case()) { + case TRule_id::kAltId1: + return GetIdentifier(ctx, node.GetAlt_id1()); + case TRule_id::kAltId2: + return GetKeywordId(ctx, node.GetAlt_id2().GetRule_keyword1()); default: Y_FAIL("You should change implementation according grammar changes"); } } -static TString Id(const TRule_in_id_expr& node, TTranslation& ctx) { - switch (node.Alt_case()) { - case TRule_in_id_expr::kAltInIdExpr1: - return ctx.Identifier(node.GetAlt_in_id_expr1().GetToken1()); - case TRule_in_id_expr::kAltInIdExpr2: - return ctx.Token(node.GetAlt_in_id_expr2().GetRule_keyword_compat1().GetToken1()); - case TRule_in_id_expr::kAltInIdExpr3: - return ctx.Token(node.GetAlt_in_id_expr3().GetRule_keyword_alter_uncompat1().GetToken1()); - default: - Y_FAIL("You should change implementation according grammar changes"); - } -} - -static TIdentifier IdEx(const TRule_id& node, TTranslation& ctx) { - switch (node.Alt_case()) { - case TRule_id::kAltId1: - return GetIdentifier(ctx, node.GetAlt_id1()); - case TRule_id::kAltId2: - return GetKeywordId(ctx, node.GetAlt_id2().GetRule_keyword1()); - default: - Y_FAIL("You should change implementation according grammar changes"); - } -} - -static TString IdOrString(const TRule_id_or_string& node, TTranslation& ctx, bool rawString = false) { +static TString IdOrString(const TRule_id_or_string& node, TTranslation& ctx, bool rawString = false) { switch (node.Alt_case()) { case TRule_id_or_string::kAltIdOrString1: { return ctx.Identifier(node.GetAlt_id_or_string1().GetToken1()); } case TRule_id_or_string::kAltIdOrString2: { - auto raw = ctx.Token(node.GetAlt_id_or_string2().GetToken1()); - return rawString ? raw : StringContent(ctx.Context(), raw); + auto raw = ctx.Token(node.GetAlt_id_or_string2().GetToken1()); + return rawString ? raw : StringContent(ctx.Context(), raw); } case TRule_id_or_string::kAltIdOrString3: return GetKeyword(ctx, node.GetAlt_id_or_string3().GetRule_keyword1()); @@ -619,21 +619,21 @@ public: TNodePtr LiteralExpr(const TRule_literal_value& node); private: - template<typename TWindowFunctionRule> - TNodePtr WindowFunctionRule(const TWindowFunctionRule& rule); - TNodePtr BindParameterRule(const TRule_bind_parameter& rule); - TNodePtr LambdaRule(const TRule_lambda& rule); - TNodePtr CastRule(const TRule_cast_expr& rule); + template<typename TWindowFunctionRule> + TNodePtr WindowFunctionRule(const TWindowFunctionRule& rule); + TNodePtr BindParameterRule(const TRule_bind_parameter& rule); + TNodePtr LambdaRule(const TRule_lambda& rule); + TNodePtr CastRule(const TRule_cast_expr& rule); TNodePtr BitCastRule(const TRule_bitcast_expr& rule); - TNodePtr ExistsRule(const TRule_exists_expr& rule); - TNodePtr CaseRule(const TRule_case_expr& rule); - + TNodePtr ExistsRule(const TRule_exists_expr& rule); + TNodePtr CaseRule(const TRule_case_expr& rule); + TNodePtr AtomExpr(const TRule_atom_expr& node); - TNodePtr InAtomExpr(const TRule_in_atom_expr& node); + TNodePtr InAtomExpr(const TRule_in_atom_expr& node); + + template<typename TUnarySubExprRule> + TNodePtr UnaryExpr(const TUnarySubExprRule& node); - template<typename TUnarySubExprRule> - TNodePtr UnaryExpr(const TUnarySubExprRule& node); - bool SqlLambdaParams(const TNodePtr& node, TVector<TString>& args); bool SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& node, TVector<TNodePtr>& exprSeq); @@ -737,8 +737,8 @@ public: { } - template<typename TCallExprRule> - bool Init(const TCallExprRule& node); + template<typename TCallExprRule> + bool Init(const TCallExprRule& node); void IncCounters(); TNodePtr BuildUdf(bool withArgsType) { @@ -814,7 +814,7 @@ private: TVector<TNodePtr> PositionalArgs; TVector<TNodePtr> NamedArgs; EAggregateMode AggMode = EAggregateMode::Normal; - TSqlExpression* UsedExpr = nullptr; + TSqlExpression* UsedExpr = nullptr; }; TNodePtr TSqlTranslation::NamedExpr(const TRule_named_expr& node, EExpr exprMode) { @@ -1022,40 +1022,40 @@ bool ExprList(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule return true; } -template<typename TCallExprRule> -bool TSqlCallExpr::Init(const TCallExprRule& node) { - // call_expr: ((id_or_string NAMESPACE id_or_string) | id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; - // OR - // in_call_expr: ((id_or_string NAMESPACE id_or_string) | in_id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; +template<typename TCallExprRule> +bool TSqlCallExpr::Init(const TCallExprRule& node) { + // call_expr: ((id_or_string NAMESPACE id_or_string) | id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; + // OR + // in_call_expr: ((id_or_string NAMESPACE id_or_string) | in_id_expr | bind_parameter) LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN; const auto& block = node.GetBlock1(); switch (block.Alt_case()) { - case TCallExprRule::TBlock1::kAlt1: { + case TCallExprRule::TBlock1::kAlt1: { auto& subblock = block.GetAlt1().GetBlock1(); Module = IdOrString(subblock.GetRule_id_or_string1(), *this); Func = IdOrString(subblock.GetRule_id_or_string3(), *this); break; } - case TCallExprRule::TBlock1::kAlt2: { - if constexpr (std::is_same_v<TCallExprRule, TRule_call_expr>) { - Func = Id(block.GetAlt2().GetRule_id_expr1(), *this); - } else { - Func = Id(block.GetAlt2().GetRule_in_id_expr1(), *this); - } + case TCallExprRule::TBlock1::kAlt2: { + if constexpr (std::is_same_v<TCallExprRule, TRule_call_expr>) { + Func = Id(block.GetAlt2().GetRule_id_expr1(), *this); + } else { + Func = Id(block.GetAlt2().GetRule_in_id_expr1(), *this); + } break; - } - case TCallExprRule::TBlock1::kAlt3: + } + case TCallExprRule::TBlock1::kAlt3: Node = GetNamedNode(NamedNodeImpl(block.GetAlt3().GetRule_bind_parameter1(), *this)); if (!Node) { return false; } break; - default: - Y_FAIL("You should change implementation according grammar changes"); + default: + Y_FAIL("You should change implementation according grammar changes"); } Pos = Ctx.Pos(); if (node.HasBlock3()) { switch (node.GetBlock3().Alt_case()) { - case TCallExprRule::TBlock3::kAlt1: { + case TCallExprRule::TBlock3::kAlt1: { const auto& alt = node.GetBlock3().GetAlt1(); if (IsDistinctOptSet(alt.GetRule_opt_set_quantifier1())) { YQL_ENSURE(AggMode == EAggregateMode::Normal); @@ -1078,11 +1078,11 @@ bool TSqlCallExpr::Init(const TCallExprRule& node) { } break; } - case TCallExprRule::TBlock3::kAlt2: + case TCallExprRule::TBlock3::kAlt2: Args.push_back(BuildColumn(Pos, "*")); break; - default: - Y_FAIL("You should change implementation according grammar changes"); + default: + Y_FAIL("You should change implementation according grammar changes"); } } return true; @@ -1225,7 +1225,7 @@ bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& s } if (strLen > 1) { auto iter = str.cend() - 1; - if (*iter == 'l' || *iter == 's' || *iter == 't' || /* deprecated */ *iter == 'b') { + if (*iter == 'l' || *iter == 's' || *iter == 't' || /* deprecated */ *iter == 'b') { --iter; } if (*iter == 'u') { @@ -1237,7 +1237,7 @@ bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& s const TString digString(str.begin() + (base == 10 ? 0 : 2), str.end() - suffix.size()); for (const char& cur: digString) { const ui64 curDigit = Char2DigitTable[static_cast<int>(cur)]; - if (curDigit >= base) { + if (curDigit >= base) { ctx.Error(ctx.Pos()) << "Failed to parse number from string: " << strOrig << ", char: '" << cur << "' is out of base: " << base; return false; @@ -1260,11 +1260,11 @@ TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node) { if (!ParseNumbers(ctx, intergerString, value, suffix)) { return {}; } - - if (suffix == "ub" || suffix == "b") { - ctx.Warning(ctx.Pos(), TIssuesIds::YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX) << "Deprecated suffix 'b' - please use 't' suffix for 8-bit integers"; - } - + + if (suffix == "ub" || suffix == "b") { + ctx.Warning(ctx.Pos(), TIssuesIds::YQL_DEPRECATED_TINY_INT_LITERAL_SUFFIX) << "Deprecated suffix 'b' - please use 't' suffix for 8-bit integers"; + } + const bool noSpaceForInt32 = value >> 31; const bool noSpaceForInt64 = value >> 63; if (suffix == "") { @@ -1278,9 +1278,9 @@ TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node) { return new TLiteralNumberNode<ui32>(ctx.Pos(), "Uint32", ToString(value)); } else if (suffix == "ul") { return new TLiteralNumberNode<ui64>(ctx.Pos(), "Uint64", ToString(value)); - } else if (suffix == "ut" || suffix == "ub") { + } else if (suffix == "ut" || suffix == "ub") { return new TLiteralNumberNode<ui8>(ctx.Pos(), "Uint8", ToString(value)); - } else if (suffix == "t" || suffix == "b") { + } else if (suffix == "t" || suffix == "b") { return new TLiteralNumberNode<i8>(ctx.Pos(), "Int8", ToString(value)); } else if (suffix == "l") { return new TLiteralNumberNode<i64>(ctx.Pos(), "Int64", ToString(value)); @@ -1326,7 +1326,7 @@ TNodePtr TSqlExpression::LiteralExpr(const TRule_literal_value& node) { } case TRule_literal_value::kAltLiteralValue3: { const TString value(Token(node.GetAlt_literal_value3().GetToken1())); - return BuildLiteralSmartString(Ctx, value); + return BuildLiteralSmartString(Ctx, value); } case TRule_literal_value::kAltLiteralValue5: { Token(node.GetAlt_literal_value5().GetToken1()); @@ -1345,34 +1345,34 @@ TNodePtr TSqlExpression::LiteralExpr(const TRule_literal_value& node) { return nullptr; } -template<typename TUnarySubExprType> -TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { +template<typename TUnarySubExprType> +TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { //unary_subexpr: (id_expr | atom_expr ) key_expr* (DOT (bind_parameter | DIGITS | id_or_string) key_expr*)* (COLLATE id)?; - // OR + // OR //in_unary_subexpr: (in_id_expr | in_atom_expr) key_expr* (DOT (bind_parameter | DIGITS | id_or_string) key_expr*)* (COLLATE id)?; TVector<INode::TIdPart> ids; auto& block = node.GetBlock1(); switch (block.Alt_case()) { - case TUnarySubExprType::TBlock1::kAlt1: { + case TUnarySubExprType::TBlock1::kAlt1: { auto& alt = block.GetAlt1(); - TString name; - if constexpr (std::is_same_v<TUnarySubExprType, TRule_unary_subexpr>) { - name = Id(alt.GetRule_id_expr1(), *this); - } else { - name = Id(alt.GetRule_in_id_expr1(), *this); - } + TString name; + if constexpr (std::is_same_v<TUnarySubExprType, TRule_unary_subexpr>) { + name = Id(alt.GetRule_id_expr1(), *this); + } else { + name = Id(alt.GetRule_in_id_expr1(), *this); + } ids.push_back(BuildColumn(Ctx.Pos())); ids.push_back(name); break; } - case TUnarySubExprType::TBlock1::kAlt2: { - TNodePtr expr; - if constexpr (std::is_same_v<TUnarySubExprType, TRule_unary_subexpr>) { - expr = AtomExpr(block.GetAlt2().GetRule_atom_expr1()); - } else { - expr = InAtomExpr(block.GetAlt2().GetRule_in_atom_expr1()); - } - + case TUnarySubExprType::TBlock1::kAlt2: { + TNodePtr expr; + if constexpr (std::is_same_v<TUnarySubExprType, TRule_unary_subexpr>) { + expr = AtomExpr(block.GetAlt2().GetRule_atom_expr1()); + } else { + expr = InAtomExpr(block.GetAlt2().GetRule_in_atom_expr1()); + } + if (!expr) { Ctx.IncrementMonCounter("sql_errors", "BadAtomExpr"); return nullptr; @@ -1397,7 +1397,7 @@ TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { for (auto& dotBlock: node.GetBlock3()) { auto bb = dotBlock.GetBlock2(); switch (bb.Alt_case()) { - case TUnarySubExprType::TBlock3::TBlock2::kAlt1: { + case TUnarySubExprType::TBlock3::TBlock2::kAlt1: { auto named = NamedNodeImpl(bb.GetAlt1().GetRule_bind_parameter1(), *this); auto namedNode = GetNamedNode(named); if (!namedNode) { @@ -1408,7 +1408,7 @@ TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { ids.back().Expr = namedNode; break; } - case TUnarySubExprType::TBlock3::TBlock2::kAlt2: { + case TUnarySubExprType::TBlock3::TBlock2::kAlt2: { const TString str(Token(bb.GetAlt2().GetToken1())); i32 pos = -1; if (!TryFromString<i32>(str, pos)) { @@ -1419,7 +1419,7 @@ TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { ids.push_back(pos); break; } - case TUnarySubExprType::TBlock3::TBlock2::kAlt3: { + case TUnarySubExprType::TBlock3::TBlock2::kAlt3: { ids.push_back(IdOrString(bb.GetAlt3().GetRule_id_or_string1(), *this)); break; } @@ -1445,61 +1445,61 @@ TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node) { return ids.size() > 1 ? BuildAccess(pos, ids, isLookup) : ids.back().Expr; } -TNodePtr TSqlExpression::BindParameterRule(const TRule_bind_parameter& rule) { - const auto namedArg = NamedNodeImpl(rule, *this); - if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { - Ctx.IncrementMonCounter("sql_features", "LambdaArgument"); - return BuildAtom(Ctx.Pos(), namedArg); - } else { - Ctx.IncrementMonCounter("sql_features", "NamedNodeUseAtom"); +TNodePtr TSqlExpression::BindParameterRule(const TRule_bind_parameter& rule) { + const auto namedArg = NamedNodeImpl(rule, *this); + if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { + Ctx.IncrementMonCounter("sql_features", "LambdaArgument"); + return BuildAtom(Ctx.Pos(), namedArg); + } else { + Ctx.IncrementMonCounter("sql_features", "NamedNodeUseAtom"); return GetNamedNode(namedArg); - } -} - -TNodePtr TSqlExpression::LambdaRule(const TRule_lambda& rule) { - const auto& alt = rule; - const bool isSqlLambda = alt.HasBlock2(); - if (!isSqlLambda) { - return SmartParenthesis(alt.GetRule_smart_parenthesis1()); - } - TSqlExpression expr(Ctx, Mode); - expr.SetSmartParenthesisMode(ESmartParenthesis::SqlLambdaParams); - auto parenthesis = expr.SmartParenthesis(alt.GetRule_smart_parenthesis1()); - if (!parenthesis) { - return {}; - } - TVector<TString> args; - if (!SqlLambdaParams(parenthesis, args)) { - return {}; - } - auto bodyBlock = alt.GetBlock2(); - Token(bodyBlock.GetToken1()); - TPosition pos(Ctx.Pos()); - TVector<TNodePtr> exprSeq; - for (const auto& arg: args) { - PushNamedNode(arg, BuildAtom(pos, arg, NYql::TNodeFlags::Default)); - } - const bool ret = SqlLambdaExprBody(Ctx, bodyBlock.GetRule_lambda_body3(), exprSeq); - for (const auto& arg : args) { - PopNamedNode(arg); - } - if (!ret) { - return {}; - } - return BuildSqlLambda(pos, std::move(args), std::move(exprSeq)); -} - -TNodePtr TSqlExpression::CastRule(const TRule_cast_expr& rule) { - Ctx.IncrementMonCounter("sql_features", "Cast"); - const auto& alt = rule; - Token(alt.GetToken1()); - TPosition pos(Ctx.Pos()); - TSqlExpression expr(Ctx, Mode); - const auto& paramOne = alt.GetRule_type_name5().HasBlock2() ? alt.GetRule_type_name5().GetBlock2().GetRule_integer2().GetToken1().GetValue() : TString(); - const auto& paramTwo = !paramOne.empty() && alt.GetRule_type_name5().GetBlock2().HasBlock3() ? alt.GetRule_type_name5().GetBlock2().GetBlock3().GetRule_integer2().GetToken1().GetValue() : TString(); - return BuildCast(Ctx, pos, expr.Build(alt.GetRule_expr3()), Id(alt.GetRule_type_name5().GetRule_id1(), *this), paramOne, paramTwo); -} - + } +} + +TNodePtr TSqlExpression::LambdaRule(const TRule_lambda& rule) { + const auto& alt = rule; + const bool isSqlLambda = alt.HasBlock2(); + if (!isSqlLambda) { + return SmartParenthesis(alt.GetRule_smart_parenthesis1()); + } + TSqlExpression expr(Ctx, Mode); + expr.SetSmartParenthesisMode(ESmartParenthesis::SqlLambdaParams); + auto parenthesis = expr.SmartParenthesis(alt.GetRule_smart_parenthesis1()); + if (!parenthesis) { + return {}; + } + TVector<TString> args; + if (!SqlLambdaParams(parenthesis, args)) { + return {}; + } + auto bodyBlock = alt.GetBlock2(); + Token(bodyBlock.GetToken1()); + TPosition pos(Ctx.Pos()); + TVector<TNodePtr> exprSeq; + for (const auto& arg: args) { + PushNamedNode(arg, BuildAtom(pos, arg, NYql::TNodeFlags::Default)); + } + const bool ret = SqlLambdaExprBody(Ctx, bodyBlock.GetRule_lambda_body3(), exprSeq); + for (const auto& arg : args) { + PopNamedNode(arg); + } + if (!ret) { + return {}; + } + return BuildSqlLambda(pos, std::move(args), std::move(exprSeq)); +} + +TNodePtr TSqlExpression::CastRule(const TRule_cast_expr& rule) { + Ctx.IncrementMonCounter("sql_features", "Cast"); + const auto& alt = rule; + Token(alt.GetToken1()); + TPosition pos(Ctx.Pos()); + TSqlExpression expr(Ctx, Mode); + const auto& paramOne = alt.GetRule_type_name5().HasBlock2() ? alt.GetRule_type_name5().GetBlock2().GetRule_integer2().GetToken1().GetValue() : TString(); + const auto& paramTwo = !paramOne.empty() && alt.GetRule_type_name5().GetBlock2().HasBlock3() ? alt.GetRule_type_name5().GetBlock2().GetBlock3().GetRule_integer2().GetToken1().GetValue() : TString(); + return BuildCast(Ctx, pos, expr.Build(alt.GetRule_expr3()), Id(alt.GetRule_type_name5().GetRule_id1(), *this), paramOne, paramTwo); +} + TNodePtr TSqlExpression::BitCastRule(const TRule_bitcast_expr& rule) { Ctx.IncrementMonCounter("sql_features", "BitCast"); const auto& alt = rule; @@ -1511,76 +1511,76 @@ TNodePtr TSqlExpression::BitCastRule(const TRule_bitcast_expr& rule) { return BuildBitCast(Ctx, pos, expr.Build(alt.GetRule_expr3()), Id(alt.GetRule_type_name5().GetRule_id1(), *this), paramOne, paramTwo); } -TNodePtr TSqlExpression::ExistsRule(const TRule_exists_expr& rule) { - Ctx.IncrementMonCounter("sql_features", "Exists"); - const auto& alt = rule; - - Token(alt.GetToken2()); - TSqlSelect select(Ctx, Mode); - TPosition pos; - auto source = select.Build(alt.GetRule_select_stmt3(), pos); - if (!source) { - Ctx.IncrementMonCounter("sql_errors", "BadSource"); - return nullptr; - } - const bool checkExist = true; - return BuildBuiltinFunc(Ctx, Ctx.Pos(), "ListHasItems", {BuildSourceNode(pos, std::move(source), checkExist)}); -} - -TNodePtr TSqlExpression::CaseRule(const TRule_case_expr& rule) { - Ctx.IncrementMonCounter("sql_features", "Case"); - const auto& alt = rule; - Token(alt.GetToken1()); - TNodePtr elseExpr; - if (alt.HasBlock4()) { - Token(alt.GetBlock4().GetToken1()); - TSqlExpression expr(Ctx, Mode); - elseExpr = expr.Build(alt.GetBlock4().GetRule_expr2()); - } else { - Ctx.IncrementMonCounter("sql_errors", "ElseIsRequired"); - Error() << "ELSE is required"; - return nullptr; - } - TVector<TNodePtr> args; - for (int i = alt.Block3Size() - 1; i >= 0; --i) { - const auto& block = alt.GetBlock3(i).GetRule_when_expr1(); - args.clear(); - Token(block.GetToken1()); - TSqlExpression condExpr(Ctx, Mode); - args.push_back(condExpr.Build(block.GetRule_expr2())); - if (alt.HasBlock2()) { - TSqlExpression expr(Ctx, Mode); - args.back() = BuildBinaryOp(Ctx.Pos(), "==", expr.Build(alt.GetBlock2().GetRule_expr1()), args.back()); - } - Token(block.GetToken3()); - TSqlExpression thenExpr(Ctx, Mode); - args.push_back(thenExpr.Build(block.GetRule_expr4())); - args.push_back(elseExpr); - if (i > 0) { - elseExpr = BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", args); - } - } - return BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", args); -} - -template<typename TWindowFunctionType> -TNodePtr TSqlExpression::WindowFunctionRule(const TWindowFunctionType& rule) { +TNodePtr TSqlExpression::ExistsRule(const TRule_exists_expr& rule) { + Ctx.IncrementMonCounter("sql_features", "Exists"); + const auto& alt = rule; + + Token(alt.GetToken2()); + TSqlSelect select(Ctx, Mode); + TPosition pos; + auto source = select.Build(alt.GetRule_select_stmt3(), pos); + if (!source) { + Ctx.IncrementMonCounter("sql_errors", "BadSource"); + return nullptr; + } + const bool checkExist = true; + return BuildBuiltinFunc(Ctx, Ctx.Pos(), "ListHasItems", {BuildSourceNode(pos, std::move(source), checkExist)}); +} + +TNodePtr TSqlExpression::CaseRule(const TRule_case_expr& rule) { + Ctx.IncrementMonCounter("sql_features", "Case"); + const auto& alt = rule; + Token(alt.GetToken1()); + TNodePtr elseExpr; + if (alt.HasBlock4()) { + Token(alt.GetBlock4().GetToken1()); + TSqlExpression expr(Ctx, Mode); + elseExpr = expr.Build(alt.GetBlock4().GetRule_expr2()); + } else { + Ctx.IncrementMonCounter("sql_errors", "ElseIsRequired"); + Error() << "ELSE is required"; + return nullptr; + } + TVector<TNodePtr> args; + for (int i = alt.Block3Size() - 1; i >= 0; --i) { + const auto& block = alt.GetBlock3(i).GetRule_when_expr1(); + args.clear(); + Token(block.GetToken1()); + TSqlExpression condExpr(Ctx, Mode); + args.push_back(condExpr.Build(block.GetRule_expr2())); + if (alt.HasBlock2()) { + TSqlExpression expr(Ctx, Mode); + args.back() = BuildBinaryOp(Ctx.Pos(), "==", expr.Build(alt.GetBlock2().GetRule_expr1()), args.back()); + } + Token(block.GetToken3()); + TSqlExpression thenExpr(Ctx, Mode); + args.push_back(thenExpr.Build(block.GetRule_expr4())); + args.push_back(elseExpr); + if (i > 0) { + elseExpr = BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", args); + } + } + return BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", args); +} + +template<typename TWindowFunctionType> +TNodePtr TSqlExpression::WindowFunctionRule(const TWindowFunctionType& rule) { // window_function: call_expr (null_treatment? OVER window_name_or_specification)?; - // OR - // in_window_function: in_call_expr (null_treatment? OVER window_name_or_specification)?; + // OR + // in_window_function: in_call_expr (null_treatment? OVER window_name_or_specification)?; const bool overWindow = rule.HasBlock2(); TSqlCallExpr call(Ctx, Mode, this); - - bool initResult; - if constexpr (std::is_same_v<TWindowFunctionType, TRule_window_function>) { - initResult = call.Init(rule.GetRule_call_expr1()); - } else { - initResult = call.Init(rule.GetRule_in_call_expr1()); - } - if (!initResult) { + + bool initResult; + if constexpr (std::is_same_v<TWindowFunctionType, TRule_window_function>) { + initResult = call.Init(rule.GetRule_call_expr1()); + } else { + initResult = call.Init(rule.GetRule_in_call_expr1()); + } + if (!initResult) { return {}; } - + call.IncCounters(); if (!overWindow) { return call.BuildCall(); @@ -1606,34 +1606,34 @@ TNodePtr TSqlExpression::AtomExpr(const TRule_atom_expr& node) { // | bind_parameter // | window_function // | lambda - // | cast_expr - // | exists_expr - // | case_expr - // | id_or_string NAMESPACE id_or_string + // | cast_expr + // | exists_expr + // | case_expr + // | id_or_string NAMESPACE id_or_string // ; - + switch (node.Alt_case()) { case TRule_atom_expr::kAltAtomExpr1: Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); return LiteralExpr(node.GetAlt_atom_expr1().GetRule_literal_value1()); - case TRule_atom_expr::kAltAtomExpr2: - return BindParameterRule(node.GetAlt_atom_expr2().GetRule_bind_parameter1()); + case TRule_atom_expr::kAltAtomExpr2: + return BindParameterRule(node.GetAlt_atom_expr2().GetRule_bind_parameter1()); case TRule_atom_expr::kAltAtomExpr3: return WindowFunctionRule(node.GetAlt_atom_expr3().GetRule_window_function1()); - case TRule_atom_expr::kAltAtomExpr4: - return LambdaRule(node.GetAlt_atom_expr4().GetRule_lambda1()); - case TRule_atom_expr::kAltAtomExpr5: - return CastRule(node.GetAlt_atom_expr5().GetRule_cast_expr1()); - case TRule_atom_expr::kAltAtomExpr6: - return ExistsRule(node.GetAlt_atom_expr6().GetRule_exists_expr1()); - case TRule_atom_expr::kAltAtomExpr7: - return CaseRule(node.GetAlt_atom_expr7().GetRule_case_expr1()); + case TRule_atom_expr::kAltAtomExpr4: + return LambdaRule(node.GetAlt_atom_expr4().GetRule_lambda1()); + case TRule_atom_expr::kAltAtomExpr5: + return CastRule(node.GetAlt_atom_expr5().GetRule_cast_expr1()); + case TRule_atom_expr::kAltAtomExpr6: + return ExistsRule(node.GetAlt_atom_expr6().GetRule_exists_expr1()); + case TRule_atom_expr::kAltAtomExpr7: + return CaseRule(node.GetAlt_atom_expr7().GetRule_case_expr1()); case TRule_atom_expr::kAltAtomExpr8: { const auto& alt = node.GetAlt_atom_expr8(); const TString module(IdOrString(alt.GetRule_id_or_string1(), *this)); TPosition pos(Ctx.Pos()); - bool rawString = true; - const TString name(IdOrString(alt.GetRule_id_or_string3(), *this, rawString)); + bool rawString = true; + const TString name(IdOrString(alt.GetRule_id_or_string3(), *this, rawString)); return BuildCallable(pos, module, name, {}); } case TRule_atom_expr::kAltAtomExpr9: @@ -1644,51 +1644,51 @@ TNodePtr TSqlExpression::AtomExpr(const TRule_atom_expr& node) { return nullptr; } -TNodePtr TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node) { - // in_atom_expr: - // literal_value - // | bind_parameter - // | in_window_function - // | smart_parenthesis - // | cast_expr - // | case_expr - // | LPAREN select_stmt RPAREN - // ; - - switch (node.Alt_case()) { - case TRule_in_atom_expr::kAltInAtomExpr1: - Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); - return LiteralExpr(node.GetAlt_in_atom_expr1().GetRule_literal_value1()); - case TRule_in_atom_expr::kAltInAtomExpr2: - return BindParameterRule(node.GetAlt_in_atom_expr2().GetRule_bind_parameter1()); - case TRule_in_atom_expr::kAltInAtomExpr3: - return WindowFunctionRule(node.GetAlt_in_atom_expr3().GetRule_in_window_function1()); - case TRule_in_atom_expr::kAltInAtomExpr4: - return SmartParenthesis(node.GetAlt_in_atom_expr4().GetRule_smart_parenthesis1()); - case TRule_in_atom_expr::kAltInAtomExpr5: - return CastRule(node.GetAlt_in_atom_expr5().GetRule_cast_expr1()); - case TRule_in_atom_expr::kAltInAtomExpr6: - return CaseRule(node.GetAlt_in_atom_expr6().GetRule_case_expr1()); - case TRule_in_atom_expr::kAltInAtomExpr7: { - Token(node.GetAlt_in_atom_expr7().GetToken1()); - TSqlSelect select(Ctx, Mode); - TPosition pos; - auto source = select.Build(node.GetAlt_in_atom_expr7().GetRule_select_stmt2(), pos); - if (!source) { - Ctx.IncrementMonCounter("sql_errors", "BadSource"); - return {}; - } - Ctx.IncrementMonCounter("sql_features", "InSubquery"); +TNodePtr TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node) { + // in_atom_expr: + // literal_value + // | bind_parameter + // | in_window_function + // | smart_parenthesis + // | cast_expr + // | case_expr + // | LPAREN select_stmt RPAREN + // ; + + switch (node.Alt_case()) { + case TRule_in_atom_expr::kAltInAtomExpr1: + Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); + return LiteralExpr(node.GetAlt_in_atom_expr1().GetRule_literal_value1()); + case TRule_in_atom_expr::kAltInAtomExpr2: + return BindParameterRule(node.GetAlt_in_atom_expr2().GetRule_bind_parameter1()); + case TRule_in_atom_expr::kAltInAtomExpr3: + return WindowFunctionRule(node.GetAlt_in_atom_expr3().GetRule_in_window_function1()); + case TRule_in_atom_expr::kAltInAtomExpr4: + return SmartParenthesis(node.GetAlt_in_atom_expr4().GetRule_smart_parenthesis1()); + case TRule_in_atom_expr::kAltInAtomExpr5: + return CastRule(node.GetAlt_in_atom_expr5().GetRule_cast_expr1()); + case TRule_in_atom_expr::kAltInAtomExpr6: + return CaseRule(node.GetAlt_in_atom_expr6().GetRule_case_expr1()); + case TRule_in_atom_expr::kAltInAtomExpr7: { + Token(node.GetAlt_in_atom_expr7().GetToken1()); + TSqlSelect select(Ctx, Mode); + TPosition pos; + auto source = select.Build(node.GetAlt_in_atom_expr7().GetRule_select_stmt2(), pos); + if (!source) { + Ctx.IncrementMonCounter("sql_errors", "BadSource"); + return {}; + } + Ctx.IncrementMonCounter("sql_features", "InSubquery"); return BuildSelectResult(pos, std::move(source), false, Mode == NSQLTranslation::ESqlMode::SUBQUERY); - } + } case TRule_in_atom_expr::kAltInAtomExpr8: return BitCastRule(node.GetAlt_in_atom_expr8().GetRule_bitcast_expr1()); - default: - AltNotImplemented("in_atom_expr", node); - } - return nullptr; -} - + default: + AltNotImplemented("in_atom_expr", node); + } + return nullptr; +} + bool TSqlExpression::SqlLambdaParams(const TNodePtr& node, TVector<TString>& args) { auto errMsg = TStringBuf("Invalid lambda arguments syntax. Lambda arguments should starts with '$' as named value."); auto tupleNodePtr = dynamic_cast<TTupleNode*>(node.Get()); @@ -1921,10 +1921,10 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node) { } if (opName != "ilike") { if (!hasPattern) { - isMatch = BuildBinaryOp(pos, "==", res, BuildLiteralSmartString(Ctx, + isMatch = BuildBinaryOp(pos, "==", res, BuildLiteralSmartString(Ctx, TStringBuilder() << "@@" << lowerBound << "@@")); } else if (!lowerBound.empty()) { - const auto& lowerBoundOp = BuildBinaryOp(pos, ">=", res, BuildLiteralSmartString(Ctx, + const auto& lowerBoundOp = BuildBinaryOp(pos, ">=", res, BuildLiteralSmartString(Ctx, TStringBuilder() << "@@" << lowerBound << "@@")); auto& isMatchCopy = isMatch; TStringBuilder upperBound; @@ -1944,7 +1944,7 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node) { pos, "And", lowerBoundOp, - BuildBinaryOp(pos, "<", res, BuildLiteralSmartString(Ctx, + BuildBinaryOp(pos, "<", res, BuildLiteralSmartString(Ctx, TStringBuilder() << "@@" << upperBound << "@@")) ); isMatch = BuildBinaryOp(pos, "And", between, isMatchCopy); @@ -1983,15 +1983,15 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node) { case TRule_cond_expr::kAltCondExpr2: { auto altInExpr = cond.GetAlt_cond_expr2(); const bool notIn = altInExpr.HasBlock1(); - auto hints = BuildTuple(pos, {}); - if (altInExpr.HasBlock3()) { - Ctx.IncrementMonCounter("sql_features", "IsCompactHint"); - auto sizeHint = BuildTuple(pos, { BuildQuotedAtom(pos, "isCompact", NYql::TNodeFlags::Default) }); - hints = BuildTuple(pos, { sizeHint }); - } + auto hints = BuildTuple(pos, {}); + if (altInExpr.HasBlock3()) { + Ctx.IncrementMonCounter("sql_features", "IsCompactHint"); + auto sizeHint = BuildTuple(pos, { BuildQuotedAtom(pos, "isCompact", NYql::TNodeFlags::Default) }); + hints = BuildTuple(pos, { sizeHint }); + } TSqlExpression inSubexpr(Ctx, Mode); - auto inRight = inSubexpr.SqlInExpr(altInExpr.GetRule_in_expr4()); - auto isIn = BuildBuiltinFunc(Ctx, pos, "In", {res, inRight, hints}); + auto inRight = inSubexpr.SqlInExpr(altInExpr.GetRule_in_expr4()); + auto isIn = BuildBuiltinFunc(Ctx, pos, "In", {res, inRight, hints}); Ctx.IncrementMonCounter("sql_features", notIn ? "NotIn" : "In"); return notIn ? BuildUnaryOp(pos, "Not", isIn) : isIn; } @@ -2001,19 +2001,19 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node) { altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt2 || altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 ; - - if (altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 && - !cond.GetAlt_cond_expr3().GetBlock1().GetAlt4().HasBlock1()) - { - Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_MISSING_IS_BEFORE_NOT_NULL) << "Missing IS keyword before NOT NULL"; - } - + + if (altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 && + !cond.GetAlt_cond_expr3().GetBlock1().GetAlt4().HasBlock1()) + { + Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_MISSING_IS_BEFORE_NOT_NULL) << "Missing IS keyword before NOT NULL"; + } + auto isNull = BuildIsNullOp(pos, res); Ctx.IncrementMonCounter("sql_features", notNoll ? "NotNull" : "Null"); return notNoll ? BuildUnaryOp(pos, "Not", isNull) : isNull; } case TRule_cond_expr::kAltCondExpr4: { - auto alt = cond.GetAlt_cond_expr4(); + auto alt = cond.GetAlt_cond_expr4(); if (alt.HasBlock1()) { Ctx.IncrementMonCounter("sql_features", "NotBetween"); return BuildBinaryOp( @@ -2032,9 +2032,9 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node) { ); } } - case TRule_cond_expr::kAltCondExpr5: { - auto alt = cond.GetAlt_cond_expr5(); - auto getNode = [](const TRule_cond_expr::TAlt5::TBlock1& b) -> const TRule_eq_subexpr& { return b.GetRule_eq_subexpr2(); }; + case TRule_cond_expr::kAltCondExpr5: { + auto alt = cond.GetAlt_cond_expr5(); + auto getNode = [](const TRule_cond_expr::TAlt5::TBlock1& b) -> const TRule_eq_subexpr& { return b.GetRule_eq_subexpr2(); }; return BinOpList(node.GetRule_eq_subexpr1(), getNode, alt.GetBlock1().begin(), alt.GetBlock1().end()); } default: @@ -2127,14 +2127,14 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be break; case SQLLexerTokens::TOKEN_EQUALS: Ctx.IncrementMonCounter("sql_binary_operations", "Equals"); - [[fallthrough]]; + [[fallthrough]]; case SQLLexerTokens::TOKEN_EQUALS2: Ctx.IncrementMonCounter("sql_binary_operations", "Equals2"); opName = "=="; break; case SQLLexerTokens::TOKEN_NOT_EQUALS: Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals"); - [[fallthrough]]; + [[fallthrough]]; case SQLLexerTokens::TOKEN_NOT_EQUALS2: Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals2"); opName = "!="; @@ -2183,7 +2183,7 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be TNodePtr TSqlExpression::SqlInExpr(const TRule_in_expr& node) { TSqlExpression expr(Ctx, Mode); expr.SetSmartParenthesisMode(TSqlExpression::ESmartParenthesis::InStatement); - auto result = expr.WrapExprShortcuts(expr.UnaryExpr(node.GetRule_in_unary_subexpr1())); + auto result = expr.WrapExprShortcuts(expr.UnaryExpr(node.GetRule_in_unary_subexpr1())); return result; } @@ -4596,7 +4596,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success const bool hasLexicalScope = withConfigure || normalizedPragma == "classicdivision"; for (auto pragmaValue : pragmaValues) { if (pragmaValue->HasAlt_pragma_value3()) { - values.push_back(TDeferredAtom(Ctx.Pos(), StringContent(Ctx, pragmaValue->GetAlt_pragma_value3().GetToken1().GetValue()))); + values.push_back(TDeferredAtom(Ctx.Pos(), StringContent(Ctx, pragmaValue->GetAlt_pragma_value3().GetToken1().GetValue()))); } else if (pragmaValue->HasAlt_pragma_value2() && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2() @@ -4748,33 +4748,33 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } - - TString codePattern = *values[1].GetLiteral(); - TString action = *values[0].GetLiteral(); - - TWarningRule rule; - TString parseError; - auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError); - switch (parseResult) { - case TWarningRule::EParseResult::PARSE_OK: - Ctx.WarningPolicy.AddRule(rule); - if (Ctx.Settings.WarnOnV0 && codePattern == "*") { - // Add exception for YQL_DEPRECATED_V0_SYNTAX - TWarningRule defaultForDeprecatedV0; - YQL_ENSURE(TWarningRule::ParseFrom(ToString(TIssueCode(TIssuesIds::YQL_DEPRECATED_V0_SYNTAX)), - "default", defaultForDeprecatedV0, - parseError) == TWarningRule::EParseResult::PARSE_OK); - Ctx.WarningPolicy.AddRule(defaultForDeprecatedV0); - } - break; - case TWarningRule::EParseResult::PARSE_PATTERN_FAIL: - case TWarningRule::EParseResult::PARSE_ACTION_FAIL: - Ctx.Error() << parseError; - return {}; - default: - Y_ENSURE(false, "Unknown parse result"); - } - + + TString codePattern = *values[1].GetLiteral(); + TString action = *values[0].GetLiteral(); + + TWarningRule rule; + TString parseError; + auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError); + switch (parseResult) { + case TWarningRule::EParseResult::PARSE_OK: + Ctx.WarningPolicy.AddRule(rule); + if (Ctx.Settings.WarnOnV0 && codePattern == "*") { + // Add exception for YQL_DEPRECATED_V0_SYNTAX + TWarningRule defaultForDeprecatedV0; + YQL_ENSURE(TWarningRule::ParseFrom(ToString(TIssueCode(TIssuesIds::YQL_DEPRECATED_V0_SYNTAX)), + "default", defaultForDeprecatedV0, + parseError) == TWarningRule::EParseResult::PARSE_OK); + Ctx.WarningPolicy.AddRule(defaultForDeprecatedV0); + } + break; + case TWarningRule::EParseResult::PARSE_PATTERN_FAIL: + case TWarningRule::EParseResult::PARSE_ACTION_FAIL: + Ctx.Error() << parseError; + return {}; + default: + Y_ENSURE(false, "Unknown parse result"); + } + Ctx.IncrementMonCounter("sql_pragma", "warning"); } else if (normalizedPragma == "greetings") { if (values.size() > 1) { @@ -4809,12 +4809,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disableunordered") { Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA) << "Use of deprecated DisableUnordered pragma. It will be dropped soon"; - } else if (normalizedPragma == "pullupflatmapoverjoin") { - Ctx.PragmaPullUpFlatMapOverJoin = true; - Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin"); - } else if (normalizedPragma == "disablepullupflatmapoverjoin") { - Ctx.PragmaPullUpFlatMapOverJoin = false; - Ctx.IncrementMonCounter("sql_pragma", "DisablePullUpFlatMapOverJoin"); + } else if (normalizedPragma == "pullupflatmapoverjoin") { + Ctx.PragmaPullUpFlatMapOverJoin = true; + Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin"); + } else if (normalizedPragma == "disablepullupflatmapoverjoin") { + Ctx.PragmaPullUpFlatMapOverJoin = false; + Ctx.IncrementMonCounter("sql_pragma", "DisablePullUpFlatMapOverJoin"); } else if (normalizedPragma == "enablesystemcolumns") { if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { Error() << "Expected single boolean argument for: " << pragma; @@ -5050,17 +5050,17 @@ TSourcePtr TSqlQuery::Build(const TRule_multiple_column_assignment& stmt) { TNodePtr TSqlQuery::Build(const TSQLParserAST& ast) { const auto& statements = ast.GetRule_sql_stmt_list(); TVector<TNodePtr> blocks; - - if (Ctx.Settings.WarnOnV0) { + + if (Ctx.Settings.WarnOnV0) { if (Ctx.Settings.V0WarnAsError->Allow()) { Error() << "SQL v0 syntax is deprecated and no longer supported. Please switch to v1: https://clubs.at.yandex-team.ru/yql/2910"; - return nullptr; - } - - Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_V0_SYNTAX) << + return nullptr; + } + + Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_V0_SYNTAX) << "SQL v0 syntax is deprecated and will stop working soon. Consider switching to v1: https://clubs.at.yandex-team.ru/yql/2910"; - } - + } + if (Ctx.Settings.V0Behavior == NSQLTranslation::EV0Behavior::Report) { AddStatementToBlocks(blocks, BuildPragma(TPosition(), "config", "flags", { TDeferredAtom(TPosition(), "SQL"), @@ -5234,8 +5234,8 @@ google::protobuf::Message* SqlAST(const TString& query, const TString& queryName #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, ""); - NProtoAST::TProtoASTBuilder<NALP::SQLParser, NALP::SQLLexer> builder(query, queryName, arena); + NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, ""); + NProtoAST::TProtoASTBuilder<NALP::SQLParser, NALP::SQLLexer> builder(query, queryName, arena); return builder.BuildAST(collector); } @@ -5244,7 +5244,7 @@ google::protobuf::Message* SqlAST(const TString& query, const TString& queryName #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - NProtoAST::TProtoASTBuilder<NALP::SQLParser, NALP::SQLLexer> builder(query, queryName, arena); + NProtoAST::TProtoASTBuilder<NALP::SQLParser, NALP::SQLLexer> builder(query, queryName, arena); return builder.BuildAST(err); } @@ -5298,7 +5298,7 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra { TAstParseResult res; TContext ctx(settings, res.Issues); - NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); + NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); google::protobuf::Message* ast(SqlAST(query, "query", collector, settings.Arena)); if (ast) { @@ -5306,9 +5306,9 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra } else { ctx.IncrementMonCounter("sql_errors", "AstError"); } - if (warningRules) { - *warningRules = ctx.WarningPolicy.GetRules(); - ctx.WarningPolicy.Clear(); + if (warningRules) { + *warningRules = ctx.WarningPolicy.GetRules(); + ctx.WarningPolicy.Clear(); } return res; } diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index fb23169305..3a70aef93c 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -34,7 +34,7 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES settings.Mode = mode; settings.Arena = &arena; settings.V0Behavior = NSQLTranslation::EV0Behavior::Report; - settings.WarnOnV0 = false; + settings.WarnOnV0 = false; auto res = SqlToYql(query, settings); if (debug == EDebugOutput::ToCerr) { Err2Str(res, debug); @@ -103,61 +103,61 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEME)").IsOk()); } - Y_UNIT_TEST(InNoHints) { - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input WHERE key IN (1,2,3)"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'()")); - }; - TWordCountHive elementStat = {{TString("SqlIn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(InHintCompact) { - // should parse COMPACT as hint - NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('isCompact))")); - }; - TWordCountHive elementStat = {{TString("SqlIn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(InHintTableSource) { - // should add tableSource as a hint - NYql::TAstParseResult res = SqlToYql("$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('tableSource))")); - }; - TWordCountHive elementStat = {{TString("SqlIn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(InHintCompactAndTableSource) { - NYql::TAstParseResult res = SqlToYql("$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - Y_UNUSED(word); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('isCompact) '('tableSource))")); - }; - TWordCountHive elementStat = {{TString("SqlIn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - } - - Y_UNIT_TEST(CompactKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); - } - + Y_UNIT_TEST(InNoHints) { + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input WHERE key IN (1,2,3)"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'()")); + }; + TWordCountHive elementStat = {{TString("SqlIn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + } + + Y_UNIT_TEST(InHintCompact) { + // should parse COMPACT as hint + NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('isCompact))")); + }; + TWordCountHive elementStat = {{TString("SqlIn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + } + + Y_UNIT_TEST(InHintTableSource) { + // should add tableSource as a hint + NYql::TAstParseResult res = SqlToYql("$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('tableSource))")); + }; + TWordCountHive elementStat = {{TString("SqlIn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + } + + Y_UNIT_TEST(InHintCompactAndTableSource) { + NYql::TAstParseResult res = SqlToYql("$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + Y_UNUSED(word); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('('isCompact) '('tableSource))")); + }; + TWordCountHive elementStat = {{TString("SqlIn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + } + + Y_UNIT_TEST(CompactKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); + } + Y_UNIT_TEST(Jubilee) { NYql::TAstParseResult res = SqlToYql("USE plato; INSERT INTO Arcadia (r2000000) VALUES (\"2M GET!!!\");"); UNIT_ASSERT(res.Root); @@ -832,70 +832,70 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { )", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(res.Root); } - - Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { - NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); - } - - Y_UNIT_TEST(WarnLegacySuffixForTinyIntLiteral) { - NYql::TAstParseResult res = SqlToYql("select 1b, 0ub, 0b0b;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:8: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" - "<main>:1:12: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" - "<main>:1:17: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" - ); - } - - Y_UNIT_TEST(WarnMultiWayJoinWithUsing) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "PRAGMA DisableSimpleColumns;\n" - "SELECT *\n" - "FROM Input1 AS a\n" - "JOIN Input2 AS b USING(key)\n" - "JOIN Input3 AS c ON a.key = c.key;\n" - ); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:5:24: Warning: Multi-way JOINs should be connected with ON clause instead of USING clause, code: 4514\n" - ); - } - - Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); - UNIT_ASSERT(res1.Root); - UNIT_ASSERT(res2.Root); - - TWordCountHive elementStat = {{TString("a\\"), 0}}; - - VerifyProgram(res1, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); - - VerifyProgram(res2, elementStat); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); - } - - Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { - UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { - UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); - } + + Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { + NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); + } + + Y_UNIT_TEST(WarnLegacySuffixForTinyIntLiteral) { + NYql::TAstParseResult res = SqlToYql("select 1b, 0ub, 0b0b;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:8: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" + "<main>:1:12: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" + "<main>:1:17: Warning: Deprecated suffix 'b' - please use 't' suffix for 8-bit integers, code: 4508\n" + ); + } + + Y_UNIT_TEST(WarnMultiWayJoinWithUsing) { + NYql::TAstParseResult res = SqlToYql( + "USE plato;\n" + "PRAGMA DisableSimpleColumns;\n" + "SELECT *\n" + "FROM Input1 AS a\n" + "JOIN Input2 AS b USING(key)\n" + "JOIN Input3 AS c ON a.key = c.key;\n" + ); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:5:24: Warning: Multi-way JOINs should be connected with ON clause instead of USING clause, code: 4514\n" + ); + } + + Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); + UNIT_ASSERT(res1.Root); + UNIT_ASSERT(res2.Root); + + TWordCountHive elementStat = {{TString("a\\"), 0}}; + + VerifyProgram(res1, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); + + VerifyProgram(res2, elementStat); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); + } + + Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { + UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); + } + + Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { + UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); + } } @@ -915,49 +915,49 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(a1, a2); } - Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); - UNIT_ASSERT(!res1.Root); - UNIT_ASSERT(!res2.Root); - - TString header = "<main>:1:15: Error: Unexpected character : syntax error...\n\n"; - TString err1 = "<main>:1:7: Error: Unexpected token '\'a\\\\\'' : cannot match to any predicted input...\n\n"; - TString err2 = "<main>:1:7: Error: Unexpected token '\"a\\\\\"' : cannot match to any predicted input...\n\n"; - UNIT_ASSERT_NO_DIFF(Err2Str(res1), header + err1); - UNIT_ASSERT_NO_DIFF(Err2Str(res2), header + err2); - } - - Y_UNIT_TEST(InvalidHexInStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\n" - "bar\n" - "\\01\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidOctalInMultilineStringIdentifier) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from `In\\0123\n\n" - "\\`\\01put`"); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:3:6: Error: Cannot parse broken identifier: Invalid octal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - + Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); + UNIT_ASSERT(!res1.Root); + UNIT_ASSERT(!res2.Root); + + TString header = "<main>:1:15: Error: Unexpected character : syntax error...\n\n"; + TString err1 = "<main>:1:7: Error: Unexpected token '\'a\\\\\'' : cannot match to any predicted input...\n\n"; + TString err2 = "<main>:1:7: Error: Unexpected token '\"a\\\\\"' : cannot match to any predicted input...\n\n"; + UNIT_ASSERT_NO_DIFF(Err2Str(res1), header + err1); + UNIT_ASSERT_NO_DIFF(Err2Str(res2), header + err2); + } + + Y_UNIT_TEST(InvalidHexInStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\n" + "bar\n" + "\\01\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(InvalidOctalInMultilineStringIdentifier) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from `In\\0123\n\n" + "\\`\\01put`"); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:3:6: Error: Cannot parse broken identifier: Invalid octal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + Y_UNIT_TEST(WrongTokenTrivial) { NYql::TAstParseResult res = SqlToYql("foo"); UNIT_ASSERT(!res.Root); @@ -967,16 +967,16 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(InvalidDoubleAtString) { NYql::TAstParseResult res = SqlToYql("select @@@@@@"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n" ); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n" ); + } + + Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { + NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); } - Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { - NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); - } - Y_UNIT_TEST(InvalidStringFromTable) { NYql::TAstParseResult res = SqlToYql("select \"FOO\"\"BAR from plato.foo"); UNIT_ASSERT(!res.Root); @@ -986,8 +986,8 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(InvalidDoubleAtStringFromTable) { NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); } Y_UNIT_TEST(SelectInvalidSyntax) { @@ -1429,23 +1429,23 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(UseInOnStrings) { NYql::TAstParseResult res = SqlToYql("select * from plato.Input where \"foo\" in \"foovalue\";"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source\n"); - } - - Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { - NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, perhaps you should remove parenthesis here, code: 4501\n"); - } - - Y_UNIT_TEST(InHintsWithKeywordClash) { - NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT [COMPACT](1,2,3)"); - UNIT_ASSERT(!res.Root); - // should try to parse last compact as call expression - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT, to use YQL functions, try YQL::COMPACT\n"); - } - + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " + "expecting tuple, list, dict or single column table source\n"); + } + + Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { + NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, perhaps you should remove parenthesis here, code: 4501\n"); + } + + Y_UNIT_TEST(InHintsWithKeywordClash) { + NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT [COMPACT](1,2,3)"); + UNIT_ASSERT(!res.Root); + // should try to parse last compact as call expression + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT, to use YQL functions, try YQL::COMPACT\n"); + } + Y_UNIT_TEST(ErrorColumnPosition) { NYql::TAstParseResult res = SqlToYql( "USE plato;\n" @@ -1616,15 +1616,15 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:50: Error: ROLLBACK isn't supported for provider: yt\n"); } - Y_UNIT_TEST(ShouldFailAsTruncatedBinaryNotSucceedAsLegacyTinyZeroInt) { - NYql::TAstParseResult res = SqlToYql("select 0b;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:9: Error: Unexpected character ';' (Unicode character <59>) : missing elements...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n" - ); - } - + Y_UNIT_TEST(ShouldFailAsTruncatedBinaryNotSucceedAsLegacyTinyZeroInt) { + NYql::TAstParseResult res = SqlToYql("select 0b;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:9: Error: Unexpected character ';' (Unicode character <59>) : missing elements...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n" + ); + } + Y_UNIT_TEST(ConvertNumberFailed0) { NYql::TAstParseResult res = SqlToYql("select 0o80l;"); UNIT_ASSERT(!res.Root); diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 739690d0fd..ef394c7a00 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -2,15 +2,15 @@ grammar SQLv1; options { language = Cpp; - memoize = true; + memoize = true; } // Input is a list of statements. sql_query: sql_stmt_list | (PRAGMA ANSI DIGITS ansi_sql_stmt_list); -sql_stmt_list: SEMICOLON* sql_stmt (SEMICOLON+ sql_stmt)* SEMICOLON* EOF; -ansi_sql_stmt_list: SEMICOLON* EOF; +sql_stmt_list: SEMICOLON* sql_stmt (SEMICOLON+ sql_stmt)* SEMICOLON* EOF; +ansi_sql_stmt_list: SEMICOLON* EOF; -lambda_body: SEMICOLON* (lambda_stmt SEMICOLON+)* RETURN expr SEMICOLON*; +lambda_body: SEMICOLON* (lambda_stmt SEMICOLON+)* RETURN expr SEMICOLON*; lambda_stmt: named_nodes_stmt | import_stmt @@ -38,17 +38,17 @@ sql_stmt_core: | define_action_or_subquery_stmt | if_stmt | for_stmt - | values_stmt - | create_user_stmt - | alter_user_stmt - | create_group_stmt - | alter_group_stmt - | drop_role_stmt + | values_stmt + | create_user_stmt + | alter_user_stmt + | create_group_stmt + | alter_group_stmt + | drop_role_stmt ; expr: or_subexpr (OR or_subexpr)* - | type_name_composite; + | type_name_composite; or_subexpr: and_subexpr (AND and_subexpr)*; @@ -56,14 +56,14 @@ and_subexpr: xor_subexpr (XOR xor_subexpr)*; xor_subexpr: eq_subexpr cond_expr?; -distinct_from_op: IS NOT? DISTINCT FROM; - +distinct_from_op: IS NOT? DISTINCT FROM; + cond_expr: NOT? match_op eq_subexpr (ESCAPE eq_subexpr)? - | NOT? IN COMPACT? in_expr - | (ISNULL | NOTNULL | IS NULL | (IS)? NOT NULL) + | NOT? IN COMPACT? in_expr + | (ISNULL | NOTNULL | IS NULL | (IS)? NOT NULL) | NOT? BETWEEN eq_subexpr AND eq_subexpr - | ((EQUALS | EQUALS2 | NOT_EQUALS | NOT_EQUALS2 | distinct_from_op) eq_subexpr)+ /* order of the eq subexpressions is reversed! */ + | ((EQUALS | EQUALS2 | NOT_EQUALS | NOT_EQUALS2 | distinct_from_op) eq_subexpr)+ /* order of the eq subexpressions is reversed! */ ; match_op: LIKE | ILIKE | GLOB | REGEXP | RLIKE | MATCH; @@ -75,9 +75,9 @@ shift_right: GREATER GREATER; rot_right: GREATER GREATER PIPE; double_question: QUESTION QUESTION; -neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)* - // trailing QUESTIONs are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x) - ((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?; +neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)* + // trailing QUESTIONs are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x) + ((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?; bit_subexpr: add_subexpr ((PLUS | MINUS) add_subexpr)*; @@ -92,12 +92,12 @@ unary_op: PLUS | MINUS | TILDA | NOT; unary_subexpr_suffix: (key_expr | invoke_expr |(DOT (bind_parameter | DIGITS | an_id_or_type)))* (COLLATE an_id)?; unary_casual_subexpr: (id_expr | atom_expr) unary_subexpr_suffix; - -in_unary_casual_subexpr: (id_expr_in | in_atom_expr) unary_subexpr_suffix; + +in_unary_casual_subexpr: (id_expr_in | in_atom_expr) unary_subexpr_suffix; unary_subexpr: unary_casual_subexpr | json_api_expr; -in_unary_subexpr: in_unary_casual_subexpr | json_api_expr; +in_unary_subexpr: in_unary_casual_subexpr | json_api_expr; list_literal: LBRACE_SQUARE expr_list? COMMA? RBRACE_SQUARE; @@ -113,10 +113,10 @@ atom_expr: literal_value | bind_parameter | lambda - | cast_expr - | exists_expr - | case_expr - | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) + | cast_expr + | exists_expr + | case_expr + | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) | value_constructor | bitcast_expr | list_literal @@ -124,39 +124,39 @@ atom_expr: | struct_literal ; -in_atom_expr: - literal_value - | bind_parameter +in_atom_expr: + literal_value + | bind_parameter | lambda - | cast_expr - | case_expr - | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) - | LPAREN select_stmt RPAREN -// TODO: resolve ANTLR error: rule in_atom_expr has non-LL(*) decision due to recursive rule invocations reachable from alts 3,8 -// | LPAREN values_stmt RPAREN + | cast_expr + | case_expr + | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) + | LPAREN select_stmt RPAREN +// TODO: resolve ANTLR error: rule in_atom_expr has non-LL(*) decision due to recursive rule invocations reachable from alts 3,8 +// | LPAREN values_stmt RPAREN | value_constructor | bitcast_expr | list_literal | dict_literal | struct_literal -; - +; + cast_expr: CAST LPAREN expr AS type_name_or_bind RPAREN; - + bitcast_expr: BITCAST LPAREN expr AS type_name_simple RPAREN; -exists_expr: EXISTS LPAREN (select_stmt | values_stmt) RPAREN; - -case_expr: CASE expr? when_expr+ (ELSE expr)? END; - +exists_expr: EXISTS LPAREN (select_stmt | values_stmt) RPAREN; + +case_expr: CASE expr? when_expr+ (ELSE expr)? END; + lambda: smart_parenthesis (ARROW ((LPAREN expr RPAREN) | (LBRACE_CURLY lambda_body RBRACE_CURLY)) )?; -in_expr: in_unary_subexpr; +in_expr: in_unary_subexpr; // ANSI SQL JSON support json_api_expr: json_value | json_exists | json_query; -jsonpath_spec: STRING_VALUE; +jsonpath_spec: STRING_VALUE; json_variable_name: id_expr | STRING_VALUE; @@ -196,31 +196,31 @@ smart_parenthesis: LPAREN named_expr_list? COMMA? RPAREN; expr_list: expr (COMMA expr)*; -pure_column_list: LPAREN an_id (COMMA an_id)* RPAREN; +pure_column_list: LPAREN an_id (COMMA an_id)* RPAREN; -pure_column_or_named: bind_parameter | an_id; +pure_column_or_named: bind_parameter | an_id; pure_column_or_named_list: LPAREN pure_column_or_named (COMMA pure_column_or_named)* RPAREN; -column_name: opt_id_prefix an_id; -without_column_name: (an_id DOT an_id) | an_id_without; +column_name: opt_id_prefix an_id; +without_column_name: (an_id DOT an_id) | an_id_without; -column_list: column_name (COMMA column_name)* COMMA?; -without_column_list: without_column_name (COMMA without_column_name)* COMMA?; +column_list: column_name (COMMA column_name)* COMMA?; +without_column_list: without_column_name (COMMA without_column_name)* COMMA?; -named_expr: expr (AS an_id_or_type)?; +named_expr: expr (AS an_id_or_type)?; named_expr_list: named_expr (COMMA named_expr)*; -invoke_expr: LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN invoke_expr_tail; +invoke_expr: LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN invoke_expr_tail; + +// null_treatment can only happen after window functions LAG/LEAD/NTH/FIRST_VALUE/LAST_VALUE +// filter_clause can only happen after aggregation functions +invoke_expr_tail: + (null_treatment | filter_clause)? (OVER window_name_or_specification)? +; -// null_treatment can only happen after window functions LAG/LEAD/NTH/FIRST_VALUE/LAST_VALUE -// filter_clause can only happen after aggregation functions -invoke_expr_tail: - (null_treatment | filter_clause)? (OVER window_name_or_specification)? -; - using_call_expr: ((an_id_or_type NAMESPACE an_id_or_type) | an_id_expr | bind_parameter | (EXTERNAL FUNCTION)) invoke_expr; - + key_expr: LBRACE_SQUARE expr RBRACE_SQUARE; when_expr: WHEN expr THEN expr; @@ -238,8 +238,8 @@ literal_value: | EMPTY_ACTION ; -bind_parameter: DOLLAR an_id_or_type; -opt_bind_parameter: bind_parameter QUESTION?; +bind_parameter: DOLLAR an_id_or_type; +opt_bind_parameter: bind_parameter QUESTION?; bind_parameter_list: bind_parameter (COMMA bind_parameter)*; named_bind_parameter: bind_parameter (AS bind_parameter)?; @@ -247,34 +247,34 @@ named_bind_parameter_list: named_bind_parameter (COMMA named_bind_parameter)*; signed_number: (PLUS | MINUS)? (integer | real); -type_name_simple: an_id_pure; +type_name_simple: an_id_pure; integer_or_bind: integer | bind_parameter; type_name_tag: id | STRING_VALUE | bind_parameter; struct_arg: type_name_tag COLON type_name_or_bind; -struct_arg_as: type_name_or_bind AS type_name_tag; +struct_arg_as: type_name_or_bind AS type_name_tag; variant_arg: (type_name_tag COLON)? type_name_or_bind; callable_arg: variant_arg (LBRACE_CURLY AUTOMAP RBRACE_CURLY)?; callable_arg_list: callable_arg (COMMA callable_arg)*; type_name_decimal: DECIMAL LPAREN integer_or_bind COMMA integer_or_bind RPAREN; type_name_optional: OPTIONAL LESS type_name_or_bind GREATER; -type_name_tuple: TUPLE (LESS (type_name_or_bind (COMMA type_name_or_bind)* COMMA?)? GREATER | NOT_EQUALS2); -type_name_struct: STRUCT (LESS (struct_arg (COMMA struct_arg)* COMMA?)? GREATER | NOT_EQUALS2); -type_name_variant: VARIANT LESS variant_arg (COMMA variant_arg)* COMMA? GREATER; +type_name_tuple: TUPLE (LESS (type_name_or_bind (COMMA type_name_or_bind)* COMMA?)? GREATER | NOT_EQUALS2); +type_name_struct: STRUCT (LESS (struct_arg (COMMA struct_arg)* COMMA?)? GREATER | NOT_EQUALS2); +type_name_variant: VARIANT LESS variant_arg (COMMA variant_arg)* COMMA? GREATER; type_name_list: LIST LESS type_name_or_bind GREATER; type_name_stream: STREAM LESS type_name_or_bind GREATER; type_name_flow: FLOW LESS type_name_or_bind GREATER; type_name_dict: DICT LESS type_name_or_bind COMMA type_name_or_bind GREATER; type_name_set: SET LESS type_name_or_bind GREATER; -type_name_enum: ENUM LESS type_name_tag (COMMA type_name_tag)* COMMA? GREATER; +type_name_enum: ENUM LESS type_name_tag (COMMA type_name_tag)* COMMA? GREATER; type_name_resource: RESOURCE LESS type_name_tag GREATER; type_name_tagged: TAGGED LESS type_name_or_bind COMMA type_name_tag GREATER; type_name_callable: CALLABLE LESS LPAREN callable_arg_list? COMMA? (LBRACE_SQUARE callable_arg_list RBRACE_SQUARE)? RPAREN ARROW type_name_or_bind GREATER; -type_name_composite: - ( type_name_optional +type_name_composite: + ( type_name_optional | type_name_tuple | type_name_struct | type_name_variant @@ -289,29 +289,29 @@ type_name_composite: | type_name_callable ) QUESTION*; -type_name: - type_name_composite - | (type_name_decimal | type_name_simple) QUESTION*; - +type_name: + type_name_composite + | (type_name_decimal | type_name_simple) QUESTION*; + type_name_or_bind: type_name | bind_parameter; -value_constructor_literal: STRING_VALUE; +value_constructor_literal: STRING_VALUE; value_constructor: - VARIANT LPAREN expr COMMA expr COMMA expr RPAREN + VARIANT LPAREN expr COMMA expr COMMA expr RPAREN | ENUM LPAREN expr COMMA expr RPAREN | CALLABLE LPAREN expr COMMA expr RPAREN ; declare_stmt: DECLARE bind_parameter AS type_name (EQUALS literal_value)?; -module_path: DOT? an_id (DOT an_id)*; +module_path: DOT? an_id (DOT an_id)*; import_stmt: IMPORT module_path SYMBOLS named_bind_parameter_list; export_stmt: EXPORT bind_parameter_list; call_action: (bind_parameter | EMPTY_ACTION) LPAREN expr_list? RPAREN; -inline_action: BEGIN define_action_or_subquery_body END DO; +inline_action: BEGIN define_action_or_subquery_body END DO; do_stmt: DO (call_action | inline_action); -pragma_stmt: PRAGMA opt_id_prefix_or_type an_id (EQUALS pragma_value | LPAREN pragma_value (COMMA pragma_value)* RPAREN)?; +pragma_stmt: PRAGMA opt_id_prefix_or_type an_id (EQUALS pragma_value | LPAREN pragma_value (COMMA pragma_value)* RPAREN)?; pragma_value: signed_number @@ -341,7 +341,7 @@ select_kind_partial: select_kind select_kind: (DISCARD)? (process_core | reduce_core | select_core) (INTO RESULT pure_column_or_named)?; process_core: - PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)? + PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)? (WITH external_call_settings)? (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)? ; @@ -351,14 +351,14 @@ external_call_settings: external_call_param (COMMA external_call_param)*; reduce_core: REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)? - ON column_list USING ALL? using_call_expr (AS an_id)? + ON column_list USING ALL? using_call_expr (AS an_id)? (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)? ; opt_set_quantifier: (ALL | DISTINCT)?; select_core: - (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* COMMA? (WITHOUT without_column_list)? (FROM join_source)? (WHERE expr)? + (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* COMMA? (WITHOUT without_column_list)? (FROM join_source)? (WHERE expr)? group_by_clause? (HAVING expr)? window_clause? ext_order_by_clause? ; @@ -366,7 +366,7 @@ order_by_clause: ORDER BY sort_specification_list; ext_order_by_clause: ASSUME? order_by_clause; -group_by_clause: GROUP COMPACT? BY opt_set_quantifier grouping_element_list; +group_by_clause: GROUP COMPACT? BY opt_set_quantifier grouping_element_list; grouping_element_list: grouping_element (COMMA grouping_element)*; @@ -396,23 +396,23 @@ result_column: | expr ((AS an_id_or_type) | an_id_pure)? ; -join_source: ANY? flatten_source (join_op ANY? flatten_source join_constraint?)*; +join_source: ANY? flatten_source (join_op ANY? flatten_source join_constraint?)*; -named_column: column_name (AS an_id)?; - -flatten_by_arg: +named_column: column_name (AS an_id)?; + +flatten_by_arg: named_column - | LPAREN named_expr_list COMMA? RPAREN + | LPAREN named_expr_list COMMA? RPAREN ; -flatten_source: named_single_source (FLATTEN ((OPTIONAL|LIST|DICT)? BY flatten_by_arg | COLUMNS))?; +flatten_source: named_single_source (FLATTEN ((OPTIONAL|LIST|DICT)? BY flatten_by_arg | COLUMNS))?; -named_single_source: single_source (((AS an_id) | an_id_pure) pure_column_list?)? (sample_clause | tablesample_clause)?; +named_single_source: single_source (((AS an_id) | an_id_pure) pure_column_list?)? (sample_clause | tablesample_clause)?; single_source: table_ref | LPAREN select_stmt RPAREN - | LPAREN values_stmt RPAREN + | LPAREN values_stmt RPAREN ; sample_clause: SAMPLE expr; @@ -425,7 +425,7 @@ repeatable_clause: REPEATABLE LPAREN expr RPAREN; join_op: COMMA - | (NATURAL)? ((LEFT (ONLY | SEMI)? | RIGHT (ONLY | SEMI)? | EXCLUSION | FULL)? (OUTER)? | INNER | CROSS) SORTED? JOIN + | (NATURAL)? ((LEFT (ONLY | SEMI)? | RIGHT (ONLY | SEMI)? | EXCLUSION | FULL)? (OUTER)? | INNER | CROSS) SORTED? JOIN ; join_constraint: @@ -440,9 +440,9 @@ into_values_source: | DEFAULT VALUES ; -values_stmt: VALUES values_source_row_list; - -values_source: values_stmt | select_stmt; +values_stmt: VALUES values_source_row_list; + +values_source: values_stmt | select_stmt; values_source_row_list: values_source_row (COMMA values_source_row)*; values_source_row: LPAREN expr_list RPAREN; @@ -480,7 +480,7 @@ alter_table_action: ; alter_table_add_column: ADD COLUMN? column_schema; -alter_table_drop_column: DROP COLUMN? an_id; +alter_table_drop_column: DROP COLUMN? an_id; alter_table_alter_column: ALTER COLUMN an_id SET family_relation; alter_table_add_column_family: ADD family_entry; alter_table_alter_column_family: ALTER FAMILY an_id SET an_id family_setting_value; @@ -496,18 +496,18 @@ alter_table_drop_changefeed: DROP CHANGEFEED an_id; column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?; family_relation: FAMILY an_id; -column_order_by_specification: an_id (ASC | DESC)?; +column_order_by_specification: an_id (ASC | DESC)?; table_constraint: - PRIMARY KEY LPAREN an_id (COMMA an_id)* RPAREN - | PARTITION BY LPAREN an_id (COMMA an_id)* RPAREN + PRIMARY KEY LPAREN an_id (COMMA an_id)* RPAREN + | PARTITION BY LPAREN an_id (COMMA an_id)* RPAREN | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN ; table_index: INDEX an_id table_index_type - (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? - ON LPAREN an_id_schema (COMMA an_id_schema)* RPAREN - (COVER LPAREN an_id_schema (COMMA an_id_schema)* RPAREN)?; + (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? + ON LPAREN an_id_schema (COMMA an_id_schema)* RPAREN + (COVER LPAREN an_id_schema (COMMA an_id_schema)* RPAREN)?; table_index_type: global_index @@ -549,36 +549,36 @@ split_boundaries: literal_value_list: LPAREN literal_value (COMMA literal_value)* RPAREN; drop_table_stmt: DROP TABLE (IF EXISTS)? simple_table_ref; - -create_user_stmt: CREATE USER role_name create_user_option?; -alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); - -create_group_stmt: CREATE GROUP role_name; -alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); - -drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; - -role_name: an_id_or_type | bind_parameter; -create_user_option: ENCRYPTED? PASSWORD expr; - -action_or_subquery_args: opt_bind_parameter (COMMA opt_bind_parameter)*; - -define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN action_or_subquery_args? RPAREN AS define_action_or_subquery_body END DEFINE; -define_action_or_subquery_body: SEMICOLON* (sql_stmt_core (SEMICOLON+ sql_stmt_core)* SEMICOLON*)?; + +create_user_stmt: CREATE USER role_name create_user_option?; +alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); + +create_group_stmt: CREATE GROUP role_name; +alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); + +drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; + +role_name: an_id_or_type | bind_parameter; +create_user_option: ENCRYPTED? PASSWORD expr; + +action_or_subquery_args: opt_bind_parameter (COMMA opt_bind_parameter)*; + +define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN action_or_subquery_args? RPAREN AS define_action_or_subquery_body END DEFINE; +define_action_or_subquery_body: SEMICOLON* (sql_stmt_core (SEMICOLON+ sql_stmt_core)* SEMICOLON*)?; if_stmt: EVALUATE? IF expr do_stmt (ELSE do_stmt)?; for_stmt: EVALUATE? FOR bind_parameter IN expr do_stmt (ELSE do_stmt)?; -table_ref: (cluster_expr DOT)? AT? (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)* COMMA?)? RPAREN | bind_parameter (LPAREN expr_list? RPAREN)? (VIEW an_id)?) table_hints?; +table_ref: (cluster_expr DOT)? AT? (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)* COMMA?)? RPAREN | bind_parameter (LPAREN expr_list? RPAREN)? (VIEW an_id)?) table_hints?; table_key: id_table_or_type (VIEW an_id)?; -table_arg: AT? named_expr (VIEW an_id)?; +table_arg: AT? named_expr (VIEW an_id)?; table_hints: WITH (table_hint | LPAREN table_hint (COMMA table_hint)* RPAREN); -table_hint: - an_id_hint - | (SCHEMA | COLUMNS) type_name_or_bind - | SCHEMA LPAREN (struct_arg_as (COMMA struct_arg_as)*)? COMMA? RPAREN -; +table_hint: + an_id_hint + | (SCHEMA | COLUMNS) type_name_or_bind + | SCHEMA LPAREN (struct_arg_as (COMMA struct_arg_as)*)? COMMA? RPAREN +; simple_table_ref: (((cluster_expr DOT)? id_or_at) | AT? bind_parameter) table_hints?; into_simple_table_ref: simple_table_ref (ERASE BY pure_column_list)?; @@ -599,11 +599,11 @@ set_target_list: LPAREN set_target (COMMA set_target)* RPAREN; // differ from 2003 for resolve conflict null_treatment: RESPECT NULLS | IGNORE NULLS; -filter_clause: FILTER LPAREN WHERE expr RPAREN; - -window_name_or_specification: window_name | window_specification; +filter_clause: FILTER LPAREN WHERE expr RPAREN; + +window_name_or_specification: window_name | window_specification; -window_name: an_id_window; +window_name: an_id_window; window_clause: WINDOW window_definition_list; @@ -623,19 +623,19 @@ window_specification_details: ; existing_window_name: window_name; -window_partition_clause: PARTITION COMPACT? BY named_expr_list; -window_order_clause: order_by_clause; - +window_partition_clause: PARTITION COMPACT? BY named_expr_list; +window_order_clause: order_by_clause; + window_frame_clause: window_frame_units window_frame_extent window_frame_exclusion?; -window_frame_units: ROWS | RANGE | GROUPS; +window_frame_units: ROWS | RANGE | GROUPS; + +window_frame_extent: window_frame_bound | window_frame_between; -window_frame_extent: window_frame_bound | window_frame_between; +window_frame_between: BETWEEN window_frame_bound AND window_frame_bound; -window_frame_between: BETWEEN window_frame_bound AND window_frame_bound; - -window_frame_bound: - CURRENT ROW - | (expr | UNBOUNDED) (PRECEDING | FOLLOWING) +window_frame_bound: + CURRENT ROW + | (expr | UNBOUNDED) (PRECEDING | FOLLOWING) ; window_frame_exclusion: EXCLUDE CURRENT ROW | EXCLUDE GROUP | EXCLUDE TIES | EXCLUDE NO OTHERS; @@ -645,8 +645,8 @@ use_stmt: USE cluster_expr; subselect_stmt: (LPAREN select_stmt RPAREN | select_unparenthesized_stmt); -// TODO: [fatal] rule named_nodes_stmt has non-LL(*) decision due to recursive rule invocations reachable from alts 1,3 -// named_nodes_stmt: bind_parameter_list EQUALS (expr | subselect_stmt | values_stmt | LPAREN values_stmt RPAREN); +// TODO: [fatal] rule named_nodes_stmt has non-LL(*) decision due to recursive rule invocations reachable from alts 1,3 +// named_nodes_stmt: bind_parameter_list EQUALS (expr | subselect_stmt | values_stmt | LPAREN values_stmt RPAREN); named_nodes_stmt: bind_parameter_list EQUALS (expr | subselect_stmt); commit_stmt: COMMIT; @@ -654,136 +654,136 @@ commit_stmt: COMMIT; rollback_stmt: ROLLBACK; // Special rules that allow to use certain keywords as identifiers. -identifier: ID_PLAIN | ID_QUOTED; -id: identifier | keyword; - -id_schema: - identifier - | keyword_compat - | keyword_expr_uncompat -// | keyword_table_uncompat - | keyword_select_uncompat -// | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat -// | keyword_schema_uncompat -; - -id_expr: - identifier - | keyword_compat -// | keyword_expr_uncompat -// | keyword_table_uncompat -// | keyword_select_uncompat - | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; - -id_expr_in: - identifier - | keyword_compat -// | keyword_expr_uncompat -// | keyword_table_uncompat -// | keyword_select_uncompat - | keyword_alter_uncompat -// | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; - -id_window: - identifier - | keyword_compat - | keyword_expr_uncompat - | keyword_table_uncompat - | keyword_select_uncompat - | keyword_alter_uncompat - | keyword_in_uncompat -// | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; - -id_table: - identifier - | keyword_compat - | keyword_expr_uncompat -// | keyword_table_uncompat - | keyword_select_uncompat -// | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; - -id_without: - identifier - | keyword_compat -// | keyword_expr_uncompat - | keyword_table_uncompat -// | keyword_select_uncompat - | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; - -id_hint: - identifier - | keyword_compat - | keyword_expr_uncompat - | keyword_table_uncompat - | keyword_select_uncompat - | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat -// | keyword_hint_uncompat - | keyword_schema_uncompat -; - -// ANSI-aware versions of various identifiers with support double-quoted identifiers when PRAGMA AnsiQuotedIdentifiers; is present -an_id: id | STRING_VALUE; -an_id_or_type: id_or_type | STRING_VALUE; -an_id_schema: id_schema | STRING_VALUE; -an_id_expr: id_expr | STRING_VALUE; -an_id_expr_in: id_expr_in | STRING_VALUE; -an_id_window: id_window | STRING_VALUE; -an_id_table: id_table | STRING_VALUE; -an_id_without: id_without | STRING_VALUE; -an_id_hint: id_hint | STRING_VALUE; -an_id_pure: identifier | STRING_VALUE; - -opt_id_prefix: (an_id DOT)?; -cluster_expr: (an_id COLON)? (pure_column_or_named | ASTERISK); - +identifier: ID_PLAIN | ID_QUOTED; +id: identifier | keyword; + +id_schema: + identifier + | keyword_compat + | keyword_expr_uncompat +// | keyword_table_uncompat + | keyword_select_uncompat +// | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat +// | keyword_schema_uncompat +; + +id_expr: + identifier + | keyword_compat +// | keyword_expr_uncompat +// | keyword_table_uncompat +// | keyword_select_uncompat + | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; + +id_expr_in: + identifier + | keyword_compat +// | keyword_expr_uncompat +// | keyword_table_uncompat +// | keyword_select_uncompat + | keyword_alter_uncompat +// | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; + +id_window: + identifier + | keyword_compat + | keyword_expr_uncompat + | keyword_table_uncompat + | keyword_select_uncompat + | keyword_alter_uncompat + | keyword_in_uncompat +// | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; + +id_table: + identifier + | keyword_compat + | keyword_expr_uncompat +// | keyword_table_uncompat + | keyword_select_uncompat +// | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; + +id_without: + identifier + | keyword_compat +// | keyword_expr_uncompat + | keyword_table_uncompat +// | keyword_select_uncompat + | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; + +id_hint: + identifier + | keyword_compat + | keyword_expr_uncompat + | keyword_table_uncompat + | keyword_select_uncompat + | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat +// | keyword_hint_uncompat + | keyword_schema_uncompat +; + +// ANSI-aware versions of various identifiers with support double-quoted identifiers when PRAGMA AnsiQuotedIdentifiers; is present +an_id: id | STRING_VALUE; +an_id_or_type: id_or_type | STRING_VALUE; +an_id_schema: id_schema | STRING_VALUE; +an_id_expr: id_expr | STRING_VALUE; +an_id_expr_in: id_expr_in | STRING_VALUE; +an_id_window: id_window | STRING_VALUE; +an_id_table: id_table | STRING_VALUE; +an_id_without: id_without | STRING_VALUE; +an_id_hint: id_hint | STRING_VALUE; +an_id_pure: identifier | STRING_VALUE; + +opt_id_prefix: (an_id DOT)?; +cluster_expr: (an_id COLON)? (pure_column_or_named | ASTERISK); + id_or_type: id | type_id; -opt_id_prefix_or_type: (an_id_or_type DOT)?; -id_or_at: AT? an_id_or_type; -id_table_or_type: an_id_table | type_id; +opt_id_prefix_or_type: (an_id_or_type DOT)?; +id_or_at: AT? an_id_or_type; +id_table_or_type: an_id_table | type_id; id_table_or_at: AT? id_table_or_type; -keyword: - keyword_compat - | keyword_expr_uncompat - | keyword_table_uncompat - | keyword_select_uncompat - | keyword_alter_uncompat - | keyword_in_uncompat - | keyword_window_uncompat - | keyword_hint_uncompat - | keyword_schema_uncompat -; +keyword: + keyword_compat + | keyword_expr_uncompat + | keyword_table_uncompat + | keyword_select_uncompat + | keyword_alter_uncompat + | keyword_in_uncompat + | keyword_window_uncompat + | keyword_hint_uncompat + | keyword_schema_uncompat +; keyword_expr_uncompat: - BETWEEN - | BITCAST + BETWEEN + | BITCAST | CASE | CAST | CUBE @@ -806,54 +806,54 @@ keyword_expr_uncompat: | RETURN | ROLLUP | SELECT - | UNBOUNDED + | UNBOUNDED | WHEN | WHERE ; keyword_table_uncompat: - ANY - | ERASE + ANY + | ERASE | STREAM ; keyword_select_uncompat: ALL | AS - | ASSUME + | ASSUME | DISTINCT - | EXCEPT + | EXCEPT | HAVING - | INTERSECT - | LIMIT - | UNION - | WINDOW - | WITHOUT + | INTERSECT + | LIMIT + | UNION + | WINDOW + | WITHOUT ; keyword_alter_uncompat: COLUMN ; -keyword_in_uncompat: - COMPACT -; - -keyword_window_uncompat: - GROUPS - | RANGE - | ROWS -; - -keyword_hint_uncompat: - SCHEMA - | COLUMNS -; - -keyword_schema_uncompat: - FAMILY -; - +keyword_in_uncompat: + COMPACT +; + +keyword_window_uncompat: + GROUPS + | RANGE + | ROWS +; + +keyword_hint_uncompat: + SCHEMA + | COLUMNS +; + +keyword_schema_uncompat: + FAMILY +; + keyword_compat: ( ABORT | ACTION @@ -885,7 +885,7 @@ keyword_compat: ( | CROSS | CURRENT | DATABASE - | DECIMAL + | DECIMAL | DECLARE | DEFAULT | DEFERRABLE @@ -901,7 +901,7 @@ keyword_compat: ( | EACH | ELSE | EMPTY - | ENCRYPTED + | ENCRYPTED | END | ERROR | ESCAPE @@ -913,7 +913,7 @@ keyword_compat: ( | EXPORT | EXTERNAL | FAIL - | FILTER + | FILTER | FLATTEN | FOLLOWING | FOR @@ -958,7 +958,7 @@ keyword_compat: ( | OVER | PARTITION | PASSING - | PASSWORD + | PASSWORD | PLAN | PRAGMA | PRECEDING @@ -984,16 +984,16 @@ keyword_compat: ( | ROW | SAMPLE | SAVEPOINT - | SEMI + | SEMI | SETS - | SORTED + | SORTED | SUBQUERY | SYMBOLS | SYNC | SYSTEM | TABLE | TABLESAMPLE - | TEMP + | TEMP | TEMPORARY | THEN | TIES @@ -1005,7 +1005,7 @@ keyword_compat: ( | UNKNOWN | UPDATE | UPSERT - | USER + | USER | USING | VACUUM | VALUES @@ -1017,7 +1017,7 @@ keyword_compat: ( ); type_id: - OPTIONAL + OPTIONAL | TUPLE | STRUCT | VARIANT @@ -1033,8 +1033,8 @@ type_id: ; bool_value: (TRUE | FALSE); -real: REAL; -integer: DIGITS | INTEGER_VALUE; +real: REAL; +integer: DIGITS | INTEGER_VALUE; // // Lexer @@ -1062,7 +1062,7 @@ ASTERISK: '*'; SLASH: '/'; BACKSLASH: '\\'; PERCENT: '%'; -SEMICOLON: ';'; +SEMICOLON: ';'; DOT: '.'; COMMA: ','; LPAREN: '('; @@ -1120,7 +1120,7 @@ ALTER: A L T E R; ANALYZE: A N A L Y Z E; AND: A N D; ANSI: A N S I; -ANY: A N Y; +ANY: A N Y; ARRAY: A R R A Y; AS: A S; ASC: A S C; @@ -1145,7 +1145,7 @@ COLLATE: C O L L A T E; COLUMN: C O L U M N; COLUMNS: C O L U M N S; COMMIT: C O M M I T; -COMPACT: C O M P A C T; +COMPACT: C O M P A C T; CONDITIONAL: C O N D I T I O N A L; CONFLICT: C O N F L I C T; CONSTRAINT: C O N S T R A I N T; @@ -1178,7 +1178,7 @@ ELSE: E L S E; ERROR: E R R O R; EMPTY: E M P T Y; EMPTY_ACTION: E M P T Y '_' A C T I O N; -ENCRYPTED: E N C R Y P T E D; +ENCRYPTED: E N C R Y P T E D; END: E N D; ENUM: E N U M; ERASE: E R A S E; @@ -1194,7 +1194,7 @@ EXPORT: E X P O R T; EXTERNAL: E X T E R N A L; FAIL: F A I L; FAMILY: F A M I L Y; -FILTER: F I L T E R; +FILTER: F I L T E R; FLATTEN: F L A T T E N; FLOW: F L O W; FOLLOWING: F O L L O W I N G; @@ -1207,7 +1207,7 @@ GLOB: G L O B; GLOBAL: G L O B A L; GROUP: G R O U P; GROUPING: G R O U P I N G; -GROUPS: G R O U P S; +GROUPS: G R O U P S; HAVING: H A V I N G; HOP: H O P; IF: I F; @@ -1227,9 +1227,9 @@ INTO: I N T O; IS: I S; ISNULL: I S N U L L; JOIN: J O I N; -JSON_EXISTS: J S O N '_' E X I S T S; -JSON_VALUE: J S O N '_' V A L U E; -JSON_QUERY: J S O N '_' Q U E R Y; +JSON_EXISTS: J S O N '_' E X I S T S; +JSON_VALUE: J S O N '_' V A L U E; +JSON_QUERY: J S O N '_' Q U E R Y; KEY: K E Y; LEFT: L E F T; LIKE: L I K E; @@ -1256,7 +1256,7 @@ OUTER: O U T E R; OVER: O V E R; PARTITION: P A R T I T I O N; PASSING: P A S S I N G; -PASSWORD: P A S S W O R D; +PASSWORD: P A S S W O R D; PLAN: P L A N; PRAGMA: P R A G M A; PRECEDING: P R E C E D I N G; @@ -1292,21 +1292,21 @@ SAMPLE: S A M P L E; SAVEPOINT: S A V E P O I N T; SCHEMA: S C H E M A; SELECT: S E L E C T; -SEMI: S E M I; +SEMI: S E M I; SET: S E T; SETS: S E T S; -SORTED: S O R T E D; +SORTED: S O R T E D; STREAM: S T R E A M; STRUCT: S T R U C T; -SUBQUERY: S U B Q U E R Y; +SUBQUERY: S U B Q U E R Y; SYMBOLS: S Y M B O L S; SYNC: S Y N C; SYSTEM: S Y S T E M; TABLE: T A B L E; TABLESAMPLE: T A B L E S A M P L E; TAGGED: T A G G E D; -TEMP: T E M P; -TEMPORARY: T E M P O R A R Y; +TEMP: T E M P; +TEMPORARY: T E M P O R A R Y; THEN: T H E N; TIES: T I E S; TO: T O; @@ -1321,7 +1321,7 @@ UNKNOWN: U N K N O W N; UPDATE: U P D A T E; UPSERT: U P S E R T; USE: U S E; -USER: U S E R; +USER: U S E R; USING: U S I N G; VACUUM: V A C U U M; VALUES: V A L U E S; @@ -1338,27 +1338,27 @@ XOR: X O R; TRUE: T R U E; FALSE: F A L S E; -// YQL Default Lexer: -// GRAMMAR_STRING_CORE_SINGLE = ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) -// GRAMMAR_STRING_CORE_DOUBLE = ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) - -// ANSI Lexer: -// GRAMMAR_STRING_CORE_SINGLE = ~QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) -// GRAMMAR_STRING_CORE_DOUBLE = ~QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) - -fragment STRING_CORE_SINGLE: @GRAMMAR_STRING_CORE_SINGLE@; -fragment STRING_CORE_DOUBLE: @GRAMMAR_STRING_CORE_DOUBLE@; - -fragment STRING_SINGLE: (QUOTE_SINGLE STRING_CORE_SINGLE* QUOTE_SINGLE); -fragment STRING_DOUBLE: (QUOTE_DOUBLE STRING_CORE_DOUBLE* QUOTE_DOUBLE); -fragment STRING_MULTILINE: (DOUBLE_AT .* DOUBLE_AT)+ AT?; +// YQL Default Lexer: +// GRAMMAR_STRING_CORE_SINGLE = ~(QUOTE_SINGLE | BACKSLASH) | (BACKSLASH .) +// GRAMMAR_STRING_CORE_DOUBLE = ~(QUOTE_DOUBLE | BACKSLASH) | (BACKSLASH .) + +// ANSI Lexer: +// GRAMMAR_STRING_CORE_SINGLE = ~QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) +// GRAMMAR_STRING_CORE_DOUBLE = ~QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) + +fragment STRING_CORE_SINGLE: @GRAMMAR_STRING_CORE_SINGLE@; +fragment STRING_CORE_DOUBLE: @GRAMMAR_STRING_CORE_DOUBLE@; + +fragment STRING_SINGLE: (QUOTE_SINGLE STRING_CORE_SINGLE* QUOTE_SINGLE); +fragment STRING_DOUBLE: (QUOTE_DOUBLE STRING_CORE_DOUBLE* QUOTE_DOUBLE); +fragment STRING_MULTILINE: (DOUBLE_AT .* DOUBLE_AT)+ AT?; STRING_VALUE: ((STRING_SINGLE | STRING_DOUBLE | STRING_MULTILINE) (U | Y | J)?); -ID_PLAIN: ('a'..'z' | 'A'..'Z' | '_') ('a'..'z' | 'A'..'Z' | '_' | DIGIT)*; +ID_PLAIN: ('a'..'z' | 'A'..'Z' | '_') ('a'..'z' | 'A'..'Z' | '_' | DIGIT)*; -fragment ID_QUOTED_CORE: '``' | '\\`' | ~'`'; -ID_QUOTED: BACKTICK ID_QUOTED_CORE* BACKTICK; +fragment ID_QUOTED_CORE: '``' | '\\`' | ~'`'; +ID_QUOTED: BACKTICK ID_QUOTED_CORE* BACKTICK; fragment DIGIT: '0'..'9'; fragment HEXDIGIT: '0'..'9' | 'a'..'f' | 'A'..'F'; @@ -1379,12 +1379,12 @@ REAL: BLOB: X QUOTE_SINGLE HEXDIGIT+ QUOTE_SINGLE; -// YQL Default Lexer: -// GRAMMAR_MULTILINE_COMMENT_CORE = . -// ANSI Lexer: -// GRAMMAR_MULTILINE_COMMENT_CORE = MULTILINE_COMMENT | . - -fragment MULTILINE_COMMENT: '/*' ( options {greedy=false;} : @GRAMMAR_MULTILINE_COMMENT_CORE@ )* '*/'; -fragment LINE_COMMENT: '--' ~('\n'|'\r')* ('\r' '\n'? | '\n' | EOF); -WS: (' '|'\r'|'\t'|'\u000C'|'\n') {$channel=HIDDEN;}; -COMMENT: (MULTILINE_COMMENT|LINE_COMMENT) {$channel=HIDDEN;}; +// YQL Default Lexer: +// GRAMMAR_MULTILINE_COMMENT_CORE = . +// ANSI Lexer: +// GRAMMAR_MULTILINE_COMMENT_CORE = MULTILINE_COMMENT | . + +fragment MULTILINE_COMMENT: '/*' ( options {greedy=false;} : @GRAMMAR_MULTILINE_COMMENT_CORE@ )* '*/'; +fragment LINE_COMMENT: '--' ~('\n'|'\r')* ('\r' '\n'? | '\n' | EOF); +WS: (' '|'\r'|'\t'|'\u000C'|'\n') {$channel=HIDDEN;}; +COMMENT: (MULTILINE_COMMENT|LINE_COMMENT) {$channel=HIDDEN;}; diff --git a/ydb/library/yql/sql/v1/aggregation.cpp b/ydb/library/yql/sql/v1/aggregation.cpp index 6688efb623..850e0e5056 100644 --- a/ydb/library/yql/sql/v1/aggregation.cpp +++ b/ydb/library/yql/sql/v1/aggregation.cpp @@ -14,18 +14,18 @@ using namespace NYql; namespace NSQLTranslationV1 { namespace { - bool BlockWindowAggregationWithoutFrameSpec(TPosition pos, TStringBuf name, ISource* src, TContext& ctx) { + bool BlockWindowAggregationWithoutFrameSpec(TPosition pos, TStringBuf name, ISource* src, TContext& ctx) { if (src) { auto winNamePtr = src->GetWindowName(); if (winNamePtr) { auto winSpecPtr = src->FindWindowSpecification(ctx, *winNamePtr); if (!winSpecPtr) { - ctx.Error(pos) << "Failed to use aggregation function " << name << " without window specification or in wrong place"; - return true; + ctx.Error(pos) << "Failed to use aggregation function " << name << " without window specification or in wrong place"; + return true; } } } - return false; + return false; } } @@ -54,7 +54,7 @@ protected: return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -73,9 +73,9 @@ protected: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -111,10 +111,10 @@ protected: } if (AggMode == EAggregateMode::Distinct) { const auto column = Expr->GetColumnName(); - YQL_ENSURE(column); + YQL_ENSURE(column); DistinctKey = *column; - YQL_ENSURE(src); - if (!IsGeneratedKeyColumn && src->GetJoin()) { + YQL_ENSURE(src); + if (!IsGeneratedKeyColumn && src->GetJoin()) { const auto sourcePtr = Expr->GetSourceName(); if (!sourcePtr || !*sourcePtr) { if (!src->IsGroupByColumn(DistinctKey)) { @@ -145,8 +145,8 @@ protected: Y("NthArg", Q("1"), "x"), Y("NthArg", Q("2"), "x"), BuildLambda(Pos, Y("value", "state"), Y("Void")), - Y("NthArg", Q("6"), "x"), - Y("NthArg", Q("7"), "x") + Y("NthArg", Q("6"), "x"), + Y("NthArg", Q("7"), "x") )) )))); } @@ -194,7 +194,7 @@ private: << adjustArgsCount << " or " << (1 + adjustArgsCount) << " arguments, given: " << exprs.size(); return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -224,9 +224,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -298,7 +298,7 @@ private: return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -314,9 +314,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -376,7 +376,7 @@ private: return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -392,9 +392,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -717,7 +717,7 @@ private: TSourcePtr FakeSource; std::multimap<TString, TNodePtr> Percentiles; TNodePtr FactoryPercentile; - const TString* Column = nullptr; + const TString* Column = nullptr; }; TAggregationPtr BuildPercentileFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { @@ -866,7 +866,7 @@ private: return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -889,9 +889,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -1001,9 +1001,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; @@ -1050,7 +1050,7 @@ private: return false; } - if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -1074,9 +1074,9 @@ private: if (!isFactory) { node.Add("Member", "row", Q(Name)); - if (IsOverWindow()) { - src->AddTmpWindowColumn(Name); - } + if (IsOverWindow()) { + src->AddTmpWindowColumn(Name); + } } return true; diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index 9dc3027115..43ef1243f1 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -65,24 +65,24 @@ public: } TVector<TString> columns; columns.reserve(Args.size()); - const bool isJoin = src->GetJoin(); - ISource* composite = src->GetCompositeSource(); + const bool isJoin = src->GetJoin(); + ISource* composite = src->GetCompositeSource(); for (const auto& node: Args) { auto namePtr = node->GetColumnName(); if (!namePtr || !*namePtr) { - ctx.Error(Pos) << "GROUPING function should use columns as arguments"; + ctx.Error(Pos) << "GROUPING function should use columns as arguments"; return false; } - TString column = *namePtr; - if (isJoin) { - auto sourceNamePtr = node->GetSourceName(); - if (sourceNamePtr && !sourceNamePtr->empty()) { - column = DotJoin(*sourceNamePtr, column); - } - } - - if (!src->IsGroupByColumn(column) && !src->IsAlias(EExprSeat::GroupBy, *namePtr) && (!composite || !composite->IsGroupByColumn(column))) { - ctx.Error(node->GetPos()) << "Column '" << column << "' is not a grouping column"; + TString column = *namePtr; + if (isJoin) { + auto sourceNamePtr = node->GetSourceName(); + if (sourceNamePtr && !sourceNamePtr->empty()) { + column = DotJoin(*sourceNamePtr, column); + } + } + + if (!src->IsGroupByColumn(column) && !src->IsAlias(EExprSeat::GroupBy, *namePtr) && (!composite || !composite->IsGroupByColumn(column))) { + ctx.Error(node->GetPos()) << "Column '" << column << "' is not a grouping column"; return false; } columns.emplace_back(column); @@ -129,35 +129,35 @@ public: return TAstListNode::DoInit(ctx, src); } - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override { - if (Args.empty() || Aggr->GetAggregationMode() != EAggregateMode::Distinct) { - return; - } - - auto& expr = Args.front(); - - // need to initialize expr before checking whether it is a column - auto clone = expr->Clone(); - if (!clone->Init(ctx, &src)) { - return; - } - - const auto column = clone->GetColumnName(); - if (column) { - return; - } - - auto tmpColumn = src.MakeLocalName("_yql_preagg_" + Name); - YQL_ENSURE(!expr->GetLabel()); - expr->SetLabel(tmpColumn); - - PreaggregateExpr = expr; - exprs.push_back(PreaggregateExpr); - expr = BuildColumn(expr->GetPos(), tmpColumn); - - Aggr->MarkKeyColumnAsGenerated(); - } - + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override { + if (Args.empty() || Aggr->GetAggregationMode() != EAggregateMode::Distinct) { + return; + } + + auto& expr = Args.front(); + + // need to initialize expr before checking whether it is a column + auto clone = expr->Clone(); + if (!clone->Init(ctx, &src)) { + return; + } + + const auto column = clone->GetColumnName(); + if (column) { + return; + } + + auto tmpColumn = src.MakeLocalName("_yql_preagg_" + Name); + YQL_ENSURE(!expr->GetLabel()); + expr->SetLabel(tmpColumn); + + PreaggregateExpr = expr; + exprs.push_back(PreaggregateExpr); + expr = BuildColumn(expr->GetPos(), tmpColumn); + + Aggr->MarkKeyColumnAsGenerated(); + } + TNodePtr DoClone() const final { TAggregationPtr aggrClone = static_cast<IAggregation*>(Aggr->Clone().Release()); return new TBasicAggrFunc(Pos, Name, aggrClone, CloneContainer(Args)); @@ -169,14 +169,14 @@ public: private: bool DoInitAggregation(TContext& ctx, ISource* src) { - if (PreaggregateExpr) { - YQL_ENSURE(PreaggregateExpr->HasState(ENodeState::Initialized)); - if (PreaggregateExpr->IsAggregated() && !PreaggregateExpr->IsAggregationKey() && !Aggr->IsOverWindow()) { - ctx.Error(Aggr->GetPos()) << "Aggregation of aggregated values is forbidden for no window functions"; - return false; - } - } - + if (PreaggregateExpr) { + YQL_ENSURE(PreaggregateExpr->HasState(ENodeState::Initialized)); + if (PreaggregateExpr->IsAggregated() && !PreaggregateExpr->IsAggregationKey() && !Aggr->IsOverWindow()) { + ctx.Error(Aggr->GetPos()) << "Aggregation of aggregated values is forbidden for no window functions"; + return false; + } + } + if (!Aggr->InitAggr(ctx, false, src, *this, Args)) { return false; } @@ -184,11 +184,11 @@ private: } void DoUpdateState() const final { - State.Set(ENodeState::Const, !Args.empty() && AllOf(Args, [](const auto& arg){ return arg->IsConstant(); })); + State.Set(ENodeState::Const, !Args.empty() && AllOf(Args, [](const auto& arg){ return arg->IsConstant(); })); State.Set(ENodeState::Aggregated); } - TNodePtr PreaggregateExpr; + TNodePtr PreaggregateExpr; protected: const TString Name; TAggregationPtr Aggr; @@ -303,12 +303,12 @@ public: return {}; } - void DoUpdateState() const override { - YQL_ENSURE(Atom); - State.Set(ENodeState::Const, Atom->IsConstant()); - State.Set(ENodeState::Aggregated, Atom->IsAggregated()); - State.Set(ENodeState::OverWindow, Atom->IsOverWindow()); - } + void DoUpdateState() const override { + YQL_ENSURE(Atom); + State.Set(ENodeState::Const, Atom->IsConstant()); + State.Set(ENodeState::Aggregated, Atom->IsAggregated()); + State.Set(ENodeState::OverWindow, Atom->IsOverWindow()); + } private: TNodePtr Node; TNodePtr Atom; @@ -443,7 +443,7 @@ public: TTableName(TPosition pos, const TVector<TNodePtr>& args, const TString& service) : TCallNode(pos, "TableName", 0, 2, args) , Service(service) - , EmptyArgs(args.empty()) + , EmptyArgs(args.empty()) { } @@ -453,23 +453,23 @@ public: } if (Args.empty()) { - if (!src) { - ctx.Error(Pos) << "Unable to use TableName() without source"; - return false; - } - - // TODO: TablePath() and TableRecordIndex() have more strict limitations - if (src->GetJoin()) { - ctx.Warning(Pos, - TIssuesIds::YQL_EMPTY_TABLENAME_RESULT) << "TableName() may produce empty result when used in ambiguous context (with JOIN)"; - } - - if (src->HasAggregations()) { - ctx.Warning(Pos, - TIssuesIds::YQL_EMPTY_TABLENAME_RESULT) << "TableName() will produce empty result when used with aggregation.\n" - "Please consult https://yql.yandex-team.ru/docs/yt/builtins/basic/#tablepath for possible workaround"; - } - + if (!src) { + ctx.Error(Pos) << "Unable to use TableName() without source"; + return false; + } + + // TODO: TablePath() and TableRecordIndex() have more strict limitations + if (src->GetJoin()) { + ctx.Warning(Pos, + TIssuesIds::YQL_EMPTY_TABLENAME_RESULT) << "TableName() may produce empty result when used in ambiguous context (with JOIN)"; + } + + if (src->HasAggregations()) { + ctx.Warning(Pos, + TIssuesIds::YQL_EMPTY_TABLENAME_RESULT) << "TableName() will produce empty result when used with aggregation.\n" + "Please consult https://yql.yandex-team.ru/docs/yt/builtins/basic/#tablepath for possible workaround"; + } + Args.push_back(Y("TablePath", Y("DependsOn", "row"))); } @@ -498,16 +498,16 @@ public: } void DoUpdateState() const override { - if (EmptyArgs) { - State.Set(ENodeState::Const, false); - } else { - TCallNode::DoUpdateState(); - } + if (EmptyArgs) { + State.Set(ENodeState::Const, false); + } else { + TCallNode::DoUpdateState(); + } } private: TString Service; - const bool EmptyArgs; + const bool EmptyArgs; }; class TYqlParseType final : public INode { @@ -542,9 +542,9 @@ public: return new TYqlParseType(Pos, Args); } - void DoUpdateState() const final { - State.Set(ENodeState::Const); - } + void DoUpdateState() const final { + State.Set(ENodeState::Const); + } private: TVector<TNodePtr> Args; }; @@ -1324,7 +1324,7 @@ private: class TYqlIn final: public TCallNode { public: TYqlIn(TPosition pos, const TVector<TNodePtr>& args) - : TCallNode(pos, "IN", 3, 3, args) + : TCallNode(pos, "IN", 3, 3, args) {} private: @@ -1336,68 +1336,68 @@ private: return false; } - auto key = Args[0]; + auto key = Args[0]; auto inNode = Args[1]; - auto hints = Args[2]; - - const auto pos = inNode->GetPos(); - - if (!key->Init(ctx, src)) { - return false; - } - - if (!inNode->Init(ctx, inNode->GetSource() ? nullptr : src)) { - return false; - } - + auto hints = Args[2]; + + const auto pos = inNode->GetPos(); + + if (!key->Init(ctx, src)) { + return false; + } + + if (!inNode->Init(ctx, inNode->GetSource() ? nullptr : src)) { + return false; + } + if (inNode->GetLiteral("String")) { - ctx.Error(pos) << "Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source"; - return false; - } - - if (inNode->GetTupleSize() == 1) { - auto singleElement = inNode->GetTupleElement(0); - // TODO: 'IN ((select ...))' is parsed exactly like 'IN (select ...)' instead of a single element tuple - if (singleElement->GetSource() || singleElement->IsSelect()) { - TStringBuf parenKind = singleElement->GetSource() ? "" : "external "; - ctx.Warning(pos, - TIssuesIds::YQL_CONST_SUBREQUEST_IN_LIST) << "Using subrequest in scalar context after IN, " - << "perhaps you should remove " - << parenKind << "parenthesis here"; - } - } - - TVector<TNodePtr> hintElements; - for (size_t i = 0; i < hints->GetTupleSize(); ++i) { - hintElements.push_back(hints->GetTupleElement(i)); - } - + ctx.Error(pos) << "Unable to use IN predicate with string argument, it won't search substring - " + "expecting tuple, list, dict or single column table source"; + return false; + } + + if (inNode->GetTupleSize() == 1) { + auto singleElement = inNode->GetTupleElement(0); + // TODO: 'IN ((select ...))' is parsed exactly like 'IN (select ...)' instead of a single element tuple + if (singleElement->GetSource() || singleElement->IsSelect()) { + TStringBuf parenKind = singleElement->GetSource() ? "" : "external "; + ctx.Warning(pos, + TIssuesIds::YQL_CONST_SUBREQUEST_IN_LIST) << "Using subrequest in scalar context after IN, " + << "perhaps you should remove " + << parenKind << "parenthesis here"; + } + } + + TVector<TNodePtr> hintElements; + for (size_t i = 0; i < hints->GetTupleSize(); ++i) { + hintElements.push_back(hints->GetTupleElement(i)); + } + if (inNode->GetSource() || inNode->IsSelect()) { - hintElements.push_back(BuildHint(pos, "tableSource")); + hintElements.push_back(BuildHint(pos, "tableSource")); } - if (!ctx.AnsiInForEmptyOrNullableItemsCollections.Defined()) { - hintElements.push_back(BuildHint(pos, "warnNoAnsi")); - } else if (*ctx.AnsiInForEmptyOrNullableItemsCollections) { - hintElements.push_back(BuildHint(pos, "ansi")); - } - - OpName = "SqlIn"; - MinArgs = MaxArgs = 3; + if (!ctx.AnsiInForEmptyOrNullableItemsCollections.Defined()) { + hintElements.push_back(BuildHint(pos, "warnNoAnsi")); + } else if (*ctx.AnsiInForEmptyOrNullableItemsCollections) { + hintElements.push_back(BuildHint(pos, "ansi")); + } + + OpName = "SqlIn"; + MinArgs = MaxArgs = 3; Args = { - inNode->GetSource() ? inNode->GetSource() : inNode, + inNode->GetSource() ? inNode->GetSource() : inNode, key, - BuildTuple(pos, hintElements) + BuildTuple(pos, hintElements) }; return TCallNode::DoInit(ctx, src); } - static TNodePtr BuildHint(TPosition pos, const TString& name) { - return BuildTuple(pos, { BuildQuotedAtom(pos, name, NYql::TNodeFlags::Default) }); - } - + static TNodePtr BuildHint(TPosition pos, const TString& name) { + return BuildTuple(pos, { BuildQuotedAtom(pos, name, NYql::TNodeFlags::Default) }); + } + TString GetOpName() const override { return "IN predicate"; } @@ -1507,19 +1507,19 @@ public: if (!ValidateArguments(ctx)) { return false; } - - bool hasError = false; - for (auto& arg: Args) { - if (!arg->Init(ctx, src)) { - hasError = true; - continue; - } - } - - if (hasError) { - return false; - } - + + bool hasError = false; + for (auto& arg: Args) { + if (!arg->Init(ctx, src)) { + hasError = true; + continue; + } + } + + if (hasError) { + return false; + } + PrecacheState(); const auto memberPos = Args[0]->GetPos(); @@ -1538,7 +1538,7 @@ public: TVector<TNodePtr> column; auto namePtr = Args[0]->GetColumnName(); if (!namePtr || !*namePtr) { - ctx.Error(Pos) << GetCallExplain() << " expects column name as first argument"; + ctx.Error(Pos) << GetCallExplain() << " expects column name as first argument"; return false; } auto memberName = *namePtr; @@ -1573,14 +1573,14 @@ public: }; template <bool Join> -class TTableRow final : public INode { +class TTableRow final : public INode { public: TTableRow(TPosition pos, const TVector<TNodePtr>& args) : TTableRow(pos, args.size()) {} TTableRow(TPosition pos, ui32 argsCount) - : INode(pos) + : INode(pos) , ArgsCount(argsCount) {} @@ -1596,56 +1596,56 @@ public: } src->AllColumns(); - const bool isJoin = src->GetJoin(); + const bool isJoin = src->GetJoin(); if (!Join && ctx.SimpleColumns && isJoin) { - TNodePtr block = Y(); - const auto& sameKeyMap = src->GetJoin()->GetSameKeysMap(); - if (sameKeyMap) { - block = L(block, Y("let", "flatSameKeys", "row")); - for (const auto& sameKeysPair: sameKeyMap) { - const auto& column = sameKeysPair.first; - auto keys = Y("Coalesce"); - auto sameSourceIter = sameKeysPair.second.begin(); - for (auto end = sameKeysPair.second.end(); sameSourceIter != end; ++sameSourceIter) { - auto addKeyNode = Q(DotJoin(*sameSourceIter, column)); - keys = L(keys, Y("TryMember", "row", addKeyNode, Y("Null"))); - } - - block = L(block, Y("let", "flatSameKeys", Y("AddMember", "flatSameKeys", Q(column), keys))); - sameSourceIter = sameKeysPair.second.begin(); - for (auto end = sameKeysPair.second.end(); sameSourceIter != end; ++sameSourceIter) { - auto removeKeyNode = Q(DotJoin(*sameSourceIter, column)); - block = L(block, Y("let", "flatSameKeys", Y("ForceRemoveMember", "flatSameKeys", removeKeyNode))); - } - } - block = L(block, Y("let", "row", "flatSameKeys")); - } - - auto members = Y(); - for (auto& joinLabel: src->GetJoin()->GetJoinLabels()) { - members = L(members, BuildQuotedAtom(Pos, joinLabel + ".")); - } - block = L(block, Y("let", "res", Y("DivePrefixMembers", "row", Q(members)))); - - for (const auto& sameKeysPair: src->GetJoin()->GetSameKeysMap()) { - const auto& column = sameKeysPair.first; - auto addMemberKeyNode = Y("Member", "row", Q(column)); - block = L(block, Y("let", "res", Y("AddMember", "res", Q(column), addMemberKeyNode))); - } - - Node = Y("block", Q(L(block, Y("return", "res")))); - } else { + TNodePtr block = Y(); + const auto& sameKeyMap = src->GetJoin()->GetSameKeysMap(); + if (sameKeyMap) { + block = L(block, Y("let", "flatSameKeys", "row")); + for (const auto& sameKeysPair: sameKeyMap) { + const auto& column = sameKeysPair.first; + auto keys = Y("Coalesce"); + auto sameSourceIter = sameKeysPair.second.begin(); + for (auto end = sameKeysPair.second.end(); sameSourceIter != end; ++sameSourceIter) { + auto addKeyNode = Q(DotJoin(*sameSourceIter, column)); + keys = L(keys, Y("TryMember", "row", addKeyNode, Y("Null"))); + } + + block = L(block, Y("let", "flatSameKeys", Y("AddMember", "flatSameKeys", Q(column), keys))); + sameSourceIter = sameKeysPair.second.begin(); + for (auto end = sameKeysPair.second.end(); sameSourceIter != end; ++sameSourceIter) { + auto removeKeyNode = Q(DotJoin(*sameSourceIter, column)); + block = L(block, Y("let", "flatSameKeys", Y("ForceRemoveMember", "flatSameKeys", removeKeyNode))); + } + } + block = L(block, Y("let", "row", "flatSameKeys")); + } + + auto members = Y(); + for (auto& joinLabel: src->GetJoin()->GetJoinLabels()) { + members = L(members, BuildQuotedAtom(Pos, joinLabel + ".")); + } + block = L(block, Y("let", "res", Y("DivePrefixMembers", "row", Q(members)))); + + for (const auto& sameKeysPair: src->GetJoin()->GetSameKeysMap()) { + const auto& column = sameKeysPair.first; + auto addMemberKeyNode = Y("Member", "row", Q(column)); + block = L(block, Y("let", "res", Y("AddMember", "res", Q(column), addMemberKeyNode))); + } + + Node = Y("block", Q(L(block, Y("return", "res")))); + } else { Node = ctx.EnableSystemColumns ? Y("RemoveSystemMembers", "row") : BuildAtom(Pos, "row", 0); - } + } return true; } TAstNode* Translate(TContext& ctx) const override { - Y_VERIFY_DEBUG(Node); - return Node->Translate(ctx); - } - - void DoUpdateState() const override { + Y_VERIFY_DEBUG(Node); + return Node->Translate(ctx); + } + + void DoUpdateState() const override { State.Set(ENodeState::Const, false); } @@ -1654,8 +1654,8 @@ public: } private: - const size_t ArgsCount; - TNodePtr Node; + const size_t ArgsCount; + TNodePtr Node; }; TTableRows::TTableRows(TPosition pos, const TVector<TNodePtr>& args) @@ -1689,206 +1689,206 @@ TNodePtr TTableRows::DoClone() const { return MakeIntrusive<TTableRows>(Pos, ArgsCount); } -TSessionWindow::TSessionWindow(TPosition pos, const TVector<TNodePtr>& args) - : INode(pos) - , Args(args) - , FakeSource(BuildFakeSource(pos)) - , Valid(false) -{} - -void TSessionWindow::MarkValid() { - YQL_ENSURE(!HasState(ENodeState::Initialized)); - Valid = true; -} - -TNodePtr TSessionWindow::BuildTraits(const TString& label) const { - YQL_ENSURE(HasState(ENodeState::Initialized)); - - auto trueNode = Y("Bool", Q("true")); - - if (Args.size() == 2) { - auto timeExpr = Args[0]; - auto timeoutExpr = Args[1]; - - auto coalesceLess = [&](auto first, auto second) { - // first < second ?? true - return Y("Coalesce", Y("<", first, second), trueNode); - }; - - auto absDelta = Y("If", - coalesceLess("prev", "curr"), - Y("-", "curr", "prev"), - Y("-", "prev", "curr")); - - auto newSessionPred = Y("And", Y("AggrNotEquals", "curr", "prev"), coalesceLess(timeoutExpr, absDelta)); - auto timeoutLambda = BuildLambda(timeoutExpr->GetPos(), Y("prev", "curr"), newSessionPred); - auto sortSpec = Y("SortTraits", Y("TypeOf", label), trueNode, BuildLambda(Pos, Y("row"), Y("PersistableRepr", timeExpr))); - - return Y("SessionWindowTraits", - Y("TypeOf", label), - sortSpec, - BuildLambda(Pos, Y("row"), timeExpr), - timeoutLambda); - } - - auto orderExpr = Args[0]; - auto initLambda = Args[1]; - auto updateLambda = Args[2]; - auto calculateLambda = Args[3]; - - auto sortSpec = Y("SortTraits", Y("TypeOf", label), trueNode, BuildLambda(Pos, Y("row"), Y("PersistableRepr", orderExpr))); - - return Y("SessionWindowTraits", - Y("TypeOf", label), - sortSpec, - initLambda, - updateLambda, - calculateLambda); -} - -bool TSessionWindow::DoInit(TContext& ctx, ISource* src) { - if (!src || src->IsFake()) { - ctx.Error(Pos) << "SessionWindow requires data source"; - return false; - } - - if (!(Args.size() == 2 || Args.size() == 4)) { - ctx.Error(Pos) << "SessionWindow requires either two or four arguments"; - return false; - } - - if (!Valid) { - ctx.Error(Pos) << "SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression"; - return false; - } - - if (Args.size() == 2) { - auto timeExpr = Args[0]; - auto timeoutExpr = Args[1]; - return timeExpr->Init(ctx, src) && timeoutExpr->Init(ctx, FakeSource.Get()); - } - - auto orderExpr = Args[0]; - auto initLambda = Args[1]; - auto updateLambda = Args[2]; - auto calculateLambda = Args[3]; - - return orderExpr->Init(ctx, src) && initLambda->Init(ctx, FakeSource.Get()) && - updateLambda->Init(ctx, FakeSource.Get()) && calculateLambda->Init(ctx, FakeSource.Get()); -} - -TAstNode* TSessionWindow::Translate(TContext&) const { - YQL_ENSURE(false, "Translate is called for SessionWindow"); - return nullptr; -} - -void TSessionWindow::DoUpdateState() const { - State.Set(ENodeState::Const, false); -} - -TNodePtr TSessionWindow::DoClone() const { - return new TSessionWindow(Pos, CloneContainer(Args)); -} - -TString TSessionWindow::GetOpName() const { - return "SessionWindow"; -} - -template<bool IsStart> -class TSessionStart final : public INode { -public: - TSessionStart(TPosition pos, const TVector<TNodePtr>& args) - : INode(pos) - , ArgsCount(args.size()) - { - } -private: - TSessionStart(TPosition pos, size_t argsCount) - : INode(pos) - , ArgsCount(argsCount) - {} - - bool DoInit(TContext& ctx, ISource* src) override { - if (!src || src->IsFake()) { - ctx.Error(Pos) << GetOpName() << " requires data source"; - return false; - } - - if (ArgsCount > 0) { - ctx.Error(Pos) << GetOpName() << " requires exactly 0 arguments"; - return false; - } - - auto windowName = src->GetWindowName(); - OverWindow = windowName != nullptr; - TNodePtr sessionWindow; - if (windowName) { - auto spec = src->FindWindowSpecification(ctx, *windowName); - if (!spec) { - return false; - } - sessionWindow = spec->Session; - if (!sessionWindow) { - ctx.Error(Pos) << GetOpName() << " can not be used with window " << *windowName << ": SessionWindow specification is missing in PARTITION BY"; - return false; - } - } else { - sessionWindow = src->GetSessionWindowSpec(); - if (!sessionWindow) { - TString extra; - if (src->IsOverWindowSource()) { - extra = ". Maybe you forgot to add OVER `window_name`?"; - } - if (src->HasAggregations()) { - ctx.Error(Pos) << GetOpName() << " can not be used here: SessionWindow specification is missing in GROUP BY" << extra; - } else { - ctx.Error(Pos) << GetOpName() << " can not be used without aggregation by SessionWindow" << extra; - } - return false; - } - - if (!IsStart) { - ctx.Error(Pos) << GetOpName() << " with GROUP BY is not supported yet"; - return false; - } - } - - YQL_ENSURE(sessionWindow->GetLabel()); - Node = Y("Member", "row", BuildQuotedAtom(Pos, sessionWindow->GetLabel())); - if (OverWindow) { - Node = Y("Member", Node, BuildQuotedAtom(Pos, IsStart ? "start" : "state")); - } - return true; - } - - TAstNode* Translate(TContext& ctx) const override { - Y_VERIFY_DEBUG(Node); - return Node->Translate(ctx); - } - - void DoUpdateState() const override { - State.Set(ENodeState::Const, false); - if (OverWindow) { - State.Set(ENodeState::OverWindow, true); - } else if (IsStart) { - State.Set(ENodeState::Aggregated, true); - } - } - - TNodePtr DoClone() const override { - return new TSessionStart<IsStart>(Pos, ArgsCount); - } - - TString GetOpName() const override { - return IsStart ? "SessionStart" : "SessionState"; - } - - const size_t ArgsCount; - bool OverWindow = false; - TNodePtr Node; -}; - - +TSessionWindow::TSessionWindow(TPosition pos, const TVector<TNodePtr>& args) + : INode(pos) + , Args(args) + , FakeSource(BuildFakeSource(pos)) + , Valid(false) +{} + +void TSessionWindow::MarkValid() { + YQL_ENSURE(!HasState(ENodeState::Initialized)); + Valid = true; +} + +TNodePtr TSessionWindow::BuildTraits(const TString& label) const { + YQL_ENSURE(HasState(ENodeState::Initialized)); + + auto trueNode = Y("Bool", Q("true")); + + if (Args.size() == 2) { + auto timeExpr = Args[0]; + auto timeoutExpr = Args[1]; + + auto coalesceLess = [&](auto first, auto second) { + // first < second ?? true + return Y("Coalesce", Y("<", first, second), trueNode); + }; + + auto absDelta = Y("If", + coalesceLess("prev", "curr"), + Y("-", "curr", "prev"), + Y("-", "prev", "curr")); + + auto newSessionPred = Y("And", Y("AggrNotEquals", "curr", "prev"), coalesceLess(timeoutExpr, absDelta)); + auto timeoutLambda = BuildLambda(timeoutExpr->GetPos(), Y("prev", "curr"), newSessionPred); + auto sortSpec = Y("SortTraits", Y("TypeOf", label), trueNode, BuildLambda(Pos, Y("row"), Y("PersistableRepr", timeExpr))); + + return Y("SessionWindowTraits", + Y("TypeOf", label), + sortSpec, + BuildLambda(Pos, Y("row"), timeExpr), + timeoutLambda); + } + + auto orderExpr = Args[0]; + auto initLambda = Args[1]; + auto updateLambda = Args[2]; + auto calculateLambda = Args[3]; + + auto sortSpec = Y("SortTraits", Y("TypeOf", label), trueNode, BuildLambda(Pos, Y("row"), Y("PersistableRepr", orderExpr))); + + return Y("SessionWindowTraits", + Y("TypeOf", label), + sortSpec, + initLambda, + updateLambda, + calculateLambda); +} + +bool TSessionWindow::DoInit(TContext& ctx, ISource* src) { + if (!src || src->IsFake()) { + ctx.Error(Pos) << "SessionWindow requires data source"; + return false; + } + + if (!(Args.size() == 2 || Args.size() == 4)) { + ctx.Error(Pos) << "SessionWindow requires either two or four arguments"; + return false; + } + + if (!Valid) { + ctx.Error(Pos) << "SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression"; + return false; + } + + if (Args.size() == 2) { + auto timeExpr = Args[0]; + auto timeoutExpr = Args[1]; + return timeExpr->Init(ctx, src) && timeoutExpr->Init(ctx, FakeSource.Get()); + } + + auto orderExpr = Args[0]; + auto initLambda = Args[1]; + auto updateLambda = Args[2]; + auto calculateLambda = Args[3]; + + return orderExpr->Init(ctx, src) && initLambda->Init(ctx, FakeSource.Get()) && + updateLambda->Init(ctx, FakeSource.Get()) && calculateLambda->Init(ctx, FakeSource.Get()); +} + +TAstNode* TSessionWindow::Translate(TContext&) const { + YQL_ENSURE(false, "Translate is called for SessionWindow"); + return nullptr; +} + +void TSessionWindow::DoUpdateState() const { + State.Set(ENodeState::Const, false); +} + +TNodePtr TSessionWindow::DoClone() const { + return new TSessionWindow(Pos, CloneContainer(Args)); +} + +TString TSessionWindow::GetOpName() const { + return "SessionWindow"; +} + +template<bool IsStart> +class TSessionStart final : public INode { +public: + TSessionStart(TPosition pos, const TVector<TNodePtr>& args) + : INode(pos) + , ArgsCount(args.size()) + { + } +private: + TSessionStart(TPosition pos, size_t argsCount) + : INode(pos) + , ArgsCount(argsCount) + {} + + bool DoInit(TContext& ctx, ISource* src) override { + if (!src || src->IsFake()) { + ctx.Error(Pos) << GetOpName() << " requires data source"; + return false; + } + + if (ArgsCount > 0) { + ctx.Error(Pos) << GetOpName() << " requires exactly 0 arguments"; + return false; + } + + auto windowName = src->GetWindowName(); + OverWindow = windowName != nullptr; + TNodePtr sessionWindow; + if (windowName) { + auto spec = src->FindWindowSpecification(ctx, *windowName); + if (!spec) { + return false; + } + sessionWindow = spec->Session; + if (!sessionWindow) { + ctx.Error(Pos) << GetOpName() << " can not be used with window " << *windowName << ": SessionWindow specification is missing in PARTITION BY"; + return false; + } + } else { + sessionWindow = src->GetSessionWindowSpec(); + if (!sessionWindow) { + TString extra; + if (src->IsOverWindowSource()) { + extra = ". Maybe you forgot to add OVER `window_name`?"; + } + if (src->HasAggregations()) { + ctx.Error(Pos) << GetOpName() << " can not be used here: SessionWindow specification is missing in GROUP BY" << extra; + } else { + ctx.Error(Pos) << GetOpName() << " can not be used without aggregation by SessionWindow" << extra; + } + return false; + } + + if (!IsStart) { + ctx.Error(Pos) << GetOpName() << " with GROUP BY is not supported yet"; + return false; + } + } + + YQL_ENSURE(sessionWindow->GetLabel()); + Node = Y("Member", "row", BuildQuotedAtom(Pos, sessionWindow->GetLabel())); + if (OverWindow) { + Node = Y("Member", Node, BuildQuotedAtom(Pos, IsStart ? "start" : "state")); + } + return true; + } + + TAstNode* Translate(TContext& ctx) const override { + Y_VERIFY_DEBUG(Node); + return Node->Translate(ctx); + } + + void DoUpdateState() const override { + State.Set(ENodeState::Const, false); + if (OverWindow) { + State.Set(ENodeState::OverWindow, true); + } else if (IsStart) { + State.Set(ENodeState::Aggregated, true); + } + } + + TNodePtr DoClone() const override { + return new TSessionStart<IsStart>(Pos, ArgsCount); + } + + TString GetOpName() const override { + return IsStart ? "SessionStart" : "SessionState"; + } + + const size_t ArgsCount; + bool OverWindow = false; + TNodePtr Node; +}; + + TNodePtr BuildUdfUserTypeArg(TPosition pos, const TVector<TNodePtr>& args, TNodePtr customUserType) { TVector<TNodePtr> argsTypeItems; for (auto& arg : args) { @@ -1942,78 +1942,78 @@ TVector<TNodePtr> BuildUdfArgs(const TContext& ctx, TPosition pos, const TVector return udfArgs; } -TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, - TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, TMaybe<TString> typeConfig) -{ - const TString fullName = module + "." + name; - TNodePtr callable; - if (to_lower(module) == "@yql") { - callable = BuildCallable(pos, module, name, {}); - } else if (!ctx.Settings.EnableGenericUdfs) { - auto varName = ctx.AddSimpleUdf(fullName); - callable = new TAstAtomNodeImpl(pos, varName, TNodeFlags::ArbitraryContent); - } - - if (callable) { - TVector<TNodePtr> applyArgs = { callable }; - applyArgs.insert(applyArgs.end(), args.begin(), args.end()); - return new TCallNodeImpl(pos, namedArgs ? "NamedApply" : "Apply", applyArgs); - } - - TVector<TNodePtr> sqlCallArgs; - sqlCallArgs.push_back(BuildQuotedAtom(pos, fullName)); - if (namedArgs) { - auto tupleNodePtr = dynamic_cast<const TTupleNode*>(positionalArgs.Get()); - YQL_ENSURE(tupleNodePtr); - TNodePtr positionalArgsNode = new TCallNodeImpl(pos, "PositionalArgs", tupleNodePtr->Elements()); - sqlCallArgs.push_back(BuildTuple(pos, { positionalArgsNode, namedArgs })); - } else { - TNodePtr positionalArgsNode = new TCallNodeImpl(pos, "PositionalArgs", args); - sqlCallArgs.push_back(BuildTuple(pos, { positionalArgsNode })); - } - - // optional arguments - if (customUserType) { - sqlCallArgs.push_back(customUserType); - } else if (typeConfig) { - sqlCallArgs.push_back(new TCallNodeImpl(pos, "TupleType", {})); - } - - if (typeConfig) { - sqlCallArgs.push_back(BuildQuotedAtom(pos, *typeConfig)); - } - - return new TCallNodeImpl(pos, "SqlCall", sqlCallArgs); -} - +TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, + TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, TMaybe<TString> typeConfig) +{ + const TString fullName = module + "." + name; + TNodePtr callable; + if (to_lower(module) == "@yql") { + callable = BuildCallable(pos, module, name, {}); + } else if (!ctx.Settings.EnableGenericUdfs) { + auto varName = ctx.AddSimpleUdf(fullName); + callable = new TAstAtomNodeImpl(pos, varName, TNodeFlags::ArbitraryContent); + } + + if (callable) { + TVector<TNodePtr> applyArgs = { callable }; + applyArgs.insert(applyArgs.end(), args.begin(), args.end()); + return new TCallNodeImpl(pos, namedArgs ? "NamedApply" : "Apply", applyArgs); + } + + TVector<TNodePtr> sqlCallArgs; + sqlCallArgs.push_back(BuildQuotedAtom(pos, fullName)); + if (namedArgs) { + auto tupleNodePtr = dynamic_cast<const TTupleNode*>(positionalArgs.Get()); + YQL_ENSURE(tupleNodePtr); + TNodePtr positionalArgsNode = new TCallNodeImpl(pos, "PositionalArgs", tupleNodePtr->Elements()); + sqlCallArgs.push_back(BuildTuple(pos, { positionalArgsNode, namedArgs })); + } else { + TNodePtr positionalArgsNode = new TCallNodeImpl(pos, "PositionalArgs", args); + sqlCallArgs.push_back(BuildTuple(pos, { positionalArgsNode })); + } + + // optional arguments + if (customUserType) { + sqlCallArgs.push_back(customUserType); + } else if (typeConfig) { + sqlCallArgs.push_back(new TCallNodeImpl(pos, "TupleType", {})); + } + + if (typeConfig) { + sqlCallArgs.push_back(BuildQuotedAtom(pos, *typeConfig)); + } + + return new TCallNodeImpl(pos, "SqlCall", sqlCallArgs); +} + class TCallableNode final: public INode { public: - TCallableNode(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce) + TCallableNode(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce) : INode(pos) , Module(module) , Name(name) , Args(args) - , ForReduce(forReduce) + , ForReduce(forReduce) {} bool DoInit(TContext& ctx, ISource* src) override { if (Module == "yql") { Node = new TFuncNodeImpl(Pos, Name); } else if (Module == "@yql") { - auto parsedName = StringContent(ctx, Pos, Name); - if (!parsedName) { - return false; - } - - const TString yql("(" + parsedName->Content + ")"); + auto parsedName = StringContent(ctx, Pos, Name); + if (!parsedName) { + return false; + } + + const TString yql("(" + parsedName->Content + ")"); TAstParseResult ast = ParseAst(yql, ctx.Pool.get()); /// TODO: do not drop warnings if (ast.IsOk()) { - const auto rootCount = ast.Root->GetChildrenCount(); - if (rootCount != 1) { - ctx.Error(Pos) << "Failed to parse YQL: expecting AST root node with single child, but got " << rootCount; - return false; - } + const auto rootCount = ast.Root->GetChildrenCount(); + if (rootCount != 1) { + ctx.Error(Pos) << "Failed to parse YQL: expecting AST root node with single child, but got " << rootCount; + return false; + } Node = AstNode(ast.Root->GetChild(0)); } else { ctx.Error(Pos) << "Failed to parse YQL: " << ast.Issues.ToString(); @@ -2043,18 +2043,18 @@ public: Module.append('2'); TMaybe<TString> typeConfig = MakeTypeConfig(to_lower(Module), Args); - if (ForReduce) { - TVector<TNodePtr> udfArgs; - udfArgs.push_back(BuildQuotedAtom(Pos, TString(Module) + "." + Name)); - udfArgs.push_back(customUserType ? customUserType : new TCallNodeImpl(Pos, "TupleType", {})); - if (typeConfig) { - udfArgs.push_back(BuildQuotedAtom(Pos, *typeConfig)); - } - Node = new TCallNodeImpl(Pos, "SqlReduceUdf", udfArgs); - } else { - auto udfArgs = BuildUdfArgs(ctx, Pos, Args, nullptr, nullptr, customUserType, typeConfig); - Node = BuildUdf(ctx, Pos, Module, Name, udfArgs); - } + if (ForReduce) { + TVector<TNodePtr> udfArgs; + udfArgs.push_back(BuildQuotedAtom(Pos, TString(Module) + "." + Name)); + udfArgs.push_back(customUserType ? customUserType : new TCallNodeImpl(Pos, "TupleType", {})); + if (typeConfig) { + udfArgs.push_back(BuildQuotedAtom(Pos, *typeConfig)); + } + Node = new TCallNodeImpl(Pos, "SqlReduceUdf", udfArgs); + } else { + auto udfArgs = BuildUdfArgs(ctx, Pos, Args, nullptr, nullptr, customUserType, typeConfig); + Node = BuildUdf(ctx, Pos, Module, Name, udfArgs); + } } return Node->Init(ctx, src); } @@ -2078,23 +2078,23 @@ public: } TNodePtr DoClone() const override { - return new TCallableNode(Pos, Module, Name, Args, ForReduce); + return new TCallableNode(Pos, Module, Name, Args, ForReduce); } - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(Node); - Node->VisitTree(func, visited); - } + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(Node); + Node->VisitTree(func, visited); + } private: TCiString Module; TString Name; TVector<TNodePtr> Args; TNodePtr Node; - const bool ForReduce; + const bool ForReduce; }; -TNodePtr BuildCallable(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce) { - return new TCallableNode(pos, module, name, args, forReduce); +TNodePtr BuildCallable(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce) { + return new TCallableNode(pos, module, name, args, forReduce); } TNodePtr BuildUdf(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args) { @@ -2179,11 +2179,11 @@ public: TNodePtr DoClone() const final { return new TScriptUdf(GetPos(), ModuleName, FuncName, CloneContainer(Args)); } - - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(Node); - Node->VisitTree(func, visited); - } + + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(Node); + Node->VisitTree(func, visited); + } private: TString ModuleName; TString FuncName; @@ -2323,14 +2323,14 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( return [functionName, realFunctionName, factoryName, type, validModes] (TPosition pos, const TVector<TNodePtr>& args, EAggregateMode aggMode, bool isFactory) -> INode::TPtr { if (!validModes.empty()) { if (!IsIn(validModes, aggMode)) { - TString errorText; - if (TVector{EAggregateMode::OverWindow} == validModes) { - errorText = TStringBuilder() - << "Can't use window function " << functionName << " without window specification (OVER keyword is missing)"; - } else { - errorText = TStringBuilder() - << "Can't use " << functionName << " in " << ToString(aggMode) << " aggregation mode"; - } + TString errorText; + if (TVector{EAggregateMode::OverWindow} == validModes) { + errorText = TStringBuilder() + << "Can't use window function " << functionName << " without window specification (OVER keyword is missing)"; + } else { + errorText = TStringBuilder() + << "Can't use " << functionName << " in " << ToString(aggMode) << " aggregation mode"; + } return INode::TPtr(new TInvalidBuiltin(pos, errorText)); } } @@ -2467,7 +2467,7 @@ struct TBuiltinFuncData { TBuiltinFuncData(): BuiltinFuncs(MakeBuiltinFuncs()), AggrFuncs(MakeAggrFuncs()), - CoreFuncs(MakeCoreFuncs()) + CoreFuncs(MakeCoreFuncs()) { } @@ -2507,7 +2507,7 @@ struct TBuiltinFuncData { {"listlength", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Length", 1, 1)}, {"listhasitems", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("HasItems", 1, 1)}, {"listextend", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListExtend", 0, -1)}, - {"listextendstrict", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListExtendStrict", 0, -1)}, + {"listextendstrict", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListExtendStrict", 0, -1)}, {"listunionall", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListUnionAll", 0, -1) }, {"listzip", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListZip", -1, -1)}, {"listzipall", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ListZipAll", -1, -1)}, @@ -2689,7 +2689,7 @@ struct TBuiltinFuncData { {"combinemembers", BuildNamedBuiltinFactoryCallback<TCombineMembers>("FlattenMembers")}, {"flattenmembers", BuildNamedBuiltinFactoryCallback<TFlattenMembers>("FlattenMembers")}, {"staticmap", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticMap", 2, 2) }, - {"staticzip", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticZip", 1, -1) }, + {"staticzip", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticZip", 1, -1) }, // File builtins {"filepath", BuildNamedBuiltinFactoryCallback<TYqlAtom>("FilePath")}, @@ -2718,16 +2718,16 @@ struct TBuiltinFuncData { // Window funcitons {"rownumber", BuildNamedArgcBuiltinFactoryCallback<TWinRowNumber>("RowNumber", 0, 0)}, - {"rank", BuildNamedArgcBuiltinFactoryCallback<TWinRank>("Rank", 0, 1)}, - {"denserank", BuildNamedArgcBuiltinFactoryCallback<TWinRank>("DenseRank", 0, 1)}, + {"rank", BuildNamedArgcBuiltinFactoryCallback<TWinRank>("Rank", 0, 1)}, + {"denserank", BuildNamedArgcBuiltinFactoryCallback<TWinRank>("DenseRank", 0, 1)}, {"lead", BuildNamedArgcBuiltinFactoryCallback<TWinLeadLag>("Lead", 1, 2)}, {"lag", BuildNamedArgcBuiltinFactoryCallback<TWinLeadLag>("Lag", 1, 2)}, - // Session window - {"sessionwindow", BuildSimpleBuiltinFactoryCallback<TSessionWindow>()}, - {"sessionstart", BuildSimpleBuiltinFactoryCallback<TSessionStart<true>>()}, - {"sessionstate", BuildSimpleBuiltinFactoryCallback<TSessionStart<false>>()}, - + // Session window + {"sessionwindow", BuildSimpleBuiltinFactoryCallback<TSessionWindow>()}, + {"sessionstart", BuildSimpleBuiltinFactoryCallback<TSessionStart<true>>()}, + {"sessionstate", BuildSimpleBuiltinFactoryCallback<TSessionStart<false>>()}, + // Hopping intervals time functions {"hopstart", BuildSimpleBuiltinFactoryCallback<THoppingTime<true>>()}, {"hopend", BuildSimpleBuiltinFactoryCallback<THoppingTime<false>>()}, @@ -2871,7 +2871,7 @@ struct TBuiltinFuncData { }; TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVector<TNodePtr>& args, - const TString& originalNameSpace, EAggregateMode aggMode, bool* mustUseNamed, bool warnOnYqlNameSpace) { + const TString& originalNameSpace, EAggregateMode aggMode, bool* mustUseNamed, bool warnOnYqlNameSpace) { const TBuiltinFuncData* funcData = Singleton<TBuiltinFuncData>(); const TBuiltinFactoryCallbackMap& builtinFuncs = funcData->BuiltinFuncs; @@ -2945,7 +2945,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec } if (ns == "yql" || ns == "@yql") { - if (warnOnYqlNameSpace && GetEnv("YQL_DETERMINISTIC_MODE").empty()) { + if (warnOnYqlNameSpace && GetEnv("YQL_DETERMINISTIC_MODE").empty()) { ctx.Warning(pos, TIssuesIds::YQL_S_EXPRESSIONS_CALL) << "It is not recommended to directly access s-expressions functions via YQL::" << Endl << "This mechanism is mostly intended for temporary workarounds or internal testing purposes"; @@ -3074,35 +3074,35 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType); return new TScriptUdf(pos, TString(scriptName), name, args); } else if (ns.empty()) { - if (auto simpleType = LookupSimpleTypeBySqlAlias(normalizedName, ctx.FlexibleTypes)) { - const auto type = ToString(*simpleType); - if (NUdf::FindDataSlot(type)) { - YQL_ENSURE(type != "Decimal"); - return new TYqlData(pos, type, args); - } - - if (type == "Void" || type == "EmptyList" || type == "EmptyDict") { - return new TCallNodeImpl(pos, type, 0, 0, args); - } else { - return new TInvalidBuiltin(pos, TStringBuilder() << "Can not create objects of type " << type); - } - } - - if (normalizedName == "decimal") { - if (args.size() == 2) { - TVector<TNodePtr> dataTypeArgs = { BuildQuotedAtom(pos, "Decimal", TNodeFlags::Default) }; - for (auto& arg : args) { - if (auto literal = arg->GetLiteral("Int32")) { - dataTypeArgs.push_back(BuildQuotedAtom(pos, *literal, TNodeFlags::Default)); - } else { - dataTypeArgs.push_back(MakeAtomFromExpression(ctx, arg).Build()); - } - } - return new TCallNodeImpl(pos, "DataType", dataTypeArgs); - } - return new TYqlData(pos, "Decimal", args); - } - + if (auto simpleType = LookupSimpleTypeBySqlAlias(normalizedName, ctx.FlexibleTypes)) { + const auto type = ToString(*simpleType); + if (NUdf::FindDataSlot(type)) { + YQL_ENSURE(type != "Decimal"); + return new TYqlData(pos, type, args); + } + + if (type == "Void" || type == "EmptyList" || type == "EmptyDict") { + return new TCallNodeImpl(pos, type, 0, 0, args); + } else { + return new TInvalidBuiltin(pos, TStringBuilder() << "Can not create objects of type " << type); + } + } + + if (normalizedName == "decimal") { + if (args.size() == 2) { + TVector<TNodePtr> dataTypeArgs = { BuildQuotedAtom(pos, "Decimal", TNodeFlags::Default) }; + for (auto& arg : args) { + if (auto literal = arg->GetLiteral("Int32")) { + dataTypeArgs.push_back(BuildQuotedAtom(pos, *literal, TNodeFlags::Default)); + } else { + dataTypeArgs.push_back(MakeAtomFromExpression(ctx, arg).Build()); + } + } + return new TCallNodeImpl(pos, "DataType", dataTypeArgs); + } + return new TYqlData(pos, "Decimal", args); + } + if (normalizedName == "tablename") { return new TTableName(pos, args, ctx.Scoped->CurrService); } @@ -3132,7 +3132,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec } if (aggMode == EAggregateMode::Distinct) { - return new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"); + return new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"); } return (*aggrCallback).second(pos, args, aggMode, true).Release(); @@ -3154,7 +3154,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec return (*aggrCallback).second(pos, args, aggMode, false).Release(); } if (aggMode == EAggregateMode::Distinct) { - return new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"); + return new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"); } auto builtinCallback = builtinFuncs.find(normalizedName); @@ -3184,11 +3184,11 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec (normalizedName == "asstruct" ? "AsStruct" : "StructType") << " requires all argument to be named"); } else if (normalizedName == "expandstruct") { - if (mustUseNamed) { - if (!*mustUseNamed) { - return new TInvalidBuiltin(pos, TStringBuilder() << "ExpandStruct requires at least one named argument"); - } - *mustUseNamed = false; + if (mustUseNamed) { + if (!*mustUseNamed) { + return new TInvalidBuiltin(pos, TStringBuilder() << "ExpandStruct requires at least one named argument"); + } + *mustUseNamed = false; } YQL_ENSURE(args.size() == 2); auto posArgs = static_cast<TTupleNode*>(args[0].Get()); @@ -3219,7 +3219,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec *mustUseNamed = false; } - TVector<TNodePtr> usedArgs = args; + TVector<TNodePtr> usedArgs = args; TNodePtr customUserType = nullptr; if (ns == "json") { @@ -3256,14 +3256,14 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec usedArgs.emplace_back(BuildYsonOptionsNode(pos, ctx.PragmaYsonAutoConvert, ctx.PragmaYsonStrict, ctx.PragmaYsonFast)); } positionalArgs = BuildTuple(pos, usedArgs); - auto encodeUtf8 = BuildLiteralBool(pos, true); + auto encodeUtf8 = BuildLiteralBool(pos, true); encodeUtf8->SetLabel("EncodeUtf8"); namedArgs = BuildStructure(pos, {encodeUtf8}); usedArgs = {positionalArgs, namedArgs}; } else if (name.StartsWith("From")) { - if (usedArgs) { - usedArgs.resize(1U); - } + if (usedArgs) { + usedArgs.resize(1U); + } name = "From"; } else if (name == "GetLength" || name.StartsWith("ConvertTo") || name.StartsWith("Parse") || name.StartsWith("SerializeJson")) { if (usedArgs.size() < 2U) { @@ -3295,7 +3295,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec } TMaybe<TString> typeConfig = MakeTypeConfig(ns, usedArgs); - return BuildSqlCall(ctx, pos, nameSpace, name, usedArgs, positionalArgs, namedArgs, customUserType, typeConfig); + return BuildSqlCall(ctx, pos, nameSpace, name, usedArgs, positionalArgs, namedArgs, customUserType, typeConfig); } } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp index 7ee43227af..c08fe96cae 100644 --- a/ydb/library/yql/sql/v1/context.cpp +++ b/ydb/library/yql/sql/v1/context.cpp @@ -24,45 +24,45 @@ TNodePtr AddTablePathPrefix(TContext& ctx, TStringBuf prefixPath, const TDeferre return path.Build(); } - if (path.GetLiteral()) { - return BuildQuotedAtom(path.Build()->GetPos(), BuildTablePath(prefixPath, *path.GetLiteral())); - } + if (path.GetLiteral()) { + return BuildQuotedAtom(path.Build()->GetPos(), BuildTablePath(prefixPath, *path.GetLiteral())); + } - auto pathNode = path.Build(); - pathNode = new TCallNodeImpl(pathNode->GetPos(), "String", { pathNode }); - auto prefixNode = BuildLiteralRawString(pathNode->GetPos(), TString(prefixPath)); + auto pathNode = path.Build(); + pathNode = new TCallNodeImpl(pathNode->GetPos(), "String", { pathNode }); + auto prefixNode = BuildLiteralRawString(pathNode->GetPos(), TString(prefixPath)); - TNodePtr buildPathNode = new TCallNodeImpl(pathNode->GetPos(), "BuildTablePath", { prefixNode, pathNode }); + TNodePtr buildPathNode = new TCallNodeImpl(pathNode->GetPos(), "BuildTablePath", { prefixNode, pathNode }); - TDeferredAtom result; - MakeTableFromExpression(ctx, buildPathNode, result); - return result.Build(); + TDeferredAtom result; + MakeTableFromExpression(ctx, buildPathNode, result); + return result.Build(); } typedef bool TContext::*TPragmaField; THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { - {"AnsiOptionalAs", &TContext::AnsiOptionalAs}, - {"WarnOnAnsiAliasShadowing", &TContext::WarnOnAnsiAliasShadowing}, - {"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin}, + {"AnsiOptionalAs", &TContext::AnsiOptionalAs}, + {"WarnOnAnsiAliasShadowing", &TContext::WarnOnAnsiAliasShadowing}, + {"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin}, {"DqEngineEnable", &TContext::DqEngineEnable}, {"DqEngineForce", &TContext::DqEngineForce}, - {"RegexUseRe2", &TContext::PragmaRegexUseRe2}, - {"OrderedColumns", &TContext::OrderedColumns}, - {"BogousStarInGroupByOverJoin", &TContext::BogousStarInGroupByOverJoin}, - {"CoalesceJoinKeysOnQualifiedAll", &TContext::CoalesceJoinKeysOnQualifiedAll}, - {"UnorderedSubqueries", &TContext::UnorderedSubqueries}, - {"FlexibleTypes", &TContext::FlexibleTypes}, + {"RegexUseRe2", &TContext::PragmaRegexUseRe2}, + {"OrderedColumns", &TContext::OrderedColumns}, + {"BogousStarInGroupByOverJoin", &TContext::BogousStarInGroupByOverJoin}, + {"CoalesceJoinKeysOnQualifiedAll", &TContext::CoalesceJoinKeysOnQualifiedAll}, + {"UnorderedSubqueries", &TContext::UnorderedSubqueries}, + {"FlexibleTypes", &TContext::FlexibleTypes}, +}; + +typedef TMaybe<bool> TContext::*TPragmaMaybeField; + +THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = { + {"AnsiRankForNullableKeys", &TContext::AnsiRankForNullableKeys}, + {"AnsiOrderByLimitInUnionAll", &TContext::AnsiOrderByLimitInUnionAll}, + {"AnsiInForEmptyOrNullableItemsCollections", &TContext::AnsiInForEmptyOrNullableItemsCollections}, }; -typedef TMaybe<bool> TContext::*TPragmaMaybeField; - -THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = { - {"AnsiRankForNullableKeys", &TContext::AnsiRankForNullableKeys}, - {"AnsiOrderByLimitInUnionAll", &TContext::AnsiOrderByLimitInUnionAll}, - {"AnsiInForEmptyOrNullableItemsCollections", &TContext::AnsiInForEmptyOrNullableItemsCollections}, -}; - } // namespace TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, @@ -76,7 +76,7 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, , IncrementMonCounterFunction(settings.IncrementCounter) , HasPendingErrors(false) , DqEngineEnable(Settings.DqDefaultAuto->Allow()) - , AnsiQuotedIdentifiers(settings.AnsiLexer) + , AnsiQuotedIdentifiers(settings.AnsiLexer) { for (auto lib : settings.Libraries) { Libraries[lib] = Nothing(); @@ -97,17 +97,17 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, bool value = true; TStringBuf key = flag; auto ptr = CTX_PRAGMA_FIELDS.FindPtr(key); - auto ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key); - if (!ptr && !ptrMaybe && key.SkipPrefix("Disable")) { + auto ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key); + if (!ptr && !ptrMaybe && key.SkipPrefix("Disable")) { value = false; ptr = CTX_PRAGMA_FIELDS.FindPtr(key); - ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key); + ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key); + } + if (ptr) { + this->*(*ptr) = value; + } else if (ptrMaybe) { + this->*(*ptrMaybe) = value; } - if (ptr) { - this->*(*ptr) = value; - } else if (ptrMaybe) { - this->*(*ptrMaybe) = value; - } } DiscoveryMode = (NSQLTranslation::ESqlMode::DISCOVERY == Settings.Mode); } @@ -151,25 +151,25 @@ IOutputStream& TContext::Info(NYql::TPosition pos) { return MakeIssue(TSeverityIds::S_INFO, TIssuesIds::INFO, pos); } -void TContext::SetWarningPolicyFor(NYql::TIssueCode code, NYql::EWarningAction action) { - TString codePattern = ToString(code); - TString actionString = ToString(action); - - TWarningRule rule; - TString parseError; - auto parseResult = TWarningRule::ParseFrom(codePattern, actionString, rule, parseError); - YQL_ENSURE(parseResult == TWarningRule::EParseResult::PARSE_OK); - WarningPolicy.AddRule(rule); -} - +void TContext::SetWarningPolicyFor(NYql::TIssueCode code, NYql::EWarningAction action) { + TString codePattern = ToString(code); + TString actionString = ToString(action); + + TWarningRule rule; + TString parseError; + auto parseResult = TWarningRule::ParseFrom(codePattern, actionString, rule, parseError); + YQL_ENSURE(parseResult == TWarningRule::EParseResult::PARSE_OK); + WarningPolicy.AddRule(rule); +} + IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TPosition pos) { if (severity == TSeverityIds::S_WARNING) { - auto action = WarningPolicy.GetAction(code); - if (action == EWarningAction::ERROR) { - severity = TSeverityIds::S_ERROR; - HasPendingErrors = true; - } else if (action == EWarningAction::DISABLE) { - return Cnull; + auto action = WarningPolicy.GetAction(code); + if (action == EWarningAction::ERROR) { + severity = TSeverityIds::S_ERROR; + HasPendingErrors = true; + } else if (action == EWarningAction::DISABLE) { + return Cnull; } } @@ -246,24 +246,24 @@ TNodePtr TContext::UniversalAlias(const TString& baseName, TNodePtr&& node) { return BuildAtom(node->GetPos(), alias, TNodeFlags::Default); } -void TContext::DeclareVariable(const TString& varName, const TNodePtr& typeNode) { +void TContext::DeclareVariable(const TString& varName, const TNodePtr& typeNode) { Variables.emplace(varName, typeNode); } -bool TContext::AddExport(TPosition pos, const TString& name) { - if (IsAnonymousName(name)) { - Error(pos) << "Can not export anonymous name " << name; - return false; - } - if (Exports.contains(name)) { - Error(pos) << "Duplicate export symbol: " << name; - return false; - } - if (!Scoped->LookupNode(name)) { - Error(pos) << "Unable to export unknown symbol: " << name; - return false; - } - Exports.emplace(name); +bool TContext::AddExport(TPosition pos, const TString& name) { + if (IsAnonymousName(name)) { + Error(pos) << "Can not export anonymous name " << name; + return false; + } + if (Exports.contains(name)) { + Error(pos) << "Duplicate export symbol: " << name; + return false; + } + if (!Scoped->LookupNode(name)) { + Error(pos) << "Unable to export unknown symbol: " << name; + return false; + } + Exports.emplace(name); return true; } @@ -336,16 +336,16 @@ void TScopedState::Clear() { *this = TScopedState(); } -TNodePtr TScopedState::LookupNode(const TString& name) { - auto mapIt = NamedNodes.find(name); - if (mapIt == NamedNodes.end()) { - return nullptr; - } - Y_VERIFY_DEBUG(!mapIt->second.empty()); - mapIt->second.front()->IsUsed = true; - return mapIt->second.front()->Node->Clone(); -} - +TNodePtr TScopedState::LookupNode(const TString& name) { + auto mapIt = NamedNodes.find(name); + if (mapIt == NamedNodes.end()) { + return nullptr; + } + Y_VERIFY_DEBUG(!mapIt->second.empty()); + mapIt->second.front()->IsUsed = true; + return mapIt->second.front()->Node->Clone(); +} + bool TContext::HasNonYtProvider(const ISource& source) const { TTableList tableList; source.GetInputTables(tableList); @@ -374,89 +374,89 @@ bool TContext::UseUnordered(const TTableRef& table) const { return YtProviderName == table.Service; } - -TMaybe<EColumnRefState> GetFunctionArgColumnStatus(TContext& ctx, const TString& module, const TString& func, size_t argIndex) { - static const TSet<TStringBuf> denyForAllArgs = { - "datatype", - "optionaltype", - "listtype", - "streamtype", - "dicttype", - "tupletype", - "resourcetype", - "taggedtype", - "varianttype", - "callabletype", - "optionalitemtype", - "listitemtype", - "streamitemtype", - "dictkeytype", - "dictpayloadtype", - "tupleelementtype", - "structmembertype", - "callableresulttype", - "callableargumenttype", - "variantunderlyingtype", - }; - static const TMap<std::pair<TStringBuf, size_t>, EColumnRefState> positionalArgsCustomStatus = { - { {"frombytes", 1}, EColumnRefState::Deny }, - { {"enum", 0}, EColumnRefState::Deny }, - { {"asenum", 0}, EColumnRefState::Deny }, - { {"variant", 1}, EColumnRefState::Deny }, - { {"variant", 2}, EColumnRefState::Deny }, - { {"asvariant", 1}, EColumnRefState::Deny }, - { {"astagged", 1}, EColumnRefState::Deny }, - { {"ensuretype", 1}, EColumnRefState::Deny }, - { {"ensuretype", 2}, EColumnRefState::Deny }, - { {"ensureconvertibleto", 1}, EColumnRefState::Deny }, - { {"ensureconvertibleto", 2}, EColumnRefState::Deny }, - - // TODO: switch to Deny here - { {"evaluateexpr", 0}, EColumnRefState::AsStringLiteral }, - { {"evaluateatom", 0}, EColumnRefState::AsStringLiteral }, - { {"evaluatetype", 0}, EColumnRefState::AsStringLiteral }, - - { {"nothing", 0}, EColumnRefState::Deny }, - { {"formattype", 0}, EColumnRefState::Deny }, - { {"instanceof", 0}, EColumnRefState::Deny }, - - { {"unpickle", 0}, EColumnRefState::Deny }, - { {"typehandle", 0}, EColumnRefState::Deny }, - - { {"listcreate", 0}, EColumnRefState::Deny }, - { {"setcreate", 0}, EColumnRefState::Deny }, - { {"dictcreate", 0}, EColumnRefState::Deny }, - { {"dictcreate", 1}, EColumnRefState::Deny }, - { {"weakfield", 1}, EColumnRefState::Deny }, - - { {"Yson::ConvertTo", 1}, EColumnRefState::Deny }, - }; - - TString normalized; - if (module.empty()) { - normalized = to_lower(func); - } else if (to_upper(module) == "YQL") { - normalized = "YQL::" + func; - } else { - normalized = module + "::" + func; - } - - if (normalized == "typeof" && argIndex == 0) { - // TODO: more such cases? - return ctx.GetTopLevelColumnReferenceState(); - } - - if (denyForAllArgs.contains(normalized)) { - return EColumnRefState::Deny; - } - - auto it = positionalArgsCustomStatus.find(std::make_pair(normalized, argIndex)); - if (it != positionalArgsCustomStatus.end()) { - return it->second; - } - return {}; -} - + +TMaybe<EColumnRefState> GetFunctionArgColumnStatus(TContext& ctx, const TString& module, const TString& func, size_t argIndex) { + static const TSet<TStringBuf> denyForAllArgs = { + "datatype", + "optionaltype", + "listtype", + "streamtype", + "dicttype", + "tupletype", + "resourcetype", + "taggedtype", + "varianttype", + "callabletype", + "optionalitemtype", + "listitemtype", + "streamitemtype", + "dictkeytype", + "dictpayloadtype", + "tupleelementtype", + "structmembertype", + "callableresulttype", + "callableargumenttype", + "variantunderlyingtype", + }; + static const TMap<std::pair<TStringBuf, size_t>, EColumnRefState> positionalArgsCustomStatus = { + { {"frombytes", 1}, EColumnRefState::Deny }, + { {"enum", 0}, EColumnRefState::Deny }, + { {"asenum", 0}, EColumnRefState::Deny }, + { {"variant", 1}, EColumnRefState::Deny }, + { {"variant", 2}, EColumnRefState::Deny }, + { {"asvariant", 1}, EColumnRefState::Deny }, + { {"astagged", 1}, EColumnRefState::Deny }, + { {"ensuretype", 1}, EColumnRefState::Deny }, + { {"ensuretype", 2}, EColumnRefState::Deny }, + { {"ensureconvertibleto", 1}, EColumnRefState::Deny }, + { {"ensureconvertibleto", 2}, EColumnRefState::Deny }, + + // TODO: switch to Deny here + { {"evaluateexpr", 0}, EColumnRefState::AsStringLiteral }, + { {"evaluateatom", 0}, EColumnRefState::AsStringLiteral }, + { {"evaluatetype", 0}, EColumnRefState::AsStringLiteral }, + + { {"nothing", 0}, EColumnRefState::Deny }, + { {"formattype", 0}, EColumnRefState::Deny }, + { {"instanceof", 0}, EColumnRefState::Deny }, + + { {"unpickle", 0}, EColumnRefState::Deny }, + { {"typehandle", 0}, EColumnRefState::Deny }, + + { {"listcreate", 0}, EColumnRefState::Deny }, + { {"setcreate", 0}, EColumnRefState::Deny }, + { {"dictcreate", 0}, EColumnRefState::Deny }, + { {"dictcreate", 1}, EColumnRefState::Deny }, + { {"weakfield", 1}, EColumnRefState::Deny }, + + { {"Yson::ConvertTo", 1}, EColumnRefState::Deny }, + }; + + TString normalized; + if (module.empty()) { + normalized = to_lower(func); + } else if (to_upper(module) == "YQL") { + normalized = "YQL::" + func; + } else { + normalized = module + "::" + func; + } + + if (normalized == "typeof" && argIndex == 0) { + // TODO: more such cases? + return ctx.GetTopLevelColumnReferenceState(); + } + + if (denyForAllArgs.contains(normalized)) { + return EColumnRefState::Deny; + } + + auto it = positionalArgsCustomStatus.find(std::make_pair(normalized, argIndex)); + if (it != positionalArgsCustomStatus.end()) { + return it->second; + } + return {}; +} + TTranslation::TTranslation(TContext& ctx) : Ctx(ctx) { @@ -471,78 +471,78 @@ IOutputStream& TTranslation::Error() { } TNodePtr TTranslation::GetNamedNode(const TString& name) { - if (name == "$_") { - Ctx.Error() << "Unable to reference anonymous name " << name; - return nullptr; - } - auto res = Ctx.Scoped->LookupNode(name); - if (!res) { + if (name == "$_") { + Ctx.Error() << "Unable to reference anonymous name " << name; + return nullptr; + } + auto res = Ctx.Scoped->LookupNode(name); + if (!res) { Ctx.Error() << "Unknown name: " << name; } - return res; + return res; } -TString TTranslation::PushNamedNode(TPosition namePos, const TString& name, const TNodeBuilderByName& builder) { - TString resultName = name; - if (IsAnonymousName(name)) { - resultName = "$_yql_anonymous_name_" + ToString(Ctx.AnonymousNameIndex++); - YQL_ENSURE(Ctx.Scoped->NamedNodes.find(resultName) == Ctx.Scoped->NamedNodes.end()); - } - auto node = builder(resultName); +TString TTranslation::PushNamedNode(TPosition namePos, const TString& name, const TNodeBuilderByName& builder) { + TString resultName = name; + if (IsAnonymousName(name)) { + resultName = "$_yql_anonymous_name_" + ToString(Ctx.AnonymousNameIndex++); + YQL_ENSURE(Ctx.Scoped->NamedNodes.find(resultName) == Ctx.Scoped->NamedNodes.end()); + } + auto node = builder(resultName); Y_VERIFY_DEBUG(node); - auto mapIt = Ctx.Scoped->NamedNodes.find(resultName); + auto mapIt = Ctx.Scoped->NamedNodes.find(resultName); if (mapIt == Ctx.Scoped->NamedNodes.end()) { - auto result = Ctx.Scoped->NamedNodes.insert(std::make_pair(resultName, TDeque<TNodeWithUsageInfoPtr>())); + auto result = Ctx.Scoped->NamedNodes.insert(std::make_pair(resultName, TDeque<TNodeWithUsageInfoPtr>())); Y_VERIFY_DEBUG(result.second); mapIt = result.first; } - mapIt->second.push_front(MakeIntrusive<TNodeWithUsageInfo>(node, namePos, Ctx.ScopeLevel)); - return resultName; + mapIt->second.push_front(MakeIntrusive<TNodeWithUsageInfo>(node, namePos, Ctx.ScopeLevel)); + return resultName; +} + +TString TTranslation::PushNamedNode(NYql::TPosition namePos, const TString &name, NSQLTranslationV1::TNodePtr node) { + return PushNamedNode(namePos, name, [node](const TString&) { return node; }); +} + +TString TTranslation::PushNamedAtom(TPosition namePos, const TString& name) { + auto buildAtom = [namePos](const TString& resultName) { + return BuildAtom(namePos, resultName); + }; + return PushNamedNode(namePos, name, buildAtom); } -TString TTranslation::PushNamedNode(NYql::TPosition namePos, const TString &name, NSQLTranslationV1::TNodePtr node) { - return PushNamedNode(namePos, name, [node](const TString&) { return node; }); -} - -TString TTranslation::PushNamedAtom(TPosition namePos, const TString& name) { - auto buildAtom = [namePos](const TString& resultName) { - return BuildAtom(namePos, resultName); - }; - return PushNamedNode(namePos, name, buildAtom); -} - void TTranslation::PopNamedNode(const TString& name) { auto mapIt = Ctx.Scoped->NamedNodes.find(name); Y_VERIFY_DEBUG(mapIt != Ctx.Scoped->NamedNodes.end()); Y_VERIFY_DEBUG(mapIt->second.size() > 0); - auto& top = mapIt->second.front(); - if (!top->IsUsed && !Ctx.HasPendingErrors && !name.StartsWith("$_")) { - Ctx.Warning(top->NamePos, TIssuesIds::YQL_UNUSED_SYMBOL) << "Symbol " << name << " is not used"; - } - mapIt->second.pop_front(); + auto& top = mapIt->second.front(); + if (!top->IsUsed && !Ctx.HasPendingErrors && !name.StartsWith("$_")) { + Ctx.Warning(top->NamePos, TIssuesIds::YQL_UNUSED_SYMBOL) << "Symbol " << name << " is not used"; + } + mapIt->second.pop_front(); if (mapIt->second.empty()) { Ctx.Scoped->NamedNodes.erase(mapIt); } } -void TTranslation::WarnUnusedNodes() const { - if (Ctx.HasPendingErrors) { - // result is not reliable in this case - return; - } - for (const auto& [name, items]: Ctx.Scoped->NamedNodes) { - if (name.StartsWith("$_")) { - continue; - } - for (const auto& item : items) { - if (!item->IsUsed && item->Level == Ctx.ScopeLevel) { - Ctx.Warning(item->NamePos, TIssuesIds::YQL_UNUSED_SYMBOL) << "Symbol " << name << " is not used"; - } - } - } -} - +void TTranslation::WarnUnusedNodes() const { + if (Ctx.HasPendingErrors) { + // result is not reliable in this case + return; + } + for (const auto& [name, items]: Ctx.Scoped->NamedNodes) { + if (name.StartsWith("$_")) { + continue; + } + for (const auto& item : items) { + if (!item->IsUsed && item->Level == Ctx.ScopeLevel) { + Ctx.Warning(item->NamePos, TIssuesIds::YQL_UNUSED_SYMBOL) << "Symbol " << name << " is not used"; + } + } + } +} + TString GetDescription(const google::protobuf::Message& node, const google::protobuf::FieldDescriptor* d) { const auto& field = node.GetReflection()->GetMessage(node, d); return field.GetReflection()->GetString(field, d->message_type()->FindFieldByName("Descr")); diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index 55b937d1fa..8951757486 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -13,40 +13,40 @@ #include <util/generic/map.h> #include <util/generic/maybe.h> #include <util/generic/set.h> -#include <util/generic/deque.h> +#include <util/generic/deque.h> #include <util/generic/vector.h> namespace NSQLTranslationV1 { - inline bool IsAnonymousName(const TString& name) { - return name == "$_"; - } - + inline bool IsAnonymousName(const TString& name) { + return name == "$_"; + } + inline bool IsStreamingService(const TString& service) { return service == NYql::RtmrProviderName || service == NYql::PqProviderName; } - struct TNodeWithUsageInfo : public TThrRefBase { - explicit TNodeWithUsageInfo(const TNodePtr& node, TPosition namePos, int level) - : Node(node) - , NamePos(namePos) - , Level(level) - {} - - TNodePtr Node; - TPosition NamePos; - int Level = 0; - bool IsUsed = false; - }; - - using TNodeWithUsageInfoPtr = TIntrusivePtr<TNodeWithUsageInfo>; - using TNamedNodesMap = THashMap<TString, TDeque<TNodeWithUsageInfoPtr>>; + struct TNodeWithUsageInfo : public TThrRefBase { + explicit TNodeWithUsageInfo(const TNodePtr& node, TPosition namePos, int level) + : Node(node) + , NamePos(namePos) + , Level(level) + {} + + TNodePtr Node; + TPosition NamePos; + int Level = 0; + bool IsUsed = false; + }; + + using TNodeWithUsageInfoPtr = TIntrusivePtr<TNodeWithUsageInfo>; + using TNamedNodesMap = THashMap<TString, TDeque<TNodeWithUsageInfoPtr>>; using TBlocks = TVector<TNodePtr>; struct TScopedState : public TThrRefBase { TString CurrService; TDeferredAtom CurrCluster; bool PragmaClassicDivision = true; - bool StrictJoinKeyTypes = false; + bool StrictJoinKeyTypes = false; TNamedNodesMap NamedNodes; struct TLocal { @@ -64,18 +64,18 @@ namespace NSQLTranslationV1 { TNodePtr WrapCluster(const TDeferredAtom& cluster, TContext& ctx); void AddExprCluster(TNodePtr expr, TContext& ctx); void Clear(); - TNodePtr LookupNode(const TString& name); + TNodePtr LookupNode(const TString& name); }; using TScopedStatePtr = TIntrusivePtr<TScopedState>; - class TColumnRefScope; - enum class EColumnRefState { - Deny, - Allow, - AsStringLiteral, - }; - + class TColumnRefScope; + enum class EColumnRefState { + Deny, + Allow, + AsStringLiteral, + }; + class TContext { public: TContext(const NSQLTranslation::TTranslationSettings& settings, @@ -93,8 +93,8 @@ namespace NSQLTranslationV1 { IOutputStream& Warning(NYql::TPosition pos, NYql::TIssueCode code); IOutputStream& Info(NYql::TPosition pos); - void SetWarningPolicyFor(NYql::TIssueCode code, NYql::EWarningAction action); - + void SetWarningPolicyFor(NYql::TIssueCode code, NYql::EWarningAction action); + template <typename TToken> const TString& Token(const TToken& token) { Position.Row = token.GetLine(); @@ -128,10 +128,10 @@ namespace NSQLTranslationV1 { TMaybe<TString> GetClusterProvider(const TString& cluster, TString& normalizedClusterName) const { auto provider = ClusterMapping.GetClusterProvider(cluster, normalizedClusterName); if (!provider) { - if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { + if (Settings.AssumeYdbOnClusterWithSlash && cluster.StartsWith('/')) { normalizedClusterName = cluster; return TString(NYql::KikimrProviderName); - } + } return Nothing(); } @@ -156,32 +156,32 @@ namespace NSQLTranslationV1 { return IntoHeading; } - void DeclareVariable(const TString& varName, const TNodePtr& typeNode); + void DeclareVariable(const TString& varName, const TNodePtr& typeNode); - bool AddExport(TPosition symbolPos, const TString& symbolName); + bool AddExport(TPosition symbolPos, const TString& symbolName); TString AddImport(const TVector<TString>& modulePath); TString AddSimpleUdf(const TString& udf); void SetPackageVersion(const TString& packageName, ui32 version); bool IsStreamingService(const TStringBuf service) const; - - bool CheckColumnReference(TPosition pos, const TString& name) { - const bool allowed = GetColumnReferenceState() != EColumnRefState::Deny; - if (!allowed) { - Error(pos) << "Column reference \"" << name << "\" is not allowed " << NoColumnErrorContext; - IncrementMonCounter("sql_errors", "ColumnReferenceInScopeIsNotAllowed"); - } - return allowed; - } - - EColumnRefState GetColumnReferenceState() const { - return ColumnReferenceState; - } - - EColumnRefState GetTopLevelColumnReferenceState() const { - return TopLevelColumnReferenceState; - } - + + bool CheckColumnReference(TPosition pos, const TString& name) { + const bool allowed = GetColumnReferenceState() != EColumnRefState::Deny; + if (!allowed) { + Error(pos) << "Column reference \"" << name << "\" is not allowed " << NoColumnErrorContext; + IncrementMonCounter("sql_errors", "ColumnReferenceInScopeIsNotAllowed"); + } + return allowed; + } + + EColumnRefState GetColumnReferenceState() const { + return ColumnReferenceState; + } + + EColumnRefState GetTopLevelColumnReferenceState() const { + return TopLevelColumnReferenceState; + } + private: IOutputStream& MakeIssue(NYql::ESeverity severity, NYql::TIssueCode code, NYql::TPosition pos); @@ -194,12 +194,12 @@ namespace NSQLTranslationV1 { THashMap<TString, TString> ClusterPathPrefixes; bool IntoHeading = true; - friend class TColumnRefScope; - - EColumnRefState ColumnReferenceState = EColumnRefState::Deny; - EColumnRefState TopLevelColumnReferenceState = EColumnRefState::Deny; - TString NoColumnErrorContext = "in current scope"; - + friend class TColumnRefScope; + + EColumnRefState ColumnReferenceState = EColumnRefState::Deny; + EColumnRefState TopLevelColumnReferenceState = EColumnRefState::Deny; + TString NoColumnErrorContext = "in current scope"; + public: THashMap<TString, TNodePtr> Variables; NSQLTranslation::TTranslationSettings Settings; @@ -211,88 +211,88 @@ namespace NSQLTranslationV1 { TMap<TString, TString> SimpleUdfs; NSQLTranslation::TIncrementMonCounterFunction IncrementMonCounterFunction; TScopedStatePtr Scoped; - int ScopeLevel = 0; - size_t AnonymousNameIndex = 0; + int ScopeLevel = 0; + size_t AnonymousNameIndex = 0; TDeque<TScopedStatePtr> AllScopes; bool HasPendingErrors; THashMap<TString, ui32> GenIndexes; - using TWinSpecsRef = std::reference_wrapper<TWinSpecs>; - TDeque<TWinSpecsRef> WinSpecsScopes; + using TWinSpecsRef = std::reference_wrapper<TWinSpecs>; + TDeque<TWinSpecsRef> WinSpecsScopes; bool PragmaRefSelect = false; bool PragmaSampleSelect = false; bool PragmaAllowDotInAlias = false; bool PragmaInferSchema = false; bool PragmaAutoCommit = false; bool SimpleColumns = true; - bool CoalesceJoinKeysOnQualifiedAll = false; + bool CoalesceJoinKeysOnQualifiedAll = false; bool PragmaDirectRead = false; bool PragmaYsonFast = true; bool PragmaYsonAutoConvert = false; bool PragmaYsonStrict = true; bool PragmaRegexUseRe2 = true; - bool PragmaPullUpFlatMapOverJoin = true; - bool WarnUnnamedColumns = false; + bool PragmaPullUpFlatMapOverJoin = true; + bool WarnUnnamedColumns = false; bool DiscoveryMode = false; bool EnableSystemColumns = true; bool DqEngineEnable = false; bool DqEngineForce = false; TMaybe<bool> JsonQueryReturnsJsonDocument; - TMaybe<bool> AnsiInForEmptyOrNullableItemsCollections; - TMaybe<bool> AnsiRankForNullableKeys = true; - TMaybe<bool> AnsiOrderByLimitInUnionAll = true; - const bool AnsiQuotedIdentifiers; - bool AnsiOptionalAs = true; - bool OrderedColumns = false; - bool PositionalUnionAll = false; - bool BogousStarInGroupByOverJoin = false; - bool UnorderedSubqueries = true; + TMaybe<bool> AnsiInForEmptyOrNullableItemsCollections; + TMaybe<bool> AnsiRankForNullableKeys = true; + TMaybe<bool> AnsiOrderByLimitInUnionAll = true; + const bool AnsiQuotedIdentifiers; + bool AnsiOptionalAs = true; + bool OrderedColumns = false; + bool PositionalUnionAll = false; + bool BogousStarInGroupByOverJoin = false; + bool UnorderedSubqueries = true; bool PragmaDataWatermarks = true; - bool WarnOnAnsiAliasShadowing = true; + bool WarnOnAnsiAliasShadowing = true; ui32 ResultRowsLimit = 0; ui64 ResultSizeLimit = 0; ui32 PragmaGroupByLimit = 1 << 5; ui32 PragmaGroupByCubeLimit = 5; - // if FlexibleTypes=true, emit TypeOrMember callable and resolve Type/Column uncertainty on type annotation stage, otherwise always emit Type - bool FlexibleTypes = false; + // if FlexibleTypes=true, emit TypeOrMember callable and resolve Type/Column uncertainty on type annotation stage, otherwise always emit Type + bool FlexibleTypes = false; THashMap<TString, TMaybe<TString>> Libraries; // alias -> optional file THashMap<TString, ui32> PackageVersions; - NYql::TWarningPolicy WarningPolicy; + NYql::TWarningPolicy WarningPolicy; TString PqReadByRtmrCluster; }; - class TColumnRefScope { - public: - TColumnRefScope(TContext& ctx, EColumnRefState state, bool isTopLevelExpr = true) - : PrevTop(ctx.TopLevelColumnReferenceState) - , Prev(ctx.ColumnReferenceState) - , PrevErr(ctx.NoColumnErrorContext) - , Ctx(ctx) - { - if (isTopLevelExpr) { - Ctx.ColumnReferenceState = Ctx.TopLevelColumnReferenceState = state; - } else { - Ctx.ColumnReferenceState = state; - } - } - - void SetNoColumnErrContext(const TString& msg) { - Ctx.NoColumnErrorContext = msg; - } - - ~TColumnRefScope() { - Ctx.TopLevelColumnReferenceState = PrevTop; - Ctx.ColumnReferenceState = Prev; - std::swap(Ctx.NoColumnErrorContext, PrevErr); - } - private: - const EColumnRefState PrevTop; - const EColumnRefState Prev; - TString PrevErr; - TContext& Ctx; - }; - - TMaybe<EColumnRefState> GetFunctionArgColumnStatus(TContext& ctx, const TString& module, const TString& func, size_t argIndex); - + class TColumnRefScope { + public: + TColumnRefScope(TContext& ctx, EColumnRefState state, bool isTopLevelExpr = true) + : PrevTop(ctx.TopLevelColumnReferenceState) + , Prev(ctx.ColumnReferenceState) + , PrevErr(ctx.NoColumnErrorContext) + , Ctx(ctx) + { + if (isTopLevelExpr) { + Ctx.ColumnReferenceState = Ctx.TopLevelColumnReferenceState = state; + } else { + Ctx.ColumnReferenceState = state; + } + } + + void SetNoColumnErrContext(const TString& msg) { + Ctx.NoColumnErrorContext = msg; + } + + ~TColumnRefScope() { + Ctx.TopLevelColumnReferenceState = PrevTop; + Ctx.ColumnReferenceState = Prev; + std::swap(Ctx.NoColumnErrorContext, PrevErr); + } + private: + const EColumnRefState PrevTop; + const EColumnRefState Prev; + TString PrevErr; + TContext& Ctx; + }; + + TMaybe<EColumnRefState> GetFunctionArgColumnStatus(TContext& ctx, const TString& module, const TString& func, size_t argIndex); + class TTranslation { protected: typedef TSet<ui32> TSetType; @@ -319,13 +319,13 @@ namespace NSQLTranslationV1 { } TNodePtr GetNamedNode(const TString& name); - - using TNodeBuilderByName = std::function<TNodePtr(const TString& effectiveName)>; - TString PushNamedNode(TPosition namePos, const TString& name, const TNodeBuilderByName& builder); - TString PushNamedNode(TPosition namePos, const TString& name, TNodePtr node); - TString PushNamedAtom(TPosition namePos, const TString& name); + + using TNodeBuilderByName = std::function<TNodePtr(const TString& effectiveName)>; + TString PushNamedNode(TPosition namePos, const TString& name, const TNodeBuilderByName& builder); + TString PushNamedNode(TPosition namePos, const TString& name, TNodePtr node); + TString PushNamedAtom(TPosition namePos, const TString& name); void PopNamedNode(const TString& name); - void WarnUnusedNodes() const; + void WarnUnusedNodes() const; template <typename TNode> void AltNotImplemented(const TString& ruleName, const TNode& node) { diff --git a/ydb/library/yql/sql/v1/insert.cpp b/ydb/library/yql/sql/v1/insert.cpp index 735fe21c53..4f4c863018 100644 --- a/ydb/library/yql/sql/v1/insert.cpp +++ b/ydb/library/yql/sql/v1/insert.cpp @@ -40,12 +40,12 @@ public: } bool AddAggregation(TContext& ctx, TAggregationPtr aggr) override { - YQL_ENSURE(aggr); - ctx.Error(aggr->GetPos()) << "Source does not allow aggregation"; + YQL_ENSURE(aggr); + ctx.Error(aggr->GetPos()) << "Source does not allow aggregation"; return false; } - TNodePtr BuildFilter(TContext& ctx, const TString& label) override { + TNodePtr BuildFilter(TContext& ctx, const TString& label) override { Y_UNUSED(ctx); Y_UNUSED(label); return nullptr; @@ -146,7 +146,7 @@ public: Y_UNUSED(ctx); auto tuple = Y(); for (const auto& row: Values) { - auto rowValues = Y("AsStruct"); // ordered struct + auto rowValues = Y("AsStruct"); // ordered struct auto column = ColumnsHint.begin(); for (auto value: row) { rowValues = L(rowValues, Q(Y(BuildQuotedAtom(Pos, *column), value))); @@ -226,7 +226,7 @@ public: return input; } auto srcColumn = Source->GetColumns()->List.begin(); - auto structObj = Y("AsStruct"); // ordered struct + auto structObj = Y("AsStruct"); // ordered struct for (auto column: ColumnsHint) { structObj = L(structObj, Q(Y(BuildQuotedAtom(Pos, column), Y("Member", "row", BuildQuotedAtom(Pos, *srcColumn)) @@ -300,7 +300,7 @@ public: if (!TableSource->Init(ctx, src) || !TableSource->InitFilters(ctx)) { return false; } - options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda()))); + options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda()))); } bool unordered = false; diff --git a/ydb/library/yql/sql/v1/join.cpp b/ydb/library/yql/sql/v1/join.cpp index 8f58391984..29f5b3ec4b 100644 --- a/ydb/library/yql/sql/v1/join.cpp +++ b/ydb/library/yql/sql/v1/join.cpp @@ -25,7 +25,7 @@ TString NormalizeJoinOp(const TString& joinOp) { struct TJoinDescr { TString Op; - TJoinLinkSettings LinkSettings; + TJoinLinkSettings LinkSettings; struct TFullColumn { ui32 Source; @@ -34,27 +34,27 @@ struct TJoinDescr { TVector<std::pair<TFullColumn, TFullColumn>> Keys; - explicit TJoinDescr(const TString& op) + explicit TJoinDescr(const TString& op) : Op(op) {} }; class TJoinBase: public IJoin { public: - TJoinBase(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags) + TJoinBase(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags) : IJoin(pos) , Sources(std::move(sources)) - , AnyFlags(std::move(anyFlags)) + , AnyFlags(std::move(anyFlags)) { - YQL_ENSURE(Sources.size() == AnyFlags.size()); + YQL_ENSURE(Sources.size() == AnyFlags.size()); + } + + void AllColumns() override { + for (auto& source: Sources) { + source->AllColumns(); + } } - void AllColumns() override { - for (auto& source: Sources) { - source->AllColumns(); - } - } - TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) override { ISource* srcByName = nullptr; if (column.IsArtificial()) { @@ -174,26 +174,26 @@ public: if (!lhs || !rhs) { return nullptr; } - TNodePtr eq(BuildBinaryOp(ctx, pos, "==", lhs, rhs)); + TNodePtr eq(BuildBinaryOp(ctx, pos, "==", lhs, rhs)); if (expr) { - expr = BuildBinaryOp(ctx, pos, "And", expr, eq); + expr = BuildBinaryOp(ctx, pos, "And", expr, eq); } else { expr = eq; } } - if (expr && Sources.size() > 2) { - ctx.Error() << "Multi-way JOINs should be connected with ON clause instead of USING clause"; - return nullptr; - } + if (expr && Sources.size() > 2) { + ctx.Error() << "Multi-way JOINs should be connected with ON clause instead of USING clause"; + return nullptr; + } return expr; } bool DoInit(TContext& ctx, ISource* src) override; - void SetupJoin(const TString& opName, TNodePtr expr, const TJoinLinkSettings& linkSettings) override { + void SetupJoin(const TString& opName, TNodePtr expr, const TJoinLinkSettings& linkSettings) override { JoinOps.push_back(opName); JoinExprs.push_back(expr); - JoinLinkSettings.push_back(linkSettings); + JoinLinkSettings.push_back(linkSettings); } bool IsStream() const override { @@ -209,7 +209,7 @@ protected: bool InitKeysOrFilters(TContext& ctx, ui32 joinIdx, TNodePtr expr) { const TString joinOp(JoinOps[joinIdx]); - const TJoinLinkSettings linkSettings(JoinLinkSettings[joinIdx]); + const TJoinLinkSettings linkSettings(JoinLinkSettings[joinIdx]); const TCallNode* op = nullptr; if (expr) { const TString opName(expr->GetOpName()); @@ -226,11 +226,11 @@ protected: ui32 idx = 0; THashMap<TString, ui32> sources; for (auto& source: Sources) { - auto label = source->GetLabel(); - if (!label) { - ctx.Error(source->GetPos()) << "JOIN: missing correlation name for source"; - return false; - } + auto label = source->GetLabel(); + if (!label) { + ctx.Error(source->GetPos()) << "JOIN: missing correlation name for source"; + return false; + } sources.insert({ source->GetLabel(), idx }); ++idx; } @@ -253,7 +253,7 @@ protected: for (auto& arg : op->GetArgs()) { const auto sourceNamePtr = arg->GetSourceName(); if (!sourceNamePtr) { - ctx.Error(expr->GetPos()) << "JOIN: each equality predicate argument must depend on exactly one JOIN input"; + ctx.Error(expr->GetPos()) << "JOIN: each equality predicate argument must depend on exactly one JOIN input"; return false; } const auto sourceName = *sourceNamePtr; @@ -340,9 +340,9 @@ protected: } if (joinIdx == JoinDescrs.size()) { - TJoinDescr newDescr(joinOp); - newDescr.LinkSettings = linkSettings; - JoinDescrs.push_back(std::move(newDescr)); + TJoinDescr newDescr(joinOp); + newDescr.LinkSettings = linkSettings; + JoinDescrs.push_back(std::move(newDescr)); } JoinDescrs.back().Keys.push_back({ { leftSourceIdx, op ? op->GetArgs()[leftArg] : nullptr}, @@ -358,11 +358,11 @@ protected: protected: TVector<TString> JoinOps; TVector<TNodePtr> JoinExprs; - TVector<TJoinLinkSettings> JoinLinkSettings; + TVector<TJoinLinkSettings> JoinLinkSettings; TVector<TJoinDescr> JoinDescrs; THashMap<TString, THashSet<TString>> SameKeyMap; - const TVector<TSourcePtr> Sources; - const TVector<bool> AnyFlags; + const TVector<TSourcePtr> Sources; + const TVector<bool> AnyFlags; TColumns JoinedColumns; bool KeysInitializing = false; bool IsColumnDone = false; @@ -405,24 +405,24 @@ protected: } }; -bool TJoinBase::DoInit(TContext& ctx, ISource* initSrc) { +bool TJoinBase::DoInit(TContext& ctx, ISource* initSrc) { for (auto& source: Sources) { - if (!source->Init(ctx, initSrc)) { + if (!source->Init(ctx, initSrc)) { return false; } - - auto src = source.Get(); - if (src->IsFlattenByExprs()) { - for (auto& expr : static_cast<ISource const*>(src)->Expressions(EExprSeat::FlattenByExpr)) { - if (!expr->Init(ctx, src)) { - return false; - } - } - } + + auto src = source.Get(); + if (src->IsFlattenByExprs()) { + for (auto& expr : static_cast<ISource const*>(src)->Expressions(EExprSeat::FlattenByExpr)) { + if (!expr->Init(ctx, src)) { + return false; + } + } + } } YQL_ENSURE(JoinOps.size() == JoinExprs.size(), "Invalid join exprs number"); - YQL_ENSURE(JoinOps.size() == JoinLinkSettings.size()); + YQL_ENSURE(JoinOps.size() == JoinLinkSettings.size()); const TSet<TString> allowedJoinOps = {"Inner", "Left", "Right", "Full", "LeftOnly", "RightOnly", "Exclusion", "LeftSemi", "RightSemi", "Cross"}; for (auto& opName: JoinOps) { @@ -470,14 +470,14 @@ bool TJoinBase::DoInit(TContext& ctx, ISource* initSrc) { } } - return ISource::DoInit(ctx, initSrc); + return ISource::DoInit(ctx, initSrc); } class TEquiJoin: public TJoinBase { public: - TEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes) - : TJoinBase(pos, std::move(sources), std::move(anyFlags)) - , StrictJoinKeyTypes(strictJoinKeyTypes) + TEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes) + : TJoinBase(pos, std::move(sources), std::move(anyFlags)) + , StrictJoinKeyTypes(strictJoinKeyTypes) { } @@ -486,43 +486,43 @@ public: TNodePtr joinTree; for (auto& descr: JoinDescrs) { auto leftBranch = joinTree; - bool leftAny = false; + bool leftAny = false; if (!leftBranch) { leftBranch = BuildQuotedAtom(Pos, Sources[descr.Keys[0].first.Source]->GetLabel()); - leftAny = AnyFlags[descr.Keys[0].first.Source]; + leftAny = AnyFlags[descr.Keys[0].first.Source]; } - bool rightAny = AnyFlags[descr.Keys[0].second.Source]; + bool rightAny = AnyFlags[descr.Keys[0].second.Source]; auto leftKeys = GetColumnNames(ctx, extraColumns, descr.Keys, true); auto rightKeys = GetColumnNames(ctx, extraColumns, descr.Keys, false); if (!leftKeys || !rightKeys) { return nullptr; } - TNodePtr linkOptions = Y(); - if (descr.LinkSettings.ForceSortedMerge) { - linkOptions = L(linkOptions, Q(Y(Q("forceSortedMerge")))); - } - if (leftAny) { - linkOptions = L(linkOptions, Q(Y(Q("left"), Q("any")))); - } - if (rightAny) { - linkOptions = L(linkOptions, Q(Y(Q("right"), Q("any")))); - } - + TNodePtr linkOptions = Y(); + if (descr.LinkSettings.ForceSortedMerge) { + linkOptions = L(linkOptions, Q(Y(Q("forceSortedMerge")))); + } + if (leftAny) { + linkOptions = L(linkOptions, Q(Y(Q("left"), Q("any")))); + } + if (rightAny) { + linkOptions = L(linkOptions, Q(Y(Q("right"), Q("any")))); + } + joinTree = Q(Y( Q(descr.Op), leftBranch, BuildQuotedAtom(Pos, Sources[descr.Keys[0].second.Source]->GetLabel()), leftKeys, rightKeys, - Q(linkOptions) + Q(linkOptions) )); } TNodePtr equiJoin(Y("EquiJoin")); bool ordered = false; - for (size_t i = 0; i < Sources.size(); ++i) { - auto& source = Sources[i]; + for (size_t i = 0; i < Sources.size(); ++i) { + auto& source = Sources[i]; auto sourceNode = source->Build(ctx); if (!sourceNode) { return nullptr; @@ -538,16 +538,16 @@ public: return nullptr; } auto block = Y(Y("let", "flatten", sourceNode)); - - if (source->IsFlattenByExprs()) { - auto premap = source->BuildPreFlattenMap(ctx); - if (!premap) { - return nullptr; - } - - block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), premap)))); - } - + + if (source->IsFlattenByExprs()) { + auto premap = source->BuildPreFlattenMap(ctx); + if (!premap) { + return nullptr; + } + + block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), premap)))); + } + block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), flatten, "res")))); sourceNode = Y("block", Q(L(block, Y("return", "flatten")))); } @@ -557,7 +557,7 @@ public: break; } if (!extraMembers) { - extraMembers = Y(); + extraMembers = Y(); } extraMembers = L( extraMembers, @@ -583,9 +583,9 @@ public: ); } auto options = Y(); - if (StrictJoinKeyTypes) { - options = L(options, Q(Y(Q("strict_keys")))); - } + if (StrictJoinKeyTypes) { + options = L(options, Q(Y(Q("strict_keys")))); + } equiJoin = L(equiJoin, joinTree, Q(options)); if (removeMembers) { equiJoin = Y(ordered ? "OrderedMap" : "Map", equiJoin, BuildLambda(Pos, Y("row"), removeMembers, "row")); @@ -597,12 +597,12 @@ public: return SameKeyMap; } - TVector<TString> GetJoinLabels() const override { - TVector<TString> labels; + TVector<TString> GetJoinLabels() const override { + TVector<TString> labels; for (auto& source: Sources) { const auto label = source->GetLabel(); YQL_ENSURE(label); - labels.push_back(label); + labels.push_back(label); } return labels; } @@ -612,10 +612,10 @@ public: for (auto& cur: Sources) { clonedSources.push_back(cur->CloneSource()); } - auto newSource = MakeIntrusive<TEquiJoin>(Pos, std::move(clonedSources), TVector<bool>(AnyFlags), StrictJoinKeyTypes); + auto newSource = MakeIntrusive<TEquiJoin>(Pos, std::move(clonedSources), TVector<bool>(AnyFlags), StrictJoinKeyTypes); newSource->JoinOps = JoinOps; newSource->JoinExprs = CloneContainer(JoinExprs); - newSource->JoinLinkSettings = JoinLinkSettings; + newSource->JoinLinkSettings = JoinLinkSettings; return newSource; } @@ -651,12 +651,12 @@ private: return Q(res); } - - const bool StrictJoinKeyTypes; + + const bool StrictJoinKeyTypes; }; -TSourcePtr BuildEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes) { - return new TEquiJoin(pos, std::move(sources), std::move(anyFlags), strictJoinKeyTypes); +TSourcePtr BuildEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes) { + return new TEquiJoin(pos, std::move(sources), std::move(anyFlags), strictJoinKeyTypes); } } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/lexer/lexer.cpp b/ydb/library/yql/sql/v1/lexer/lexer.cpp index 236de4f6c9..3f58f9d236 100644 --- a/ydb/library/yql/sql/v1/lexer/lexer.cpp +++ b/ydb/library/yql/sql/v1/lexer/lexer.cpp @@ -1,65 +1,65 @@ -#include "lexer.h" - +#include "lexer.h" + #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h> #include <ydb/library/yql/parser/proto_ast/gen/v1/SQLv1Lexer.h> #include <ydb/library/yql/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> - -#if defined(_tsan_enabled_) -#include <util/system/mutex.h> -#endif - -namespace NALPDefault { -extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; -} - -namespace NALPAnsi { -extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; -} - -namespace NSQLTranslationV1 { - -namespace { - -#if defined(_tsan_enabled_) -TMutex SanitizerSQLTranslationMutex; -#endif - -using NSQLTranslation::ILexer; -using NSQLTranslation::TParsedTokenList; - -class TV1Lexer : public ILexer { -public: - explicit TV1Lexer(bool ansi) - : Ansi(ansi) - { - } - - bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override { - issues.Clear(); -#if defined(_tsan_enabled_) - TGuard<TMutex> grd(SanitizerSQLTranslationMutex); -#endif - NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, ""); - if (Ansi) { - NProtoAST::TLexerTokensCollector<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName); - tokens = tokensCollector.CollectTokens(collector); - } else { - NProtoAST::TLexerTokensCollector<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName); - tokens = tokensCollector.CollectTokens(collector); - } - - return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); - } - -private: - const bool Ansi; -}; - -} // namespace - -NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi) { - return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi)); -} - -} // namespace NSQLTranslationV1 + +#if defined(_tsan_enabled_) +#include <util/system/mutex.h> +#endif + +namespace NALPDefault { +extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; +} + +namespace NALPAnsi { +extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; +} + +namespace NSQLTranslationV1 { + +namespace { + +#if defined(_tsan_enabled_) +TMutex SanitizerSQLTranslationMutex; +#endif + +using NSQLTranslation::ILexer; +using NSQLTranslation::TParsedTokenList; + +class TV1Lexer : public ILexer { +public: + explicit TV1Lexer(bool ansi) + : Ansi(ansi) + { + } + + bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override { + issues.Clear(); +#if defined(_tsan_enabled_) + TGuard<TMutex> grd(SanitizerSQLTranslationMutex); +#endif + NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, ""); + if (Ansi) { + NProtoAST::TLexerTokensCollector<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName); + tokens = tokensCollector.CollectTokens(collector); + } else { + NProtoAST::TLexerTokensCollector<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName); + tokens = tokensCollector.CollectTokens(collector); + } + + return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } + +private: + const bool Ansi; +}; + +} // namespace + +NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi) { + return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi)); +} + +} // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/lexer/lexer.h b/ydb/library/yql/sql/v1/lexer/lexer.h index e693417832..312c1e2072 100644 --- a/ydb/library/yql/sql/v1/lexer/lexer.h +++ b/ydb/library/yql/sql/v1/lexer/lexer.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/parser/lexer_common/lexer.h> - -namespace NSQLTranslationV1 { - -NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi); - -} + +namespace NSQLTranslationV1 { + +NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi); + +} diff --git a/ydb/library/yql/sql/v1/lexer/ya.make b/ydb/library/yql/sql/v1/lexer/ya.make index 98c8ae2b14..8a26fa9474 100644 --- a/ydb/library/yql/sql/v1/lexer/ya.make +++ b/ydb/library/yql/sql/v1/lexer/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( g:yql g:yql_ydb_core -) - -PEERDIR( +) + +PEERDIR( ydb/library/yql/core/issue/protos ydb/library/yql/parser/proto_ast ydb/library/yql/parser/proto_ast/gen/v1 ydb/library/yql/parser/proto_ast/gen/v1_ansi ydb/library/yql/parser/proto_ast/gen/v1_proto -) - -SRCS( - lexer.cpp -) - -END() +) + +SRCS( + lexer.cpp +) + +END() diff --git a/ydb/library/yql/sql/v1/list_builtin.cpp b/ydb/library/yql/sql/v1/list_builtin.cpp index 085019c117..bc4edc2e4c 100644 --- a/ydb/library/yql/sql/v1/list_builtin.cpp +++ b/ydb/library/yql/sql/v1/list_builtin.cpp @@ -109,10 +109,10 @@ bool TListCreateBuiltin::DoInit(TContext& ctx, ISource* src) { return true; } -void TListCreateBuiltin::DoUpdateState() const { - State.Set(ENodeState::Const); -} - +void TListCreateBuiltin::DoUpdateState() const { + State.Set(ENodeState::Const); +} + bool TDictCreateBuiltin::DoInit(TContext& ctx, ISource* src) { if (Args.size() != 2) { ctx.Error(Pos) << OpName << " requires two parameters"; @@ -129,10 +129,10 @@ bool TDictCreateBuiltin::DoInit(TContext& ctx, ISource* src) { return true; } -void TDictCreateBuiltin::DoUpdateState() const { - State.Set(ENodeState::Const); -} - +void TDictCreateBuiltin::DoUpdateState() const { + State.Set(ENodeState::Const); +} + bool TSetCreateBuiltin::DoInit(TContext& ctx, ISource* src) { if (Args.size() != 1) { ctx.Error(Pos) << OpName << " requires one parameter"; @@ -147,8 +147,8 @@ bool TSetCreateBuiltin::DoInit(TContext& ctx, ISource* src) { return true; } -void TSetCreateBuiltin::DoUpdateState() const { - State.Set(ENodeState::Const); -} - +void TSetCreateBuiltin::DoUpdateState() const { + State.Set(ENodeState::Const); +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/list_builtin.h b/ydb/library/yql/sql/v1/list_builtin.h index a4495e85d9..a8077b464a 100644 --- a/ydb/library/yql/sql/v1/list_builtin.h +++ b/ydb/library/yql/sql/v1/list_builtin.h @@ -134,7 +134,7 @@ public: {} bool DoInit(TContext& ctx, ISource* src) override; - void DoUpdateState() const override; + void DoUpdateState() const override; TNodePtr DoClone() const final { return new TListCreateBuiltin(Pos, CloneContainer(Args)); @@ -149,7 +149,7 @@ public: {} bool DoInit(TContext& ctx, ISource* src) override; - void DoUpdateState() const override; + void DoUpdateState() const override; TNodePtr DoClone() const final { return new TDictCreateBuiltin(Pos, CloneContainer(Args)); @@ -164,7 +164,7 @@ public: {} bool DoInit(TContext& ctx, ISource* src) override; - void DoUpdateState() const override; + void DoUpdateState() const override; TNodePtr DoClone() const final { return new TSetCreateBuiltin(Pos, CloneContainer(Args)); diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index 8a0f308b6e..870bbd9cc7 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -12,7 +12,7 @@ #include <util/generic/hash_set.h> #include <util/stream/str.h> #include <util/string/cast.h> -#include <util/string/escape.h> +#include <util/string/escape.h> #include <util/string/subst.h> using namespace NYql; @@ -71,23 +71,23 @@ const TString& INode::GetLabel() const { return Label; } -TMaybe<TPosition> INode::GetLabelPos() const { - return LabelPos; -} - -void INode::SetLabel(const TString& label, TMaybe<TPosition> pos) { +TMaybe<TPosition> INode::GetLabelPos() const { + return LabelPos; +} + +void INode::SetLabel(const TString& label, TMaybe<TPosition> pos) { Label = label; - LabelPos = pos; + LabelPos = pos; +} + +bool INode::IsImplicitLabel() const { + return ImplicitLabel; +} + +void INode::MarkImplicitLabel(bool isImplicitLabel) { + ImplicitLabel = isImplicitLabel; } -bool INode::IsImplicitLabel() const { - return ImplicitLabel; -} - -void INode::MarkImplicitLabel(bool isImplicitLabel) { - ImplicitLabel = isImplicitLabel; -} - void INode::SetCountHint(bool isCount) { State.Set(ENodeState::CountHint, isCount); } @@ -100,10 +100,10 @@ bool INode::IsConstant() const { return HasState(ENodeState::Const); } -bool INode::MaybeConstant() const { - return HasState(ENodeState::MaybeConst); -} - +bool INode::MaybeConstant() const { + return HasState(ENodeState::MaybeConst); +} + bool INode::IsAggregated() const { return HasState(ENodeState::Aggregated); } @@ -136,14 +136,14 @@ bool INode::IsIntegerLiteral() const { return false; } -INode::TPtr INode::ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const { - Y_UNUSED(ctx); - if (IsNull()) { - return BuildLiteralNull(pos); - } - return new TCallNodeImpl(pos, opName, { Clone() }); -} - +INode::TPtr INode::ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const { + Y_UNUSED(ctx); + if (IsNull()) { + return BuildLiteralNull(pos); + } + return new TCallNodeImpl(pos, opName, { Clone() }); +} + bool INode::IsAsterisk() const { return false; } @@ -244,8 +244,8 @@ TNodePtr INode::Clone() const { clone = const_cast<INode*>(this); } else { YQL_ENSURE(!State.Test(ENodeState::Initialized), "Clone should be for uninitialized or persistent node"); - clone->SetLabel(Label, LabelPos); - clone->MarkImplicitLabel(ImplicitLabel); + clone->SetLabel(Label, LabelPos); + clone->MarkImplicitLabel(ImplicitLabel); } return clone; } @@ -254,12 +254,12 @@ TAggregationPtr INode::GetAggregation() const { return {}; } -void INode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { - Y_UNUSED(ctx); - Y_UNUSED(src); - Y_UNUSED(exprs); -} - +void INode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { + Y_UNUSED(ctx); + Y_UNUSED(src); + Y_UNUSED(exprs); +} + INode::TPtr INode::WindowSpecFunc(const TPtr& type) const { Y_UNUSED(type); return {}; @@ -292,17 +292,17 @@ const TString* INode::ModuleName() const { return nullptr; } -void INode::VisitTree(const TVisitFunc& func) const { - TVisitNodeSet visited; - VisitTree(func, visited); -} - -void INode::VisitTree(const TVisitFunc& func, TVisitNodeSet& visited) const { - if (visited.emplace(this).second && HasState(ENodeState::Initialized) && func(*this)) { - DoVisitChildren(func, visited); - } -} - +void INode::VisitTree(const TVisitFunc& func) const { + TVisitNodeSet visited; + VisitTree(func, visited); +} + +void INode::VisitTree(const TVisitFunc& func, TVisitNodeSet& visited) const { + if (visited.emplace(this).second && HasState(ENodeState::Initialized) && func(*this)) { + DoVisitChildren(func, visited); + } +} + TNodePtr INode::ShallowCopy() const { Y_VERIFY_DEBUG(false, "Node is not copyable"); return nullptr; @@ -326,23 +326,23 @@ void INode::PrecacheState() const { State.Set(ENodeState::Precached); } -void INode::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { - Y_UNUSED(func); - Y_UNUSED(visited); -} - +void INode::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { + Y_UNUSED(func); + Y_UNUSED(visited); +} + void INode::DoAdd(TNodePtr node) { Y_UNUSED(node); Y_VERIFY_DEBUG(false, "Node is not expandable"); } -void MergeHints(TTableHints& base, const TTableHints& overrides) { - for (auto& i : overrides) { - base[i.first] = i.second; - } -} - -TAstAtomNode::TAstAtomNode(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg) +void MergeHints(TTableHints& base, const TTableHints& overrides) { + for (auto& i : overrides) { + base[i.first] = i.second; + } +} + +TAstAtomNode::TAstAtomNode(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg) : INode(pos) , Content(content) , Flags(flags) @@ -355,7 +355,7 @@ TAstAtomNode::~TAstAtomNode() } void TAstAtomNode::DoUpdateState() const { - State.Set(ENodeState::Const); + State.Set(ENodeState::Const); } TAstNode* TAstAtomNode::Translate(TContext& ctx) const { @@ -409,7 +409,7 @@ TAstNode* TAstListNode::Translate(TContext& ctx) const { auto listPos = Pos; for (auto& node: Nodes) { if (node) { - auto astNode = node->Translate(ctx); + auto astNode = node->Translate(ctx); if (!astNode) { return nullptr; } @@ -433,36 +433,36 @@ void TAstListNode::UpdateStateByListNodes(const TVector<TNodePtr>& nodes) const std::map<ENodeState, TAttributesFlags> flags; for (auto& node: nodes) { const bool isNodeConst = node->IsConstant(); - const bool isNodeMaybeConst = node->MaybeConstant(); + const bool isNodeMaybeConst = node->MaybeConstant(); for (auto state: checkStates) { if (node->HasState(state)) { flags[state].has = true; - } else if (!isNodeConst && !isNodeMaybeConst) { + } else if (!isNodeConst && !isNodeMaybeConst) { flags[state].all = false; } - - if (!isNodeConst) { - isConst = false; - } + + if (!isNodeConst) { + isConst = false; + } } } State.Set(ENodeState::Const, isConst); for (auto& flag: flags) { State.Set(flag.first, flag.second.has && flag.second.all); } - State.Set(ENodeState::MaybeConst, !isConst && AllOf(nodes, [](const auto& node) { return node->IsConstant() || node->MaybeConstant(); })); + State.Set(ENodeState::MaybeConst, !isConst && AllOf(nodes, [](const auto& node) { return node->IsConstant() || node->MaybeConstant(); })); } void TAstListNode::DoUpdateState() const { UpdateStateByListNodes(Nodes); } -void TAstListNode::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { - for (auto& node : Nodes) { - node->VisitTree(func, visited); - } -} - +void TAstListNode::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { + for (auto& node : Nodes) { + node->VisitTree(func, visited); + } +} + TAstListNode::TAstListNode(const TAstListNode& node) : INode(node.Pos) , Nodes(node.Nodes) @@ -475,9 +475,9 @@ TAstListNode::TAstListNode(TPosition pos, TVector<TNodePtr>&& nodes) : INode(pos) , Nodes(std::move(nodes)) { - for (const auto& node: Nodes) { - YQL_ENSURE(node, "Null ptr passed as list element"); - } + for (const auto& node: Nodes) { + YQL_ENSURE(node, "Null ptr passed as list element"); + } } TNodePtr TAstListNode::ShallowCopy() const { @@ -497,18 +497,18 @@ TAstListNodeImpl::TAstListNodeImpl(TPosition pos) TAstListNodeImpl::TAstListNodeImpl(TPosition pos, TVector<TNodePtr> nodes) : TAstListNode(pos) { - for (const auto& node: nodes) { - YQL_ENSURE(node, "Null ptr passed as list element"); - } + for (const auto& node: nodes) { + YQL_ENSURE(node, "Null ptr passed as list element"); + } Nodes.swap(nodes); } -void TAstListNodeImpl::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { - for (auto& node : Nodes) { - node->CollectPreaggregateExprs(ctx, src, exprs); - } -} - +void TAstListNodeImpl::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { + for (auto& node : Nodes) { + node->CollectPreaggregateExprs(ctx, src, exprs); + } +} + TNodePtr TAstListNodeImpl::DoClone() const { return new TAstListNodeImpl(Pos, CloneContainer(Nodes)); } @@ -520,20 +520,20 @@ TCallNode::TCallNode(TPosition pos, const TString& opName, i32 minArgs, i32 maxA , MaxArgs(maxArgs) , Args(args) { - for (const auto& arg: Args) { - YQL_ENSURE(arg, "Null ptr passed as call argument"); - } + for (const auto& arg: Args) { + YQL_ENSURE(arg, "Null ptr passed as call argument"); + } } TString TCallNode::GetOpName() const { return OpName; } -namespace { -const TString* DeriveCommonSourceName(const TVector<TNodePtr> &nodes) { +namespace { +const TString* DeriveCommonSourceName(const TVector<TNodePtr> &nodes) { const TString* name = nullptr; - for (auto& node: nodes) { - auto n = node->GetSourceName(); + for (auto& node: nodes) { + auto n = node->GetSourceName(); if (!n) { continue; } @@ -545,12 +545,12 @@ const TString* DeriveCommonSourceName(const TVector<TNodePtr> &nodes) { return name; } -} - -const TString* TCallNode::GetSourceName() const { - return DeriveCommonSourceName(Args); -} - +} + +const TString* TCallNode::GetSourceName() const { + return DeriveCommonSourceName(Args); +} + const TVector<TNodePtr>& TCallNode::GetArgs() const { return Args; } @@ -569,12 +569,12 @@ TString TCallNode::GetCallExplain() const { return std::move(sb); } -void TCallNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { - for (auto& arg : Args) { - arg->CollectPreaggregateExprs(ctx, src, exprs); - } -} - +void TCallNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { + for (auto& arg : Args) { + arg->CollectPreaggregateExprs(ctx, src, exprs); + } +} + bool TCallNode::ValidateArguments(TContext& ctx) const { const auto argsCount = static_cast<i32>(Args.size()); if (MinArgs >= 0 && MaxArgs == MinArgs && argsCount != MinArgs) { @@ -701,16 +701,16 @@ void TWinAggrEmulation::DoUpdateState() const { bool TWinAggrEmulation::DoInit(TContext& ctx, ISource* src) { if (!src) { - ctx.Error(Pos) << "Unable to use window function " << OpName << " without source"; + ctx.Error(Pos) << "Unable to use window function " << OpName << " without source"; return false; } if (!src->IsOverWindowSource()) { - ctx.Error(Pos) << "Failed to use window function " << OpName << " without window specification"; + ctx.Error(Pos) << "Failed to use window function " << OpName << " without window specification"; return false; } if (!src->AddFuncOverWindow(ctx, this)) { - ctx.Error(Pos) << "Failed to use window function " << OpName << " without window specification or in wrong place"; + ctx.Error(Pos) << "Failed to use window function " << OpName << " without window specification or in wrong place"; return false; } @@ -756,15 +756,15 @@ bool TWinLeadLag::DoInit(TContext& ctx, ISource* src) { return false; } if (Args.size() >= 1) { - Args[0] = BuildLambda(Pos, Y("row"), Args[0]); + Args[0] = BuildLambda(Pos, Y("row"), Args[0]); } return true; } -TWinRank::TWinRank(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args) - : TWinAggrEmulation(pos, opName, minArgs, maxArgs, args) +TWinRank::TWinRank(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args) + : TWinAggrEmulation(pos, opName, minArgs, maxArgs, args) { - + } bool TExternalFunctionConfig::DoInit(TContext& ctx, ISource* src) { @@ -782,68 +782,68 @@ INode::TPtr TExternalFunctionConfig::DoClone() const { return {}; } -bool TWinRank::DoInit(TContext& ctx, ISource* src) { - if (!ValidateArguments(ctx)) { - return false; - } - - if (!src) { - ctx.Error(Pos) << "Unable to use window function: " << OpName << " without source"; - return false; - } - - auto winNamePtr = src->GetWindowName(); - if (!winNamePtr) { - ctx.Error(Pos) << "Failed to use window function: " << OpName << " without window"; - return false; - } - - auto winSpecPtr = src->FindWindowSpecification(ctx, *winNamePtr); - if (!winSpecPtr) { - return false; - } - - const auto& orderSpec = winSpecPtr->OrderBy; - if (orderSpec.empty()) { - if (Args.empty()) { - ctx.Warning(GetPos(), TIssuesIds::YQL_RANK_WITHOUT_ORDER_BY) << - OpName << "() is used with unordered window - all rows will be considered equal to each other"; - } else { - ctx.Warning(GetPos(), TIssuesIds::YQL_RANK_WITHOUT_ORDER_BY) << - OpName << "(<expression>) is used with unordered window - the result is likely to be undefined"; - } - } - - if (Args.empty()) { - for (const auto& spec: orderSpec) { - Args.push_back(spec->OrderExpr->Clone()); - } - - if (Args.size() != 1) { - Args = {BuildTuple(GetPos(), Args)}; - } - } - - YQL_ENSURE(Args.size() == 1); - - TVector<TNodePtr> optionsElements; - if (!ctx.AnsiRankForNullableKeys.Defined()) { - optionsElements.push_back(BuildTuple(Pos, { BuildQuotedAtom(Pos, "warnNoAnsi", NYql::TNodeFlags::Default) })); - } else if (*ctx.AnsiRankForNullableKeys) { - optionsElements.push_back(BuildTuple(Pos, { BuildQuotedAtom(Pos, "ansi", NYql::TNodeFlags::Default) })); - } - Args.push_back(BuildTuple(Pos, optionsElements)); - - MinArgs = MaxArgs = 2; - if (!TWinAggrEmulation::DoInit(ctx, src)) { - return false; - } - - YQL_ENSURE(Args.size() == 2); - Args[0] = BuildLambda(Pos, Y("row"), Args[0]); - return true; -} - +bool TWinRank::DoInit(TContext& ctx, ISource* src) { + if (!ValidateArguments(ctx)) { + return false; + } + + if (!src) { + ctx.Error(Pos) << "Unable to use window function: " << OpName << " without source"; + return false; + } + + auto winNamePtr = src->GetWindowName(); + if (!winNamePtr) { + ctx.Error(Pos) << "Failed to use window function: " << OpName << " without window"; + return false; + } + + auto winSpecPtr = src->FindWindowSpecification(ctx, *winNamePtr); + if (!winSpecPtr) { + return false; + } + + const auto& orderSpec = winSpecPtr->OrderBy; + if (orderSpec.empty()) { + if (Args.empty()) { + ctx.Warning(GetPos(), TIssuesIds::YQL_RANK_WITHOUT_ORDER_BY) << + OpName << "() is used with unordered window - all rows will be considered equal to each other"; + } else { + ctx.Warning(GetPos(), TIssuesIds::YQL_RANK_WITHOUT_ORDER_BY) << + OpName << "(<expression>) is used with unordered window - the result is likely to be undefined"; + } + } + + if (Args.empty()) { + for (const auto& spec: orderSpec) { + Args.push_back(spec->OrderExpr->Clone()); + } + + if (Args.size() != 1) { + Args = {BuildTuple(GetPos(), Args)}; + } + } + + YQL_ENSURE(Args.size() == 1); + + TVector<TNodePtr> optionsElements; + if (!ctx.AnsiRankForNullableKeys.Defined()) { + optionsElements.push_back(BuildTuple(Pos, { BuildQuotedAtom(Pos, "warnNoAnsi", NYql::TNodeFlags::Default) })); + } else if (*ctx.AnsiRankForNullableKeys) { + optionsElements.push_back(BuildTuple(Pos, { BuildQuotedAtom(Pos, "ansi", NYql::TNodeFlags::Default) })); + } + Args.push_back(BuildTuple(Pos, optionsElements)); + + MinArgs = MaxArgs = 2; + if (!TWinAggrEmulation::DoInit(ctx, src)) { + return false; + } + + YQL_ENSURE(Args.size() == 2); + Args[0] = BuildLambda(Pos, Y("row"), Args[0]); + return true; +} + class TQuotedAtomNode: public TAstListNode { public: TQuotedAtomNode(TPosition pos, const TString& content, ui32 flags) @@ -962,34 +962,34 @@ TSortSpecificationPtr TSortSpecification::Clone() const { return res; } -TFrameBoundPtr TFrameBound::Clone() const { - auto res = MakeIntrusive<TFrameBound>(); - res->Pos = Pos; - res->Bound = SafeClone(Bound); - res->Settings = Settings; - return res; -} - -TFrameSpecificationPtr TFrameSpecification::Clone() const { - YQL_ENSURE(FrameBegin); - YQL_ENSURE(FrameEnd); - auto res = MakeIntrusive<TFrameSpecification>(); - res->FrameType = FrameType; - res->FrameBegin = FrameBegin->Clone(); - res->FrameEnd = FrameEnd->Clone(); - res->FrameExclusion = FrameExclusion; - return res; -} - +TFrameBoundPtr TFrameBound::Clone() const { + auto res = MakeIntrusive<TFrameBound>(); + res->Pos = Pos; + res->Bound = SafeClone(Bound); + res->Settings = Settings; + return res; +} + +TFrameSpecificationPtr TFrameSpecification::Clone() const { + YQL_ENSURE(FrameBegin); + YQL_ENSURE(FrameEnd); + auto res = MakeIntrusive<TFrameSpecification>(); + res->FrameType = FrameType; + res->FrameBegin = FrameBegin->Clone(); + res->FrameEnd = FrameEnd->Clone(); + res->FrameExclusion = FrameExclusion; + return res; +} + TWindowSpecificationPtr TWindowSpecification::Clone() const { - YQL_ENSURE(Frame); + YQL_ENSURE(Frame); auto res = MakeIntrusive<TWindowSpecification>(); res->ExistingWindowName = ExistingWindowName; res->Partitions = CloneContainer(Partitions); - res->IsCompact = IsCompact; + res->IsCompact = IsCompact; res->OrderBy = CloneContainer(OrderBy); - res->Session = SafeClone(Session); - res->Frame = Frame->Clone(); + res->Session = SafeClone(Session); + res->Frame = Frame->Clone(); return res; } @@ -1003,11 +1003,11 @@ THoppingWindowSpecPtr THoppingWindowSpec::Clone() const { return res; } -TColumnNode::TColumnNode(TPosition pos, const TString& column, const TString& source, bool maybeType) +TColumnNode::TColumnNode(TPosition pos, const TString& column, const TString& source, bool maybeType) : INode(pos) , ColumnName(column) , Source(source) - , MaybeType(maybeType) + , MaybeType(maybeType) { } @@ -1070,14 +1070,14 @@ bool TColumnNode::DoInit(TContext& ctx, ISource* src) { if (IsAsterisk()) { Node = AstNode("row"); } else { - TString callable; - if (MaybeType) { - callable = Reliable && !UseSource ? "SqlPlainColumnOrType" : "SqlColumnOrType"; - } else { - // TODO: consider replacing Member -> SqlPlainColumn - callable = Reliable && !UseSource ? "Member" : "SqlColumn"; - } - Node = Y(callable, "row", ColumnExpr ? Y("EvaluateAtom", ColumnExpr) : BuildQuotedAtom(Pos, *GetColumnName())); + TString callable; + if (MaybeType) { + callable = Reliable && !UseSource ? "SqlPlainColumnOrType" : "SqlColumnOrType"; + } else { + // TODO: consider replacing Member -> SqlPlainColumn + callable = Reliable && !UseSource ? "Member" : "SqlColumn"; + } + Node = Y(callable, "row", ColumnExpr ? Y("EvaluateAtom", ColumnExpr) : BuildQuotedAtom(Pos, *GetColumnName())); if (UseSource) { YQL_ENSURE(Source); Node = L(Node, BuildQuotedAtom(Pos, Source)); @@ -1112,13 +1112,13 @@ bool TColumnNode::IsReliable() const { return Reliable; } -bool TColumnNode::CanBeType() const { - return MaybeType; -} - +bool TColumnNode::CanBeType() const { + return MaybeType; +} + TNodePtr TColumnNode::DoClone() const { YQL_ENSURE(!Node, "TColumnNode::Clone: Node should not be initialized"); - auto copy = ColumnExpr ? new TColumnNode(Pos, ColumnExpr, Source) : new TColumnNode(Pos, ColumnName, Source, MaybeType); + auto copy = ColumnExpr ? new TColumnNode(Pos, ColumnExpr, Source) : new TColumnNode(Pos, ColumnName, Source, MaybeType); copy->GroupKey = GroupKey; copy->Artificial = Artificial; copy->Reliable = Reliable; @@ -1129,7 +1129,7 @@ TNodePtr TColumnNode::DoClone() const { void TColumnNode::DoUpdateState() const { State.Set(ENodeState::Const, false); - State.Set(ENodeState::MaybeConst, MaybeType); + State.Set(ENodeState::MaybeConst, MaybeType); State.Set(ENodeState::Aggregated, GroupKey); State.Set(ENodeState::AggregationKey, GroupKey); } @@ -1161,8 +1161,8 @@ void TColumnNode::ResetColumn(const TNodePtr& column, const TString& source) { const TString TColumnNode::Empty; TNodePtr BuildColumn(TPosition pos, const TString& column, const TString& source) { - bool maybeType = false; - return new TColumnNode(pos, column, source, maybeType); + bool maybeType = false; + return new TColumnNode(pos, column, source, maybeType); } TNodePtr BuildColumn(TPosition pos, const TNodePtr& column, const TString& source) { @@ -1173,11 +1173,11 @@ TNodePtr BuildColumn(TPosition pos, const TDeferredAtom& column, const TString& return column.GetLiteral() ? BuildColumn(pos, *column.GetLiteral(), source) : BuildColumn(pos, column.Build(), source); } -TNodePtr BuildColumnOrType(TPosition pos, const TString& column) { - TString source = ""; - bool maybeType = true; - return new TColumnNode(pos, column, source, maybeType); -} +TNodePtr BuildColumnOrType(TPosition pos, const TString& column) { + TString source = ""; + bool maybeType = true; + return new TColumnNode(pos, column, source, maybeType); +} ITableKeys::ITableKeys(TPosition pos) : INode(pos) @@ -1219,14 +1219,14 @@ const TString& IAggregation::GetName() const { return Name; } -EAggregateMode IAggregation::GetAggregationMode() const { - return AggMode; -} - -void IAggregation::MarkKeyColumnAsGenerated() { - IsGeneratedKeyColumn = true; -} - +EAggregateMode IAggregation::GetAggregationMode() const { + return AggMode; +} + +void IAggregation::MarkKeyColumnAsGenerated() { + IsGeneratedKeyColumn = true; +} + IAggregation::IAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) : INode(pos), Name(name), Func(func), AggMode(aggMode) {} @@ -1275,7 +1275,7 @@ TSourcePtr ISource::CloneSource() const { result->NamedExprs[i] = CloneContainer(NamedExprs[i]); } result->FlattenColumns = FlattenColumns; - result->FlattenMode = FlattenMode; + result->FlattenMode = FlattenMode; return result; } @@ -1332,17 +1332,17 @@ void ISource::SetCompactGroupBy(bool compactGroupBy) { bool ISource::AddExpressions(TContext& ctx, const TVector<TNodePtr>& expressions, EExprSeat exprSeat) { YQL_ENSURE(exprSeat < EExprSeat::Max); - THashSet<TString> names; - THashSet<TString> aliasSet; - // TODO: merge FlattenBy with FlattenByExpr - const bool isFlatten = (exprSeat == EExprSeat::FlattenBy || exprSeat == EExprSeat::FlattenByExpr); - THashSet<TString>& aliases = isFlatten ? FlattenByAliases : aliasSet; + THashSet<TString> names; + THashSet<TString> aliasSet; + // TODO: merge FlattenBy with FlattenByExpr + const bool isFlatten = (exprSeat == EExprSeat::FlattenBy || exprSeat == EExprSeat::FlattenByExpr); + THashSet<TString>& aliases = isFlatten ? FlattenByAliases : aliasSet; for (const auto& expr: expressions) { const auto& alias = expr->GetLabel(); const auto& columnNamePtr = expr->GetColumnName(); if (alias) { - ExprAliases.insert(alias); - if (!aliases.emplace(alias).second) { + ExprAliases.insert(alias); + if (!aliases.emplace(alias).second) { ctx.Error(expr->GetPos()) << "Duplicate alias found: " << alias << " in " << exprSeat << " section"; return false; } @@ -1361,7 +1361,7 @@ bool ISource::AddExpressions(TContext& ctx, const TVector<TNodePtr>& expressions ctx.Error(expr->GetPos()) << "Duplicate column name found: " << columnName << " in " << exprSeat << " section"; return false; } - if (!alias && aliases.contains(columnName)) { + if (!alias && aliases.contains(columnName)) { ctx.Error(expr->GetPos()) << "Collision between alias and column name: " << columnName << " in " << exprSeat << " section"; return false; } @@ -1375,17 +1375,17 @@ bool ISource::AddExpressions(TContext& ctx, const TVector<TNodePtr>& expressions } } } - - if (exprSeat == EExprSeat::GroupBy) { - if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { - if (SessionWindow) { - ctx.Error(expr->GetPos()) << "Duplicate session window specification:"; + + if (exprSeat == EExprSeat::GroupBy) { + if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { + if (SessionWindow) { + ctx.Error(expr->GetPos()) << "Duplicate session window specification:"; ctx.Error(SessionWindow->GetPos()) << "Previous session window is declared here"; - return false; - } - SessionWindow = expr; - } - } + return false; + } + SessionWindow = expr; + } + } Expressions(exprSeat).emplace_back(expr); } return true; @@ -1416,7 +1416,7 @@ TString ISource::MakeLocalName(const TString& name) { bool ISource::AddAggregation(TContext& ctx, TAggregationPtr aggr) { Y_UNUSED(ctx); - YQL_ENSURE(aggr); + YQL_ENSURE(aggr); Aggregations.push_back(aggr); return true; } @@ -1451,25 +1451,25 @@ THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const { return HoppingWindowSpec; } -TNodePtr ISource::GetSessionWindowSpec() const { - return SessionWindow; -} - +TNodePtr ISource::GetSessionWindowSpec() const { + return SessionWindow; +} + TWindowSpecificationPtr ISource::FindWindowSpecification(TContext& ctx, const TString& windowName) const { auto winIter = WinSpecs.find(windowName); if (winIter == WinSpecs.end()) { - ctx.Error(Pos) << "Unable to find window specification for window '" << windowName << "'"; + ctx.Error(Pos) << "Unable to find window specification for window '" << windowName << "'"; return {}; } - YQL_ENSURE(winIter->second); - return winIter->second; + YQL_ENSURE(winIter->second); + return winIter->second; } inline TVector<TNodePtr>& ISource::Expressions(EExprSeat exprSeat) { return NamedExprs[static_cast<size_t>(exprSeat)]; } -const TVector<TNodePtr>& ISource::Expressions(EExprSeat exprSeat) const { +const TVector<TNodePtr>& ISource::Expressions(EExprSeat exprSeat) const { return NamedExprs[static_cast<size_t>(exprSeat)]; } @@ -1486,11 +1486,11 @@ inline TNodePtr ISource::AliasOrColumn(const TNodePtr& node, bool withSource) { } } } - return BuildQuotedAtom(node->GetPos(), result); + return BuildQuotedAtom(node->GetPos(), result); } bool ISource::AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func) { - YQL_ENSURE(func->IsOverWindow()); + YQL_ENSURE(func->IsOverWindow()); if (func->IsDistinct()) { ctx.Error(func->GetPos()) << "Aggregation with distinct is not allowed over window: " << windowName; return false; @@ -1498,7 +1498,7 @@ bool ISource::AddAggregationOverWindow(TContext& ctx, const TString& windowName, if (!FindWindowSpecification(ctx, windowName)) { return false; } - AggregationOverWindow[windowName].emplace_back(std::move(func)); + AggregationOverWindow[windowName].emplace_back(std::move(func)); return true; } @@ -1506,7 +1506,7 @@ bool ISource::AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodeP if (!FindWindowSpecification(ctx, windowName)) { return false; } - FuncOverWindow[windowName].emplace_back(std::move(func)); + FuncOverWindow[windowName].emplace_back(std::move(func)); return true; } @@ -1522,10 +1522,10 @@ bool ISource::IsFlattenByColumns() const { return !Expressions(EExprSeat::FlattenBy).empty(); } -bool ISource::IsFlattenByExprs() const { - return !Expressions(EExprSeat::FlattenByExpr).empty(); -} - +bool ISource::IsFlattenByExprs() const { + return !Expressions(EExprSeat::FlattenByExpr).empty(); +} + bool ISource::IsAlias(EExprSeat exprSeat, const TString& column) const { for (const auto& exprNode: Expressions(exprSeat)) { const auto& labelName = exprNode->GetLabel(); @@ -1537,8 +1537,8 @@ bool ISource::IsAlias(EExprSeat exprSeat, const TString& column) const { } bool ISource::IsExprAlias(const TString& column) const { - std::array<EExprSeat, 5> exprSeats = {{EExprSeat::FlattenBy, EExprSeat::FlattenByExpr, EExprSeat::GroupBy, - EExprSeat::WindowPartitionBy, EExprSeat::DistinctAggr}}; + std::array<EExprSeat, 5> exprSeats = {{EExprSeat::FlattenBy, EExprSeat::FlattenByExpr, EExprSeat::GroupBy, + EExprSeat::WindowPartitionBy, EExprSeat::DistinctAggr}}; for (auto seat: exprSeats) { if (IsAlias(seat, column)) { return true; @@ -1573,8 +1573,8 @@ const TString* ISource::GetWindowName() const { } bool ISource::IsCalcOverWindow() const { - return !AggregationOverWindow.empty() || !FuncOverWindow.empty() || - AnyOf(WinSpecs, [](const auto& item) { return item.second->Session; }); + return !AggregationOverWindow.empty() || !FuncOverWindow.empty() || + AnyOf(WinSpecs, [](const auto& item) { return item.second->Session; }); } bool ISource::IsOverWindowSource() const { @@ -1602,18 +1602,18 @@ bool ISource::SetSamplingOptions(TContext& ctx, Y_UNUSED(mode); Y_UNUSED(samplingRate); Y_UNUSED(samplingSeed); - ctx.Error() << "Sampling is only supported for table sources"; + ctx.Error() << "Sampling is only supported for table sources"; return false; } -bool ISource::SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints) { +bool ISource::SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints) { Y_UNUSED(pos); - Y_UNUSED(contextHints); - if (hints) { - ctx.Error() << "Explicit hints are only supported for table sources"; - return false; - } - return true; + Y_UNUSED(contextHints); + if (hints) { + ctx.Error() << "Explicit hints are only supported for table sources"; + return false; + } + return true; } bool ISource::CalculateGroupingHint(TContext& ctx, const TVector<TString>& columns, ui64& hint) const { @@ -1623,11 +1623,11 @@ bool ISource::CalculateGroupingHint(TContext& ctx, const TVector<TString>& colum return false; } -TNodePtr ISource::BuildFilter(TContext& ctx, const TString& label) { - return Filters.empty() ? nullptr : Y(ctx.UseUnordered(*this) ? "OrderedFilter" : "Filter", label, BuildFilterLambda()); +TNodePtr ISource::BuildFilter(TContext& ctx, const TString& label) { + return Filters.empty() ? nullptr : Y(ctx.UseUnordered(*this) ? "OrderedFilter" : "Filter", label, BuildFilterLambda()); } -TNodePtr ISource::BuildFilterLambda() { +TNodePtr ISource::BuildFilterLambda() { if (Filters.empty()) { return BuildLambda(Pos, Y("row"), Y("Bool", Q("true"))); } @@ -1659,56 +1659,56 @@ TNodePtr ISource::BuildFlattenColumns(const TString& label) { return Y(Y("let", "res", Y("Just", Y("FlattenStructs", label)))); } -namespace { - -TNodePtr BuildLambdaBodyForExprAliases(TPosition pos, const TVector<TNodePtr>& exprs) { - auto structObj = BuildAtom(pos, "row", TNodeFlags::Default); - for (const auto& exprNode: exprs) { - const auto name = exprNode->GetLabel(); - YQL_ENSURE(name); - structObj = structObj->Y("ForceRemoveMember", structObj, structObj->Q(name)); - if (dynamic_cast<const TSessionWindow*>(exprNode.Get())) { - continue; - } - structObj = structObj->Y("AddMember", structObj, structObj->Q(name), exprNode); - } - return structObj->Y("AsList", structObj); -} - -} - -TNodePtr ISource::BuildPreaggregatedMap(TContext& ctx) { - Y_UNUSED(ctx); - const auto& groupByExprs = Expressions(EExprSeat::GroupBy); - const auto& distinctAggrExprs = Expressions(EExprSeat::DistinctAggr); - YQL_ENSURE(groupByExprs || distinctAggrExprs); - - TNodePtr res; - if (groupByExprs) { - auto body = BuildLambdaBodyForExprAliases(Pos, groupByExprs); - res = Y("FlatMap", "core", BuildLambda(Pos, Y("row"), body)); - } - - if (distinctAggrExprs) { - auto body = BuildLambdaBodyForExprAliases(Pos, distinctAggrExprs); - auto lambda = BuildLambda(Pos, Y("row"), body); - res = res ? Y("FlatMap", res, lambda) : Y("FlatMap", "core", lambda); - } - return res; -} - -TNodePtr ISource::BuildPreFlattenMap(TContext& ctx) { - Y_UNUSED(ctx); - YQL_ENSURE(IsFlattenByExprs()); - return BuildLambdaBodyForExprAliases(Pos, Expressions(EExprSeat::FlattenByExpr)); -} - -TNodePtr ISource::BuildPrewindowMap(TContext& ctx) { +namespace { + +TNodePtr BuildLambdaBodyForExprAliases(TPosition pos, const TVector<TNodePtr>& exprs) { + auto structObj = BuildAtom(pos, "row", TNodeFlags::Default); + for (const auto& exprNode: exprs) { + const auto name = exprNode->GetLabel(); + YQL_ENSURE(name); + structObj = structObj->Y("ForceRemoveMember", structObj, structObj->Q(name)); + if (dynamic_cast<const TSessionWindow*>(exprNode.Get())) { + continue; + } + structObj = structObj->Y("AddMember", structObj, structObj->Q(name), exprNode); + } + return structObj->Y("AsList", structObj); +} + +} + +TNodePtr ISource::BuildPreaggregatedMap(TContext& ctx) { + Y_UNUSED(ctx); + const auto& groupByExprs = Expressions(EExprSeat::GroupBy); + const auto& distinctAggrExprs = Expressions(EExprSeat::DistinctAggr); + YQL_ENSURE(groupByExprs || distinctAggrExprs); + + TNodePtr res; + if (groupByExprs) { + auto body = BuildLambdaBodyForExprAliases(Pos, groupByExprs); + res = Y("FlatMap", "core", BuildLambda(Pos, Y("row"), body)); + } + + if (distinctAggrExprs) { + auto body = BuildLambdaBodyForExprAliases(Pos, distinctAggrExprs); + auto lambda = BuildLambda(Pos, Y("row"), body); + res = res ? Y("FlatMap", res, lambda) : Y("FlatMap", "core", lambda); + } + return res; +} + +TNodePtr ISource::BuildPreFlattenMap(TContext& ctx) { + Y_UNUSED(ctx); + YQL_ENSURE(IsFlattenByExprs()); + return BuildLambdaBodyForExprAliases(Pos, Expressions(EExprSeat::FlattenByExpr)); +} + +TNodePtr ISource::BuildPrewindowMap(TContext& ctx) { auto feed = BuildAtom(Pos, "row", TNodeFlags::Default); for (const auto& exprNode: Expressions(EExprSeat::WindowPartitionBy)) { const auto name = exprNode->GetLabel(); - if (name && !dynamic_cast<const TSessionWindow*>(exprNode.Get())) { - feed = Y("AddMember", feed, Q(name), exprNode); + if (name && !dynamic_cast<const TSessionWindow*>(exprNode.Get())) { + feed = Y("AddMember", feed, Q(name), exprNode); } } return Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Y("AsList", feed))); @@ -1718,22 +1718,22 @@ bool ISource::BuildSamplingLambda(TNodePtr& node) { if (!SamplingRate) { return true; } - auto res = Y("Coalesce", Y("SafeCast", SamplingRate, Y("DataType", Q("Double"))), Y("Double", Q("0"))); + auto res = Y("Coalesce", Y("SafeCast", SamplingRate, Y("DataType", Q("Double"))), Y("Double", Q("0"))); res = Y("/", res, Y("Double", Q("100"))); res = Y(Y("let", "res", Y("OptionalIf", Y("<", Y("Random", Y("DependsOn", "row")), res), "row"))); node = BuildLambda(GetPos(), Y("row"), res, "res"); return !!node; } -bool ISource::SetSamplingRate(TContext& ctx, TNodePtr samplingRate) { - if (samplingRate) { - if (!samplingRate->Init(ctx, this)) { - return false; - } - SamplingRate = Y("Ensure", samplingRate, Y(">=", samplingRate, Y("Double", Q("0"))), Y("String", Q("\"Expected sampling rate to be nonnegative\""))); - SamplingRate = Y("Ensure", SamplingRate, Y("<=", SamplingRate, Y("Double", Q("100"))), Y("String", Q("\"Sampling rate is over 100%\""))); +bool ISource::SetSamplingRate(TContext& ctx, TNodePtr samplingRate) { + if (samplingRate) { + if (!samplingRate->Init(ctx, this)) { + return false; + } + SamplingRate = Y("Ensure", samplingRate, Y(">=", samplingRate, Y("Double", Q("0"))), Y("String", Q("\"Expected sampling rate to be nonnegative\""))); + SamplingRate = Y("Ensure", SamplingRate, Y("<=", SamplingRate, Y("Double", Q("100"))), Y("String", Q("\"Sampling rate is over 100%\""))); } - return true; + return true; } TNodePtr ISource::BuildAggregation(const TString& label) { @@ -1742,9 +1742,9 @@ TNodePtr ISource::BuildAggregation(const TString& label) { } auto keysTuple = Y(); - YQL_ENSURE(GroupKeys.size() == OrderedGroupKeys.size()); - for (const auto& key: OrderedGroupKeys) { - YQL_ENSURE(GroupKeys.contains(key)); + YQL_ENSURE(GroupKeys.size() == OrderedGroupKeys.size()); + for (const auto& key: OrderedGroupKeys) { + YQL_ENSURE(GroupKeys.contains(key)); keysTuple = L(keysTuple, BuildQuotedAtom(Pos, key)); } @@ -1786,15 +1786,15 @@ TNodePtr ISource::BuildAggregation(const TString& label) { options = L(options, Q(Y(Q("hopping"), hoppingTraits))); } - if (SessionWindow) { - YQL_ENSURE(SessionWindow->GetLabel()); - auto sessionWindow = dynamic_cast<TSessionWindow*>(SessionWindow.Get()); - YQL_ENSURE(sessionWindow); - options = L(options, Q(Y(Q("session"), - Q(Y(BuildQuotedAtom(Pos, SessionWindow->GetLabel()), sessionWindow->BuildTraits(label)))))); - } - - return Y("AssumeColumnOrderPartial", Y("Aggregate", label, Q(keysTuple), Q(aggrArgs), Q(options)), Q(keysTuple)); + if (SessionWindow) { + YQL_ENSURE(SessionWindow->GetLabel()); + auto sessionWindow = dynamic_cast<TSessionWindow*>(SessionWindow.Get()); + YQL_ENSURE(sessionWindow); + options = L(options, Q(Y(Q("session"), + Q(Y(BuildQuotedAtom(Pos, SessionWindow->GetLabel()), sessionWindow->BuildTraits(label)))))); + } + + return Y("AssumeColumnOrderPartial", Y("Aggregate", label, Q(keysTuple), Q(aggrArgs), Q(options)), Q(keysTuple)); } TMaybe<TString> ISource::FindColumnMistype(const TString& name) const { @@ -1806,170 +1806,170 @@ void ISource::AddDependentSource(ISource* usedSource) { UsedSources.push_back(usedSource); } -class TYqlFrameBound final: public TCallNode { -public: - TYqlFrameBound(TPosition pos, TNodePtr bound) - : TCallNode(pos, "EvaluateExpr", 1, 1, { bound }) - , FakeSource(BuildFakeSource(pos)) - { - } - - bool DoInit(TContext& ctx, ISource* src) override { - if (!ValidateArguments(ctx)) { - return false; - } - - if (!Args[0]->Init(ctx, FakeSource.Get())) { - return false; - } - - return TCallNode::DoInit(ctx, src); - } - - TNodePtr DoClone() const final { - return new TYqlFrameBound(Pos, Args[0]->Clone()); - } -private: - TSourcePtr FakeSource; -}; - -TNodePtr BuildFrameNode(const TFrameBound& frame) { - YQL_ENSURE(frame.Settings != FrameUndefined); - - TNodePtr node = frame.Bound; - TPosition pos = frame.Pos; - if (frame.Settings == FrameCurrentRow) { - node = new TCallNodeImpl(pos, "Int32", { BuildQuotedAtom(pos, "0", TNodeFlags::Default) }); - } else if (!node) { - node = BuildLiteralVoid(pos); - } else if (node->IsLiteral()) { - YQL_ENSURE(node->GetLiteralType() == "Int32"); - auto value = node->GetLiteralValue(); - YQL_ENSURE(!value.StartsWith('-')); - if (frame.Settings == FramePreceding) { - value = "-" + value; - } - node = new TCallNodeImpl(pos, "Int32", { BuildQuotedAtom(pos, value, TNodeFlags::Default) }); - } else { - if (frame.Settings == FramePreceding) { - node = new TCallNodeImpl(pos, "Minus", { node->Clone() }); - } - node = new TYqlFrameBound(pos, node); - } - return node; -} - -TNodePtr ISource::BuildWindowFrame(const TFrameSpecification& spec, bool isCompact) { - YQL_ENSURE(spec.FrameType == FrameByRows); - YQL_ENSURE(spec.FrameExclusion == FrameExclNone); - YQL_ENSURE(spec.FrameBegin); - YQL_ENSURE(spec.FrameEnd); - - auto frameBeginNode = BuildFrameNode(*spec.FrameBegin); - auto frameEndNode = BuildFrameNode(*spec.FrameEnd); - - auto begin = Q(Y(Q("begin"), frameBeginNode)); - auto end = Q(Y(Q("end"), frameEndNode)); - - return isCompact ? Q(Y(begin, end, Q(Y(Q("compact"))))) : Q(Y(begin, end)); -} - -class TSessionWindowTraits final: public TCallNode { -public: - TSessionWindowTraits(TPosition pos, const TVector<TNodePtr>& args) - : TCallNode(pos, "SessionWindowTraits", args) - , FakeSource(BuildFakeSource(pos)) - { - YQL_ENSURE(args.size() == 4); - } - - bool DoInit(TContext& ctx, ISource* src) override { - if (!ValidateArguments(ctx)) { - return false; - } - - if (!Args.back()->Init(ctx, FakeSource.Get())) { - return false; - } - - return TCallNode::DoInit(ctx, src); - } - - TNodePtr DoClone() const final { - return new TSessionWindowTraits(Pos, CloneContainer(Args)); - } -private: - TSourcePtr FakeSource; -}; - -TNodePtr ISource::BuildCalcOverWindow(TContext& ctx, const TString& label) { - YQL_ENSURE(IsCalcOverWindow()); - - TSet<TString> usedWindows; - for (auto& it : AggregationOverWindow) { - usedWindows.insert(it.first); - } - for (auto& it : FuncOverWindow) { - usedWindows.insert(it.first); - } - for (auto& it : WinSpecs) { - if (it.second->Session) { - usedWindows.insert(it.first); - } - } - - YQL_ENSURE(!usedWindows.empty()); - - const bool onePartition = usedWindows.size() == 1; +class TYqlFrameBound final: public TCallNode { +public: + TYqlFrameBound(TPosition pos, TNodePtr bound) + : TCallNode(pos, "EvaluateExpr", 1, 1, { bound }) + , FakeSource(BuildFakeSource(pos)) + { + } + + bool DoInit(TContext& ctx, ISource* src) override { + if (!ValidateArguments(ctx)) { + return false; + } + + if (!Args[0]->Init(ctx, FakeSource.Get())) { + return false; + } + + return TCallNode::DoInit(ctx, src); + } + + TNodePtr DoClone() const final { + return new TYqlFrameBound(Pos, Args[0]->Clone()); + } +private: + TSourcePtr FakeSource; +}; + +TNodePtr BuildFrameNode(const TFrameBound& frame) { + YQL_ENSURE(frame.Settings != FrameUndefined); + + TNodePtr node = frame.Bound; + TPosition pos = frame.Pos; + if (frame.Settings == FrameCurrentRow) { + node = new TCallNodeImpl(pos, "Int32", { BuildQuotedAtom(pos, "0", TNodeFlags::Default) }); + } else if (!node) { + node = BuildLiteralVoid(pos); + } else if (node->IsLiteral()) { + YQL_ENSURE(node->GetLiteralType() == "Int32"); + auto value = node->GetLiteralValue(); + YQL_ENSURE(!value.StartsWith('-')); + if (frame.Settings == FramePreceding) { + value = "-" + value; + } + node = new TCallNodeImpl(pos, "Int32", { BuildQuotedAtom(pos, value, TNodeFlags::Default) }); + } else { + if (frame.Settings == FramePreceding) { + node = new TCallNodeImpl(pos, "Minus", { node->Clone() }); + } + node = new TYqlFrameBound(pos, node); + } + return node; +} + +TNodePtr ISource::BuildWindowFrame(const TFrameSpecification& spec, bool isCompact) { + YQL_ENSURE(spec.FrameType == FrameByRows); + YQL_ENSURE(spec.FrameExclusion == FrameExclNone); + YQL_ENSURE(spec.FrameBegin); + YQL_ENSURE(spec.FrameEnd); + + auto frameBeginNode = BuildFrameNode(*spec.FrameBegin); + auto frameEndNode = BuildFrameNode(*spec.FrameEnd); + + auto begin = Q(Y(Q("begin"), frameBeginNode)); + auto end = Q(Y(Q("end"), frameEndNode)); + + return isCompact ? Q(Y(begin, end, Q(Y(Q("compact"))))) : Q(Y(begin, end)); +} + +class TSessionWindowTraits final: public TCallNode { +public: + TSessionWindowTraits(TPosition pos, const TVector<TNodePtr>& args) + : TCallNode(pos, "SessionWindowTraits", args) + , FakeSource(BuildFakeSource(pos)) + { + YQL_ENSURE(args.size() == 4); + } + + bool DoInit(TContext& ctx, ISource* src) override { + if (!ValidateArguments(ctx)) { + return false; + } + + if (!Args.back()->Init(ctx, FakeSource.Get())) { + return false; + } + + return TCallNode::DoInit(ctx, src); + } + + TNodePtr DoClone() const final { + return new TSessionWindowTraits(Pos, CloneContainer(Args)); + } +private: + TSourcePtr FakeSource; +}; + +TNodePtr ISource::BuildCalcOverWindow(TContext& ctx, const TString& label) { + YQL_ENSURE(IsCalcOverWindow()); + + TSet<TString> usedWindows; + for (auto& it : AggregationOverWindow) { + usedWindows.insert(it.first); + } + for (auto& it : FuncOverWindow) { + usedWindows.insert(it.first); + } + for (auto& it : WinSpecs) { + if (it.second->Session) { + usedWindows.insert(it.first); + } + } + + YQL_ENSURE(!usedWindows.empty()); + + const bool onePartition = usedWindows.size() == 1; const auto useLabel = onePartition ? label : "partitioning"; const auto listType = Y("TypeOf", useLabel); auto framesProcess = Y(); auto resultNode = onePartition ? Y() : Y(Y("let", "partitioning", label)); - - for (const auto& name : usedWindows) { - auto spec = FindWindowSpecification(ctx, name); - YQL_ENSURE(spec); - - auto aggsIter = AggregationOverWindow.find(name); - auto funcsIter = FuncOverWindow.find(name); - - const auto& aggs = (aggsIter == AggregationOverWindow.end()) ? TVector<TAggregationPtr>() : aggsIter->second; - const auto& funcs = (funcsIter == FuncOverWindow.end()) ? TVector<TNodePtr>() : funcsIter->second; - - auto frames = Y(); - auto callOnFrame = Y("WinOnRows", BuildWindowFrame(*spec->Frame, spec->IsCompact)); - for (auto& agg : aggs) { - auto winTraits = agg->WindowTraits(listType); - callOnFrame = L(callOnFrame, winTraits); - } - for (auto& func : funcs) { - auto winSpec = func->WindowSpecFunc(listType); - callOnFrame = L(callOnFrame, winSpec); - } - frames = L(frames, callOnFrame); - + + for (const auto& name : usedWindows) { + auto spec = FindWindowSpecification(ctx, name); + YQL_ENSURE(spec); + + auto aggsIter = AggregationOverWindow.find(name); + auto funcsIter = FuncOverWindow.find(name); + + const auto& aggs = (aggsIter == AggregationOverWindow.end()) ? TVector<TAggregationPtr>() : aggsIter->second; + const auto& funcs = (funcsIter == FuncOverWindow.end()) ? TVector<TNodePtr>() : funcsIter->second; + + auto frames = Y(); + auto callOnFrame = Y("WinOnRows", BuildWindowFrame(*spec->Frame, spec->IsCompact)); + for (auto& agg : aggs) { + auto winTraits = agg->WindowTraits(listType); + callOnFrame = L(callOnFrame, winTraits); + } + for (auto& func : funcs) { + auto winSpec = func->WindowSpecFunc(listType); + callOnFrame = L(callOnFrame, winSpec); + } + frames = L(frames, callOnFrame); + auto keysTuple = Y(); for (const auto& key: spec->Partitions) { - if (!dynamic_cast<TSessionWindow*>(key.Get())) { - keysTuple = L(keysTuple, AliasOrColumn(key, GetJoin())); - } - } - - auto sortSpec = spec->OrderBy.empty() ? Y("Void") : BuildSortSpec(spec->OrderBy, useLabel, true, false); - if (spec->Session) { - TString label = spec->Session->GetLabel(); - YQL_ENSURE(label); - auto sessionWindow = dynamic_cast<TSessionWindow*>(spec->Session.Get()); - YQL_ENSURE(sessionWindow); - auto labelNode = BuildQuotedAtom(sessionWindow->GetPos(), label); - - auto sessionTraits = sessionWindow->BuildTraits(useLabel); - framesProcess = Y("CalcOverSessionWindow", useLabel, Q(keysTuple), sortSpec, Q(frames), sessionTraits, Q(Y(labelNode))); - } else { - YQL_ENSURE(aggs || funcs); - framesProcess = Y("CalcOverWindow", useLabel, Q(keysTuple), sortSpec, Q(frames)); - } - + if (!dynamic_cast<TSessionWindow*>(key.Get())) { + keysTuple = L(keysTuple, AliasOrColumn(key, GetJoin())); + } + } + + auto sortSpec = spec->OrderBy.empty() ? Y("Void") : BuildSortSpec(spec->OrderBy, useLabel, true, false); + if (spec->Session) { + TString label = spec->Session->GetLabel(); + YQL_ENSURE(label); + auto sessionWindow = dynamic_cast<TSessionWindow*>(spec->Session.Get()); + YQL_ENSURE(sessionWindow); + auto labelNode = BuildQuotedAtom(sessionWindow->GetPos(), label); + + auto sessionTraits = sessionWindow->BuildTraits(useLabel); + framesProcess = Y("CalcOverSessionWindow", useLabel, Q(keysTuple), sortSpec, Q(frames), sessionTraits, Q(Y(labelNode))); + } else { + YQL_ENSURE(aggs || funcs); + framesProcess = Y("CalcOverWindow", useLabel, Q(keysTuple), sortSpec, Q(frames)); + } + if (!onePartition) { resultNode = L(resultNode, Y("let", "partitioning", framesProcess)); } @@ -2009,7 +2009,7 @@ bool ISource::IsTableSource() const { return false; } -bool ISource::ShouldUseSourceAsColumn(const TString& source) const { +bool ISource::ShouldUseSourceAsColumn(const TString& source) const { Y_UNUSED(source); return false; } @@ -2075,10 +2075,10 @@ void ISource::FillSortParts(const TVector<TSortSpecificationPtr>& orderBy, TNode sortKeySelector = BuildLambda(Pos, Y("row"), expr); } -TNodePtr ISource::BuildSortSpec(const TVector<TSortSpecificationPtr>& orderBy, const TString& label, bool traits, bool assume) { +TNodePtr ISource::BuildSortSpec(const TVector<TSortSpecificationPtr>& orderBy, const TString& label, bool traits, bool assume) { YQL_ENSURE(!orderBy.empty()); TNodePtr dirsNode; - TNodePtr keySelectorNode; + TNodePtr keySelectorNode; FillSortParts(orderBy, dirsNode, keySelectorNode); if (traits) { return Y("SortTraits", Y("TypeOf", label), dirsNode, keySelectorNode); @@ -2102,126 +2102,126 @@ IJoin* IJoin::GetJoin() { return this; } -namespace { -bool UnescapeQuoted(const TString& str, TPosition& pos, char quoteChar, TString& result, TString& error) { - result = error = {}; - - size_t readBytes = 0; - TStringBuf atom(str); - TStringOutput sout(result); - atom.Skip(1); - result.reserve(str.size()); - - auto unescapeResult = UnescapeArbitraryAtom(atom, quoteChar, &sout, &readBytes); - if (unescapeResult != EUnescapeResult::OK) { - TTextWalker walker(pos); - walker.Advance(atom.Trunc(readBytes)); - error = UnescapeResultToString(unescapeResult); +namespace { +bool UnescapeQuoted(const TString& str, TPosition& pos, char quoteChar, TString& result, TString& error) { + result = error = {}; + + size_t readBytes = 0; + TStringBuf atom(str); + TStringOutput sout(result); + atom.Skip(1); + result.reserve(str.size()); + + auto unescapeResult = UnescapeArbitraryAtom(atom, quoteChar, &sout, &readBytes); + if (unescapeResult != EUnescapeResult::OK) { + TTextWalker walker(pos); + walker.Advance(atom.Trunc(readBytes)); + error = UnescapeResultToString(unescapeResult); return false; } - return true; -} - -TString UnescapeAnsiQuoted(const TString& str) { - YQL_ENSURE(str.length() >= 2); - YQL_ENSURE(str[0] == str[str.length() - 1]); - YQL_ENSURE(str[0] == '\'' || str[0] == '"'); - - TString quote(1, str[0]); - TString replace(2, str[0]); - - TString result = str.substr(1, str.length() - 2); - SubstGlobal(result, replace, quote); - return result; -} - -enum class EStringContentMode : int { - Default = 0, - AnsiIdent, - TypedStringLiteral, -}; - -TMaybe<TStringContent> -StringContentInternal(TContext& ctx, TPosition pos, const TString& input, EStringContentMode mode) { - TStringContent result; - if (mode == EStringContentMode::AnsiIdent) { - if (!(input.size() >= 2 && input.StartsWith('"') && input.EndsWith('"'))) { - ctx.Error(pos) << "Expected double quoted identifier, got string literal"; - return {}; - } - - result.Flags = NYql::TNodeFlags::ArbitraryContent; - result.Content = UnescapeAnsiQuoted(input); - return result; - } - - TString str = input; - if (mode == EStringContentMode::TypedStringLiteral) { - if (str.EndsWith("u") || str.EndsWith("U")) { - str = str.substr(0, str.Size() - 1); - result.Type = NKikimr::NUdf::EDataSlot::Utf8; - } else if (str.EndsWith("y")) { - str = str.substr(0, str.Size() - 1); - result.Type = NKikimr::NUdf::EDataSlot::Yson; - } else if (str.EndsWith("j")) { - str = str.substr(0, str.Size() - 1); - result.Type = NKikimr::NUdf::EDataSlot::Json; - } - } - - if (mode == EStringContentMode::Default && result.Type != NKikimr::NUdf::EDataSlot::String) { - ctx.Error(pos) << "Type suffix is not allowed here"; - return {}; - } - + return true; +} + +TString UnescapeAnsiQuoted(const TString& str) { + YQL_ENSURE(str.length() >= 2); + YQL_ENSURE(str[0] == str[str.length() - 1]); + YQL_ENSURE(str[0] == '\'' || str[0] == '"'); + + TString quote(1, str[0]); + TString replace(2, str[0]); + + TString result = str.substr(1, str.length() - 2); + SubstGlobal(result, replace, quote); + return result; +} + +enum class EStringContentMode : int { + Default = 0, + AnsiIdent, + TypedStringLiteral, +}; + +TMaybe<TStringContent> +StringContentInternal(TContext& ctx, TPosition pos, const TString& input, EStringContentMode mode) { + TStringContent result; + if (mode == EStringContentMode::AnsiIdent) { + if (!(input.size() >= 2 && input.StartsWith('"') && input.EndsWith('"'))) { + ctx.Error(pos) << "Expected double quoted identifier, got string literal"; + return {}; + } + + result.Flags = NYql::TNodeFlags::ArbitraryContent; + result.Content = UnescapeAnsiQuoted(input); + return result; + } + + TString str = input; + if (mode == EStringContentMode::TypedStringLiteral) { + if (str.EndsWith("u") || str.EndsWith("U")) { + str = str.substr(0, str.Size() - 1); + result.Type = NKikimr::NUdf::EDataSlot::Utf8; + } else if (str.EndsWith("y")) { + str = str.substr(0, str.Size() - 1); + result.Type = NKikimr::NUdf::EDataSlot::Yson; + } else if (str.EndsWith("j")) { + str = str.substr(0, str.Size() - 1); + result.Type = NKikimr::NUdf::EDataSlot::Json; + } + } + + if (mode == EStringContentMode::Default && result.Type != NKikimr::NUdf::EDataSlot::String) { + ctx.Error(pos) << "Type suffix is not allowed here"; + return {}; + } + bool doubleQuoted = (str.StartsWith('"') && str.EndsWith('"')); bool singleQuoted = !doubleQuoted && (str.StartsWith('\'') && str.EndsWith('\'')); if (str.size() >= 2 && (doubleQuoted || singleQuoted)) { - result.Flags = NYql::TNodeFlags::ArbitraryContent; - if (ctx.Settings.AnsiLexer) { - YQL_ENSURE(singleQuoted); - result.Content = UnescapeAnsiQuoted(str); - } else { - TString error; - if (!UnescapeQuoted(str, pos, str[0], result.Content, error)) { - ctx.Error(pos) << "Failed to parse string literal: " << error; - return {}; - } - } + result.Flags = NYql::TNodeFlags::ArbitraryContent; + if (ctx.Settings.AnsiLexer) { + YQL_ENSURE(singleQuoted); + result.Content = UnescapeAnsiQuoted(str); + } else { + TString error; + if (!UnescapeQuoted(str, pos, str[0], result.Content, error)) { + ctx.Error(pos) << "Failed to parse string literal: " << error; + return {}; + } + } } else if (str.size() >= 4 && str.StartsWith("@@") && str.EndsWith("@@")) { - result.Flags = TNodeFlags::MultilineContent; + result.Flags = TNodeFlags::MultilineContent; TString s = str.substr(2, str.length() - 4); SubstGlobal(s, "@@@@", "@@"); - result.Content.swap(s); + result.Content.swap(s); } else { - ctx.Error(pos) << "Invalid string literal: " << EscapeC(str); - return {}; - } - - if (!NKikimr::NMiniKQL::IsValidStringValue(result.Type, result.Content)) { - ctx.Error() << "Invalid value " << result.Content.Quote() << " for type " << result.Type; - return {}; - } - - return result; -} -} // namespace - -TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input) { - if (ctx.AnsiQuotedIdentifiers && input.StartsWith('"')) { - ctx.Error() << "Expected string literal, got quoted identifier"; - return {}; - } - - return StringContentInternal(ctx, pos, input, EStringContentMode::Default); -} - -TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input) { - return StringContentInternal(ctx, pos, input, - (ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default); -} - + ctx.Error(pos) << "Invalid string literal: " << EscapeC(str); + return {}; + } + + if (!NKikimr::NMiniKQL::IsValidStringValue(result.Type, result.Content)) { + ctx.Error() << "Invalid value " << result.Content.Quote() << " for type " << result.Type; + return {}; + } + + return result; +} +} // namespace + +TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input) { + if (ctx.AnsiQuotedIdentifiers && input.StartsWith('"')) { + ctx.Error() << "Expected string literal, got quoted identifier"; + return {}; + } + + return StringContentInternal(ctx, pos, input, EStringContentMode::Default); +} + +TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input) { + return StringContentInternal(ctx, pos, input, + (ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default); +} + TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr) : ColumnName(columnName) , Expr(expr) @@ -2246,17 +2246,17 @@ TString IdContent(TContext& ctx, const TString& s) { unescapedStr.reserve(s.size()); size_t readBytes = 0; - TPosition pos = ctx.Pos(); - pos.Column += skipSymbols - 1; - - auto unescapeResult = UnescapeArbitraryAtom(atom, endSym, &sout, &readBytes); - if (unescapeResult != EUnescapeResult::OK) { - TTextWalker walker(pos); - walker.Advance(atom.Trunc(readBytes)); - ctx.Error(pos) << "Cannot parse broken identifier: " << UnescapeResultToString(unescapeResult); + TPosition pos = ctx.Pos(); + pos.Column += skipSymbols - 1; + + auto unescapeResult = UnescapeArbitraryAtom(atom, endSym, &sout, &readBytes); + if (unescapeResult != EUnescapeResult::OK) { + TTextWalker walker(pos); + walker.Advance(atom.Trunc(readBytes)); + ctx.Error(pos) << "Cannot parse broken identifier: " << UnescapeResultToString(unescapeResult); return {}; } - + if (readBytes != atom.size()) { ctx.Error() << "The identifier not parsed completely"; return {}; @@ -2265,46 +2265,46 @@ TString IdContent(TContext& ctx, const TString& s) { return unescapedStr; } -TString IdContentFromString(TContext& ctx, const TString& str) { - if (!ctx.AnsiQuotedIdentifiers) { - ctx.Error() << "String literal can not be used here"; - return {}; - } - auto parsed = StringContentInternal(ctx, ctx.Pos(), str, EStringContentMode::AnsiIdent); - if (!parsed) { - return {}; - } - - return parsed->Content; -} - - -namespace { -class TInvalidLiteralNode final: public INode { -public: - TInvalidLiteralNode(TPosition pos) - : INode(pos) - { - } - - bool DoInit(TContext& ctx, ISource* source) override { - Y_UNUSED(ctx); - Y_UNUSED(source); - return false; - } - - TAstNode* Translate(TContext& ctx) const override { - Y_UNUSED(ctx); - return nullptr; - } - - TPtr DoClone() const override { - return {}; - } -}; - -} - +TString IdContentFromString(TContext& ctx, const TString& str) { + if (!ctx.AnsiQuotedIdentifiers) { + ctx.Error() << "String literal can not be used here"; + return {}; + } + auto parsed = StringContentInternal(ctx, ctx.Pos(), str, EStringContentMode::AnsiIdent); + if (!parsed) { + return {}; + } + + return parsed->Content; +} + + +namespace { +class TInvalidLiteralNode final: public INode { +public: + TInvalidLiteralNode(TPosition pos) + : INode(pos) + { + } + + bool DoInit(TContext& ctx, ISource* source) override { + Y_UNUSED(ctx); + Y_UNUSED(source); + return false; + } + + TAstNode* Translate(TContext& ctx) const override { + Y_UNUSED(ctx); + return nullptr; + } + + TPtr DoClone() const override { + return {}; + } +}; + +} + TLiteralNode::TLiteralNode(TPosition pos, bool isNull) : TAstListNode(pos) , Null(isNull) @@ -2323,14 +2323,14 @@ TLiteralNode::TLiteralNode(TPosition pos, const TString& type, const TString& va Add(Type, BuildQuotedAtom(Pos, Value)); } -TLiteralNode::TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags) +TLiteralNode::TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags) : TAstListNode(pos) , Null(false) , Void(false) , Type("String") - , Value(value) + , Value(value) { - Add(Type, BuildQuotedAtom(pos, Value, nodeFlags)); + Add(Type, BuildQuotedAtom(pos, Value, nodeFlags)); } TLiteralNode::TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags, const TString& type) @@ -2374,14 +2374,14 @@ TNodePtr TLiteralNode::DoClone() const { } template<typename T> -TLiteralNumberNode<T>::TLiteralNumberNode(TPosition pos, const TString& type, const TString& value, bool implicitType) +TLiteralNumberNode<T>::TLiteralNumberNode(TPosition pos, const TString& type, const TString& value, bool implicitType) : TLiteralNode(pos, type, value) - , ImplicitType(implicitType) + , ImplicitType(implicitType) {} template<typename T> TNodePtr TLiteralNumberNode<T>::DoClone() const { - return new TLiteralNumberNode<T>(Pos, Type, Value, ImplicitType); + return new TLiteralNumberNode<T>(Pos, Type, Value, ImplicitType); } template<typename T> @@ -2389,7 +2389,7 @@ bool TLiteralNumberNode<T>::DoInit(TContext& ctx, ISource* src) { Y_UNUSED(src); T val; if (!TryFromString(Value, val)) { - ctx.Error(Pos) << "Failed to parse " << Value << " as integer literal of " << Type << " type: value out of range for " << Type; + ctx.Error(Pos) << "Failed to parse " << Value << " as integer literal of " << Type << " type: value out of range for " << Type; return false; } return true; @@ -2400,38 +2400,38 @@ bool TLiteralNumberNode<T>::IsIntegerLiteral() const { return std::numeric_limits<T>::is_integer; } -template<typename T> -TNodePtr TLiteralNumberNode<T>::ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const { - YQL_ENSURE(!Value.empty()); - if (opName == "Minus" && IsIntegerLiteral() && Value[0] != '-') { - if (ImplicitType) { - ui64 val = FromString<ui64>(Value); - TString negated = "-" + Value; - if (val <= ui64(std::numeric_limits<i32>::max()) + 1) { - // negated value fits in Int32 - i32 v; - YQL_ENSURE(TryFromString(negated, v)); - return new TLiteralNumberNode<i32>(pos, "Int32", negated); - } - if (val <= ui64(std::numeric_limits<i64>::max()) + 1) { - // negated value fits in Int64 - i64 v; - YQL_ENSURE(TryFromString(negated, v)); - return new TLiteralNumberNode<i64>(pos, "Int64", negated); - } - - ctx.Error(pos) << "Failed to parse negative integer: " << negated << ", number limit overflow"; - return {}; - } - - if (std::numeric_limits<T>::is_signed) { - return new TLiteralNumberNode<T>(pos, Type, "-" + Value); - } - } - return INode::ApplyUnaryOp(ctx, pos, opName); -} - - +template<typename T> +TNodePtr TLiteralNumberNode<T>::ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const { + YQL_ENSURE(!Value.empty()); + if (opName == "Minus" && IsIntegerLiteral() && Value[0] != '-') { + if (ImplicitType) { + ui64 val = FromString<ui64>(Value); + TString negated = "-" + Value; + if (val <= ui64(std::numeric_limits<i32>::max()) + 1) { + // negated value fits in Int32 + i32 v; + YQL_ENSURE(TryFromString(negated, v)); + return new TLiteralNumberNode<i32>(pos, "Int32", negated); + } + if (val <= ui64(std::numeric_limits<i64>::max()) + 1) { + // negated value fits in Int64 + i64 v; + YQL_ENSURE(TryFromString(negated, v)); + return new TLiteralNumberNode<i64>(pos, "Int64", negated); + } + + ctx.Error(pos) << "Failed to parse negative integer: " << negated << ", number limit overflow"; + return {}; + } + + if (std::numeric_limits<T>::is_signed) { + return new TLiteralNumberNode<T>(pos, Type, "-" + Value); + } + } + return INode::ApplyUnaryOp(ctx, pos, opName); +} + + template class TLiteralNumberNode<i32>; template class TLiteralNumberNode<i64>; template class TLiteralNumberNode<ui32>; @@ -2451,43 +2451,43 @@ TNodePtr BuildLiteralVoid(TPosition pos) { return new TLiteralNode(pos, false); } -TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value) { - auto unescaped = StringContent(ctx, ctx.Pos(), value); - if (!unescaped) { +TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value) { + auto unescaped = StringContent(ctx, ctx.Pos(), value); + if (!unescaped) { return new TInvalidLiteralNode(ctx.Pos()); } - YQL_ENSURE(unescaped->Type == NKikimr::NUdf::EDataSlot::String); - return new TLiteralNode(ctx.Pos(), unescaped->Content, unescaped->Flags, "String"); -} + YQL_ENSURE(unescaped->Type == NKikimr::NUdf::EDataSlot::String); + return new TLiteralNode(ctx.Pos(), unescaped->Content, unescaped->Flags, "String"); +} -TMaybe<TExprOrIdent> BuildLiteralTypedSmartStringOrId(TContext& ctx, const TString& value) { - TExprOrIdent result; - if (ctx.AnsiQuotedIdentifiers && value.StartsWith('"')) { - auto unescaped = StringContentInternal(ctx, ctx.Pos(), value, EStringContentMode::AnsiIdent); - if (!unescaped) { - return {}; - } - result.Ident = unescaped->Content; - return result; +TMaybe<TExprOrIdent> BuildLiteralTypedSmartStringOrId(TContext& ctx, const TString& value) { + TExprOrIdent result; + if (ctx.AnsiQuotedIdentifiers && value.StartsWith('"')) { + auto unescaped = StringContentInternal(ctx, ctx.Pos(), value, EStringContentMode::AnsiIdent); + if (!unescaped) { + return {}; + } + result.Ident = unescaped->Content; + return result; + } + auto unescaped = StringContentInternal(ctx, ctx.Pos(), value, EStringContentMode::TypedStringLiteral); + if (!unescaped) { + return {}; } - auto unescaped = StringContentInternal(ctx, ctx.Pos(), value, EStringContentMode::TypedStringLiteral); - if (!unescaped) { - return {}; - } - TString type = ToString(unescaped->Type); - result.Expr = new TLiteralNode(ctx.Pos(), unescaped->Content, unescaped->Flags, type); - return result; + TString type = ToString(unescaped->Type); + result.Expr = new TLiteralNode(ctx.Pos(), unescaped->Content, unescaped->Flags, type); + return result; } - -TNodePtr BuildLiteralRawString(TPosition pos, const TString& value, bool isUtf8) { - return new TLiteralNode(pos, isUtf8 ? "Utf8" : "String", value); + +TNodePtr BuildLiteralRawString(TPosition pos, const TString& value, bool isUtf8) { + return new TLiteralNode(pos, isUtf8 ? "Utf8" : "String", value); } -TNodePtr BuildLiteralBool(TPosition pos, bool value) { - return new TLiteralNode(pos, "Bool", value ? "true" : "false"); +TNodePtr BuildLiteralBool(TPosition pos, bool value) { + return new TLiteralNode(pos, "Bool", value ? "true" : "false"); } TAsteriskNode::TAsteriskNode(TPosition pos) @@ -2534,16 +2534,16 @@ const TString* TDeferredAtom::GetLiteral() const { return Explicit.Get(); } -bool TDeferredAtom::GetLiteral(TString& value, TContext& ctx) const { - if (Explicit) { - value = *Explicit; - return true; - } - - ctx.Error(Node ? Node->GetPos() : ctx.Pos()) << "Expected literal value"; - return false; -} - +bool TDeferredAtom::GetLiteral(TString& value, TContext& ctx) const { + if (Explicit) { + value = *Explicit; + return true; + } + + ctx.Error(Node ? Node->GetPos() : ctx.Pos()) << "Expected literal value"; + return false; +} + TNodePtr TDeferredAtom::Build() const { return Node; } @@ -2594,77 +2594,77 @@ TNodePtr TTupleNode::DoClone() const { return new TTupleNode(Pos, CloneContainer(Exprs)); } -void TTupleNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { - for (auto& expr : Exprs) { - expr->CollectPreaggregateExprs(ctx, src, exprs); - } -} - -const TString* TTupleNode::GetSourceName() const { - return DeriveCommonSourceName(Exprs); -} - +void TTupleNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { + for (auto& expr : Exprs) { + expr->CollectPreaggregateExprs(ctx, src, exprs); + } +} + +const TString* TTupleNode::GetSourceName() const { + return DeriveCommonSourceName(Exprs); +} + TNodePtr BuildTuple(TPosition pos, const TVector<TNodePtr>& exprs) { return new TTupleNode(pos, exprs); } -TStructNode::TStructNode(TPosition pos, const TVector<TNodePtr>& exprs, const TVector<TNodePtr>& labels, bool ordered) +TStructNode::TStructNode(TPosition pos, const TVector<TNodePtr>& exprs, const TVector<TNodePtr>& labels, bool ordered) : TAstListNode(pos) , Exprs(exprs) - , Labels(labels) - , Ordered(ordered) -{ - YQL_ENSURE(Labels.empty() || Labels.size() == Exprs.size()); -} + , Labels(labels) + , Ordered(ordered) +{ + YQL_ENSURE(Labels.empty() || Labels.size() == Exprs.size()); +} bool TStructNode::DoInit(TContext& ctx, ISource* src) { - Nodes.push_back(BuildAtom(Pos, (Ordered || Exprs.size() < 2) ? "AsStruct" : "AsStructUnordered", TNodeFlags::Default)); - size_t i = 0; + Nodes.push_back(BuildAtom(Pos, (Ordered || Exprs.size() < 2) ? "AsStruct" : "AsStructUnordered", TNodeFlags::Default)); + size_t i = 0; for (const auto& expr : Exprs) { - TNodePtr label; - if (Labels.empty()) { - if (!expr->GetLabel()) { - ctx.Error(expr->GetPos()) << "Structure does not allow anonymous members"; - return false; - } - label = BuildQuotedAtom(expr->GetPos(), expr->GetLabel()); - } else { - label = Labels[i++]; - } - Nodes.push_back(Q(Y(label, expr))); + TNodePtr label; + if (Labels.empty()) { + if (!expr->GetLabel()) { + ctx.Error(expr->GetPos()) << "Structure does not allow anonymous members"; + return false; + } + label = BuildQuotedAtom(expr->GetPos(), expr->GetLabel()); + } else { + label = Labels[i++]; + } + Nodes.push_back(Q(Y(label, expr))); } return TAstListNode::DoInit(ctx, src); } TNodePtr TStructNode::DoClone() const { - return new TStructNode(Pos, CloneContainer(Exprs), CloneContainer(Labels), Ordered); -} - -void TStructNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { - for (auto& expr : Exprs) { - expr->CollectPreaggregateExprs(ctx, src, exprs); - } -} - -const TString* TStructNode::GetSourceName() const { - return DeriveCommonSourceName(Exprs); -} - + return new TStructNode(Pos, CloneContainer(Exprs), CloneContainer(Labels), Ordered); +} + +void TStructNode::CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) { + for (auto& expr : Exprs) { + expr->CollectPreaggregateExprs(ctx, src, exprs); + } +} + +const TString* TStructNode::GetSourceName() const { + return DeriveCommonSourceName(Exprs); +} + TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprs) { - bool ordered = false; - return new TStructNode(pos, exprs, {}, ordered); -} - -TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels) { - bool ordered = false; - return new TStructNode(pos, exprsUnlabeled, labels, ordered); -} - -TNodePtr BuildOrderedStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels) { - bool ordered = true; - return new TStructNode(pos, exprsUnlabeled, labels, ordered); -} - + bool ordered = false; + return new TStructNode(pos, exprs, {}, ordered); +} + +TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels) { + bool ordered = false; + return new TStructNode(pos, exprsUnlabeled, labels, ordered); +} + +TNodePtr BuildOrderedStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels) { + bool ordered = true; + return new TStructNode(pos, exprsUnlabeled, labels, ordered); +} + TListOfNamedNodes::TListOfNamedNodes(TPosition pos, TVector<TNodePtr>&& exprs) : INode(pos) , Exprs(std::move(exprs)) @@ -2684,12 +2684,12 @@ TNodePtr TListOfNamedNodes::DoClone() const { return {}; } -void TListOfNamedNodes::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { - for (auto& expr : Exprs) { - expr->VisitTree(func, visited); - } -} - +void TListOfNamedNodes::DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const { + for (auto& expr : Exprs) { + expr->VisitTree(func, visited); + } +} + TNodePtr BuildListOfNamedNodes(TPosition pos, TVector<TNodePtr>&& exprs) { return new TListOfNamedNodes(pos, std::move(exprs)); } @@ -2731,7 +2731,7 @@ public: , IsLookup(isLookup) , ColumnOnly(false) , IsColumnRequired(false) - , AccessOpName("AccessNode") + , AccessOpName("AccessNode") { Y_VERIFY_DEBUG(Ids.size() > 1); Y_VERIFY_DEBUG(Ids[0].Expr); @@ -2865,19 +2865,19 @@ public: protected: void DoUpdateState() const override { - YQL_ENSURE(Node); - State.Set(ENodeState::Const, Node->IsConstant()); - State.Set(ENodeState::MaybeConst, Node->MaybeConstant()); - State.Set(ENodeState::Aggregated, Node->IsAggregated()); - State.Set(ENodeState::AggregationKey, Node->HasState(ENodeState::AggregationKey)); - State.Set(ENodeState::OverWindow, Node->IsOverWindow()); - } - - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(Node); - Node->VisitTree(func, visited); - } - + YQL_ENSURE(Node); + State.Set(ENodeState::Const, Node->IsConstant()); + State.Set(ENodeState::MaybeConst, Node->MaybeConstant()); + State.Set(ENodeState::Aggregated, Node->IsAggregated()); + State.Set(ENodeState::AggregationKey, Node->HasState(ENodeState::AggregationKey)); + State.Set(ENodeState::OverWindow, Node->IsOverWindow()); + } + + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(Node); + Node->VisitTree(func, visited); + } + bool CheckColumnId(TPosition pos, TContext& ctx, const TIdPart& id, const TString& where, bool checkLookup) { if (id.Name.empty()) { ctx.Error(pos) << where << " name can not be empty"; @@ -2894,14 +2894,14 @@ protected: return AccessOpName; } - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override { - for (auto& id : Ids) { - if (id.Expr) { - id.Expr->CollectPreaggregateExprs(ctx, src, exprs); - } - } - } - + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override { + for (auto& id : Ids) { + if (id.Expr) { + id.Expr->CollectPreaggregateExprs(ctx, src, exprs); + } + } + } + private: TNodePtr Node; TVector<TIdPart> Ids; @@ -2915,138 +2915,138 @@ TNodePtr BuildAccess(TPosition pos, const TVector<INode::TIdPart>& ids, bool isL return new TAccessNode(pos, ids, isLookup); } -void WarnIfAliasFromSelectIsUsedInGroupBy(TContext& ctx, const TVector<TNodePtr>& selectTerms, const TVector<TNodePtr>& groupByTerms, - const TVector<TNodePtr>& groupByExprTerms) -{ - THashMap<TString, TNodePtr> termsByLabel; - for (auto& term : selectTerms) { - auto label = term->GetLabel(); - if (!label || term->IsOverWindow()) { - continue; - } - - auto column = term->GetColumnName(); - - // do not warn for trivial renaming such as '[X.]foo AS foo' - if (column && *column == label) { - continue; - } - - // skip terms with aggregation functions inside - bool hasAggregationFunction = false; - auto visitor = [&](const INode& current) { - hasAggregationFunction = hasAggregationFunction || current.GetAggregation(); - return !hasAggregationFunction; - }; - - term->VisitTree(visitor); - if (!hasAggregationFunction) { - termsByLabel[label] = term; - } - } - - if (termsByLabel.empty()) { - return; - } - - bool found = false; - auto visitor = [&](const INode& current) { - if (found) { - return false; - } - - if (auto columnName = current.GetColumnName()) { - // do not warn if source name is set - auto src = current.GetSourceName(); - if (src && *src) { - return true; - } - auto it = termsByLabel.find(*columnName); - if (it != termsByLabel.end()) { - found = true; - ctx.Warning(current.GetPos(), TIssuesIds::YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY) - << "GROUP BY will aggregate by column `" << *columnName << "` instead of aggregating by SELECT expression with same alias"; - ctx.Warning(it->second->GetPos(), TIssuesIds::YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY) - << "You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details"; - return false; - } - } - - return true; - }; - - TVector<TNodePtr> originalGroupBy; - { - THashSet<TString> groupByExprLabels; - for (auto& expr : groupByExprTerms) { - auto label = expr->GetLabel(); - YQL_ENSURE(label); - groupByExprLabels.insert(label); - } - - originalGroupBy = groupByTerms; - EraseIf(originalGroupBy, [&](const TNodePtr& node) { - auto column = node->GetColumnName(); - auto src = node->GetSourceName(); - - return (!src || src->empty()) && column && groupByExprLabels.contains(*column); - }); - - originalGroupBy.insert(originalGroupBy.end(), groupByExprTerms.begin(), groupByExprTerms.end()); - } - - for (auto& groupByTerm : originalGroupBy) { - groupByTerm->VisitTree(visitor); - if (found) { - return; - } - } -} - -bool ValidateAllNodesForAggregation(TContext& ctx, const TVector<TNodePtr>& nodes) { - for (auto& node: nodes) { - if (!node->HasState(ENodeState::Initialized) || node->IsConstant() || node->MaybeConstant()) { - continue; - } - // TODO: "!node->IsOverWindow()" doesn't look right here - if (!node->IsAggregated() && !node->IsOverWindow()) { - // locate column which is not a key column and not aggregated - const INode* found = nullptr; - auto visitor = [&found](const INode& current) { - if (found || current.IsAggregated() || current.IsOverWindow()) { - return false; - } - - if (dynamic_cast<const TColumnNode*>(¤t) || dynamic_cast<const TAccessNode*>(¤t)) { - found = ¤t; - return false; - } - return true; - }; - - node->VisitTree(visitor); - if (found) { - TString columnName; - if (auto col = found->GetColumnName(); col && *col) { - columnName = "`"; - if (auto src = found->GetSourceName(); src && *src) { - columnName += DotJoin(*src, *col); - } else { - columnName += *col; - } - columnName += "` "; - } - ctx.Error(found->GetPos()) << "Column " << columnName << "must either be a key column in GROUP BY or it should be used in aggregation function"; - } else { - ctx.Error(node->GetPos()) << "Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery"; - } - - return false; - } - } - return true; -} - +void WarnIfAliasFromSelectIsUsedInGroupBy(TContext& ctx, const TVector<TNodePtr>& selectTerms, const TVector<TNodePtr>& groupByTerms, + const TVector<TNodePtr>& groupByExprTerms) +{ + THashMap<TString, TNodePtr> termsByLabel; + for (auto& term : selectTerms) { + auto label = term->GetLabel(); + if (!label || term->IsOverWindow()) { + continue; + } + + auto column = term->GetColumnName(); + + // do not warn for trivial renaming such as '[X.]foo AS foo' + if (column && *column == label) { + continue; + } + + // skip terms with aggregation functions inside + bool hasAggregationFunction = false; + auto visitor = [&](const INode& current) { + hasAggregationFunction = hasAggregationFunction || current.GetAggregation(); + return !hasAggregationFunction; + }; + + term->VisitTree(visitor); + if (!hasAggregationFunction) { + termsByLabel[label] = term; + } + } + + if (termsByLabel.empty()) { + return; + } + + bool found = false; + auto visitor = [&](const INode& current) { + if (found) { + return false; + } + + if (auto columnName = current.GetColumnName()) { + // do not warn if source name is set + auto src = current.GetSourceName(); + if (src && *src) { + return true; + } + auto it = termsByLabel.find(*columnName); + if (it != termsByLabel.end()) { + found = true; + ctx.Warning(current.GetPos(), TIssuesIds::YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY) + << "GROUP BY will aggregate by column `" << *columnName << "` instead of aggregating by SELECT expression with same alias"; + ctx.Warning(it->second->GetPos(), TIssuesIds::YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY) + << "You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details"; + return false; + } + } + + return true; + }; + + TVector<TNodePtr> originalGroupBy; + { + THashSet<TString> groupByExprLabels; + for (auto& expr : groupByExprTerms) { + auto label = expr->GetLabel(); + YQL_ENSURE(label); + groupByExprLabels.insert(label); + } + + originalGroupBy = groupByTerms; + EraseIf(originalGroupBy, [&](const TNodePtr& node) { + auto column = node->GetColumnName(); + auto src = node->GetSourceName(); + + return (!src || src->empty()) && column && groupByExprLabels.contains(*column); + }); + + originalGroupBy.insert(originalGroupBy.end(), groupByExprTerms.begin(), groupByExprTerms.end()); + } + + for (auto& groupByTerm : originalGroupBy) { + groupByTerm->VisitTree(visitor); + if (found) { + return; + } + } +} + +bool ValidateAllNodesForAggregation(TContext& ctx, const TVector<TNodePtr>& nodes) { + for (auto& node: nodes) { + if (!node->HasState(ENodeState::Initialized) || node->IsConstant() || node->MaybeConstant()) { + continue; + } + // TODO: "!node->IsOverWindow()" doesn't look right here + if (!node->IsAggregated() && !node->IsOverWindow()) { + // locate column which is not a key column and not aggregated + const INode* found = nullptr; + auto visitor = [&found](const INode& current) { + if (found || current.IsAggregated() || current.IsOverWindow()) { + return false; + } + + if (dynamic_cast<const TColumnNode*>(¤t) || dynamic_cast<const TAccessNode*>(¤t)) { + found = ¤t; + return false; + } + return true; + }; + + node->VisitTree(visitor); + if (found) { + TString columnName; + if (auto col = found->GetColumnName(); col && *col) { + columnName = "`"; + if (auto src = found->GetSourceName(); src && *src) { + columnName += DotJoin(*src, *col); + } else { + columnName += *col; + } + columnName += "` "; + } + ctx.Error(found->GetPos()) << "Column " << columnName << "must either be a key column in GROUP BY or it should be used in aggregation function"; + } else { + ctx.Error(node->GetPos()) << "Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery"; + } + + return false; + } + } + return true; +} + class TBindNode: public TAstListNode { public: TBindNode(TPosition pos, const TString& module, const TString& alias) @@ -3056,7 +3056,7 @@ public: } TPtr DoClone() const final { - return {}; + return {}; } }; @@ -3077,12 +3077,12 @@ public: protected: TPtr DoClone() const final { - return {}; + return {}; + } + + void DoUpdateState() const final { + State.Set(ENodeState::Const); } - - void DoUpdateState() const final { - State.Set(ENodeState::Const); - } }; TNodePtr BuildLambda(TPosition pos, TNodePtr params, TNodePtr body, const TString& resName) { @@ -3093,26 +3093,26 @@ TNodePtr BuildDataType(TPosition pos, const TString& typeName) { return new TCallNodeImpl(pos, "DataType", {BuildQuotedAtom(pos, typeName, TNodeFlags::Default)}); } -TNodePtr BuildSimpleType(TContext& ctx, TPosition pos, const TString& typeName, bool dataOnly) { - auto found = LookupSimpleTypeBySqlAlias(typeName, ctx.FlexibleTypes); - if (!found) { - ctx.Error(pos) << "Unknown simple type '" << typeName << "'"; - return {}; - } - - auto type = ToString(*found); - if (type == "Void" || type == "Unit" || type == "Generic" || type == "EmptyList" || type == "EmptyDict") { - if (dataOnly) { - ctx.Error(pos) << "Only data types are allowed here, but got: '" << typeName << "'"; - return {}; - } - type += "Type"; - return new TCallNodeImpl(pos, type, {}); - } - - return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, type, TNodeFlags::Default) }); -} - +TNodePtr BuildSimpleType(TContext& ctx, TPosition pos, const TString& typeName, bool dataOnly) { + auto found = LookupSimpleTypeBySqlAlias(typeName, ctx.FlexibleTypes); + if (!found) { + ctx.Error(pos) << "Unknown simple type '" << typeName << "'"; + return {}; + } + + auto type = ToString(*found); + if (type == "Void" || type == "Unit" || type == "Generic" || type == "EmptyList" || type == "EmptyDict") { + if (dataOnly) { + ctx.Error(pos) << "Only data types are allowed here, but got: '" << typeName << "'"; + return {}; + } + type += "Type"; + return new TCallNodeImpl(pos, type, {}); + } + + return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, type, TNodeFlags::Default) }); +} + TString TypeByAlias(const TString& alias, bool normalize) { TString type(alias); TCiString typeAlias(alias); @@ -3137,7 +3137,7 @@ TNodePtr BuildIsNullOp(TPosition pos, TNodePtr a) { return nullptr; } if (a->IsNull()) { - return BuildLiteralBool(pos, true); + return BuildLiteralBool(pos, true); } return new TCallNodeImpl(pos, "Not", {new TCallNodeImpl(pos, "Exists", {a})}); } @@ -3157,21 +3157,21 @@ TBinaryOpNode::TBinaryOpNode(TPosition pos, const TString& opName, TNodePtr a, T { } -TNodePtr BuildBinaryOp(TContext& ctx, TPosition pos, const TString& opName, TNodePtr a, TNodePtr b) { +TNodePtr BuildBinaryOp(TContext& ctx, TPosition pos, const TString& opName, TNodePtr a, TNodePtr b) { if (!a || !b) { return nullptr; } - - static const THashSet<TStringBuf> nullSafeOps = {"IsDistinctFrom", "IsNotDistinctFrom"}; - if (!nullSafeOps.contains(opName)) { - const bool bothArgNull = a->IsNull() && b->IsNull(); - const bool oneArgNull = a->IsNull() || b->IsNull(); - - if (bothArgNull || (opName != "Or" && oneArgNull)) { - ctx.Warning(pos, TIssuesIds::YQL_OPERATION_WILL_RETURN_NULL) << "Binary operation " << opName << " will return NULL here"; - } - } - + + static const THashSet<TStringBuf> nullSafeOps = {"IsDistinctFrom", "IsNotDistinctFrom"}; + if (!nullSafeOps.contains(opName)) { + const bool bothArgNull = a->IsNull() && b->IsNull(); + const bool oneArgNull = a->IsNull() || b->IsNull(); + + if (bothArgNull || (opName != "Or" && oneArgNull)) { + ctx.Warning(pos, TIssuesIds::YQL_OPERATION_WILL_RETURN_NULL) << "Binary operation " << opName << " will return NULL here"; + } + } + return new TBinaryOpNode(pos, opName, a, b); } @@ -3202,15 +3202,15 @@ public: void DoUpdateState() const override { State.Set(ENodeState::Const, FuncNode->IsConstant()); - State.Set(ENodeState::MaybeConst, FuncNode->MaybeConstant()); + State.Set(ENodeState::MaybeConst, FuncNode->MaybeConstant()); State.Set(ENodeState::Aggregated, FuncNode->IsAggregated()); State.Set(ENodeState::OverWindow, true); } - - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(FuncNode); - FuncNode->VisitTree(func, visited); - } + + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(FuncNode); + FuncNode->VisitTree(func, visited); + } protected: const TString WindowName; TNodePtr FuncNode; @@ -3229,9 +3229,9 @@ public: , Strict(strict) { auto udf = Y("Udf", Q(Fast ? "Yson2.Options" : "Yson.Options")); - auto autoConvertNode = BuildLiteralBool(pos, autoConvert); + auto autoConvertNode = BuildLiteralBool(pos, autoConvert); autoConvertNode->SetLabel("AutoConvert"); - auto strictNode = BuildLiteralBool(pos, strict); + auto strictNode = BuildLiteralBool(pos, strict); strictNode->SetLabel("Strict"); Node = Y("NamedApply", udf, Q(Y()), BuildStructure(pos, { autoConvertNode, strictNode })); } @@ -3251,10 +3251,10 @@ public: return new TYsonOptionsNode(Pos, AutoConvert, Strict); } - void DoUpdateState() const override { - State.Set(ENodeState::Const, true); - } - + void DoUpdateState() const override { + State.Set(ENodeState::Const, true); + } + protected: TNodePtr Node; const bool AutoConvert; @@ -3298,10 +3298,10 @@ public: return new TDoCall(Pos, Node->Clone()); } - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(Node); - Node->VisitTree(func, visited); - } + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(Node); + Node->VisitTree(func, visited); + } private: TNodePtr Node; TSourcePtr FakeSource; @@ -3407,10 +3407,10 @@ public: return {}; } - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_VERIFY_DEBUG(Node); - Node->VisitTree(func, visited); - } + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { + Y_VERIFY_DEBUG(Node); + Node->VisitTree(func, visited); + } protected: TNodePtr Node; const int EnsureTupleSize; diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index aeda93b001..de82b45a23 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -16,10 +16,10 @@ #include <library/cpp/enumbitset/enumbitset.h> -#include <array> -#include <functional> +#include <array> +#include <functional> #include <variant> - + namespace NSQLTranslationV1 { constexpr const size_t SQL_MAX_INLINE_SCRIPT_LEN = 24; @@ -32,7 +32,7 @@ namespace NSQLTranslationV1 { Initialized, CountHint, Const, - MaybeConst, + MaybeConst, Aggregated, AggregationKey, OverWindow, @@ -120,17 +120,17 @@ namespace NSQLTranslationV1 { TPosition GetPos() const; const TString& GetLabel() const; - TMaybe<TPosition> GetLabelPos() const; - void SetLabel(const TString& label, TMaybe<TPosition> pos = {}); - bool IsImplicitLabel() const; - void MarkImplicitLabel(bool isImplicitLabel); + TMaybe<TPosition> GetLabelPos() const; + void SetLabel(const TString& label, TMaybe<TPosition> pos = {}); + bool IsImplicitLabel() const; + void MarkImplicitLabel(bool isImplicitLabel); void SetCountHint(bool isCount); bool GetCountHint() const; bool Init(TContext& ctx, ISource* src); bool IsConstant() const; - bool MaybeConstant() const; + bool MaybeConstant() const; bool IsAggregated() const; bool IsAggregationKey() const; bool IsOverWindow() const; @@ -144,7 +144,7 @@ namespace NSQLTranslationV1 { virtual TString GetLiteralType() const; virtual TString GetLiteralValue() const; virtual bool IsIntegerLiteral() const; - virtual TPtr ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const; + virtual TPtr ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const; virtual bool IsAsterisk() const; virtual const TString* SubqueryAlias() const; virtual TString GetOpName() const; @@ -161,7 +161,7 @@ namespace NSQLTranslationV1 { virtual TVector<INode::TPtr>* ContentListPtr(); virtual TAstNode* Translate(TContext& ctx) const = 0; virtual TAggregationPtr GetAggregation() const; - virtual void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs); + virtual void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs); virtual TPtr WindowSpecFunc(const TPtr& type) const; virtual bool SetViewName(TContext& ctx, TPosition pos, const TString& view); void UseAsInner(); @@ -170,12 +170,12 @@ namespace NSQLTranslationV1 { virtual const TString* FuncName() const; virtual const TString* ModuleName() const; - using TVisitFunc = std::function<bool (const INode&)>; - using TVisitNodeSet = std::unordered_set<const INode*>; - - void VisitTree(const TVisitFunc& func) const; - void VisitTree(const TVisitFunc& func, TVisitNodeSet& visited) const; - + using TVisitFunc = std::function<bool (const INode&)>; + using TVisitNodeSet = std::unordered_set<const INode*>; + + void VisitTree(const TVisitFunc& func) const; + void VisitTree(const TVisitFunc& func, TVisitNodeSet& visited) const; + TPtr AstNode() const; TPtr AstNode(TAstNode* node) const; TPtr AstNode(TPtr node) const; @@ -221,7 +221,7 @@ namespace NSQLTranslationV1 { virtual TPtr DoClone() const = 0; void PrecacheState() const; - virtual void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const; + virtual void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const; private: virtual bool DoInit(TContext& ctx, ISource* src); virtual void DoAdd(TPtr node); @@ -229,15 +229,15 @@ namespace NSQLTranslationV1 { protected: TPosition Pos; TString Label; - TMaybe<TPosition> LabelPos; - bool ImplicitLabel = false; + TMaybe<TPosition> LabelPos; + bool ImplicitLabel = false; mutable TNodeState State; - bool AsInner = false; + bool AsInner = false; }; typedef INode::TPtr TNodePtr; - using TTableHints = TMap<TString, TVector<TNodePtr>>; - void MergeHints(TTableHints& base, const TTableHints& overrides); + using TTableHints = TMap<TString, TVector<TNodePtr>>; + void MergeHints(TTableHints& base, const TTableHints& overrides); template<class T> inline T SafeClone(const T& node) { @@ -256,7 +256,7 @@ namespace NSQLTranslationV1 { class TAstAtomNode: public INode { public: - TAstAtomNode(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg); + TAstAtomNode(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg); ~TAstAtomNode() override; @@ -278,12 +278,12 @@ namespace NSQLTranslationV1 { class TAstAtomNodeImpl final: public TAstAtomNode { public: - TAstAtomNodeImpl(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg = false) - : TAstAtomNode(pos, content, flags, isOptionalArg) + TAstAtomNodeImpl(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg = false) + : TAstAtomNode(pos, content, flags, isOptionalArg) {} TNodePtr DoClone() const final { - return new TAstAtomNodeImpl(Pos, Content, Flags, IsOptionalArg_); + return new TAstAtomNodeImpl(Pos, Content, Flags, IsOptionalArg_); } }; @@ -313,7 +313,7 @@ namespace NSQLTranslationV1 { TPtr ShallowCopy() const override; bool DoInit(TContext& ctx, ISource* src) override; void DoAdd(TNodePtr node) override; - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const override; + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const override; void DoUpdateState() const override; @@ -328,10 +328,10 @@ namespace NSQLTranslationV1 { public: TAstListNodeImpl(TPosition pos); TAstListNodeImpl(TPosition pos, TVector<TNodePtr> nodes); - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; protected: - TNodePtr DoClone() const final; + TNodePtr DoClone() const final; }; class TCallNode: public TAstListNode { @@ -350,7 +350,7 @@ namespace NSQLTranslationV1 { bool DoInit(TContext& ctx, ISource* src) override; bool ValidateArguments(TContext& ctx) const; TString GetCallExplain() const; - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; protected: TString OpName; @@ -446,15 +446,15 @@ namespace NSQLTranslationV1 { TWinLeadLag(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args); }; - class TWinRank final: public TWinAggrEmulation { - TPtr DoClone() const final { - return CallNodeClone<TWinRank>(); - } - bool DoInit(TContext& ctx, ISource* src) override; - public: - TWinRank(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args); - }; - + class TWinRank final: public TWinAggrEmulation { + TPtr DoClone() const final { + return CallNodeClone<TWinRank>(); + } + bool DoInit(TContext& ctx, ISource* src) override; + public: + TWinRank(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args); + }; + class ITableKeys: public INode { public: enum class EBuildKeysMode { @@ -490,7 +490,7 @@ namespace NSQLTranslationV1 { TDeferredAtom(TPosition pos, const TString& str); TDeferredAtom(TNodePtr node, TContext& ctx); const TString* GetLiteral() const; - bool GetLiteral(TString& value, TContext& ctx) const; + bool GetLiteral(TString& value, TContext& ctx) const; TNodePtr Build() const; TString GetRepr() const; bool Empty() const; @@ -566,43 +566,43 @@ namespace NSQLTranslationV1 { enum EFrameType { FrameByRows, FrameByRange, - FrameByGroups, + FrameByGroups, }; enum EFrameExclusions { - FrameExclNone, // same as EXCLUDE NO OTHERS + FrameExclNone, // same as EXCLUDE NO OTHERS FrameExclCurRow, FrameExclGroup, FrameExclTies, }; - enum EFrameSettings { - // keep order - FrameUndefined, - FramePreceding, - FrameCurrentRow, - FrameFollowing, - }; - - struct TFrameBound: public TSimpleRefCount<TFrameBound> { - TPosition Pos; - TNodePtr Bound; - EFrameSettings Settings = FrameUndefined; - - TIntrusivePtr<TFrameBound> Clone() const; - ~TFrameBound() {} - }; - typedef TIntrusivePtr<TFrameBound> TFrameBoundPtr; - - - struct TFrameSpecification: public TSimpleRefCount<TFrameSpecification> { - EFrameType FrameType = FrameByRows; - TFrameBoundPtr FrameBegin; - TFrameBoundPtr FrameEnd; + enum EFrameSettings { + // keep order + FrameUndefined, + FramePreceding, + FrameCurrentRow, + FrameFollowing, + }; + + struct TFrameBound: public TSimpleRefCount<TFrameBound> { + TPosition Pos; + TNodePtr Bound; + EFrameSettings Settings = FrameUndefined; + + TIntrusivePtr<TFrameBound> Clone() const; + ~TFrameBound() {} + }; + typedef TIntrusivePtr<TFrameBound> TFrameBoundPtr; + + + struct TFrameSpecification: public TSimpleRefCount<TFrameSpecification> { + EFrameType FrameType = FrameByRows; + TFrameBoundPtr FrameBegin; + TFrameBoundPtr FrameEnd; EFrameExclusions FrameExclusion = FrameExclNone; - TIntrusivePtr<TFrameSpecification> Clone() const; - ~TFrameSpecification() {} + TIntrusivePtr<TFrameSpecification> Clone() const; + ~TFrameSpecification() {} }; - typedef TIntrusivePtr<TFrameSpecification> TFrameSpecificationPtr; + typedef TIntrusivePtr<TFrameSpecification> TFrameSpecificationPtr; struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> { TNodePtr TimeExtractor; @@ -614,36 +614,36 @@ namespace NSQLTranslationV1 { TIntrusivePtr<THoppingWindowSpec> Clone() const; ~THoppingWindowSpec() {} }; - typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr; - - struct TWindowSpecification: public TSimpleRefCount<TWindowSpecification> { - TMaybe<TString> ExistingWindowName; - TVector<TNodePtr> Partitions; - bool IsCompact = false; - TVector<TSortSpecificationPtr> OrderBy; - TNodePtr Session; - TFrameSpecificationPtr Frame; - - TIntrusivePtr<TWindowSpecification> Clone() const; - ~TWindowSpecification() {} - }; + typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr; + + struct TWindowSpecification: public TSimpleRefCount<TWindowSpecification> { + TMaybe<TString> ExistingWindowName; + TVector<TNodePtr> Partitions; + bool IsCompact = false; + TVector<TSortSpecificationPtr> OrderBy; + TNodePtr Session; + TFrameSpecificationPtr Frame; + + TIntrusivePtr<TWindowSpecification> Clone() const; + ~TWindowSpecification() {} + }; typedef TIntrusivePtr<TWindowSpecification> TWindowSpecificationPtr; typedef TMap<TString, TWindowSpecificationPtr> TWinSpecs; typedef TVector<TTableRef> TTableList; - void WarnIfAliasFromSelectIsUsedInGroupBy(TContext& ctx, const TVector<TNodePtr>& selectTerms, const TVector<TNodePtr>& groupByTerms, - const TVector<TNodePtr>& groupByExprTerms); + void WarnIfAliasFromSelectIsUsedInGroupBy(TContext& ctx, const TVector<TNodePtr>& selectTerms, const TVector<TNodePtr>& groupByTerms, + const TVector<TNodePtr>& groupByExprTerms); bool ValidateAllNodesForAggregation(TContext& ctx, const TVector<TNodePtr>& nodes); struct TWriteSettings { - bool Discard = false; + bool Discard = false; TDeferredAtom Label; }; class TColumnNode final: public INode { public: - TColumnNode(TPosition pos, const TString& column, const TString& source, bool maybeType); + TColumnNode(TPosition pos, const TString& column, const TString& source, bool maybeType); TColumnNode(TPosition pos, const TNodePtr& column, const TString& source); virtual ~TColumnNode(); @@ -661,7 +661,7 @@ namespace NSQLTranslationV1 { void SetAsNotReliable(); bool IsReliable() const; bool IsUseSourceAsColumn() const; - bool CanBeType() const; + bool CanBeType() const; private: bool DoInit(TContext& ctx, ISource* src) override; @@ -680,7 +680,7 @@ namespace NSQLTranslationV1 { bool Reliable = true; bool UseSource = false; bool UseSourceAsColumn = false; - bool MaybeType = false; + bool MaybeType = false; }; class TArgPlaceholderNode final: public INode @@ -717,15 +717,15 @@ namespace NSQLTranslationV1 { TPtr GetTupleElement(size_t index) const override; TNodePtr DoClone() const final; private: - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; - const TString* GetSourceName() const override; - + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; + const TString* GetSourceName() const override; + const TVector<TNodePtr> Exprs; }; class TStructNode: public TAstListNode { public: - TStructNode(TPosition pos, const TVector<TNodePtr>& exprs, const TVector<TNodePtr>& labels, bool ordered); + TStructNode(TPosition pos, const TVector<TNodePtr>& exprs, const TVector<TNodePtr>& labels, bool ordered); bool DoInit(TContext& ctx, ISource* src) override; TNodePtr DoClone() const final; @@ -734,12 +734,12 @@ namespace NSQLTranslationV1 { } private: - void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; - const TString* GetSourceName() const override; - + void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override; + const TString* GetSourceName() const override; + const TVector<TNodePtr> Exprs; - const TVector<TNodePtr> Labels; - const bool Ordered; + const TVector<TNodePtr> Labels; + const bool Ordered; }; class IAggregation: public INode { @@ -764,9 +764,9 @@ namespace NSQLTranslationV1 { const TString& GetName() const; - EAggregateMode GetAggregationMode() const; - void MarkKeyColumnAsGenerated(); - + EAggregateMode GetAggregationMode() const; + void MarkKeyColumnAsGenerated(); + virtual void Join(IAggregation* aggr); private: @@ -780,15 +780,15 @@ namespace NSQLTranslationV1 { TString Func; const EAggregateMode AggMode; TString DistinctKey; - bool IsGeneratedKeyColumn = false; + bool IsGeneratedKeyColumn = false; }; enum class EExprSeat: int { Open = 0, - FlattenByExpr, + FlattenByExpr, FlattenBy, GroupBy, - DistinctAggr, + DistinctAggr, WindowPartitionBy, Max }; @@ -835,37 +835,37 @@ namespace NSQLTranslationV1 { virtual bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func); virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec); virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const; - virtual TNodePtr GetSessionWindowSpec() const; + virtual TNodePtr GetSessionWindowSpec() const; virtual bool IsCompositeSource() const; virtual bool IsGroupByColumn(const TString& column) const; virtual bool IsFlattenByColumns() const; - virtual bool IsFlattenByExprs() const; + virtual bool IsFlattenByExprs() const; virtual bool IsCalcOverWindow() const; virtual bool IsOverWindowSource() const; virtual bool IsStream() const; virtual EOrderKind GetOrderKind() const; virtual TWriteSettings GetWriteSettings() const; virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed); - virtual bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints); + virtual bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints); virtual bool CalculateGroupingHint(TContext& ctx, const TVector<TString>& columns, ui64& hint) const; - virtual TNodePtr BuildFilter(TContext& ctx, const TString& label); - virtual TNodePtr BuildFilterLambda(); + virtual TNodePtr BuildFilter(TContext& ctx, const TString& label); + virtual TNodePtr BuildFilterLambda(); virtual TNodePtr BuildFlattenByColumns(const TString& label); virtual TNodePtr BuildFlattenColumns(const TString& label); - virtual TNodePtr BuildPreaggregatedMap(TContext& ctx); - virtual TNodePtr BuildPreFlattenMap(TContext& ctx); - virtual TNodePtr BuildPrewindowMap(TContext& ctx); + virtual TNodePtr BuildPreaggregatedMap(TContext& ctx); + virtual TNodePtr BuildPreFlattenMap(TContext& ctx); + virtual TNodePtr BuildPrewindowMap(TContext& ctx); virtual TNodePtr BuildAggregation(const TString& label); - virtual TNodePtr BuildCalcOverWindow(TContext& ctx, const TString& label); + virtual TNodePtr BuildCalcOverWindow(TContext& ctx, const TString& label); virtual TNodePtr BuildSort(TContext& ctx, const TString& label); virtual TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label); virtual bool BuildSamplingLambda(TNodePtr& node); - virtual bool SetSamplingRate(TContext& ctx, TNodePtr samplingRate); + virtual bool SetSamplingRate(TContext& ctx, TNodePtr samplingRate); virtual IJoin* GetJoin(); virtual ISource* GetCompositeSource(); virtual bool IsSelect() const; virtual bool IsTableSource() const; - virtual bool ShouldUseSourceAsColumn(const TString& source) const; + virtual bool ShouldUseSourceAsColumn(const TString& source) const; virtual bool IsJoinKeysInitializing() const; virtual const TString* GetWindowName() const; @@ -880,7 +880,7 @@ namespace NSQLTranslationV1 { bool IsExprAlias(const TString& label) const; bool IsExprSeat(EExprSeat exprSeat, EExprType type = EExprType::WithExpression) const; TString GetGroupByColumnAlias(const TString& column) const; - const TVector<TNodePtr>& Expressions(EExprSeat exprSeat) const; + const TVector<TNodePtr>& Expressions(EExprSeat exprSeat) const; virtual TWindowSpecificationPtr FindWindowSpecification(TContext& ctx, const TString& windowName) const; @@ -891,15 +891,15 @@ namespace NSQLTranslationV1 { virtual TAstNode* Translate(TContext& ctx) const; void FillSortParts(const TVector<TSortSpecificationPtr>& orderBy, TNodePtr& sortKeySelector, TNodePtr& sortDirection); - TNodePtr BuildSortSpec(const TVector<TSortSpecificationPtr>& orderBy, const TString& label, bool traits, bool assume); + TNodePtr BuildSortSpec(const TVector<TSortSpecificationPtr>& orderBy, const TString& label, bool traits, bool assume); TVector<TNodePtr>& Expressions(EExprSeat exprSeat); TNodePtr AliasOrColumn(const TNodePtr& node, bool withSource); - TNodePtr BuildWindowFrame(const TFrameSpecification& spec, bool isCompact); - + TNodePtr BuildWindowFrame(const TFrameSpecification& spec, bool isCompact); + THashSet<TString> ExprAliases; - THashSet<TString> FlattenByAliases; + THashSet<TString> FlattenByAliases; THashMap<TString, TString> GroupByColumnAliases; TVector<TNodePtr> Filters; bool CompactGroupBy = false; @@ -907,11 +907,11 @@ namespace NSQLTranslationV1 { TVector<TString> OrderedGroupKeys; std::array<TVector<TNodePtr>, static_cast<unsigned>(EExprSeat::Max)> NamedExprs; TVector<TAggregationPtr> Aggregations; - TMap<TString, TVector<TAggregationPtr>> AggregationOverWindow; - TMap<TString, TVector<TNodePtr>> FuncOverWindow; + TMap<TString, TVector<TAggregationPtr>> AggregationOverWindow; + TMap<TString, TVector<TNodePtr>> FuncOverWindow; TWinSpecs WinSpecs; THoppingWindowSpecPtr HoppingWindowSpec; - TNodePtr SessionWindow; + TNodePtr SessionWindow; TVector<ISource*> UsedSources; TString FlattenMode; bool FlattenColumns = false; @@ -930,19 +930,19 @@ namespace NSQLTranslationV1 { return cloneArgs; } - struct TJoinLinkSettings { - bool ForceSortedMerge = false; - }; - + struct TJoinLinkSettings { + bool ForceSortedMerge = false; + }; + class IJoin: public ISource { public: virtual ~IJoin(); virtual IJoin* GetJoin(); virtual TNodePtr BuildJoinKeys(TContext& ctx, const TVector<TDeferredAtom>& names) = 0; - virtual void SetupJoin(const TString& joinOp, TNodePtr joinExpr, const TJoinLinkSettings& linkSettings) = 0; + virtual void SetupJoin(const TString& joinOp, TNodePtr joinExpr, const TJoinLinkSettings& linkSettings) = 0; virtual const THashMap<TString, THashSet<TString>>& GetSameKeysMap() const = 0; - virtual TVector<TString> GetJoinLabels() const = 0; + virtual TVector<TString> GetJoinLabels() const = 0; protected: IJoin(TPosition pos); @@ -955,7 +955,7 @@ namespace NSQLTranslationV1 { TVector<TNodePtr>* ContentListPtr() override; TAstNode* Translate(TContext& ctx) const override; TPtr DoClone() const final; - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final; + void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final; private: TVector<TNodePtr> Exprs; TString Meaning; @@ -965,7 +965,7 @@ namespace NSQLTranslationV1 { public: TLiteralNode(TPosition pos, bool isNull); TLiteralNode(TPosition pos, const TString& type, const TString& value); - TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags); + TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags); TLiteralNode(TPosition pos, const TString& value, ui32 nodeFlags, const TString& type); bool IsNull() const override; const TString* GetLiteral(const TString& type) const override; @@ -992,13 +992,13 @@ namespace NSQLTranslationV1 { template<typename T> class TLiteralNumberNode: public TLiteralNode { public: - TLiteralNumberNode(TPosition pos, const TString& type, const TString& value, bool implicitType = false); + TLiteralNumberNode(TPosition pos, const TString& type, const TString& value, bool implicitType = false); TPtr DoClone() const override final; bool DoInit(TContext& ctx, ISource* src) override; bool IsIntegerLiteral() const override; - TPtr ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const override; - private: - const bool ImplicitType; + TPtr ApplyUnaryOp(TContext& ctx, TPosition pos, const TString& opName) const override; + private: + const bool ImplicitType; }; struct TTableArg { @@ -1025,32 +1025,32 @@ namespace NSQLTranslationV1 { TNodePtr Node; }; - class TSessionWindow final : public INode { - public: - TSessionWindow(TPosition pos, const TVector<TNodePtr>& args); - void MarkValid(); - TNodePtr BuildTraits(const TString& label) const; - private: - bool DoInit(TContext& ctx, ISource* src) override; - TAstNode* Translate(TContext&) const override; - void DoUpdateState() const override; - TNodePtr DoClone() const override; - TString GetOpName() const override; - - TVector<TNodePtr> Args; - TSourcePtr FakeSource; - TNodePtr Node; - bool Valid; - }; - - struct TStringContent { - TString Content; + class TSessionWindow final : public INode { + public: + TSessionWindow(TPosition pos, const TVector<TNodePtr>& args); + void MarkValid(); + TNodePtr BuildTraits(const TString& label) const; + private: + bool DoInit(TContext& ctx, ISource* src) override; + TAstNode* Translate(TContext&) const override; + void DoUpdateState() const override; + TNodePtr DoClone() const override; + TString GetOpName() const override; + + TVector<TNodePtr> Args; + TSourcePtr FakeSource; + TNodePtr Node; + bool Valid; + }; + + struct TStringContent { + TString Content; NYql::NUdf::EDataSlot Type = NYql::NUdf::EDataSlot::String; - ui32 Flags = NYql::TNodeFlags::Default; - }; - - TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input); - TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input); + ui32 Flags = NYql::TNodeFlags::Default; + }; + + TMaybe<TStringContent> StringContent(TContext& ctx, TPosition pos, const TString& input); + TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, const TString& input); struct TTtlSettings { TIdentifier ColumnName; @@ -1162,13 +1162,13 @@ namespace NSQLTranslationV1 { } }; - struct TRoleParameters { - TMaybe<TDeferredAtom> Password; - bool IsPasswordEncrypted = false; - }; - + struct TRoleParameters { + TMaybe<TDeferredAtom> Password; + bool IsPasswordEncrypted = false; + }; + TString IdContent(TContext& ctx, const TString& str); - TString IdContentFromString(TContext& ctx, const TString& str); + TString IdContentFromString(TContext& ctx, const TString& str); TTableHints GetContextHints(TContext& ctx); TString TypeByAlias(const TString& alias, bool normalize = true); @@ -1180,24 +1180,24 @@ namespace NSQLTranslationV1 { TNodePtr BuildLiteralNull(TPosition pos); TNodePtr BuildLiteralVoid(TPosition pos); /// String is checked as quotable, support escaping and multiline - TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value); - - struct TExprOrIdent { - TNodePtr Expr; - TString Ident; - }; - TMaybe<TExprOrIdent> BuildLiteralTypedSmartStringOrId(TContext& ctx, const TString& value); - - TNodePtr BuildLiteralRawString(TPosition pos, const TString& value, bool isUtf8 = false); - TNodePtr BuildLiteralBool(TPosition pos, bool value); + TNodePtr BuildLiteralSmartString(TContext& ctx, const TString& value); + + struct TExprOrIdent { + TNodePtr Expr; + TString Ident; + }; + TMaybe<TExprOrIdent> BuildLiteralTypedSmartStringOrId(TContext& ctx, const TString& value); + + TNodePtr BuildLiteralRawString(TPosition pos, const TString& value, bool isUtf8 = false); + TNodePtr BuildLiteralBool(TPosition pos, bool value); TNodePtr BuildEmptyAction(TPosition pos); TNodePtr BuildTuple(TPosition pos, const TVector<TNodePtr>& exprs); - + TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprs); - TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels); - TNodePtr BuildOrderedStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels); - + TNodePtr BuildStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels); + TNodePtr BuildOrderedStructure(TPosition pos, const TVector<TNodePtr>& exprsUnlabeled, const TVector<TNodePtr>& labels); + TNodePtr BuildListOfNamedNodes(TPosition pos, TVector<TNodePtr>&& exprs); TNodePtr BuildArgPlaceholder(TPosition pos, const TString& name); @@ -1205,14 +1205,14 @@ namespace NSQLTranslationV1 { TNodePtr BuildColumn(TPosition pos, const TString& column = TString(), const TString& source = TString()); TNodePtr BuildColumn(TPosition pos, const TNodePtr& column, const TString& source = TString()); TNodePtr BuildColumn(TPosition pos, const TDeferredAtom& column, const TString& source = TString()); - TNodePtr BuildColumnOrType(TPosition pos, const TString& column = TString()); + TNodePtr BuildColumnOrType(TPosition pos, const TString& column = TString()); TNodePtr BuildAccess(TPosition pos, const TVector<INode::TIdPart>& ids, bool isLookup); TNodePtr BuildBind(TPosition pos, const TString& module, const TString& alias); TNodePtr BuildLambda(TPosition pos, TNodePtr params, TNodePtr body, const TString& resName = TString()); TNodePtr BuildDataType(TPosition pos, const TString& typeName); - TNodePtr BuildSimpleType(TContext& ctx, TPosition pos, const TString& typeName, bool dataOnly); + TNodePtr BuildSimpleType(TContext& ctx, TPosition pos, const TString& typeName, bool dataOnly); TNodePtr BuildIsNullOp(TPosition pos, TNodePtr a); - TNodePtr BuildBinaryOp(TContext& ctx, TPosition pos, const TString& opName, TNodePtr a, TNodePtr b); + TNodePtr BuildBinaryOp(TContext& ctx, TPosition pos, const TString& opName, TNodePtr a, TNodePtr b); TNodePtr BuildCalcOverWindow(TPosition pos, const TString& windowName, TNodePtr call); TNodePtr BuildYsonOptionsNode(TPosition pos, bool autoConvert, bool strict, bool fastYson); @@ -1238,7 +1238,7 @@ namespace NSQLTranslationV1 { // Implemented in builtin.cpp - TNodePtr BuildCallable(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce = false); + TNodePtr BuildCallable(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce = false); TNodePtr BuildUdf(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args); TNodePtr BuildBuiltinFunc( TContext& ctx, @@ -1248,24 +1248,24 @@ namespace NSQLTranslationV1 { const TString& nameSpace = TString(), EAggregateMode aggMode = EAggregateMode::Normal, bool* mustUseNamed = nullptr, - bool warnOnYqlNameSpace = true + bool warnOnYqlNameSpace = true ); // Implemented in join.cpp TString NormalizeJoinOp(const TString& joinOp); - TSourcePtr BuildEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes); + TSourcePtr BuildEquiJoin(TPosition pos, TVector<TSourcePtr>&& sources, TVector<bool>&& anyFlags, bool strictJoinKeyTypes); // Implemented in select.cpp TNodePtr BuildSubquery(TSourcePtr source, const TString& alias, bool inSubquery, int ensureTupleSize, TScopedStatePtr scoped); TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex = -1); TNodePtr BuildSourceNode(TPosition pos, TSourcePtr source, bool checkExist = false); TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources); - TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom = false); - TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList = false); + TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom = false); + TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList = false); TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label = TString()); TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& service, const TDeferredAtom& cluster, const TString& label = TString()); TSourcePtr BuildRefColumnSource(TPosition pos, const TString& partExpression); - TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings); + TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings); TSourcePtr BuildOverWindowSource(TPosition pos, const TString& windowName, ISource* origSource); TNodePtr BuildOrderBy(TPosition pos, const TVector<TNodePtr>& keys, const TVector<bool>& order); @@ -1324,14 +1324,14 @@ namespace NSQLTranslationV1 { TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, TScopedStatePtr scoped); - TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped); - TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped); - TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped); - TNodePtr BuildRenameUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped); - TNodePtr BuildAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, - TScopedStatePtr scoped); - TNodePtr BuildRenameGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped); - TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped); + TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped); + TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped); + TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped); + TNodePtr BuildRenameUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped); + TNodePtr BuildAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, + TScopedStatePtr scoped); + TNodePtr BuildRenameGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped); + TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped); TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options, TScopedStatePtr scoped); TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index c0c0c28008..725356b9f5 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -169,14 +169,14 @@ public: } TCiString func(Func); - if (func != "object") { - for (auto& arg: Args) { - if (arg.Expr->GetLabel()) { - ctx.Error(Pos) << "Named arguments are not supported for table function " << to_upper(Func); - return nullptr; - } - } - } + if (func != "object") { + for (auto& arg: Args) { + if (arg.Expr->GetLabel()) { + ctx.Error(Pos) << "Named arguments are not supported for table function " << to_upper(Func); + return nullptr; + } + } + } if (func == "concat_strict") { auto tuple = Y(); for (auto& arg: Args) { @@ -406,42 +406,42 @@ public: return L(Y("DataTables")); } - else if (func == "object") { - const size_t positionalArgs = 2; - auto result = Y("MrObject"); - auto settings = Y(); - //TVector<TNodePtr> settings; - size_t argc = 0; - for (ui32 index = 0; index < Args.size(); ++index) { - auto& arg = Args[index]; - if (arg.HasAt) { - ctx.Error(arg.Expr->GetPos()) << "Temporary tables are not supported here"; - return nullptr; - } - - if (!arg.View.empty()) { - ctx.Error(Pos) << to_upper(Func) << " doesn't supports views"; - return nullptr; - } - - if (!arg.Expr->GetLabel()) { - ExtractTableName(ctx, arg); - result = L(result, arg.Id.Build()); - ++argc; - } else { - settings = L(settings, Q(Y(BuildQuotedAtom(arg.Expr->GetPos(), arg.Expr->GetLabel()), arg.Expr))); - } - } - - if (argc != positionalArgs) { - ctx.Error(Pos) << to_upper(Func) << " requires exacty " << positionalArgs << " positional args, but got " << argc; - return nullptr; - } - - result = L(result, Q(settings)); - return result; - } - + else if (func == "object") { + const size_t positionalArgs = 2; + auto result = Y("MrObject"); + auto settings = Y(); + //TVector<TNodePtr> settings; + size_t argc = 0; + for (ui32 index = 0; index < Args.size(); ++index) { + auto& arg = Args[index]; + if (arg.HasAt) { + ctx.Error(arg.Expr->GetPos()) << "Temporary tables are not supported here"; + return nullptr; + } + + if (!arg.View.empty()) { + ctx.Error(Pos) << to_upper(Func) << " doesn't supports views"; + return nullptr; + } + + if (!arg.Expr->GetLabel()) { + ExtractTableName(ctx, arg); + result = L(result, arg.Id.Build()); + ++argc; + } else { + settings = L(settings, Q(Y(BuildQuotedAtom(arg.Expr->GetPos(), arg.Expr->GetLabel()), arg.Expr))); + } + } + + if (argc != positionalArgs) { + ctx.Error(Pos) << to_upper(Func) << " requires exacty " << positionalArgs << " positional args, but got " << argc; + return nullptr; + } + + result = L(result, Q(settings)); + return result; + } + ctx.Error(Pos) << "Unknown table name preprocessor: " << Func; return nullptr; } @@ -468,14 +468,14 @@ public: bool DoInit(TContext& ctx, ISource* src) override { for (auto& hint: Hints) { - TString hintName = hint.first; - TMaybe<TIssue> normalizeError = NormalizeName(Pos, hintName); + TString hintName = hint.first; + TMaybe<TIssue> normalizeError = NormalizeName(Pos, hintName); if (!normalizeError.Empty()) { ctx.Error() << normalizeError->Message; ctx.IncrementMonCounter("sql_errors", "NormalizeHintError"); return false; } - TNodePtr option = Y(BuildQuotedAtom(Pos, hintName)); + TNodePtr option = Y(BuildQuotedAtom(Pos, hintName)); for (auto& x : hint.second) { if (!x->Init(ctx, src)) { return false; @@ -1055,333 +1055,333 @@ TNodePtr BuildDropTable(TPosition pos, const TTableRef& tr, TScopedStatePtr scop return new TDropTableNode(pos, tr, scoped); } -class TCreateRole final: public TAstListNode { -public: - TCreateRole(TPosition pos, bool isUser, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped) - : TAstListNode(pos) - , IsUser(isUser) - , Service(service) - , Cluster(cluster) - , Name(name) - , Params(params) - , Scoped(scoped) - { - FakeSource = BuildFakeSource(pos); - scoped->UseCluster(service, cluster); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(src); - auto name = Name.Build(); - TNodePtr password; - if (Params && Params->Password) { - password = Params->Password->Build(); - } - TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); - - if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { - return false; - } - if (password && !password->Init(ctx, FakeSource.Get())) { - return false; - } - - auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "createUser" : "createGroup")))); - if (Params) { - if (Params->IsPasswordEncrypted) { - options = L(options, Q(Y(Q("passwordEncrypted")))); - } - if (Params->Password) { - options = L(options, Q(Y(Q("password"), password))); - } else { - options = L(options, Q(Y(Q("nullPassword")))); - } - } - - Add("block", Q(Y( - Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), - Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), - Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - return TAstListNode::DoInit(ctx, FakeSource.Get()); - } - - TPtr DoClone() const final { - return {}; - } -private: - const bool IsUser; - const TString Service; - TDeferredAtom Cluster; - TDeferredAtom Name; - const TMaybe<TRoleParameters> Params; - TScopedStatePtr Scoped; - TSourcePtr FakeSource; -}; - -TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped) { - bool isUser = true; - return new TCreateRole(pos, isUser, service, cluster, name, params, scoped); -} - -TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped) { - bool isUser = false; - return new TCreateRole(pos, isUser, service, cluster, name, {}, scoped); -} - -class TAlterUser final: public TAstListNode { -public: - TAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped) - : TAstListNode(pos) - , Service(service) - , Cluster(cluster) - , Name(name) - , Params(params) - , Scoped(scoped) - { - FakeSource = BuildFakeSource(pos); - scoped->UseCluster(service, cluster); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(src); - auto name = Name.Build(); - TNodePtr password; - if (Params.Password) { - password = Params.Password->Build(); - } - TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); - - if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { - return false; - } - if (password && !password->Init(ctx, FakeSource.Get())) { - return false; - } - - auto options = Y(Q(Y(Q("mode"), Q("alterUser")))); - if (Params.IsPasswordEncrypted) { - options = L(options, Q(Y(Q("passwordEncrypted")))); - } - if (Params.Password) { - options = L(options, Q(Y(Q("password"), password))); - } else { - options = L(options, Q(Y(Q("nullPassword")))); - } - - Add("block", Q(Y( - Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), - Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), - Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - return TAstListNode::DoInit(ctx, FakeSource.Get()); - } - - TPtr DoClone() const final { - return {}; - } -private: - const TString Service; - TDeferredAtom Cluster; - TDeferredAtom Name; - const TRoleParameters Params; - TScopedStatePtr Scoped; - TSourcePtr FakeSource; -}; - -TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped) { - return new TAlterUser(pos, service, cluster, name, params, scoped); -} - -class TRenameRole final: public TAstListNode { -public: - TRenameRole(TPosition pos, bool isUser, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) - : TAstListNode(pos) - , IsUser(isUser) - , Service(service) - , Cluster(cluster) - , Name(name) - , NewName(newName) - , Scoped(scoped) - { - FakeSource = BuildFakeSource(pos); - scoped->UseCluster(service, cluster); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(src); - auto name = Name.Build(); - auto newName = NewName.Build(); - TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); - - if (!name->Init(ctx, FakeSource.Get()) || - !newName->Init(ctx, FakeSource.Get()) || - !cluster->Init(ctx, FakeSource.Get())) - { - return false; - } - - auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "renameUser" : "renameGroup")))); - options = L(options, Q(Y(Q("newName"), newName))); - - Add("block", Q(Y( - Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), - Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), - Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - return TAstListNode::DoInit(ctx, FakeSource.Get()); - } - - TPtr DoClone() const final { - return {}; - } -private: - const bool IsUser; - const TString Service; - TDeferredAtom Cluster; - TDeferredAtom Name; - TDeferredAtom NewName; - TScopedStatePtr Scoped; - TSourcePtr FakeSource; -}; - -TNodePtr BuildRenameUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) { - const bool isUser = true; - return new TRenameRole(pos, isUser, service, cluster, name, newName, scoped); -} - -TNodePtr BuildRenameGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) { - const bool isUser = false; - return new TRenameRole(pos, isUser, service, cluster, name, newName, scoped); -} - -class TAlterGroup final: public TAstListNode { -public: - TAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, TScopedStatePtr scoped) - : TAstListNode(pos) - , Service(service) - , Cluster(cluster) - , Name(name) - , ToChange(toChange) - , IsDrop(isDrop) - , Scoped(scoped) - { - FakeSource = BuildFakeSource(pos); - scoped->UseCluster(service, cluster); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(src); - auto name = Name.Build(); - TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); - - if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { - return false; - } - - TVector<TNodePtr> toChange; - for (auto& item : ToChange) { - toChange.push_back(item.Build()); - if (!toChange.back()->Init(ctx, FakeSource.Get())) { - return false; - } - } - - auto options = Y(Q(Y(Q("mode"), Q(IsDrop ? "dropUsersFromGroup" : "addUsersToGroup")))); - options = L(options, Q(Y(Q("roles"), Q(new TAstListNodeImpl(Pos, std::move(toChange)))))); - - Add("block", Q(Y( - Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), - Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), - Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) - ))); - - return TAstListNode::DoInit(ctx, FakeSource.Get()); - } - - TPtr DoClone() const final { - return {}; - } -private: - const TString Service; - TDeferredAtom Cluster; - TDeferredAtom Name; - TVector<TDeferredAtom> ToChange; - const bool IsDrop; - TScopedStatePtr Scoped; - TSourcePtr FakeSource; -}; - -TNodePtr BuildAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, - TScopedStatePtr scoped) -{ - return new TAlterGroup(pos, service, cluster, name, toChange, isDrop, scoped); -} - -class TDropRoles final: public TAstListNode { -public: - TDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped) - : TAstListNode(pos) - , Service(service) - , Cluster(cluster) - , ToDrop(toDrop) - , IsUser(isUser) - , Force(force) - , Scoped(scoped) - { - FakeSource = BuildFakeSource(pos); - scoped->UseCluster(service, cluster); - } - - bool DoInit(TContext& ctx, ISource* src) override { - Y_UNUSED(src); - TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); - - if (!cluster->Init(ctx, FakeSource.Get())) { - return false; - } - - auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "dropUser" : "dropGroup")))); - if (Force) { - options = L(options, Q(Y(Q("force")))); - } - - - auto block = Y(Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster))); - for (auto& item : ToDrop) { - auto name = item.Build(); - if (!name->Init(ctx, FakeSource.Get())) { - return false; - } - - block = L(block, Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options)))); - } - block = L(block, Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world"))); - Add("block", Q(block)); - - return TAstListNode::DoInit(ctx, FakeSource.Get()); - } - - TPtr DoClone() const final { - return {}; - } -private: - const TString Service; - TDeferredAtom Cluster; - TVector<TDeferredAtom> ToDrop; - const bool IsUser; - const bool Force; - TScopedStatePtr Scoped; - TSourcePtr FakeSource; -}; - -TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped) { - return new TDropRoles(pos, service, cluster, toDrop, isUser, force, scoped); -} - +class TCreateRole final: public TAstListNode { +public: + TCreateRole(TPosition pos, bool isUser, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped) + : TAstListNode(pos) + , IsUser(isUser) + , Service(service) + , Cluster(cluster) + , Name(name) + , Params(params) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + auto name = Name.Build(); + TNodePtr password; + if (Params && Params->Password) { + password = Params->Password->Build(); + } + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { + return false; + } + if (password && !password->Init(ctx, FakeSource.Get())) { + return false; + } + + auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "createUser" : "createGroup")))); + if (Params) { + if (Params->IsPasswordEncrypted) { + options = L(options, Q(Y(Q("passwordEncrypted")))); + } + if (Params->Password) { + options = L(options, Q(Y(Q("password"), password))); + } else { + options = L(options, Q(Y(Q("nullPassword")))); + } + } + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), + Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + const bool IsUser; + const TString Service; + TDeferredAtom Cluster; + TDeferredAtom Name; + const TMaybe<TRoleParameters> Params; + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped) { + bool isUser = true; + return new TCreateRole(pos, isUser, service, cluster, name, params, scoped); +} + +TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped) { + bool isUser = false; + return new TCreateRole(pos, isUser, service, cluster, name, {}, scoped); +} + +class TAlterUser final: public TAstListNode { +public: + TAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped) + : TAstListNode(pos) + , Service(service) + , Cluster(cluster) + , Name(name) + , Params(params) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + auto name = Name.Build(); + TNodePtr password; + if (Params.Password) { + password = Params.Password->Build(); + } + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { + return false; + } + if (password && !password->Init(ctx, FakeSource.Get())) { + return false; + } + + auto options = Y(Q(Y(Q("mode"), Q("alterUser")))); + if (Params.IsPasswordEncrypted) { + options = L(options, Q(Y(Q("passwordEncrypted")))); + } + if (Params.Password) { + options = L(options, Q(Y(Q("password"), password))); + } else { + options = L(options, Q(Y(Q("nullPassword")))); + } + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), + Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + const TString Service; + TDeferredAtom Cluster; + TDeferredAtom Name; + const TRoleParameters Params; + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped) { + return new TAlterUser(pos, service, cluster, name, params, scoped); +} + +class TRenameRole final: public TAstListNode { +public: + TRenameRole(TPosition pos, bool isUser, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) + : TAstListNode(pos) + , IsUser(isUser) + , Service(service) + , Cluster(cluster) + , Name(name) + , NewName(newName) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + auto name = Name.Build(); + auto newName = NewName.Build(); + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + if (!name->Init(ctx, FakeSource.Get()) || + !newName->Init(ctx, FakeSource.Get()) || + !cluster->Init(ctx, FakeSource.Get())) + { + return false; + } + + auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "renameUser" : "renameGroup")))); + options = L(options, Q(Y(Q("newName"), newName))); + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), + Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + const bool IsUser; + const TString Service; + TDeferredAtom Cluster; + TDeferredAtom Name; + TDeferredAtom NewName; + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildRenameUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) { + const bool isUser = true; + return new TRenameRole(pos, isUser, service, cluster, name, newName, scoped); +} + +TNodePtr BuildRenameGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TDeferredAtom& newName, TScopedStatePtr scoped) { + const bool isUser = false; + return new TRenameRole(pos, isUser, service, cluster, name, newName, scoped); +} + +class TAlterGroup final: public TAstListNode { +public: + TAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, TScopedStatePtr scoped) + : TAstListNode(pos) + , Service(service) + , Cluster(cluster) + , Name(name) + , ToChange(toChange) + , IsDrop(isDrop) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + auto name = Name.Build(); + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + if (!name->Init(ctx, FakeSource.Get()) || !cluster->Init(ctx, FakeSource.Get())) { + return false; + } + + TVector<TNodePtr> toChange; + for (auto& item : ToChange) { + toChange.push_back(item.Build()); + if (!toChange.back()->Init(ctx, FakeSource.Get())) { + return false; + } + } + + auto options = Y(Q(Y(Q("mode"), Q(IsDrop ? "dropUsersFromGroup" : "addUsersToGroup")))); + options = L(options, Q(Y(Q("roles"), Q(new TAstListNodeImpl(Pos, std::move(toChange)))))); + + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster)), + Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + const TString Service; + TDeferredAtom Cluster; + TDeferredAtom Name; + TVector<TDeferredAtom> ToChange; + const bool IsDrop; + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildAlterGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TVector<TDeferredAtom>& toChange, bool isDrop, + TScopedStatePtr scoped) +{ + return new TAlterGroup(pos, service, cluster, name, toChange, isDrop, scoped); +} + +class TDropRoles final: public TAstListNode { +public: + TDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped) + : TAstListNode(pos) + , Service(service) + , Cluster(cluster) + , ToDrop(toDrop) + , IsUser(isUser) + , Force(force) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(service, cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + TNodePtr cluster = Scoped->WrapCluster(Cluster, ctx); + + if (!cluster->Init(ctx, FakeSource.Get())) { + return false; + } + + auto options = Y(Q(Y(Q("mode"), Q(IsUser ? "dropUser" : "dropGroup")))); + if (Force) { + options = L(options, Q(Y(Q("force")))); + } + + + auto block = Y(Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), cluster))); + for (auto& item : ToDrop) { + auto name = item.Build(); + if (!name->Init(ctx, FakeSource.Get())) { + return false; + } + + block = L(block, Y("let", "world", Y(TString(WriteName), "world", "sink", Y("Key", Q(Y(Q("role"), Y("String", name)))), Y("Void"), Q(options)))); + } + block = L(block, Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world"))); + Add("block", Q(block)); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + const TString Service; + TDeferredAtom Cluster; + TVector<TDeferredAtom> ToDrop; + const bool IsUser; + const bool Force; + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool force, TScopedStatePtr scoped) { + return new TDropRoles(pos, service, cluster, toDrop, isUser, force, scoped); +} + static const TMap<EWriteColumnMode, TString> columnModeToStrMapMR { {EWriteColumnMode::Default, ""}, {EWriteColumnMode::Insert, "append"}, @@ -1622,28 +1622,28 @@ public: } for (auto& nodes: Scoped->NamedNodes) { - if (src || ctx.Exports.contains(nodes.first)) { - auto& item = nodes.second.front(); - if (!item->Node->Init(ctx, src)) { - hasError = true; - continue; - } - - // Some constants may be used directly by YQL code and need to be translated without reference from SQL AST - if (item->Node->IsConstant() || ctx.Exports.contains(nodes.first)) { - Add(Y("let", BuildAtom(item->Node->GetPos(), nodes.first), item->Node)); - } - } - } - + if (src || ctx.Exports.contains(nodes.first)) { + auto& item = nodes.second.front(); + if (!item->Node->Init(ctx, src)) { + hasError = true; + continue; + } + + // Some constants may be used directly by YQL code and need to be translated without reference from SQL AST + if (item->Node->IsConstant() || ctx.Exports.contains(nodes.first)) { + Add(Y("let", BuildAtom(item->Node->GetPos(), nodes.first), item->Node)); + } + } + } + if (ctx.Settings.Mode != NSQLTranslation::ESqlMode::LIBRARY) { auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName))); auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName))); - for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) { + for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) { Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), - BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); + BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()), + BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction())))))); } if (ctx.ResultSizeLimit > 0) { @@ -1651,10 +1651,10 @@ public: BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit))))); } - if (!ctx.PragmaPullUpFlatMapOverJoin) { + if (!ctx.PragmaPullUpFlatMapOverJoin) { Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin")))); - } + BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin")))); + } if (ctx.DiscoveryMode) { Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, @@ -1680,11 +1680,11 @@ public: Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName)))); } - - if (ctx.OrderedColumns) { - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, - BuildQuotedAtom(Pos, "OrderedColumns")))); - } + + if (ctx.OrderedColumns) { + Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + BuildQuotedAtom(Pos, "OrderedColumns")))); + } if (ctx.PqReadByRtmrCluster) { auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all")); @@ -1718,7 +1718,7 @@ public: for (const auto& x : Scoped->Local.ExprClusters) { auto& data = Scoped->Local.ExprClustersMap[x.Get()]; auto& node = data.second; - + if (!node->Init(ctx, nullptr)) { hasError = true; continue; @@ -1738,8 +1738,8 @@ public: Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1")))); } } else { - const auto& ref = block->GetLabel(); - Add(Y("let", ref ? ref : "world", block)); + const auto& ref = block->GetLabel(); + Add(Y("let", ref ? ref : "world", block)); } } @@ -1864,7 +1864,7 @@ TNodePtr BuildPragma(TPosition pos, const TString& prefix, const TString& name, class TSqlLambda final: public TAstListNode { public: - TSqlLambda(TPosition pos, TVector<TString>&& args, TVector<TNodePtr>&& exprSeq) + TSqlLambda(TPosition pos, TVector<TString>&& args, TVector<TNodePtr>&& exprSeq) : TAstListNode(pos) , Args(args) , ExprSeq(exprSeq) @@ -1891,14 +1891,14 @@ public: body = Y("block", Q(L(body, Y("return", *end)))); auto args = Y(); for (const auto& arg: Args) { - args = L(args, BuildAtom(GetPos(), arg)); + args = L(args, BuildAtom(GetPos(), arg)); } Add("lambda", Q(args), body); return TAstListNode::DoInit(ctx, src); } TPtr DoClone() const final { - return {}; + return {}; } void DoUpdateState() const override { @@ -1933,20 +1933,20 @@ public: } Add(IsEvaluate ? "EvaluateIf!" : "If!"); Add("world"); - auto coalesced = Y("Coalesce", Predicate, Y("Bool", Q("false"))); + auto coalesced = Y("Coalesce", Predicate, Y("Bool", Q("false"))); Add(IsEvaluate ? Y("EvaluateExpr", Y("EnsureType", coalesced, Y("DataType", Q("Bool")))) : coalesced); if (!ThenNode->Init(ctx, FakeSource.Get())) { return{}; } - Add(ThenNode); + Add(ThenNode); if (ElseNode) { if (!ElseNode->Init(ctx, FakeSource.Get())) { return{}; } - Add(ElseNode); + Add(ElseNode); } return TAstListNode::DoInit(ctx, src); @@ -1986,18 +1986,18 @@ public: } Add(IsEvaluate ? "EvaluateFor!" : "For!"); Add("world"); - Add(IsEvaluate ? Y("EvaluateExpr", List) : List); - + Add(IsEvaluate ? Y("EvaluateExpr", List) : List); + if (!BodyNode->Init(ctx, FakeSource.Get())) { return{}; } - Add(BodyNode); + Add(BodyNode); if (ElseNode) { if (!ElseNode->Init(ctx, FakeSource.Get())) { return{}; } - Add(ElseNode); + Add(ElseNode); } return TAstListNode::DoInit(ctx, src); diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp index 78b85f5a41..2841f05a5b 100644 --- a/ydb/library/yql/sql/v1/select.cpp +++ b/ydb/library/yql/sql/v1/select.cpp @@ -119,11 +119,11 @@ public: if (IsSubquery()) { /// should be not used? auto columnsPtr = Source->GetColumns(); - if (columnsPtr && (columnsPtr->All || columnsPtr->QualifiedAll)) { - Node = Y("SingleMember", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0")))); - } else if (columnsPtr && columnsPtr->List.size() == 1) { - Node = Y("Member", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0"))), Q(columnsPtr->List.front())); - } else { + if (columnsPtr && (columnsPtr->All || columnsPtr->QualifiedAll)) { + Node = Y("SingleMember", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0")))); + } else if (columnsPtr && columnsPtr->List.size() == 1) { + Node = Y("Member", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0"))), Q(columnsPtr->List.front())); + } else { ctx.Error(Pos) << "Source used in expression should contain one concrete column"; return false; } @@ -161,76 +161,76 @@ TNodePtr BuildSourceNode(TPosition pos, TSourcePtr source, bool checkExist) { class TFakeSource: public ISource { public: - TFakeSource(TPosition pos, bool missingFrom) + TFakeSource(TPosition pos, bool missingFrom) : ISource(pos) - , MissingFrom(missingFrom) + , MissingFrom(missingFrom) {} bool IsFake() const override { return true; } - TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) override { - // TODO: fix column reference scope - with proper scopes error below should happen earlier - if (column.CanBeType()) { - return true; - } - ctx.Error(Pos) << (MissingFrom ? "Column references are not allowed without FROM" : "Source does not allow column references"); - ctx.Error(column.GetPos()) << "Column reference " - << (column.GetColumnName() ? "'" + *column.GetColumnName() + "'" : "(expr)"); - return {}; - } - + TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) override { + // TODO: fix column reference scope - with proper scopes error below should happen earlier + if (column.CanBeType()) { + return true; + } + ctx.Error(Pos) << (MissingFrom ? "Column references are not allowed without FROM" : "Source does not allow column references"); + ctx.Error(column.GetPos()) << "Column reference " + << (column.GetColumnName() ? "'" + *column.GetColumnName() + "'" : "(expr)"); + return {}; + } + bool AddFilter(TContext& ctx, TNodePtr filter) override { Y_UNUSED(filter); - auto pos = filter ? filter->GetPos() : Pos; - ctx.Error(pos) << (MissingFrom ? "Filtering is not allowed without FROM" : "Source does not allow filtering"); + auto pos = filter ? filter->GetPos() : Pos; + ctx.Error(pos) << (MissingFrom ? "Filtering is not allowed without FROM" : "Source does not allow filtering"); return false; } TNodePtr Build(TContext& ctx) override { Y_UNUSED(ctx); - return Y("AsList", Y("AsStruct")); + return Y("AsList", Y("AsStruct")); } bool AddGroupKey(TContext& ctx, const TString& column) override { Y_UNUSED(column); - ctx.Error(Pos) << "Grouping is not allowed " << (MissingFrom ? "without FROM" : "in this context"); + ctx.Error(Pos) << "Grouping is not allowed " << (MissingFrom ? "without FROM" : "in this context"); return false; } bool AddAggregation(TContext& ctx, TAggregationPtr aggr) override { - YQL_ENSURE(aggr); - ctx.Error(aggr->GetPos()) << "Aggregation is not allowed " << (MissingFrom ? "without FROM" : "in this context"); + YQL_ENSURE(aggr); + ctx.Error(aggr->GetPos()) << "Aggregation is not allowed " << (MissingFrom ? "without FROM" : "in this context"); return false; } - bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func) override { - Y_UNUSED(windowName); - YQL_ENSURE(func); - ctx.Error(func->GetPos()) << "Aggregation is not allowed " << (MissingFrom ? "without FROM" : "in this context"); - return false; - } - - bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func) override { - Y_UNUSED(windowName); - YQL_ENSURE(func); - ctx.Error(func->GetPos()) << "Window functions are not allowed " << (MissingFrom ? "without FROM" : "in this context"); - return false; - } - - TWindowSpecificationPtr FindWindowSpecification(TContext& ctx, const TString& windowName) const override { - Y_UNUSED(windowName); - ctx.Error(Pos) << "Window and aggregation functions are not allowed " << (MissingFrom ? "without FROM" : "in this context"); - return {}; - } - + bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func) override { + Y_UNUSED(windowName); + YQL_ENSURE(func); + ctx.Error(func->GetPos()) << "Aggregation is not allowed " << (MissingFrom ? "without FROM" : "in this context"); + return false; + } + + bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func) override { + Y_UNUSED(windowName); + YQL_ENSURE(func); + ctx.Error(func->GetPos()) << "Window functions are not allowed " << (MissingFrom ? "without FROM" : "in this context"); + return false; + } + + TWindowSpecificationPtr FindWindowSpecification(TContext& ctx, const TString& windowName) const override { + Y_UNUSED(windowName); + ctx.Error(Pos) << "Window and aggregation functions are not allowed " << (MissingFrom ? "without FROM" : "in this context"); + return {}; + } + bool IsGroupByColumn(const TString& column) const override { Y_UNUSED(column); return false; } - TNodePtr BuildFilter(TContext& ctx, const TString& label) override { + TNodePtr BuildFilter(TContext& ctx, const TString& label) override { Y_UNUSED(ctx); Y_UNUSED(label); return nullptr; @@ -242,22 +242,22 @@ public: } TPtr DoClone() const final { - return new TFakeSource(Pos, MissingFrom); + return new TFakeSource(Pos, MissingFrom); } -private: - const bool MissingFrom; +private: + const bool MissingFrom; }; -TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom) { - return new TFakeSource(pos, missingFrom); +TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom) { + return new TFakeSource(pos, missingFrom); } class TNodeSource: public ISource { public: - TNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList) + TNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList) : ISource(pos) , Node(node) - , WrapToList(wrapToList) + , WrapToList(wrapToList) { YQL_ENSURE(Node); FakeSource = BuildFakeSource(pos); @@ -267,10 +267,10 @@ public: UseAllColumns = true; } - bool ShouldUseSourceAsColumn(const TString& source) const final { - return source && source != GetLabel(); - } - + bool ShouldUseSourceAsColumn(const TString& source) const final { + return source && source != GetLabel(); + } + TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) final { Y_UNUSED(ctx); if (UseAllColumns) { @@ -281,7 +281,7 @@ public: AllColumns(); } else { if (column.GetColumnName()) { - Columns.insert(*column.GetColumnName()); + Columns.insert(*column.GetColumnName()); } else { AllColumns(); } @@ -290,18 +290,18 @@ public: return true; } - bool DoInit(TContext& ctx, ISource* src) final { - if (!Node->Init(ctx, FakeSource.Get())) { - return false; - } - return ISource::DoInit(ctx, src); - } - + bool DoInit(TContext& ctx, ISource* src) final { + if (!Node->Init(ctx, FakeSource.Get())) { + return false; + } + return ISource::DoInit(ctx, src); + } + TNodePtr Build(TContext& ctx) final { - auto nodeAst = AstNode(Node); - if (WrapToList) { - nodeAst = Y("ToList", nodeAst); - } + auto nodeAst = AstNode(Node); + if (WrapToList) { + nodeAst = Y("ToList", nodeAst); + } if (UseAllColumns) { return nodeAst; @@ -316,19 +316,19 @@ public: } TPtr DoClone() const final { - return new TNodeSource(Pos, SafeClone(Node), WrapToList); + return new TNodeSource(Pos, SafeClone(Node), WrapToList); } private: TNodePtr Node; - bool WrapToList; + bool WrapToList; TSourcePtr FakeSource; - TSet<TString> Columns; + TSet<TString> Columns; bool UseAllColumns = false; }; -TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList) { - return new TNodeSource(pos, node, wrapToList); +TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList) { + return new TNodeSource(pos, node, wrapToList); } class IProxySource: public ISource { @@ -362,7 +362,7 @@ protected: return ret; } - bool ShouldUseSourceAsColumn(const TString& source) const override { + bool ShouldUseSourceAsColumn(const TString& source) const override { return Source->ShouldUseSourceAsColumn(source); } @@ -416,7 +416,7 @@ protected: return true; } const auto* name = column.GetColumnName(); - if (name && !column.CanBeType() && !Columns.IsColumnPossible(ctx, *name) && !IsAlias(EExprSeat::GroupBy, *name) && !IsAlias(EExprSeat::DistinctAggr, *name)) { + if (name && !column.CanBeType() && !Columns.IsColumnPossible(ctx, *name) && !IsAlias(EExprSeat::GroupBy, *name) && !IsAlias(EExprSeat::DistinctAggr, *name)) { if (column.IsReliable()) { TStringBuilder sb; sb << "Column " << *name << " is not in source column set"; @@ -514,7 +514,7 @@ public: } else { block = Y(Y("let", ref, input)); } - auto filter = source->BuildFilter(ctx, ref); + auto filter = source->BuildFilter(ctx, ref); if (filter) { block = L(block, Y("let", ref, filter)); } @@ -572,14 +572,14 @@ public: if (!Node->Init(ctx, src)) { return false; } - if (src && Subquery->GetSource()->IsSelect()) { + if (src && Subquery->GetSource()->IsSelect()) { auto columnsPtr = &Columns; - if (columnsPtr && (columnsPtr->All || columnsPtr->QualifiedAll)) { - Node = Y("SingleMember", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0")))); - } else if (columnsPtr && columnsPtr->List.size() == 1) { - Node = Y("Member", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0"))), Q(columnsPtr->List.front())); - } else { - ctx.Error(Pos) << "Source used in expression should contain one concrete column"; + if (columnsPtr && (columnsPtr->All || columnsPtr->QualifiedAll)) { + Node = Y("SingleMember", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0")))); + } else if (columnsPtr && columnsPtr->List.size() == 1) { + Node = Y("Member", Y("SqlAccess", Q("dict"), Y("Take", Node, Y("Uint64", Q("1"))), Y("Uint64", Q("0"))), Q(columnsPtr->List.front())); + } else { + ctx.Error(Pos) << "Source used in expression should contain one concrete column"; return false; } } @@ -611,7 +611,7 @@ public: ctx.Error(pos) << "'Repeatable' keyword is not supported for subqueries"; return false; } - return SetSamplingRate(ctx, samplingRate); + return SetSamplingRate(ctx, samplingRate); } bool IsStream() const override { @@ -647,7 +647,7 @@ public: TTableSource(TPosition pos, const TTableRef& table, const TString& label) : IRealSource(pos) , Table(table) - , FakeSource(BuildFakeSource(pos)) + , FakeSource(BuildFakeSource(pos)) { SetLabel(label.empty() ? Table.ShortName() : label); } @@ -657,7 +657,7 @@ public: ISource::GetInputTables(tableList); } - bool ShouldUseSourceAsColumn(const TString& source) const override { + bool ShouldUseSourceAsColumn(const TString& source) const override { return source && source != GetLabel(); } @@ -674,8 +674,8 @@ public: TPosition pos, ESampleMode mode, TNodePtr samplingRate, - TNodePtr samplingSeed) override - { + TNodePtr samplingSeed) override + { TString modeName; if (!samplingSeed) { samplingSeed = Y("Int32", Q("0")); @@ -694,19 +694,19 @@ public: break; } - if (!samplingRate->Init(ctx, FakeSource.Get())) { - return false; - } - - auto ensureLow = Y("Ensure", "samplingRate", Y(">=", "samplingRate", Y("Double", Q("0"))), Y("String", BuildQuotedAtom(pos, "Expected sampling rate to be nonnegative"))); - auto ensureHigh = Y("Ensure", "samplingRate", Y("<=", "samplingRate", Y("Double", Q("100"))), Y("String", BuildQuotedAtom(pos, "Sampling rate is over 100%"))); - - auto block(Y(Y("let", "samplingRate", samplingRate))); - block = L(block, Y("let", "samplingRate", ensureLow)); - block = L(block, Y("let", "samplingRate", ensureHigh)); - - samplingRate = Y("block", Q(L(block, Y("return", "samplingRate")))); - + if (!samplingRate->Init(ctx, FakeSource.Get())) { + return false; + } + + auto ensureLow = Y("Ensure", "samplingRate", Y(">=", "samplingRate", Y("Double", Q("0"))), Y("String", BuildQuotedAtom(pos, "Expected sampling rate to be nonnegative"))); + auto ensureHigh = Y("Ensure", "samplingRate", Y("<=", "samplingRate", Y("Double", Q("100"))), Y("String", BuildQuotedAtom(pos, "Sampling rate is over 100%"))); + + auto block(Y(Y("let", "samplingRate", samplingRate))); + block = L(block, Y("let", "samplingRate", ensureLow)); + block = L(block, Y("let", "samplingRate", ensureHigh)); + + samplingRate = Y("block", Q(L(block, Y("return", "samplingRate")))); + auto sampleSettings = Q(Y(Q(modeName), Y("EvaluateAtom", Y("ToString", samplingRate)), Y("EvaluateAtom", Y("ToString", samplingSeed)))); auto sampleOption = Q(Y(Q("sample"), sampleSettings)); if (Table.Options) { @@ -722,9 +722,9 @@ public: bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints) override { Y_UNUSED(ctx); - TTableHints merged = contextHints; - MergeHints(merged, hints); - Table.Options = BuildInputOptions(pos, merged); + TTableHints merged = contextHints; + MergeHints(merged, hints); + Table.Options = BuildInputOptions(pos, merged); return true; } @@ -752,8 +752,8 @@ public: } protected: TTableRef Table; -private: - const TSourcePtr FakeSource; +private: + const TSourcePtr FakeSource; }; TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label) { @@ -780,11 +780,11 @@ public: return true; } - bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints) override { + bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints) override { Y_UNUSED(ctx); HintsPos = pos; Hints = hints; - ContextHints = contextHints; + ContextHints = contextHints; return true; } @@ -795,7 +795,7 @@ public: return true; } - bool ShouldUseSourceAsColumn(const TString& source) const override { + bool ShouldUseSourceAsColumn(const TString& source) const override { return source && source != GetLabel(); } @@ -808,8 +808,8 @@ public: return IProxySource::AddColumn(ctx, column); } - bool DoInit(TContext& ctx, ISource* initSrc) override { - Y_UNUSED(initSrc); + bool DoInit(TContext& ctx, ISource* initSrc) override { + Y_UNUSED(initSrc); auto source = Node->GetSource(); if (!source) { NewSource = TryMakeSourceFromExpression(ctx, Service, Cluster, Node); @@ -834,7 +834,7 @@ public: } if (HintsPos) { - if (!source->SetTableHints(ctx, *HintsPos, Hints, ContextHints)) { + if (!source->SetTableHints(ctx, *HintsPos, Hints, ContextHints)) { return false; } } @@ -842,13 +842,13 @@ public: source->SetLabel(Label); if (!NewSource) { Node->UseAsInner(); - if (!Node->Init(ctx, nullptr)) { + if (!Node->Init(ctx, nullptr)) { return false; } } SetSource(source); - if (NewSource && !NewSource->Init(ctx, nullptr)) { + if (NewSource && !NewSource->Init(ctx, nullptr)) { return false; } @@ -891,7 +891,7 @@ private: TMaybe<TPosition> HintsPos; TTableHints Hints; - TTableHints ContextHints; + TTableHints ContextHints; }; TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& service, const TDeferredAtom& cluster, const TString& label) { @@ -912,7 +912,7 @@ static bool IsComparableExpression(TContext& ctx, const TNodePtr& expr, bool ass ctx.Error(expr->GetPos()) << "Unable to " << sqlConstruction << " aggregated values"; return false; } - if (expr->GetColumnName()) { + if (expr->GetColumnName()) { return true; } if (expr->GetOpName().empty()) { @@ -999,10 +999,10 @@ public: } } - if (!Udf->Init(ctx, src)) { - return false; - } - + if (!Udf->Init(ctx, src)) { + return false; + } + if (Udf->GetLabel().empty()) { Columns.SetAll(); } else { @@ -1052,7 +1052,7 @@ public: } TNodePtr expr = BuildAtom(Pos, "partitionStream"); - processPartitions = Y("SqlReduce", "partitionStream", BuildQuotedAtom(Pos, "byAllList", TNodeFlags::Default), Udf, expr); + processPartitions = Y("SqlReduce", "partitionStream", BuildQuotedAtom(Pos, "byAllList", TNodeFlags::Default), Udf, expr); } else { switch (Mode) { case ReduceMode::ByAll: { @@ -1061,14 +1061,14 @@ public: if (!columnPtr || *columnPtr != "*") { expr = Y("Map", "partitionStream", BuildLambda(Pos, Y("keyPair"), Q(L(Y(),\ Y("Nth", "keyPair", Q(ToString("0"))),\ - Y("Map", Y("Nth", "keyPair", Q(ToString("1"))), BuildLambda(Pos, Y("row"), Args[0])))))); + Y("Map", Y("Nth", "keyPair", Q(ToString("1"))), BuildLambda(Pos, Y("row"), Args[0])))))); } - processPartitions = Y("SqlReduce", "partitionStream", BuildQuotedAtom(Pos, "byAll", TNodeFlags::Default), Udf, expr); + processPartitions = Y("SqlReduce", "partitionStream", BuildQuotedAtom(Pos, "byAll", TNodeFlags::Default), Udf, expr); break; } case ReduceMode::ByPartition: { - processPartitions = Y("SqlReduce", "partitionStream", extractKeyLambda, Udf, - BuildLambda(Pos, Y("row"), Args[0])); + processPartitions = Y("SqlReduce", "partitionStream", extractKeyLambda, Udf, + BuildLambda(Pos, Y("row"), Args[0])); break; } default: @@ -1077,7 +1077,7 @@ public: } TNodePtr sortDirection; - TNodePtr sortKeySelector; + TNodePtr sortKeySelector; FillSortParts(OrderBy, sortDirection, sortKeySelector); if (!OrderBy.empty()) { sortKeySelector = BuildLambda(Pos, Y("row"), Y("SqlExtractKey", "row", sortKeySelector)); @@ -1088,7 +1088,7 @@ public: auto inputLabel = ListCall ? "inputRowsList" : "core"; auto block(Y(Y("let", inputLabel, input))); - auto filter = Source->BuildFilter(ctx, inputLabel); + auto filter = Source->BuildFilter(ctx, inputLabel); if (filter) { block = L(block, Y("let", inputLabel, filter)); } @@ -1102,7 +1102,7 @@ public: block = L(block, Y("let", "core", Y("AutoDemuxList", partitionByKey))); if (Having) { block = L(block, Y("let", "core", - Y("Filter", "core", BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false"))))) + Y("Filter", "core", BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false"))))) )); } return Y("block", Q(L(block, Y("return", "core")))); @@ -1114,7 +1114,7 @@ public: return nullptr; } - return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true)); + return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true)); } EOrderKind GetOrderKind() const override { @@ -1158,55 +1158,55 @@ TSourcePtr BuildReduce(TPosition pos, std::move(args), udf, having, settings, assumeOrderBy, listCall); } -namespace { - -bool InitAndGetGroupKey(TContext& ctx, const TNodePtr& expr, ISource* src, TStringBuf where, TString& keyColumn) { - keyColumn.clear(); - - YQL_ENSURE(src); - const bool isJoin = src->GetJoin(); - - if (!expr->Init(ctx, src)) { - return false; - } - - auto keyNamePtr = expr->GetColumnName(); - if (keyNamePtr && expr->GetLabel().empty()) { - keyColumn = *keyNamePtr; - auto sourceNamePtr = expr->GetSourceName(); - auto columnNode = dynamic_cast<TColumnNode*>(expr.Get()); - if (isJoin && (!columnNode || !columnNode->IsArtificial())) { - if (!sourceNamePtr || sourceNamePtr->empty()) { - if (!src->IsAlias(EExprSeat::GroupBy, keyColumn)) { - ctx.Error(expr->GetPos()) << "Columns in " << where << " should have correlation name, error in key: " << keyColumn; - return false; - } - } else { - keyColumn = DotJoin(*sourceNamePtr, keyColumn); - } - } - } - - return true; -} - -} - +namespace { + +bool InitAndGetGroupKey(TContext& ctx, const TNodePtr& expr, ISource* src, TStringBuf where, TString& keyColumn) { + keyColumn.clear(); + + YQL_ENSURE(src); + const bool isJoin = src->GetJoin(); + + if (!expr->Init(ctx, src)) { + return false; + } + + auto keyNamePtr = expr->GetColumnName(); + if (keyNamePtr && expr->GetLabel().empty()) { + keyColumn = *keyNamePtr; + auto sourceNamePtr = expr->GetSourceName(); + auto columnNode = dynamic_cast<TColumnNode*>(expr.Get()); + if (isJoin && (!columnNode || !columnNode->IsArtificial())) { + if (!sourceNamePtr || sourceNamePtr->empty()) { + if (!src->IsAlias(EExprSeat::GroupBy, keyColumn)) { + ctx.Error(expr->GetPos()) << "Columns in " << where << " should have correlation name, error in key: " << keyColumn; + return false; + } + } else { + keyColumn = DotJoin(*sourceNamePtr, keyColumn); + } + } + } + + return true; +} + +} + class TCompositeSelect: public IRealSource { public: - TCompositeSelect(TPosition pos, TSourcePtr source, TSourcePtr originalSource, const TWriteSettings& settings) + TCompositeSelect(TPosition pos, TSourcePtr source, TSourcePtr originalSource, const TWriteSettings& settings) : IRealSource(pos) , Source(std::move(source)) - , OriginalSource(std::move(originalSource)) + , OriginalSource(std::move(originalSource)) , Settings(settings) { YQL_ENSURE(Source); } - void SetSubselects(TVector<TSourcePtr>&& subselects, TVector<TNodePtr>&& grouping, TVector<TNodePtr>&& groupByExpr) { + void SetSubselects(TVector<TSourcePtr>&& subselects, TVector<TNodePtr>&& grouping, TVector<TNodePtr>&& groupByExpr) { Subselects = std::move(subselects); - Grouping = std::move(grouping); - GroupByExpr = std::move(groupByExpr); + Grouping = std::move(grouping); + GroupByExpr = std::move(groupByExpr); Y_VERIFY_DEBUG(Subselects.size() > 1); } @@ -1231,37 +1231,37 @@ public: if (!Source->InitFilters(ctx)) { return false; } - - if (!CalculateGroupingCols(ctx, src)) { - return false; - } - - auto origSrc = OriginalSource.Get(); - if (!origSrc->Init(ctx, src)) { - return false; - } - - if (origSrc->IsFlattenByColumns() || origSrc->IsFlattenColumns()) { - Flatten = origSrc->IsFlattenByColumns() ? - origSrc->BuildFlattenByColumns("row") : - origSrc->BuildFlattenColumns("row"); - if (!Flatten || !Flatten->Init(ctx, src)) { - return false; - } - } - - if (origSrc->IsFlattenByExprs()) { - for (auto& expr : static_cast<ISource const*>(origSrc)->Expressions(EExprSeat::FlattenByExpr)) { - if (!expr->Init(ctx, origSrc)) { - return false; - } - } - PreFlattenMap = origSrc->BuildPreFlattenMap(ctx); - if (!PreFlattenMap) { - return false; - } - } - + + if (!CalculateGroupingCols(ctx, src)) { + return false; + } + + auto origSrc = OriginalSource.Get(); + if (!origSrc->Init(ctx, src)) { + return false; + } + + if (origSrc->IsFlattenByColumns() || origSrc->IsFlattenColumns()) { + Flatten = origSrc->IsFlattenByColumns() ? + origSrc->BuildFlattenByColumns("row") : + origSrc->BuildFlattenColumns("row"); + if (!Flatten || !Flatten->Init(ctx, src)) { + return false; + } + } + + if (origSrc->IsFlattenByExprs()) { + for (auto& expr : static_cast<ISource const*>(origSrc)->Expressions(EExprSeat::FlattenByExpr)) { + if (!expr->Init(ctx, origSrc)) { + return false; + } + } + PreFlattenMap = origSrc->BuildPreFlattenMap(ctx); + if (!PreFlattenMap) { + return false; + } + } + for (const auto& select: Subselects) { select->SetLabel(Label); if (AsInner) { @@ -1287,15 +1287,15 @@ public: TNodePtr Build(TContext& ctx) override { auto input = Source->Build(ctx); auto block(Y(Y("let", "composite", input))); - - bool ordered = ctx.UseUnordered(*this); - if (PreFlattenMap) { - block = L(block, Y("let", "composite", Y(ordered ? "OrderedFlatMap" : "FlatMap", "composite", BuildLambda(Pos, Y("row"), PreFlattenMap)))); - } - if (Flatten) { - block = L(block, Y("let", "composite", Y(ordered ? "OrderedFlatMap" : "FlatMap", "composite", BuildLambda(Pos, Y("row"), Flatten, "res")))); - } - auto filter = Source->BuildFilter(ctx, "composite"); + + bool ordered = ctx.UseUnordered(*this); + if (PreFlattenMap) { + block = L(block, Y("let", "composite", Y(ordered ? "OrderedFlatMap" : "FlatMap", "composite", BuildLambda(Pos, Y("row"), PreFlattenMap)))); + } + if (Flatten) { + block = L(block, Y("let", "composite", Y(ordered ? "OrderedFlatMap" : "FlatMap", "composite", BuildLambda(Pos, Y("row"), Flatten, "res")))); + } + auto filter = Source->BuildFilter(ctx, "composite"); if (filter) { block = L(block, Y("let", "composite", filter)); } @@ -1313,7 +1313,7 @@ public: } bool IsGroupByColumn(const TString& column) const override { - YQL_ENSURE(!GroupingCols.empty()); + YQL_ENSURE(!GroupingCols.empty()); return GroupingCols.contains(column); } @@ -1342,48 +1342,48 @@ public: } TNodePtr DoClone() const final { - auto newSource = MakeIntrusive<TCompositeSelect>(Pos, Source->CloneSource(), OriginalSource->CloneSource(), Settings); - newSource->SetSubselects(CloneContainer(Subselects), CloneContainer(Grouping), CloneContainer(GroupByExpr)); + auto newSource = MakeIntrusive<TCompositeSelect>(Pos, Source->CloneSource(), OriginalSource->CloneSource(), Settings); + newSource->SetSubselects(CloneContainer(Subselects), CloneContainer(Grouping), CloneContainer(GroupByExpr)); return newSource; } private: - bool CalculateGroupingCols(TContext& ctx, ISource* initSrc) { - auto origSrc = OriginalSource->CloneSource(); - if (!origSrc->Init(ctx, initSrc)) { - return false; - } - - bool hasError = false; - for (auto& expr: GroupByExpr) { - if (!expr->Init(ctx, origSrc.Get()) || !IsComparableExpression(ctx, expr, false, "GROUP BY")) { - hasError = true; - } - } - if (!origSrc->AddExpressions(ctx, GroupByExpr, EExprSeat::GroupBy)) { - hasError = true; - } - - YQL_ENSURE(!Grouping.empty()); - for (auto& grouping : Grouping) { - TString keyColumn; - if (!InitAndGetGroupKey(ctx, grouping, origSrc.Get(), "grouping sets", keyColumn)) { - hasError = true; - } else if (!keyColumn.empty()) { - GroupingCols.insert(keyColumn); - } - } - - return !hasError; - } - + bool CalculateGroupingCols(TContext& ctx, ISource* initSrc) { + auto origSrc = OriginalSource->CloneSource(); + if (!origSrc->Init(ctx, initSrc)) { + return false; + } + + bool hasError = false; + for (auto& expr: GroupByExpr) { + if (!expr->Init(ctx, origSrc.Get()) || !IsComparableExpression(ctx, expr, false, "GROUP BY")) { + hasError = true; + } + } + if (!origSrc->AddExpressions(ctx, GroupByExpr, EExprSeat::GroupBy)) { + hasError = true; + } + + YQL_ENSURE(!Grouping.empty()); + for (auto& grouping : Grouping) { + TString keyColumn; + if (!InitAndGetGroupKey(ctx, grouping, origSrc.Get(), "grouping sets", keyColumn)) { + hasError = true; + } else if (!keyColumn.empty()) { + GroupingCols.insert(keyColumn); + } + } + + return !hasError; + } + TSourcePtr Source; - TSourcePtr OriginalSource; - TNodePtr Flatten; - TNodePtr PreFlattenMap; + TSourcePtr OriginalSource; + TNodePtr Flatten; + TNodePtr PreFlattenMap; const TWriteSettings Settings; TVector<TSourcePtr> Subselects; - TVector<TNodePtr> Grouping; - TVector<TNodePtr> GroupByExpr; + TVector<TNodePtr> Grouping; + TVector<TNodePtr> GroupByExpr; TSet<TString> GroupingCols; }; @@ -1451,92 +1451,92 @@ public: auto src = Source.Get(); bool hasError = false; - - if (src->IsFlattenByExprs()) { - for (auto& expr : static_cast<ISource const*>(src)->Expressions(EExprSeat::FlattenByExpr)) { - if (!expr->Init(ctx, src)) { - hasError = true; - continue; - } - } - } - - if (hasError) { - return false; - } - + + if (src->IsFlattenByExprs()) { + for (auto& expr : static_cast<ISource const*>(src)->Expressions(EExprSeat::FlattenByExpr)) { + if (!expr->Init(ctx, src)) { + hasError = true; + continue; + } + } + } + + if (hasError) { + return false; + } + src->SetCompactGroupBy(CompactGroupBy); - - for (auto& term: Terms) { - term->CollectPreaggregateExprs(ctx, *src, DistinctAggrExpr); - } - - if (Having) { - Having->CollectPreaggregateExprs(ctx, *src, DistinctAggrExpr); - } - + + for (auto& term: Terms) { + term->CollectPreaggregateExprs(ctx, *src, DistinctAggrExpr); + } + + if (Having) { + Having->CollectPreaggregateExprs(ctx, *src, DistinctAggrExpr); + } + for (auto& expr: GroupByExpr) { - if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { + if (auto sessionWindow = dynamic_cast<TSessionWindow*>(expr.Get())) { if (Source->IsStream()) { ctx.Error(Pos) << "SessionWindow is unsupported for streaming sources"; - return false; + return false; + } + sessionWindow->MarkValid(); + } + + // need to collect and Init() preaggregated exprs before calling Init() on GROUP BY expression + TVector<TNodePtr> distinctAggrsInGroupBy; + expr->CollectPreaggregateExprs(ctx, *src, distinctAggrsInGroupBy); + for (auto& distinct : distinctAggrsInGroupBy) { + if (!distinct->Init(ctx, src)) { + return false; } - sessionWindow->MarkValid(); - } - - // need to collect and Init() preaggregated exprs before calling Init() on GROUP BY expression - TVector<TNodePtr> distinctAggrsInGroupBy; - expr->CollectPreaggregateExprs(ctx, *src, distinctAggrsInGroupBy); - for (auto& distinct : distinctAggrsInGroupBy) { - if (!distinct->Init(ctx, src)) { - return false; - } - } - DistinctAggrExpr.insert(DistinctAggrExpr.end(), distinctAggrsInGroupBy.begin(), distinctAggrsInGroupBy.end()); - + } + DistinctAggrExpr.insert(DistinctAggrExpr.end(), distinctAggrsInGroupBy.begin(), distinctAggrsInGroupBy.end()); + if (!expr->Init(ctx, src) || !IsComparableExpression(ctx, expr, false, "GROUP BY")) { hasError = true; } } - if (hasError || !src->AddExpressions(ctx, GroupByExpr, EExprSeat::GroupBy)) { - return false; + if (hasError || !src->AddExpressions(ctx, GroupByExpr, EExprSeat::GroupBy)) { + return false; + } + + for (auto& expr: DistinctAggrExpr) { + if (!expr->Init(ctx, src)) { + hasError = true; + } + } + if (hasError || !src->AddExpressions(ctx, DistinctAggrExpr, EExprSeat::DistinctAggr)) { + return false; } - - for (auto& expr: DistinctAggrExpr) { - if (!expr->Init(ctx, src)) { - hasError = true; - } - } - if (hasError || !src->AddExpressions(ctx, DistinctAggrExpr, EExprSeat::DistinctAggr)) { - return false; - } - + /// grouped expressions are available in filters if (!Source->InitFilters(ctx)) { - return false; + return false; } - + for (auto& expr: GroupBy) { - TString usedColumn; - if (!InitAndGetGroupKey(ctx, expr, src, "GROUP BY", usedColumn)) { + TString usedColumn; + if (!InitAndGetGroupKey(ctx, expr, src, "GROUP BY", usedColumn)) { hasError = true; - } else if (usedColumn) { + } else if (usedColumn) { if (!src->AddGroupKey(ctx, usedColumn)) { hasError = true; } } } - - if (hasError) { - return false; - } - + + if (hasError) { + return false; + } + if (Having && !Having->Init(ctx, src)) { - return false; + return false; } src->AddWindowSpecs(WinSpecs); - const bool isJoin = Source->GetJoin(); + const bool isJoin = Source->GetJoin(); if (!InitSelect(ctx, src, isJoin, hasError)) { return false; } @@ -1548,50 +1548,50 @@ public: src->BuildFlattenByColumns("row") : src->BuildFlattenColumns("row"); if (!Flatten || !Flatten->Init(ctx, src)) { - return false; + return false; + } + } + + if (src->IsFlattenByExprs()) { + PreFlattenMap = src->BuildPreFlattenMap(ctx); + if (!PreFlattenMap) { + return false; } } - - if (src->IsFlattenByExprs()) { - PreFlattenMap = src->BuildPreFlattenMap(ctx); - if (!PreFlattenMap) { - return false; - } - } - - if (GroupByExpr || DistinctAggrExpr) { - PreaggregatedMap = src->BuildPreaggregatedMap(ctx); - if (!PreaggregatedMap) { - return false; + + if (GroupByExpr || DistinctAggrExpr) { + PreaggregatedMap = src->BuildPreaggregatedMap(ctx); + if (!PreaggregatedMap) { + return false; } } if (Aggregate) { if (!Aggregate->Init(ctx, src)) { - return false; + return false; } if (Having) { Aggregate = Y( "Filter", Aggregate, - BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false")))) + BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false")))) ); } } else if (Having) { - if (Distinct) { - Aggregate = Y( - "Filter", - "core", - BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false")))) - ); - ctx.Warning(Having->GetPos(), TIssuesIds::YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT) - << "The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead."; - } else { - ctx.Error(Having->GetPos()) << "HAVING with meaning GROUP BY () should be with aggregation function."; - return false; - } + if (Distinct) { + Aggregate = Y( + "Filter", + "core", + BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false")))) + ); + ctx.Warning(Having->GetPos(), TIssuesIds::YQL_HAVING_WITHOUT_AGGREGATION_IN_SELECT_DISTINCT) + << "The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead."; + } else { + ctx.Error(Having->GetPos()) << "HAVING with meaning GROUP BY () should be with aggregation function."; + return false; + } } else if (!Distinct && !GroupBy.empty()) { ctx.Error(Pos) << "No aggregations were specified"; - return false; + return false; } if (hasError) { return false; @@ -1599,14 +1599,14 @@ public: if (src->IsCalcOverWindow()) { if (src->IsExprSeat(EExprSeat::WindowPartitionBy, EExprType::WithExpression)) { - PrewindowMap = src->BuildPrewindowMap(ctx); + PrewindowMap = src->BuildPrewindowMap(ctx); if (!PrewindowMap) { - return false; + return false; } } - CalcOverWindow = src->BuildCalcOverWindow(ctx, "core"); - if (!CalcOverWindow || !CalcOverWindow->Init(ctx, src)) { - return false; + CalcOverWindow = src->BuildCalcOverWindow(ctx, "core"); + if (!CalcOverWindow || !CalcOverWindow->Init(ctx, src)) { + return false; } } @@ -1621,21 +1621,21 @@ public: bool ordered = ctx.UseUnordered(*this); auto block(Y(Y("let", "core", input))); - if (PreFlattenMap) { - block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), PreFlattenMap)))); - } + if (PreFlattenMap) { + block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), PreFlattenMap)))); + } if (Flatten) { block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Flatten, "res")))); } if (PreaggregatedMap) { - block = L(block, Y("let", "core", PreaggregatedMap)); + block = L(block, Y("let", "core", PreaggregatedMap)); if (Source->IsCompositeSource() && !Columns.QualifiedAll) { block = L(block, Y("let", "preaggregated", "core")); } } else if (Source->IsCompositeSource() && !Columns.QualifiedAll) { block = L(block, Y("let", "origcore", "core")); } - auto filter = Source->BuildFilter(ctx, "core"); + auto filter = Source->BuildFilter(ctx, "core"); if (filter) { block = L(block, Y("let", "core", filter)); } @@ -1649,13 +1649,13 @@ public: if (CalcOverWindow) { block = L(block, Y("let", "core", CalcOverWindow)); } - - block = L(block, Y("let", "core", Y("PersistableRepr", BuildSqlProject(ctx, ordered)))); - - if (Distinct) { - block = L(block, Y("let", "core", Y("PersistableRepr", Y("SqlAggregateAll", Y("RemoveSystemMembers", "core"))))); - } - + + block = L(block, Y("let", "core", Y("PersistableRepr", BuildSqlProject(ctx, ordered)))); + + if (Distinct) { + block = L(block, Y("let", "core", Y("PersistableRepr", Y("SqlAggregateAll", Y("RemoveSystemMembers", "core"))))); + } + return Y("block", Q(L(block, Y("return", "core")))); } @@ -1665,7 +1665,7 @@ public: return nullptr; } - return Y("let", label, BuildSortSpec(OrderBy, label, false, AssumeSorted)); + return Y("let", label, BuildSortSpec(OrderBy, label, false, AssumeSorted)); } TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label) override { @@ -1757,36 +1757,36 @@ public: } return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr), CloneContainer(GroupBy), CompactGroupBy, AssumeSorted, CloneContainer(OrderBy), - SafeClone(Having), newSpecs, SafeClone(HoppingWindowSpec), + SafeClone(Having), newSpecs, SafeClone(HoppingWindowSpec), CloneContainer(Terms), Distinct, Without, SelectStream, Settings); } private: bool InitSelect(TContext& ctx, ISource* src, bool isJoin, bool& hasError) { - for (auto& [name, winSpec] : WinSpecs) { - for (size_t i = 0; i < winSpec->Partitions.size(); ++i) { - auto partitionNode = winSpec->Partitions[i]; - if (auto sessionWindow = dynamic_cast<TSessionWindow*>(partitionNode.Get())) { - if (winSpec->Session) { - ctx.Error(partitionNode->GetPos()) << "Duplicate session window specification:"; - ctx.Error(winSpec->Session->GetPos()) << "Previous session window is declared here"; - hasError = true; - continue; - } - sessionWindow->MarkValid(); - winSpec->Session = partitionNode; + for (auto& [name, winSpec] : WinSpecs) { + for (size_t i = 0; i < winSpec->Partitions.size(); ++i) { + auto partitionNode = winSpec->Partitions[i]; + if (auto sessionWindow = dynamic_cast<TSessionWindow*>(partitionNode.Get())) { + if (winSpec->Session) { + ctx.Error(partitionNode->GetPos()) << "Duplicate session window specification:"; + ctx.Error(winSpec->Session->GetPos()) << "Previous session window is declared here"; + hasError = true; + continue; + } + sessionWindow->MarkValid(); + winSpec->Session = partitionNode; } - + if (!partitionNode->Init(ctx, src)) { hasError = true; continue; } if (!partitionNode->GetLabel() && !partitionNode->GetColumnName()) { - TString label = TStringBuilder() << "group_" << name << "_" << i; - partitionNode->SetLabel(label); + TString label = TStringBuilder() << "group_" << name << "_" << i; + partitionNode->SetLabel(label); } } - if (!src->AddExpressions(ctx, winSpec->Partitions, EExprSeat::WindowPartitionBy)) { + if (!src->AddExpressions(ctx, winSpec->Partitions, EExprSeat::WindowPartitionBy)) { hasError = true; } } @@ -1799,7 +1799,7 @@ private: } for (auto& term: Terms) { - if (!term->Init(ctx, src)) { + if (!term->Init(ctx, src)) { hasError = true; continue; } @@ -1816,10 +1816,10 @@ private: } else { label = TStringBuilder() << "column" << Columns.List.size(); hasName = false; - if (ctx.WarnUnnamedColumns) { - ctx.Warning(term->GetPos(), TIssuesIds::YQL_UNNAMED_COLUMN) - << "Autogenerated column name " << label << " will be used for expression"; - } + if (ctx.WarnUnnamedColumns) { + ctx.Warning(term->GetPos(), TIssuesIds::YQL_UNNAMED_COLUMN) + << "Autogenerated column name " << label << " will be used for expression"; + } } } if (!Columns.Add(&label, false, false, true, hasName)) { @@ -1829,32 +1829,32 @@ private: } } - CompositeTerms = Y(); - if (!hasError && Source->IsCompositeSource() && !Columns.All && !Columns.QualifiedAll && !Columns.List.empty()) { - auto compositeSrcPtr = static_cast<TCompositeSelect*>(Source->GetCompositeSource()); - if (compositeSrcPtr) { - const auto& groupings = compositeSrcPtr->GetGroupingCols(); - for (const auto& column: groupings) { - if (Source->IsGroupByColumn(column)) { - continue; - } - const TString tableName = (GroupByExpr || DistinctAggrExpr) ? "preaggregated" : "origcore"; - CompositeTerms = L(CompositeTerms, Y("let", "row", Y("AddMember", "row", BuildQuotedAtom(Pos, column), Y("Nothing", Y("MatchType", - Y("StructMemberType", Y("ListItemType", Y("TypeOf", tableName)), Q(column)), - Q("Optional"), Y("lambda", Q(Y("item")), "item"), Y("lambda", Q(Y("item")), Y("OptionalType", "item"))))))); - } - } - } - - for (auto iter: WinSpecs) { - auto winSpec = *iter.second; - for (auto orderSpec: winSpec.OrderBy) { - if (!orderSpec->OrderExpr->Init(ctx, src)) { - hasError = true; - } - } - } - + CompositeTerms = Y(); + if (!hasError && Source->IsCompositeSource() && !Columns.All && !Columns.QualifiedAll && !Columns.List.empty()) { + auto compositeSrcPtr = static_cast<TCompositeSelect*>(Source->GetCompositeSource()); + if (compositeSrcPtr) { + const auto& groupings = compositeSrcPtr->GetGroupingCols(); + for (const auto& column: groupings) { + if (Source->IsGroupByColumn(column)) { + continue; + } + const TString tableName = (GroupByExpr || DistinctAggrExpr) ? "preaggregated" : "origcore"; + CompositeTerms = L(CompositeTerms, Y("let", "row", Y("AddMember", "row", BuildQuotedAtom(Pos, column), Y("Nothing", Y("MatchType", + Y("StructMemberType", Y("ListItemType", Y("TypeOf", tableName)), Q(column)), + Q("Optional"), Y("lambda", Q(Y("item")), "item"), Y("lambda", Q(Y("item")), Y("OptionalType", "item"))))))); + } + } + } + + for (auto iter: WinSpecs) { + auto winSpec = *iter.second; + for (auto orderSpec: winSpec.OrderBy) { + if (!orderSpec->OrderExpr->Init(ctx, src)) { + hasError = true; + } + } + } + if (Columns.All || Columns.QualifiedAll) { Source->AllColumns(); } @@ -1872,8 +1872,8 @@ private: hasError = true; } if (!src->IsCompositeSource() && !Distinct && !Columns.All && src->HasAggregations()) { - WarnIfAliasFromSelectIsUsedInGroupBy(ctx, Terms, GroupBy, GroupByExpr); - + WarnIfAliasFromSelectIsUsedInGroupBy(ctx, Terms, GroupBy, GroupByExpr); + /// verify select aggregation compatibility TVector<TNodePtr> exprs(Terms); if (Having) { @@ -1905,175 +1905,175 @@ private: } SetLabel(label); - return !hasError; + return !hasError; } - TNodePtr PrepareJoinCoalesce(TContext& ctx, const TNodePtr& base, bool multipleQualifiedAll, const TVector<TString>& coalesceLabels) { - const bool isJoin = Source->GetJoin(); - const bool needCoalesce = isJoin && ctx.SimpleColumns && - (Columns.All || multipleQualifiedAll || ctx.CoalesceJoinKeysOnQualifiedAll); + TNodePtr PrepareJoinCoalesce(TContext& ctx, const TNodePtr& base, bool multipleQualifiedAll, const TVector<TString>& coalesceLabels) { + const bool isJoin = Source->GetJoin(); + const bool needCoalesce = isJoin && ctx.SimpleColumns && + (Columns.All || multipleQualifiedAll || ctx.CoalesceJoinKeysOnQualifiedAll); - if (!needCoalesce) { - return base; - } + if (!needCoalesce) { + return base; + } - auto terms = base; - const auto& sameKeyMap = Source->GetJoin()->GetSameKeysMap(); - if (sameKeyMap) { - terms = L(terms, Y("let", "flatSameKeys", "row")); - for (const auto& [key, sources]: sameKeyMap) { - auto coalesceKeys = Y(); - for (const auto& label : coalesceLabels) { - if (sources.contains(label)) { - coalesceKeys = L(coalesceKeys, Q(DotJoin(label, key))); + auto terms = base; + const auto& sameKeyMap = Source->GetJoin()->GetSameKeysMap(); + if (sameKeyMap) { + terms = L(terms, Y("let", "flatSameKeys", "row")); + for (const auto& [key, sources]: sameKeyMap) { + auto coalesceKeys = Y(); + for (const auto& label : coalesceLabels) { + if (sources.contains(label)) { + coalesceKeys = L(coalesceKeys, Q(DotJoin(label, key))); } } - terms = L(terms, Y("let", "flatSameKeys", Y("CoalesceMembers", "flatSameKeys", Q(coalesceKeys)))); - } - terms = L(terms, Y("let", "row", "flatSameKeys")); - } - - return terms; - } - - TNodePtr BuildSqlProject(TContext& ctx, bool ordered) { - auto sqlProjectArgs = Y(); - const bool isJoin = Source->GetJoin(); - const bool haveCompositeTerms = Source->IsCompositeSource() && !Columns.All && !Columns.QualifiedAll && !Columns.List.empty(); - - if (Columns.All) { - YQL_ENSURE(Columns.List.empty()); - auto terms = PrepareWithout(Y()); - auto options = Y(); - if (isJoin && ctx.SimpleColumns) { - terms = PrepareJoinCoalesce(ctx, terms, false, Source->GetJoin()->GetJoinLabels()); - - auto members = Y(); - for (auto& source : Source->GetJoin()->GetJoinLabels()) { - YQL_ENSURE(!source.empty()); - members = L(members, BuildQuotedAtom(Pos, source + ".")); - } - if (GroupByExpr.empty() || ctx.BogousStarInGroupByOverJoin) { - terms = L(terms, Y("let", "res", Y("DivePrefixMembers", "row", Q(members)))); - } else { - auto groupExprStruct = Y("AsStruct"); - for (auto node : GroupByExpr) { - auto label = node->GetLabel(); - YQL_ENSURE(label); - if (Source->IsGroupByColumn(label)) { - auto name = BuildQuotedAtom(Pos, label); - groupExprStruct = L(groupExprStruct, Q(Y(name, Y("Member", "row", name)))); - } - } - auto groupColumnsStruct = Y("DivePrefixMembers", "row", Q(members)); - - terms = L(terms, Y("let", "res", Y("FlattenMembers", Q(Y(BuildQuotedAtom(Pos, ""), groupExprStruct)), - Q(Y(BuildQuotedAtom(Pos, ""), groupColumnsStruct))))); - } - options = L(options, Q(Y(Q("divePrefix"), Q(members)))); - } else { - terms = L(terms, Y("let", "res", "row")); - } - sqlProjectArgs = L(sqlProjectArgs, Y("SqlProjectStarItem", "projectCoreType", BuildQuotedAtom(Pos, ""), BuildLambda(Pos, Y("row"), terms, "res"), Q(options))); - } else { - YQL_ENSURE(!Columns.List.empty()); - YQL_ENSURE(Columns.List.size() == Terms.size()); - - TVector<TString> coalesceLabels; - bool multipleQualifiedAll = false; - - if (isJoin && ctx.SimpleColumns) { - THashSet<TString> starTerms; - for (auto& term: Terms) { - if (term->IsAsterisk()) { - auto sourceName = term->GetSourceName(); - YQL_ENSURE(*sourceName && !sourceName->empty()); - YQL_ENSURE(Columns.QualifiedAll); - starTerms.insert(*sourceName); + terms = L(terms, Y("let", "flatSameKeys", Y("CoalesceMembers", "flatSameKeys", Q(coalesceKeys)))); + } + terms = L(terms, Y("let", "row", "flatSameKeys")); + } + + return terms; + } + + TNodePtr BuildSqlProject(TContext& ctx, bool ordered) { + auto sqlProjectArgs = Y(); + const bool isJoin = Source->GetJoin(); + const bool haveCompositeTerms = Source->IsCompositeSource() && !Columns.All && !Columns.QualifiedAll && !Columns.List.empty(); + + if (Columns.All) { + YQL_ENSURE(Columns.List.empty()); + auto terms = PrepareWithout(Y()); + auto options = Y(); + if (isJoin && ctx.SimpleColumns) { + terms = PrepareJoinCoalesce(ctx, terms, false, Source->GetJoin()->GetJoinLabels()); + + auto members = Y(); + for (auto& source : Source->GetJoin()->GetJoinLabels()) { + YQL_ENSURE(!source.empty()); + members = L(members, BuildQuotedAtom(Pos, source + ".")); + } + if (GroupByExpr.empty() || ctx.BogousStarInGroupByOverJoin) { + terms = L(terms, Y("let", "res", Y("DivePrefixMembers", "row", Q(members)))); + } else { + auto groupExprStruct = Y("AsStruct"); + for (auto node : GroupByExpr) { + auto label = node->GetLabel(); + YQL_ENSURE(label); + if (Source->IsGroupByColumn(label)) { + auto name = BuildQuotedAtom(Pos, label); + groupExprStruct = L(groupExprStruct, Q(Y(name, Y("Member", "row", name)))); + } } - } - - TVector<TString> matched; - TVector<TString> unmatched; - for (auto& label : Source->GetJoin()->GetJoinLabels()) { - if (starTerms.contains(label)) { - matched.push_back(label); - } else { - unmatched.push_back(label); + auto groupColumnsStruct = Y("DivePrefixMembers", "row", Q(members)); + + terms = L(terms, Y("let", "res", Y("FlattenMembers", Q(Y(BuildQuotedAtom(Pos, ""), groupExprStruct)), + Q(Y(BuildQuotedAtom(Pos, ""), groupColumnsStruct))))); + } + options = L(options, Q(Y(Q("divePrefix"), Q(members)))); + } else { + terms = L(terms, Y("let", "res", "row")); + } + sqlProjectArgs = L(sqlProjectArgs, Y("SqlProjectStarItem", "projectCoreType", BuildQuotedAtom(Pos, ""), BuildLambda(Pos, Y("row"), terms, "res"), Q(options))); + } else { + YQL_ENSURE(!Columns.List.empty()); + YQL_ENSURE(Columns.List.size() == Terms.size()); + + TVector<TString> coalesceLabels; + bool multipleQualifiedAll = false; + + if (isJoin && ctx.SimpleColumns) { + THashSet<TString> starTerms; + for (auto& term: Terms) { + if (term->IsAsterisk()) { + auto sourceName = term->GetSourceName(); + YQL_ENSURE(*sourceName && !sourceName->empty()); + YQL_ENSURE(Columns.QualifiedAll); + starTerms.insert(*sourceName); } - } - - coalesceLabels.insert(coalesceLabels.end(), matched.begin(), matched.end()); - coalesceLabels.insert(coalesceLabels.end(), unmatched.begin(), unmatched.end()); - - multipleQualifiedAll = starTerms.size() > 1; - } - - auto column = Columns.List.begin(); - for (auto& term: Terms) { - auto sourceName = term->GetSourceName(); - if (!term->IsAsterisk()) { - auto body = Y(); - if (haveCompositeTerms) { - body = L(body, Y("let", "row", Y("Apply", "addCompositTerms", "row"))); - } - body = L(body, Y("let", "res", term)); - TPosition lambdaPos = Pos; - TPosition aliasPos = Pos; - if (term->IsImplicitLabel() && ctx.WarnOnAnsiAliasShadowing) { - // TODO: recanonize for positions below - lambdaPos = term->GetPos(); - aliasPos = term->GetLabelPos() ? *term->GetLabelPos() : lambdaPos; - } - auto projectItem = Y("SqlProjectItem", "projectCoreType", BuildQuotedAtom(aliasPos, *column), BuildLambda(lambdaPos, Y("row"), body, "res")); - if (term->IsImplicitLabel() && ctx.WarnOnAnsiAliasShadowing) { - projectItem = L(projectItem, Q(Y(Q(Y(Q("warnShadow")))))); - } - sqlProjectArgs = L(sqlProjectArgs, projectItem); - } else { - auto terms = PrepareWithout(Y()); - auto options = Y(); - if (ctx.SimpleColumns && !isJoin) { - terms = L(terms, Y("let", "res", "row")); - } else { - terms = PrepareJoinCoalesce(ctx, terms, multipleQualifiedAll, coalesceLabels); - - auto members = isJoin ? Y() : Y("FlattenMembers"); - if (isJoin) { - members = L(members, BuildQuotedAtom(Pos, *sourceName + ".")); - if (ctx.SimpleColumns) { - options = L(options, Q(Y(Q("divePrefix"), Q(members)))); - } - members = Y(ctx.SimpleColumns ? "DivePrefixMembers" : "SelectMembers", "row", Q(members)); - } else { - auto prefix = BuildQuotedAtom(Pos, ctx.SimpleColumns ? "" : *sourceName + "."); - members = L(members, Q(Y(prefix, "row"))); - if (!ctx.SimpleColumns) { - options = L(options, Q(Y(Q("addPrefix"), prefix))); - } - } - - terms = L(terms, Y("let", "res", members)); + } + + TVector<TString> matched; + TVector<TString> unmatched; + for (auto& label : Source->GetJoin()->GetJoinLabels()) { + if (starTerms.contains(label)) { + matched.push_back(label); + } else { + unmatched.push_back(label); } - sqlProjectArgs = L(sqlProjectArgs, Y("SqlProjectStarItem", "projectCoreType", BuildQuotedAtom(Pos, *sourceName), BuildLambda(Pos, Y("row"), terms, "res"), Q(options))); } - ++column; + + coalesceLabels.insert(coalesceLabels.end(), matched.begin(), matched.end()); + coalesceLabels.insert(coalesceLabels.end(), unmatched.begin(), unmatched.end()); + + multipleQualifiedAll = starTerms.size() > 1; } + + auto column = Columns.List.begin(); + for (auto& term: Terms) { + auto sourceName = term->GetSourceName(); + if (!term->IsAsterisk()) { + auto body = Y(); + if (haveCompositeTerms) { + body = L(body, Y("let", "row", Y("Apply", "addCompositTerms", "row"))); + } + body = L(body, Y("let", "res", term)); + TPosition lambdaPos = Pos; + TPosition aliasPos = Pos; + if (term->IsImplicitLabel() && ctx.WarnOnAnsiAliasShadowing) { + // TODO: recanonize for positions below + lambdaPos = term->GetPos(); + aliasPos = term->GetLabelPos() ? *term->GetLabelPos() : lambdaPos; + } + auto projectItem = Y("SqlProjectItem", "projectCoreType", BuildQuotedAtom(aliasPos, *column), BuildLambda(lambdaPos, Y("row"), body, "res")); + if (term->IsImplicitLabel() && ctx.WarnOnAnsiAliasShadowing) { + projectItem = L(projectItem, Q(Y(Q(Y(Q("warnShadow")))))); + } + sqlProjectArgs = L(sqlProjectArgs, projectItem); + } else { + auto terms = PrepareWithout(Y()); + auto options = Y(); + if (ctx.SimpleColumns && !isJoin) { + terms = L(terms, Y("let", "res", "row")); + } else { + terms = PrepareJoinCoalesce(ctx, terms, multipleQualifiedAll, coalesceLabels); + + auto members = isJoin ? Y() : Y("FlattenMembers"); + if (isJoin) { + members = L(members, BuildQuotedAtom(Pos, *sourceName + ".")); + if (ctx.SimpleColumns) { + options = L(options, Q(Y(Q("divePrefix"), Q(members)))); + } + members = Y(ctx.SimpleColumns ? "DivePrefixMembers" : "SelectMembers", "row", Q(members)); + } else { + auto prefix = BuildQuotedAtom(Pos, ctx.SimpleColumns ? "" : *sourceName + "."); + members = L(members, Q(Y(prefix, "row"))); + if (!ctx.SimpleColumns) { + options = L(options, Q(Y(Q("addPrefix"), prefix))); + } + } + + terms = L(terms, Y("let", "res", members)); + } + sqlProjectArgs = L(sqlProjectArgs, Y("SqlProjectStarItem", "projectCoreType", BuildQuotedAtom(Pos, *sourceName), BuildLambda(Pos, Y("row"), terms, "res"), Q(options))); + } + ++column; + } + } + + auto block(Y(Y("let", "projectCoreType", Y("TypeOf", "core")))); + if (haveCompositeTerms) { + block = L(block, Y("let", "addCompositTerms", BuildLambda(Pos, Y("row"), CompositeTerms, "row"))); } - - auto block(Y(Y("let", "projectCoreType", Y("TypeOf", "core")))); - if (haveCompositeTerms) { - block = L(block, Y("let", "addCompositTerms", BuildLambda(Pos, Y("row"), CompositeTerms, "row"))); - } - - block = L(block, Y("let", "core", Y(ordered ? "OrderedSqlProject" : "SqlProject", "core", Q(sqlProjectArgs)))); - return Y("block", Q(L(block, Y("return", "core")))); + + block = L(block, Y("let", "core", Y(ordered ? "OrderedSqlProject" : "SqlProject", "core", Q(sqlProjectArgs)))); + return Y("block", Q(L(block, Y("return", "core")))); } private: TSourcePtr Source; TVector<TNodePtr> GroupByExpr; - TVector<TNodePtr> DistinctAggrExpr; + TVector<TNodePtr> DistinctAggrExpr; TVector<TNodePtr> GroupBy; bool AssumeSorted = false; bool CompactGroupBy = false; @@ -2081,12 +2081,12 @@ private: TNodePtr Having; TWinSpecs WinSpecs; TNodePtr Flatten; - TNodePtr PreFlattenMap; + TNodePtr PreFlattenMap; TNodePtr PreaggregatedMap; TNodePtr PrewindowMap; TNodePtr Aggregate; TNodePtr CalcOverWindow; - TNodePtr CompositeTerms; + TNodePtr CompositeTerms; TVector<TNodePtr> Terms; TVector<TNodePtr> Without; const bool Distinct; @@ -2207,12 +2207,12 @@ public: produce = L(produce, term); } - if (hasError) { - return false; - } - + if (hasError) { + return false; + } + if (ListCall && !WithExtFunction) { - YQL_ENSURE(listPosIndex.Defined()); + YQL_ENSURE(listPosIndex.Defined()); produce = L(produce, Q(ToString(*listPosIndex))); } @@ -2263,7 +2263,7 @@ public: auto block(Y(Y("let", inputLabel, input))); - auto filter = Source->BuildFilter(ctx, inputLabel); + auto filter = Source->BuildFilter(ctx, inputLabel); if (filter) { block = L(block, Y("let", inputLabel, filter)); } @@ -2292,7 +2292,7 @@ public: return nullptr; } - return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true)); + return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true)); } EOrderKind GetOrderKind() const override { @@ -2328,8 +2328,8 @@ private: terms = Y(Y("let", "res", Y("ToSequence", Terms.front()))); } else { Y_VERIFY_DEBUG(Columns.List.size() == Terms.size()); - terms = L(Y(), Y("let", "res", - L(Y("AsStructUnordered"), Q(Y(BuildQuotedAtom(Pos, Columns.List.front()), Terms.front()))))); + terms = L(Y(), Y("let", "res", + L(Y("AsStructUnordered"), Q(Y(BuildQuotedAtom(Pos, Columns.List.front()), Terms.front()))))); terms = L(terms, Y("let", "res", Y("Just", "res"))); } return terms; @@ -2387,14 +2387,14 @@ public: return CompositeSelect ? true : Source->InitFilters(ctx); } - TNodePtr BuildFilter(TContext& ctx, const TString& label) override { - return CompositeSelect ? nullptr : Source->BuildFilter(ctx, label); + TNodePtr BuildFilter(TContext& ctx, const TString& label) override { + return CompositeSelect ? nullptr : Source->BuildFilter(ctx, label); + } + + IJoin* GetJoin() override { + return Source->GetJoin(); } - IJoin* GetJoin() override { - return Source->GetJoin(); - } - bool IsCompositeSource() const override { return true; } @@ -2427,11 +2427,11 @@ public: } TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) override { - if (const TString* columnName = column.GetColumnName()) { - if (columnName && IsExprAlias(*columnName)) { - return true; - } - } + if (const TString* columnName = column.GetColumnName()) { + if (columnName && IsExprAlias(*columnName)) { + return true; + } + } return Source->AddColumn(ctx, column); } @@ -2447,12 +2447,12 @@ private: mutable TSet<TString> GroupByColumns; }; - -namespace { -TSourcePtr DoBuildSelectCore( + +namespace { +TSourcePtr DoBuildSelectCore( TContext& ctx, TPosition pos, - TSourcePtr originalSource, + TSourcePtr originalSource, TSourcePtr source, const TVector<TNodePtr>& groupByExpr, const TVector<TNodePtr>& groupBy, @@ -2475,16 +2475,16 @@ TSourcePtr DoBuildSelectCore( if (groupBy.size() == 1) { /// actualy no big idea to use grouping function in this case (result allways 0) auto contentPtr = groupBy.front()->ContentListPtr(); - source = new TNestedProxySource(pos, *contentPtr, source); - return DoBuildSelectCore(ctx, pos, originalSource, source, groupByExpr, *contentPtr, compactGroupBy, + source = new TNestedProxySource(pos, *contentPtr, source); + return DoBuildSelectCore(ctx, pos, originalSource, source, groupByExpr, *contentPtr, compactGroupBy, assumeSorted, orderBy, having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); } /// \todo some smart merge logic, generalize common part of grouping (expr, flatten, etc)? - TIntrusivePtr<TCompositeSelect> compositeSelect = new TCompositeSelect(pos, std::move(source), originalSource->CloneSource(), settings); + TIntrusivePtr<TCompositeSelect> compositeSelect = new TCompositeSelect(pos, std::move(source), originalSource->CloneSource(), settings); size_t totalGroups = 0; TVector<TSourcePtr> subselects; - TVector<TNodePtr> groupingCols; + TVector<TNodePtr> groupingCols; for (auto& grouping: groupBy) { auto contentPtr = grouping->ContentListPtr(); TVector<TNodePtr> cache(1, nullptr); @@ -2492,8 +2492,8 @@ TSourcePtr DoBuildSelectCore( cache[0] = grouping; contentPtr = &cache; } - groupingCols.insert(groupingCols.end(), contentPtr->cbegin(), contentPtr->cend()); - TSourcePtr proxySource = new TNestedProxySource(compositeSelect.Get(), CloneContainer(*contentPtr)); + groupingCols.insert(groupingCols.end(), contentPtr->cbegin(), contentPtr->cend()); + TSourcePtr proxySource = new TNestedProxySource(compositeSelect.Get(), CloneContainer(*contentPtr)); if (!subselects.empty()) { /// clone terms for others usage TVector<TNodePtr> termsCopy; @@ -2504,7 +2504,7 @@ TSourcePtr DoBuildSelectCore( } totalGroups += contentPtr->size(); TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr), - CloneContainer(*contentPtr), compactGroupBy, assumeSorted, orderBy, SafeClone(having), winSpecs, + CloneContainer(*contentPtr), compactGroupBy, assumeSorted, orderBy, SafeClone(having), winSpecs, hoppingWindowSpec, terms, distinct, without, selectStream, settings); subselects.emplace_back(selectCore); } @@ -2512,40 +2512,40 @@ TSourcePtr DoBuildSelectCore( ctx.Error(pos) << "Unable to GROUP BY more than " << ctx.PragmaGroupByLimit << " groups, you try use " << totalGroups << " groups"; return nullptr; } - compositeSelect->SetSubselects(std::move(subselects), std::move(groupingCols), CloneContainer(groupByExpr)); + compositeSelect->SetSubselects(std::move(subselects), std::move(groupingCols), CloneContainer(groupByExpr)); return compositeSelect; } -} - -TSourcePtr BuildSelectCore( - TContext& ctx, - TPosition pos, - TSourcePtr source, - const TVector<TNodePtr>& groupByExpr, - const TVector<TNodePtr>& groupBy, - bool compactGroupBy, - bool assumeSorted, - const TVector<TSortSpecificationPtr>& orderBy, - TNodePtr having, - TWinSpecs&& winSpecs, - THoppingWindowSpecPtr hoppingWindowSpec, - TVector<TNodePtr>&& terms, - bool distinct, - TVector<TNodePtr>&& without, +} + +TSourcePtr BuildSelectCore( + TContext& ctx, + TPosition pos, + TSourcePtr source, + const TVector<TNodePtr>& groupByExpr, + const TVector<TNodePtr>& groupBy, + bool compactGroupBy, + bool assumeSorted, + const TVector<TSortSpecificationPtr>& orderBy, + TNodePtr having, + TWinSpecs&& winSpecs, + THoppingWindowSpecPtr hoppingWindowSpec, + TVector<TNodePtr>&& terms, + bool distinct, + TVector<TNodePtr>&& without, bool selectStream, - const TWriteSettings& settings) -{ - return DoBuildSelectCore(ctx, pos, source, source, groupByExpr, groupBy, compactGroupBy, assumeSorted, orderBy, + const TWriteSettings& settings) +{ + return DoBuildSelectCore(ctx, pos, source, source, groupByExpr, groupBy, compactGroupBy, assumeSorted, orderBy, having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); -} - +} + class TUnionAll: public IRealSource { public: - TUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings) + TUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings) : IRealSource(pos) , Sources(std::move(sources)) - , Settings(settings) + , Settings(settings) { } @@ -2562,24 +2562,24 @@ public: } bool DoInit(TContext& ctx, ISource* src) override { - bool first = true; + bool first = true; for (auto& s: Sources) { s->UseAsInner(); if (!s->Init(ctx, src)) { return false; } - if (!ctx.PositionalUnionAll || first) { - auto c = s->GetColumns(); - Y_VERIFY_DEBUG(c); - Columns.Merge(*c); - first = false; - } + if (!ctx.PositionalUnionAll || first) { + auto c = s->GetColumns(); + Y_VERIFY_DEBUG(c); + Columns.Merge(*c); + first = false; + } } return true; } TNodePtr Build(TContext& ctx) override { - auto res = ctx.PositionalUnionAll ? Y("UnionAllPositional") : Y("UnionAll"); + auto res = ctx.PositionalUnionAll ? Y("UnionAllPositional") : Y("UnionAll"); for (auto& s: Sources) { auto input = s->Build(ctx); if (!input) { @@ -2601,24 +2601,24 @@ public: } TNodePtr DoClone() const final { - return MakeIntrusive<TUnionAll>(Pos, CloneContainer(Sources), Settings); + return MakeIntrusive<TUnionAll>(Pos, CloneContainer(Sources), Settings); + } + + bool IsSelect() const override { + return true; + } + + TWriteSettings GetWriteSettings() const override { + return Settings; } - bool IsSelect() const override { - return true; - } - - TWriteSettings GetWriteSettings() const override { - return Settings; - } - private: TVector<TSourcePtr> Sources; - const TWriteSettings Settings; + const TWriteSettings Settings; }; -TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings) { - return new TUnionAll(pos, std::move(sources), settings); +TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources, const TWriteSettings& settings) { + return new TUnionAll(pos, std::move(sources), settings); } class TOverWindowSource: public IProxySource { @@ -2670,10 +2670,10 @@ public: return Source->FindWindowSpecification(ctx, windowName); } - TNodePtr GetSessionWindowSpec() const override { - return Source->GetSessionWindowSpec(); - } - + TNodePtr GetSessionWindowSpec() const override { + return Source->GetSessionWindowSpec(); + } + TNodePtr DoClone() const final { return {}; } @@ -2721,16 +2721,16 @@ public: Source->UseAsInner(); } - if (IgnoreSort()) { - ctx.Warning(Source->GetPos(), TIssuesIds::YQL_ORDER_BY_WITHOUT_LIMIT_IN_SUBQUERY) << "ORDER BY without LIMIT in subquery will be ignored"; - } - + if (IgnoreSort()) { + ctx.Warning(Source->GetPos(), TIssuesIds::YQL_ORDER_BY_WITHOUT_LIMIT_IN_SUBQUERY) << "ORDER BY without LIMIT in subquery will be ignored"; + } + if (!Source->Init(ctx, src)) { return false; } src = Source.Get(); if (SkipTake) { - FakeSource = BuildFakeSource(SkipTake->GetPos()); + FakeSource = BuildFakeSource(SkipTake->GetPos()); if (!SkipTake->Init(ctx, FakeSource.Get())) { return false; } @@ -2748,12 +2748,12 @@ public: auto block(Y(Y("let", label, input))); auto sortNode = Source->BuildSort(ctx, label); - if (sortNode && !IgnoreSort()) { - block = L(block, sortNode); + if (sortNode && !IgnoreSort()) { + block = L(block, sortNode); } if (SkipTake) { - block = L(block, SkipTake); + block = L(block, SkipTake); } TNodePtr sample; @@ -2785,7 +2785,7 @@ public: ctx.Error(pos) << "'Repeatable' keyword is not supported for subqueries"; return false; } - return SetSamplingRate(ctx, samplingRate); + return SetSamplingRate(ctx, samplingRate); } bool IsSelect() const override { @@ -2796,13 +2796,13 @@ public: return MakeIntrusive<TSelect>(Pos, Source->CloneSource(), SafeClone(SkipTake)); } protected: - bool IgnoreSort() const { - return AsInner && !SkipTake && EOrderKind::Sort == Source->GetOrderKind(); - } - + bool IgnoreSort() const { + return AsInner && !SkipTake && EOrderKind::Sort == Source->GetOrderKind(); + } + TSourcePtr Source; TNodePtr SkipTake; - TSourcePtr FakeSource; + TSourcePtr FakeSource; }; TSourcePtr BuildSelect(TPosition pos, TSourcePtr source, TNodePtr skipTake) { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 04a607890a..ab5a864788 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -23,7 +23,7 @@ #include <google/protobuf/repeated_field.h> -#include <util/charset/wide.h> +#include <util/charset/wide.h> #include <util/generic/array_ref.h> #include <util/generic/set.h> #include <util/generic/ylimits.h> @@ -34,16 +34,16 @@ #include <util/string/hex.h> #include <util/string/join.h> -#if defined(_tsan_enabled_) -#include <util/system/mutex.h> -#endif - +#if defined(_tsan_enabled_) +#include <util/system/mutex.h> +#endif + using namespace NYql; namespace NSQLTranslationV1 { -using NALPDefault::SQLv1LexerTokens; - +using NALPDefault::SQLv1LexerTokens; + #if defined(_tsan_enabled_) TMutex SanitizerSQLTranslationMutex; #endif @@ -61,36 +61,36 @@ TIdentifier GetIdentifier(TTranslation& ctx, const TToken& node) { } inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) { - // keyword: - // keyword_compat - // | keyword_expr_uncompat - // | keyword_table_uncompat - // | keyword_select_uncompat - // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; + // keyword: + // keyword_compat + // | keyword_expr_uncompat + // | keyword_table_uncompat + // | keyword_select_uncompat + // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; switch (node.Alt_case()) { case TRule_keyword::kAltKeyword1: - return GetIdentifier(ctx, node.GetAlt_keyword1().GetRule_keyword_compat1()); + return GetIdentifier(ctx, node.GetAlt_keyword1().GetRule_keyword_compat1()); case TRule_keyword::kAltKeyword2: - return GetIdentifier(ctx, node.GetAlt_keyword2().GetRule_keyword_expr_uncompat1()); + return GetIdentifier(ctx, node.GetAlt_keyword2().GetRule_keyword_expr_uncompat1()); case TRule_keyword::kAltKeyword3: return GetIdentifier(ctx, node.GetAlt_keyword3().GetRule_keyword_table_uncompat1()); - case TRule_keyword::kAltKeyword4: - return GetIdentifier(ctx, node.GetAlt_keyword4().GetRule_keyword_select_uncompat1()); - case TRule_keyword::kAltKeyword5: - return GetIdentifier(ctx, node.GetAlt_keyword5().GetRule_keyword_alter_uncompat1()); - case TRule_keyword::kAltKeyword6: - return GetIdentifier(ctx, node.GetAlt_keyword6().GetRule_keyword_in_uncompat1()); - case TRule_keyword::kAltKeyword7: - return GetIdentifier(ctx, node.GetAlt_keyword7().GetRule_keyword_window_uncompat1()); - case TRule_keyword::kAltKeyword8: - return GetIdentifier(ctx, node.GetAlt_keyword8().GetRule_keyword_hint_uncompat1()); - case TRule_keyword::kAltKeyword9: - return GetIdentifier(ctx, node.GetAlt_keyword9().GetRule_keyword_schema_uncompat1()); + case TRule_keyword::kAltKeyword4: + return GetIdentifier(ctx, node.GetAlt_keyword4().GetRule_keyword_select_uncompat1()); + case TRule_keyword::kAltKeyword5: + return GetIdentifier(ctx, node.GetAlt_keyword5().GetRule_keyword_alter_uncompat1()); + case TRule_keyword::kAltKeyword6: + return GetIdentifier(ctx, node.GetAlt_keyword6().GetRule_keyword_in_uncompat1()); + case TRule_keyword::kAltKeyword7: + return GetIdentifier(ctx, node.GetAlt_keyword7().GetRule_keyword_window_uncompat1()); + case TRule_keyword::kAltKeyword8: + return GetIdentifier(ctx, node.GetAlt_keyword8().GetRule_keyword_hint_uncompat1()); + case TRule_keyword::kAltKeyword9: + return GetIdentifier(ctx, node.GetAlt_keyword9().GetRule_keyword_schema_uncompat1()); default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -100,21 +100,21 @@ inline TString GetKeyword(TTranslation& ctx, const TRule_keyword& node) { return GetKeywordId(ctx, node).Name; } -template <typename TRule> -inline TString GetKeyword(TTranslation& ctx, const TRule& node) { - return GetIdentifier(ctx, node).Name; +template <typename TRule> +inline TString GetKeyword(TTranslation& ctx, const TRule& node) { + return GetIdentifier(ctx, node).Name; +} + +static TString Id(const TRule_identifier& node, TTranslation& ctx) { + // identifier: ID_PLAIN | ID_QUOTED; + return ctx.Identifier(node.GetToken1()); } -static TString Id(const TRule_identifier& node, TTranslation& ctx) { - // identifier: ID_PLAIN | ID_QUOTED; - return ctx.Identifier(node.GetToken1()); -} - static TString Id(const TRule_id& node, TTranslation& ctx) { - // id: identifier | keyword; + // id: identifier | keyword; switch (node.Alt_case()) { case TRule_id::kAltId1: - return Id(node.GetAlt_id1().GetRule_identifier1(), ctx); + return Id(node.GetAlt_id1().GetRule_identifier1(), ctx); case TRule_id::kAltId2: return GetKeyword(ctx, node.GetAlt_id2().GetRule_keyword1()); default: @@ -134,106 +134,106 @@ static TString Id(const TRule_id_or_type& node, TTranslation& ctx) { } static TString Id(const TRule_id_schema& node, TTranslation& ctx) { - //id_schema: - // identifier - // | keyword_compat - // | keyword_expr_uncompat - // // | keyword_table_uncompat - // | keyword_select_uncompat - // // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // // | keyword_schema_uncompat - //; + //id_schema: + // identifier + // | keyword_compat + // | keyword_expr_uncompat + // // | keyword_table_uncompat + // | keyword_select_uncompat + // // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // // | keyword_schema_uncompat + //; switch (node.Alt_case()) { case TRule_id_schema::kAltIdSchema1: - return Id(node.GetAlt_id_schema1().GetRule_identifier1(), ctx); + return Id(node.GetAlt_id_schema1().GetRule_identifier1(), ctx); case TRule_id_schema::kAltIdSchema2: - return GetKeyword(ctx, node.GetAlt_id_schema2().GetRule_keyword_compat1()); - case TRule_id_schema::kAltIdSchema3: - return GetKeyword(ctx, node.GetAlt_id_schema3().GetRule_keyword_expr_uncompat1()); - case TRule_id_schema::kAltIdSchema4: - return GetKeyword(ctx, node.GetAlt_id_schema4().GetRule_keyword_select_uncompat1()); - case TRule_id_schema::kAltIdSchema5: - return GetKeyword(ctx, node.GetAlt_id_schema5().GetRule_keyword_in_uncompat1()); - case TRule_id_schema::kAltIdSchema6: - return GetKeyword(ctx, node.GetAlt_id_schema6().GetRule_keyword_window_uncompat1()); - case TRule_id_schema::kAltIdSchema7: - return GetKeyword(ctx, node.GetAlt_id_schema7().GetRule_keyword_hint_uncompat1()); + return GetKeyword(ctx, node.GetAlt_id_schema2().GetRule_keyword_compat1()); + case TRule_id_schema::kAltIdSchema3: + return GetKeyword(ctx, node.GetAlt_id_schema3().GetRule_keyword_expr_uncompat1()); + case TRule_id_schema::kAltIdSchema4: + return GetKeyword(ctx, node.GetAlt_id_schema4().GetRule_keyword_select_uncompat1()); + case TRule_id_schema::kAltIdSchema5: + return GetKeyword(ctx, node.GetAlt_id_schema5().GetRule_keyword_in_uncompat1()); + case TRule_id_schema::kAltIdSchema6: + return GetKeyword(ctx, node.GetAlt_id_schema6().GetRule_keyword_window_uncompat1()); + case TRule_id_schema::kAltIdSchema7: + return GetKeyword(ctx, node.GetAlt_id_schema7().GetRule_keyword_hint_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_or_type& node, TTranslation& ctx) { + // an_id_or_type: id_or_type | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_or_type::kAltAnIdOrType1: + return Id(node.GetAlt_an_id_or_type1().GetRule_id_or_type1(), ctx); + case TRule_an_id_or_type::kAltAnIdOrType2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_or_type2().GetToken1())); default: Y_FAIL("You should change implementation according to grammar changes"); } } -static TString Id(const TRule_an_id_or_type& node, TTranslation& ctx) { - // an_id_or_type: id_or_type | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_or_type::kAltAnIdOrType1: - return Id(node.GetAlt_an_id_or_type1().GetRule_id_or_type1(), ctx); - case TRule_an_id_or_type::kAltAnIdOrType2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_or_type2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - static std::pair<bool, TString> Id(const TRule_id_or_at& node, TTranslation& ctx) { bool hasAt = node.HasBlock1(); - return std::make_pair(hasAt, Id(node.GetRule_an_id_or_type2(), ctx) ); + return std::make_pair(hasAt, Id(node.GetRule_an_id_or_type2(), ctx) ); } static TString Id(const TRule_id_table& node, TTranslation& ctx) { - //id_table: - // identifier - // | keyword_compat - // | keyword_expr_uncompat - // // | keyword_table_uncompat - // | keyword_select_uncompat - // // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; + //id_table: + // identifier + // | keyword_compat + // | keyword_expr_uncompat + // // | keyword_table_uncompat + // | keyword_select_uncompat + // // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; + switch (node.Alt_case()) { + case TRule_id_table::kAltIdTable1: + return Id(node.GetAlt_id_table1().GetRule_identifier1(), ctx); + case TRule_id_table::kAltIdTable2: + return GetKeyword(ctx, node.GetAlt_id_table2().GetRule_keyword_compat1()); + case TRule_id_table::kAltIdTable3: + return GetKeyword(ctx, node.GetAlt_id_table3().GetRule_keyword_expr_uncompat1()); + case TRule_id_table::kAltIdTable4: + return GetKeyword(ctx, node.GetAlt_id_table4().GetRule_keyword_select_uncompat1()); + case TRule_id_table::kAltIdTable5: + return GetKeyword(ctx, node.GetAlt_id_table5().GetRule_keyword_in_uncompat1()); + case TRule_id_table::kAltIdTable6: + return GetKeyword(ctx, node.GetAlt_id_table6().GetRule_keyword_window_uncompat1()); + case TRule_id_table::kAltIdTable7: + return GetKeyword(ctx, node.GetAlt_id_table7().GetRule_keyword_hint_uncompat1()); + case TRule_id_table::kAltIdTable8: + return GetKeyword(ctx, node.GetAlt_id_table8().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_table& node, TTranslation& ctx) { + // an_id_table: id_table | STRING_VALUE; switch (node.Alt_case()) { - case TRule_id_table::kAltIdTable1: - return Id(node.GetAlt_id_table1().GetRule_identifier1(), ctx); - case TRule_id_table::kAltIdTable2: - return GetKeyword(ctx, node.GetAlt_id_table2().GetRule_keyword_compat1()); - case TRule_id_table::kAltIdTable3: - return GetKeyword(ctx, node.GetAlt_id_table3().GetRule_keyword_expr_uncompat1()); - case TRule_id_table::kAltIdTable4: - return GetKeyword(ctx, node.GetAlt_id_table4().GetRule_keyword_select_uncompat1()); - case TRule_id_table::kAltIdTable5: - return GetKeyword(ctx, node.GetAlt_id_table5().GetRule_keyword_in_uncompat1()); - case TRule_id_table::kAltIdTable6: - return GetKeyword(ctx, node.GetAlt_id_table6().GetRule_keyword_window_uncompat1()); - case TRule_id_table::kAltIdTable7: - return GetKeyword(ctx, node.GetAlt_id_table7().GetRule_keyword_hint_uncompat1()); - case TRule_id_table::kAltIdTable8: - return GetKeyword(ctx, node.GetAlt_id_table8().GetRule_keyword_schema_uncompat1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_table& node, TTranslation& ctx) { - // an_id_table: id_table | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_table::kAltAnIdTable1: - return Id(node.GetAlt_an_id_table1().GetRule_id_table1(), ctx); - case TRule_an_id_table::kAltAnIdTable2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_table2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - + case TRule_an_id_table::kAltAnIdTable1: + return Id(node.GetAlt_an_id_table1().GetRule_id_table1(), ctx); + case TRule_an_id_table::kAltAnIdTable2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_table2().GetToken1())); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) { switch (node.Alt_case()) { case TRule_id_table_or_type::kAltIdTableOrType1: - return Id(node.GetAlt_id_table_or_type1().GetRule_an_id_table1(), ctx); + return Id(node.GetAlt_id_table_or_type1().GetRule_an_id_table1(), ctx); case TRule_id_table_or_type::kAltIdTableOrType2: return ctx.Identifier(node.GetAlt_id_table_or_type2().GetRule_type_id1().GetToken1()); default: @@ -242,273 +242,273 @@ static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) { } static TString Id(const TRule_id_expr& node, TTranslation& ctx) { - //id_expr: - // identifier - // | keyword_compat - // // | keyword_expr_uncompat - // // | keyword_table_uncompat - // // | keyword_select_uncompat - // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; + //id_expr: + // identifier + // | keyword_compat + // // | keyword_expr_uncompat + // // | keyword_table_uncompat + // // | keyword_select_uncompat + // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; switch (node.Alt_case()) { case TRule_id_expr::kAltIdExpr1: - return Id(node.GetAlt_id_expr1().GetRule_identifier1(), ctx); + return Id(node.GetAlt_id_expr1().GetRule_identifier1(), ctx); case TRule_id_expr::kAltIdExpr2: - return GetKeyword(ctx, node.GetAlt_id_expr2().GetRule_keyword_compat1()); + return GetKeyword(ctx, node.GetAlt_id_expr2().GetRule_keyword_compat1()); case TRule_id_expr::kAltIdExpr3: - return GetKeyword(ctx, node.GetAlt_id_expr3().GetRule_keyword_alter_uncompat1()); - case TRule_id_expr::kAltIdExpr4: - return GetKeyword(ctx, node.GetAlt_id_expr4().GetRule_keyword_in_uncompat1()); - case TRule_id_expr::kAltIdExpr5: - return GetKeyword(ctx, node.GetAlt_id_expr5().GetRule_keyword_window_uncompat1()); - case TRule_id_expr::kAltIdExpr6: - return GetKeyword(ctx, node.GetAlt_id_expr6().GetRule_keyword_hint_uncompat1()); - case TRule_id_expr::kAltIdExpr7: - return GetKeyword(ctx, node.GetAlt_id_expr7().GetRule_keyword_schema_uncompat1()); + return GetKeyword(ctx, node.GetAlt_id_expr3().GetRule_keyword_alter_uncompat1()); + case TRule_id_expr::kAltIdExpr4: + return GetKeyword(ctx, node.GetAlt_id_expr4().GetRule_keyword_in_uncompat1()); + case TRule_id_expr::kAltIdExpr5: + return GetKeyword(ctx, node.GetAlt_id_expr5().GetRule_keyword_window_uncompat1()); + case TRule_id_expr::kAltIdExpr6: + return GetKeyword(ctx, node.GetAlt_id_expr6().GetRule_keyword_hint_uncompat1()); + case TRule_id_expr::kAltIdExpr7: + return GetKeyword(ctx, node.GetAlt_id_expr7().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static bool IsQuotedId(const TRule_id_expr& node, TTranslation& ctx) { + if (node.Alt_case() != TRule_id_expr::kAltIdExpr1) { + return false; + } + const auto& id = ctx.Token(node.GetAlt_id_expr1().GetRule_identifier1().GetToken1()); + // identifier: ID_PLAIN | ID_QUOTED; + return id.StartsWith('`'); +} + +static TString Id(const TRule_id_expr_in& node, TTranslation& ctx) { + //id_expr_in: + // identifier + // | keyword_compat + // // | keyword_expr_uncompat + // // | keyword_table_uncompat + // // | keyword_select_uncompat + // | keyword_alter_uncompat + // // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; + switch (node.Alt_case()) { + case TRule_id_expr_in::kAltIdExprIn1: + return Id(node.GetAlt_id_expr_in1().GetRule_identifier1(), ctx); + case TRule_id_expr_in::kAltIdExprIn2: + return GetKeyword(ctx, node.GetAlt_id_expr_in2().GetRule_keyword_compat1()); + case TRule_id_expr_in::kAltIdExprIn3: + return GetKeyword(ctx, node.GetAlt_id_expr_in3().GetRule_keyword_alter_uncompat1()); + case TRule_id_expr_in::kAltIdExprIn4: + return GetKeyword(ctx, node.GetAlt_id_expr_in4().GetRule_keyword_window_uncompat1()); + case TRule_id_expr_in::kAltIdExprIn5: + return GetKeyword(ctx, node.GetAlt_id_expr_in5().GetRule_keyword_hint_uncompat1()); + case TRule_id_expr_in::kAltIdExprIn6: + return GetKeyword(ctx, node.GetAlt_id_expr_in6().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_id_window& node, TTranslation& ctx) { + //id_window: + // identifier + // | keyword_compat + // | keyword_expr_uncompat + // | keyword_table_uncompat + // | keyword_select_uncompat + // | keyword_alter_uncompat + // | keyword_in_uncompat + // // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; + switch (node.Alt_case()) { + case TRule_id_window::kAltIdWindow1: + return Id(node.GetAlt_id_window1().GetRule_identifier1(), ctx); + case TRule_id_window::kAltIdWindow2: + return GetKeyword(ctx, node.GetAlt_id_window2().GetRule_keyword_compat1()); + case TRule_id_window::kAltIdWindow3: + return GetKeyword(ctx, node.GetAlt_id_window3().GetRule_keyword_expr_uncompat1()); + case TRule_id_window::kAltIdWindow4: + return GetKeyword(ctx, node.GetAlt_id_window4().GetRule_keyword_table_uncompat1()); + case TRule_id_window::kAltIdWindow5: + return GetKeyword(ctx, node.GetAlt_id_window5().GetRule_keyword_select_uncompat1()); + case TRule_id_window::kAltIdWindow6: + return GetKeyword(ctx, node.GetAlt_id_window6().GetRule_keyword_alter_uncompat1()); + case TRule_id_window::kAltIdWindow7: + return GetKeyword(ctx, node.GetAlt_id_window7().GetRule_keyword_in_uncompat1()); + case TRule_id_window::kAltIdWindow8: + return GetKeyword(ctx, node.GetAlt_id_window8().GetRule_keyword_hint_uncompat1()); + case TRule_id_window::kAltIdWindow9: + return GetKeyword(ctx, node.GetAlt_id_window9().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_id_without& node, TTranslation& ctx) { + //id_without: + // identifier + // | keyword_compat + // // | keyword_expr_uncompat + // | keyword_table_uncompat + // // | keyword_select_uncompat + // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; + switch (node.Alt_case()) { + case TRule_id_without::kAltIdWithout1: + return Id(node.GetAlt_id_without1().GetRule_identifier1(), ctx); + case TRule_id_without::kAltIdWithout2: + return GetKeyword(ctx, node.GetAlt_id_without2().GetRule_keyword_compat1()); + case TRule_id_without::kAltIdWithout3: + return GetKeyword(ctx, node.GetAlt_id_without3().GetRule_keyword_table_uncompat1()); + case TRule_id_without::kAltIdWithout4: + return GetKeyword(ctx, node.GetAlt_id_without4().GetRule_keyword_alter_uncompat1()); + case TRule_id_without::kAltIdWithout5: + return GetKeyword(ctx, node.GetAlt_id_without5().GetRule_keyword_in_uncompat1()); + case TRule_id_without::kAltIdWithout6: + return GetKeyword(ctx, node.GetAlt_id_without6().GetRule_keyword_window_uncompat1()); + case TRule_id_without::kAltIdWithout7: + return GetKeyword(ctx, node.GetAlt_id_without7().GetRule_keyword_hint_uncompat1()); + case TRule_id_without::kAltIdWithout8: + return GetKeyword(ctx, node.GetAlt_id_without8().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_id_hint& node, TTranslation& ctx) { + //id_hint: + // identifier + // | keyword_compat + // | keyword_expr_uncompat + // | keyword_table_uncompat + // | keyword_select_uncompat + // | keyword_alter_uncompat + // | keyword_in_uncompat + // | keyword_window_uncompat + // // | keyword_hint_uncompat + // | keyword_schema_uncompat + //; + switch (node.Alt_case()) { + case TRule_id_hint::kAltIdHint1: + return Id(node.GetAlt_id_hint1().GetRule_identifier1(), ctx); + case TRule_id_hint::kAltIdHint2: + return GetKeyword(ctx, node.GetAlt_id_hint2().GetRule_keyword_compat1()); + case TRule_id_hint::kAltIdHint3: + return GetKeyword(ctx, node.GetAlt_id_hint3().GetRule_keyword_expr_uncompat1()); + case TRule_id_hint::kAltIdHint4: + return GetKeyword(ctx, node.GetAlt_id_hint4().GetRule_keyword_table_uncompat1()); + case TRule_id_hint::kAltIdHint5: + return GetKeyword(ctx, node.GetAlt_id_hint5().GetRule_keyword_select_uncompat1()); + case TRule_id_hint::kAltIdHint6: + return GetKeyword(ctx, node.GetAlt_id_hint6().GetRule_keyword_alter_uncompat1()); + case TRule_id_hint::kAltIdHint7: + return GetKeyword(ctx, node.GetAlt_id_hint7().GetRule_keyword_in_uncompat1()); + case TRule_id_hint::kAltIdHint8: + return GetKeyword(ctx, node.GetAlt_id_hint8().GetRule_keyword_window_uncompat1()); + case TRule_id_hint::kAltIdHint9: + return GetKeyword(ctx, node.GetAlt_id_hint9().GetRule_keyword_schema_uncompat1()); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id& node, TTranslation& ctx) { + // an_id: id | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id::kAltAnId1: + return Id(node.GetAlt_an_id1().GetRule_id1(), ctx); + case TRule_an_id::kAltAnId2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id2().GetToken1())); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_schema& node, TTranslation& ctx) { + // an_id_schema: id_schema | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_schema::kAltAnIdSchema1: + return Id(node.GetAlt_an_id_schema1().GetRule_id_schema1(), ctx); + case TRule_an_id_schema::kAltAnIdSchema2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_schema2().GetToken1())); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_expr& node, TTranslation& ctx) { + // an_id_expr: id_expr | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_expr::kAltAnIdExpr1: + return Id(node.GetAlt_an_id_expr1().GetRule_id_expr1(), ctx); + case TRule_an_id_expr::kAltAnIdExpr2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_expr2().GetToken1())); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_window& node, TTranslation& ctx) { + // an_id_window: id_window | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_window::kAltAnIdWindow1: + return Id(node.GetAlt_an_id_window1().GetRule_id_window1(), ctx); + case TRule_an_id_window::kAltAnIdWindow2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_window2().GetToken1())); + default: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + +static TString Id(const TRule_an_id_without& node, TTranslation& ctx) { + // an_id_without: id_without | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_without::kAltAnIdWithout1: + return Id(node.GetAlt_an_id_without1().GetRule_id_without1(), ctx); + case TRule_an_id_without::kAltAnIdWithout2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_without2().GetToken1())); default: Y_FAIL("You should change implementation according to grammar changes"); } } -static bool IsQuotedId(const TRule_id_expr& node, TTranslation& ctx) { - if (node.Alt_case() != TRule_id_expr::kAltIdExpr1) { - return false; - } - const auto& id = ctx.Token(node.GetAlt_id_expr1().GetRule_identifier1().GetToken1()); - // identifier: ID_PLAIN | ID_QUOTED; - return id.StartsWith('`'); -} - -static TString Id(const TRule_id_expr_in& node, TTranslation& ctx) { - //id_expr_in: - // identifier - // | keyword_compat - // // | keyword_expr_uncompat - // // | keyword_table_uncompat - // // | keyword_select_uncompat - // | keyword_alter_uncompat - // // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; - switch (node.Alt_case()) { - case TRule_id_expr_in::kAltIdExprIn1: - return Id(node.GetAlt_id_expr_in1().GetRule_identifier1(), ctx); - case TRule_id_expr_in::kAltIdExprIn2: - return GetKeyword(ctx, node.GetAlt_id_expr_in2().GetRule_keyword_compat1()); - case TRule_id_expr_in::kAltIdExprIn3: - return GetKeyword(ctx, node.GetAlt_id_expr_in3().GetRule_keyword_alter_uncompat1()); - case TRule_id_expr_in::kAltIdExprIn4: - return GetKeyword(ctx, node.GetAlt_id_expr_in4().GetRule_keyword_window_uncompat1()); - case TRule_id_expr_in::kAltIdExprIn5: - return GetKeyword(ctx, node.GetAlt_id_expr_in5().GetRule_keyword_hint_uncompat1()); - case TRule_id_expr_in::kAltIdExprIn6: - return GetKeyword(ctx, node.GetAlt_id_expr_in6().GetRule_keyword_schema_uncompat1()); - default: +static TString Id(const TRule_an_id_hint& node, TTranslation& ctx) { + // an_id_hint: id_hint | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_hint::kAltAnIdHint1: + return Id(node.GetAlt_an_id_hint1().GetRule_id_hint1(), ctx); + case TRule_an_id_hint::kAltAnIdHint2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_hint2().GetToken1())); + default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_id_window& node, TTranslation& ctx) { - //id_window: - // identifier - // | keyword_compat - // | keyword_expr_uncompat - // | keyword_table_uncompat - // | keyword_select_uncompat - // | keyword_alter_uncompat - // | keyword_in_uncompat - // // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; - switch (node.Alt_case()) { - case TRule_id_window::kAltIdWindow1: - return Id(node.GetAlt_id_window1().GetRule_identifier1(), ctx); - case TRule_id_window::kAltIdWindow2: - return GetKeyword(ctx, node.GetAlt_id_window2().GetRule_keyword_compat1()); - case TRule_id_window::kAltIdWindow3: - return GetKeyword(ctx, node.GetAlt_id_window3().GetRule_keyword_expr_uncompat1()); - case TRule_id_window::kAltIdWindow4: - return GetKeyword(ctx, node.GetAlt_id_window4().GetRule_keyword_table_uncompat1()); - case TRule_id_window::kAltIdWindow5: - return GetKeyword(ctx, node.GetAlt_id_window5().GetRule_keyword_select_uncompat1()); - case TRule_id_window::kAltIdWindow6: - return GetKeyword(ctx, node.GetAlt_id_window6().GetRule_keyword_alter_uncompat1()); - case TRule_id_window::kAltIdWindow7: - return GetKeyword(ctx, node.GetAlt_id_window7().GetRule_keyword_in_uncompat1()); - case TRule_id_window::kAltIdWindow8: - return GetKeyword(ctx, node.GetAlt_id_window8().GetRule_keyword_hint_uncompat1()); - case TRule_id_window::kAltIdWindow9: - return GetKeyword(ctx, node.GetAlt_id_window9().GetRule_keyword_schema_uncompat1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_id_without& node, TTranslation& ctx) { - //id_without: - // identifier - // | keyword_compat - // // | keyword_expr_uncompat - // | keyword_table_uncompat - // // | keyword_select_uncompat - // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; - switch (node.Alt_case()) { - case TRule_id_without::kAltIdWithout1: - return Id(node.GetAlt_id_without1().GetRule_identifier1(), ctx); - case TRule_id_without::kAltIdWithout2: - return GetKeyword(ctx, node.GetAlt_id_without2().GetRule_keyword_compat1()); - case TRule_id_without::kAltIdWithout3: - return GetKeyword(ctx, node.GetAlt_id_without3().GetRule_keyword_table_uncompat1()); - case TRule_id_without::kAltIdWithout4: - return GetKeyword(ctx, node.GetAlt_id_without4().GetRule_keyword_alter_uncompat1()); - case TRule_id_without::kAltIdWithout5: - return GetKeyword(ctx, node.GetAlt_id_without5().GetRule_keyword_in_uncompat1()); - case TRule_id_without::kAltIdWithout6: - return GetKeyword(ctx, node.GetAlt_id_without6().GetRule_keyword_window_uncompat1()); - case TRule_id_without::kAltIdWithout7: - return GetKeyword(ctx, node.GetAlt_id_without7().GetRule_keyword_hint_uncompat1()); - case TRule_id_without::kAltIdWithout8: - return GetKeyword(ctx, node.GetAlt_id_without8().GetRule_keyword_schema_uncompat1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_id_hint& node, TTranslation& ctx) { - //id_hint: - // identifier - // | keyword_compat - // | keyword_expr_uncompat - // | keyword_table_uncompat - // | keyword_select_uncompat - // | keyword_alter_uncompat - // | keyword_in_uncompat - // | keyword_window_uncompat - // // | keyword_hint_uncompat - // | keyword_schema_uncompat - //; - switch (node.Alt_case()) { - case TRule_id_hint::kAltIdHint1: - return Id(node.GetAlt_id_hint1().GetRule_identifier1(), ctx); - case TRule_id_hint::kAltIdHint2: - return GetKeyword(ctx, node.GetAlt_id_hint2().GetRule_keyword_compat1()); - case TRule_id_hint::kAltIdHint3: - return GetKeyword(ctx, node.GetAlt_id_hint3().GetRule_keyword_expr_uncompat1()); - case TRule_id_hint::kAltIdHint4: - return GetKeyword(ctx, node.GetAlt_id_hint4().GetRule_keyword_table_uncompat1()); - case TRule_id_hint::kAltIdHint5: - return GetKeyword(ctx, node.GetAlt_id_hint5().GetRule_keyword_select_uncompat1()); - case TRule_id_hint::kAltIdHint6: - return GetKeyword(ctx, node.GetAlt_id_hint6().GetRule_keyword_alter_uncompat1()); - case TRule_id_hint::kAltIdHint7: - return GetKeyword(ctx, node.GetAlt_id_hint7().GetRule_keyword_in_uncompat1()); - case TRule_id_hint::kAltIdHint8: - return GetKeyword(ctx, node.GetAlt_id_hint8().GetRule_keyword_window_uncompat1()); - case TRule_id_hint::kAltIdHint9: - return GetKeyword(ctx, node.GetAlt_id_hint9().GetRule_keyword_schema_uncompat1()); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id& node, TTranslation& ctx) { - // an_id: id | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id::kAltAnId1: - return Id(node.GetAlt_an_id1().GetRule_id1(), ctx); - case TRule_an_id::kAltAnId2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id2().GetToken1())); - default: + } +} + +static TString Id(const TRule_an_id_pure& node, TTranslation& ctx) { + // an_id_pure: identifier | STRING_VALUE; + switch (node.Alt_case()) { + case TRule_an_id_pure::kAltAnIdPure1: + return Id(node.GetAlt_an_id_pure1().GetRule_identifier1(), ctx); + case TRule_an_id_pure::kAltAnIdPure2: + return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_pure2().GetToken1())); + default: Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_schema& node, TTranslation& ctx) { - // an_id_schema: id_schema | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_schema::kAltAnIdSchema1: - return Id(node.GetAlt_an_id_schema1().GetRule_id_schema1(), ctx); - case TRule_an_id_schema::kAltAnIdSchema2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_schema2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_expr& node, TTranslation& ctx) { - // an_id_expr: id_expr | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_expr::kAltAnIdExpr1: - return Id(node.GetAlt_an_id_expr1().GetRule_id_expr1(), ctx); - case TRule_an_id_expr::kAltAnIdExpr2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_expr2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_window& node, TTranslation& ctx) { - // an_id_window: id_window | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_window::kAltAnIdWindow1: - return Id(node.GetAlt_an_id_window1().GetRule_id_window1(), ctx); - case TRule_an_id_window::kAltAnIdWindow2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_window2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_without& node, TTranslation& ctx) { - // an_id_without: id_without | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_without::kAltAnIdWithout1: - return Id(node.GetAlt_an_id_without1().GetRule_id_without1(), ctx); - case TRule_an_id_without::kAltAnIdWithout2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_without2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_hint& node, TTranslation& ctx) { - // an_id_hint: id_hint | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_hint::kAltAnIdHint1: - return Id(node.GetAlt_an_id_hint1().GetRule_id_hint1(), ctx); - case TRule_an_id_hint::kAltAnIdHint2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_hint2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -static TString Id(const TRule_an_id_pure& node, TTranslation& ctx) { - // an_id_pure: identifier | STRING_VALUE; - switch (node.Alt_case()) { - case TRule_an_id_pure::kAltAnIdPure1: - return Id(node.GetAlt_an_id_pure1().GetRule_identifier1(), ctx); - case TRule_an_id_pure::kAltAnIdPure2: - return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id_pure2().GetToken1())); - default: - Y_FAIL("You should change implementation according to grammar changes"); - } -} - -template<typename TRule> -static TIdentifier IdEx(const TRule& node, TTranslation& ctx) { + } +} + +template<typename TRule> +static TIdentifier IdEx(const TRule& node, TTranslation& ctx) { const TString name(Id(node, ctx)); const TPosition pos(ctx.Context().Pos()); return TIdentifier(pos, name); @@ -518,53 +518,53 @@ static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& c if (!node.HasBlock1()) { return defaultStr; } - return Id(node.GetBlock1().GetRule_an_id1(), ctx); + return Id(node.GetBlock1().GetRule_an_id1(), ctx); } static TString OptIdPrefixAsStr(const TRule_opt_id_prefix_or_type& node, TTranslation& ctx, const TString& defaultStr = {}) { if (!node.HasBlock1()) { return defaultStr; } - return Id(node.GetBlock1().GetRule_an_id_or_type1(), ctx); + return Id(node.GetBlock1().GetRule_an_id_or_type1(), ctx); } static void PureColumnListStr(const TRule_pure_column_list& node, TTranslation& ctx, TVector<TString>& outList) { - outList.push_back(Id(node.GetRule_an_id2(), ctx)); + outList.push_back(Id(node.GetRule_an_id2(), ctx)); for (auto& block: node.GetBlock3()) { - outList.push_back(Id(block.GetRule_an_id2(), ctx)); - } -} - -static bool NamedNodeImpl(const TRule_bind_parameter& node, TString& name, TTranslation& ctx) { - // bind_parameter: DOLLAR an_id_or_type; - auto id = Id(node.GetRule_an_id_or_type2(), ctx); - auto dollar = ctx.Token(node.GetToken1()); - if (id.empty()) { - ctx.Error() << "Empty symbol name is not allowed"; - return false; - } - - name = dollar + id; - return true; -} - -static bool NamedNodeImpl(const TRule_opt_bind_parameter& node, TString& name, bool& isOptional, TTranslation& ctx) { - // opt_bind_parameter: bind_parameter QUESTION?; - isOptional = false; - if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name, ctx)) { + outList.push_back(Id(block.GetRule_an_id2(), ctx)); + } +} + +static bool NamedNodeImpl(const TRule_bind_parameter& node, TString& name, TTranslation& ctx) { + // bind_parameter: DOLLAR an_id_or_type; + auto id = Id(node.GetRule_an_id_or_type2(), ctx); + auto dollar = ctx.Token(node.GetToken1()); + if (id.empty()) { + ctx.Error() << "Empty symbol name is not allowed"; + return false; + } + + name = dollar + id; + return true; +} + +static bool NamedNodeImpl(const TRule_opt_bind_parameter& node, TString& name, bool& isOptional, TTranslation& ctx) { + // opt_bind_parameter: bind_parameter QUESTION?; + isOptional = false; + if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name, ctx)) { return false; } - isOptional = node.HasBlock2(); - return true; + isOptional = node.HasBlock2(); + return true; } static TDeferredAtom PureColumnOrNamed(const TRule_pure_column_or_named& node, TTranslation& ctx) { switch (node.Alt_case()) { case TRule_pure_column_or_named::kAltPureColumnOrNamed1: { - TString named; - if (!NamedNodeImpl(node.GetAlt_pure_column_or_named1().GetRule_bind_parameter1(), named, ctx)) { - return {}; - } + TString named; + if (!NamedNodeImpl(node.GetAlt_pure_column_or_named1().GetRule_bind_parameter1(), named, ctx)) { + return {}; + } auto namedNode = ctx.GetNamedNode(named); if (!namedNode) { return {}; @@ -574,7 +574,7 @@ static TDeferredAtom PureColumnOrNamed(const TRule_pure_column_or_named& node, T } case TRule_pure_column_or_named::kAltPureColumnOrNamed2: - return TDeferredAtom(ctx.Context().Pos(), Id(node.GetAlt_pure_column_or_named2().GetRule_an_id1(), ctx)); + return TDeferredAtom(ctx.Context().Pos(), Id(node.GetAlt_pure_column_or_named2().GetRule_an_id1(), ctx)); default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -713,7 +713,7 @@ static std::pair<TString, TString> TableKeyImpl(const TRule_table_key& node, TTr auto name(Id(node.GetRule_id_table_or_type1(), ctx)); TString view; if (node.HasBlock2()) { - view = Id(node.GetBlock2().GetRule_an_id2(), ctx); + view = Id(node.GetBlock2().GetRule_an_id2(), ctx); ctx.Context().IncrementMonCounter("sql_features", "View"); } @@ -722,7 +722,7 @@ static std::pair<TString, TString> TableKeyImpl(const TRule_table_key& node, TTr /// \return optional prefix static TString ColumnNameAsStr(TTranslation& ctx, const TRule_column_name& node, TString& id) { - id = Id(node.GetRule_an_id2(), ctx); + id = Id(node.GetRule_an_id2(), ctx); return OptIdPrefixAsStr(node.GetRule_opt_id_prefix1(), ctx); } @@ -742,10 +742,10 @@ static void FillTargetList(TTranslation& ctx, const TRule_set_target_list& node, TTableHints GetContextHints(TContext& ctx) { TTableHints hints; if (ctx.PragmaInferSchema) { - hints["infer_schema"] = {}; + hints["infer_schema"] = {}; } if (ctx.PragmaDirectRead) { - hints["direct_read"] = {}; + hints["direct_read"] = {}; } return hints; @@ -794,13 +794,13 @@ static bool PackageVersionFromString(const TString& s, ui32& version) { return TryFromString(s, version); } -class TSqlQuery; - -struct TSymbolNameWithPos { - TString Name; - TPosition Pos; -}; - +class TSqlQuery; + +struct TSymbolNameWithPos { + TString Name; + TPosition Pos; +}; + class TSqlTranslation: public TTranslation { protected: TSqlTranslation(TContext& ctx, NSQLTranslation::ESqlMode mode) @@ -817,27 +817,27 @@ protected: GroupBy, SqlLambdaParams, }; - TNodePtr NamedExpr(const TRule_named_expr& node, EExpr exprMode = EExpr::Regular); - bool NamedExprList(const TRule_named_expr_list& node, TVector<TNodePtr>& exprs, EExpr exprMode = EExpr::Regular); - bool BindList(const TRule_bind_parameter_list& node, TVector<TSymbolNameWithPos>& bindNames); - bool ActionOrSubqueryArgs(const TRule_action_or_subquery_args& node, TVector<TSymbolNameWithPos>& bindNames, ui32& optionalArgsCount); + TNodePtr NamedExpr(const TRule_named_expr& node, EExpr exprMode = EExpr::Regular); + bool NamedExprList(const TRule_named_expr_list& node, TVector<TNodePtr>& exprs, EExpr exprMode = EExpr::Regular); + bool BindList(const TRule_bind_parameter_list& node, TVector<TSymbolNameWithPos>& bindNames); + bool ActionOrSubqueryArgs(const TRule_action_or_subquery_args& node, TVector<TSymbolNameWithPos>& bindNames, ui32& optionalArgsCount); bool ModulePath(const TRule_module_path& node, TVector<TString>& path); - bool NamedBindList(const TRule_named_bind_parameter_list& node, TVector<TSymbolNameWithPos>& names, - TVector<TSymbolNameWithPos>& aliases); - bool NamedBindParam(const TRule_named_bind_parameter& node, TSymbolNameWithPos& name, TSymbolNameWithPos& alias); - TNodePtr NamedNode(const TRule_named_nodes_stmt& rule, TVector<TSymbolNameWithPos>& names); + bool NamedBindList(const TRule_named_bind_parameter_list& node, TVector<TSymbolNameWithPos>& names, + TVector<TSymbolNameWithPos>& aliases); + bool NamedBindParam(const TRule_named_bind_parameter& node, TSymbolNameWithPos& name, TSymbolNameWithPos& alias); + TNodePtr NamedNode(const TRule_named_nodes_stmt& rule, TVector<TSymbolNameWithPos>& names); - bool ImportStatement(const TRule_import_stmt& stmt, TVector<TString>* namesPtr = nullptr); + bool ImportStatement(const TRule_import_stmt& stmt, TVector<TString>* namesPtr = nullptr); TNodePtr DoStatement(const TRule_do_stmt& stmt, bool makeLambda, const TVector<TString>& args = {}); - bool DefineActionOrSubqueryStatement(const TRule_define_action_or_subquery_stmt& stmt); - bool DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& blocks, const TRule_define_action_or_subquery_body& body); + bool DefineActionOrSubqueryStatement(const TRule_define_action_or_subquery_stmt& stmt); + bool DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& blocks, const TRule_define_action_or_subquery_body& body); TNodePtr IfStatement(const TRule_if_stmt& stmt); TNodePtr ForStatement(const TRule_for_stmt& stmt); TMaybe<TTableArg> TableArgImpl(const TRule_table_arg& node); bool TableRefImpl(const TRule_table_ref& node, TTableRef& result, bool unorderedSubquery); TMaybe<TSourcePtr> AsTableImpl(const TRule_table_ref& node); bool ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, TString& service, TDeferredAtom& cluster); - bool ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding); + bool ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding); TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node); bool CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params); @@ -851,16 +851,16 @@ protected: bool alter = false); bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings); - TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); - TNodePtr TypeDecimal(const TRule_type_name_decimal& node); - TNodePtr AddOptionals(const TNodePtr& node, size_t optionalCount); + TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); + TNodePtr TypeDecimal(const TRule_type_name_decimal& node); + TNodePtr AddOptionals(const TNodePtr& node, size_t optionalCount); TMaybe<std::pair<TVector<TNodePtr>, bool>> CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted); TNodePtr IntegerOrBind(const TRule_integer_or_bind& node); TNodePtr TypeNameTag(const TRule_type_name_tag& node); TNodePtr TypeNodeOrBind(const TRule_type_name_or_bind& node); - TNodePtr TypeNode(const TRule_type_name& node); - TNodePtr TypeNode(const TRule_type_name_composite& node); + TNodePtr TypeNode(const TRule_type_name& node); + TNodePtr TypeNode(const TRule_type_name_composite& node); TNodePtr ValueConstructorLiteral(const TRule_value_constructor_literal& node); TNodePtr ValueConstructor(const TRule_value_constructor& node); TNodePtr ListLiteral(const TRule_list_literal& node); @@ -869,27 +869,27 @@ protected: TMaybe<TTableHints> TableHintsImpl(const TRule_table_hints& node); bool TableHintImpl(const TRule_table_hint& rule, TTableHints& hints); bool SimpleTableRefImpl(const TRule_simple_table_ref& node, TTableRef& result); - TWindowSpecificationPtr WindowSpecification(const TRule_window_specification_details& rule); - bool OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy); - bool SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs); - - bool IsDistinctOptSet(const TRule_opt_set_quantifier& node) const; - bool IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const; - - bool RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles); - bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) ; -private: - static bool IsValidFrameSettings(TContext& ctx, const TFrameBound& begin, const TFrameBound& end); - static TString FrameSettingsToString(EFrameSettings settings, bool isUnbounded); - - bool FrameBound(const TRule_window_frame_bound& rule, TFrameBoundPtr& bound); - bool FrameClause(const TRule_window_frame_clause& node, TFrameSpecificationPtr& frameSpec); - bool SortSpecification(const TRule_sort_specification& node, TVector<TSortSpecificationPtr>& sortSpecs); - - bool ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, bool allowBinding, TString& service, TDeferredAtom& cluster, bool& isBinding); - bool ApplyTableBinding(const TString& binding, TTableRef& tr, TTableHints& hints); - bool StructLiteralItem(TVector<TNodePtr>& labels, const TRule_expr& label, TVector<TNodePtr>& values, const TRule_expr& value); -protected: + TWindowSpecificationPtr WindowSpecification(const TRule_window_specification_details& rule); + bool OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy); + bool SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs); + + bool IsDistinctOptSet(const TRule_opt_set_quantifier& node) const; + bool IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const; + + bool RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles); + bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) ; +private: + static bool IsValidFrameSettings(TContext& ctx, const TFrameBound& begin, const TFrameBound& end); + static TString FrameSettingsToString(EFrameSettings settings, bool isUnbounded); + + bool FrameBound(const TRule_window_frame_bound& rule, TFrameBoundPtr& bound); + bool FrameClause(const TRule_window_frame_clause& node, TFrameSpecificationPtr& frameSpec); + bool SortSpecification(const TRule_sort_specification& node, TVector<TSortSpecificationPtr>& sortSpecs); + + bool ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, bool allowBinding, TString& service, TDeferredAtom& cluster, bool& isBinding); + bool ApplyTableBinding(const TString& binding, TTableRef& tr, TTableHints& hints); + bool StructLiteralItem(TVector<TNodePtr>& labels, const TRule_expr& label, TVector<TNodePtr>& values, const TRule_expr& value); +protected: NSQLTranslation::ESqlMode Mode; }; @@ -902,7 +902,7 @@ public: SqlLambdaParams, }; - TSqlExpression(TContext& ctx, NSQLTranslation::ESqlMode mode) + TSqlExpression(TContext& ctx, NSQLTranslation::ESqlMode mode) : TSqlTranslation(ctx, mode) { } @@ -910,15 +910,15 @@ public: TNodePtr Build(const TRule_expr& node) { // expr: // or_subexpr (OR or_subexpr)* - // | type_name_composite + // | type_name_composite switch (node.Alt_case()) { case TRule_expr::kAltExpr1: { auto getNode = [](const TRule_expr_TAlt1_TBlock2& b) -> const TRule_or_subexpr& { return b.GetRule_or_subexpr2(); }; return BinOper("Or", node.GetAlt_expr1().GetRule_or_subexpr1(), getNode, - node.GetAlt_expr1().GetBlock2().begin(), node.GetAlt_expr1().GetBlock2().end(), {}); + node.GetAlt_expr1().GetBlock2().begin(), node.GetAlt_expr1().GetBlock2().end(), {}); } case TRule_expr::kAltExpr2: { - return TypeNode(node.GetAlt_expr2().GetRule_type_name_composite1()); + return TypeNode(node.GetAlt_expr2().GetRule_type_name_composite1()); } default: Y_FAIL("You should change implementation according to grammar changes"); @@ -929,46 +929,46 @@ public: SmartParenthesisMode = mode; } - TMaybe<TExprOrIdent> LiteralExpr(const TRule_literal_value& node); + TMaybe<TExprOrIdent> LiteralExpr(const TRule_literal_value& node); private: - struct TTrailingQuestions { - size_t Count = 0; - TPosition Pos; - }; - - TNodePtr BindParameterRule(const TRule_bind_parameter& rule, const TTrailingQuestions& tail); - TNodePtr LambdaRule(const TRule_lambda& rule); - TNodePtr CastRule(const TRule_cast_expr& rule); + struct TTrailingQuestions { + size_t Count = 0; + TPosition Pos; + }; + + TNodePtr BindParameterRule(const TRule_bind_parameter& rule, const TTrailingQuestions& tail); + TNodePtr LambdaRule(const TRule_lambda& rule); + TNodePtr CastRule(const TRule_cast_expr& rule); TNodePtr BitCastRule(const TRule_bitcast_expr& rule); - TNodePtr ExistsRule(const TRule_exists_expr& rule); - TNodePtr CaseRule(const TRule_case_expr& rule); - - TMaybe<TExprOrIdent> AtomExpr(const TRule_atom_expr& node, const TTrailingQuestions& tail); - TMaybe<TExprOrIdent> InAtomExpr(const TRule_in_atom_expr& node, const TTrailingQuestions& tail); + TNodePtr ExistsRule(const TRule_exists_expr& rule); + TNodePtr CaseRule(const TRule_case_expr& rule); + + TMaybe<TExprOrIdent> AtomExpr(const TRule_atom_expr& node, const TTrailingQuestions& tail); + TMaybe<TExprOrIdent> InAtomExpr(const TRule_in_atom_expr& node, const TTrailingQuestions& tail); - TNodePtr JsonInputArg(const TRule_json_common_args& node); + TNodePtr JsonInputArg(const TRule_json_common_args& node); TNodePtr JsonPathSpecification(const TRule_jsonpath_spec& node); TNodePtr JsonReturningTypeRule(const TRule_type_name_simple& node); - TNodePtr JsonValueCaseHandler(const TRule_json_case_handler& node, EJsonValueHandlerMode& mode); - void AddJsonValueCaseHandlers(const TRule_json_value& node, TVector<TNodePtr>& children); - void AddJsonVariable(const TRule_json_variable& node, TVector<TNodePtr>& children); - void AddJsonVariables(const TRule_json_variables& node, TVector<TNodePtr>& children); - TNodePtr JsonVariables(const TRule_json_common_args& node); - void AddJsonCommonArgs(const TRule_json_common_args& node, TVector<TNodePtr>& children); - TNodePtr JsonValueExpr(const TRule_json_value& node); - void AddJsonExistsHandler(const TRule_json_exists& node, TVector<TNodePtr>& children); - TNodePtr JsonExistsExpr(const TRule_json_exists& node); - EJsonQueryWrap JsonQueryWrapper(const TRule_json_query& node); + TNodePtr JsonValueCaseHandler(const TRule_json_case_handler& node, EJsonValueHandlerMode& mode); + void AddJsonValueCaseHandlers(const TRule_json_value& node, TVector<TNodePtr>& children); + void AddJsonVariable(const TRule_json_variable& node, TVector<TNodePtr>& children); + void AddJsonVariables(const TRule_json_variables& node, TVector<TNodePtr>& children); + TNodePtr JsonVariables(const TRule_json_common_args& node); + void AddJsonCommonArgs(const TRule_json_common_args& node, TVector<TNodePtr>& children); + TNodePtr JsonValueExpr(const TRule_json_value& node); + void AddJsonExistsHandler(const TRule_json_exists& node, TVector<TNodePtr>& children); + TNodePtr JsonExistsExpr(const TRule_json_exists& node); + EJsonQueryWrap JsonQueryWrapper(const TRule_json_query& node); EJsonQueryHandler JsonQueryHandler(const TRule_json_query_handler& node); - TNodePtr JsonQueryExpr(const TRule_json_query& node); - TNodePtr JsonApiExpr(const TRule_json_api_expr& node); + TNodePtr JsonQueryExpr(const TRule_json_query& node); + TNodePtr JsonApiExpr(const TRule_json_api_expr& node); template<typename TUnaryCasualExprRule> - TNodePtr UnaryCasualExpr(const TUnaryCasualExprRule& node, const TTrailingQuestions& tail); + TNodePtr UnaryCasualExpr(const TUnaryCasualExprRule& node, const TTrailingQuestions& tail); + + template<typename TUnarySubExprRule> + TNodePtr UnaryExpr(const TUnarySubExprRule& node, const TTrailingQuestions& tail); - template<typename TUnarySubExprRule> - TNodePtr UnaryExpr(const TUnarySubExprRule& node, const TTrailingQuestions& tail); - bool SqlLambdaParams(const TNodePtr& node, TVector<TSymbolNameWithPos>& args, ui32& optionalArgumentsCount); bool SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& node, TVector<TNodePtr>& exprSeq); bool SqlLambdaExprBody(TContext& ctx, const TRule_expr& node, TVector<TNodePtr>& exprSeq); @@ -978,106 +978,106 @@ private: return expr.Build(node.GetRule_expr2()); } - TNodePtr SubExpr(const TRule_con_subexpr& node, const TTrailingQuestions& tail); - TNodePtr SubExpr(const TRule_xor_subexpr& node, const TTrailingQuestions& tail); + TNodePtr SubExpr(const TRule_con_subexpr& node, const TTrailingQuestions& tail); + TNodePtr SubExpr(const TRule_xor_subexpr& node, const TTrailingQuestions& tail); - TNodePtr SubExpr(const TRule_mul_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_mul_subexpr& node, const TTrailingQuestions& tail) { // mul_subexpr: con_subexpr (DOUBLE_PIPE con_subexpr)*; auto getNode = [](const TRule_mul_subexpr::TBlock2& b) -> const TRule_con_subexpr& { return b.GetRule_con_subexpr2(); }; - return BinOper("Concat", node.GetRule_con_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + return BinOper("Concat", node.GetRule_con_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); } - TNodePtr SubExpr(const TRule_add_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_add_subexpr& node, const TTrailingQuestions& tail) { // add_subexpr: mul_subexpr ((ASTERISK | SLASH | PERCENT) mul_subexpr)*; auto getNode = [](const TRule_add_subexpr::TBlock2& b) -> const TRule_mul_subexpr& { return b.GetRule_mul_subexpr2(); }; - return BinOpList(node.GetRule_mul_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + return BinOpList(node.GetRule_mul_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); } - TNodePtr SubExpr(const TRule_bit_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_bit_subexpr& node, const TTrailingQuestions& tail) { // bit_subexpr: add_subexpr ((PLUS | MINUS) add_subexpr)*; auto getNode = [](const TRule_bit_subexpr::TBlock2& b) -> const TRule_add_subexpr& { return b.GetRule_add_subexpr2(); }; - return BinOpList(node.GetRule_add_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); - } - - TNodePtr SubExpr(const TRule_neq_subexpr& node, const TTrailingQuestions& tailExternal) { - //neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)* - // // trailing QUESTIONS are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x) - // ((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?; - YQL_ENSURE(tailExternal.Count == 0); - TTrailingQuestions tail; - if (node.HasBlock3() && node.GetBlock3().Alt_case() == TRule_neq_subexpr::TBlock3::kAlt2) { - auto& questions = node.GetBlock3().GetAlt2(); - tail.Count = questions.GetBlock1().size(); - tail.Pos = Ctx.TokenPosition(questions.GetBlock1().begin()->GetToken1()); - YQL_ENSURE(tail.Count > 0); - } - + return BinOpList(node.GetRule_add_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + } + + TNodePtr SubExpr(const TRule_neq_subexpr& node, const TTrailingQuestions& tailExternal) { + //neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)* + // // trailing QUESTIONS are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x) + // ((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?; + YQL_ENSURE(tailExternal.Count == 0); + TTrailingQuestions tail; + if (node.HasBlock3() && node.GetBlock3().Alt_case() == TRule_neq_subexpr::TBlock3::kAlt2) { + auto& questions = node.GetBlock3().GetAlt2(); + tail.Count = questions.GetBlock1().size(); + tail.Pos = Ctx.TokenPosition(questions.GetBlock1().begin()->GetToken1()); + YQL_ENSURE(tail.Count > 0); + } + auto getNode = [](const TRule_neq_subexpr::TBlock2& b) -> const TRule_bit_subexpr& { return b.GetRule_bit_subexpr2(); }; - auto result = BinOpList(node.GetRule_bit_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + auto result = BinOpList(node.GetRule_bit_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); if (!result) { return {}; } if (node.HasBlock3()) { - auto& block = node.GetBlock3(); - if (block.Alt_case() == TRule_neq_subexpr::TBlock3::kAlt1) { - TSqlExpression altExpr(Ctx, Mode); - auto altResult = SubExpr(block.GetAlt1().GetRule_neq_subexpr2(), {}); - if (!altResult) { - return {}; - } - const TVector<TNodePtr> args({result, altResult}); - Token(block.GetAlt1().GetRule_double_question1().GetToken1()); - result = BuildBuiltinFunc(Ctx, Ctx.Pos(), "Coalesce", args); + auto& block = node.GetBlock3(); + if (block.Alt_case() == TRule_neq_subexpr::TBlock3::kAlt1) { + TSqlExpression altExpr(Ctx, Mode); + auto altResult = SubExpr(block.GetAlt1().GetRule_neq_subexpr2(), {}); + if (!altResult) { + return {}; + } + const TVector<TNodePtr> args({result, altResult}); + Token(block.GetAlt1().GetRule_double_question1().GetToken1()); + result = BuildBuiltinFunc(Ctx, Ctx.Pos(), "Coalesce", args); } } return result; } - TNodePtr SubExpr(const TRule_eq_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_eq_subexpr& node, const TTrailingQuestions& tail) { // eq_subexpr: neq_subexpr ((LESS | LESS_OR_EQ | GREATER | GREATER_OR_EQ) neq_subexpr)*; auto getNode = [](const TRule_eq_subexpr::TBlock2& b) -> const TRule_neq_subexpr& { return b.GetRule_neq_subexpr2(); }; - return BinOpList(node.GetRule_neq_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + return BinOpList(node.GetRule_neq_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); } - TNodePtr SubExpr(const TRule_or_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_or_subexpr& node, const TTrailingQuestions& tail) { // or_subexpr: and_subexpr (AND and_subexpr)*; auto getNode = [](const TRule_or_subexpr::TBlock2& b) -> const TRule_and_subexpr& { return b.GetRule_and_subexpr2(); }; - return BinOper("And", node.GetRule_and_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + return BinOper("And", node.GetRule_and_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); } - TNodePtr SubExpr(const TRule_and_subexpr& node, const TTrailingQuestions& tail) { + TNodePtr SubExpr(const TRule_and_subexpr& node, const TTrailingQuestions& tail) { // and_subexpr: xor_subexpr (XOR xor_subexpr)*; auto getNode = [](const TRule_and_subexpr::TBlock2& b) -> const TRule_xor_subexpr& { return b.GetRule_xor_subexpr2(); }; - return BinOper("Xor", node.GetRule_xor_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); + return BinOper("Xor", node.GetRule_xor_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail); } template <typename TNode, typename TGetNode, typename TIter> - TNodePtr BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); + TNodePtr BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); template <typename TGetNode, typename TIter> - TNodePtr BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); + TNodePtr BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); + + template <typename TGetNode, typename TIter> + TNodePtr BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); - template <typename TGetNode, typename TIter> - TNodePtr BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); - TNodePtr BinOperList(const TString& opName, TVector<TNodePtr>::const_iterator begin, TVector<TNodePtr>::const_iterator end) const; - struct TCaseBranch { - TNodePtr Pred; - TNodePtr Value; - }; - TCaseBranch ReduceCaseBranches(TVector<TCaseBranch>::const_iterator begin, TVector<TCaseBranch>::const_iterator end) const; - + struct TCaseBranch { + TNodePtr Pred; + TNodePtr Value; + }; + TCaseBranch ReduceCaseBranches(TVector<TCaseBranch>::const_iterator begin, TVector<TCaseBranch>::const_iterator end) const; + template <typename TNode, typename TGetNode, typename TIter> - TNodePtr BinOper(const TString& operName, const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); - - TNodePtr SqlInExpr(const TRule_in_expr& node, const TTrailingQuestions& tail); - - void UnexpectedQuestionToken(const TTrailingQuestions& tail) { - YQL_ENSURE(tail.Count > 0); - Ctx.Error(tail.Pos) << "Unexpected token '?' at the end of expression"; - } - + TNodePtr BinOper(const TString& operName, const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail); + + TNodePtr SqlInExpr(const TRule_in_expr& node, const TTrailingQuestions& tail); + + void UnexpectedQuestionToken(const TTrailingQuestions& tail) { + YQL_ENSURE(tail.Count > 0); + Ctx.Error(tail.Pos) << "Unexpected token '?' at the end of expression"; + } + TNodePtr SmartParenthesis(const TRule_smart_parenthesis& node); ESmartParenthesis SmartParenthesisMode = ESmartParenthesis::Default; @@ -1087,7 +1087,7 @@ private: class TSqlCallExpr: public TSqlTranslation { public: - TSqlCallExpr(TContext& ctx, NSQLTranslation::ESqlMode mode) + TSqlCallExpr(TContext& ctx, NSQLTranslation::ESqlMode mode) : TSqlTranslation(ctx, mode) { } @@ -1100,28 +1100,28 @@ public: , Node(call.Node) , Args(args) , AggMode(call.AggMode) - , DistinctAllowed(call.DistinctAllowed) - , UsingCallExpr(call.UsingCallExpr) + , DistinctAllowed(call.DistinctAllowed) + , UsingCallExpr(call.UsingCallExpr) , IsExternalCall(call.IsExternalCall) , CallConfig(call.CallConfig) { } - void AllowDistinct() { - DistinctAllowed = true; - } - + void AllowDistinct() { + DistinctAllowed = true; + } + void InitName(const TString& name); void InitExpr(const TNodePtr& expr); - bool Init(const TRule_using_call_expr& node); + bool Init(const TRule_using_call_expr& node); bool Init(const TRule_value_constructor& node); bool Init(const TRule_invoke_expr& node); bool ConfigureExternalCall(const TRule_external_call_settings& node); void IncCounters(); - TNodePtr BuildUdf(bool forReduce) { - auto result = Node ? Node : BuildCallable(Pos, Module, Func, Args, forReduce); + TNodePtr BuildUdf(bool forReduce) { + auto result = Node ? Node : BuildCallable(Pos, Module, Func, Args, forReduce); if (to_lower(Module) == "tensorflow" && Func == "RunBatch") { Args.erase(Args.begin() + 2); } @@ -1130,11 +1130,11 @@ public: TNodePtr BuildCall() { TVector<TNodePtr> args; - bool warnOnYqlNameSpace = true; + bool warnOnYqlNameSpace = true; if (Node && !Node->FuncName()) { Module = "YQL"; Func = NamedArgs.empty() ? "Apply" : "NamedApply"; - warnOnYqlNameSpace = false; + warnOnYqlNameSpace = false; args.push_back(Node); } @@ -1170,7 +1170,7 @@ public: args.insert(args.end(), Args.begin(), Args.end()); } - auto result = BuildBuiltinFunc(Ctx, Pos, Func, args, Module, AggMode, &mustUseNamed, warnOnYqlNameSpace); + auto result = BuildBuiltinFunc(Ctx, Pos, Func, args, Module, AggMode, &mustUseNamed, warnOnYqlNameSpace); if (mustUseNamed) { Error() << "Named args are used for call, but unsupported by function: " << Func; return nullptr; @@ -1205,9 +1205,9 @@ public: } private: - bool ExtractCallParam(const TRule_external_call_param& node); - bool FillArg(const TString& module, const TString& func, size_t& idx, const TRule_named_expr& node); - bool FillArgs(const TRule_named_expr_list& node); + bool ExtractCallParam(const TRule_external_call_param& node); + bool FillArg(const TString& module, const TString& func, size_t& idx, const TRule_named_expr& node); + bool FillArgs(const TRule_named_expr_list& node); private: TPosition Pos; @@ -1219,14 +1219,14 @@ private: TVector<TNodePtr> NamedArgs; EAggregateMode AggMode = EAggregateMode::Normal; TString WindowName; - bool DistinctAllowed = false; - bool UsingCallExpr = false; + bool DistinctAllowed = false; + bool UsingCallExpr = false; bool IsExternalCall = false; TFunctionConfig CallConfig; }; -TNodePtr TSqlTranslation::NamedExpr(const TRule_named_expr& node, EExpr exprMode) { - TSqlExpression expr(Ctx, Mode); +TNodePtr TSqlTranslation::NamedExpr(const TRule_named_expr& node, EExpr exprMode) { + TSqlExpression expr(Ctx, Mode); if (exprMode == EExpr::GroupBy) { expr.SetSmartParenthesisMode(TSqlExpression::ESmartParenthesis::GroupBy); } else if (exprMode == EExpr::SqlLambdaParams) { @@ -1239,18 +1239,18 @@ TNodePtr TSqlTranslation::NamedExpr(const TRule_named_expr& node, EExpr exprMode } if (node.HasBlock2()) { exprNode = SafeClone(exprNode); - exprNode->SetLabel(Id(node.GetBlock2().GetRule_an_id_or_type2(), *this)); + exprNode->SetLabel(Id(node.GetBlock2().GetRule_an_id_or_type2(), *this)); } return exprNode; } -bool TSqlTranslation::NamedExprList(const TRule_named_expr_list& node, TVector<TNodePtr>& exprs, EExpr exprMode) { - exprs.emplace_back(NamedExpr(node.GetRule_named_expr1(), exprMode)); +bool TSqlTranslation::NamedExprList(const TRule_named_expr_list& node, TVector<TNodePtr>& exprs, EExpr exprMode) { + exprs.emplace_back(NamedExpr(node.GetRule_named_expr1(), exprMode)); if (!exprs.back()) { return false; } for (auto& b: node.GetBlock2()) { - exprs.emplace_back(NamedExpr(b.GetRule_named_expr2(), exprMode)); + exprs.emplace_back(NamedExpr(b.GetRule_named_expr2(), exprMode)); if (!exprs.back()) { return false; } @@ -1258,52 +1258,52 @@ bool TSqlTranslation::NamedExprList(const TRule_named_expr_list& node, TVector<T return true; } -bool TSqlTranslation::BindList(const TRule_bind_parameter_list& node, TVector<TSymbolNameWithPos>& bindNames) { - bindNames.clear(); - - TString name; - if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name, *this)) { - return false; - } - - bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); - for (auto& b: node.GetBlock2()) { - if (!NamedNodeImpl(b.GetRule_bind_parameter2(), name, *this)) { - return false; - } - - bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); - } - return true; -} - -bool TSqlTranslation::ActionOrSubqueryArgs(const TRule_action_or_subquery_args& node, TVector<TSymbolNameWithPos>& bindNames, ui32& optionalArgsCount) { - bindNames.clear(); - optionalArgsCount = 0; - - TString name; - bool isOptional = false; - if (!NamedNodeImpl(node.GetRule_opt_bind_parameter1(), name, isOptional, *this)) { - return false; - } - - if (isOptional) { - optionalArgsCount++; - } - bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); +bool TSqlTranslation::BindList(const TRule_bind_parameter_list& node, TVector<TSymbolNameWithPos>& bindNames) { + bindNames.clear(); + + TString name; + if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name, *this)) { + return false; + } + + bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); + for (auto& b: node.GetBlock2()) { + if (!NamedNodeImpl(b.GetRule_bind_parameter2(), name, *this)) { + return false; + } + + bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); + } + return true; +} + +bool TSqlTranslation::ActionOrSubqueryArgs(const TRule_action_or_subquery_args& node, TVector<TSymbolNameWithPos>& bindNames, ui32& optionalArgsCount) { + bindNames.clear(); + optionalArgsCount = 0; + + TString name; + bool isOptional = false; + if (!NamedNodeImpl(node.GetRule_opt_bind_parameter1(), name, isOptional, *this)) { + return false; + } + + if (isOptional) { + optionalArgsCount++; + } + bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); for (auto& b: node.GetBlock2()) { - if (!NamedNodeImpl(b.GetRule_opt_bind_parameter2(), name, isOptional, *this)) { - return false; - } - - if (isOptional) { - optionalArgsCount++; - } else if (optionalArgsCount > 0) { - Context().Error() << "Non-optional argument can not follow optional one"; - return false; - } - bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); + if (!NamedNodeImpl(b.GetRule_opt_bind_parameter2(), name, isOptional, *this)) { + return false; + } + + if (isOptional) { + optionalArgsCount++; + } else if (optionalArgsCount > 0) { + Context().Error() << "Non-optional argument can not follow optional one"; + return false; + } + bindNames.emplace_back(TSymbolNameWithPos{name, Ctx.Pos()}); } return true; } @@ -1312,90 +1312,90 @@ bool TSqlTranslation::ModulePath(const TRule_module_path& node, TVector<TString> if (node.HasBlock1()) { path.emplace_back(TString()); } - path.emplace_back(Id(node.GetRule_an_id2(), *this)); + path.emplace_back(Id(node.GetRule_an_id2(), *this)); for (auto& b: node.GetBlock3()) { - path.emplace_back(Id(b.GetRule_an_id2(), *this)); + path.emplace_back(Id(b.GetRule_an_id2(), *this)); } return true; } -bool TSqlTranslation::NamedBindList(const TRule_named_bind_parameter_list& node, TVector<TSymbolNameWithPos>& names, - TVector<TSymbolNameWithPos>& aliases) -{ - names.clear(); - aliases.clear(); - TSymbolNameWithPos name; - TSymbolNameWithPos alias; - - if (!NamedBindParam(node.GetRule_named_bind_parameter1(), name, alias)) { - return false; - } - names.push_back(name); - aliases.push_back(alias); - +bool TSqlTranslation::NamedBindList(const TRule_named_bind_parameter_list& node, TVector<TSymbolNameWithPos>& names, + TVector<TSymbolNameWithPos>& aliases) +{ + names.clear(); + aliases.clear(); + TSymbolNameWithPos name; + TSymbolNameWithPos alias; + + if (!NamedBindParam(node.GetRule_named_bind_parameter1(), name, alias)) { + return false; + } + names.push_back(name); + aliases.push_back(alias); + for (auto& b: node.GetBlock2()) { - if (!NamedBindParam(b.GetRule_named_bind_parameter2(), name, alias)) { - return false; - } - names.push_back(name); - aliases.push_back(alias); + if (!NamedBindParam(b.GetRule_named_bind_parameter2(), name, alias)) { + return false; + } + names.push_back(name); + aliases.push_back(alias); } - return true; + return true; } -bool TSqlTranslation::NamedBindParam(const TRule_named_bind_parameter& node, TSymbolNameWithPos& name, TSymbolNameWithPos& alias) { - name = alias = {}; - if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name.Name, *this)) { - return false; - } - name.Pos = Ctx.Pos(); +bool TSqlTranslation::NamedBindParam(const TRule_named_bind_parameter& node, TSymbolNameWithPos& name, TSymbolNameWithPos& alias) { + name = alias = {}; + if (!NamedNodeImpl(node.GetRule_bind_parameter1(), name.Name, *this)) { + return false; + } + name.Pos = Ctx.Pos(); if (node.HasBlock2()) { - if (!NamedNodeImpl(node.GetBlock2().GetRule_bind_parameter2(), alias.Name, *this)) { - return false; - } - alias.Pos = Ctx.Pos(); + if (!NamedNodeImpl(node.GetBlock2().GetRule_bind_parameter2(), alias.Name, *this)) { + return false; + } + alias.Pos = Ctx.Pos(); } - return true; + return true; } TMaybe<TTableArg> TSqlTranslation::TableArgImpl(const TRule_table_arg& node) { TTableArg ret; ret.HasAt = node.HasBlock1(); - TColumnRefScope scope(Ctx, EColumnRefState::AsStringLiteral); - ret.Expr = NamedExpr(node.GetRule_named_expr2()); + TColumnRefScope scope(Ctx, EColumnRefState::AsStringLiteral); + ret.Expr = NamedExpr(node.GetRule_named_expr2()); if (!ret.Expr) { return Nothing(); } if (node.HasBlock3()) { - ret.View = Id(node.GetBlock3().GetRule_an_id2(), *this); + ret.View = Id(node.GetBlock3().GetRule_an_id2(), *this); Context().IncrementMonCounter("sql_features", "View"); } return ret; } -bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, TString& service, TDeferredAtom& cluster) { - bool allowBinding = false; - bool isBinding; - return ClusterExpr(node, allowWildcard, allowBinding, service, cluster, isBinding); -} - -bool TSqlTranslation::ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding) { - bool allowWildcard = false; - bool allowBinding = true; - return ClusterExpr(node, allowWildcard, allowBinding, service, cluster, isBinding); -} - -bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, bool allowBinding, TString& service, - TDeferredAtom& cluster, bool& isBinding) -{ +bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, TString& service, TDeferredAtom& cluster) { + bool allowBinding = false; + bool isBinding; + return ClusterExpr(node, allowWildcard, allowBinding, service, cluster, isBinding); +} + +bool TSqlTranslation::ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding) { + bool allowWildcard = false; + bool allowBinding = true; + return ClusterExpr(node, allowWildcard, allowBinding, service, cluster, isBinding); +} + +bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, bool allowBinding, TString& service, + TDeferredAtom& cluster, bool& isBinding) +{ service = ""; cluster = TDeferredAtom(); - isBinding = false; + isBinding = false; if (node.HasBlock1()) { - service = to_lower(Id(node.GetBlock1().GetRule_an_id1(), *this)); - allowBinding = false; + service = to_lower(Id(node.GetBlock1().GetRule_an_id1(), *this)); + allowBinding = false; if (service != YtProviderName && service != KikimrProviderName && service != RtmrProviderName && service != StatProviderName) { @@ -1413,10 +1413,10 @@ bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWild if (value.GetLiteral()) { TString clusterName = *value.GetLiteral(); - if (allowBinding && to_lower(clusterName) == "bindings") { - isBinding = true; - return true; - } + if (allowBinding && to_lower(clusterName) == "bindings") { + isBinding = true; + return true; + } TString normalizedClusterName; auto foundProvider = Ctx.GetClusterProvider(clusterName, normalizedClusterName); if (!foundProvider) { @@ -1460,135 +1460,135 @@ bool TSqlTranslation::ClusterExpr(const TRule_cluster_expr& node, bool allowWild bool ExprList(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule_expr_list& node); -bool TSqlTranslation::ApplyTableBinding(const TString& binding, TTableRef& tr, TTableHints& hints) { - const auto& settings = Context().Settings; - auto pit = settings.PrivateBindings.find(binding); - auto sit = settings.ScopedBindings.find(binding); - - if (pit == settings.PrivateBindings.end() && sit == settings.ScopedBindings.end()) { - Ctx.Error() << "Table binding `" << binding << "` is not defined"; - return false; - } - - if (pit != settings.PrivateBindings.end() && sit != settings.ScopedBindings.end()) { - // warn - Ctx.Warning(Ctx.Pos(), - TIssuesIds::YQL_TABLE_BINDING_DUPLICATE) << "Table binding `" << binding << "` is defined both as private and scoped binding"; - } - - const auto& bindSettings = (pit != settings.PrivateBindings.end()) ? pit->second : sit->second; - +bool TSqlTranslation::ApplyTableBinding(const TString& binding, TTableRef& tr, TTableHints& hints) { + const auto& settings = Context().Settings; + auto pit = settings.PrivateBindings.find(binding); + auto sit = settings.ScopedBindings.find(binding); + + if (pit == settings.PrivateBindings.end() && sit == settings.ScopedBindings.end()) { + Ctx.Error() << "Table binding `" << binding << "` is not defined"; + return false; + } + + if (pit != settings.PrivateBindings.end() && sit != settings.ScopedBindings.end()) { + // warn + Ctx.Warning(Ctx.Pos(), + TIssuesIds::YQL_TABLE_BINDING_DUPLICATE) << "Table binding `" << binding << "` is defined both as private and scoped binding"; + } + + const auto& bindSettings = (pit != settings.PrivateBindings.end()) ? pit->second : sit->second; + if (!IsIn({S3ProviderName, PqProviderName}, bindSettings.ClusterType)) { - Ctx.Error() << "Cluster type " << bindSettings.ClusterType << " is not supported for table bindings"; - return false; - } - - // ordered map ensures AST stability - TMap<TString, TString> kvs(bindSettings.Settings.begin(), bindSettings.Settings.end()); - auto pullSettingOrFail = [&](const TString& name, TString& value) -> bool { - auto it = kvs.find(name); - if (it == kvs.end()) { - Ctx.Error() << name << " is not found for " << binding; - return false; - } - value = it->second; - kvs.erase(it); - return true; - }; - - TString func = "object"; - TString cluster; - TString path; - TString format; - - if (!pullSettingOrFail("cluster", cluster) || - !pullSettingOrFail("path", path) || - !pullSettingOrFail("format", format)) - { - return false; - } - - MergeHints(hints, GetTableFuncHints(func)); - if (auto it = kvs.find("schema"); it != kvs.end()) { - TNodePtr schema = BuildQuotedAtom(Ctx.Pos(), it->second); - - TNodePtr type = new TCallNodeImpl(Ctx.Pos(), "SqlTypeFromYson", { schema }); - TNodePtr columns = new TCallNodeImpl(Ctx.Pos(), "SqlColumnOrderFromYson", { schema }); - - hints["user_schema"] = { type, columns }; - kvs.erase(it); - } - - TVector<TTableArg> args; - for (auto& arg : { path, format }) { - args.emplace_back(); + Ctx.Error() << "Cluster type " << bindSettings.ClusterType << " is not supported for table bindings"; + return false; + } + + // ordered map ensures AST stability + TMap<TString, TString> kvs(bindSettings.Settings.begin(), bindSettings.Settings.end()); + auto pullSettingOrFail = [&](const TString& name, TString& value) -> bool { + auto it = kvs.find(name); + if (it == kvs.end()) { + Ctx.Error() << name << " is not found for " << binding; + return false; + } + value = it->second; + kvs.erase(it); + return true; + }; + + TString func = "object"; + TString cluster; + TString path; + TString format; + + if (!pullSettingOrFail("cluster", cluster) || + !pullSettingOrFail("path", path) || + !pullSettingOrFail("format", format)) + { + return false; + } + + MergeHints(hints, GetTableFuncHints(func)); + if (auto it = kvs.find("schema"); it != kvs.end()) { + TNodePtr schema = BuildQuotedAtom(Ctx.Pos(), it->second); + + TNodePtr type = new TCallNodeImpl(Ctx.Pos(), "SqlTypeFromYson", { schema }); + TNodePtr columns = new TCallNodeImpl(Ctx.Pos(), "SqlColumnOrderFromYson", { schema }); + + hints["user_schema"] = { type, columns }; + kvs.erase(it); + } + + TVector<TTableArg> args; + for (auto& arg : { path, format }) { + args.emplace_back(); args.back().Expr = BuildLiteralRawString(Ctx.Pos(), arg); - } - - for (auto& [key, value] : kvs) { - YQL_ENSURE(!key.empty()); - args.emplace_back(); + } + + for (auto& [key, value] : kvs) { + YQL_ENSURE(!key.empty()); + args.emplace_back(); args.back().Expr = BuildLiteralRawString(Ctx.Pos(), value); - args.back().Expr->SetLabel(key); - } - - tr.Service = bindSettings.ClusterType; - tr.Cluster = TDeferredAtom(Ctx.Pos(), cluster); - tr.Keys = BuildTableKeys(Ctx.Pos(), tr.Service, tr.Cluster, func, args); - return true; -} - + args.back().Expr->SetLabel(key); + } + + tr.Service = bindSettings.ClusterType; + tr.Cluster = TDeferredAtom(Ctx.Pos(), cluster); + tr.Keys = BuildTableKeys(Ctx.Pos(), tr.Service, tr.Cluster, func, args); + return true; +} + bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& result, bool unorderedSubquery) { - // table_ref: - // (cluster_expr DOT)? AT? - // (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)*)? RPAREN | - // bind_parameter (LPAREN expr_list? RPAREN)? (VIEW an_id)?) - // table_hints?; + // table_ref: + // (cluster_expr DOT)? AT? + // (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)*)? RPAREN | + // bind_parameter (LPAREN expr_list? RPAREN)? (VIEW an_id)?) + // table_hints?; if (Mode == NSQLTranslation::ESqlMode::LIMITED_VIEW && node.HasBlock1()) { Ctx.Error() << "Cluster should not be used in limited view"; return false; } auto service = Context().Scoped->CurrService; auto cluster = Context().Scoped->CurrCluster; - const bool hasAt = node.HasBlock2(); - bool isBinding = false; + const bool hasAt = node.HasBlock2(); + bool isBinding = false; if (node.HasBlock1()) { - const auto& clusterExpr = node.GetBlock1().GetRule_cluster_expr1(); - bool result = !hasAt ? - ClusterExprOrBinding(clusterExpr, service, cluster, isBinding) : ClusterExpr(clusterExpr, false, service, cluster); - if (!result) { - return false; + const auto& clusterExpr = node.GetBlock1().GetRule_cluster_expr1(); + bool result = !hasAt ? + ClusterExprOrBinding(clusterExpr, service, cluster, isBinding) : ClusterExpr(clusterExpr, false, service, cluster); + if (!result) { + return false; } } TTableRef tr(Context().MakeName("table"), service, cluster, nullptr); TPosition pos(Context().Pos()); - TTableHints hints = GetContextHints(Ctx); + TTableHints hints = GetContextHints(Ctx); TTableHints tableHints; auto& block = node.GetBlock3(); switch (block.Alt_case()) { case TRule_table_ref::TBlock3::kAlt1: { - if (!isBinding && cluster.Empty()) { + if (!isBinding && cluster.Empty()) { Ctx.Error() << "No cluster name given and no default cluster is selected"; return false; } auto pair = TableKeyImpl(block.GetAlt1().GetRule_table_key1(), *this, hasAt); - if (isBinding) { - TString binding = pair.first; - TString view = pair.second; - if (!view.empty()) { - YQL_ENSURE(view != "@"); - Ctx.Error() << "VIEW is not supported for table bindings"; - return false; - } - - if (!ApplyTableBinding(binding, tr, tableHints)) { - return false; - } - } else { - tr.Keys = BuildTableKey(pos, service, cluster, TDeferredAtom(pos, pair.first), pair.second); - } + if (isBinding) { + TString binding = pair.first; + TString view = pair.second; + if (!view.empty()) { + YQL_ENSURE(view != "@"); + Ctx.Error() << "VIEW is not supported for table bindings"; + return false; + } + + if (!ApplyTableBinding(binding, tr, tableHints)) { + return false; + } + } else { + tr.Keys = BuildTableKey(pos, service, cluster, TDeferredAtom(pos, pair.first), pair.second); + } break; } case TRule_table_ref::TBlock3::kAlt2: { @@ -1598,7 +1598,7 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul } auto& alt = block.GetAlt2(); - const TString func(Id(alt.GetRule_an_id_expr1(), *this)); + const TString func(Id(alt.GetRule_an_id_expr1(), *this)); TVector<TTableArg> args; if (alt.HasBlock3()) { auto& argsBlock = alt.GetBlock3(); @@ -1679,7 +1679,7 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul values.push_back(nodePtr); values.push_back(new TAstAtomNodeImpl(Ctx.Pos(), "world", TNodeFlags::Default)); - TSqlExpression sqlExpr(Ctx, Mode); + TSqlExpression sqlExpr(Ctx, Mode); if (alt.GetBlock2().HasBlock2() && !ExprList(sqlExpr, values, alt.GetBlock2().GetBlock2().GetRule_expr_list1())) { return false; } @@ -1725,8 +1725,8 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul Y_FAIL("You should change implementation according to grammar changes"); } - MergeHints(hints, tableHints); - + MergeHints(hints, tableHints); + if (node.HasBlock4()) { auto tmp = TableHintsImpl(node.GetBlock4().GetRule_table_hints1()); if (!tmp) { @@ -1734,7 +1734,7 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul return false; } - MergeHints(hints, *tmp); + MergeHints(hints, *tmp); } if (!hints.empty()) { @@ -1754,7 +1754,7 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) { if (block.Alt_case() == TRule_table_ref::TBlock3::kAlt2) { auto& alt = block.GetAlt2(); - TCiString func(Id(alt.GetRule_an_id_expr1(), *this)); + TCiString func(Id(alt.GetRule_an_id_expr1(), *this)); if (func == "as_table") { if (node.HasBlock1()) { @@ -1782,7 +1782,7 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) { return TMaybe<TSourcePtr>(nullptr); } - return BuildNodeSource(Ctx.Pos(), arg->Expr, true); + return BuildNodeSource(Ctx.Pos(), arg->Expr, true); } } @@ -1791,7 +1791,7 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) { TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) { const bool nullable = !node.HasBlock4() || !node.GetBlock4().HasBlock1(); - const TString name(Id(node.GetRule_an_id_schema1(), *this)); + const TString name(Id(node.GetRule_an_id_schema1(), *this)); const TPosition pos(Context().Pos()); const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2()); if (!type) { @@ -2126,48 +2126,48 @@ namespace { bool WithoutAlpha(const std::string_view& literal) { return literal.cend() == std::find_if(literal.cbegin(), literal.cend(), [](char c) { return std::isalpha(c) || (c & '\x80'); }); } - - template<typename TChar> - struct TSplitResult { - TBasicString<TChar> Prefix; - TBasicString<TChar> Suffix; - }; - - template<typename TChar> - TSplitResult<TChar> SplitPattern(const TBasicString<TChar>& pattern, TMaybe<char> escape, - bool& inEscape, bool& hasPattern, bool& isSimple) - { - inEscape = hasPattern = false; - isSimple = true; - TSplitResult<TChar> result; - TBasicString<TChar> left, right, current; - for (const auto c : pattern) { - if (inEscape) { - current.append(c); - inEscape = false; - } else if (escape && c == static_cast<TChar>(*escape)) { - inEscape = true; - } else if (c == '%' || c == '_') { - if (result.Prefix.empty() && !hasPattern) - std::swap(result.Prefix, current); - else if (left.empty()) - std::swap(left, current); - else if (right.empty()) - std::swap(right, current); - else - isSimple = false; - hasPattern = true; - if (c == '_') { - isSimple = false; - } - } else { - current.append(c); - } - } - - result.Suffix = std::move(current); - return result; - } + + template<typename TChar> + struct TSplitResult { + TBasicString<TChar> Prefix; + TBasicString<TChar> Suffix; + }; + + template<typename TChar> + TSplitResult<TChar> SplitPattern(const TBasicString<TChar>& pattern, TMaybe<char> escape, + bool& inEscape, bool& hasPattern, bool& isSimple) + { + inEscape = hasPattern = false; + isSimple = true; + TSplitResult<TChar> result; + TBasicString<TChar> left, right, current; + for (const auto c : pattern) { + if (inEscape) { + current.append(c); + inEscape = false; + } else if (escape && c == static_cast<TChar>(*escape)) { + inEscape = true; + } else if (c == '%' || c == '_') { + if (result.Prefix.empty() && !hasPattern) + std::swap(result.Prefix, current); + else if (left.empty()) + std::swap(left, current); + else if (right.empty()) + std::swap(right, current); + else + isSimple = false; + hasPattern = true; + if (c == '_') { + isSimple = false; + } + } else { + current.append(c); + } + } + + result.Suffix = std::move(current); + return result; + } } bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, @@ -2322,10 +2322,10 @@ TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) { return BuildQuotedAtom(Ctx.Pos(), ToString(value), TNodeFlags::ArbitraryContent); } case TRule_integer_or_bind::kAltIntegerOrBind2: { - TString bindName; - if (!NamedNodeImpl(node.GetAlt_integer_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { - return {}; - } + TString bindName; + if (!NamedNodeImpl(node.GetAlt_integer_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { + return {}; + } auto namedNode = GetNamedNode(bindName); if (!namedNode) { return {}; @@ -2346,19 +2346,19 @@ TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) { return atom.Build(); } case TRule_type_name_tag::kAltTypeNameTag2: { - auto value = Token(node.GetAlt_type_name_tag2().GetToken1()); - auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value); - if (!parsed) { + auto value = Token(node.GetAlt_type_name_tag2().GetToken1()); + auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value); + if (!parsed) { return {}; } - auto atom = TDeferredAtom(Ctx.Pos(), parsed->Content); + auto atom = TDeferredAtom(Ctx.Pos(), parsed->Content); return atom.Build(); } case TRule_type_name_tag::kAltTypeNameTag3: { - TString bindName; - if (!NamedNodeImpl(node.GetAlt_type_name_tag3().GetRule_bind_parameter1(), bindName, *this)) { - return {}; - } + TString bindName; + if (!NamedNodeImpl(node.GetAlt_type_name_tag3().GetRule_bind_parameter1(), bindName, *this)) { + return {}; + } auto namedNode = GetNamedNode(bindName); if (!namedNode) { return {}; @@ -2372,16 +2372,16 @@ TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) { } } -TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed) { - const TString origName = Id(node.GetRule_an_id_pure1(), *this); - if (origName.empty()) { - return {}; - } - return BuildSimpleType(Ctx, Ctx.Pos(), origName, onlyDataAllowed); -} - -TNodePtr TSqlTranslation::TypeDecimal(const TRule_type_name_decimal& node) { - auto pos = Ctx.Pos(); +TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed) { + const TString origName = Id(node.GetRule_an_id_pure1(), *this); + if (origName.empty()) { + return {}; + } + return BuildSimpleType(Ctx, Ctx.Pos(), origName, onlyDataAllowed); +} + +TNodePtr TSqlTranslation::TypeDecimal(const TRule_type_name_decimal& node) { + auto pos = Ctx.Pos(); auto flags = TNodeFlags::Default; auto paramOne = IntegerOrBind(node.GetRule_integer_or_bind3()); @@ -2392,21 +2392,21 @@ TNodePtr TSqlTranslation::TypeDecimal(const TRule_type_name_decimal& node) { if (!paramTwo) { return {}; } - return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, "Decimal", flags), paramOne, paramTwo }); -} - -TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCount) { - TNodePtr result = node; - if (node) { - TPosition pos = node->GetPos(); - for (size_t i = 0; i < optionalCount; ++i) { - result = new TCallNodeImpl(pos, "OptionalType", { result }); - } - } - return result; -} - - + return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, "Decimal", flags), paramOne, paramTwo }); +} + +TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCount) { + TNodePtr result = node; + if (node) { + TPosition pos = node->GetPos(); + for (size_t i = 0; i < optionalCount; ++i) { + result = new TCallNodeImpl(pos, "OptionalType", { result }); + } + } + return result; +} + + TMaybe<std::pair<TVector<TNodePtr>, bool>> TSqlTranslation::CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted) { auto pos = Ctx.Pos(); auto flags = TNodeFlags::Default; @@ -2473,10 +2473,10 @@ TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) { return TypeNode(node.GetAlt_type_name_or_bind1().GetRule_type_name1()); } case TRule_type_name_or_bind::kAltTypeNameOrBind2: { - TString bindName; - if (!NamedNodeImpl(node.GetAlt_type_name_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { - return {}; - } + TString bindName; + if (!NamedNodeImpl(node.GetAlt_type_name_or_bind2().GetRule_bind_parameter1(), bindName, *this)) { + return {}; + } return GetNamedNode(bindName); } default: @@ -2484,55 +2484,55 @@ TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) { } } -TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) { - //type_name: - // type_name_composite - // | (type_name_decimal | type_name_simple) QUESTION*; - if (node.Alt_case() == TRule_type_name::kAltTypeName1) { - return TypeNode(node.GetAlt_type_name1().GetRule_type_name_composite1()); - } - +TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) { + //type_name: + // type_name_composite + // | (type_name_decimal | type_name_simple) QUESTION*; + if (node.Alt_case() == TRule_type_name::kAltTypeName1) { + return TypeNode(node.GetAlt_type_name1().GetRule_type_name_composite1()); + } + + TNodePtr result; + TPosition pos = Ctx.Pos(); + + auto& alt = node.GetAlt_type_name2(); + auto& block = alt.GetBlock1(); + switch (block.Alt_case()) { + case TRule_type_name::TAlt2::TBlock1::kAlt1: { + auto& decimalType = block.GetAlt1().GetRule_type_name_decimal1(); + result = TypeDecimal(decimalType); + break; + } + case TRule_type_name::TAlt2::TBlock1::kAlt2: { + auto& simpleType = block.GetAlt2().GetRule_type_name_simple1(); + result = TypeSimple(simpleType, false); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + return AddOptionals(result, alt.GetBlock2().size()); +} + +TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { + //type_name_composite: + // ( type_name_optional + // | type_name_tuple + // | type_name_struct + // | type_name_variant + // | type_name_list + // | type_name_stream + // | type_name_flow + // | type_name_dict + // | type_name_set + // | type_name_enum + // | type_name_resource + // | type_name_tagged + // | type_name_callable + // ) QUESTION*; TNodePtr result; TPosition pos = Ctx.Pos(); - - auto& alt = node.GetAlt_type_name2(); - auto& block = alt.GetBlock1(); - switch (block.Alt_case()) { - case TRule_type_name::TAlt2::TBlock1::kAlt1: { - auto& decimalType = block.GetAlt1().GetRule_type_name_decimal1(); - result = TypeDecimal(decimalType); - break; - } - case TRule_type_name::TAlt2::TBlock1::kAlt2: { - auto& simpleType = block.GetAlt2().GetRule_type_name_simple1(); - result = TypeSimple(simpleType, false); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - return AddOptionals(result, alt.GetBlock2().size()); -} - -TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { - //type_name_composite: - // ( type_name_optional - // | type_name_tuple - // | type_name_struct - // | type_name_variant - // | type_name_list - // | type_name_stream - // | type_name_flow - // | type_name_dict - // | type_name_set - // | type_name_enum - // | type_name_resource - // | type_name_tagged - // | type_name_callable - // ) QUESTION*; - TNodePtr result; - TPosition pos = Ctx.Pos(); auto flags = TNodeFlags::Default; auto wrapOneParamType = [&] (const TRule_type_name_or_bind& param, const char* type) -> TNodePtr { @@ -2548,87 +2548,87 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { auto& block = node.GetBlock1(); switch (block.Alt_case()) { - case TRule_type_name_composite_TBlock1::kAlt1: { - auto& optionalType = block.GetAlt1().GetRule_type_name_optional1(); + case TRule_type_name_composite_TBlock1::kAlt1: { + auto& optionalType = block.GetAlt1().GetRule_type_name_optional1(); result = wrapOneParamType(optionalType.GetRule_type_name_or_bind3(), "OptionalType"); break; } - case TRule_type_name_composite_TBlock1::kAlt2: { - auto& tupleType = block.GetAlt2().GetRule_type_name_tuple1(); + case TRule_type_name_composite_TBlock1::kAlt2: { + auto& tupleType = block.GetAlt2().GetRule_type_name_tuple1(); TVector<TNodePtr> items; items.push_back(BuildAtom(pos, "TupleType", flags)); - - switch (tupleType.GetBlock2().Alt_case()) { - case TRule_type_name_tuple::TBlock2::kAlt1: { - if (tupleType.GetBlock2().GetAlt1().HasBlock2()) { - auto typeNode = TypeNodeOrBind(tupleType.GetBlock2().GetAlt1().GetBlock2().GetRule_type_name_or_bind1()); + + switch (tupleType.GetBlock2().Alt_case()) { + case TRule_type_name_tuple::TBlock2::kAlt1: { + if (tupleType.GetBlock2().GetAlt1().HasBlock2()) { + auto typeNode = TypeNodeOrBind(tupleType.GetBlock2().GetAlt1().GetBlock2().GetRule_type_name_or_bind1()); if (!typeNode) { return {}; } items.push_back(typeNode); - for (auto& arg : tupleType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) { - auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind2()); - if (!typeNode) { - return {}; - } - items.push_back(typeNode); - } + for (auto& arg : tupleType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) { + auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind2()); + if (!typeNode) { + return {}; + } + items.push_back(typeNode); + } } [[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME } - case TRule_type_name_tuple::TBlock2::kAlt2: - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - + case TRule_type_name_tuple::TBlock2::kAlt2: + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + result = new TAstListNodeImpl(pos, items); break; } - case TRule_type_name_composite_TBlock1::kAlt3: { - auto& structType = block.GetAlt3().GetRule_type_name_struct1(); + case TRule_type_name_composite_TBlock1::kAlt3: { + auto& structType = block.GetAlt3().GetRule_type_name_struct1(); TVector<TNodePtr> items; items.push_back(BuildAtom(pos, "StructType", flags)); - - switch (structType.GetBlock2().Alt_case()) { - case TRule_type_name_struct::TBlock2::kAlt1: { - if (structType.GetBlock2().GetAlt1().HasBlock2()) { - auto& structArg = structType.GetBlock2().GetAlt1().GetBlock2().GetRule_struct_arg1(); - auto typeNode = TypeNodeOrBind(structArg.GetRule_type_name_or_bind3()); - if (!typeNode) { - return {}; - } - auto tag = TypeNameTag(structArg.GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); - for (auto& arg : structType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) { - auto typeNode = TypeNodeOrBind(arg.GetRule_struct_arg2().GetRule_type_name_or_bind3()); - if (!typeNode) { - return {}; - } - auto tag = TypeNameTag(arg.GetRule_struct_arg2().GetRule_type_name_tag1()); - if (!tag) { - return {}; - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); - } - } + + switch (structType.GetBlock2().Alt_case()) { + case TRule_type_name_struct::TBlock2::kAlt1: { + if (structType.GetBlock2().GetAlt1().HasBlock2()) { + auto& structArg = structType.GetBlock2().GetAlt1().GetBlock2().GetRule_struct_arg1(); + auto typeNode = TypeNodeOrBind(structArg.GetRule_type_name_or_bind3()); + if (!typeNode) { + return {}; + } + auto tag = TypeNameTag(structArg.GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); + for (auto& arg : structType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) { + auto typeNode = TypeNodeOrBind(arg.GetRule_struct_arg2().GetRule_type_name_or_bind3()); + if (!typeNode) { + return {}; + } + auto tag = TypeNameTag(arg.GetRule_struct_arg2().GetRule_type_name_tag1()); + if (!tag) { + return {}; + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode }))); + } + } [[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME } - case TRule_type_name_struct::TBlock2::kAlt2: - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); + case TRule_type_name_struct::TBlock2::kAlt2: + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); } - + result = new TAstListNodeImpl(pos, items); break; } - case TRule_type_name_composite_TBlock1::kAlt4: { - auto& variantType = block.GetAlt4().GetRule_type_name_variant1(); + case TRule_type_name_composite_TBlock1::kAlt4: { + auto& variantType = block.GetAlt4().GetRule_type_name_variant1(); TVector<TNodePtr> items; bool overStruct = false; auto& variantArg = variantType.GetRule_variant_arg3(); @@ -2676,23 +2676,23 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); break; } - case TRule_type_name_composite_TBlock1::kAlt5: { - auto& listType = block.GetAlt5().GetRule_type_name_list1(); + case TRule_type_name_composite_TBlock1::kAlt5: { + auto& listType = block.GetAlt5().GetRule_type_name_list1(); result = wrapOneParamType(listType.GetRule_type_name_or_bind3(), "ListType"); break; } - case TRule_type_name_composite_TBlock1::kAlt6: { - auto& streamType = block.GetAlt6().GetRule_type_name_stream1(); + case TRule_type_name_composite_TBlock1::kAlt6: { + auto& streamType = block.GetAlt6().GetRule_type_name_stream1(); result = wrapOneParamType(streamType.GetRule_type_name_or_bind3(), "StreamType"); break; } - case TRule_type_name_composite_TBlock1::kAlt7: { - auto& flowType = block.GetAlt7().GetRule_type_name_flow1(); + case TRule_type_name_composite_TBlock1::kAlt7: { + auto& flowType = block.GetAlt7().GetRule_type_name_flow1(); result = wrapOneParamType(flowType.GetRule_type_name_or_bind3(), "FlowType"); break; } - case TRule_type_name_composite_TBlock1::kAlt8: { - auto& dictType = block.GetAlt8().GetRule_type_name_dict1(); + case TRule_type_name_composite_TBlock1::kAlt8: { + auto& dictType = block.GetAlt8().GetRule_type_name_dict1(); TVector<TNodePtr> items; items.push_back(BuildAtom(pos, "DictType", flags)); auto typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind3()); @@ -2708,8 +2708,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, items); break; } - case TRule_type_name_composite_TBlock1::kAlt9: { - auto& setType = block.GetAlt9().GetRule_type_name_set1(); + case TRule_type_name_composite_TBlock1::kAlt9: { + auto& setType = block.GetAlt9().GetRule_type_name_set1(); auto typeNode = TypeNodeOrBind(setType.GetRule_type_name_or_bind3()); if (!typeNode) { return {}; @@ -2717,17 +2717,17 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, { BuildAtom(pos, "DictType", flags), typeNode, makeVoid() }); break; } - case TRule_type_name_composite_TBlock1::kAlt10: { - auto& enumType = block.GetAlt10().GetRule_type_name_enum1(); + case TRule_type_name_composite_TBlock1::kAlt10: { + auto& enumType = block.GetAlt10().GetRule_type_name_enum1(); TVector<TNodePtr> items; items.push_back(BuildAtom(pos, "StructType", flags)); - auto tag = TypeNameTag(enumType.GetRule_type_name_tag3()); - if (!tag) { - return {}; - } - items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() }))); - for (auto& arg : enumType.GetBlock4()) { - auto tag = TypeNameTag(arg.GetRule_type_name_tag2()); + auto tag = TypeNameTag(enumType.GetRule_type_name_tag3()); + if (!tag) { + return {}; + } + items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() }))); + for (auto& arg : enumType.GetBlock4()) { + auto tag = TypeNameTag(arg.GetRule_type_name_tag2()); if (!tag) { return {}; } @@ -2737,8 +2737,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode }); break; } - case TRule_type_name_composite_TBlock1::kAlt11: { - auto& resourceType = block.GetAlt11().GetRule_type_name_resource1(); + case TRule_type_name_composite_TBlock1::kAlt11: { + auto& resourceType = block.GetAlt11().GetRule_type_name_resource1(); auto tag = TypeNameTag(resourceType.GetRule_type_name_tag3()); if (!tag) { return {}; @@ -2746,8 +2746,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, { BuildAtom(pos, "ResourceType", flags), tag }); break; } - case TRule_type_name_composite_TBlock1::kAlt12: { - auto& taggedType = block.GetAlt12().GetRule_type_name_tagged1(); + case TRule_type_name_composite_TBlock1::kAlt12: { + auto& taggedType = block.GetAlt12().GetRule_type_name_tagged1(); auto typeNode = TypeNodeOrBind(taggedType.GetRule_type_name_or_bind3()); if (!typeNode) { return {}; @@ -2759,8 +2759,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { result = new TAstListNodeImpl(pos, { BuildAtom(pos, "TaggedType", flags), typeNode, tag }); break; } - case TRule_type_name_composite_TBlock1::kAlt13: { - auto& callableType = block.GetAlt13().GetRule_type_name_callable1(); + case TRule_type_name_composite_TBlock1::kAlt13: { + auto& callableType = block.GetAlt13().GetRule_type_name_callable1(); TMaybe<std::pair<TVector<TNodePtr>, bool>> requiredArgs, optionalArgs; bool namedArgsStarted = false; size_t optionalArgsCount = 0; @@ -2810,7 +2810,7 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) { Y_FAIL("You should change implementation according to grammar changes"); } - return AddOptionals(result, node.GetBlock2().size()); + return AddOptionals(result, node.GetBlock2().size()); } bool Expr(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule_expr& node) { @@ -2836,7 +2836,7 @@ bool ExprList(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule } TNodePtr TSqlTranslation::ValueConstructorLiteral(const TRule_value_constructor_literal& node) { - return BuildLiteralSmartString(Ctx, Token(node.GetToken1())); + return BuildLiteralSmartString(Ctx, Token(node.GetToken1())); } TNodePtr TSqlTranslation::ValueConstructor(const TRule_value_constructor& node) { @@ -2885,8 +2885,8 @@ TNodePtr TSqlTranslation::DictLiteral(const TRule_dict_literal& node) { for (auto& b : list.GetBlock3()) { sqlExpr.Token(b.GetToken1()); - const bool isSetCurr = !b.HasBlock3(); - if (isSetCurr != isSet) { + const bool isSetCurr = !b.HasBlock3(); + if (isSetCurr != isSet) { Error() << "Expected keys/values pair or keys, but got mix of them"; return nullptr; } @@ -2915,70 +2915,70 @@ TNodePtr TSqlTranslation::DictLiteral(const TRule_dict_literal& node) { return new TAstListNodeImpl(Ctx.Pos(), std::move(values)); } -bool TSqlTranslation::StructLiteralItem(TVector<TNodePtr>& labels, const TRule_expr& label, TVector<TNodePtr>& values, const TRule_expr& value) { - // label expr - { - TColumnRefScope scope(Ctx, EColumnRefState::AsStringLiteral, /* topLevel */ false); +bool TSqlTranslation::StructLiteralItem(TVector<TNodePtr>& labels, const TRule_expr& label, TVector<TNodePtr>& values, const TRule_expr& value) { + // label expr + { + TColumnRefScope scope(Ctx, EColumnRefState::AsStringLiteral, /* topLevel */ false); TSqlExpression sqlExpr(Ctx, Mode); - if (!Expr(sqlExpr, labels, label)) { - return false; + if (!Expr(sqlExpr, labels, label)) { + return false; } TDeferredAtom atom; - MakeTableFromExpression(Ctx, labels.back(), atom); - labels.back() = atom.Build(); - if (!labels.back()) { - return false; - } - } + MakeTableFromExpression(Ctx, labels.back(), atom); + labels.back() = atom.Build(); + if (!labels.back()) { + return false; + } + } - // value expr - { - TSqlExpression sqlExpr(Ctx, Mode); - if (!Expr(sqlExpr, values, value)) { - return false; + // value expr + { + TSqlExpression sqlExpr(Ctx, Mode); + if (!Expr(sqlExpr, values, value)) { + return false; } - } + } + + return true; +} - return true; -} +TNodePtr TSqlTranslation::StructLiteral(const TRule_struct_literal& node) { + TVector<TNodePtr> labels; + TVector<TNodePtr> values; + TPosition pos = Ctx.TokenPosition(node.GetToken1()); + if (node.HasBlock2()) { + const auto& list = node.GetBlock2().GetRule_expr_struct_list1(); -TNodePtr TSqlTranslation::StructLiteral(const TRule_struct_literal& node) { - TVector<TNodePtr> labels; - TVector<TNodePtr> values; - TPosition pos = Ctx.TokenPosition(node.GetToken1()); - if (node.HasBlock2()) { - const auto& list = node.GetBlock2().GetRule_expr_struct_list1(); + if (!StructLiteralItem(labels, list.GetRule_expr1(), values, list.GetRule_expr3())) { + return {}; + } - if (!StructLiteralItem(labels, list.GetRule_expr1(), values, list.GetRule_expr3())) { - return {}; - } - - for (auto& b : list.GetBlock4()) { - if (!StructLiteralItem(labels, b.GetRule_expr2(), values, b.GetRule_expr4())) { - return {}; + for (auto& b : list.GetBlock4()) { + if (!StructLiteralItem(labels, b.GetRule_expr2(), values, b.GetRule_expr4())) { + return {}; } } } - return BuildStructure(pos, values, labels); + return BuildStructure(pos, values, labels); } bool TSqlTranslation::TableHintImpl(const TRule_table_hint& rule, TTableHints& hints) { - // table_hint: - // an_id_hint - // | (SCHEMA | COLUMNS) type_name_or_bind - // | SCHEMA LPAREN (struct_arg_as (COMMA struct_arg_as)*)? COMMA? RPAREN + // table_hint: + // an_id_hint + // | (SCHEMA | COLUMNS) type_name_or_bind + // | SCHEMA LPAREN (struct_arg_as (COMMA struct_arg_as)*)? COMMA? RPAREN switch (rule.Alt_case()) { - case TRule_table_hint::kAltTableHint1: { - const TString id = Id(rule.GetAlt_table_hint1().GetRule_an_id_hint1(), *this); - const auto idLower = to_lower(id); - if (idLower == "schema" || idLower == "columns") { - Error() << "Expected type after " << to_upper(id); - return false; - } - hints[id] = {}; + case TRule_table_hint::kAltTableHint1: { + const TString id = Id(rule.GetAlt_table_hint1().GetRule_an_id_hint1(), *this); + const auto idLower = to_lower(id); + if (idLower == "schema" || idLower == "columns") { + Error() << "Expected type after " << to_upper(id); + return false; + } + hints[id] = {}; break; - } + } case TRule_table_hint::kAltTableHint2: { const auto& alt2 = rule.GetAlt_table_hint2(); @@ -2987,52 +2987,52 @@ bool TSqlTranslation::TableHintImpl(const TRule_table_hint& rule, TTableHints& h return false; } - hints["user_" + to_lower(alt2.GetToken1().GetValue())] = { node }; + hints["user_" + to_lower(alt2.GetToken1().GetValue())] = { node }; + break; + } + + case TRule_table_hint::kAltTableHint3: { + const auto& alt = rule.GetAlt_table_hint3(); + TVector<TNodePtr> labels; + TVector<TNodePtr> structTypeItems; + if (alt.HasBlock3()) { + auto processItem = [&](const TRule_struct_arg_as& arg) { + auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind1()); + if (!typeNode) { + return false; + } + + auto pos = Ctx.Pos(); + + auto tag = TypeNameTag(arg.GetRule_type_name_tag3()); + if (!tag) { + return false; + } + + labels.push_back(tag); + structTypeItems.push_back(BuildTuple(pos, { tag, typeNode })); + return true; + }; + + if (!processItem(alt.GetBlock3().GetRule_struct_arg_as1())) { + return false; + } + + for (auto& entry : alt.GetBlock3().GetBlock2()) { + if (!processItem(entry.GetRule_struct_arg_as2())) { + return false; + } + } + } + + TPosition pos = Ctx.TokenPosition(alt.GetToken1()); + auto labelsTuple = BuildTuple(pos, labels); + TNodePtr structType = new TCallNodeImpl(pos, "StructType", structTypeItems); + + hints["user_" + to_lower(alt.GetToken1().GetValue())] = { structType, labelsTuple }; break; } - case TRule_table_hint::kAltTableHint3: { - const auto& alt = rule.GetAlt_table_hint3(); - TVector<TNodePtr> labels; - TVector<TNodePtr> structTypeItems; - if (alt.HasBlock3()) { - auto processItem = [&](const TRule_struct_arg_as& arg) { - auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind1()); - if (!typeNode) { - return false; - } - - auto pos = Ctx.Pos(); - - auto tag = TypeNameTag(arg.GetRule_type_name_tag3()); - if (!tag) { - return false; - } - - labels.push_back(tag); - structTypeItems.push_back(BuildTuple(pos, { tag, typeNode })); - return true; - }; - - if (!processItem(alt.GetBlock3().GetRule_struct_arg_as1())) { - return false; - } - - for (auto& entry : alt.GetBlock3().GetBlock2()) { - if (!processItem(entry.GetRule_struct_arg_as2())) { - return false; - } - } - } - - TPosition pos = Ctx.TokenPosition(alt.GetToken1()); - auto labelsTuple = BuildTuple(pos, labels); - TNodePtr structType = new TCallNodeImpl(pos, "StructType", structTypeItems); - - hints["user_" + to_lower(alt.GetToken1().GetValue())] = { structType, labelsTuple }; - break; - } - default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -3102,11 +3102,11 @@ bool TSqlTranslation::SimpleTableRefImpl(const TRule_simple_table_ref& node, TTa } auto at = node.GetBlock1().GetAlt2().HasBlock1(); - TString bindName; - if (!NamedNodeImpl(node.GetBlock1().GetAlt2().GetRule_bind_parameter2(), bindName, *this)) { - return false; - } - auto named = GetNamedNode(bindName); + TString bindName; + if (!NamedNodeImpl(node.GetBlock1().GetAlt2().GetRule_bind_parameter2(), bindName, *this)) { + return false; + } + auto named = GetNamedNode(bindName); if (!named) { return false; } @@ -3155,31 +3155,31 @@ bool TSqlCallExpr::Init(const TRule_value_constructor& node) { if (!Expr(expr, Args, ctor.GetRule_expr5())) { return false; } - if (!Expr(expr, Args, ctor.GetRule_expr7())) { + if (!Expr(expr, Args, ctor.GetRule_expr7())) { return false; } break; } - case TRule_value_constructor::kAltValueConstructor2: { - auto& ctor = node.GetAlt_value_constructor2(); + case TRule_value_constructor::kAltValueConstructor2: { + auto& ctor = node.GetAlt_value_constructor2(); Func = "Enum"; TSqlExpression expr(Ctx, Mode); if (!Expr(expr, Args, ctor.GetRule_expr3())) { return false; } - if (!Expr(expr, Args, ctor.GetRule_expr5())) { + if (!Expr(expr, Args, ctor.GetRule_expr5())) { return false; } break; } - case TRule_value_constructor::kAltValueConstructor3: { - auto& ctor = node.GetAlt_value_constructor3(); + case TRule_value_constructor::kAltValueConstructor3: { + auto& ctor = node.GetAlt_value_constructor3(); Func = "Callable"; - TSqlExpression expr(Ctx, Mode); + TSqlExpression expr(Ctx, Mode); if (!Expr(expr, Args, ctor.GetRule_expr3())) { return false; } - if (!Expr(expr, Args, ctor.GetRule_expr5())) { + if (!Expr(expr, Args, ctor.GetRule_expr5())) { return false; } break; @@ -3191,84 +3191,84 @@ bool TSqlCallExpr::Init(const TRule_value_constructor& node) { return true; } -bool TSqlCallExpr::ExtractCallParam(const TRule_external_call_param& node) { +bool TSqlCallExpr::ExtractCallParam(const TRule_external_call_param& node) { TString paramName = Id(node.GetRule_an_id1(), *this); paramName = to_lower(paramName); - if (CallConfig.contains(paramName)) { - Ctx.Error() << "WITH " << to_upper(paramName).Quote() - << " clause should be specified only once"; - return false; - } - - const bool optimizeForParam = paramName == "optimize_for"; - const auto columnRefState = optimizeForParam ? EColumnRefState::AsStringLiteral : EColumnRefState::Deny; - - TColumnRefScope scope(Ctx, columnRefState); - if (optimizeForParam) { - scope.SetNoColumnErrContext("in external call params"); - } - - TSqlExpression expression(Ctx, Mode); - auto value = expression.Build(node.GetRule_expr3()); - if (value && optimizeForParam) { - TDeferredAtom atom; - MakeTableFromExpression(Ctx, value, atom); - value = new TCallNodeImpl(Ctx.Pos(), "String", { atom.Build() }); - } - - if (!value) { - return false; - } - - CallConfig[paramName] = value; - return true; + if (CallConfig.contains(paramName)) { + Ctx.Error() << "WITH " << to_upper(paramName).Quote() + << " clause should be specified only once"; + return false; + } + + const bool optimizeForParam = paramName == "optimize_for"; + const auto columnRefState = optimizeForParam ? EColumnRefState::AsStringLiteral : EColumnRefState::Deny; + + TColumnRefScope scope(Ctx, columnRefState); + if (optimizeForParam) { + scope.SetNoColumnErrContext("in external call params"); + } + + TSqlExpression expression(Ctx, Mode); + auto value = expression.Build(node.GetRule_expr3()); + if (value && optimizeForParam) { + TDeferredAtom atom; + MakeTableFromExpression(Ctx, value, atom); + value = new TCallNodeImpl(Ctx.Pos(), "String", { atom.Build() }); + } + + if (!value) { + return false; + } + + CallConfig[paramName] = value; + return true; } bool TSqlCallExpr::ConfigureExternalCall(const TRule_external_call_settings& node) { - bool success = ExtractCallParam(node.GetRule_external_call_param1()); - for (auto& block: node.GetBlock2()) { - success = ExtractCallParam(block.GetRule_external_call_param2()) && success; + bool success = ExtractCallParam(node.GetRule_external_call_param1()); + for (auto& block: node.GetBlock2()) { + success = ExtractCallParam(block.GetRule_external_call_param2()) && success; } return success; } -bool TSqlCallExpr::Init(const TRule_using_call_expr& node) { +bool TSqlCallExpr::Init(const TRule_using_call_expr& node) { // using_call_expr: ((an_id_or_type NAMESPACE an_id_or_type) | an_id_expr | bind_parameter | (EXTERNAL FUNCTION)) invoke_expr; const auto& block = node.GetBlock1(); switch (block.Alt_case()) { - case TRule_using_call_expr::TBlock1::kAlt1: { + case TRule_using_call_expr::TBlock1::kAlt1: { auto& subblock = block.GetAlt1().GetBlock1(); - Module = Id(subblock.GetRule_an_id_or_type1(), *this); - Func = Id(subblock.GetRule_an_id_or_type3(), *this); + Module = Id(subblock.GetRule_an_id_or_type1(), *this); + Func = Id(subblock.GetRule_an_id_or_type3(), *this); break; } - case TRule_using_call_expr::TBlock1::kAlt2: { - Func = Id(block.GetAlt2().GetRule_an_id_expr1(), *this); + case TRule_using_call_expr::TBlock1::kAlt2: { + Func = Id(block.GetAlt2().GetRule_an_id_expr1(), *this); break; - } - case TRule_using_call_expr::TBlock1::kAlt3: { - TString bindName; - if (!NamedNodeImpl(block.GetAlt3().GetRule_bind_parameter1(), bindName, *this)) { - return false; - } - Node = GetNamedNode(bindName); + } + case TRule_using_call_expr::TBlock1::kAlt3: { + TString bindName; + if (!NamedNodeImpl(block.GetAlt3().GetRule_bind_parameter1(), bindName, *this)) { + return false; + } + Node = GetNamedNode(bindName); if (!Node) { return false; } break; - } + } case TRule_using_call_expr::TBlock1::kAlt4: { IsExternalCall = true; break; } - default: + default: Y_FAIL("You should change implementation according to grammar changes"); } - YQL_ENSURE(!DistinctAllowed); - UsingCallExpr = true; - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + YQL_ENSURE(!DistinctAllowed); + UsingCallExpr = true; + TColumnRefScope scope(Ctx, EColumnRefState::Allow); return Init(node.GetRule_invoke_expr2()); } @@ -3281,81 +3281,81 @@ void TSqlCallExpr::InitExpr(const TNodePtr& expr) { Node = expr; } -bool TSqlCallExpr::FillArg(const TString& module, const TString& func, size_t& idx, const TRule_named_expr& node) { - const bool isNamed = node.HasBlock2(); - - TMaybe<EColumnRefState> status; - // TODO: support named args - if (!isNamed) { - status = GetFunctionArgColumnStatus(Ctx, module, func, idx); - } - - TNodePtr expr; - if (status) { - TColumnRefScope scope(Ctx, *status, /* isTopLevel = */ false); - expr = NamedExpr(node); - } else { - expr = NamedExpr(node); - } - - if (!expr) { - return false; - } - - Args.emplace_back(std::move(expr)); - if (!isNamed) { - ++idx; - } - return true; -} - -bool TSqlCallExpr::FillArgs(const TRule_named_expr_list& node) { - TString module = Module; - TString func = Func; - if (Node && Node->FuncName()) { - module = Node->ModuleName() ? *Node->ModuleName() : "YQL"; - func = *Node->FuncName(); - } - - size_t idx = 0; - if (!FillArg(module, func, idx, node.GetRule_named_expr1())) { - return false; - } - - for (auto& b: node.GetBlock2()) { - if (!FillArg(module, func, idx, b.GetRule_named_expr2())) { - return false; - } - } - - return true; -} - +bool TSqlCallExpr::FillArg(const TString& module, const TString& func, size_t& idx, const TRule_named_expr& node) { + const bool isNamed = node.HasBlock2(); + + TMaybe<EColumnRefState> status; + // TODO: support named args + if (!isNamed) { + status = GetFunctionArgColumnStatus(Ctx, module, func, idx); + } + + TNodePtr expr; + if (status) { + TColumnRefScope scope(Ctx, *status, /* isTopLevel = */ false); + expr = NamedExpr(node); + } else { + expr = NamedExpr(node); + } + + if (!expr) { + return false; + } + + Args.emplace_back(std::move(expr)); + if (!isNamed) { + ++idx; + } + return true; +} + +bool TSqlCallExpr::FillArgs(const TRule_named_expr_list& node) { + TString module = Module; + TString func = Func; + if (Node && Node->FuncName()) { + module = Node->ModuleName() ? *Node->ModuleName() : "YQL"; + func = *Node->FuncName(); + } + + size_t idx = 0; + if (!FillArg(module, func, idx, node.GetRule_named_expr1())) { + return false; + } + + for (auto& b: node.GetBlock2()) { + if (!FillArg(module, func, idx, b.GetRule_named_expr2())) { + return false; + } + } + + return true; +} + bool TSqlCallExpr::Init(const TRule_invoke_expr& node) { - // invoke_expr: LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN invoke_expr_tail; - // invoke_expr_tail: - // (null_treatment | filter_clause)? (OVER window_name_or_specification)? - // ; + // invoke_expr: LPAREN (opt_set_quantifier named_expr_list COMMA? | ASTERISK)? RPAREN invoke_expr_tail; + // invoke_expr_tail: + // (null_treatment | filter_clause)? (OVER window_name_or_specification)? + // ; Pos = Ctx.Pos(); if (node.HasBlock2()) { switch (node.GetBlock2().Alt_case()) { case TRule_invoke_expr::TBlock2::kAlt1: { const auto& alt = node.GetBlock2().GetAlt1(); - TPosition distinctPos; - if (IsDistinctOptSet(alt.GetRule_opt_set_quantifier1(), distinctPos)) { - if (!DistinctAllowed) { - if (UsingCallExpr) { - Ctx.Error(distinctPos) << "DISTINCT can not be used in PROCESS/REDUCE"; - } else { - Ctx.Error(distinctPos) << "DISTINCT can only be used in aggregation functions"; - } - return false; - } + TPosition distinctPos; + if (IsDistinctOptSet(alt.GetRule_opt_set_quantifier1(), distinctPos)) { + if (!DistinctAllowed) { + if (UsingCallExpr) { + Ctx.Error(distinctPos) << "DISTINCT can not be used in PROCESS/REDUCE"; + } else { + Ctx.Error(distinctPos) << "DISTINCT can only be used in aggregation functions"; + } + return false; + } YQL_ENSURE(AggMode == EAggregateMode::Normal); AggMode = EAggregateMode::Distinct; Ctx.IncrementMonCounter("sql_features", "DistinctInCallExpr"); } - if (!FillArgs(alt.GetRule_named_expr_list2())) { + if (!FillArgs(alt.GetRule_named_expr_list2())) { return false; } for (const auto& arg : Args) { @@ -3384,69 +3384,69 @@ bool TSqlCallExpr::Init(const TRule_invoke_expr& node) { } } - const auto& tail = node.GetRule_invoke_expr_tail4(); - - if (tail.HasBlock1()) { + const auto& tail = node.GetRule_invoke_expr_tail4(); + + if (tail.HasBlock1()) { if (IsExternalCall) { Ctx.Error() << "Additional clause after EXTERNAL FUNCTION(...) is not supported"; return false; } - switch (tail.GetBlock1().Alt_case()) { - case TRule_invoke_expr_tail::TBlock1::kAlt1: { - if (!tail.HasBlock2()) { - Ctx.Error() << "RESPECT/IGNORE NULLS can only be used with window functions"; - return false; - } - const auto& alt = tail.GetBlock1().GetAlt1(); - if (alt.GetRule_null_treatment1().Alt_case() == TRule_null_treatment::kAltNullTreatment2) { - SetIgnoreNulls(); - } - break; - } - case TRule_invoke_expr_tail::TBlock1::kAlt2: { - Ctx.Error() << "FILTER clause is not supported yet"; - return false; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - } - - if (tail.HasBlock2()) { - if (AggMode == EAggregateMode::Distinct) { - Ctx.Error() << "DISTINCT is not yet supported in window functions"; - return false; - } + switch (tail.GetBlock1().Alt_case()) { + case TRule_invoke_expr_tail::TBlock1::kAlt1: { + if (!tail.HasBlock2()) { + Ctx.Error() << "RESPECT/IGNORE NULLS can only be used with window functions"; + return false; + } + const auto& alt = tail.GetBlock1().GetAlt1(); + if (alt.GetRule_null_treatment1().Alt_case() == TRule_null_treatment::kAltNullTreatment2) { + SetIgnoreNulls(); + } + break; + } + case TRule_invoke_expr_tail::TBlock1::kAlt2: { + Ctx.Error() << "FILTER clause is not supported yet"; + return false; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + } + + if (tail.HasBlock2()) { + if (AggMode == EAggregateMode::Distinct) { + Ctx.Error() << "DISTINCT is not yet supported in window functions"; + return false; + } SetOverWindow(); - auto winRule = tail.GetBlock2().GetRule_window_name_or_specification2(); - switch (winRule.Alt_case()) { - case TRule_window_name_or_specification::kAltWindowNameOrSpecification1: { - WindowName = Id(winRule.GetAlt_window_name_or_specification1().GetRule_window_name1().GetRule_an_id_window1(), *this); - break; - } - case TRule_window_name_or_specification::kAltWindowNameOrSpecification2: { - if (!Ctx.WinSpecsScopes) { - auto pos = Ctx.TokenPosition(tail.GetBlock2().GetToken1()); - Ctx.Error(pos) << "Window and aggregation functions are not allowed in this context"; - return false; - } - - TWindowSpecificationPtr spec = WindowSpecification( - winRule.GetAlt_window_name_or_specification2().GetRule_window_specification1().GetRule_window_specification_details2()); - if (!spec) { - return false; - } - - WindowName = Ctx.MakeName("_yql_anonymous_window"); - TWinSpecs& specs = Ctx.WinSpecsScopes.back(); - YQL_ENSURE(!specs.contains(WindowName)); - specs[WindowName] = spec; - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } + auto winRule = tail.GetBlock2().GetRule_window_name_or_specification2(); + switch (winRule.Alt_case()) { + case TRule_window_name_or_specification::kAltWindowNameOrSpecification1: { + WindowName = Id(winRule.GetAlt_window_name_or_specification1().GetRule_window_name1().GetRule_an_id_window1(), *this); + break; + } + case TRule_window_name_or_specification::kAltWindowNameOrSpecification2: { + if (!Ctx.WinSpecsScopes) { + auto pos = Ctx.TokenPosition(tail.GetBlock2().GetToken1()); + Ctx.Error(pos) << "Window and aggregation functions are not allowed in this context"; + return false; + } + + TWindowSpecificationPtr spec = WindowSpecification( + winRule.GetAlt_window_name_or_specification2().GetRule_window_specification1().GetRule_window_specification_details2()); + if (!spec) { + return false; + } + + WindowName = Ctx.MakeName("_yql_anonymous_window"); + TWinSpecs& specs = Ctx.WinSpecsScopes.back(); + YQL_ENSURE(!specs.contains(WindowName)); + specs[WindowName] = spec; + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } Ctx.IncrementMonCounter("sql_features", "WindowFunctionOver"); } @@ -3486,72 +3486,72 @@ public: private: bool SelectTerm(TVector<TNodePtr>& terms, const TRule_result_column& node); bool ValidateSelectColumns(const TVector<TNodePtr>& terms); - bool ColumnName(TVector<TNodePtr>& keys, const TRule_column_name& node); - bool ColumnName(TVector<TNodePtr>& keys, const TRule_without_column_name& node); - template<typename TRule> - bool ColumnList(TVector<TNodePtr>& keys, const TRule& node); + bool ColumnName(TVector<TNodePtr>& keys, const TRule_column_name& node); + bool ColumnName(TVector<TNodePtr>& keys, const TRule_without_column_name& node); + template<typename TRule> + bool ColumnList(TVector<TNodePtr>& keys, const TRule& node); bool NamedColumn(TVector<TNodePtr>& columnList, const TRule_named_column& node); - TSourcePtr SingleSource(const TRule_single_source& node, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos, bool unorderedSubquery); - TSourcePtr NamedSingleSource(const TRule_named_single_source& node, bool unorderedSubquery); - bool FlattenByArg(const TString& sourceLabel, TVector<TNodePtr>& flattenByColumns, TVector<TNodePtr>& flattenByExprs, const TRule_flatten_by_arg& node); + TSourcePtr SingleSource(const TRule_single_source& node, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos, bool unorderedSubquery); + TSourcePtr NamedSingleSource(const TRule_named_single_source& node, bool unorderedSubquery); + bool FlattenByArg(const TString& sourceLabel, TVector<TNodePtr>& flattenByColumns, TVector<TNodePtr>& flattenByExprs, const TRule_flatten_by_arg& node); TSourcePtr FlattenSource(const TRule_flatten_source& node); TSourcePtr JoinSource(const TRule_join_source& node); - bool JoinOp(ISource* join, const TRule_join_source::TBlock3& block, TMaybe<TPosition> anyPos); + bool JoinOp(ISource* join, const TRule_join_source::TBlock3& block, TMaybe<TPosition> anyPos); TNodePtr JoinExpr(ISource*, const TRule_join_constraint& node); TSourcePtr ProcessCore(const TRule_process_core& node, const TWriteSettings& settings, TPosition& selectPos); TSourcePtr ReduceCore(const TRule_reduce_core& node, const TWriteSettings& settings, TPosition& selectPos); - - struct TSelectKindPlacement { - bool IsFirstInSelectOp = false; - bool IsLastInSelectOp = false; - }; - - TSourcePtr SelectCore(const TRule_select_core& node, const TWriteSettings& settings, TPosition& selectPos, - TMaybe<TSelectKindPlacement> placement, TVector<TSortSpecificationPtr>& selectOpOrederBy, bool& selectOpAssumeOrderBy); - + + struct TSelectKindPlacement { + bool IsFirstInSelectOp = false; + bool IsLastInSelectOp = false; + }; + + TSourcePtr SelectCore(const TRule_select_core& node, const TWriteSettings& settings, TPosition& selectPos, + TMaybe<TSelectKindPlacement> placement, TVector<TSortSpecificationPtr>& selectOpOrederBy, bool& selectOpAssumeOrderBy); + bool WindowDefinition(const TRule_window_definition& node, TWinSpecs& winSpecs); bool WindowClause(const TRule_window_clause& node, TWinSpecs& winSpecs); - - struct TSelectKindResult { - TSourcePtr Source; - TWriteSettings Settings; - - TVector<TSortSpecificationPtr> SelectOpOrderBy; - bool SelectOpAssumeOrderBy = false; - TNodePtr SelectOpSkipTake; - - inline explicit operator bool() const { - return static_cast<bool>(Source); - } - }; - - bool ValidateLimitOrderByWithSelectOp(TMaybe<TSelectKindPlacement> placement, TStringBuf what); - bool NeedPassLimitOrderByToUnderlyingSelect(TMaybe<TSelectKindPlacement> placement); - - template<typename TRule> - TSourcePtr Build(const TRule& node, TPosition pos, TSelectKindResult&& first); - - - TSelectKindResult SelectKind(const TRule_select_kind& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); - TSelectKindResult SelectKind(const TRule_select_kind_partial& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); - TSelectKindResult SelectKind(const TRule_select_kind_parenthesis& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); + + struct TSelectKindResult { + TSourcePtr Source; + TWriteSettings Settings; + + TVector<TSortSpecificationPtr> SelectOpOrderBy; + bool SelectOpAssumeOrderBy = false; + TNodePtr SelectOpSkipTake; + + inline explicit operator bool() const { + return static_cast<bool>(Source); + } + }; + + bool ValidateLimitOrderByWithSelectOp(TMaybe<TSelectKindPlacement> placement, TStringBuf what); + bool NeedPassLimitOrderByToUnderlyingSelect(TMaybe<TSelectKindPlacement> placement); + + template<typename TRule> + TSourcePtr Build(const TRule& node, TPosition pos, TSelectKindResult&& first); + + + TSelectKindResult SelectKind(const TRule_select_kind& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); + TSelectKindResult SelectKind(const TRule_select_kind_partial& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); + TSelectKindResult SelectKind(const TRule_select_kind_parenthesis& node, TPosition& selectPos, TMaybe<TSelectKindPlacement> placement); +}; + +class TSqlValues: public TSqlTranslation { +public: + TSqlValues(TContext& ctx, NSQLTranslation::ESqlMode mode) + : TSqlTranslation(ctx, mode) + { + } + + TSourcePtr Build(const TRule_values_stmt& node, TPosition& valuesPos, const TVector<TString>& derivedColumns = {}, TPosition derivedColumnsPos = TPosition()); +protected: + bool BuildRows(const TRule_values_source_row_list& node, TVector<TVector<TNodePtr>>& rows); + +private: + bool BuildRow(const TRule_values_source_row& inRow, TVector<TNodePtr>& outRow); }; -class TSqlValues: public TSqlTranslation { -public: - TSqlValues(TContext& ctx, NSQLTranslation::ESqlMode mode) - : TSqlTranslation(ctx, mode) - { - } - - TSourcePtr Build(const TRule_values_stmt& node, TPosition& valuesPos, const TVector<TString>& derivedColumns = {}, TPosition derivedColumnsPos = TPosition()); -protected: - bool BuildRows(const TRule_values_source_row_list& node, TVector<TVector<TNodePtr>>& rows); - -private: - bool BuildRow(const TRule_values_source_row& inRow, TVector<TNodePtr>& outRow); -}; - class TGroupByClause: public TSqlTranslation { enum class EGroupByFeatures { Begin, @@ -3582,24 +3582,24 @@ public: {} bool Build(const TRule_group_by_clause& node, bool stream); - bool ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext); + bool ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext); void SetFeatures(const TString& field) const; TVector<TNodePtr>& Content(); TMap<TString, TNodePtr>& Aliases(); - THoppingWindowSpecPtr GetHoppingWindow() const; + THoppingWindowSpecPtr GetHoppingWindow() const; bool IsCompactGroupBy() const; private: - TMaybe<TVector<TNodePtr>> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const; - bool ResolveGroupByAndGrouping(); - bool GroupingElement(const TRule_grouping_element& node, EGroupByFeatures featureContext); + TMaybe<TVector<TNodePtr>> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const; + bool ResolveGroupByAndGrouping(); + bool GroupingElement(const TRule_grouping_element& node, EGroupByFeatures featureContext); void FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& collection, bool& hasEmpty) const; - bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext); - bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext); + bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext); + bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext); bool HoppingWindow(const TRule_hopping_window_specification& node); - bool AllowUnnamed(TPosition pos, EGroupByFeatures featureContext); + bool AllowUnnamed(TPosition pos, EGroupByFeatures featureContext); TGroupingSetFeatures& Features(); const TGroupingSetFeatures& Features() const; @@ -3632,7 +3632,7 @@ bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& s } if (strLen > 1) { auto iter = str.cend() - 1; - if (*iter == 'l' || *iter == 's' || *iter == 't' || /* deprecated */ *iter == 'b') { + if (*iter == 'l' || *iter == 's' || *iter == 't' || /* deprecated */ *iter == 'b') { --iter; } if (*iter == 'u') { @@ -3644,22 +3644,22 @@ bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& s const TString digString(str.begin() + (base == 10 ? 0 : 2), str.end() - suffix.size()); for (const char& cur: digString) { const ui64 curDigit = Char2DigitTable[static_cast<int>(cur)]; - if (curDigit >= base) { + if (curDigit >= base) { ctx.Error(ctx.Pos()) << "Failed to parse number from string: " << strOrig << ", char: '" << cur << "' is out of base: " << base; return false; } - - ui64 curValue = value; + + ui64 curValue = value; value *= base; - bool overflow = ((value / base) != curValue); - if (!overflow) { - curValue = value; - value += curDigit; - overflow = value < curValue; - } - - if (overflow) { + bool overflow = ((value / base) != curValue); + if (!overflow) { + curValue = value; + value += curDigit; + overflow = value < curValue; + } + + if (overflow) { ctx.Error(ctx.Pos()) << "Failed to parse number from string: " << strOrig << ", number limit overflow"; return false; } @@ -3674,17 +3674,17 @@ TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node) { if (!ParseNumbers(ctx, intergerString, value, suffix)) { return {}; } - + const bool noSpaceForInt32 = value >> 31; const bool noSpaceForInt64 = value >> 63; if (suffix == "") { - bool implicitType = true; + bool implicitType = true; if (noSpaceForInt64) { - return new TLiteralNumberNode<ui64>(ctx.Pos(), "Uint64", ToString(value), implicitType); + return new TLiteralNumberNode<ui64>(ctx.Pos(), "Uint64", ToString(value), implicitType); } else if (noSpaceForInt32) { - return new TLiteralNumberNode<i64>(ctx.Pos(), "Int64", ToString(value), implicitType); + return new TLiteralNumberNode<i64>(ctx.Pos(), "Int64", ToString(value), implicitType); } - return new TLiteralNumberNode<i32>(ctx.Pos(), "Int32", ToString(value), implicitType); + return new TLiteralNumberNode<i32>(ctx.Pos(), "Int32", ToString(value), implicitType); } else if (suffix == "u") { return new TLiteralNumberNode<ui32>(ctx.Pos(), "Uint32", ToString(value)); } else if (suffix == "ul") { @@ -3716,93 +3716,93 @@ TNodePtr LiteralReal(TContext& ctx, const TRule_real& node) { } } -TMaybe<TExprOrIdent> TSqlExpression::LiteralExpr(const TRule_literal_value& node) { - TExprOrIdent result; +TMaybe<TExprOrIdent> TSqlExpression::LiteralExpr(const TRule_literal_value& node) { + TExprOrIdent result; switch (node.Alt_case()) { case TRule_literal_value::kAltLiteralValue1: { - result.Expr = LiteralNumber(Ctx, node.GetAlt_literal_value1().GetRule_integer1()); - break; + result.Expr = LiteralNumber(Ctx, node.GetAlt_literal_value1().GetRule_integer1()); + break; } case TRule_literal_value::kAltLiteralValue2: { - result.Expr = LiteralReal(Ctx, node.GetAlt_literal_value2().GetRule_real1()); - break; + result.Expr = LiteralReal(Ctx, node.GetAlt_literal_value2().GetRule_real1()); + break; } case TRule_literal_value::kAltLiteralValue3: { const TString value(Token(node.GetAlt_literal_value3().GetToken1())); - return BuildLiteralTypedSmartStringOrId(Ctx, value); + return BuildLiteralTypedSmartStringOrId(Ctx, value); } case TRule_literal_value::kAltLiteralValue5: { Token(node.GetAlt_literal_value5().GetToken1()); - result.Expr = BuildLiteralNull(Ctx.Pos()); - break; + result.Expr = BuildLiteralNull(Ctx.Pos()); + break; } case TRule_literal_value::kAltLiteralValue9: { - const TString value(to_lower(Token(node.GetAlt_literal_value9().GetRule_bool_value1().GetToken1()))); - result.Expr = BuildLiteralBool(Ctx.Pos(), FromString<bool>(value)); - break; + const TString value(to_lower(Token(node.GetAlt_literal_value9().GetRule_bool_value1().GetToken1()))); + result.Expr = BuildLiteralBool(Ctx.Pos(), FromString<bool>(value)); + break; } case TRule_literal_value::kAltLiteralValue10: { - result.Expr = BuildEmptyAction(Ctx.Pos()); - break; + result.Expr = BuildEmptyAction(Ctx.Pos()); + break; } default: AltNotImplemented("literal_value", node); } - if (!result.Expr) { - return {}; - } - return result; + if (!result.Expr) { + return {}; + } + return result; } -template<typename TUnarySubExprType> -TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node, const TTrailingQuestions& tail) { +template<typename TUnarySubExprType> +TNodePtr TSqlExpression::UnaryExpr(const TUnarySubExprType& node, const TTrailingQuestions& tail) { if constexpr (std::is_same_v<TUnarySubExprType, TRule_unary_subexpr>) { if (node.Alt_case() == TRule_unary_subexpr::kAltUnarySubexpr1) { - return UnaryCasualExpr(node.GetAlt_unary_subexpr1().GetRule_unary_casual_subexpr1(), tail); - } else if (tail.Count) { - UnexpectedQuestionToken(tail); - return {}; + return UnaryCasualExpr(node.GetAlt_unary_subexpr1().GetRule_unary_casual_subexpr1(), tail); + } else if (tail.Count) { + UnexpectedQuestionToken(tail); + return {}; } else { return JsonApiExpr(node.GetAlt_unary_subexpr2().GetRule_json_api_expr1()); } } else { if (node.Alt_case() == TRule_in_unary_subexpr::kAltInUnarySubexpr1) { - return UnaryCasualExpr(node.GetAlt_in_unary_subexpr1().GetRule_in_unary_casual_subexpr1(), tail); - } else if (tail.Count) { - UnexpectedQuestionToken(tail); - return {}; + return UnaryCasualExpr(node.GetAlt_in_unary_subexpr1().GetRule_in_unary_casual_subexpr1(), tail); + } else if (tail.Count) { + UnexpectedQuestionToken(tail); + return {}; } else { - return JsonApiExpr(node.GetAlt_in_unary_subexpr2().GetRule_json_api_expr1()); + return JsonApiExpr(node.GetAlt_in_unary_subexpr2().GetRule_json_api_expr1()); } } } TNodePtr TSqlExpression::JsonPathSpecification(const TRule_jsonpath_spec& node) { /* - jsonpath_spec: STRING_VALUE; + jsonpath_spec: STRING_VALUE; */ - TString value = Token(node.GetToken1()); - TPosition pos = Ctx.Pos(); + TString value = Token(node.GetToken1()); + TPosition pos = Ctx.Pos(); - auto parsed = StringContent(Ctx, pos, value); - if (!parsed) { + auto parsed = StringContent(Ctx, pos, value); + if (!parsed) { return nullptr; } - return new TCallNodeImpl(pos, "Utf8", {BuildQuotedAtom(pos, parsed->Content, parsed->Flags)}); + return new TCallNodeImpl(pos, "Utf8", {BuildQuotedAtom(pos, parsed->Content, parsed->Flags)}); } TNodePtr TSqlExpression::JsonReturningTypeRule(const TRule_type_name_simple& node) { /* (RETURNING type_name_simple)? */ - return TypeSimple(node, /* onlyDataAllowed */ true); + return TypeSimple(node, /* onlyDataAllowed */ true); } -TNodePtr TSqlExpression::JsonInputArg(const TRule_json_common_args& node) { - /* - json_common_args: expr COMMA jsonpath_spec (PASSING json_variables)?; - */ - TNodePtr jsonExpr = Build(node.GetRule_expr1()); +TNodePtr TSqlExpression::JsonInputArg(const TRule_json_common_args& node) { + /* + json_common_args: expr COMMA jsonpath_spec (PASSING json_variables)?; + */ + TNodePtr jsonExpr = Build(node.GetRule_expr1()); if (!jsonExpr || jsonExpr->IsNull()) { jsonExpr = new TCallNodeImpl(Ctx.Pos(), "Nothing", { new TCallNodeImpl(Ctx.Pos(), "OptionalType", {BuildDataType(Ctx.Pos(), "Json")}) @@ -3812,7 +3812,7 @@ TNodePtr TSqlExpression::JsonInputArg(const TRule_json_common_args& node) { return jsonExpr; } -void TSqlExpression::AddJsonVariable(const TRule_json_variable& node, TVector<TNodePtr>& children) { +void TSqlExpression::AddJsonVariable(const TRule_json_variable& node, TVector<TNodePtr>& children) { /* json_variable: expr AS json_variable_name; */ @@ -3821,57 +3821,57 @@ void TSqlExpression::AddJsonVariable(const TRule_json_variable& node, TVector<TN TPosition namePos = Ctx.Pos(); ui32 nameFlags = 0; - expr = Build(node.GetRule_expr1()); - const auto& nameRule = node.GetRule_json_variable_name3(); - switch (nameRule.GetAltCase()) { - case TRule_json_variable_name::kAltJsonVariableName1: - rawName = Id(nameRule.GetAlt_json_variable_name1().GetRule_id_expr1(), *this); - nameFlags = TNodeFlags::ArbitraryContent; - break; - case TRule_json_variable_name::kAltJsonVariableName2: { - const auto& token = nameRule.GetAlt_json_variable_name2().GetToken1(); - namePos = GetPos(token); - auto parsed = StringContentOrIdContent(Ctx, namePos, token.GetValue()); - if (!parsed) { - return; - } - rawName = parsed->Content; - nameFlags = parsed->Flags; - break; + expr = Build(node.GetRule_expr1()); + const auto& nameRule = node.GetRule_json_variable_name3(); + switch (nameRule.GetAltCase()) { + case TRule_json_variable_name::kAltJsonVariableName1: + rawName = Id(nameRule.GetAlt_json_variable_name1().GetRule_id_expr1(), *this); + nameFlags = TNodeFlags::ArbitraryContent; + break; + case TRule_json_variable_name::kAltJsonVariableName2: { + const auto& token = nameRule.GetAlt_json_variable_name2().GetToken1(); + namePos = GetPos(token); + auto parsed = StringContentOrIdContent(Ctx, namePos, token.GetValue()); + if (!parsed) { + return; + } + rawName = parsed->Content; + nameFlags = parsed->Flags; + break; } - default: - Y_FAIL("You should change implementation according to grammar changes"); + default: + Y_FAIL("You should change implementation according to grammar changes"); } TNodePtr nameExpr = BuildQuotedAtom(namePos, rawName, nameFlags); children.push_back(BuildTuple(namePos, {nameExpr, expr})); } -void TSqlExpression::AddJsonVariables(const TRule_json_variables& node, TVector<TNodePtr>& children) { +void TSqlExpression::AddJsonVariables(const TRule_json_variables& node, TVector<TNodePtr>& children) { /* json_variables: json_variable (COMMA json_variable)*; */ - AddJsonVariable(node.GetRule_json_variable1(), children); + AddJsonVariable(node.GetRule_json_variable1(), children); for (size_t i = 0; i < node.Block2Size(); i++) { AddJsonVariable(node.GetBlock2(i).GetRule_json_variable2(), children); } } -TNodePtr TSqlExpression::JsonVariables(const TRule_json_common_args& node) { +TNodePtr TSqlExpression::JsonVariables(const TRule_json_common_args& node) { /* - json_common_args: expr COMMA jsonpath_spec (PASSING json_variables)?; + json_common_args: expr COMMA jsonpath_spec (PASSING json_variables)?; */ TVector<TNodePtr> variables; TPosition pos = Ctx.Pos(); if (node.HasBlock4()) { const auto& block = node.GetBlock4(); pos = GetPos(block.GetToken1()); - AddJsonVariables(block.GetRule_json_variables2(), variables); + AddJsonVariables(block.GetRule_json_variables2(), variables); } return new TCallNodeImpl(pos, "JsonVariables", variables); } -void TSqlExpression::AddJsonCommonArgs(const TRule_json_common_args& node, TVector<TNodePtr>& children) { +void TSqlExpression::AddJsonCommonArgs(const TRule_json_common_args& node, TVector<TNodePtr>& children) { /* json_common_args: expr COMMA jsonpath_spec (PASSING json_variables)?; */ @@ -3884,31 +3884,31 @@ void TSqlExpression::AddJsonCommonArgs(const TRule_json_common_args& node, TVect children.push_back(variables); } -TNodePtr TSqlExpression::JsonValueCaseHandler(const TRule_json_case_handler& node, EJsonValueHandlerMode& mode) { +TNodePtr TSqlExpression::JsonValueCaseHandler(const TRule_json_case_handler& node, EJsonValueHandlerMode& mode) { /* json_case_handler: ERROR | NULL | (DEFAULT expr); */ - switch (node.GetAltCase()) { - case TRule_json_case_handler::kAltJsonCaseHandler1: { - const auto pos = GetPos(node.GetAlt_json_case_handler1().GetToken1()); - mode = EJsonValueHandlerMode::Error; - return new TCallNodeImpl(pos, "Null", {}); - } - case TRule_json_case_handler::kAltJsonCaseHandler2: { - const auto pos = GetPos(node.GetAlt_json_case_handler2().GetToken1()); - mode = EJsonValueHandlerMode::DefaultValue; - return new TCallNodeImpl(pos, "Null", {}); - } - case TRule_json_case_handler::kAltJsonCaseHandler3: - mode = EJsonValueHandlerMode::DefaultValue; - return Build(node.GetAlt_json_case_handler3().GetBlock1().GetRule_expr2()); - default: - Y_FAIL("You should change implementation according to grammar changes"); + switch (node.GetAltCase()) { + case TRule_json_case_handler::kAltJsonCaseHandler1: { + const auto pos = GetPos(node.GetAlt_json_case_handler1().GetToken1()); + mode = EJsonValueHandlerMode::Error; + return new TCallNodeImpl(pos, "Null", {}); + } + case TRule_json_case_handler::kAltJsonCaseHandler2: { + const auto pos = GetPos(node.GetAlt_json_case_handler2().GetToken1()); + mode = EJsonValueHandlerMode::DefaultValue; + return new TCallNodeImpl(pos, "Null", {}); + } + case TRule_json_case_handler::kAltJsonCaseHandler3: + mode = EJsonValueHandlerMode::DefaultValue; + return Build(node.GetAlt_json_case_handler3().GetBlock1().GetRule_expr2()); + default: + Y_FAIL("You should change implementation according to grammar changes"); } } -void TSqlExpression::AddJsonValueCaseHandlers(const TRule_json_value& node, TVector<TNodePtr>& children) { +void TSqlExpression::AddJsonValueCaseHandlers(const TRule_json_value& node, TVector<TNodePtr>& children) { /* json_case_handler* */ @@ -3945,7 +3945,7 @@ void TSqlExpression::AddJsonValueCaseHandlers(const TRule_json_value& node, TVec } EJsonValueHandlerMode currentMode; - TNodePtr currentHandler = JsonValueCaseHandler(block.GetRule_json_case_handler1(), currentMode); + TNodePtr currentHandler = JsonValueCaseHandler(block.GetRule_json_case_handler1(), currentMode); if (isEmptyClause) { onEmpty = currentHandler; @@ -3970,42 +3970,42 @@ void TSqlExpression::AddJsonValueCaseHandlers(const TRule_json_value& node, TVec children.push_back(onError); } -TNodePtr TSqlExpression::JsonValueExpr(const TRule_json_value& node) { +TNodePtr TSqlExpression::JsonValueExpr(const TRule_json_value& node) { /* json_value: JSON_VALUE LPAREN json_common_args (RETURNING type_name_simple)? - (json_case_handler ON (EMPTY | ERROR))* + (json_case_handler ON (EMPTY | ERROR))* RPAREN; */ TVector<TNodePtr> children; - AddJsonCommonArgs(node.GetRule_json_common_args3(), children); + AddJsonCommonArgs(node.GetRule_json_common_args3(), children); AddJsonValueCaseHandlers(node, children); if (node.HasBlock4()) { auto returningType = JsonReturningTypeRule(node.GetBlock4().GetRule_type_name_simple2()); - if (!returningType) { - return {}; - } + if (!returningType) { + return {}; + } children.push_back(returningType); } return new TCallNodeImpl(GetPos(node.GetToken1()), "JsonValue", children); } -void TSqlExpression::AddJsonExistsHandler(const TRule_json_exists& node, TVector<TNodePtr>& children) { - /* - json_exists: JSON_EXISTS LPAREN - json_common_args - json_exists_handler? - RPAREN; - */ - auto buildJustBool = [&](const TPosition& pos, bool value) { +void TSqlExpression::AddJsonExistsHandler(const TRule_json_exists& node, TVector<TNodePtr>& children) { + /* + json_exists: JSON_EXISTS LPAREN + json_common_args + json_exists_handler? + RPAREN; + */ + auto buildJustBool = [&](const TPosition& pos, bool value) { return new TCallNodeImpl(pos, "Just", {BuildLiteralBool(pos, value)}); }; if (!node.HasBlock4()) { - children.push_back(buildJustBool(Ctx.Pos(), false)); + children.push_back(buildJustBool(Ctx.Pos(), false)); return; } @@ -4019,11 +4019,11 @@ void TSqlExpression::AddJsonExistsHandler(const TRule_json_exists& node, TVector }); children.push_back(nothingNode); } else if (mode != "error") { - children.push_back(buildJustBool(pos, FromString<bool>(mode))); + children.push_back(buildJustBool(pos, FromString<bool>(mode))); } } -TNodePtr TSqlExpression::JsonExistsExpr(const TRule_json_exists& node) { +TNodePtr TSqlExpression::JsonExistsExpr(const TRule_json_exists& node) { /* json_exists: JSON_EXISTS LPAREN json_common_args @@ -4031,21 +4031,21 @@ TNodePtr TSqlExpression::JsonExistsExpr(const TRule_json_exists& node) { RPAREN; */ TVector<TNodePtr> children; - AddJsonCommonArgs(node.GetRule_json_common_args3(), children); + AddJsonCommonArgs(node.GetRule_json_common_args3(), children); AddJsonExistsHandler(node, children); return new TCallNodeImpl(GetPos(node.GetToken1()), "JsonExists", children); } -EJsonQueryWrap TSqlExpression::JsonQueryWrapper(const TRule_json_query& node) { +EJsonQueryWrap TSqlExpression::JsonQueryWrapper(const TRule_json_query& node) { /* - json_query: JSON_QUERY LPAREN - json_common_args - (json_query_wrapper WRAPPER)? - (json_query_handler ON EMPTY)? - (json_query_handler ON ERROR)? - RPAREN; + json_query: JSON_QUERY LPAREN + json_common_args + (json_query_wrapper WRAPPER)? + (json_query_handler ON EMPTY)? + (json_query_handler ON ERROR)? + RPAREN; */ // default behaviour - no wrapping if (!node.HasBlock4()) { @@ -4090,18 +4090,18 @@ EJsonQueryHandler TSqlExpression::JsonQueryHandler(const TRule_json_query_handle } } -TNodePtr TSqlExpression::JsonQueryExpr(const TRule_json_query& node) { +TNodePtr TSqlExpression::JsonQueryExpr(const TRule_json_query& node) { /* json_query: JSON_QUERY LPAREN json_common_args - (json_query_wrapper WRAPPER)? + (json_query_wrapper WRAPPER)? (json_query_handler ON EMPTY)? (json_query_handler ON ERROR)? RPAREN; */ TVector<TNodePtr> children; - AddJsonCommonArgs(node.GetRule_json_common_args3(), children); + AddJsonCommonArgs(node.GetRule_json_common_args3(), children); auto addChild = [&](TPosition pos, const TString& content) { children.push_back(BuildQuotedAtom(pos, content, TNodeFlags::Default)); @@ -4130,82 +4130,82 @@ TNodePtr TSqlExpression::JsonQueryExpr(const TRule_json_query& node) { return new TCallNodeImpl(GetPos(node.GetToken1()), "JsonQuery", children); } -TNodePtr TSqlExpression::JsonApiExpr(const TRule_json_api_expr& node) { +TNodePtr TSqlExpression::JsonApiExpr(const TRule_json_api_expr& node) { /* json_api_expr: json_value | json_exists | json_query; */ TPosition pos = Ctx.Pos(); TNodePtr result = nullptr; - switch (node.GetAltCase()) { - case TRule_json_api_expr::kAltJsonApiExpr1: { - const auto& jsonValue = node.GetAlt_json_api_expr1().GetRule_json_value1(); - pos = GetPos(jsonValue.GetToken1()); - result = JsonValueExpr(jsonValue); - break; - } - case TRule_json_api_expr::kAltJsonApiExpr2: { - const auto& jsonExists = node.GetAlt_json_api_expr2().GetRule_json_exists1(); - pos = GetPos(jsonExists.GetToken1()); - result = JsonExistsExpr(jsonExists); - break; - } - case TRule_json_api_expr::kAltJsonApiExpr3: { - const auto& jsonQuery = node.GetAlt_json_api_expr3().GetRule_json_query1(); - pos = GetPos(jsonQuery.GetToken1()); - result = JsonQueryExpr(jsonQuery); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); + switch (node.GetAltCase()) { + case TRule_json_api_expr::kAltJsonApiExpr1: { + const auto& jsonValue = node.GetAlt_json_api_expr1().GetRule_json_value1(); + pos = GetPos(jsonValue.GetToken1()); + result = JsonValueExpr(jsonValue); + break; + } + case TRule_json_api_expr::kAltJsonApiExpr2: { + const auto& jsonExists = node.GetAlt_json_api_expr2().GetRule_json_exists1(); + pos = GetPos(jsonExists.GetToken1()); + result = JsonExistsExpr(jsonExists); + break; + } + case TRule_json_api_expr::kAltJsonApiExpr3: { + const auto& jsonQuery = node.GetAlt_json_api_expr3().GetRule_json_query1(); + pos = GetPos(jsonQuery.GetToken1()); + result = JsonQueryExpr(jsonQuery); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); } return result; } template<typename TUnaryCasualExprRule> -TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const TTrailingQuestions& tail) { +TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const TTrailingQuestions& tail) { // unary_casual_subexpr: (id_expr | atom_expr) unary_subexpr_suffix; - // OR - // in_unary_casual_subexpr: (id_expr_in | in_atom_expr) unary_subexpr_suffix; + // OR + // in_unary_casual_subexpr: (id_expr_in | in_atom_expr) unary_subexpr_suffix; // where // unary_subexpr_suffix: (key_expr | invoke_expr |(DOT (bind_parameter | DIGITS | id)))* (COLLATE id)?; - const auto& suffix = node.GetRule_unary_subexpr_suffix2(); - const bool suffixIsEmpty = suffix.GetBlock1().empty() && !suffix.HasBlock2(); + const auto& suffix = node.GetRule_unary_subexpr_suffix2(); + const bool suffixIsEmpty = suffix.GetBlock1().empty() && !suffix.HasBlock2(); TString name; TNodePtr expr; - bool typePossible = false; + bool typePossible = false; auto& block = node.GetBlock1(); switch (block.Alt_case()) { case TUnaryCasualExprRule::TBlock1::kAlt1: { auto& alt = block.GetAlt1(); if constexpr (std::is_same_v<TUnaryCasualExprRule, TRule_unary_casual_subexpr>) { - name = Id(alt.GetRule_id_expr1(), *this); - typePossible = !IsQuotedId(alt.GetRule_id_expr1(), *this); - } else { - // type was never possible here - name = Id(alt.GetRule_id_expr_in1(), *this); - } + name = Id(alt.GetRule_id_expr1(), *this); + typePossible = !IsQuotedId(alt.GetRule_id_expr1(), *this); + } else { + // type was never possible here + name = Id(alt.GetRule_id_expr_in1(), *this); + } break; } case TUnaryCasualExprRule::TBlock1::kAlt2: { auto& alt = block.GetAlt2(); - TMaybe<TExprOrIdent> exprOrId; + TMaybe<TExprOrIdent> exprOrId; if constexpr (std::is_same_v<TUnaryCasualExprRule, TRule_unary_casual_subexpr>) { - exprOrId = AtomExpr(alt.GetRule_atom_expr1(), suffixIsEmpty ? tail : TTrailingQuestions{}); - } else { - exprOrId = InAtomExpr(alt.GetRule_in_atom_expr1(), suffixIsEmpty ? tail : TTrailingQuestions{}); - } - - if (!exprOrId) { + exprOrId = AtomExpr(alt.GetRule_atom_expr1(), suffixIsEmpty ? tail : TTrailingQuestions{}); + } else { + exprOrId = InAtomExpr(alt.GetRule_in_atom_expr1(), suffixIsEmpty ? tail : TTrailingQuestions{}); + } + + if (!exprOrId) { Ctx.IncrementMonCounter("sql_errors", "BadAtomExpr"); return nullptr; } - if (!exprOrId->Expr) { - name = exprOrId->Ident; - } else { - expr = exprOrId->Expr; - } + if (!exprOrId->Expr) { + name = exprOrId->Ident; + } else { + expr = exprOrId->Expr; + } break; } default: @@ -4250,26 +4250,26 @@ TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const if (!isColumnRef) { lastExpr = expr; } else { - const bool flexibleTypes = Ctx.FlexibleTypes; - bool columnOrType = false; - if (auto simpleType = LookupSimpleTypeBySqlAlias(name, flexibleTypes); simpleType && typePossible && suffixIsEmpty) { - auto columnRefsState = Ctx.GetColumnReferenceState(); - if (tail.Count > 0 || columnRefsState == EColumnRefState::Deny || !flexibleTypes) { - // a type - return AddOptionals(BuildSimpleType(Ctx, Ctx.Pos(), name, false), tail.Count); - } - // type or column: ambiguity will be resolved on type annotation stage - columnOrType = columnRefsState == EColumnRefState::Allow; - } - if (tail.Count) { - UnexpectedQuestionToken(tail); - return {}; - } - if (!Ctx.CheckColumnReference(Ctx.Pos(), name)) { + const bool flexibleTypes = Ctx.FlexibleTypes; + bool columnOrType = false; + if (auto simpleType = LookupSimpleTypeBySqlAlias(name, flexibleTypes); simpleType && typePossible && suffixIsEmpty) { + auto columnRefsState = Ctx.GetColumnReferenceState(); + if (tail.Count > 0 || columnRefsState == EColumnRefState::Deny || !flexibleTypes) { + // a type + return AddOptionals(BuildSimpleType(Ctx, Ctx.Pos(), name, false), tail.Count); + } + // type or column: ambiguity will be resolved on type annotation stage + columnOrType = columnRefsState == EColumnRefState::Allow; + } + if (tail.Count) { + UnexpectedQuestionToken(tail); + return {}; + } + if (!Ctx.CheckColumnReference(Ctx.Pos(), name)) { return nullptr; } - ids.push_back(columnOrType ? BuildColumnOrType(Ctx.Pos()) : BuildColumn(Ctx.Pos())); + ids.push_back(columnOrType ? BuildColumnOrType(Ctx.Pos()) : BuildColumn(Ctx.Pos())); ids.push_back(name); } @@ -4297,9 +4297,9 @@ TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const } case TRule_unary_subexpr_suffix::TBlock1::kAlt2: { // invoke_expr - cannot be a column, function name - TSqlCallExpr call(Ctx, Mode); + TSqlCallExpr call(Ctx, Mode); if (isFirstElem && !name.Empty()) { - call.AllowDistinct(); + call.AllowDistinct(); call.InitName(name); } else { call.InitExpr(lastExpr); @@ -4330,10 +4330,10 @@ TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const auto bb = b.GetAlt3().GetBlock1().GetBlock2(); switch (bb.Alt_case()) { case TRule_unary_subexpr_suffix_TBlock1_TAlt3_TBlock1_TBlock2::kAlt1: { - TString named; - if (!NamedNodeImpl(bb.GetAlt1().GetRule_bind_parameter1(), named, *this)) { - return nullptr; - } + TString named; + if (!NamedNodeImpl(bb.GetAlt1().GetRule_bind_parameter1(), named, *this)) { + return nullptr; + } auto namedNode = GetNamedNode(named); if (!namedNode) { return nullptr; @@ -4384,75 +4384,75 @@ TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const return lastExpr; } -TNodePtr TSqlExpression::BindParameterRule(const TRule_bind_parameter& rule, const TTrailingQuestions& tail) { - TString namedArg; - if (!NamedNodeImpl(rule, namedArg, *this)) { - return {}; - } - if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { - Ctx.IncrementMonCounter("sql_features", "LambdaArgument"); - if (tail.Count > 1) { - Ctx.Error(tail.Pos) << "Expecting at most one '?' token here (for optional lambda parameters), but got " << tail.Count; - return {}; - } - return BuildAtom(Ctx.Pos(), namedArg, NYql::TNodeFlags::ArbitraryContent, tail.Count != 0); - } - if (tail.Count) { - UnexpectedQuestionToken(tail); - return {}; - } - Ctx.IncrementMonCounter("sql_features", "NamedNodeUseAtom"); - return GetNamedNode(namedArg); -} - -TNodePtr TSqlExpression::LambdaRule(const TRule_lambda& rule) { - const auto& alt = rule; - const bool isSqlLambda = alt.HasBlock2(); - if (!isSqlLambda) { - return SmartParenthesis(alt.GetRule_smart_parenthesis1()); - } - - TNodePtr parenthesis; - { - // we allow column reference here to postpone error and report it with better description in SqlLambdaParams - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - TSqlExpression expr(Ctx, Mode); - expr.SetSmartParenthesisMode(ESmartParenthesis::SqlLambdaParams); - parenthesis = expr.SmartParenthesis(alt.GetRule_smart_parenthesis1()); - } - if (!parenthesis) { - return {}; - } +TNodePtr TSqlExpression::BindParameterRule(const TRule_bind_parameter& rule, const TTrailingQuestions& tail) { + TString namedArg; + if (!NamedNodeImpl(rule, namedArg, *this)) { + return {}; + } + if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { + Ctx.IncrementMonCounter("sql_features", "LambdaArgument"); + if (tail.Count > 1) { + Ctx.Error(tail.Pos) << "Expecting at most one '?' token here (for optional lambda parameters), but got " << tail.Count; + return {}; + } + return BuildAtom(Ctx.Pos(), namedArg, NYql::TNodeFlags::ArbitraryContent, tail.Count != 0); + } + if (tail.Count) { + UnexpectedQuestionToken(tail); + return {}; + } + Ctx.IncrementMonCounter("sql_features", "NamedNodeUseAtom"); + return GetNamedNode(namedArg); +} + +TNodePtr TSqlExpression::LambdaRule(const TRule_lambda& rule) { + const auto& alt = rule; + const bool isSqlLambda = alt.HasBlock2(); + if (!isSqlLambda) { + return SmartParenthesis(alt.GetRule_smart_parenthesis1()); + } + + TNodePtr parenthesis; + { + // we allow column reference here to postpone error and report it with better description in SqlLambdaParams + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TSqlExpression expr(Ctx, Mode); + expr.SetSmartParenthesisMode(ESmartParenthesis::SqlLambdaParams); + parenthesis = expr.SmartParenthesis(alt.GetRule_smart_parenthesis1()); + } + if (!parenthesis) { + return {}; + } ui32 optionalArgumentsCount = 0; - TVector<TSymbolNameWithPos> args; + TVector<TSymbolNameWithPos> args; if (!SqlLambdaParams(parenthesis, args, optionalArgumentsCount)) { - return {}; - } - auto bodyBlock = alt.GetBlock2(); - Token(bodyBlock.GetToken1()); - TPosition pos(Ctx.Pos()); - TVector<TNodePtr> exprSeq; - for (auto& arg: args) { - arg.Name = PushNamedAtom(arg.Pos, arg.Name); - } + return {}; + } + auto bodyBlock = alt.GetBlock2(); + Token(bodyBlock.GetToken1()); + TPosition pos(Ctx.Pos()); + TVector<TNodePtr> exprSeq; + for (auto& arg: args) { + arg.Name = PushNamedAtom(arg.Pos, arg.Name); + } bool ret = false; - TColumnRefScope scope(Ctx, EColumnRefState::Deny); - scope.SetNoColumnErrContext("in lambda function"); + TColumnRefScope scope(Ctx, EColumnRefState::Deny); + scope.SetNoColumnErrContext("in lambda function"); if (bodyBlock.GetBlock2().HasAlt1()) { ret = SqlLambdaExprBody(Ctx, bodyBlock.GetBlock2().GetAlt1().GetBlock1().GetRule_expr2(), exprSeq); } else { ret = SqlLambdaExprBody(Ctx, bodyBlock.GetBlock2().GetAlt2().GetBlock1().GetRule_lambda_body2(), exprSeq); } - TVector<TString> argNames; - for (const auto& arg : args) { - argNames.push_back(arg.Name); - PopNamedNode(arg.Name); - } - if (!ret) { - return {}; - } + TVector<TString> argNames; + for (const auto& arg : args) { + argNames.push_back(arg.Name); + PopNamedNode(arg.Name); + } + if (!ret) { + return {}; + } auto lambdaNode = BuildSqlLambda(pos, std::move(argNames), std::move(exprSeq)); if (optionalArgumentsCount > 0) { @@ -4463,315 +4463,315 @@ TNodePtr TSqlExpression::LambdaRule(const TRule_lambda& rule) { } return lambdaNode; -} - -TNodePtr TSqlExpression::CastRule(const TRule_cast_expr& rule) { - Ctx.IncrementMonCounter("sql_features", "Cast"); - const auto& alt = rule; - Token(alt.GetToken1()); - TPosition pos(Ctx.Pos()); - TSqlExpression expr(Ctx, Mode); - auto exprNode = expr.Build(rule.GetRule_expr3()); - if (!exprNode) { - return {}; - } - auto type = TypeNodeOrBind(rule.GetRule_type_name_or_bind5()); - if (!type) { - return {}; - } - return new TCallNodeImpl(pos, "SafeCast", {exprNode, type}); -} - +} + +TNodePtr TSqlExpression::CastRule(const TRule_cast_expr& rule) { + Ctx.IncrementMonCounter("sql_features", "Cast"); + const auto& alt = rule; + Token(alt.GetToken1()); + TPosition pos(Ctx.Pos()); + TSqlExpression expr(Ctx, Mode); + auto exprNode = expr.Build(rule.GetRule_expr3()); + if (!exprNode) { + return {}; + } + auto type = TypeNodeOrBind(rule.GetRule_type_name_or_bind5()); + if (!type) { + return {}; + } + return new TCallNodeImpl(pos, "SafeCast", {exprNode, type}); +} + TNodePtr TSqlExpression::BitCastRule(const TRule_bitcast_expr& rule) { Ctx.IncrementMonCounter("sql_features", "BitCast"); const auto& alt = rule; Token(alt.GetToken1()); TPosition pos(Ctx.Pos()); TSqlExpression expr(Ctx, Mode); - auto exprNode = expr.Build(rule.GetRule_expr3()); - if (!exprNode) { - return {}; - } - auto type = TypeSimple(rule.GetRule_type_name_simple5(), true); - if (!type) { - return {}; - } - return new TCallNodeImpl(pos, "BitCast", {exprNode, type}); -} - -TNodePtr TSqlExpression::ExistsRule(const TRule_exists_expr& rule) { - Ctx.IncrementMonCounter("sql_features", "Exists"); - - TPosition pos; - TSourcePtr source; - Token(rule.GetToken2()); - switch (rule.GetBlock3().Alt_case()) { - case TRule_exists_expr::TBlock3::kAlt1: { - const auto& alt = rule.GetBlock3().GetAlt1().GetRule_select_stmt1(); - TSqlSelect select(Ctx, Mode); - source = select.Build(alt, pos); - break; - } - case TRule_exists_expr::TBlock3::kAlt2: { - const auto& alt = rule.GetBlock3().GetAlt2().GetRule_values_stmt1(); - TSqlValues values(Ctx, Mode); - source = values.Build(alt, pos); - break; - } - default: - AltNotImplemented("exists_expr", rule.GetBlock3()); - } - - if (!source) { - Ctx.IncrementMonCounter("sql_errors", "BadSource"); - return nullptr; - } - const bool checkExist = true; - return BuildBuiltinFunc(Ctx, Ctx.Pos(), "ListHasItems", {BuildSourceNode(pos, std::move(source), checkExist)}); -} - -TNodePtr TSqlExpression::CaseRule(const TRule_case_expr& rule) { - // case_expr: CASE expr? when_expr+ (ELSE expr)? END; - // when_expr: WHEN expr THEN expr; - Ctx.IncrementMonCounter("sql_features", "Case"); - const auto& alt = rule; - Token(alt.GetToken1()); - TNodePtr elseExpr; - if (alt.HasBlock4()) { - Token(alt.GetBlock4().GetToken1()); - TSqlExpression expr(Ctx, Mode); - elseExpr = expr.Build(alt.GetBlock4().GetRule_expr2()); - } else { - Ctx.IncrementMonCounter("sql_errors", "ElseIsRequired"); - Error() << "ELSE is required"; - return {}; - } - - TNodePtr caseExpr; - if (alt.HasBlock2()) { - TSqlExpression expr(Ctx, Mode); - caseExpr = expr.Build(alt.GetBlock2().GetRule_expr1()); - if (!caseExpr) { - return {}; - } - } - - TVector<TCaseBranch> branches; - for (size_t i = 0; i < alt.Block3Size(); ++i) { - branches.emplace_back(); - const auto& block = alt.GetBlock3(i).GetRule_when_expr1(); - Token(block.GetToken1()); - TSqlExpression condExpr(Ctx, Mode); - branches.back().Pred = condExpr.Build(block.GetRule_expr2()); - if (caseExpr) { - branches.back().Pred = BuildBinaryOp(Ctx, Ctx.Pos(), "==", caseExpr, branches.back().Pred); - } - if (!branches.back().Pred) { - return {}; - } - Token(block.GetToken3()); - TSqlExpression thenExpr(Ctx, Mode); - branches.back().Value = thenExpr.Build(block.GetRule_expr4()); - if (!branches.back().Value) { - return {}; - } - } - auto final = ReduceCaseBranches(branches.begin(), branches.end()); - return BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", { final.Pred, final.Value, elseExpr }); -} - -TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const TTrailingQuestions& tail) { + auto exprNode = expr.Build(rule.GetRule_expr3()); + if (!exprNode) { + return {}; + } + auto type = TypeSimple(rule.GetRule_type_name_simple5(), true); + if (!type) { + return {}; + } + return new TCallNodeImpl(pos, "BitCast", {exprNode, type}); +} + +TNodePtr TSqlExpression::ExistsRule(const TRule_exists_expr& rule) { + Ctx.IncrementMonCounter("sql_features", "Exists"); + + TPosition pos; + TSourcePtr source; + Token(rule.GetToken2()); + switch (rule.GetBlock3().Alt_case()) { + case TRule_exists_expr::TBlock3::kAlt1: { + const auto& alt = rule.GetBlock3().GetAlt1().GetRule_select_stmt1(); + TSqlSelect select(Ctx, Mode); + source = select.Build(alt, pos); + break; + } + case TRule_exists_expr::TBlock3::kAlt2: { + const auto& alt = rule.GetBlock3().GetAlt2().GetRule_values_stmt1(); + TSqlValues values(Ctx, Mode); + source = values.Build(alt, pos); + break; + } + default: + AltNotImplemented("exists_expr", rule.GetBlock3()); + } + + if (!source) { + Ctx.IncrementMonCounter("sql_errors", "BadSource"); + return nullptr; + } + const bool checkExist = true; + return BuildBuiltinFunc(Ctx, Ctx.Pos(), "ListHasItems", {BuildSourceNode(pos, std::move(source), checkExist)}); +} + +TNodePtr TSqlExpression::CaseRule(const TRule_case_expr& rule) { + // case_expr: CASE expr? when_expr+ (ELSE expr)? END; + // when_expr: WHEN expr THEN expr; + Ctx.IncrementMonCounter("sql_features", "Case"); + const auto& alt = rule; + Token(alt.GetToken1()); + TNodePtr elseExpr; + if (alt.HasBlock4()) { + Token(alt.GetBlock4().GetToken1()); + TSqlExpression expr(Ctx, Mode); + elseExpr = expr.Build(alt.GetBlock4().GetRule_expr2()); + } else { + Ctx.IncrementMonCounter("sql_errors", "ElseIsRequired"); + Error() << "ELSE is required"; + return {}; + } + + TNodePtr caseExpr; + if (alt.HasBlock2()) { + TSqlExpression expr(Ctx, Mode); + caseExpr = expr.Build(alt.GetBlock2().GetRule_expr1()); + if (!caseExpr) { + return {}; + } + } + + TVector<TCaseBranch> branches; + for (size_t i = 0; i < alt.Block3Size(); ++i) { + branches.emplace_back(); + const auto& block = alt.GetBlock3(i).GetRule_when_expr1(); + Token(block.GetToken1()); + TSqlExpression condExpr(Ctx, Mode); + branches.back().Pred = condExpr.Build(block.GetRule_expr2()); + if (caseExpr) { + branches.back().Pred = BuildBinaryOp(Ctx, Ctx.Pos(), "==", caseExpr, branches.back().Pred); + } + if (!branches.back().Pred) { + return {}; + } + Token(block.GetToken3()); + TSqlExpression thenExpr(Ctx, Mode); + branches.back().Value = thenExpr.Build(block.GetRule_expr4()); + if (!branches.back().Value) { + return {}; + } + } + auto final = ReduceCaseBranches(branches.begin(), branches.end()); + return BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", { final.Pred, final.Value, elseExpr }); +} + +TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const TTrailingQuestions& tail) { // atom_expr: // literal_value // | bind_parameter // | lambda - // | cast_expr - // | exists_expr - // | case_expr - // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) + // | cast_expr + // | exists_expr + // | case_expr + // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) // | value_constructor // | bitcast_expr - // | list_literal - // | dict_literal - // | struct_literal + // | list_literal + // | dict_literal + // | struct_literal // ; - if (node.Alt_case() != TRule_atom_expr::kAltAtomExpr2 && tail.Count) { - UnexpectedQuestionToken(tail); - return {}; - } - TExprOrIdent result; + if (node.Alt_case() != TRule_atom_expr::kAltAtomExpr2 && tail.Count) { + UnexpectedQuestionToken(tail); + return {}; + } + TExprOrIdent result; switch (node.Alt_case()) { case TRule_atom_expr::kAltAtomExpr1: Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); return LiteralExpr(node.GetAlt_atom_expr1().GetRule_literal_value1()); - case TRule_atom_expr::kAltAtomExpr2: - result.Expr = BindParameterRule(node.GetAlt_atom_expr2().GetRule_bind_parameter1(), tail); - break; + case TRule_atom_expr::kAltAtomExpr2: + result.Expr = BindParameterRule(node.GetAlt_atom_expr2().GetRule_bind_parameter1(), tail); + break; case TRule_atom_expr::kAltAtomExpr3: - result.Expr = LambdaRule(node.GetAlt_atom_expr3().GetRule_lambda1()); - break; - case TRule_atom_expr::kAltAtomExpr4: - result.Expr = CastRule(node.GetAlt_atom_expr4().GetRule_cast_expr1()); - break; - case TRule_atom_expr::kAltAtomExpr5: - result.Expr = ExistsRule(node.GetAlt_atom_expr5().GetRule_exists_expr1()); - break; - case TRule_atom_expr::kAltAtomExpr6: - result.Expr = CaseRule(node.GetAlt_atom_expr6().GetRule_case_expr1()); - break; + result.Expr = LambdaRule(node.GetAlt_atom_expr3().GetRule_lambda1()); + break; + case TRule_atom_expr::kAltAtomExpr4: + result.Expr = CastRule(node.GetAlt_atom_expr4().GetRule_cast_expr1()); + break; + case TRule_atom_expr::kAltAtomExpr5: + result.Expr = ExistsRule(node.GetAlt_atom_expr5().GetRule_exists_expr1()); + break; + case TRule_atom_expr::kAltAtomExpr6: + result.Expr = CaseRule(node.GetAlt_atom_expr6().GetRule_case_expr1()); + break; case TRule_atom_expr::kAltAtomExpr7: { const auto& alt = node.GetAlt_atom_expr7(); - TString module(Id(alt.GetRule_an_id_or_type1(), *this)); + TString module(Id(alt.GetRule_an_id_or_type1(), *this)); TPosition pos(Ctx.Pos()); TString name; switch (alt.GetBlock3().Alt_case()) { case TRule_atom_expr::TAlt7::TBlock3::kAlt1: name = Id(alt.GetBlock3().GetAlt1().GetRule_id_or_type1(), *this); break; - case TRule_atom_expr::TAlt7::TBlock3::kAlt2: { - name = Token(alt.GetBlock3().GetAlt2().GetToken1()); - if (Ctx.AnsiQuotedIdentifiers && name.StartsWith('"')) { - // same as previous case - name = IdContentFromString(Ctx, name); - } else { - module = "@" + module; - } + case TRule_atom_expr::TAlt7::TBlock3::kAlt2: { + name = Token(alt.GetBlock3().GetAlt2().GetToken1()); + if (Ctx.AnsiQuotedIdentifiers && name.StartsWith('"')) { + // same as previous case + name = IdContentFromString(Ctx, name); + } else { + module = "@" + module; + } break; - } + } default: Y_FAIL("Unsigned number: you should change implementation according to grammar changes"); } - result.Expr = BuildCallable(pos, module, name, {}); - break; + result.Expr = BuildCallable(pos, module, name, {}); + break; } case TRule_atom_expr::kAltAtomExpr8: { - result.Expr = ValueConstructor(node.GetAlt_atom_expr8().GetRule_value_constructor1()); - break; + result.Expr = ValueConstructor(node.GetAlt_atom_expr8().GetRule_value_constructor1()); + break; } case TRule_atom_expr::kAltAtomExpr9: - result.Expr = BitCastRule(node.GetAlt_atom_expr9().GetRule_bitcast_expr1()); - break; + result.Expr = BitCastRule(node.GetAlt_atom_expr9().GetRule_bitcast_expr1()); + break; case TRule_atom_expr::kAltAtomExpr10: - result.Expr = ListLiteral(node.GetAlt_atom_expr10().GetRule_list_literal1()); - break; + result.Expr = ListLiteral(node.GetAlt_atom_expr10().GetRule_list_literal1()); + break; case TRule_atom_expr::kAltAtomExpr11: - result.Expr = DictLiteral(node.GetAlt_atom_expr11().GetRule_dict_literal1()); - break; + result.Expr = DictLiteral(node.GetAlt_atom_expr11().GetRule_dict_literal1()); + break; case TRule_atom_expr::kAltAtomExpr12: - result.Expr = StructLiteral(node.GetAlt_atom_expr12().GetRule_struct_literal1()); - break; + result.Expr = StructLiteral(node.GetAlt_atom_expr12().GetRule_struct_literal1()); + break; default: AltNotImplemented("atom_expr", node); } - if (!result.Expr) { - return {}; - } - return result; + if (!result.Expr) { + return {}; + } + return result; } -TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node, const TTrailingQuestions& tail) { - // in_atom_expr: - // literal_value - // | bind_parameter +TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node, const TTrailingQuestions& tail) { + // in_atom_expr: + // literal_value + // | bind_parameter // | lambda - // | cast_expr - // | case_expr - // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) - // | LPAREN select_stmt RPAREN + // | cast_expr + // | case_expr + // | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE) + // | LPAREN select_stmt RPAREN // | value_constructor // | bitcast_expr - // | list_literal - // | dict_literal - // | struct_literal - // ; - if (node.Alt_case() != TRule_in_atom_expr::kAltInAtomExpr2 && tail.Count) { - UnexpectedQuestionToken(tail); - return {}; - } - TExprOrIdent result; - switch (node.Alt_case()) { - case TRule_in_atom_expr::kAltInAtomExpr1: - Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); - return LiteralExpr(node.GetAlt_in_atom_expr1().GetRule_literal_value1()); - case TRule_in_atom_expr::kAltInAtomExpr2: - result.Expr = BindParameterRule(node.GetAlt_in_atom_expr2().GetRule_bind_parameter1(), tail); - break; - case TRule_in_atom_expr::kAltInAtomExpr3: - result.Expr = LambdaRule(node.GetAlt_in_atom_expr3().GetRule_lambda1()); - break; - case TRule_in_atom_expr::kAltInAtomExpr4: - result.Expr = CastRule(node.GetAlt_in_atom_expr4().GetRule_cast_expr1()); - break; - case TRule_in_atom_expr::kAltInAtomExpr5: - result.Expr = CaseRule(node.GetAlt_in_atom_expr5().GetRule_case_expr1()); - break; + // | list_literal + // | dict_literal + // | struct_literal + // ; + if (node.Alt_case() != TRule_in_atom_expr::kAltInAtomExpr2 && tail.Count) { + UnexpectedQuestionToken(tail); + return {}; + } + TExprOrIdent result; + switch (node.Alt_case()) { + case TRule_in_atom_expr::kAltInAtomExpr1: + Ctx.IncrementMonCounter("sql_features", "LiteralExpr"); + return LiteralExpr(node.GetAlt_in_atom_expr1().GetRule_literal_value1()); + case TRule_in_atom_expr::kAltInAtomExpr2: + result.Expr = BindParameterRule(node.GetAlt_in_atom_expr2().GetRule_bind_parameter1(), tail); + break; + case TRule_in_atom_expr::kAltInAtomExpr3: + result.Expr = LambdaRule(node.GetAlt_in_atom_expr3().GetRule_lambda1()); + break; + case TRule_in_atom_expr::kAltInAtomExpr4: + result.Expr = CastRule(node.GetAlt_in_atom_expr4().GetRule_cast_expr1()); + break; + case TRule_in_atom_expr::kAltInAtomExpr5: + result.Expr = CaseRule(node.GetAlt_in_atom_expr5().GetRule_case_expr1()); + break; case TRule_in_atom_expr::kAltInAtomExpr6: { const auto& alt = node.GetAlt_in_atom_expr6(); - TString module(Id(alt.GetRule_an_id_or_type1(), *this)); + TString module(Id(alt.GetRule_an_id_or_type1(), *this)); TPosition pos(Ctx.Pos()); TString name; switch (alt.GetBlock3().Alt_case()) { case TRule_in_atom_expr::TAlt6::TBlock3::kAlt1: name = Id(alt.GetBlock3().GetAlt1().GetRule_id_or_type1(), *this); break; - case TRule_in_atom_expr::TAlt6::TBlock3::kAlt2: { - name = Token(alt.GetBlock3().GetAlt2().GetToken1()); - if (Ctx.AnsiQuotedIdentifiers && name.StartsWith('"')) { - // same as previous case - name = IdContentFromString(Ctx, name); - } else { - module = "@" + module; - } + case TRule_in_atom_expr::TAlt6::TBlock3::kAlt2: { + name = Token(alt.GetBlock3().GetAlt2().GetToken1()); + if (Ctx.AnsiQuotedIdentifiers && name.StartsWith('"')) { + // same as previous case + name = IdContentFromString(Ctx, name); + } else { + module = "@" + module; + } break; - } + } default: Y_FAIL("You should change implementation according to grammar changes"); } - result.Expr = BuildCallable(pos, module, name, {}); - break; - } - case TRule_in_atom_expr::kAltInAtomExpr7: { - Token(node.GetAlt_in_atom_expr7().GetToken1()); - // reset column reference scope (select will reenable it where needed) - TColumnRefScope scope(Ctx, EColumnRefState::Deny); - TSqlSelect select(Ctx, Mode); - TPosition pos; - auto source = select.Build(node.GetAlt_in_atom_expr7().GetRule_select_stmt2(), pos); - if (!source) { - Ctx.IncrementMonCounter("sql_errors", "BadSource"); - return {}; - } - Ctx.IncrementMonCounter("sql_features", "InSubquery"); - result.Expr = BuildSelectResult(pos, std::move(source), false, Mode == NSQLTranslation::ESqlMode::SUBQUERY, Ctx.Scoped); - break; - } + result.Expr = BuildCallable(pos, module, name, {}); + break; + } + case TRule_in_atom_expr::kAltInAtomExpr7: { + Token(node.GetAlt_in_atom_expr7().GetToken1()); + // reset column reference scope (select will reenable it where needed) + TColumnRefScope scope(Ctx, EColumnRefState::Deny); + TSqlSelect select(Ctx, Mode); + TPosition pos; + auto source = select.Build(node.GetAlt_in_atom_expr7().GetRule_select_stmt2(), pos); + if (!source) { + Ctx.IncrementMonCounter("sql_errors", "BadSource"); + return {}; + } + Ctx.IncrementMonCounter("sql_features", "InSubquery"); + result.Expr = BuildSelectResult(pos, std::move(source), false, Mode == NSQLTranslation::ESqlMode::SUBQUERY, Ctx.Scoped); + break; + } case TRule_in_atom_expr::kAltInAtomExpr8: { - result.Expr = ValueConstructor(node.GetAlt_in_atom_expr8().GetRule_value_constructor1()); - break; + result.Expr = ValueConstructor(node.GetAlt_in_atom_expr8().GetRule_value_constructor1()); + break; } case TRule_in_atom_expr::kAltInAtomExpr9: - result.Expr = BitCastRule(node.GetAlt_in_atom_expr9().GetRule_bitcast_expr1()); - break; + result.Expr = BitCastRule(node.GetAlt_in_atom_expr9().GetRule_bitcast_expr1()); + break; case TRule_in_atom_expr::kAltInAtomExpr10: - result.Expr = ListLiteral(node.GetAlt_in_atom_expr10().GetRule_list_literal1()); - break; + result.Expr = ListLiteral(node.GetAlt_in_atom_expr10().GetRule_list_literal1()); + break; case TRule_in_atom_expr::kAltInAtomExpr11: - result.Expr = DictLiteral(node.GetAlt_in_atom_expr11().GetRule_dict_literal1()); - break; + result.Expr = DictLiteral(node.GetAlt_in_atom_expr11().GetRule_dict_literal1()); + break; case TRule_in_atom_expr::kAltInAtomExpr12: - result.Expr = StructLiteral(node.GetAlt_in_atom_expr12().GetRule_struct_literal1()); - break; - default: - AltNotImplemented("in_atom_expr", node); - } - if (!result.Expr) { - return {}; - } - return result; -} - + result.Expr = StructLiteral(node.GetAlt_in_atom_expr12().GetRule_struct_literal1()); + break; + default: + AltNotImplemented("in_atom_expr", node); + } + if (!result.Expr) { + return {}; + } + return result; +} + bool TSqlExpression::SqlLambdaParams(const TNodePtr& node, TVector<TSymbolNameWithPos>& args, ui32& optionalArgumentsCount) { - args.clear(); + args.clear(); optionalArgumentsCount = 0; auto errMsg = TStringBuf("Invalid lambda arguments syntax. Lambda arguments should start with '$' as named value."); auto tupleNodePtr = dynamic_cast<TTupleNode*>(node.Get()); @@ -4793,17 +4793,17 @@ bool TSqlExpression::SqlLambdaParams(const TNodePtr& node, TVector<TSymbolNameWi return false; } - if (!IsAnonymousName(*contentPtr) && !dupArgsChecker.insert(*contentPtr).second) { + if (!IsAnonymousName(*contentPtr) && !dupArgsChecker.insert(*contentPtr).second) { Ctx.Error(argPtr->GetPos()) << "Duplicate lambda argument parametr: '" << *contentPtr << "'."; return false; } - args.push_back(TSymbolNameWithPos{*contentPtr, argPtr->GetPos()}); + args.push_back(TSymbolNameWithPos{*contentPtr, argPtr->GetPos()}); } return true; } bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_expr& node, TVector<TNodePtr>& exprSeq) { - TSqlExpression expr(ctx, ctx.Settings.Mode); + TSqlExpression expr(ctx, ctx.Settings.Mode); TNodePtr nodeExpr = expr.Build(node); if (!nodeExpr) { return false; @@ -4813,15 +4813,15 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_expr& node, TV } bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& node, TVector<TNodePtr>& exprSeq) { - TSqlExpression expr(ctx, ctx.Settings.Mode); + TSqlExpression expr(ctx, ctx.Settings.Mode); TVector<TString> localNames; bool hasError = false; - for (auto& block: node.GetBlock2()) { + for (auto& block: node.GetBlock2()) { const auto& rule = block.GetRule_lambda_stmt1(); switch (rule.Alt_case()) { case TRule_lambda_stmt::kAltLambdaStmt1: { - TVector<TSymbolNameWithPos> names; - auto nodeExpr = NamedNode(rule.GetAlt_lambda_stmt1().GetRule_named_nodes_stmt1(), names); + TVector<TSymbolNameWithPos> names; + auto nodeExpr = NamedNode(rule.GetAlt_lambda_stmt1().GetRule_named_nodes_stmt1(), names); if (!nodeExpr) { hasError = true; continue; @@ -4836,22 +4836,22 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n exprSeq.back()->SetLabel(ref); for (size_t i = 0; i < names.size(); ++i) { TNodePtr nthExpr = nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i))); - names[i].Name = PushNamedAtom(names[i].Pos, names[i].Name); - nthExpr->SetLabel(names[i].Name); - localNames.push_back(names[i].Name); + names[i].Name = PushNamedAtom(names[i].Pos, names[i].Name); + nthExpr->SetLabel(names[i].Name); + localNames.push_back(names[i].Name); exprSeq.push_back(nthExpr); } } else { - auto& symbol = names.front(); - symbol.Name = PushNamedAtom(symbol.Pos, symbol.Name); - nodeExpr->SetLabel(symbol.Name); - localNames.push_back(symbol.Name); + auto& symbol = names.front(); + symbol.Name = PushNamedAtom(symbol.Pos, symbol.Name); + nodeExpr->SetLabel(symbol.Name); + localNames.push_back(symbol.Name); exprSeq.push_back(nodeExpr); } break; } case TRule_lambda_stmt::kAltLambdaStmt2: { - if (!ImportStatement(rule.GetAlt_lambda_stmt2().GetRule_import_stmt1(), &localNames)) { + if (!ImportStatement(rule.GetAlt_lambda_stmt2().GetRule_import_stmt1(), &localNames)) { hasError = true; } break; @@ -4863,7 +4863,7 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n TNodePtr nodeExpr; if (!hasError) { - nodeExpr = expr.Build(node.GetRule_expr4()); + nodeExpr = expr.Build(node.GetRule_expr4()); } for (const auto& name : localNames) { @@ -4877,11 +4877,11 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n return true; } -TNodePtr TSqlExpression::SubExpr(const TRule_con_subexpr& node, const TTrailingQuestions& tail) { +TNodePtr TSqlExpression::SubExpr(const TRule_con_subexpr& node, const TTrailingQuestions& tail) { // con_subexpr: unary_subexpr | unary_op unary_subexpr; switch (node.Alt_case()) { case TRule_con_subexpr::kAltConSubexpr1: - return UnaryExpr(node.GetAlt_con_subexpr1().GetRule_unary_subexpr1(), tail); + return UnaryExpr(node.GetAlt_con_subexpr1().GetRule_unary_subexpr1(), tail); case TRule_con_subexpr::kAltConSubexpr2: { Ctx.IncrementMonCounter("sql_features", "UnaryOperation"); TString opName; @@ -4899,8 +4899,8 @@ TNodePtr TSqlExpression::SubExpr(const TRule_con_subexpr& node, const TTrailingQ return nullptr; } Ctx.IncrementMonCounter("sql_unary_operations", opName); - auto expr = UnaryExpr(node.GetAlt_con_subexpr2().GetRule_unary_subexpr2(), tail); - return expr ? expr->ApplyUnaryOp(Ctx, pos, opName) : expr; + auto expr = UnaryExpr(node.GetAlt_con_subexpr2().GetRule_unary_subexpr2(), tail); + return expr ? expr->ApplyUnaryOp(Ctx, pos, opName) : expr; } default: Y_FAIL("You should change implementation according to grammar changes"); @@ -4908,9 +4908,9 @@ TNodePtr TSqlExpression::SubExpr(const TRule_con_subexpr& node, const TTrailingQ return nullptr; } -TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQuestions& tail) { +TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQuestions& tail) { // xor_subexpr: eq_subexpr cond_expr?; - TNodePtr res(SubExpr(node.GetRule_eq_subexpr1(), node.HasBlock2() ? TTrailingQuestions{} : tail)); + TNodePtr res(SubExpr(node.GetRule_eq_subexpr1(), node.HasBlock2() ? TTrailingQuestions{} : tail)); if (!res) { return {}; } @@ -4922,7 +4922,7 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ const auto& matchOp = cond.GetAlt_cond_expr1(); const bool notMatch = matchOp.HasBlock1(); const TCiString& opName = Token(matchOp.GetRule_match_op2().GetToken1()); - const auto& pattern = SubExpr(cond.GetAlt_cond_expr1().GetRule_eq_subexpr3(), matchOp.HasBlock4() ? TTrailingQuestions{} : tail); + const auto& pattern = SubExpr(cond.GetAlt_cond_expr1().GetRule_eq_subexpr3(), matchOp.HasBlock4() ? TTrailingQuestions{} : tail); if (!pattern) { return {}; } @@ -4935,7 +4935,7 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ if (matchOp.HasBlock4()) { const auto& escapeBlock = matchOp.GetBlock4(); - TNodePtr escapeExpr = SubExpr(escapeBlock.GetRule_eq_subexpr2(), tail); + TNodePtr escapeExpr = SubExpr(escapeBlock.GetRule_eq_subexpr2(), tail); if (!escapeExpr) { return {}; } @@ -4954,12 +4954,12 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ Error() << "please choose any other character"; return nullptr; } - if (!IsAscii(escapeLiteral->front())) { - Ctx.IncrementMonCounter("sql_errors", "LikeUnsupportedEscapeChar"); - Error() << "Non-ASCII symbols are not supported in ESCAPE clause, "; - Error() << "please choose ASCII character"; - return nullptr; - } + if (!IsAscii(escapeLiteral->front())) { + Ctx.IncrementMonCounter("sql_errors", "LikeUnsupportedEscapeChar"); + Error() << "Non-ASCII symbols are not supported in ESCAPE clause, "; + Error() << "please choose ASCII character"; + return nullptr; + } escaperArgs.push_back(BuildLiteralRawString(pos, *escapeLiteral)); } else { Ctx.IncrementMonCounter("sql_errors", "LikeNotLiteralEscape"); @@ -4972,7 +4972,7 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ if (opName == "ilike") { Ctx.IncrementMonCounter("sql_features", "CaseInsensitiveLike"); } - auto csModeLiteral = BuildLiteralBool(pos, opName != "ilike"); + auto csModeLiteral = BuildLiteralBool(pos, opName != "ilike"); csModeLiteral->SetLabel("CaseSensitive"); auto csOption = BuildStructure(pos, { csModeLiteral }); auto optionsApply = new TCallNodeImpl(pos, "NamedApply", { re2options, BuildTuple(pos, {}), csOption }); @@ -4984,36 +4984,36 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ const auto& matcher = BuildUdf(Ctx, pos, "Re2", "Match", { runConfig }); isMatch = new TCallNodeImpl(pos, "Apply", { matcher, res }); - bool isUtf8 = false; - const TString* literalPattern = pattern->GetLiteral("String"); - if (!literalPattern) { - literalPattern = pattern->GetLiteral("Utf8"); - isUtf8 = literalPattern != nullptr; - } - - if (literalPattern) { - TString prefix, suffix; + bool isUtf8 = false; + const TString* literalPattern = pattern->GetLiteral("String"); + if (!literalPattern) { + literalPattern = pattern->GetLiteral("Utf8"); + isUtf8 = literalPattern != nullptr; + } + + if (literalPattern) { + TString prefix, suffix; bool inEscape = false; bool hasPattern = false; bool isSimple = true; - - TMaybe<char> escape; - if (escapeLiteral) { - escape = escapeLiteral->front(); + + TMaybe<char> escape; + if (escapeLiteral) { + escape = escapeLiteral->front(); } - bool mayIgnoreCase; - if (isUtf8) { - auto splitResult = SplitPattern(UTF8ToUTF32<false>(*literalPattern), escape, inEscape, hasPattern, isSimple); - prefix = WideToUTF8(splitResult.Prefix); - suffix = WideToUTF8(splitResult.Suffix); - mayIgnoreCase = ToLowerUTF8(*literalPattern) == ToUpperUTF8(*literalPattern); - } else { - auto splitResult = SplitPattern(*literalPattern, escape, inEscape, hasPattern, isSimple); - prefix = splitResult.Prefix; - suffix = splitResult.Suffix; - mayIgnoreCase = WithoutAlpha(*literalPattern); - } + bool mayIgnoreCase; + if (isUtf8) { + auto splitResult = SplitPattern(UTF8ToUTF32<false>(*literalPattern), escape, inEscape, hasPattern, isSimple); + prefix = WideToUTF8(splitResult.Prefix); + suffix = WideToUTF8(splitResult.Suffix); + mayIgnoreCase = ToLowerUTF8(*literalPattern) == ToUpperUTF8(*literalPattern); + } else { + auto splitResult = SplitPattern(*literalPattern, escape, inEscape, hasPattern, isSimple); + prefix = splitResult.Prefix; + suffix = splitResult.Suffix; + mayIgnoreCase = WithoutAlpha(*literalPattern); + } if (inEscape) { Ctx.IncrementMonCounter("sql_errors", "LikeEscapeSymbolEnd"); @@ -5021,13 +5021,13 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ return nullptr; } - if (opName == "like" || mayIgnoreCase) { + if (opName == "like" || mayIgnoreCase) { //TODO: Drop regex if (isSimple) {} if (!(hasPattern || suffix.empty())) { - isMatch = BuildBinaryOp(Ctx, pos, "==", res, BuildLiteralRawString(pos, suffix, isUtf8)); + isMatch = BuildBinaryOp(Ctx, pos, "==", res, BuildLiteralRawString(pos, suffix, isUtf8)); } else if (!prefix.empty()) { - const auto& lowerBoundOp = BuildBinaryOp(Ctx, pos, "StartsWith", res, BuildLiteralRawString(pos, prefix, isUtf8)); + const auto& lowerBoundOp = BuildBinaryOp(Ctx, pos, "StartsWith", res, BuildLiteralRawString(pos, prefix, isUtf8)); isMatch = BuildBinaryOp(Ctx, pos, "And", lowerBoundOp, isMatch); } } @@ -5062,70 +5062,70 @@ TNodePtr TSqlExpression::SubExpr(const TRule_xor_subexpr& node, const TTrailingQ AltNotImplemented("match_op", cond); return nullptr; } - return (notMatch && isMatch) ? isMatch->ApplyUnaryOp(Ctx, pos, "Not") : isMatch; + return (notMatch && isMatch) ? isMatch->ApplyUnaryOp(Ctx, pos, "Not") : isMatch; } case TRule_cond_expr::kAltCondExpr2: { auto altInExpr = cond.GetAlt_cond_expr2(); const bool notIn = altInExpr.HasBlock1(); - auto hints = BuildTuple(pos, {}); - if (altInExpr.HasBlock3()) { - Ctx.IncrementMonCounter("sql_features", "IsCompactHint"); - auto sizeHint = BuildTuple(pos, { BuildQuotedAtom(pos, "isCompact", NYql::TNodeFlags::Default) }); - hints = BuildTuple(pos, { sizeHint }); - } + auto hints = BuildTuple(pos, {}); + if (altInExpr.HasBlock3()) { + Ctx.IncrementMonCounter("sql_features", "IsCompactHint"); + auto sizeHint = BuildTuple(pos, { BuildQuotedAtom(pos, "isCompact", NYql::TNodeFlags::Default) }); + hints = BuildTuple(pos, { sizeHint }); + } TSqlExpression inSubexpr(Ctx, Mode); - auto inRight = inSubexpr.SqlInExpr(altInExpr.GetRule_in_expr4(), tail); - auto isIn = BuildBuiltinFunc(Ctx, pos, "In", {res, inRight, hints}); + auto inRight = inSubexpr.SqlInExpr(altInExpr.GetRule_in_expr4(), tail); + auto isIn = BuildBuiltinFunc(Ctx, pos, "In", {res, inRight, hints}); Ctx.IncrementMonCounter("sql_features", notIn ? "NotIn" : "In"); - return (notIn && isIn) ? isIn->ApplyUnaryOp(Ctx, pos, "Not") : isIn; + return (notIn && isIn) ? isIn->ApplyUnaryOp(Ctx, pos, "Not") : isIn; } case TRule_cond_expr::kAltCondExpr3: { - if (tail.Count) { - UnexpectedQuestionToken(tail); - return {}; - } + if (tail.Count) { + UnexpectedQuestionToken(tail); + return {}; + } auto altCase = cond.GetAlt_cond_expr3().GetBlock1().Alt_case(); const bool notNoll = altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt2 || altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 ; - - if (altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 && - !cond.GetAlt_cond_expr3().GetBlock1().GetAlt4().HasBlock1()) - { - Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_MISSING_IS_BEFORE_NOT_NULL) << "Missing IS keyword before NOT NULL"; - } - + + if (altCase == TRule_cond_expr::TAlt3::TBlock1::kAlt4 && + !cond.GetAlt_cond_expr3().GetBlock1().GetAlt4().HasBlock1()) + { + Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_MISSING_IS_BEFORE_NOT_NULL) << "Missing IS keyword before NOT NULL"; + } + auto isNull = BuildIsNullOp(pos, res); Ctx.IncrementMonCounter("sql_features", notNoll ? "NotNull" : "Null"); - return (notNoll && isNull) ? isNull->ApplyUnaryOp(Ctx, pos, "Not") : isNull; + return (notNoll && isNull) ? isNull->ApplyUnaryOp(Ctx, pos, "Not") : isNull; } case TRule_cond_expr::kAltCondExpr4: { - auto alt = cond.GetAlt_cond_expr4(); + auto alt = cond.GetAlt_cond_expr4(); if (alt.HasBlock1()) { Ctx.IncrementMonCounter("sql_features", "NotBetween"); return BuildBinaryOp( - Ctx, + Ctx, pos, "Or", - BuildBinaryOp(Ctx, pos, "<", res, SubExpr(alt.GetRule_eq_subexpr3(), {})), - BuildBinaryOp(Ctx, pos, ">", res, SubExpr(alt.GetRule_eq_subexpr5(), tail)) + BuildBinaryOp(Ctx, pos, "<", res, SubExpr(alt.GetRule_eq_subexpr3(), {})), + BuildBinaryOp(Ctx, pos, ">", res, SubExpr(alt.GetRule_eq_subexpr5(), tail)) ); } else { Ctx.IncrementMonCounter("sql_features", "Between"); return BuildBinaryOp( - Ctx, + Ctx, pos, "And", - BuildBinaryOp(Ctx, pos, ">=", res, SubExpr(alt.GetRule_eq_subexpr3(), {})), - BuildBinaryOp(Ctx, pos, "<=", res, SubExpr(alt.GetRule_eq_subexpr5(), tail)) + BuildBinaryOp(Ctx, pos, ">=", res, SubExpr(alt.GetRule_eq_subexpr3(), {})), + BuildBinaryOp(Ctx, pos, "<=", res, SubExpr(alt.GetRule_eq_subexpr5(), tail)) ); } } - case TRule_cond_expr::kAltCondExpr5: { - auto alt = cond.GetAlt_cond_expr5(); - auto getNode = [](const TRule_cond_expr::TAlt5::TBlock1& b) -> const TRule_eq_subexpr& { return b.GetRule_eq_subexpr2(); }; - return BinOpList(node.GetRule_eq_subexpr1(), getNode, alt.GetBlock1().begin(), alt.GetBlock1().end(), tail); + case TRule_cond_expr::kAltCondExpr5: { + auto alt = cond.GetAlt_cond_expr5(); + auto getNode = [](const TRule_cond_expr::TAlt5::TBlock1& b) -> const TRule_eq_subexpr& { return b.GetRule_eq_subexpr2(); }; + return BinOpList(node.GetRule_eq_subexpr1(), getNode, alt.GetBlock1().begin(), alt.GetBlock1().end(), tail); } default: Ctx.IncrementMonCounter("sql_errors", "UnknownConditionExpr"); @@ -5141,57 +5141,57 @@ TNodePtr TSqlExpression::BinOperList(const TString& opName, TVector<TNodePtr>::c const size_t opCount = end - begin; Y_VERIFY_DEBUG(opCount >= 2); if (opCount == 2) { - return BuildBinaryOp(Ctx, pos, opName, *begin, *(begin+1)); + return BuildBinaryOp(Ctx, pos, opName, *begin, *(begin+1)); } if (opCount == 3) { - return BuildBinaryOp(Ctx, pos, opName, BuildBinaryOp(Ctx, pos, opName, *begin, *(begin+1)), *(begin+2)); + return BuildBinaryOp(Ctx, pos, opName, BuildBinaryOp(Ctx, pos, opName, *begin, *(begin+1)), *(begin+2)); } else { auto mid = begin + opCount / 2; - return BuildBinaryOp(Ctx, pos, opName, BinOperList(opName, begin, mid), BinOperList(opName, mid, end)); - } -} - -TSqlExpression::TCaseBranch TSqlExpression::ReduceCaseBranches(TVector<TCaseBranch>::const_iterator begin, TVector<TCaseBranch>::const_iterator end) const { - YQL_ENSURE(begin < end); - const size_t branchCount = end - begin; - if (branchCount == 1) { - return *begin; - } - - auto mid = begin + branchCount / 2; - auto left = ReduceCaseBranches(begin, mid); - auto right = ReduceCaseBranches(mid, end); - - TVector<TNodePtr> preds; - preds.reserve(branchCount); - for (auto it = begin; it != end; ++it) { - preds.push_back(it->Pred); - } - - TCaseBranch result; - result.Pred = new TCallNodeImpl(Ctx.Pos(), "Or", preds); - result.Value = BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", { left.Pred, left.Value, right.Value }); - return result; -} - + return BuildBinaryOp(Ctx, pos, opName, BinOperList(opName, begin, mid), BinOperList(opName, mid, end)); + } +} + +TSqlExpression::TCaseBranch TSqlExpression::ReduceCaseBranches(TVector<TCaseBranch>::const_iterator begin, TVector<TCaseBranch>::const_iterator end) const { + YQL_ENSURE(begin < end); + const size_t branchCount = end - begin; + if (branchCount == 1) { + return *begin; + } + + auto mid = begin + branchCount / 2; + auto left = ReduceCaseBranches(begin, mid); + auto right = ReduceCaseBranches(mid, end); + + TVector<TNodePtr> preds; + preds.reserve(branchCount); + for (auto it = begin; it != end; ++it) { + preds.push_back(it->Pred); + } + + TCaseBranch result; + result.Pred = new TCallNodeImpl(Ctx.Pos(), "Or", preds); + result.Value = BuildBuiltinFunc(Ctx, Ctx.Pos(), "If", { left.Pred, left.Value, right.Value }); + return result; +} + template <typename TNode, typename TGetNode, typename TIter> -TNodePtr TSqlExpression::BinOper(const TString& opName, const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { +TNodePtr TSqlExpression::BinOper(const TString& opName, const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { if (begin == end) { - return SubExpr(node, tail); + return SubExpr(node, tail); } Ctx.IncrementMonCounter("sql_binary_operations", opName); const size_t listSize = end - begin; TVector<TNodePtr> nodes; nodes.reserve(1 + listSize); - nodes.push_back(SubExpr(node, {})); + nodes.push_back(SubExpr(node, {})); for (; begin != end; ++begin) { - nodes.push_back(SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); + nodes.push_back(SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); } return BinOperList(opName, nodes.begin(), nodes.end()); } template <typename TNode, typename TGetNode, typename TIter> -TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { - TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); +TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { + TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); while (begin != end) { Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); Token(begin->GetToken1()); @@ -5230,7 +5230,7 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be case SQLv1LexerTokens::TOKEN_SLASH: opName = "/"; Ctx.IncrementMonCounter("sql_binary_operations", "Divide"); - if (!Ctx.Scoped->PragmaClassicDivision && partialResult) { + if (!Ctx.Scoped->PragmaClassicDivision && partialResult) { partialResult = new TCallNodeImpl(pos, "SafeCast", {std::move(partialResult), BuildDataType(pos, "Double")}); } break; @@ -5244,7 +5244,7 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be return nullptr; } - partialResult = BuildBinaryOp(Ctx, pos, opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); + partialResult = BuildBinaryOp(Ctx, pos, opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); ++begin; } @@ -5252,8 +5252,8 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be } template <typename TGetNode, typename TIter> -TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { - TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); +TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { + TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); while (begin != end) { Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); TString opName; @@ -5327,85 +5327,85 @@ TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNo Y_FAIL("You should change implementation according to grammar changes"); } - partialResult = BuildBinaryOp(Ctx, Ctx.Pos(), opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); + partialResult = BuildBinaryOp(Ctx, Ctx.Pos(), opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); + ++begin; + } + + return partialResult; +} + +template <typename TGetNode, typename TIter> +TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { + TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); + while (begin != end) { + Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); + TString opName; + switch (begin->GetBlock1().Alt_case()) { + case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt1: { + Token(begin->GetBlock1().GetAlt1().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_EQUALS) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + Ctx.IncrementMonCounter("sql_binary_operations", "Equals"); + opName = "=="; + break; + } + case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt2: { + Token(begin->GetBlock1().GetAlt2().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt2().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_EQUALS2) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + Ctx.IncrementMonCounter("sql_binary_operations", "Equals2"); + opName = "=="; + break; + } + case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt3: { + Token(begin->GetBlock1().GetAlt3().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_NOT_EQUALS) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals"); + opName = "!="; + break; + } + case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt4: { + Token(begin->GetBlock1().GetAlt4().GetToken1()); + auto tokenId = begin->GetBlock1().GetAlt4().GetToken1().GetId(); + if (tokenId != SQLv1LexerTokens::TOKEN_NOT_EQUALS2) { + Error() << "Unsupported binary operation token: " << tokenId; + return {}; + } + Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals2"); + opName = "!="; + break; + } + case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt5: { + Token(begin->GetBlock1().GetAlt5().GetRule_distinct_from_op1().GetToken1()); + opName = begin->GetBlock1().GetAlt5().GetRule_distinct_from_op1().HasBlock2() ? "IsNotDistinctFrom" : "IsDistinctFrom"; + Ctx.IncrementMonCounter("sql_binary_operations", opName); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + partialResult = BuildBinaryOp(Ctx, Ctx.Pos(), opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); ++begin; } return partialResult; } -template <typename TGetNode, typename TIter> -TNodePtr TSqlExpression::BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) { - TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{}); - while (begin != end) { - Ctx.IncrementMonCounter("sql_features", "BinaryOperation"); - TString opName; - switch (begin->GetBlock1().Alt_case()) { - case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt1: { - Token(begin->GetBlock1().GetAlt1().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_EQUALS) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - Ctx.IncrementMonCounter("sql_binary_operations", "Equals"); - opName = "=="; - break; - } - case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt2: { - Token(begin->GetBlock1().GetAlt2().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt2().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_EQUALS2) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - Ctx.IncrementMonCounter("sql_binary_operations", "Equals2"); - opName = "=="; - break; - } - case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt3: { - Token(begin->GetBlock1().GetAlt3().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_NOT_EQUALS) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals"); - opName = "!="; - break; - } - case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt4: { - Token(begin->GetBlock1().GetAlt4().GetToken1()); - auto tokenId = begin->GetBlock1().GetAlt4().GetToken1().GetId(); - if (tokenId != SQLv1LexerTokens::TOKEN_NOT_EQUALS2) { - Error() << "Unsupported binary operation token: " << tokenId; - return {}; - } - Ctx.IncrementMonCounter("sql_binary_operations", "NotEquals2"); - opName = "!="; - break; - } - case TRule_cond_expr::TAlt5::TBlock1::TBlock1::kAlt5: { - Token(begin->GetBlock1().GetAlt5().GetRule_distinct_from_op1().GetToken1()); - opName = begin->GetBlock1().GetAlt5().GetRule_distinct_from_op1().HasBlock2() ? "IsNotDistinctFrom" : "IsDistinctFrom"; - Ctx.IncrementMonCounter("sql_binary_operations", opName); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - partialResult = BuildBinaryOp(Ctx, Ctx.Pos(), opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{})); - ++begin; - } - - return partialResult; -} - -TNodePtr TSqlExpression::SqlInExpr(const TRule_in_expr& node, const TTrailingQuestions& tail) { +TNodePtr TSqlExpression::SqlInExpr(const TRule_in_expr& node, const TTrailingQuestions& tail) { TSqlExpression expr(Ctx, Mode); expr.SetSmartParenthesisMode(TSqlExpression::ESmartParenthesis::InStatement); - auto result = expr.UnaryExpr(node.GetRule_in_unary_subexpr1(), tail); + auto result = expr.UnaryExpr(node.GetRule_in_unary_subexpr1(), tail); return result; } @@ -5416,11 +5416,11 @@ TNodePtr TSqlExpression::SmartParenthesis(const TRule_smart_parenthesis& node) { const bool isTuple = node.HasBlock3(); bool expectTuple = SmartParenthesisMode == ESmartParenthesis::InStatement; EExpr mode = EExpr::Regular; - if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { + if (SmartParenthesisMode == ESmartParenthesis::SqlLambdaParams) { mode = EExpr::SqlLambdaParams; expectTuple = true; } - if (node.HasBlock2() && !NamedExprList(node.GetBlock2().GetRule_named_expr_list1(), exprs, mode)) { + if (node.HasBlock2() && !NamedExprList(node.GetBlock2().GetRule_named_expr_list1(), exprs, mode)) { return {}; } @@ -5455,17 +5455,17 @@ TNodePtr TSqlExpression::SmartParenthesis(const TRule_smart_parenthesis& node) { return (hasUnnamed || expectTuple || exprs.size() == 0) ? BuildTuple(pos, exprs) : BuildStructure(pos, exprs); } -TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector<TSymbolNameWithPos>& names) { +TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector<TSymbolNameWithPos>& names) { // named_nodes_stmt: bind_parameter_list EQUALS (expr | subselect_stmt); // subselect_stmt: (LPAREN select_stmt RPAREN | select_unparenthesized_stmt); - if (!BindList(rule.GetRule_bind_parameter_list1(), names)) { - return {}; - } + if (!BindList(rule.GetRule_bind_parameter_list1(), names)) { + return {}; + } TNodePtr nodeExpr = nullptr; switch (rule.GetBlock3().Alt_case()) { case TRule_named_nodes_stmt::TBlock3::kAlt1: { - TSqlExpression expr(Ctx, Mode); + TSqlExpression expr(Ctx, Mode); auto result = expr.Build(rule.GetBlock3().GetAlt1().GetRule_expr1()); return result; } @@ -5505,57 +5505,57 @@ TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector< } } -bool TSqlTranslation::ImportStatement(const TRule_import_stmt& stmt, TVector<TString>* namesPtr) { +bool TSqlTranslation::ImportStatement(const TRule_import_stmt& stmt, TVector<TString>* namesPtr) { TVector<TString> modulePath; if (!ModulePath(stmt.GetRule_module_path2(), modulePath)) { return false; } - TVector<TSymbolNameWithPos> names; - TVector<TSymbolNameWithPos> aliases; - if (!NamedBindList(stmt.GetRule_named_bind_parameter_list4(), names, aliases)) { + TVector<TSymbolNameWithPos> names; + TVector<TSymbolNameWithPos> aliases; + if (!NamedBindList(stmt.GetRule_named_bind_parameter_list4(), names, aliases)) { return false; } - YQL_ENSURE(names.size() == aliases.size()); + YQL_ENSURE(names.size() == aliases.size()); const TString moduleAlias = Ctx.AddImport(std::move(modulePath)); if (!moduleAlias) { return false; } - - for (size_t i = 0; i < names.size(); ++i) { - auto& name = names[i]; - auto& alias = aliases[i]; - - auto& var = alias.Name ? alias : name; - if (IsAnonymousName(var.Name)) { - Ctx.Error(var.Pos) << "Can not import anonymous name " << var.Name; - return false; - } - - auto builder = [&](const TString& realName) { - YQL_ENSURE(realName == var.Name); - auto atom = BuildQuotedAtom(name.Pos, name.Name); - return atom->Y("bind", moduleAlias, atom); - }; - - var.Name = PushNamedNode(var.Pos, var.Name, builder); + + for (size_t i = 0; i < names.size(); ++i) { + auto& name = names[i]; + auto& alias = aliases[i]; + + auto& var = alias.Name ? alias : name; + if (IsAnonymousName(var.Name)) { + Ctx.Error(var.Pos) << "Can not import anonymous name " << var.Name; + return false; + } + + auto builder = [&](const TString& realName) { + YQL_ENSURE(realName == var.Name); + auto atom = BuildQuotedAtom(name.Pos, name.Name); + return atom->Y("bind", moduleAlias, atom); + }; + + var.Name = PushNamedNode(var.Pos, var.Name, builder); if (namesPtr) { - namesPtr->push_back(var.Name); + namesPtr->push_back(var.Name); } } return true; } -bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, TMaybe<TPosition> anyPos) { - // block: (join_op (ANY)? flatten_source join_constraint?) - // join_op: - // COMMA - // | (NATURAL)? ((LEFT (ONLY | SEMI)? | RIGHT (ONLY | SEMI)? | EXCLUSION | FULL)? (OUTER)? | INNER | CROSS) SORTED? JOIN - //; +bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, TMaybe<TPosition> anyPos) { + // block: (join_op (ANY)? flatten_source join_constraint?) + // join_op: + // COMMA + // | (NATURAL)? ((LEFT (ONLY | SEMI)? | RIGHT (ONLY | SEMI)? | EXCLUSION | FULL)? (OUTER)? | INNER | CROSS) SORTED? JOIN + //; const auto& node = block.GetRule_join_op1(); switch (node.Alt_case()) { case TRule_join_op::kAltJoinOp1: Ctx.IncrementMonCounter("sql_join_operations", "CartesianProduct"); - Error() << "Cartesian product of tables is not supported. Please use explicit CROSS JOIN"; + Error() << "Cartesian product of tables is not supported. Please use explicit CROSS JOIN"; return false; case TRule_join_op::kAltJoinOp2: { auto alt = node.GetAlt_join_op2(); @@ -5565,8 +5565,8 @@ bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, return false; } TString joinOp("Inner"); - TJoinLinkSettings linkSettings; - linkSettings.ForceSortedMerge = alt.HasBlock3(); + TJoinLinkSettings linkSettings; + linkSettings.ForceSortedMerge = alt.HasBlock3(); switch (alt.GetBlock2().Alt_case()) { case TRule_join_op::TAlt2::TBlock2::kAlt1: if (alt.GetBlock2().GetAlt1().HasBlock1()) { @@ -5601,17 +5601,17 @@ bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, return false; } } - if (alt.GetBlock2().GetAlt1().HasBlock2()) { - TString normalizedOp = alt.GetBlock2().GetAlt1().HasBlock1() ? joinOp : ""; - normalizedOp.to_upper(); - if (!(normalizedOp == "LEFT" || normalizedOp == "RIGHT" || normalizedOp == "FULL")) { - Token(alt.GetBlock2().GetAlt1().GetBlock2().GetToken1()); - Error() << "Invalid join type: " << normalizedOp << (normalizedOp.empty() ? "" : " ") << "OUTER JOIN. " - << "OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL"; - Ctx.IncrementMonCounter("sql_errors", "BadJoinType"); - return false; - } - } + if (alt.GetBlock2().GetAlt1().HasBlock2()) { + TString normalizedOp = alt.GetBlock2().GetAlt1().HasBlock1() ? joinOp : ""; + normalizedOp.to_upper(); + if (!(normalizedOp == "LEFT" || normalizedOp == "RIGHT" || normalizedOp == "FULL")) { + Token(alt.GetBlock2().GetAlt1().GetBlock2().GetToken1()); + Error() << "Invalid join type: " << normalizedOp << (normalizedOp.empty() ? "" : " ") << "OUTER JOIN. " + << "OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL"; + Ctx.IncrementMonCounter("sql_errors", "BadJoinType"); + return false; + } + } break; case TRule_join_op::TAlt2::TBlock2::kAlt2: joinOp = Token(alt.GetBlock2().GetAlt2().GetToken1()); @@ -5630,14 +5630,14 @@ bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, Ctx.IncrementMonCounter("sql_join_operations", joinOp); TNodePtr joinKeyExpr; - if (block.HasBlock4()) { + if (block.HasBlock4()) { if (joinOp == "Cross") { Error() << "Cross join should not have ON or USING expression"; Ctx.IncrementMonCounter("sql_errors", "BadJoinExpr"); return false; } - joinKeyExpr = JoinExpr(join, block.GetBlock4().GetRule_join_constraint1()); + joinKeyExpr = JoinExpr(join, block.GetBlock4().GetRule_join_constraint1()); if (!joinKeyExpr) { Ctx.IncrementMonCounter("sql_errors", "BadJoinExpr"); return false; @@ -5651,14 +5651,14 @@ bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block, } } - if (joinOp == "Cross" && anyPos) { - Ctx.Error(*anyPos) << "ANY should not be used with Cross JOIN"; - Ctx.IncrementMonCounter("sql_errors", "BadJoinAny"); - return false; - } - + if (joinOp == "Cross" && anyPos) { + Ctx.Error(*anyPos) << "ANY should not be used with Cross JOIN"; + Ctx.IncrementMonCounter("sql_errors", "BadJoinAny"); + return false; + } + Y_VERIFY_DEBUG(join->GetJoin()); - join->GetJoin()->SetupJoin(joinOp, joinKeyExpr, linkSettings); + join->GetJoin()->SetupJoin(joinOp, joinKeyExpr, linkSettings); break; } default: @@ -5674,7 +5674,7 @@ TNodePtr TSqlSelect::JoinExpr(ISource* join, const TRule_join_constraint& node) case TRule_join_constraint::kAltJoinConstraint1: { auto& alt = node.GetAlt_join_constraint1(); Token(alt.GetToken1()); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression expr(Ctx, Mode); return expr.Build(alt.GetRule_expr2()); } @@ -5698,91 +5698,91 @@ TNodePtr TSqlSelect::JoinExpr(ISource* join, const TRule_join_constraint& node) return nullptr; } -bool TSqlSelect::FlattenByArg(const TString& sourceLabel, TVector<TNodePtr>& flattenByColumns, TVector<TNodePtr>& flattenByExprs, - const TRule_flatten_by_arg& node) -{ - // flatten_by_arg: - // named_column - // | LPAREN named_expr_list COMMA? RPAREN - // ; - - flattenByColumns.clear(); - flattenByExprs.clear(); - - TVector<TNodePtr> namedExprs; +bool TSqlSelect::FlattenByArg(const TString& sourceLabel, TVector<TNodePtr>& flattenByColumns, TVector<TNodePtr>& flattenByExprs, + const TRule_flatten_by_arg& node) +{ + // flatten_by_arg: + // named_column + // | LPAREN named_expr_list COMMA? RPAREN + // ; + + flattenByColumns.clear(); + flattenByExprs.clear(); + + TVector<TNodePtr> namedExprs; switch (node.Alt_case()) { - case TRule_flatten_by_arg::kAltFlattenByArg1: { - TVector<TNodePtr> columns; - if (!NamedColumn(columns, node.GetAlt_flatten_by_arg1().GetRule_named_column1())) { - return false; - } - YQL_ENSURE(columns.size() == 1); - auto& column = columns.back(); - auto columnNamePtr = column->GetColumnName(); - YQL_ENSURE(columnNamePtr && *columnNamePtr); - - auto sourcePtr = column->GetSourceName(); - const bool isEmptySource = !sourcePtr || !*sourcePtr; - if (isEmptySource || *sourcePtr == sourceLabel) { - // select * from T flatten by x - // select * from T as s flatten by x - // select * from T as s flatten by s.x - flattenByColumns.emplace_back(std::move(column)); - } else { - // select * from T as s flatten by x.y as z - if (!column->GetLabel()) { - Ctx.Error(column->GetPos()) << "Unnamed expression after FLATTEN BY is not allowed"; - return false; - } - flattenByColumns.emplace_back(BuildColumn(column->GetPos(), column->GetLabel())); - - TVector<INode::TIdPart> ids; - ids.push_back(BuildColumn(column->GetPos())); - ids.push_back(*sourcePtr); - ids.push_back(*columnNamePtr); - auto node = BuildAccess(column->GetPos(), ids, false); - node->SetLabel(column->GetLabel()); - flattenByExprs.emplace_back(std::move(node)); - } - - break; - } - case TRule_flatten_by_arg::kAltFlattenByArg2: { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - if (!NamedExprList(node.GetAlt_flatten_by_arg2().GetRule_named_expr_list2(), namedExprs) || Ctx.HasPendingErrors) { - return false; - } - for (auto& namedExprNode : namedExprs) { - YQL_ENSURE(!namedExprNode->ContentListPtr()); - - auto sourcePtr = namedExprNode->GetSourceName(); - const bool isEmptySource = !sourcePtr || !*sourcePtr; - auto columnNamePtr = namedExprNode->GetColumnName(); - if (columnNamePtr && (isEmptySource || *sourcePtr == sourceLabel)) { - namedExprNode->AssumeColumn(); - flattenByColumns.emplace_back(std::move(namedExprNode)); - } else { - auto nodeLabel = namedExprNode->GetLabel(); - if (!nodeLabel) { - Ctx.Error(namedExprNode->GetPos()) << "Unnamed expression after FLATTEN BY is not allowed"; - return false; - } - flattenByColumns.emplace_back(BuildColumn(namedExprNode->GetPos(), nodeLabel)); - flattenByExprs.emplace_back(std::move(namedExprNode)); - } - } - break; - } + case TRule_flatten_by_arg::kAltFlattenByArg1: { + TVector<TNodePtr> columns; + if (!NamedColumn(columns, node.GetAlt_flatten_by_arg1().GetRule_named_column1())) { + return false; + } + YQL_ENSURE(columns.size() == 1); + auto& column = columns.back(); + auto columnNamePtr = column->GetColumnName(); + YQL_ENSURE(columnNamePtr && *columnNamePtr); + + auto sourcePtr = column->GetSourceName(); + const bool isEmptySource = !sourcePtr || !*sourcePtr; + if (isEmptySource || *sourcePtr == sourceLabel) { + // select * from T flatten by x + // select * from T as s flatten by x + // select * from T as s flatten by s.x + flattenByColumns.emplace_back(std::move(column)); + } else { + // select * from T as s flatten by x.y as z + if (!column->GetLabel()) { + Ctx.Error(column->GetPos()) << "Unnamed expression after FLATTEN BY is not allowed"; + return false; + } + flattenByColumns.emplace_back(BuildColumn(column->GetPos(), column->GetLabel())); + + TVector<INode::TIdPart> ids; + ids.push_back(BuildColumn(column->GetPos())); + ids.push_back(*sourcePtr); + ids.push_back(*columnNamePtr); + auto node = BuildAccess(column->GetPos(), ids, false); + node->SetLabel(column->GetLabel()); + flattenByExprs.emplace_back(std::move(node)); + } + + break; + } + case TRule_flatten_by_arg::kAltFlattenByArg2: { + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + if (!NamedExprList(node.GetAlt_flatten_by_arg2().GetRule_named_expr_list2(), namedExprs) || Ctx.HasPendingErrors) { + return false; + } + for (auto& namedExprNode : namedExprs) { + YQL_ENSURE(!namedExprNode->ContentListPtr()); + + auto sourcePtr = namedExprNode->GetSourceName(); + const bool isEmptySource = !sourcePtr || !*sourcePtr; + auto columnNamePtr = namedExprNode->GetColumnName(); + if (columnNamePtr && (isEmptySource || *sourcePtr == sourceLabel)) { + namedExprNode->AssumeColumn(); + flattenByColumns.emplace_back(std::move(namedExprNode)); + } else { + auto nodeLabel = namedExprNode->GetLabel(); + if (!nodeLabel) { + Ctx.Error(namedExprNode->GetPos()) << "Unnamed expression after FLATTEN BY is not allowed"; + return false; + } + flattenByColumns.emplace_back(BuildColumn(namedExprNode->GetPos(), nodeLabel)); + flattenByExprs.emplace_back(std::move(namedExprNode)); + } + } + break; + } default: - Ctx.IncrementMonCounter("sql_errors", "UnknownFlattenByArg"); - AltNotImplemented("flatten_by_arg", node); - return false; + Ctx.IncrementMonCounter("sql_errors", "UnknownFlattenByArg"); + AltNotImplemented("flatten_by_arg", node); + return false; } - return true; + return true; } TSourcePtr TSqlSelect::FlattenSource(const TRule_flatten_source& node) { - auto source = NamedSingleSource(node.GetRule_named_single_source1(), true); + auto source = NamedSingleSource(node.GetRule_named_single_source1(), true); if (!source) { return nullptr; } @@ -5796,21 +5796,21 @@ TSourcePtr TSqlSelect::FlattenSource(const TRule_flatten_source& node) { mode = to_lower(Token(flatten2.GetAlt1().GetBlock1().GetToken1())); } - TVector<TNodePtr> flattenByColumns; - TVector<TNodePtr> flattenByExprs; - if (!FlattenByArg(source->GetLabel(), flattenByColumns, flattenByExprs, flatten2.GetAlt1().GetRule_flatten_by_arg3())) { + TVector<TNodePtr> flattenByColumns; + TVector<TNodePtr> flattenByExprs; + if (!FlattenByArg(source->GetLabel(), flattenByColumns, flattenByExprs, flatten2.GetAlt1().GetRule_flatten_by_arg3())) { return nullptr; } Ctx.IncrementMonCounter("sql_features", "FlattenByColumns"); - if (!source->AddExpressions(Ctx, flattenByColumns, EExprSeat::FlattenBy)) { + if (!source->AddExpressions(Ctx, flattenByColumns, EExprSeat::FlattenBy)) { + return nullptr; + } + + if (!source->AddExpressions(Ctx, flattenByExprs, EExprSeat::FlattenByExpr)) { return nullptr; } - if (!source->AddExpressions(Ctx, flattenByExprs, EExprSeat::FlattenByExpr)) { - return nullptr; - } - source->SetFlattenByMode(mode); break; } @@ -5829,60 +5829,60 @@ TSourcePtr TSqlSelect::FlattenSource(const TRule_flatten_source& node) { } TSourcePtr TSqlSelect::JoinSource(const TRule_join_source& node) { - // join_source: (ANY)? flatten_source (join_op (ANY)? flatten_source join_constraint?)*; - if (node.HasBlock1() && !node.Block3Size()) { - Error() << "ANY is not allowed without JOIN"; - return nullptr; - } - - TSourcePtr source(FlattenSource(node.GetRule_flatten_source2())); + // join_source: (ANY)? flatten_source (join_op (ANY)? flatten_source join_constraint?)*; + if (node.HasBlock1() && !node.Block3Size()) { + Error() << "ANY is not allowed without JOIN"; + return nullptr; + } + + TSourcePtr source(FlattenSource(node.GetRule_flatten_source2())); if (!source) { return nullptr; } - - if (node.Block3Size()) { + + if (node.Block3Size()) { TPosition pos(Ctx.Pos()); TVector<TSourcePtr> sources; - TVector<TMaybe<TPosition>> anyPositions; - TVector<bool> anyFlags; - + TVector<TMaybe<TPosition>> anyPositions; + TVector<bool> anyFlags; + sources.emplace_back(std::move(source)); - anyPositions.emplace_back(node.HasBlock1() ? Ctx.TokenPosition(node.GetBlock1().GetToken1()) : TMaybe<TPosition>()); - anyFlags.push_back(bool(anyPositions.back())); - - for (auto& block: node.GetBlock3()) { - sources.emplace_back(FlattenSource(block.GetRule_flatten_source3())); + anyPositions.emplace_back(node.HasBlock1() ? Ctx.TokenPosition(node.GetBlock1().GetToken1()) : TMaybe<TPosition>()); + anyFlags.push_back(bool(anyPositions.back())); + + for (auto& block: node.GetBlock3()) { + sources.emplace_back(FlattenSource(block.GetRule_flatten_source3())); if (!sources.back()) { Ctx.IncrementMonCounter("sql_errors", "NoJoinWith"); return nullptr; } - - anyPositions.emplace_back(block.HasBlock2() ? Ctx.TokenPosition(block.GetBlock2().GetToken1()) : TMaybe<TPosition>()); - anyFlags.push_back(bool(anyPositions.back())); - } - - source = BuildEquiJoin(pos, std::move(sources), std::move(anyFlags), Ctx.Scoped->StrictJoinKeyTypes); - size_t idx = 1; - for (auto& block: node.GetBlock3()) { - YQL_ENSURE(idx < anyPositions.size()); - TMaybe<TPosition> leftAny = (idx == 1) ? anyPositions[0] : Nothing(); - TMaybe<TPosition> rightAny = anyPositions[idx]; - - if (!JoinOp(source.Get(), block, leftAny ? leftAny : rightAny)) { + + anyPositions.emplace_back(block.HasBlock2() ? Ctx.TokenPosition(block.GetBlock2().GetToken1()) : TMaybe<TPosition>()); + anyFlags.push_back(bool(anyPositions.back())); + } + + source = BuildEquiJoin(pos, std::move(sources), std::move(anyFlags), Ctx.Scoped->StrictJoinKeyTypes); + size_t idx = 1; + for (auto& block: node.GetBlock3()) { + YQL_ENSURE(idx < anyPositions.size()); + TMaybe<TPosition> leftAny = (idx == 1) ? anyPositions[0] : Nothing(); + TMaybe<TPosition> rightAny = anyPositions[idx]; + + if (!JoinOp(source.Get(), block, leftAny ? leftAny : rightAny)) { Ctx.IncrementMonCounter("sql_errors", "NoJoinOp"); return nullptr; } - ++idx; + ++idx; } } - + return source; } bool TSqlSelect::SelectTerm(TVector<TNodePtr>& terms, const TRule_result_column& node) { // result_column: // opt_id_prefix ASTERISK - // | expr ((AS an_id) | an_id_pure)? + // | expr ((AS an_id) | an_id_pure)? // ; switch (node.Alt_case()) { case TRule_result_column::kAltResultColumn1: { @@ -5896,7 +5896,7 @@ bool TSqlSelect::SelectTerm(TVector<TNodePtr>& terms, const TRule_result_column& } case TRule_result_column::kAltResultColumn2: { auto alt = node.GetAlt_result_column2(); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression expr(Ctx, Mode); TNodePtr term(expr.Build(alt.GetRule_expr1())); if (!term) { @@ -5904,26 +5904,26 @@ bool TSqlSelect::SelectTerm(TVector<TNodePtr>& terms, const TRule_result_column& return false; } if (alt.HasBlock2()) { - TString label; - bool implicitLabel = false; - switch (alt.GetBlock2().Alt_case()) { - case TRule_result_column_TAlt2_TBlock2::kAlt1: + TString label; + bool implicitLabel = false; + switch (alt.GetBlock2().Alt_case()) { + case TRule_result_column_TAlt2_TBlock2::kAlt1: label = Id(alt.GetBlock2().GetAlt1().GetBlock1().GetRule_an_id_or_type2(), *this); - break; - case TRule_result_column_TAlt2_TBlock2::kAlt2: - label = Id(alt.GetBlock2().GetAlt2().GetRule_an_id_pure1(), *this); - if (!Ctx.AnsiOptionalAs) { - // AS is mandatory - Ctx.Error() << "Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility"; - return false; - } - implicitLabel = true; - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - term->SetLabel(label, Ctx.Pos()); - term->MarkImplicitLabel(implicitLabel); + break; + case TRule_result_column_TAlt2_TBlock2::kAlt2: + label = Id(alt.GetBlock2().GetAlt2().GetRule_an_id_pure1(), *this); + if (!Ctx.AnsiOptionalAs) { + // AS is mandatory + Ctx.Error() << "Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility"; + return false; + } + implicitLabel = true; + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + term->SetLabel(label, Ctx.Pos()); + term->MarkImplicitLabel(implicitLabel); } terms.push_back(term); break; @@ -5954,7 +5954,7 @@ bool TSqlSelect::ValidateSelectColumns(const TVector<TNodePtr>& terms) { if (term->IsAsterisk()) { const auto& source = *term->GetSourceName(); if (source.empty() && terms.ysize() > 1) { - Ctx.Error(term->GetPos()) << "Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias)."; + Ctx.Error(term->GetPos()) << "Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias)."; return false; } else if (!asteriskSources.insert(source).second) { Ctx.Error(term->GetPos()) << "Unable to use twice same quialified asterisk. Invalid source: " << source; @@ -5975,7 +5975,7 @@ bool TSqlSelect::ValidateSelectColumns(const TVector<TNodePtr>& terms) { return true; } -TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos, bool unorderedSubquery) { +TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos, bool unorderedSubquery) { switch (node.Alt_case()) { case TRule_single_source::kAltSingleSource1: { const auto& alt = node.GetAlt_single_source1(); @@ -6016,9 +6016,9 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVect } case TRule_single_source::kAltSingleSource3: { const auto& alt = node.GetAlt_single_source3(); - TPosition pos; - return TSqlValues(Ctx, Mode).Build(alt.GetRule_values_stmt2(), pos, derivedColumns, derivedColumnsPos); - } + TPosition pos; + return TSqlValues(Ctx, Mode).Build(alt.GetRule_values_stmt2(), pos, derivedColumns, derivedColumnsPos); + } default: AltNotImplemented("single_source", node); Ctx.IncrementMonCounter("sql_errors", "UnknownSingleSource"); @@ -6026,55 +6026,55 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVect } } -TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, bool unorderedSubquery) { - // named_single_source: single_source (((AS an_id) | an_id_pure) pure_column_list?)? (sample_clause | tablesample_clause)?; - TVector<TString> derivedColumns; - TPosition derivedColumnsPos; - if (node.HasBlock2() && node.GetBlock2().HasBlock2()) { - const auto& columns = node.GetBlock2().GetBlock2().GetRule_pure_column_list1(); - Token(columns.GetToken1()); - derivedColumnsPos = Ctx.Pos(); - +TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, bool unorderedSubquery) { + // named_single_source: single_source (((AS an_id) | an_id_pure) pure_column_list?)? (sample_clause | tablesample_clause)?; + TVector<TString> derivedColumns; + TPosition derivedColumnsPos; + if (node.HasBlock2() && node.GetBlock2().HasBlock2()) { + const auto& columns = node.GetBlock2().GetBlock2().GetRule_pure_column_list1(); + Token(columns.GetToken1()); + derivedColumnsPos = Ctx.Pos(); + if (node.GetRule_single_source1().Alt_case() != TRule_single_source::kAltSingleSource3) { - Error() << "Derived column list is only supported for VALUES"; - return nullptr; - } - - PureColumnListStr(columns, *this, derivedColumns); - } - - auto singleSource = SingleSource(node.GetRule_single_source1(), derivedColumns, derivedColumnsPos, unorderedSubquery); + Error() << "Derived column list is only supported for VALUES"; + return nullptr; + } + + PureColumnListStr(columns, *this, derivedColumns); + } + + auto singleSource = SingleSource(node.GetRule_single_source1(), derivedColumns, derivedColumnsPos, unorderedSubquery); if (!singleSource) { return nullptr; } - if (node.HasBlock2()) { - TString label; - switch (node.GetBlock2().GetBlock1().Alt_case()) { - case TRule_named_single_source_TBlock2_TBlock1::kAlt1: - label = Id(node.GetBlock2().GetBlock1().GetAlt1().GetBlock1().GetRule_an_id2(), *this); - break; - case TRule_named_single_source_TBlock2_TBlock1::kAlt2: - label = Id(node.GetBlock2().GetBlock1().GetAlt2().GetRule_an_id_pure1(), *this); - if (!Ctx.AnsiOptionalAs) { - // AS is mandatory - Ctx.Error() << "Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility"; - return {}; - } - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); - } + if (node.HasBlock2()) { + TString label; + switch (node.GetBlock2().GetBlock1().Alt_case()) { + case TRule_named_single_source_TBlock2_TBlock1::kAlt1: + label = Id(node.GetBlock2().GetBlock1().GetAlt1().GetBlock1().GetRule_an_id2(), *this); + break; + case TRule_named_single_source_TBlock2_TBlock1::kAlt2: + label = Id(node.GetBlock2().GetBlock1().GetAlt2().GetRule_an_id_pure1(), *this); + if (!Ctx.AnsiOptionalAs) { + // AS is mandatory + Ctx.Error() << "Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility"; + return {}; + } + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); + } singleSource->SetLabel(label); } - if (node.HasBlock3()) { + if (node.HasBlock3()) { ESampleMode mode = ESampleMode::Auto; TSqlExpression expr(Ctx, Mode); TNodePtr samplingRateNode; TNodePtr samplingSeedNode; - const auto& sampleBlock = node.GetBlock3(); + const auto& sampleBlock = node.GetBlock3(); TPosition pos; switch (sampleBlock.Alt_case()) { - case TRule_named_single_source::TBlock3::kAlt1: + case TRule_named_single_source::TBlock3::kAlt1: { const auto& sampleExpr = sampleBlock.GetAlt1().GetRule_sample_clause1().GetRule_expr2(); samplingRateNode = expr.Build(sampleExpr); @@ -6085,7 +6085,7 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, Ctx.IncrementMonCounter("sql_features", "SampleClause"); } break; - case TRule_named_single_source::TBlock3::kAlt2: + case TRule_named_single_source::TBlock3::kAlt2: { const auto& tableSampleClause = sampleBlock.GetAlt2().GetRule_tablesample_clause1(); const auto& modeToken = tableSampleClause.GetRule_sampling_mode2().GetToken1(); @@ -6126,68 +6126,68 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, return singleSource; } -bool TSqlSelect::ColumnName(TVector<TNodePtr>& keys, const TRule_column_name& node) { +bool TSqlSelect::ColumnName(TVector<TNodePtr>& keys, const TRule_column_name& node) { const auto sourceName = OptIdPrefixAsStr(node.GetRule_opt_id_prefix1(), *this); - const auto columnName = Id(node.GetRule_an_id2(), *this); - if (columnName.empty()) { - // TDOD: Id() should return TMaybe<TString> - if (!Ctx.HasPendingErrors) { - Ctx.Error() << "Empty column name is not allowed"; - } - return false; - } - keys.push_back(BuildColumn(Ctx.Pos(), columnName, sourceName)); - return true; -} - -bool TSqlSelect::ColumnName(TVector<TNodePtr>& keys, const TRule_without_column_name& node) { - // without_column_name: (an_id DOT an_id) | an_id_without; - TString sourceName; - TString columnName; - switch (node.Alt_case()) { - case TRule_without_column_name::kAltWithoutColumnName1: - sourceName = Id(node.GetAlt_without_column_name1().GetBlock1().GetRule_an_id1(), *this); - columnName = Id(node.GetAlt_without_column_name1().GetBlock1().GetRule_an_id3(), *this); - break; - case TRule_without_column_name::kAltWithoutColumnName2: - columnName = Id(node.GetAlt_without_column_name2().GetRule_an_id_without1(), *this); - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - if (columnName.empty()) { - // TDOD: Id() should return TMaybe<TString> - if (!Ctx.HasPendingErrors) { - Ctx.Error() << "Empty column name is not allowed"; - } - return false; - } + const auto columnName = Id(node.GetRule_an_id2(), *this); + if (columnName.empty()) { + // TDOD: Id() should return TMaybe<TString> + if (!Ctx.HasPendingErrors) { + Ctx.Error() << "Empty column name is not allowed"; + } + return false; + } + keys.push_back(BuildColumn(Ctx.Pos(), columnName, sourceName)); + return true; +} + +bool TSqlSelect::ColumnName(TVector<TNodePtr>& keys, const TRule_without_column_name& node) { + // without_column_name: (an_id DOT an_id) | an_id_without; + TString sourceName; + TString columnName; + switch (node.Alt_case()) { + case TRule_without_column_name::kAltWithoutColumnName1: + sourceName = Id(node.GetAlt_without_column_name1().GetBlock1().GetRule_an_id1(), *this); + columnName = Id(node.GetAlt_without_column_name1().GetBlock1().GetRule_an_id3(), *this); + break; + case TRule_without_column_name::kAltWithoutColumnName2: + columnName = Id(node.GetAlt_without_column_name2().GetRule_an_id_without1(), *this); + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + if (columnName.empty()) { + // TDOD: Id() should return TMaybe<TString> + if (!Ctx.HasPendingErrors) { + Ctx.Error() << "Empty column name is not allowed"; + } + return false; + } keys.push_back(BuildColumn(Ctx.Pos(), columnName, sourceName)); return true; } -template<typename TRule> -bool TSqlSelect::ColumnList(TVector<TNodePtr>& keys, const TRule& node) { - bool result; - if constexpr (std::is_same_v<TRule, TRule_column_list>) { - result = ColumnName(keys, node.GetRule_column_name1()); - } else { - result = ColumnName(keys, node.GetRule_without_column_name1()); - } - - if (!result) { +template<typename TRule> +bool TSqlSelect::ColumnList(TVector<TNodePtr>& keys, const TRule& node) { + bool result; + if constexpr (std::is_same_v<TRule, TRule_column_list>) { + result = ColumnName(keys, node.GetRule_column_name1()); + } else { + result = ColumnName(keys, node.GetRule_without_column_name1()); + } + + if (!result) { return false; } - + for (auto b: node.GetBlock2()) { Token(b.GetToken1()); - if constexpr (std::is_same_v<TRule, TRule_column_list>) { - result = ColumnName(keys, b.GetRule_column_name2()); - } else { - result = ColumnName(keys, b.GetRule_without_column_name2()); - } - if (!result) { + if constexpr (std::is_same_v<TRule, TRule_column_list>) { + result = ColumnName(keys, b.GetRule_column_name2()); + } else { + result = ColumnName(keys, b.GetRule_without_column_name2()); + } + if (!result) { return false; } } @@ -6199,16 +6199,16 @@ bool TSqlSelect::NamedColumn(TVector<TNodePtr>& columnList, const TRule_named_co return false; } if (node.HasBlock2()) { - const auto label = Id(node.GetBlock2().GetRule_an_id2(), *this); + const auto label = Id(node.GetBlock2().GetRule_an_id2(), *this); columnList.back()->SetLabel(label); } return true; } -bool TSqlTranslation::SortSpecification(const TRule_sort_specification& node, TVector<TSortSpecificationPtr>& sortSpecs) { +bool TSqlTranslation::SortSpecification(const TRule_sort_specification& node, TVector<TSortSpecificationPtr>& sortSpecs) { bool asc = true; TSqlExpression expr(Ctx, Mode); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TNodePtr exprNode = expr.Build(node.GetRule_expr1()); if (!exprNode) { return false; @@ -6239,7 +6239,7 @@ bool TSqlTranslation::SortSpecification(const TRule_sort_specification& node, TV return true; } -bool TSqlTranslation::SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs) { +bool TSqlTranslation::SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs) { if (!SortSpecification(node.GetRule_sort_specification1(), sortSpecs)) { return false; } @@ -6252,75 +6252,75 @@ bool TSqlTranslation::SortSpecificationList(const TRule_sort_specification_list& return true; } -bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node) const { - TPosition pos; - return node.HasBlock1() && node.GetBlock1().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DISTINCT; -} - -bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const { - if (node.HasBlock1() && node.GetBlock1().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DISTINCT) { - distinctPos = Ctx.TokenPosition(node.GetBlock1().GetToken1()); - return true; - } - return false; -} - -bool TSqlTranslation::RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles) { - // role_name: an_id_or_type | bind_parameter; - switch (node.Alt_case()) { - case TRule_role_name::kAltRoleName1: { - TString name = Id(node.GetAlt_role_name1().GetRule_an_id_or_type1(), *this); - result = TDeferredAtom(Ctx.Pos(), name); - break; - } - case TRule_role_name::kAltRoleName2: { - TString paramName; - if (!NamedNodeImpl(node.GetAlt_role_name2().GetRule_bind_parameter1(), paramName, *this)) { - return false; - } - auto named = GetNamedNode(paramName); - if (!named) { - return false; - } - - result = MakeAtomFromExpression(Ctx, named); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - if (auto literalName = result.GetLiteral(); literalName && !allowSystemRoles) { - static const THashSet<TStringBuf> systemRoles = { "current_role", "current_user", "session_user" }; - if (systemRoles.contains(to_lower(*literalName))) { - Ctx.Error() << "System role " << to_upper(*literalName) << " can not be used here"; - return false; - } - } - - return true; -} - -bool TSqlTranslation::RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) { - // create_user_option: ENCRYPTED? PASSWORD expr; - result = TRoleParameters{}; - - TSqlExpression expr(Ctx, Mode); - TNodePtr password = expr.Build(node.GetRule_expr3()); - if (!password) { - return false; - } - - result.IsPasswordEncrypted = node.HasBlock1(); - if (!password->IsNull()) { - result.Password = MakeAtomFromExpression(Ctx, password); - } - - return true; -} - +bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node) const { + TPosition pos; + return node.HasBlock1() && node.GetBlock1().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DISTINCT; +} + +bool TSqlTranslation::IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const { + if (node.HasBlock1() && node.GetBlock1().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DISTINCT) { + distinctPos = Ctx.TokenPosition(node.GetBlock1().GetToken1()); + return true; + } + return false; +} + +bool TSqlTranslation::RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles) { + // role_name: an_id_or_type | bind_parameter; + switch (node.Alt_case()) { + case TRule_role_name::kAltRoleName1: { + TString name = Id(node.GetAlt_role_name1().GetRule_an_id_or_type1(), *this); + result = TDeferredAtom(Ctx.Pos(), name); + break; + } + case TRule_role_name::kAltRoleName2: { + TString paramName; + if (!NamedNodeImpl(node.GetAlt_role_name2().GetRule_bind_parameter1(), paramName, *this)) { + return false; + } + auto named = GetNamedNode(paramName); + if (!named) { + return false; + } + + result = MakeAtomFromExpression(Ctx, named); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + if (auto literalName = result.GetLiteral(); literalName && !allowSystemRoles) { + static const THashSet<TStringBuf> systemRoles = { "current_role", "current_user", "session_user" }; + if (systemRoles.contains(to_lower(*literalName))) { + Ctx.Error() << "System role " << to_upper(*literalName) << " can not be used here"; + return false; + } + } + + return true; +} + +bool TSqlTranslation::RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) { + // create_user_option: ENCRYPTED? PASSWORD expr; + result = TRoleParameters{}; + + TSqlExpression expr(Ctx, Mode); + TNodePtr password = expr.Build(node.GetRule_expr3()); + if (!password) { + return false; + } + + result.IsPasswordEncrypted = node.HasBlock1(); + if (!password->IsNull()) { + result.Password = MakeAtomFromExpression(Ctx, password); + } + + return true; +} + TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteSettings& settings, TPosition& selectPos) { - // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)? + // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)? // (WITH external_call_settings)? // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)? @@ -6331,16 +6331,16 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS selectPos = startPos; } - const bool hasUsing = node.HasBlock5(); - const bool unorderedSubquery = hasUsing; - TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source3(), unorderedSubquery)); + const bool hasUsing = node.HasBlock5(); + const bool unorderedSubquery = hasUsing; + TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source3(), unorderedSubquery)); if (!source) { return nullptr; } if (node.GetBlock4().size()) { TVector<TSourcePtr> sources(1, source); for (auto& s: node.GetBlock4()) { - sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), unorderedSubquery)); + sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), unorderedSubquery)); if (!sources.back()) { return nullptr; } @@ -6358,7 +6358,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS const auto& block5 = node.GetBlock5(); if (block5.HasBlock5()) { TSqlExpression expr(Ctx, Mode); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TNodePtr where = expr.Build(block5.GetBlock5().GetRule_expr2()); if (!where || !source->AddFilter(Ctx, where)) { return nullptr; @@ -6375,7 +6375,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS bool listCall = false; TSqlCallExpr call(Ctx, Mode); - bool initRet = call.Init(block5.GetRule_using_call_expr2()); + bool initRet = call.Init(block5.GetRule_using_call_expr2()); if (initRet) { call.IncCounters(); } @@ -6409,7 +6409,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS TSqlCallExpr finalCall(call, args); TNodePtr with(finalCall.IsExternal() ? finalCall.BuildCall() : finalCall.BuildUdf(/* forReduce = */ false)); - if (!with) { + if (!with) { return {}; } args = finalCall.GetArgs(); @@ -6417,7 +6417,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS listCall = true; if (block5.HasBlock3()) { - with->SetLabel(Id(block5.GetBlock3().GetRule_an_id2(), *this)); + with->SetLabel(Id(block5.GetBlock3().GetRule_an_id2(), *this)); } if (call.IsExternal() && block5.HasBlock7()) { @@ -6438,7 +6438,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSettings& settings, TPosition& selectPos) { // REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)? - // ON column_list USING ALL? using_call_expr (AS an_id)? + // ON column_list USING ALL? using_call_expr (AS an_id)? // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)? Token(node.GetToken1()); TPosition startPos(Ctx.Pos()); @@ -6446,14 +6446,14 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet selectPos = startPos; } - TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source2(), true)); + TSourcePtr source(NamedSingleSource(node.GetRule_named_single_source2(), true)); if (!source) { return {}; } if (node.GetBlock3().size()) { TVector<TSourcePtr> sources(1, source); for (auto& s: node.GetBlock3()) { - sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), true)); + sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), true)); if (!sources.back()) { return nullptr; } @@ -6475,7 +6475,7 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet } if (node.HasBlock11()) { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression expr(Ctx, Mode); TNodePtr where = expr.Build(node.GetBlock11().GetRule_expr2()); if (!where || !source->AddFilter(Ctx, where)) { @@ -6488,7 +6488,7 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet TNodePtr having; if (node.HasBlock12()) { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression expr(Ctx, Mode); having = expr.Build(node.GetBlock12().GetRule_expr2()); if (!having) { @@ -6498,7 +6498,7 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet bool listCall = false; TSqlCallExpr call(Ctx, Mode); - bool initRet = call.Init(node.GetRule_using_call_expr9()); + bool initRet = call.Init(node.GetRule_using_call_expr9()); if (initRet) { call.IncCounters(); } @@ -6520,13 +6520,13 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet TSqlCallExpr finalCall(call, args); - TNodePtr udf(finalCall.BuildUdf(/* forReduce = */ true)); - if (!udf) { + TNodePtr udf(finalCall.BuildUdf(/* forReduce = */ true)); + if (!udf) { return {}; } if (node.HasBlock10()) { - udf->SetLabel(Id(node.GetBlock10().GetRule_an_id2(), *this)); + udf->SetLabel(Id(node.GetBlock10().GetRule_an_id2(), *this)); } const auto reduceMode = node.HasBlock8() ? ReduceMode::ByAll : ReduceMode::ByPartition; @@ -6543,13 +6543,13 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet settings, assumeOrderBy, listCall); } -TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSettings& settings, TPosition& selectPos, - TMaybe<TSelectKindPlacement> placement, TVector<TSortSpecificationPtr>& selectOpOrederBy, bool& selectOpAssumeOrderBy) -{ - // (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* COMMA? (WITHOUT column_list)? (FROM join_source)? (WHERE expr)? - // group_by_clause? (HAVING expr)? window_clause? ext_order_by_clause? - selectOpOrederBy = {}; - selectOpAssumeOrderBy = false; +TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSettings& settings, TPosition& selectPos, + TMaybe<TSelectKindPlacement> placement, TVector<TSortSpecificationPtr>& selectOpOrederBy, bool& selectOpAssumeOrderBy) +{ + // (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* COMMA? (WITHOUT column_list)? (FROM join_source)? (WHERE expr)? + // group_by_clause? (HAVING expr)? window_clause? ext_order_by_clause? + selectOpOrederBy = {}; + selectOpAssumeOrderBy = false; if (node.HasBlock1()) { Token(node.GetBlock1().GetToken1()); } else { @@ -6566,9 +6566,9 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet Ctx.IncrementMonCounter("sql_features", "DistinctInSelect"); } - TSourcePtr source(BuildFakeSource(selectPos, /* missingFrom = */ true)); - if (node.HasBlock1() && node.HasBlock9()) { - Token(node.GetBlock9().GetToken1()); + TSourcePtr source(BuildFakeSource(selectPos, /* missingFrom = */ true)); + if (node.HasBlock1() && node.HasBlock9()) { + Token(node.GetBlock9().GetToken1()); Ctx.IncrementMonCounter("sql_errors", "DoubleFrom"); Ctx.Error() << "Only one FROM clause is allowed"; return nullptr; @@ -6576,8 +6576,8 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet if (node.HasBlock1()) { source = JoinSource(node.GetBlock1().GetRule_join_source2()); Ctx.IncrementMonCounter("sql_features", "FromInFront"); - } else if (node.HasBlock9()) { - source = JoinSource(node.GetBlock9().GetRule_join_source2()); + } else if (node.HasBlock9()) { + source = JoinSource(node.GetBlock9().GetRule_join_source2()); } if (!source) { return nullptr; @@ -6585,21 +6585,21 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet const bool selectStream = node.HasBlock3(); TVector<TNodePtr> without; - if (node.HasBlock8()) { - if (!ColumnList(without, node.GetBlock8().GetRule_without_column_list2())) { + if (node.HasBlock8()) { + if (!ColumnList(without, node.GetBlock8().GetRule_without_column_list2())) { return nullptr; } } - if (node.HasBlock10()) { - auto block = node.GetBlock10(); + if (node.HasBlock10()) { + auto block = node.GetBlock10(); Token(block.GetToken1()); TPosition pos(Ctx.Pos()); - TNodePtr where; - { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - TSqlExpression expr(Ctx, Mode); - where = expr.Build(block.GetRule_expr2()); - } + TNodePtr where; + { + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TSqlExpression expr(Ctx, Mode); + where = expr.Build(block.GetRule_expr2()); + } if (!where) { Ctx.IncrementMonCounter("sql_errors", "WhereInvalid"); return nullptr; @@ -6615,7 +6615,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet TVector<TNodePtr> groupByExpr, groupBy; THoppingWindowSpecPtr hoppingWindowSpec; bool compactGroupBy = false; - if (node.HasBlock11()) { + if (node.HasBlock11()) { TGroupByClause clause(Ctx, Mode); if (!clause.Build(node.GetBlock11().GetRule_group_by_clause1(), source->IsStream())) { return nullptr; @@ -6631,10 +6631,10 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } TNodePtr having; - if (node.HasBlock12()) { + if (node.HasBlock12()) { TSqlExpression expr(Ctx, Mode); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - having = expr.Build(node.GetBlock12().GetRule_expr2()); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + having = expr.Build(node.GetBlock12().GetRule_expr2()); if (!having) { return nullptr; } @@ -6642,12 +6642,12 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet } TWinSpecs windowSpec; - if (node.HasBlock13()) { + if (node.HasBlock13()) { if (source->IsStream()) { Ctx.Error() << "WINDOW is not allowed in streaming queries"; return nullptr; } - if (!WindowClause(node.GetBlock13().GetRule_window_clause1(), windowSpec)) { + if (!WindowClause(node.GetBlock13().GetRule_window_clause1(), windowSpec)) { return nullptr; } Ctx.IncrementMonCounter("sql_features", "WindowClause"); @@ -6655,21 +6655,21 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet bool assumeSorted = false; TVector<TSortSpecificationPtr> orderBy; - if (node.HasBlock14()) { - auto& orderBlock = node.GetBlock14().GetRule_ext_order_by_clause1(); - assumeSorted = orderBlock.HasBlock1(); - - Token(orderBlock.GetRule_order_by_clause2().GetToken1()); - + if (node.HasBlock14()) { + auto& orderBlock = node.GetBlock14().GetRule_ext_order_by_clause1(); + assumeSorted = orderBlock.HasBlock1(); + + Token(orderBlock.GetRule_order_by_clause2().GetToken1()); + if (source->IsStream()) { Ctx.Error() << "ORDER BY is not allowed in streaming queries"; return nullptr; } - - if (!ValidateLimitOrderByWithSelectOp(placement, "ORDER BY")) { - return nullptr; - } - + + if (!ValidateLimitOrderByWithSelectOp(placement, "ORDER BY")) { + return nullptr; + } + if (!OrderByClause(orderBlock.GetRule_order_by_clause2(), orderBy)) { return nullptr; } @@ -6677,40 +6677,40 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet ? (assumeSorted ? "AssumeOrderBy" : "OrderBy") : (assumeSorted ? "AssumeOrderByExpr" : "OrderByExpr") ); - - if (!NeedPassLimitOrderByToUnderlyingSelect(placement)) { - selectOpOrederBy.swap(orderBy); - std::swap(selectOpAssumeOrderBy, assumeSorted); - } + + if (!NeedPassLimitOrderByToUnderlyingSelect(placement)) { + selectOpOrederBy.swap(orderBy); + std::swap(selectOpAssumeOrderBy, assumeSorted); + } } - + TVector<TNodePtr> terms; - { - class TScopedWinSpecs { - public: - TScopedWinSpecs(TContext& ctx, TWinSpecs& specs) - : Ctx(ctx) - { - Ctx.WinSpecsScopes.push_back(std::ref(specs)); - } - ~TScopedWinSpecs() { - Ctx.WinSpecsScopes.pop_back(); - } - private: - TContext& Ctx; - }; - - - TScopedWinSpecs scoped(Ctx, windowSpec); - if (!SelectTerm(terms, node.GetRule_result_column5())) { + { + class TScopedWinSpecs { + public: + TScopedWinSpecs(TContext& ctx, TWinSpecs& specs) + : Ctx(ctx) + { + Ctx.WinSpecsScopes.push_back(std::ref(specs)); + } + ~TScopedWinSpecs() { + Ctx.WinSpecsScopes.pop_back(); + } + private: + TContext& Ctx; + }; + + + TScopedWinSpecs scoped(Ctx, windowSpec); + if (!SelectTerm(terms, node.GetRule_result_column5())) { return nullptr; } - for (auto block: node.GetBlock6()) { - if (!SelectTerm(terms, block.GetRule_result_column2())) { - return nullptr; - } - } - + for (auto block: node.GetBlock6()) { + if (!SelectTerm(terms, block.GetRule_result_column2())) { + return nullptr; + } + } + } if (!ValidateSelectColumns(terms)) { return nullptr; @@ -6719,252 +6719,252 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet std::move(windowSpec), hoppingWindowSpec, std::move(terms), distinct, std::move(without), selectStream, settings); } -TString TSqlTranslation::FrameSettingsToString(EFrameSettings settings, bool isUnbounded) { - TString result; - switch (settings) { - case FramePreceding: - result = "PRECEDING"; break; - case FrameCurrentRow: - YQL_ENSURE(!isUnbounded); - result = "CURRENT ROW"; break; - case FrameFollowing: - result = "FOLLOWING"; break; +TString TSqlTranslation::FrameSettingsToString(EFrameSettings settings, bool isUnbounded) { + TString result; + switch (settings) { + case FramePreceding: + result = "PRECEDING"; break; + case FrameCurrentRow: + YQL_ENSURE(!isUnbounded); + result = "CURRENT ROW"; break; + case FrameFollowing: + result = "FOLLOWING"; break; default: - Y_FAIL("Unexpected frame settings"); - } - - return (isUnbounded ? "UNBOUNDED " : "") + result; -} - -bool CheckFrameBoundLiteral(TContext& ctx, const TFrameBound& bound, TMaybe<i32>& boundValue) { - boundValue = {}; - auto node = bound.Bound; - if (node && node->IsLiteral()) { - auto type = node->GetLiteralType(); - if (type != "Int32") { - ctx.Error(node->GetPos()) << "Expecting Int32 as frame bound value, but got " << type << " literal"; - return false; - } - - i32 value = FromString<i32>(node->GetLiteralValue()); - if (value < 0) { - ctx.Error(node->GetPos()) << "Expecting non-negative value for frame bound, but got " << value; - return false; - } - - boundValue = value; - } - - return true; -} - -bool TSqlTranslation::IsValidFrameSettings(TContext& ctx, const TFrameBound& begin, const TFrameBound& end) { - Y_UNUSED(ctx); - YQL_ENSURE(begin.Settings != FrameUndefined); - YQL_ENSURE(end.Settings != FrameUndefined); - - const bool beginUnbounded = !begin.Bound && begin.Settings != FrameCurrentRow; - const bool endUnbounded = !end.Bound && end.Settings != FrameCurrentRow; - - if (beginUnbounded && begin.Settings == FrameFollowing) { - ctx.Error(begin.Pos) << "Frame cannot start from " << FrameSettingsToString(begin.Settings, beginUnbounded); - return false; - } - - if (endUnbounded && end.Settings == FramePreceding) { - ctx.Error(end.Pos) << "Frame cannot end with " << FrameSettingsToString(end.Settings, endUnbounded); - return false; - } - - if (begin.Settings > end.Settings) { - ctx.Error(begin.Pos) << "Frame cannot start from " << FrameSettingsToString(begin.Settings, beginUnbounded) - << " and end with " << FrameSettingsToString(end.Settings, endUnbounded); - return false; - } - - TMaybe<i32> beginValue; - TMaybe<i32> endValue; - - if (!CheckFrameBoundLiteral(ctx, begin, beginValue) || !CheckFrameBoundLiteral(ctx, end, endValue)) { - return false; - } - - if (beginValue.Defined() && endValue.Defined()) { - if (begin.Settings == FramePreceding) { - beginValue = 0 - *beginValue; - } - if (end.Settings == FramePreceding) { - endValue = 0 - *endValue; - } - - if (*beginValue > *endValue) { - YQL_ENSURE(begin.Bound); - ctx.Warning(begin.Bound->GetPos(), TIssuesIds::YQL_EMPTY_WINDOW_FRAME) << "Used frame specification implies empty window frame"; - } - } - - return true; -} - -bool TSqlTranslation::FrameBound(const TRule_window_frame_bound& rule, TFrameBoundPtr& bound) { - // window_frame_bound: - // CURRENT ROW - // | (expr | UNBOUNDED) (PRECEDING | FOLLOWING) - // ; - bound = new TFrameBound; + Y_FAIL("Unexpected frame settings"); + } + + return (isUnbounded ? "UNBOUNDED " : "") + result; +} + +bool CheckFrameBoundLiteral(TContext& ctx, const TFrameBound& bound, TMaybe<i32>& boundValue) { + boundValue = {}; + auto node = bound.Bound; + if (node && node->IsLiteral()) { + auto type = node->GetLiteralType(); + if (type != "Int32") { + ctx.Error(node->GetPos()) << "Expecting Int32 as frame bound value, but got " << type << " literal"; + return false; + } + + i32 value = FromString<i32>(node->GetLiteralValue()); + if (value < 0) { + ctx.Error(node->GetPos()) << "Expecting non-negative value for frame bound, but got " << value; + return false; + } + + boundValue = value; + } + + return true; +} + +bool TSqlTranslation::IsValidFrameSettings(TContext& ctx, const TFrameBound& begin, const TFrameBound& end) { + Y_UNUSED(ctx); + YQL_ENSURE(begin.Settings != FrameUndefined); + YQL_ENSURE(end.Settings != FrameUndefined); + + const bool beginUnbounded = !begin.Bound && begin.Settings != FrameCurrentRow; + const bool endUnbounded = !end.Bound && end.Settings != FrameCurrentRow; + + if (beginUnbounded && begin.Settings == FrameFollowing) { + ctx.Error(begin.Pos) << "Frame cannot start from " << FrameSettingsToString(begin.Settings, beginUnbounded); + return false; + } + + if (endUnbounded && end.Settings == FramePreceding) { + ctx.Error(end.Pos) << "Frame cannot end with " << FrameSettingsToString(end.Settings, endUnbounded); + return false; + } + + if (begin.Settings > end.Settings) { + ctx.Error(begin.Pos) << "Frame cannot start from " << FrameSettingsToString(begin.Settings, beginUnbounded) + << " and end with " << FrameSettingsToString(end.Settings, endUnbounded); + return false; + } + + TMaybe<i32> beginValue; + TMaybe<i32> endValue; + + if (!CheckFrameBoundLiteral(ctx, begin, beginValue) || !CheckFrameBoundLiteral(ctx, end, endValue)) { + return false; + } + + if (beginValue.Defined() && endValue.Defined()) { + if (begin.Settings == FramePreceding) { + beginValue = 0 - *beginValue; + } + if (end.Settings == FramePreceding) { + endValue = 0 - *endValue; + } + + if (*beginValue > *endValue) { + YQL_ENSURE(begin.Bound); + ctx.Warning(begin.Bound->GetPos(), TIssuesIds::YQL_EMPTY_WINDOW_FRAME) << "Used frame specification implies empty window frame"; + } + } + + return true; +} + +bool TSqlTranslation::FrameBound(const TRule_window_frame_bound& rule, TFrameBoundPtr& bound) { + // window_frame_bound: + // CURRENT ROW + // | (expr | UNBOUNDED) (PRECEDING | FOLLOWING) + // ; + bound = new TFrameBound; switch (rule.Alt_case()) { case TRule_window_frame_bound::kAltWindowFrameBound1: - bound->Pos = GetPos(rule.GetAlt_window_frame_bound1().GetToken1()); - bound->Settings = FrameCurrentRow; - break; - case TRule_window_frame_bound::kAltWindowFrameBound2: { - auto block = rule.GetAlt_window_frame_bound2().GetBlock1(); - switch (block.Alt_case()) { - case TRule_window_frame_bound_TAlt2_TBlock1::kAlt1: { - TSqlExpression boundExpr(Ctx, Mode); - bound->Bound = boundExpr.Build(block.GetAlt1().GetRule_expr1()); - if (!bound->Bound) { - return false; - } - bound->Pos = bound->Bound->GetPos(); - break; - } - case TRule_window_frame_bound_TAlt2_TBlock1::kAlt2: - bound->Pos = GetPos(block.GetAlt2().GetToken1()); - break; - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - const TString settingToken = to_lower(Token(rule.GetAlt_window_frame_bound2().GetToken2())); - if (settingToken == "preceding") { - bound->Settings = FramePreceding; - } else if (settingToken == "following") { - bound->Settings = FrameFollowing; - } else { - Y_FAIL("You should change implementation according to grammar changes"); - } - break; - } + bound->Pos = GetPos(rule.GetAlt_window_frame_bound1().GetToken1()); + bound->Settings = FrameCurrentRow; + break; + case TRule_window_frame_bound::kAltWindowFrameBound2: { + auto block = rule.GetAlt_window_frame_bound2().GetBlock1(); + switch (block.Alt_case()) { + case TRule_window_frame_bound_TAlt2_TBlock1::kAlt1: { + TSqlExpression boundExpr(Ctx, Mode); + bound->Bound = boundExpr.Build(block.GetAlt1().GetRule_expr1()); + if (!bound->Bound) { + return false; + } + bound->Pos = bound->Bound->GetPos(); + break; + } + case TRule_window_frame_bound_TAlt2_TBlock1::kAlt2: + bound->Pos = GetPos(block.GetAlt2().GetToken1()); + break; + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + const TString settingToken = to_lower(Token(rule.GetAlt_window_frame_bound2().GetToken2())); + if (settingToken == "preceding") { + bound->Settings = FramePreceding; + } else if (settingToken == "following") { + bound->Settings = FrameFollowing; + } else { + Y_FAIL("You should change implementation according to grammar changes"); + } + break; + } default: Y_FAIL("FrameClause: frame bound not corresond to grammar changes"); } - return true; + return true; } -bool TSqlTranslation::FrameClause(const TRule_window_frame_clause& rule, TFrameSpecificationPtr& frameSpec) { - frameSpec = new TFrameSpecification; +bool TSqlTranslation::FrameClause(const TRule_window_frame_clause& rule, TFrameSpecificationPtr& frameSpec) { + frameSpec = new TFrameSpecification; const TString frameUnitStr = to_lower(Token(rule.GetRule_window_frame_units1().GetToken1())); if (frameUnitStr == "rows") { - frameSpec->FrameType = EFrameType::FrameByRows; + frameSpec->FrameType = EFrameType::FrameByRows; } else if (frameUnitStr == "range") { - frameSpec->FrameType = EFrameType::FrameByRange; - } else if (frameUnitStr == "groups") { - frameSpec->FrameType = EFrameType::FrameByGroups; + frameSpec->FrameType = EFrameType::FrameByRange; + } else if (frameUnitStr == "groups") { + frameSpec->FrameType = EFrameType::FrameByGroups; } else { Ctx.Error() << "Unknown frame type in window specification: " << frameUnitStr; return false; } - if (frameSpec->FrameType != EFrameType::FrameByRows) { - Ctx.Error() << "Only ROWS is supported as window frame unit"; - return false; - } - + if (frameSpec->FrameType != EFrameType::FrameByRows) { + Ctx.Error() << "Only ROWS is supported as window frame unit"; + return false; + } + auto frameExtent = rule.GetRule_window_frame_extent2(); switch (frameExtent.Alt_case()) { - case TRule_window_frame_extent::kAltWindowFrameExtent1: { - auto start = frameExtent.GetAlt_window_frame_extent1().GetRule_window_frame_bound1(); - if (!FrameBound(start, frameSpec->FrameBegin)) { + case TRule_window_frame_extent::kAltWindowFrameExtent1: { + auto start = frameExtent.GetAlt_window_frame_extent1().GetRule_window_frame_bound1(); + if (!FrameBound(start, frameSpec->FrameBegin)) { return false; } - - frameSpec->FrameEnd = new TFrameBound; - frameSpec->FrameEnd->Pos = frameSpec->FrameBegin->Pos; - frameSpec->FrameEnd->Settings = FrameCurrentRow; + + frameSpec->FrameEnd = new TFrameBound; + frameSpec->FrameEnd->Pos = frameSpec->FrameBegin->Pos; + frameSpec->FrameEnd->Settings = FrameCurrentRow; break; - } + } case TRule_window_frame_extent::kAltWindowFrameExtent2: { auto between = frameExtent.GetAlt_window_frame_extent2().GetRule_window_frame_between1(); - if (!FrameBound(between.GetRule_window_frame_bound2(), frameSpec->FrameBegin)) { + if (!FrameBound(between.GetRule_window_frame_bound2(), frameSpec->FrameBegin)) { return false; } - if (!FrameBound(between.GetRule_window_frame_bound4(), frameSpec->FrameEnd)) { + if (!FrameBound(between.GetRule_window_frame_bound4(), frameSpec->FrameEnd)) { return false; } break; } default: - Y_FAIL("FrameClause: frame extent not correspond to grammar changes"); - } - YQL_ENSURE(frameSpec->FrameBegin); - YQL_ENSURE(frameSpec->FrameEnd); - // check for literal - if (!IsValidFrameSettings(Ctx, *frameSpec->FrameBegin, *frameSpec->FrameEnd)) - { - return false; - } - + Y_FAIL("FrameClause: frame extent not correspond to grammar changes"); + } + YQL_ENSURE(frameSpec->FrameBegin); + YQL_ENSURE(frameSpec->FrameEnd); + // check for literal + if (!IsValidFrameSettings(Ctx, *frameSpec->FrameBegin, *frameSpec->FrameEnd)) + { + return false; + } + if (rule.HasBlock3()) { switch (rule.GetBlock3().GetRule_window_frame_exclusion1().Alt_case()) { case TRule_window_frame_exclusion::kAltWindowFrameExclusion1: - frameSpec->FrameExclusion = FrameExclCurRow; + frameSpec->FrameExclusion = FrameExclCurRow; break; case TRule_window_frame_exclusion::kAltWindowFrameExclusion2: - frameSpec->FrameExclusion = FrameExclGroup; + frameSpec->FrameExclusion = FrameExclGroup; break; case TRule_window_frame_exclusion::kAltWindowFrameExclusion3: - frameSpec->FrameExclusion = FrameExclTies; + frameSpec->FrameExclusion = FrameExclTies; break; case TRule_window_frame_exclusion::kAltWindowFrameExclusion4: - frameSpec->FrameExclusion = FrameExclNone; + frameSpec->FrameExclusion = FrameExclNone; break; default: - Y_FAIL("FrameClause: frame exclusion not correspond to grammar changes"); + Y_FAIL("FrameClause: frame exclusion not correspond to grammar changes"); } } - - if (frameSpec->FrameExclusion != FrameExclNone) { - Ctx.Error() << "Frame exclusion is not supported yet"; - return false; - } - + + if (frameSpec->FrameExclusion != FrameExclNone) { + Ctx.Error() << "Frame exclusion is not supported yet"; + return false; + } + return true; } -TWindowSpecificationPtr TSqlTranslation::WindowSpecification(const TRule_window_specification_details& rule) { - /* - window_specification_details: - existing_window_name? - window_partition_clause? - window_order_clause? - window_frame_clause? - */ +TWindowSpecificationPtr TSqlTranslation::WindowSpecification(const TRule_window_specification_details& rule) { + /* + window_specification_details: + existing_window_name? + window_partition_clause? + window_order_clause? + window_frame_clause? + */ TWindowSpecificationPtr winSpecPtr = new TWindowSpecification; if (rule.HasBlock1()) { Ctx.Error() << "Existing window name is not supported in window specification yet!"; return {}; } if (rule.HasBlock2()) { - /* - window_partition_clause: PARTITION COMPACT? BY named_expr_list; - */ - auto& partitionClause = rule.GetBlock2().GetRule_window_partition_clause1(); - winSpecPtr->IsCompact = partitionClause.HasBlock2(); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - if (!NamedExprList(partitionClause.GetRule_named_expr_list4(), winSpecPtr->Partitions)) { + /* + window_partition_clause: PARTITION COMPACT? BY named_expr_list; + */ + auto& partitionClause = rule.GetBlock2().GetRule_window_partition_clause1(); + winSpecPtr->IsCompact = partitionClause.HasBlock2(); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + if (!NamedExprList(partitionClause.GetRule_named_expr_list4(), winSpecPtr->Partitions)) { return {}; } - // ignore empty unnamed tuples: - // "PARTITION BY (), foo(x) as y, (), (z)" is allowed and will work exactly the same as - // "PARTITION BY foo(x) as y, z" - auto removed = std::remove_if(winSpecPtr->Partitions.begin(), winSpecPtr->Partitions.end(), - [](const TNodePtr& partitionNode) { - return !partitionNode->GetLabel() && !partitionNode->GetColumnName() && - dynamic_cast<TTupleNode*>(partitionNode.Get()) != nullptr && - partitionNode->GetTupleSize() == 0; - }); - winSpecPtr->Partitions.erase(removed, winSpecPtr->Partitions.end()); - + // ignore empty unnamed tuples: + // "PARTITION BY (), foo(x) as y, (), (z)" is allowed and will work exactly the same as + // "PARTITION BY foo(x) as y, z" + auto removed = std::remove_if(winSpecPtr->Partitions.begin(), winSpecPtr->Partitions.end(), + [](const TNodePtr& partitionNode) { + return !partitionNode->GetLabel() && !partitionNode->GetColumnName() && + dynamic_cast<TTupleNode*>(partitionNode.Get()) != nullptr && + partitionNode->GetTupleSize() == 0; + }); + winSpecPtr->Partitions.erase(removed, winSpecPtr->Partitions.end()); + } if (rule.HasBlock3()) { if (!OrderByClause(rule.GetBlock3().GetRule_window_order_clause1().GetRule_order_by_clause1(), winSpecPtr->OrderBy)) { @@ -6972,35 +6972,35 @@ TWindowSpecificationPtr TSqlTranslation::WindowSpecification(const TRule_window_ } } if (rule.HasBlock4()) { - if (!FrameClause(rule.GetBlock4().GetRule_window_frame_clause1(), winSpecPtr->Frame)) { - return {}; - } - } else { - winSpecPtr->Frame = new TFrameSpecification; - winSpecPtr->Frame->FrameType = EFrameType::FrameByRows; - winSpecPtr->Frame->FrameExclusion = EFrameExclusions::FrameExclNone; - - winSpecPtr->Frame->FrameBegin = new TFrameBound; - winSpecPtr->Frame->FrameBegin->Settings = EFrameSettings::FramePreceding; - - const bool ordered = !winSpecPtr->OrderBy.empty(); - winSpecPtr->Frame->FrameEnd = new TFrameBound; - if (ordered) { - // ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - winSpecPtr->Frame->FrameEnd->Settings = EFrameSettings::FrameCurrentRow; - } else { - // ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - winSpecPtr->Frame->FrameEnd->Settings = EFrameSettings::FrameFollowing; - } - - // TODO: According to standard this should be RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW with order by clause - // TODO: and RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING without order by + if (!FrameClause(rule.GetBlock4().GetRule_window_frame_clause1(), winSpecPtr->Frame)) { + return {}; + } + } else { + winSpecPtr->Frame = new TFrameSpecification; + winSpecPtr->Frame->FrameType = EFrameType::FrameByRows; + winSpecPtr->Frame->FrameExclusion = EFrameExclusions::FrameExclNone; + + winSpecPtr->Frame->FrameBegin = new TFrameBound; + winSpecPtr->Frame->FrameBegin->Settings = EFrameSettings::FramePreceding; + + const bool ordered = !winSpecPtr->OrderBy.empty(); + winSpecPtr->Frame->FrameEnd = new TFrameBound; + if (ordered) { + // ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + winSpecPtr->Frame->FrameEnd->Settings = EFrameSettings::FrameCurrentRow; + } else { + // ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + winSpecPtr->Frame->FrameEnd->Settings = EFrameSettings::FrameFollowing; + } + + // TODO: According to standard this should be RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW with order by clause + // TODO: and RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING without order by } return winSpecPtr; } bool TSqlSelect::WindowDefinition(const TRule_window_definition& rule, TWinSpecs& winSpecs) { - const TString windowName = Id(rule.GetRule_new_window_name1().GetRule_window_name1().GetRule_an_id_window1(), *this); + const TString windowName = Id(rule.GetRule_new_window_name1().GetRule_window_name1().GetRule_an_id_window1(), *this); if (winSpecs.contains(windowName)) { Ctx.Error() << "Unable to declare window with same name: " << windowName; return false; @@ -7026,39 +7026,39 @@ bool TSqlSelect::WindowClause(const TRule_window_clause& rule, TWinSpecs& winSpe return true; } -bool TSqlTranslation::OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy) { +bool TSqlTranslation::OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy) { return SortSpecificationList(node.GetRule_sort_specification_list3(), orderBy); } bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) { CompactGroupBy = node.HasBlock2(); - TPosition distinctPos; - if (IsDistinctOptSet(node.GetRule_opt_set_quantifier4(), distinctPos)) { - Ctx.Error(distinctPos) << "DISTINCT is not supported in GROUP BY clause yet!"; + TPosition distinctPos; + if (IsDistinctOptSet(node.GetRule_opt_set_quantifier4(), distinctPos)) { + Ctx.Error(distinctPos) << "DISTINCT is not supported in GROUP BY clause yet!"; Ctx.IncrementMonCounter("sql_errors", "DistinctInGroupByNotSupported"); return false; } - if (!ParseList(node.GetRule_grouping_element_list5(), EGroupByFeatures::Ordinary)) { + if (!ParseList(node.GetRule_grouping_element_list5(), EGroupByFeatures::Ordinary)) { return false; } - if (!ResolveGroupByAndGrouping()) { - return false; - } - if (stream) { - if (!HoppingWindowSpec) { - Ctx.Error() << "Streaming group by query must have a hopping window specification."; - return false; - } + if (!ResolveGroupByAndGrouping()) { + return false; + } + if (stream) { + if (!HoppingWindowSpec) { + Ctx.Error() << "Streaming group by query must have a hopping window specification."; + return false; + } } return true; } -bool TGroupByClause::ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext) { - if (!GroupingElement(groupingListNode.GetRule_grouping_element1(), featureContext)) { +bool TGroupByClause::ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext) { + if (!GroupingElement(groupingListNode.GetRule_grouping_element1(), featureContext)) { return false; } for (auto b: groupingListNode.GetBlock2()) { - if (!GroupingElement(b.GetRule_grouping_element2(), featureContext)) { + if (!GroupingElement(b.GetRule_grouping_element2(), featureContext)) { return false; } } @@ -7096,7 +7096,7 @@ TMap<TString, TNodePtr>& TGroupByClause::Aliases() { return GroupSetContext->NodeAliases; } -THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() const { +THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() const { return HoppingWindowSpec; } @@ -7104,23 +7104,23 @@ bool TGroupByClause::IsCompactGroupBy() const { return CompactGroupBy; } -TMaybe<TVector<TNodePtr>> TGroupByClause::MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const { +TMaybe<TVector<TNodePtr>> TGroupByClause::MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const { TVector<TNodePtr> content; for (const auto& leftNode: lhs) { auto leftPtr = leftNode->ContentListPtr(); - if (!leftPtr) { - // TODO: shouldn't happen - Ctx.Error() << "Unable to multiply grouping sets"; - return {}; - } + if (!leftPtr) { + // TODO: shouldn't happen + Ctx.Error() << "Unable to multiply grouping sets"; + return {}; + } for (const auto& rightNode: rhs) { TVector<TNodePtr> mulItem(leftPtr->begin(), leftPtr->end()); auto rightPtr = rightNode->ContentListPtr(); - if (!rightPtr) { - // TODO: shouldn't happen - Ctx.Error() << "Unable to multiply grouping sets"; - return {}; - } + if (!rightPtr) { + // TODO: shouldn't happen + Ctx.Error() << "Unable to multiply grouping sets"; + return {}; + } mulItem.insert(mulItem.end(), rightPtr->begin(), rightPtr->end()); content.push_back(BuildListOfNamedNodes(Ctx.Pos(), std::move(mulItem))); } @@ -7128,61 +7128,61 @@ TMaybe<TVector<TNodePtr>> TGroupByClause::MultiplyGroupingSets(const TVector<TNo return content; } -bool TGroupByClause::ResolveGroupByAndGrouping() { +bool TGroupByClause::ResolveGroupByAndGrouping() { auto listPos = std::find_if(GroupBySet.begin(), GroupBySet.end(), [](const TNodePtr& node) { return node->ContentListPtr(); }); if (listPos == GroupBySet.end()) { - return true; + return true; } auto curContent = *(*listPos)->ContentListPtr(); if (listPos != GroupBySet.begin()) { TVector<TNodePtr> emulate(GroupBySet.begin(), listPos); TVector<TNodePtr> emulateContent(1, BuildListOfNamedNodes(Ctx.Pos(), std::move(emulate))); - auto mult = MultiplyGroupingSets(emulateContent, curContent); - if (!mult) { - return false; - } - curContent = *mult; + auto mult = MultiplyGroupingSets(emulateContent, curContent); + if (!mult) { + return false; + } + curContent = *mult; } for (++listPos; listPos != GroupBySet.end(); ++listPos) { auto newElem = (*listPos)->ContentListPtr(); if (newElem) { - auto mult = MultiplyGroupingSets(curContent, *newElem); - if (!mult) { - return false; - } - curContent = *mult; + auto mult = MultiplyGroupingSets(curContent, *newElem); + if (!mult) { + return false; + } + curContent = *mult; } else { TVector<TNodePtr> emulate(1, *listPos); TVector<TNodePtr> emulateContent(1, BuildListOfNamedNodes(Ctx.Pos(), std::move(emulate))); - auto mult = MultiplyGroupingSets(curContent, emulateContent); - if (!mult) { - return false; - } - curContent = *mult; + auto mult = MultiplyGroupingSets(curContent, emulateContent); + if (!mult) { + return false; + } + curContent = *mult; } } TVector<TNodePtr> result(1, BuildListOfNamedNodes(Ctx.Pos(), std::move(curContent))); std::swap(result, GroupBySet); - return true; + return true; } -bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupByFeatures featureContext) { +bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupByFeatures featureContext) { TSourcePtr res; TVector<TNodePtr> emptyContent; switch (node.Alt_case()) { case TRule_grouping_element::kAltGroupingElement1: - if (!OrdinaryGroupingSet(node.GetAlt_grouping_element1().GetRule_ordinary_grouping_set1(), featureContext)) { + if (!OrdinaryGroupingSet(node.GetAlt_grouping_element1().GetRule_ordinary_grouping_set1(), featureContext)) { return false; } Features().Set(EGroupByFeatures::Ordinary); break; case TRule_grouping_element::kAltGroupingElement2: { TGroupByClause subClause(Ctx, Mode, GroupSetContext); - if (!subClause.OrdinaryGroupingSetList(node.GetAlt_grouping_element2().GetRule_rollup_list1().GetRule_ordinary_grouping_set_list3(), - EGroupByFeatures::Rollup)) - { + if (!subClause.OrdinaryGroupingSetList(node.GetAlt_grouping_element2().GetRule_rollup_list1().GetRule_ordinary_grouping_set_list3(), + EGroupByFeatures::Rollup)) + { return false; } auto& content = subClause.Content(); @@ -7199,9 +7199,9 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupB } case TRule_grouping_element::kAltGroupingElement3: { TGroupByClause subClause(Ctx, Mode, GroupSetContext); - if (!subClause.OrdinaryGroupingSetList(node.GetAlt_grouping_element3().GetRule_cube_list1().GetRule_ordinary_grouping_set_list3(), - EGroupByFeatures::Cube)) - { + if (!subClause.OrdinaryGroupingSetList(node.GetAlt_grouping_element3().GetRule_cube_list1().GetRule_ordinary_grouping_set_list3(), + EGroupByFeatures::Cube)) + { return false; } auto& content = subClause.Content(); @@ -7228,7 +7228,7 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupB case TRule_grouping_element::kAltGroupingElement4: { auto listNode = node.GetAlt_grouping_element4().GetRule_grouping_sets_specification1().GetRule_grouping_element_list4(); TGroupByClause subClause(Ctx, Mode, GroupSetContext); - if (!subClause.ParseList(listNode, EGroupByFeatures::GroupingSet)) { + if (!subClause.ParseList(listNode, EGroupByFeatures::GroupingSet)) { return false; } auto& content = subClause.Content(); @@ -7276,12 +7276,12 @@ void TGroupByClause::FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& col collection.push_back(elem); } -bool TGroupByClause::OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext) { - TNodePtr namedExprNode; - { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - namedExprNode = NamedExpr(node.GetRule_named_expr1(), EExpr::GroupBy); - } +bool TGroupByClause::OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext) { + TNodePtr namedExprNode; + { + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + namedExprNode = NamedExpr(node.GetRule_named_expr1(), EExpr::GroupBy); + } if (!namedExprNode) { return false; } @@ -7301,10 +7301,10 @@ bool TGroupByClause::OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node continue; } - if (!AllowUnnamed(content->GetPos(), featureContext)) { - return false; - } - + if (!AllowUnnamed(content->GetPos(), featureContext)) { + return false; + } + content->SetLabel(label = GenerateGroupByExprName()); } if (!AddAlias(label, content)) { @@ -7318,9 +7318,9 @@ bool TGroupByClause::OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node } if (!nodeLabel && !namedExprNode->GetColumnName()) { - if (!AllowUnnamed(namedExprNode->GetPos(), featureContext)) { - return false; - } + if (!AllowUnnamed(namedExprNode->GetPos(), featureContext)) { + return false; + } namedExprNode->SetLabel(nodeLabel = GenerateGroupByExprName()); } if (nodeLabel) { @@ -7334,12 +7334,12 @@ bool TGroupByClause::OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node return true; } -bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext) { - if (!OrdinaryGroupingSet(node.GetRule_ordinary_grouping_set1(), featureContext)) { +bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext) { + if (!OrdinaryGroupingSet(node.GetRule_ordinary_grouping_set1(), featureContext)) { return false; } for (auto& block: node.GetBlock2()) { - if (!OrdinaryGroupingSet(block.GetRule_ordinary_grouping_set2(), featureContext)) { + if (!OrdinaryGroupingSet(block.GetRule_ordinary_grouping_set2(), featureContext)) { return false; } } @@ -7353,7 +7353,7 @@ bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& nod } HoppingWindowSpec = new THoppingWindowSpec; { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression expr(Ctx, Mode); HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3()); if (!HoppingWindowSpec->TimeExtractor) { @@ -7412,27 +7412,27 @@ bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& nod return true; } -bool TGroupByClause::AllowUnnamed(TPosition pos, EGroupByFeatures featureContext) { - TStringBuf feature; - switch (featureContext) { - case EGroupByFeatures::Ordinary: - return true; - case EGroupByFeatures::Rollup: - feature = "ROLLUP"; - break; - case EGroupByFeatures::Cube: - feature = "CUBE"; - break; - case EGroupByFeatures::GroupingSet: - feature = "GROUPING SETS"; - break; - default: - YQL_ENSURE(false, "Unknown feature"); - } - - Ctx.Error(pos) << "Unnamed expressions are not supported in " << feature << ". Please use '<expr> AS <name>'."; - Ctx.IncrementMonCounter("sql_errors", "GroupBySetNoAliasOrColumn"); - return false; +bool TGroupByClause::AllowUnnamed(TPosition pos, EGroupByFeatures featureContext) { + TStringBuf feature; + switch (featureContext) { + case EGroupByFeatures::Ordinary: + return true; + case EGroupByFeatures::Rollup: + feature = "ROLLUP"; + break; + case EGroupByFeatures::Cube: + feature = "CUBE"; + break; + case EGroupByFeatures::GroupingSet: + feature = "GROUPING SETS"; + break; + default: + YQL_ENSURE(false, "Unknown feature"); + } + + Ctx.Error(pos) << "Unnamed expressions are not supported in " << feature << ". Please use '<expr> AS <name>'."; + Ctx.IncrementMonCounter("sql_errors", "GroupBySetNoAliasOrColumn"); + return false; } TGroupByClause::TGroupingSetFeatures& TGroupByClause::Features() { @@ -7457,43 +7457,43 @@ TString TGroupByClause::GenerateGroupByExprName() { return TStringBuilder() << AutogenerateNamePrefix << GroupSetContext->UnnamedCount++; } -bool TSqlSelect::ValidateLimitOrderByWithSelectOp(TMaybe<TSelectKindPlacement> placement, TStringBuf what) { - if (!placement.Defined()) { - // not in select_op chain - return true; - } - if (!Ctx.AnsiOrderByLimitInUnionAll.Defined()) { - if (!placement->IsLastInSelectOp) { - Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) - << what << " will not be allowed here for ANSI compliant UNION ALL.\n" - << "For details please consult documentation on PRAGMA AnsiOrderByLimitInUnionAll"; - } else { - Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) - << what << " will be applied to last subquery in UNION ALL, not to entire UNION ALL.\n" - << "For ANSI compliant behavior please use PRAGMA AnsiOrderByLimitInUnionAll"; - } - return true; - } - - if (*Ctx.AnsiOrderByLimitInUnionAll && !placement->IsLastInSelectOp) { - Ctx.Error() << what << " within UNION ALL is only allowed after last subquery"; - return false; - } - return true; -} - -bool TSqlSelect::NeedPassLimitOrderByToUnderlyingSelect(TMaybe<TSelectKindPlacement> placement) { - if (!Ctx.AnsiOrderByLimitInUnionAll.Defined() || !*Ctx.AnsiOrderByLimitInUnionAll) { - return true; - } - return !placement.Defined() || !placement->IsLastInSelectOp; -} - -TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_partial& node, TPosition& selectPos, - TMaybe<TSelectKindPlacement> placement) -{ - auto res = SelectKind(node.GetRule_select_kind1(), selectPos, placement); - if (!res) { +bool TSqlSelect::ValidateLimitOrderByWithSelectOp(TMaybe<TSelectKindPlacement> placement, TStringBuf what) { + if (!placement.Defined()) { + // not in select_op chain + return true; + } + if (!Ctx.AnsiOrderByLimitInUnionAll.Defined()) { + if (!placement->IsLastInSelectOp) { + Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) + << what << " will not be allowed here for ANSI compliant UNION ALL.\n" + << "For details please consult documentation on PRAGMA AnsiOrderByLimitInUnionAll"; + } else { + Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) + << what << " will be applied to last subquery in UNION ALL, not to entire UNION ALL.\n" + << "For ANSI compliant behavior please use PRAGMA AnsiOrderByLimitInUnionAll"; + } + return true; + } + + if (*Ctx.AnsiOrderByLimitInUnionAll && !placement->IsLastInSelectOp) { + Ctx.Error() << what << " within UNION ALL is only allowed after last subquery"; + return false; + } + return true; +} + +bool TSqlSelect::NeedPassLimitOrderByToUnderlyingSelect(TMaybe<TSelectKindPlacement> placement) { + if (!Ctx.AnsiOrderByLimitInUnionAll.Defined() || !*Ctx.AnsiOrderByLimitInUnionAll) { + return true; + } + return !placement.Defined() || !placement->IsLastInSelectOp; +} + +TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_partial& node, TPosition& selectPos, + TMaybe<TSelectKindPlacement> placement) +{ + auto res = SelectKind(node.GetRule_select_kind1(), selectPos, placement); + if (!res) { return {}; } TPosition startPos(Ctx.Pos()); @@ -7505,10 +7505,10 @@ TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_par Token(block.GetToken1()); TPosition pos(Ctx.Pos()); - if (!ValidateLimitOrderByWithSelectOp(placement, "LIMIT")) { - return {}; - } - + if (!ValidateLimitOrderByWithSelectOp(placement, "LIMIT")) { + return {}; + } + TSqlExpression takeExpr(Ctx, Mode); auto take = takeExpr.Build(block.GetRule_expr2()); if (!take) { @@ -7530,24 +7530,24 @@ TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_par Ctx.IncrementMonCounter("sql_features", "LimitOffset"); } } - - auto st = BuildSkipTake(pos, skip, take); - if (NeedPassLimitOrderByToUnderlyingSelect(placement)) { - skipTake = st; - } else { - res.SelectOpSkipTake = st; - } - + + auto st = BuildSkipTake(pos, skip, take); + if (NeedPassLimitOrderByToUnderlyingSelect(placement)) { + skipTake = st; + } else { + res.SelectOpSkipTake = st; + } + Ctx.IncrementMonCounter("sql_features", "Limit"); } - - res.Source = BuildSelect(startPos, std::move(res.Source), skipTake); - return res; + + res.Source = BuildSelect(startPos, std::move(res.Source), skipTake); + return res; } -TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind& node, TPosition& selectPos, - TMaybe<TSelectKindPlacement> placement) -{ +TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind& node, TPosition& selectPos, + TMaybe<TSelectKindPlacement> placement) +{ const bool discard = node.HasBlock1(); const bool hasLabel = node.HasBlock3(); if ((discard || hasLabel) && (Mode == NSQLTranslation::ESqlMode::LIMITED_VIEW || Mode == NSQLTranslation::ESqlMode::SUBQUERY)) { @@ -7570,47 +7570,47 @@ TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind& no settings.Label = PureColumnOrNamed(node.GetBlock3().GetRule_pure_column_or_named3(), *this); } - TSelectKindResult res; - if (placement.Defined()) { - if (placement->IsFirstInSelectOp) { - res.Settings.Discard = settings.Discard; - } else if (settings.Discard) { - auto discardPos = Ctx.TokenPosition(node.GetBlock1().GetToken1()); - if (Ctx.AnsiOrderByLimitInUnionAll.Defined() && *Ctx.AnsiOrderByLimitInUnionAll) { - Ctx.Error(discardPos) << "DISCARD within UNION ALL is only allowed before first subquery"; - return {}; - } - Ctx.Warning(discardPos, TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) - << "DISCARD will be ignored here. Please use DISCARD before first subquery in UNION ALL if you want to discard entire UNION ALL result"; - } - - if (placement->IsLastInSelectOp) { - res.Settings.Label = settings.Label; - } else if (!settings.Label.Empty()) { - auto labelPos = Ctx.TokenPosition(node.GetBlock3().GetToken1()); - if (Ctx.AnsiOrderByLimitInUnionAll.Defined() && *Ctx.AnsiOrderByLimitInUnionAll) { - Ctx.Error(labelPos) << "INTO RESULT within UNION ALL is only allowed after last subquery"; - return {}; - } - Ctx.Warning(labelPos, TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) - << "INTO RESULT will be ignored here. Please use INTO RESULT after last subquery in UNION ALL if you want label entire UNION ALL result"; - } - - settings = {}; - } - + TSelectKindResult res; + if (placement.Defined()) { + if (placement->IsFirstInSelectOp) { + res.Settings.Discard = settings.Discard; + } else if (settings.Discard) { + auto discardPos = Ctx.TokenPosition(node.GetBlock1().GetToken1()); + if (Ctx.AnsiOrderByLimitInUnionAll.Defined() && *Ctx.AnsiOrderByLimitInUnionAll) { + Ctx.Error(discardPos) << "DISCARD within UNION ALL is only allowed before first subquery"; + return {}; + } + Ctx.Warning(discardPos, TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) + << "DISCARD will be ignored here. Please use DISCARD before first subquery in UNION ALL if you want to discard entire UNION ALL result"; + } + + if (placement->IsLastInSelectOp) { + res.Settings.Label = settings.Label; + } else if (!settings.Label.Empty()) { + auto labelPos = Ctx.TokenPosition(node.GetBlock3().GetToken1()); + if (Ctx.AnsiOrderByLimitInUnionAll.Defined() && *Ctx.AnsiOrderByLimitInUnionAll) { + Ctx.Error(labelPos) << "INTO RESULT within UNION ALL is only allowed after last subquery"; + return {}; + } + Ctx.Warning(labelPos, TIssuesIds::YQL_LIMIT_ORDER_BY_WITH_UNION) + << "INTO RESULT will be ignored here. Please use INTO RESULT after last subquery in UNION ALL if you want label entire UNION ALL result"; + } + + settings = {}; + } + switch (node.GetBlock2().Alt_case()) { case TRule_select_kind_TBlock2::kAlt1: - res.Source = ProcessCore(node.GetBlock2().GetAlt1().GetRule_process_core1(), settings, selectPos); + res.Source = ProcessCore(node.GetBlock2().GetAlt1().GetRule_process_core1(), settings, selectPos); break; case TRule_select_kind_TBlock2::kAlt2: - res.Source = ReduceCore(node.GetBlock2().GetAlt2().GetRule_reduce_core1(), settings, selectPos); + res.Source = ReduceCore(node.GetBlock2().GetAlt2().GetRule_reduce_core1(), settings, selectPos); break; - case TRule_select_kind_TBlock2::kAlt3: { - res.Source = SelectCore(node.GetBlock2().GetAlt3().GetRule_select_core1(), settings, selectPos, - placement, res.SelectOpOrderBy, res.SelectOpAssumeOrderBy); + case TRule_select_kind_TBlock2::kAlt3: { + res.Source = SelectCore(node.GetBlock2().GetAlt3().GetRule_select_core1(), settings, selectPos, + placement, res.SelectOpOrderBy, res.SelectOpAssumeOrderBy); break; - } + } default: Y_FAIL("You should change implementation according to grammar changes"); } @@ -7618,55 +7618,55 @@ TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind& no return res; } -TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_parenthesis& node, TPosition& selectPos, - TMaybe<TSelectKindPlacement> placement) -{ +TSqlSelect::TSelectKindResult TSqlSelect::SelectKind(const TRule_select_kind_parenthesis& node, TPosition& selectPos, + TMaybe<TSelectKindPlacement> placement) +{ if (node.Alt_case() == TRule_select_kind_parenthesis::kAltSelectKindParenthesis1) { - return SelectKind(node.GetAlt_select_kind_parenthesis1().GetRule_select_kind_partial1(), selectPos, placement); + return SelectKind(node.GetAlt_select_kind_parenthesis1().GetRule_select_kind_partial1(), selectPos, placement); } else { - return SelectKind(node.GetAlt_select_kind_parenthesis2().GetRule_select_kind_partial2(), selectPos, {}); + return SelectKind(node.GetAlt_select_kind_parenthesis2().GetRule_select_kind_partial2(), selectPos, {}); } } template<typename TRule> -TSourcePtr TSqlSelect::Build(const TRule& node, TPosition pos, TSelectKindResult&& first) { +TSourcePtr TSqlSelect::Build(const TRule& node, TPosition pos, TSelectKindResult&& first) { TPosition unionPos = pos; // Position of first select TVector<TSourcePtr> sources; - sources.emplace_back(std::move(first.Source)); - - TVector<TSortSpecificationPtr> orderBy; - TNodePtr skipTake; - TWriteSettings settings; - settings.Discard = first.Settings.Discard; - bool assumeOrderBy = false; - auto blocks = node.GetBlock2(); - for (int i = 0; i < blocks.size(); ++i) { - auto& b = blocks[i]; - const bool last = (i + 1 == blocks.size()); - TSelectKindPlacement placement; - placement.IsLastInSelectOp = last; - - TSelectKindResult next = SelectKind(b.GetRule_select_kind_parenthesis2(), pos, placement); + sources.emplace_back(std::move(first.Source)); + + TVector<TSortSpecificationPtr> orderBy; + TNodePtr skipTake; + TWriteSettings settings; + settings.Discard = first.Settings.Discard; + bool assumeOrderBy = false; + auto blocks = node.GetBlock2(); + for (int i = 0; i < blocks.size(); ++i) { + auto& b = blocks[i]; + const bool last = (i + 1 == blocks.size()); + TSelectKindPlacement placement; + placement.IsLastInSelectOp = last; + + TSelectKindResult next = SelectKind(b.GetRule_select_kind_parenthesis2(), pos, placement); if (!next) { return nullptr; } - - if (last) { - orderBy = next.SelectOpOrderBy; - assumeOrderBy = next.SelectOpAssumeOrderBy; - skipTake = next.SelectOpSkipTake; - settings.Label = next.Settings.Label; - } - + + if (last) { + orderBy = next.SelectOpOrderBy; + assumeOrderBy = next.SelectOpAssumeOrderBy; + skipTake = next.SelectOpSkipTake; + settings.Label = next.Settings.Label; + } + switch (b.GetRule_select_op1().Alt_case()) { case TRule_select_op::kAltSelectOp1: { - const bool isUnionAll = b.GetRule_select_op1().GetAlt_select_op1().HasBlock2(); + const bool isUnionAll = b.GetRule_select_op1().GetAlt_select_op1().HasBlock2(); if (!isUnionAll) { Token(b.GetRule_select_op1().GetAlt_select_op1().GetToken1()); Ctx.Error() << "UNION without quantifier ALL is not supported yet. Did you mean UNION ALL?"; return nullptr; } else { - sources.emplace_back(std::move(next.Source)); + sources.emplace_back(std::move(next.Source)); } break; } @@ -7680,44 +7680,44 @@ TSourcePtr TSqlSelect::Build(const TRule& node, TPosition pos, TSelectKindResult return std::move(sources[0]); } - TSourcePtr result; - if (orderBy) { - result = BuildUnionAll(unionPos, std::move(sources), {}); - - TVector<TNodePtr> groupByExpr; - TVector<TNodePtr> groupBy; - bool compactGroupBy = false; - TNodePtr having; - TWinSpecs winSpecs; - THoppingWindowSpecPtr hoppingWindowSpec; - bool distinct = false; - TVector<TNodePtr> without; - bool stream = false; - - TVector<TNodePtr> terms; - terms.push_back(BuildColumn(unionPos, "*", "")); - - result = BuildSelectCore(Ctx, unionPos, std::move(result), groupByExpr, groupBy, compactGroupBy, - assumeOrderBy, orderBy, having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), - distinct, std::move(without), stream, settings); - } else { - result = BuildUnionAll(unionPos, std::move(sources), settings); - } - - if (skipTake || orderBy) { - result = BuildSelect(unionPos, std::move(result), skipTake); - } - return result; + TSourcePtr result; + if (orderBy) { + result = BuildUnionAll(unionPos, std::move(sources), {}); + + TVector<TNodePtr> groupByExpr; + TVector<TNodePtr> groupBy; + bool compactGroupBy = false; + TNodePtr having; + TWinSpecs winSpecs; + THoppingWindowSpecPtr hoppingWindowSpec; + bool distinct = false; + TVector<TNodePtr> without; + bool stream = false; + + TVector<TNodePtr> terms; + terms.push_back(BuildColumn(unionPos, "*", "")); + + result = BuildSelectCore(Ctx, unionPos, std::move(result), groupByExpr, groupBy, compactGroupBy, + assumeOrderBy, orderBy, having, std::move(winSpecs), hoppingWindowSpec, std::move(terms), + distinct, std::move(without), stream, settings); + } else { + result = BuildUnionAll(unionPos, std::move(sources), settings); + } + + if (skipTake || orderBy) { + result = BuildSelect(unionPos, std::move(result), skipTake); + } + return result; } TSourcePtr TSqlSelect::Build(const TRule_select_stmt& node, TPosition& selectPos) { - TMaybe<TSelectKindPlacement> placement; - if (!node.GetBlock2().empty()) { - placement.ConstructInPlace(); - placement->IsFirstInSelectOp = true; - } - - auto res = SelectKind(node.GetRule_select_kind_parenthesis1(), selectPos, placement); + TMaybe<TSelectKindPlacement> placement; + if (!node.GetBlock2().empty()) { + placement.ConstructInPlace(); + placement->IsFirstInSelectOp = true; + } + + auto res = SelectKind(node.GetRule_select_kind_parenthesis1(), selectPos, placement); if (!res) { return nullptr; } @@ -7726,13 +7726,13 @@ TSourcePtr TSqlSelect::Build(const TRule_select_stmt& node, TPosition& selectPos } TSourcePtr TSqlSelect::Build(const TRule_select_unparenthesized_stmt& node, TPosition& selectPos) { - TMaybe<TSelectKindPlacement> placement; - if (!node.GetBlock2().empty()) { - placement.ConstructInPlace(); - placement->IsFirstInSelectOp = true; - } - - auto res = SelectKind(node.GetRule_select_kind_partial1(), selectPos, placement); + TMaybe<TSelectKindPlacement> placement; + if (!node.GetBlock2().empty()) { + placement.ConstructInPlace(); + placement->IsFirstInSelectOp = true; + } + + auto res = SelectKind(node.GetRule_select_kind_partial1(), selectPos, placement); if (!res) { return nullptr; } @@ -7740,85 +7740,85 @@ TSourcePtr TSqlSelect::Build(const TRule_select_unparenthesized_stmt& node, TPos return Build(node, selectPos, std::move(res)); } -TSourcePtr TSqlValues::Build(const TRule_values_stmt& node, TPosition& valuesPos, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos) { - Token(node.GetToken1()); - valuesPos = Ctx.Pos(); - - TVector<TVector<TNodePtr>> rows; - const auto& rowList = node.GetRule_values_source_row_list2(); - if (!BuildRows(rowList, rows)) { - return nullptr; - } - - YQL_ENSURE(!rows.empty()); - const size_t columnsCount = rows.back().size(); - if (derivedColumns.size() > columnsCount) { - Ctx.Error(derivedColumnsPos) << "Derived column list size exceeds column count in VALUES"; - return nullptr; - } - - auto columns = derivedColumns; - if (Ctx.WarnUnnamedColumns && columns.size() < columnsCount) { - Ctx.Warning(valuesPos, TIssuesIds::YQL_UNNAMED_COLUMN) - << "Autogenerated column names column" << columns.size() << "...column" << columnsCount - 1 << " will be used here"; - } - - while (columns.size() < columnsCount) { - columns.push_back(TStringBuilder() << "column" << columns.size()); - } - - TVector<TNodePtr> labels; - for (size_t i = 0; i < columnsCount; ++i) { - labels.push_back(BuildQuotedAtom(derivedColumnsPos, columns[i])); - } - - TVector<TNodePtr> items; - for (auto& row : rows) { - YQL_ENSURE(!row.empty()); - YQL_ENSURE(row.size() == columnsCount); - items.push_back(BuildOrderedStructure(row.front()->GetPos(), row, labels)); - } - auto list = new TCallNodeImpl(valuesPos, "AsList", items); - list = new TCallNodeImpl(valuesPos, "PersistableRepr", { list }); - list = new TCallNodeImpl(valuesPos, "AssumeColumnOrder", { list, BuildTuple(valuesPos, labels) }); - auto result = BuildNodeSource(valuesPos, list, false); - result->AllColumns(); - return result; -} - -bool TSqlValues::BuildRows(const TRule_values_source_row_list& node, TVector<TVector<TNodePtr>>& rows) { - rows = TVector<TVector<TNodePtr>> {{}}; - - - if (!BuildRow(node.GetRule_values_source_row1(), rows.back())) { - return false; - } - - const size_t rowSize = rows.back().size(); - - for (const auto& valuesSourceRow: node.GetBlock2()) { - rows.push_back({}); - if (!BuildRow(valuesSourceRow.GetRule_values_source_row2(), rows.back())) { - return false; - } - if (rows.back().size() != rowSize) { - Token(valuesSourceRow.GetRule_values_source_row2().GetToken1()); - Error() << "All VALUES items should have same size: expecting " << rowSize << ", got " << rows.back().size(); - return false; - } - } - return true; -} - -bool TSqlValues::BuildRow(const TRule_values_source_row& inRow, TVector<TNodePtr>& outRow) { - TSqlExpression sqlExpr(Ctx, Mode); - return ExprList(sqlExpr, outRow, inRow.GetRule_expr_list2()); -} - -class TSqlIntoValues: public TSqlValues { +TSourcePtr TSqlValues::Build(const TRule_values_stmt& node, TPosition& valuesPos, const TVector<TString>& derivedColumns, TPosition derivedColumnsPos) { + Token(node.GetToken1()); + valuesPos = Ctx.Pos(); + + TVector<TVector<TNodePtr>> rows; + const auto& rowList = node.GetRule_values_source_row_list2(); + if (!BuildRows(rowList, rows)) { + return nullptr; + } + + YQL_ENSURE(!rows.empty()); + const size_t columnsCount = rows.back().size(); + if (derivedColumns.size() > columnsCount) { + Ctx.Error(derivedColumnsPos) << "Derived column list size exceeds column count in VALUES"; + return nullptr; + } + + auto columns = derivedColumns; + if (Ctx.WarnUnnamedColumns && columns.size() < columnsCount) { + Ctx.Warning(valuesPos, TIssuesIds::YQL_UNNAMED_COLUMN) + << "Autogenerated column names column" << columns.size() << "...column" << columnsCount - 1 << " will be used here"; + } + + while (columns.size() < columnsCount) { + columns.push_back(TStringBuilder() << "column" << columns.size()); + } + + TVector<TNodePtr> labels; + for (size_t i = 0; i < columnsCount; ++i) { + labels.push_back(BuildQuotedAtom(derivedColumnsPos, columns[i])); + } + + TVector<TNodePtr> items; + for (auto& row : rows) { + YQL_ENSURE(!row.empty()); + YQL_ENSURE(row.size() == columnsCount); + items.push_back(BuildOrderedStructure(row.front()->GetPos(), row, labels)); + } + auto list = new TCallNodeImpl(valuesPos, "AsList", items); + list = new TCallNodeImpl(valuesPos, "PersistableRepr", { list }); + list = new TCallNodeImpl(valuesPos, "AssumeColumnOrder", { list, BuildTuple(valuesPos, labels) }); + auto result = BuildNodeSource(valuesPos, list, false); + result->AllColumns(); + return result; +} + +bool TSqlValues::BuildRows(const TRule_values_source_row_list& node, TVector<TVector<TNodePtr>>& rows) { + rows = TVector<TVector<TNodePtr>> {{}}; + + + if (!BuildRow(node.GetRule_values_source_row1(), rows.back())) { + return false; + } + + const size_t rowSize = rows.back().size(); + + for (const auto& valuesSourceRow: node.GetBlock2()) { + rows.push_back({}); + if (!BuildRow(valuesSourceRow.GetRule_values_source_row2(), rows.back())) { + return false; + } + if (rows.back().size() != rowSize) { + Token(valuesSourceRow.GetRule_values_source_row2().GetToken1()); + Error() << "All VALUES items should have same size: expecting " << rowSize << ", got " << rows.back().size(); + return false; + } + } + return true; +} + +bool TSqlValues::BuildRow(const TRule_values_source_row& inRow, TVector<TNodePtr>& outRow) { + TSqlExpression sqlExpr(Ctx, Mode); + return ExprList(sqlExpr, outRow, inRow.GetRule_expr_list2()); +} + +class TSqlIntoValues: public TSqlValues { public: TSqlIntoValues(TContext& ctx, NSQLTranslation::ESqlMode mode) - : TSqlValues(ctx, mode) + : TSqlValues(ctx, mode) { } @@ -7854,8 +7854,8 @@ TSourcePtr TSqlIntoValues::ValuesSource(const TRule_values_source& node, TVector switch (node.Alt_case()) { case TRule_values_source::kAltValuesSource1: { TVector<TVector<TNodePtr>> rows {{}}; - const auto& rowList = node.GetAlt_values_source1().GetRule_values_stmt1().GetRule_values_source_row_list2(); - if (!BuildRows(rowList, rows)) { + const auto& rowList = node.GetAlt_values_source1().GetRule_values_stmt1().GetRule_values_source_row_list2(); + if (!BuildRows(rowList, rows)) { return nullptr; } return BuildWriteValues(pos, operationName, columnsHint, rows); @@ -7995,11 +7995,11 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { } case TRule_simple_table_ref_TBlock1::AltCase::kAlt2: { auto at = tableRef.GetBlock1().GetAlt2().HasBlock1(); - TString name; - if (!NamedNodeImpl(tableRef.GetBlock1().GetAlt2().GetRule_bind_parameter2(), name, *this)) { - return nullptr; - } - auto named = GetNamedNode(name); + TString name; + if (!NamedNodeImpl(tableRef.GetBlock1().GetAlt2().GetRule_bind_parameter2(), name, *this)) { + return nullptr; + } + auto named = GetNamedNode(name); if (!named) { return nullptr; } @@ -8205,10 +8205,10 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda TNodePtr action; switch (callAction.GetBlock1().GetAltCase()) { case TRule_call_action_TBlock1::kAlt1: { - TString bindName; - if (!NamedNodeImpl(callAction.GetBlock1().GetAlt1().GetRule_bind_parameter1(), bindName, *this)) { - return nullptr; - } + TString bindName; + if (!NamedNodeImpl(callAction.GetBlock1().GetAlt1().GetRule_bind_parameter1(), bindName, *this)) { + return nullptr; + } action = GetNamedNode(bindName); if (!action) { return nullptr; @@ -8229,7 +8229,7 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda values.push_back(action); values.push_back(new TAstAtomNodeImpl(Ctx.Pos(), "world", TNodeFlags::Default)); - TSqlExpression sqlExpr(Ctx, Mode); + TSqlExpression sqlExpr(Ctx, Mode); if (callAction.HasBlock3() && !ExprList(sqlExpr, values, callAction.GetBlock3().GetRule_expr_list1())) { return nullptr; } @@ -8256,14 +8256,14 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda Ctx.AllScopes.push_back(Ctx.Scoped); *Ctx.Scoped = *saveScoped; Ctx.Scoped->Local = TScopedState::TLocal{}; - Ctx.ScopeLevel++; + Ctx.ScopeLevel++; TSqlQuery query(Ctx, Ctx.Settings.Mode, false); TBlocks innerBlocks; - const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, body); - auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; - WarnUnusedNodes(); - Ctx.ScopeLevel--; + const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, body); + auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; + WarnUnusedNodes(); + Ctx.ScopeLevel--; Ctx.Scoped = saveScoped; if (!ret) { @@ -8340,8 +8340,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } case TRule_sql_stmt_core::kAltSqlStmtCore3: { Ctx.BodyPart(); - TVector<TSymbolNameWithPos> names; - auto nodeExpr = NamedNode(core.GetAlt_sql_stmt_core3().GetRule_named_nodes_stmt1(), names); + TVector<TSymbolNameWithPos> names; + auto nodeExpr = NamedNode(core.GetAlt_sql_stmt_core3().GetRule_named_nodes_stmt1(), names); if (!nodeExpr) { return false; } @@ -8359,9 +8359,9 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } } else { if (names.size() > 1) { - auto tupleRes = BuildTupleResult(nodeExpr, names.size()); + auto tupleRes = BuildTupleResult(nodeExpr, names.size()); for (size_t i = 0; i < names.size(); ++i) { - nodes.push_back(nodeExpr->Y("Nth", tupleRes, nodeExpr->Q(ToString(i)))); + nodes.push_back(nodeExpr->Y("Nth", tupleRes, nodeExpr->Q(ToString(i)))); } } else { nodes.push_back(std::move(nodeExpr)); @@ -8369,7 +8369,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } for (size_t i = 0; i < names.size(); ++i) { - PushNamedNode(names[i].Pos, names[i].Name, nodes[i]); + PushNamedNode(names[i].Pos, names[i].Name, nodes[i]); } break; } @@ -8474,7 +8474,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } break; case TRule_sql_stmt_core::kAltSqlStmtCore13: - if (!ImportStatement(core.GetAlt_sql_stmt_core13().GetRule_import_stmt1())) { + if (!ImportStatement(core.GetAlt_sql_stmt_core13().GetRule_import_stmt1())) { return false; } break; @@ -8517,7 +8517,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } case TRule_sql_stmt_core::kAltSqlStmtCore17: { Ctx.BodyPart(); - if (!DefineActionOrSubqueryStatement(core.GetAlt_sql_stmt_core17().GetRule_define_action_or_subquery_stmt1())) { + if (!DefineActionOrSubqueryStatement(core.GetAlt_sql_stmt_core17().GetRule_define_action_or_subquery_stmt1())) { return false; } @@ -8543,220 +8543,220 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& blocks.push_back(node); break; } - case TRule_sql_stmt_core::kAltSqlStmtCore20: { - Ctx.BodyPart(); - TSqlValues values(Ctx, Mode); - TPosition pos; - auto source = values.Build(core.GetAlt_sql_stmt_core20().GetRule_values_stmt1(), pos, {}, TPosition()); - if (!source) { - return false; - } - blocks.emplace_back(BuildSelectResult(pos, std::move(source), - Mode != NSQLTranslation::ESqlMode::LIMITED_VIEW && Mode != NSQLTranslation::ESqlMode::SUBQUERY, Mode == NSQLTranslation::ESqlMode::SUBQUERY, - Ctx.Scoped)); - break; - } - case TRule_sql_stmt_core::kAltSqlStmtCore21: { - // create_user_stmt: CREATE USER role_name create_user_option?; - Ctx.BodyPart(); - auto& node = core.GetAlt_sql_stmt_core21().GetRule_create_user_stmt1(); - - Ctx.Token(node.GetToken1()); - const TPosition pos = Ctx.Pos(); - - TString service = Ctx.Scoped->CurrService; - TDeferredAtom cluster = Ctx.Scoped->CurrCluster; - if (cluster.Empty()) { - Error() << "USE statement is missing - no default cluster is selected"; - return false; - } - - TDeferredAtom roleName; - bool allowSystemRoles = false; - if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { - return false; - } - - TMaybe<TRoleParameters> roleParams; - if (node.HasBlock4()) { - roleParams.ConstructInPlace(); - if (!RoleParameters(node.GetBlock4().GetRule_create_user_option1(), *roleParams)) { - return false; - } - } - - AddStatementToBlocks(blocks, BuildCreateUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped)); - break; - } - case TRule_sql_stmt_core::kAltSqlStmtCore22: { - // alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); - Ctx.BodyPart(); - auto& node = core.GetAlt_sql_stmt_core22().GetRule_alter_user_stmt1(); - - Ctx.Token(node.GetToken1()); - const TPosition pos = Ctx.Pos(); - - TString service = Ctx.Scoped->CurrService; - TDeferredAtom cluster = Ctx.Scoped->CurrCluster; - if (cluster.Empty()) { - Error() << "USE statement is missing - no default cluster is selected"; - return false; - } - - TDeferredAtom roleName; - { - bool allowSystemRoles = true; - if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { - return false; - } - } - - TNodePtr stmt; - switch (node.GetBlock4().Alt_case()) { - case TRule_alter_user_stmt_TBlock4::kAlt1: { - TRoleParameters roleParams; - if (!RoleParameters(node.GetBlock4().GetAlt1().GetRule_create_user_option2(), roleParams)) { - return false; - } - stmt = BuildAlterUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped); - break; - } - case TRule_alter_user_stmt_TBlock4::kAlt2: { - TDeferredAtom tgtRoleName; - bool allowSystemRoles = false; - if (!RoleNameClause(node.GetBlock4().GetAlt2().GetRule_role_name3(), tgtRoleName, allowSystemRoles)) { - return false; - } - stmt = BuildRenameUser(pos, service, cluster, roleName, tgtRoleName,Ctx.Scoped); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - AddStatementToBlocks(blocks, stmt); - break; - } - case TRule_sql_stmt_core::kAltSqlStmtCore23: { - // create_group_stmt: CREATE GROUP role_name; - Ctx.BodyPart(); - auto& node = core.GetAlt_sql_stmt_core23().GetRule_create_group_stmt1(); - - Ctx.Token(node.GetToken1()); - const TPosition pos = Ctx.Pos(); - - TString service = Ctx.Scoped->CurrService; - TDeferredAtom cluster = Ctx.Scoped->CurrCluster; - if (cluster.Empty()) { - Error() << "USE statement is missing - no default cluster is selected"; - return false; - } - - TDeferredAtom roleName; - bool allowSystemRoles = false; - if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { - return false; - } - - AddStatementToBlocks(blocks, BuildCreateGroup(pos, service, cluster, roleName, Ctx.Scoped)); - break; - } - case TRule_sql_stmt_core::kAltSqlStmtCore24: { - // alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); - Ctx.BodyPart(); - auto& node = core.GetAlt_sql_stmt_core24().GetRule_alter_group_stmt1(); - - Ctx.Token(node.GetToken1()); - const TPosition pos = Ctx.Pos(); - - TString service = Ctx.Scoped->CurrService; - TDeferredAtom cluster = Ctx.Scoped->CurrCluster; - if (cluster.Empty()) { - Error() << "USE statement is missing - no default cluster is selected"; - return false; - } - - TDeferredAtom roleName; - { - bool allowSystemRoles = true; - if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { - return false; - } - } - - TNodePtr stmt; - switch (node.GetBlock4().Alt_case()) { - case TRule_alter_group_stmt_TBlock4::kAlt1: { - auto& addDropNode = node.GetBlock4().GetAlt1(); - const bool isDrop = addDropNode.GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DROP; - TVector<TDeferredAtom> roles; - bool allowSystemRoles = false; - roles.emplace_back(); - if (!RoleNameClause(addDropNode.GetRule_role_name3(), roles.back(), allowSystemRoles)) { - return false; - } - - for (auto& item : addDropNode.GetBlock4()) { - roles.emplace_back(); - if (!RoleNameClause(item.GetRule_role_name2(), roles.back(), allowSystemRoles)) { - return false; - } - } - - stmt = BuildAlterGroup(pos, service, cluster, roleName, roles, isDrop, Ctx.Scoped); - break; - } - case TRule_alter_group_stmt_TBlock4::kAlt2: { - TDeferredAtom tgtRoleName; - bool allowSystemRoles = false; - if (!RoleNameClause(node.GetBlock4().GetAlt2().GetRule_role_name3(), tgtRoleName, allowSystemRoles)) { - return false; - } - stmt = BuildRenameGroup(pos, service, cluster, roleName, tgtRoleName, Ctx.Scoped); - break; - } - default: - Y_FAIL("You should change implementation according to grammar changes"); - } - - AddStatementToBlocks(blocks, stmt); - break; - } - case TRule_sql_stmt_core::kAltSqlStmtCore25: { - // drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; - Ctx.BodyPart(); - auto& node = core.GetAlt_sql_stmt_core25().GetRule_drop_role_stmt1(); - - Ctx.Token(node.GetToken1()); - const TPosition pos = Ctx.Pos(); - - TString service = Ctx.Scoped->CurrService; - TDeferredAtom cluster = Ctx.Scoped->CurrCluster; - if (cluster.Empty()) { - Error() << "USE statement is missing - no default cluster is selected"; - return false; - } - - const bool isUser = node.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_USER; - const bool force = node.HasBlock3(); - - TVector<TDeferredAtom> roles; - bool allowSystemRoles = true; - roles.emplace_back(); - if (!RoleNameClause(node.GetRule_role_name4(), roles.back(), allowSystemRoles)) { - return false; - } - - for (auto& item : node.GetBlock5()) { - roles.emplace_back(); - if (!RoleNameClause(item.GetRule_role_name2(), roles.back(), allowSystemRoles)) { - return false; - } - } - - AddStatementToBlocks(blocks, BuildDropRoles(pos, service, cluster, roles, isUser, force, Ctx.Scoped)); - break; - } + case TRule_sql_stmt_core::kAltSqlStmtCore20: { + Ctx.BodyPart(); + TSqlValues values(Ctx, Mode); + TPosition pos; + auto source = values.Build(core.GetAlt_sql_stmt_core20().GetRule_values_stmt1(), pos, {}, TPosition()); + if (!source) { + return false; + } + blocks.emplace_back(BuildSelectResult(pos, std::move(source), + Mode != NSQLTranslation::ESqlMode::LIMITED_VIEW && Mode != NSQLTranslation::ESqlMode::SUBQUERY, Mode == NSQLTranslation::ESqlMode::SUBQUERY, + Ctx.Scoped)); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore21: { + // create_user_stmt: CREATE USER role_name create_user_option?; + Ctx.BodyPart(); + auto& node = core.GetAlt_sql_stmt_core21().GetRule_create_user_stmt1(); + + Ctx.Token(node.GetToken1()); + const TPosition pos = Ctx.Pos(); + + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + if (cluster.Empty()) { + Error() << "USE statement is missing - no default cluster is selected"; + return false; + } + + TDeferredAtom roleName; + bool allowSystemRoles = false; + if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { + return false; + } + + TMaybe<TRoleParameters> roleParams; + if (node.HasBlock4()) { + roleParams.ConstructInPlace(); + if (!RoleParameters(node.GetBlock4().GetRule_create_user_option1(), *roleParams)) { + return false; + } + } + + AddStatementToBlocks(blocks, BuildCreateUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped)); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore22: { + // alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); + Ctx.BodyPart(); + auto& node = core.GetAlt_sql_stmt_core22().GetRule_alter_user_stmt1(); + + Ctx.Token(node.GetToken1()); + const TPosition pos = Ctx.Pos(); + + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + if (cluster.Empty()) { + Error() << "USE statement is missing - no default cluster is selected"; + return false; + } + + TDeferredAtom roleName; + { + bool allowSystemRoles = true; + if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { + return false; + } + } + + TNodePtr stmt; + switch (node.GetBlock4().Alt_case()) { + case TRule_alter_user_stmt_TBlock4::kAlt1: { + TRoleParameters roleParams; + if (!RoleParameters(node.GetBlock4().GetAlt1().GetRule_create_user_option2(), roleParams)) { + return false; + } + stmt = BuildAlterUser(pos, service, cluster, roleName, roleParams, Ctx.Scoped); + break; + } + case TRule_alter_user_stmt_TBlock4::kAlt2: { + TDeferredAtom tgtRoleName; + bool allowSystemRoles = false; + if (!RoleNameClause(node.GetBlock4().GetAlt2().GetRule_role_name3(), tgtRoleName, allowSystemRoles)) { + return false; + } + stmt = BuildRenameUser(pos, service, cluster, roleName, tgtRoleName,Ctx.Scoped); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + AddStatementToBlocks(blocks, stmt); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore23: { + // create_group_stmt: CREATE GROUP role_name; + Ctx.BodyPart(); + auto& node = core.GetAlt_sql_stmt_core23().GetRule_create_group_stmt1(); + + Ctx.Token(node.GetToken1()); + const TPosition pos = Ctx.Pos(); + + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + if (cluster.Empty()) { + Error() << "USE statement is missing - no default cluster is selected"; + return false; + } + + TDeferredAtom roleName; + bool allowSystemRoles = false; + if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { + return false; + } + + AddStatementToBlocks(blocks, BuildCreateGroup(pos, service, cluster, roleName, Ctx.Scoped)); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore24: { + // alter_group_stmt: ALTER GROUP role_name ((ADD|DROP) USER role_name (COMMA role_name)* COMMA? | RENAME TO role_name); + Ctx.BodyPart(); + auto& node = core.GetAlt_sql_stmt_core24().GetRule_alter_group_stmt1(); + + Ctx.Token(node.GetToken1()); + const TPosition pos = Ctx.Pos(); + + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + if (cluster.Empty()) { + Error() << "USE statement is missing - no default cluster is selected"; + return false; + } + + TDeferredAtom roleName; + { + bool allowSystemRoles = true; + if (!RoleNameClause(node.GetRule_role_name3(), roleName, allowSystemRoles)) { + return false; + } + } + + TNodePtr stmt; + switch (node.GetBlock4().Alt_case()) { + case TRule_alter_group_stmt_TBlock4::kAlt1: { + auto& addDropNode = node.GetBlock4().GetAlt1(); + const bool isDrop = addDropNode.GetToken1().GetId() == SQLv1LexerTokens::TOKEN_DROP; + TVector<TDeferredAtom> roles; + bool allowSystemRoles = false; + roles.emplace_back(); + if (!RoleNameClause(addDropNode.GetRule_role_name3(), roles.back(), allowSystemRoles)) { + return false; + } + + for (auto& item : addDropNode.GetBlock4()) { + roles.emplace_back(); + if (!RoleNameClause(item.GetRule_role_name2(), roles.back(), allowSystemRoles)) { + return false; + } + } + + stmt = BuildAlterGroup(pos, service, cluster, roleName, roles, isDrop, Ctx.Scoped); + break; + } + case TRule_alter_group_stmt_TBlock4::kAlt2: { + TDeferredAtom tgtRoleName; + bool allowSystemRoles = false; + if (!RoleNameClause(node.GetBlock4().GetAlt2().GetRule_role_name3(), tgtRoleName, allowSystemRoles)) { + return false; + } + stmt = BuildRenameGroup(pos, service, cluster, roleName, tgtRoleName, Ctx.Scoped); + break; + } + default: + Y_FAIL("You should change implementation according to grammar changes"); + } + + AddStatementToBlocks(blocks, stmt); + break; + } + case TRule_sql_stmt_core::kAltSqlStmtCore25: { + // drop_role_stmt: DROP (USER|GROUP) (IF EXISTS)? role_name (COMMA role_name)* COMMA?; + Ctx.BodyPart(); + auto& node = core.GetAlt_sql_stmt_core25().GetRule_drop_role_stmt1(); + + Ctx.Token(node.GetToken1()); + const TPosition pos = Ctx.Pos(); + + TString service = Ctx.Scoped->CurrService; + TDeferredAtom cluster = Ctx.Scoped->CurrCluster; + if (cluster.Empty()) { + Error() << "USE statement is missing - no default cluster is selected"; + return false; + } + + const bool isUser = node.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_USER; + const bool force = node.HasBlock3(); + + TVector<TDeferredAtom> roles; + bool allowSystemRoles = true; + roles.emplace_back(); + if (!RoleNameClause(node.GetRule_role_name4(), roles.back(), allowSystemRoles)) { + return false; + } + + for (auto& item : node.GetBlock5()) { + roles.emplace_back(); + if (!RoleNameClause(item.GetRule_role_name2(), roles.back(), allowSystemRoles)) { + return false; + } + } + + AddStatementToBlocks(blocks, BuildDropRoles(pos, service, cluster, roles, isUser, force, Ctx.Scoped)); + break; + } default: Ctx.IncrementMonCounter("sql_errors", "UnknownStatement" + internalStatementName); AltNotImplemented("sql_stmt_core", core); @@ -8771,15 +8771,15 @@ bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) { TNodePtr defaultValue; if (stmt.HasBlock5()) { TSqlExpression sqlExpr(Ctx, Mode); - auto exprOrId = sqlExpr.LiteralExpr(stmt.GetBlock5().GetRule_literal_value2()); - if (!exprOrId) { + auto exprOrId = sqlExpr.LiteralExpr(stmt.GetBlock5().GetRule_literal_value2()); + if (!exprOrId) { return false; } - if (!exprOrId->Expr) { - Ctx.Error() << "Identifier is not expected here"; - return false; - } - defaultValue = exprOrId->Expr; + if (!exprOrId->Expr) { + Ctx.Error() << "Identifier is not expected here"; + return false; + } + defaultValue = exprOrId->Expr; } if (defaultValue) { Error() << "DEFAULT value not supported yet"; @@ -8789,22 +8789,22 @@ bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) { Error() << "DECLARE statement should be in beginning of query, but it's possible to use PRAGMA or USE before it"; return false; } - - TString varName; - if (!NamedNodeImpl(stmt.GetRule_bind_parameter2(), varName, *this)) { - return false; - } + + TString varName; + if (!NamedNodeImpl(stmt.GetRule_bind_parameter2(), varName, *this)) { + return false; + } const auto varPos = Ctx.Pos(); const auto typeNode = TypeNode(stmt.GetRule_type_name4()); if (!typeNode) { return false; } - if (IsAnonymousName(varName)) { - Ctx.Error(varPos) << "Can not use anonymous name '" << varName << "' in DECLARE statement"; - return false; - } - varName = PushNamedAtom(varPos, varName); - Ctx.DeclareVariable(varName, typeNode); + if (IsAnonymousName(varName)) { + Ctx.Error(varPos) << "Can not use anonymous name '" << varName << "' in DECLARE statement"; + return false; + } + varName = PushNamedAtom(varPos, varName); + Ctx.DeclareVariable(varName, typeNode); return true; } @@ -8814,17 +8814,17 @@ bool TSqlQuery::ExportStatement(const TRule_export_stmt& stmt) { return false; } - TVector<TSymbolNameWithPos> bindNames; + TVector<TSymbolNameWithPos> bindNames; if (!BindList(stmt.GetRule_bind_parameter_list2(), bindNames)) { return false; } - - for (auto& bindName : bindNames) { - if (!Ctx.AddExport(bindName.Pos, bindName.Name)) { - return false; - } - } - return true; + + for (auto& bindName : bindNames) { + if (!Ctx.AddExport(bindName.Pos, bindName.Name)) { + return false; + } + } + return true; } bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTableParameters& params) { @@ -9135,7 +9135,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success success = false; const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this); const TString& lowerPrefix = to_lower(prefix); - const TString pragma(Id(stmt.GetRule_an_id3(), *this)); + const TString pragma(Id(stmt.GetRule_an_id3(), *this)); TString normalizedPragma(pragma); TMaybe<TIssue> normalizeError = NormalizeName(Ctx.Pos(), normalizedPragma); if (!normalizeError.Empty()) { @@ -9158,17 +9158,17 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } const bool withConfigure = prefix || normalizedPragma == "file" || normalizedPragma == "folder" || normalizedPragma == "udf"; - static const THashSet<TStringBuf> lexicalScopePragmas = {"classicdivision", "strictjoinkeytypes", "disablestrictjoinkeytypes"}; - const bool hasLexicalScope = withConfigure || lexicalScopePragmas.contains(normalizedPragma); + static const THashSet<TStringBuf> lexicalScopePragmas = {"classicdivision", "strictjoinkeytypes", "disablestrictjoinkeytypes"}; + const bool hasLexicalScope = withConfigure || lexicalScopePragmas.contains(normalizedPragma); for (auto pragmaValue : pragmaValues) { if (pragmaValue->HasAlt_pragma_value3()) { - auto value = Token(pragmaValue->GetAlt_pragma_value3().GetToken1()); - auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value); - if (!parsed) { - return {}; - } - - values.push_back(TDeferredAtom(Ctx.Pos(), parsed->Content)); + auto value = Token(pragmaValue->GetAlt_pragma_value3().GetToken1()); + auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value); + if (!parsed) { + return {}; + } + + values.push_back(TDeferredAtom(Ctx.Pos(), parsed->Content)); } else if (pragmaValue->HasAlt_pragma_value2() && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2() @@ -9177,15 +9177,15 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success pragmaValueDefault = true; } else if (withConfigure && pragmaValue->HasAlt_pragma_value5()) { - TString bindName; - if (!NamedNodeImpl(pragmaValue->GetAlt_pragma_value5().GetRule_bind_parameter1(), bindName, *this)) { - return {}; - } + TString bindName; + if (!NamedNodeImpl(pragmaValue->GetAlt_pragma_value5().GetRule_bind_parameter1(), bindName, *this)) { + return {}; + } auto namedNode = GetNamedNode(bindName); if (!namedNode) { return {}; } - + TDeferredAtom atom; MakeTableFromExpression(Ctx, namedNode, atom); values.push_back(atom); @@ -9230,20 +9230,20 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } ui32 version = 0; - TString versionString; - TString packageName; - if (!values[0].GetLiteral(packageName, Ctx) || !values[1].GetLiteral(versionString, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - if (!PackageVersionFromString(versionString, version)) { + TString versionString; + TString packageName; + if (!values[0].GetLiteral(packageName, Ctx) || !values[1].GetLiteral(versionString, Ctx)) { + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + if (!PackageVersionFromString(versionString, version)) { Error() << "Unable to parse package version, possible values 0, 1, draft, release"; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } - Ctx.SetPackageVersion(packageName, version); + Ctx.SetPackageVersion(packageName, version); Ctx.IncrementMonCounter("sql_pragma", "PackageVersion"); success = true; return BuildPragma(Ctx.Pos(), TString(ConfigProviderName), "SetPackageVersion", TVector<TDeferredAtom>{ values[0], TDeferredAtom(values[1].Build()->GetPos(), ToString(version)) }, false); @@ -9273,12 +9273,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success return{}; } - TString alias; + TString alias; if (!values[0].GetLiteral(alias, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return{}; - } - + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return{}; + } + TMaybe<TString> file; if (values.size() == 2) { file.ConstructInPlace(); @@ -9299,24 +9299,24 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.PragmaAutoCommit = true; Ctx.IncrementMonCounter("sql_pragma", "AutoCommit"); } else if (normalizedPragma == "tablepathprefix") { - TString value; - TMaybe<TString> arg; - - if (values.size() == 1 || values.size() == 2) { - if (!values.front().GetLiteral(value, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + TString value; + TMaybe<TString> arg; + + if (values.size() == 1 || values.size() == 2) { + if (!values.front().GetLiteral(value, Ctx)) { + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } - - if (values.size() == 2) { - arg = value; - if (!values.back().GetLiteral(value, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - } - - if (!Ctx.SetPathPrefix(value, arg)) { + + if (values.size() == 2) { + arg = value; + if (!values.back().GetLiteral(value, Ctx)) { + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + } + + if (!Ctx.SetPathPrefix(value, arg)) { return {}; } } else { @@ -9328,42 +9328,42 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.IncrementMonCounter("sql_pragma", "PathPrefix"); } else if (normalizedPragma == "groupbylimit") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByLimit)) { - Error() << "Expected unsigned integer literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByLimit)) { + Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.IncrementMonCounter("sql_pragma", "GroupByLimit"); } else if (normalizedPragma == "groupbycubelimit") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByCubeLimit)) { - Error() << "Expected unsigned integer literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByCubeLimit)) { + Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.IncrementMonCounter("sql_pragma", "GroupByCubeLimit"); - } else if (normalizedPragma == "simplecolumns") { + } else if (normalizedPragma == "simplecolumns") { Ctx.SimpleColumns = true; Ctx.IncrementMonCounter("sql_pragma", "SimpleColumns"); - } else if (normalizedPragma == "disablesimplecolumns") { + } else if (normalizedPragma == "disablesimplecolumns") { Ctx.SimpleColumns = false; Ctx.IncrementMonCounter("sql_pragma", "DisableSimpleColumns"); - } else if (normalizedPragma == "coalescejoinkeysonqualifiedall") { - Ctx.CoalesceJoinKeysOnQualifiedAll = true; - Ctx.IncrementMonCounter("sql_pragma", "CoalesceJoinKeysOnQualifiedAll"); - } else if (normalizedPragma == "disablecoalescejoinkeysonqualifiedall") { - Ctx.CoalesceJoinKeysOnQualifiedAll = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableCoalesceJoinKeysOnQualifiedAll"); + } else if (normalizedPragma == "coalescejoinkeysonqualifiedall") { + Ctx.CoalesceJoinKeysOnQualifiedAll = true; + Ctx.IncrementMonCounter("sql_pragma", "CoalesceJoinKeysOnQualifiedAll"); + } else if (normalizedPragma == "disablecoalescejoinkeysonqualifiedall") { + Ctx.CoalesceJoinKeysOnQualifiedAll = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableCoalesceJoinKeysOnQualifiedAll"); } else if (normalizedPragma == "resultrowslimit") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultRowsLimit)) { - Error() << "Expected unsigned integer literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultRowsLimit)) { + Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.IncrementMonCounter("sql_pragma", "ResultRowsLimit"); } else if (normalizedPragma == "resultsizelimit") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultSizeLimit)) { - Error() << "Expected unsigned integer literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.ResultSizeLimit)) { + Error() << "Expected unsigned integer literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } @@ -9375,69 +9375,69 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } - - TString action; - TString codePattern; - if (!values[0].GetLiteral(action, Ctx) || !values[1].GetLiteral(codePattern, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - TWarningRule rule; - TString parseError; - auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError); - switch (parseResult) { - case TWarningRule::EParseResult::PARSE_OK: - break; - case TWarningRule::EParseResult::PARSE_PATTERN_FAIL: - case TWarningRule::EParseResult::PARSE_ACTION_FAIL: - Ctx.Error() << parseError; - return {}; - default: - Y_ENSURE(false, "Unknown parse result"); - } - - Ctx.WarningPolicy.AddRule(rule); - if (rule.GetPattern() == "*" && rule.GetAction() == EWarningAction::ERROR) { - // Keep 'unused symbol' warning as warning unless explicitly set to error - Ctx.SetWarningPolicyFor(TIssuesIds::YQL_UNUSED_SYMBOL, EWarningAction::DEFAULT); - } - + + TString action; + TString codePattern; + if (!values[0].GetLiteral(action, Ctx) || !values[1].GetLiteral(codePattern, Ctx)) { + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + TWarningRule rule; + TString parseError; + auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError); + switch (parseResult) { + case TWarningRule::EParseResult::PARSE_OK: + break; + case TWarningRule::EParseResult::PARSE_PATTERN_FAIL: + case TWarningRule::EParseResult::PARSE_ACTION_FAIL: + Ctx.Error() << parseError; + return {}; + default: + Y_ENSURE(false, "Unknown parse result"); + } + + Ctx.WarningPolicy.AddRule(rule); + if (rule.GetPattern() == "*" && rule.GetAction() == EWarningAction::ERROR) { + // Keep 'unused symbol' warning as warning unless explicitly set to error + Ctx.SetWarningPolicyFor(TIssuesIds::YQL_UNUSED_SYMBOL, EWarningAction::DEFAULT); + } + Ctx.IncrementMonCounter("sql_pragma", "warning"); } else if (normalizedPragma == "greetings") { if (values.size() > 1) { - Error() << "Multiple arguments are not expected for " << pragma; + Error() << "Multiple arguments are not expected for " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; - } - - if (values.empty()) { + } + + if (values.empty()) { values.emplace_back(TDeferredAtom(Ctx.Pos(), "Hello, world! And best wishes from the YQL Team!")); } - - TString arg; - if (!values.front().GetLiteral(arg, Ctx)) { - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - Ctx.Info(Ctx.Pos()) << arg; + + TString arg; + if (!values.front().GetLiteral(arg, Ctx)) { + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + Ctx.Info(Ctx.Pos()) << arg; } else if (normalizedPragma == "warningmsg") { - if (values.size() != 1 || !values[0].GetLiteral()) { - Error() << "Expected string literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral()) { + Error() << "Expected string literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_PRAGMA_WARNING_MSG) << *values[0].GetLiteral(); } else if (normalizedPragma == "errormsg") { - if (values.size() != 1 || !values[0].GetLiteral()) { - Error() << "Expected string literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral()) { + Error() << "Expected string literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.Error(Ctx.Pos()) << *values[0].GetLiteral(); } else if (normalizedPragma == "classicdivision") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.Scoped->PragmaClassicDivision)) { - Error() << "Expected boolean literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.Scoped->PragmaClassicDivision)) { + Error() << "Expected boolean literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } @@ -9445,42 +9445,42 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disableunordered") { Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA) << "Use of deprecated DisableUnordered pragma. It will be dropped soon"; - } else if (normalizedPragma == "pullupflatmapoverjoin") { - Ctx.PragmaPullUpFlatMapOverJoin = true; - Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin"); - } else if (normalizedPragma == "disablepullupflatmapoverjoin") { - Ctx.PragmaPullUpFlatMapOverJoin = false; - Ctx.IncrementMonCounter("sql_pragma", "DisablePullUpFlatMapOverJoin"); - } else if (normalizedPragma == "allowunnamedcolumns") { - Ctx.WarnUnnamedColumns = false; - Ctx.IncrementMonCounter("sql_pragma", "AllowUnnamedColumns"); - } else if (normalizedPragma == "warnunnamedcolumns") { - Ctx.WarnUnnamedColumns = true; - Ctx.IncrementMonCounter("sql_pragma", "WarnUnnamedColumns"); + } else if (normalizedPragma == "pullupflatmapoverjoin") { + Ctx.PragmaPullUpFlatMapOverJoin = true; + Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin"); + } else if (normalizedPragma == "disablepullupflatmapoverjoin") { + Ctx.PragmaPullUpFlatMapOverJoin = false; + Ctx.IncrementMonCounter("sql_pragma", "DisablePullUpFlatMapOverJoin"); + } else if (normalizedPragma == "allowunnamedcolumns") { + Ctx.WarnUnnamedColumns = false; + Ctx.IncrementMonCounter("sql_pragma", "AllowUnnamedColumns"); + } else if (normalizedPragma == "warnunnamedcolumns") { + Ctx.WarnUnnamedColumns = true; + Ctx.IncrementMonCounter("sql_pragma", "WarnUnnamedColumns"); } else if (normalizedPragma == "discoverymode") { Ctx.DiscoveryMode = true; Ctx.IncrementMonCounter("sql_pragma", "DiscoveryMode"); } else if (normalizedPragma == "enablesystemcolumns") { - if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { - Error() << "Expected boolean literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) { + Error() << "Expected boolean literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } Ctx.IncrementMonCounter("sql_pragma", "EnableSystemColumns"); - } else if (normalizedPragma == "ansiinforemptyornullableitemscollections") { - Ctx.AnsiInForEmptyOrNullableItemsCollections = true; - Ctx.IncrementMonCounter("sql_pragma", "AnsiInForEmptyOrNullableItemsCollections"); - } else if (normalizedPragma == "disableansiinforemptyornullableitemscollections") { - Ctx.AnsiInForEmptyOrNullableItemsCollections = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiInForEmptyOrNullableItemsCollections"); + } else if (normalizedPragma == "ansiinforemptyornullableitemscollections") { + Ctx.AnsiInForEmptyOrNullableItemsCollections = true; + Ctx.IncrementMonCounter("sql_pragma", "AnsiInForEmptyOrNullableItemsCollections"); + } else if (normalizedPragma == "disableansiinforemptyornullableitemscollections") { + Ctx.AnsiInForEmptyOrNullableItemsCollections = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiInForEmptyOrNullableItemsCollections"); } else if (normalizedPragma == "dqengine") { Ctx.IncrementMonCounter("sql_pragma", "DqEngine"); - if (values.size() != 1 || !values[0].GetLiteral() + if (values.size() != 1 || !values[0].GetLiteral() || ! (*values[0].GetLiteral() == "disable" || *values[0].GetLiteral() == "auto" || *values[0].GetLiteral() == "force")) { Error() << "Expected `disable|auto|force' argument for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; } if (*values[0].GetLiteral() == "disable") { Ctx.DqEngineEnable = false; @@ -9492,32 +9492,32 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.DqEngineEnable = true; Ctx.DqEngineForce = false; } - } else if (normalizedPragma == "ansirankfornullablekeys") { - Ctx.AnsiRankForNullableKeys = true; - Ctx.IncrementMonCounter("sql_pragma", "AnsiRankForNullableKeys"); - } else if (normalizedPragma == "disableansirankfornullablekeys") { - Ctx.AnsiRankForNullableKeys = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiRankForNullableKeys"); - } else if (normalizedPragma == "ansiorderbylimitinunionall") { - Ctx.AnsiOrderByLimitInUnionAll = true; - Ctx.IncrementMonCounter("sql_pragma", "AnsiOrderByLimitInUnionAll"); - } else if (normalizedPragma == "disableansiorderbylimitinunionall") { - Ctx.AnsiOrderByLimitInUnionAll = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiOrderByLimitInUnionAll"); - } else if (normalizedPragma == "ansioptionalas") { - Ctx.AnsiOptionalAs = true; - Ctx.IncrementMonCounter("sql_pragma", "AnsiOptionalAs"); - } else if (normalizedPragma == "disableansioptionalas") { - Ctx.AnsiOptionalAs = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiOptionalAs"); - } else if (normalizedPragma == "warnonansialiasshadowing") { - Ctx.WarnOnAnsiAliasShadowing = true; - Ctx.IncrementMonCounter("sql_pragma", "WarnOnAnsiAliasShadowing"); - } else if (normalizedPragma == "disablewarnonansialiasshadowing") { - Ctx.WarnOnAnsiAliasShadowing = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableWarnOnAnsiAliasShadowing"); + } else if (normalizedPragma == "ansirankfornullablekeys") { + Ctx.AnsiRankForNullableKeys = true; + Ctx.IncrementMonCounter("sql_pragma", "AnsiRankForNullableKeys"); + } else if (normalizedPragma == "disableansirankfornullablekeys") { + Ctx.AnsiRankForNullableKeys = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiRankForNullableKeys"); + } else if (normalizedPragma == "ansiorderbylimitinunionall") { + Ctx.AnsiOrderByLimitInUnionAll = true; + Ctx.IncrementMonCounter("sql_pragma", "AnsiOrderByLimitInUnionAll"); + } else if (normalizedPragma == "disableansiorderbylimitinunionall") { + Ctx.AnsiOrderByLimitInUnionAll = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiOrderByLimitInUnionAll"); + } else if (normalizedPragma == "ansioptionalas") { + Ctx.AnsiOptionalAs = true; + Ctx.IncrementMonCounter("sql_pragma", "AnsiOptionalAs"); + } else if (normalizedPragma == "disableansioptionalas") { + Ctx.AnsiOptionalAs = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiOptionalAs"); + } else if (normalizedPragma == "warnonansialiasshadowing") { + Ctx.WarnOnAnsiAliasShadowing = true; + Ctx.IncrementMonCounter("sql_pragma", "WarnOnAnsiAliasShadowing"); + } else if (normalizedPragma == "disablewarnonansialiasshadowing") { + Ctx.WarnOnAnsiAliasShadowing = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableWarnOnAnsiAliasShadowing"); } else if (normalizedPragma == "regexusere2") { - if (values.size() != 1U || !values.front().GetLiteral() || !TryFromString(*values.front().GetLiteral(), Ctx.PragmaRegexUseRe2)) { + if (values.size() != 1U || !values.front().GetLiteral() || !TryFromString(*values.front().GetLiteral(), Ctx.PragmaRegexUseRe2)) { Error() << "Expected 'true' or 'false' for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; @@ -9529,20 +9529,20 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disablejsonqueryreturnsjsondocument") { Ctx.JsonQueryReturnsJsonDocument = false; Ctx.IncrementMonCounter("sql_pragma", "DisableJsonQueryReturnsJsonDocument"); - } else if (normalizedPragma == "orderedcolumns") { - Ctx.OrderedColumns = true; - Ctx.IncrementMonCounter("sql_pragma", "OrderedColumns"); - } else if (normalizedPragma == "disableorderedcolumns") { - Ctx.OrderedColumns = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableOrderedColumns"); - } else if (normalizedPragma == "positionalunionall") { - Ctx.PositionalUnionAll = true; - // PositionalUnionAll implies OrderedColumns - Ctx.OrderedColumns = true; - Ctx.IncrementMonCounter("sql_pragma", "PositionalUnionAll"); + } else if (normalizedPragma == "orderedcolumns") { + Ctx.OrderedColumns = true; + Ctx.IncrementMonCounter("sql_pragma", "OrderedColumns"); + } else if (normalizedPragma == "disableorderedcolumns") { + Ctx.OrderedColumns = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableOrderedColumns"); + } else if (normalizedPragma == "positionalunionall") { + Ctx.PositionalUnionAll = true; + // PositionalUnionAll implies OrderedColumns + Ctx.OrderedColumns = true; + Ctx.IncrementMonCounter("sql_pragma", "PositionalUnionAll"); } else if (normalizedPragma == "pqreadby") { - if (values.size() != 1 || !values[0].GetLiteral()) { - Error() << "Expected string literal as a single argument for: " << pragma; + if (values.size() != 1 || !values[0].GetLiteral()) { + Error() << "Expected string literal as a single argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; } @@ -9559,21 +9559,21 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success Ctx.PqReadByRtmrCluster = *values[0].GetLiteral(); Ctx.IncrementMonCounter("sql_pragma", "PqReadBy"); - } else if (normalizedPragma == "bogousstaringroupbyoverjoin") { - Ctx.BogousStarInGroupByOverJoin = true; + } else if (normalizedPragma == "bogousstaringroupbyoverjoin") { + Ctx.BogousStarInGroupByOverJoin = true; Ctx.IncrementMonCounter("sql_pragma", "BogousStarInGroupByOverJoin"); - } else if (normalizedPragma == "strictjoinkeytypes") { - Ctx.Scoped->StrictJoinKeyTypes = true; - Ctx.IncrementMonCounter("sql_pragma", "StrictJoinKeyTypes"); - } else if (normalizedPragma == "disablestrictjoinkeytypes") { - Ctx.Scoped->StrictJoinKeyTypes = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableStrictJoinKeyTypes"); - } else if (normalizedPragma == "unorderedsubqueries") { - Ctx.UnorderedSubqueries = true; - Ctx.IncrementMonCounter("sql_pragma", "UnorderedSubqueries"); - } else if (normalizedPragma == "disableunorderedsubqueries") { - Ctx.UnorderedSubqueries = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableUnorderedSubqueries"); + } else if (normalizedPragma == "strictjoinkeytypes") { + Ctx.Scoped->StrictJoinKeyTypes = true; + Ctx.IncrementMonCounter("sql_pragma", "StrictJoinKeyTypes"); + } else if (normalizedPragma == "disablestrictjoinkeytypes") { + Ctx.Scoped->StrictJoinKeyTypes = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableStrictJoinKeyTypes"); + } else if (normalizedPragma == "unorderedsubqueries") { + Ctx.UnorderedSubqueries = true; + Ctx.IncrementMonCounter("sql_pragma", "UnorderedSubqueries"); + } else if (normalizedPragma == "disableunorderedsubqueries") { + Ctx.UnorderedSubqueries = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableUnorderedSubqueries"); } else if (normalizedPragma == "datawatermarks") { if (values.size() != 1 || !values[0].GetLiteral() || ! (*values[0].GetLiteral() == "enable" || *values[0].GetLiteral() == "disable")) @@ -9590,12 +9590,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } Ctx.IncrementMonCounter("sql_pragma", "DataWatermarks"); - } else if (normalizedPragma == "flexibletypes") { - Ctx.FlexibleTypes = true; - Ctx.IncrementMonCounter("sql_pragma", "FlexibleTypes"); - } else if (normalizedPragma == "disableflexibletypes") { - Ctx.FlexibleTypes = false; - Ctx.IncrementMonCounter("sql_pragma", "DisableFlexibleTypes"); + } else if (normalizedPragma == "flexibletypes") { + Ctx.FlexibleTypes = true; + Ctx.IncrementMonCounter("sql_pragma", "FlexibleTypes"); + } else if (normalizedPragma == "disableflexibletypes") { + Ctx.FlexibleTypes = false; + Ctx.IncrementMonCounter("sql_pragma", "DisableFlexibleTypes"); } else { Error() << "Unknown pragma: " << pragma; Ctx.IncrementMonCounter("sql_errors", "UnknownPragma"); @@ -9621,7 +9621,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success if (values.size() == 0U) { Ctx.PragmaYsonStrict = true; success = true; - } else if (values.size() == 1U && values.front().GetLiteral() && TryFromString(*values.front().GetLiteral(), Ctx.PragmaYsonStrict)) { + } else if (values.size() == 1U && values.front().GetLiteral() && TryFromString(*values.front().GetLiteral(), Ctx.PragmaYsonStrict)) { success = true; } else { Error() << "Expected 'true', 'false' or no parameter for: " << pragma; @@ -9635,7 +9635,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success return {}; } bool pragmaYsonDisableStrict; - if (values.size() == 1U && values.front().GetLiteral() && TryFromString(*values.front().GetLiteral(), pragmaYsonDisableStrict)) { + if (values.size() == 1U && values.front().GetLiteral() && TryFromString(*values.front().GetLiteral(), pragmaYsonDisableStrict)) { Ctx.PragmaYsonStrict = !pragmaYsonDisableStrict; success = true; } else { @@ -9699,7 +9699,7 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) { case TRule_delete_stmt_TBlock4::kAlt1: { const auto& alt = stmt.GetBlock4().GetAlt1(); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression sqlExpr(Ctx, Mode); auto whereExpr = sqlExpr.Build(alt.GetRule_expr2()); if (!whereExpr) { @@ -9748,7 +9748,7 @@ TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) { auto source = BuildTableSource(Ctx.Pos(), table); if (alt.HasBlock3()) { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression sqlExpr(Ctx, Mode); auto whereExpr = sqlExpr.Build(alt.GetBlock3().GetRule_expr2()); if (!whereExpr) { @@ -9790,7 +9790,7 @@ TSourcePtr TSqlQuery::Build(const TRule_set_clause_choice& stmt) { bool TSqlQuery::FillSetClause(const TRule_set_clause& node, TVector<TString>& targetList, TVector<TNodePtr>& values) { targetList.push_back(ColumnNameAsSingleStr(*this, node.GetRule_set_target1().GetRule_column_name1())); - TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TColumnRefScope scope(Ctx, EColumnRefState::Allow); TSqlExpression sqlExpr(Ctx, Mode); if (!Expr(sqlExpr, values, node.GetRule_expr3())) { return false; @@ -9876,29 +9876,29 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); } - auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); - WarnUnusedNodes(); - return result; -} - -bool TSqlTranslation::DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& blocks, const TRule_define_action_or_subquery_body& body) { - if (body.HasBlock2()) { - if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1())) { - return false; - } - - for (const auto& nestedStmtItem : body.GetBlock2().GetBlock2()) { - const auto& nestedStmt = nestedStmtItem.GetRule_sql_stmt_core2(); - if (!query.Statement(blocks, nestedStmt)) { - return false; - } - } - } - - return true; -} - -bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_or_subquery_stmt& stmt) { + auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); + WarnUnusedNodes(); + return result; +} + +bool TSqlTranslation::DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& blocks, const TRule_define_action_or_subquery_body& body) { + if (body.HasBlock2()) { + if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1())) { + return false; + } + + for (const auto& nestedStmtItem : body.GetBlock2().GetBlock2()) { + const auto& nestedStmt = nestedStmtItem.GetRule_sql_stmt_core2(); + if (!query.Statement(blocks, nestedStmt)) { + return false; + } + } + } + + return true; +} + +bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_or_subquery_stmt& stmt) { auto kind = Ctx.Token(stmt.GetToken2()); const bool isSubquery = to_lower(kind) == "subquery"; if (!isSubquery && Mode == NSQLTranslation::ESqlMode::SUBQUERY) { @@ -9906,19 +9906,19 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ return false; } - TString actionName; - if (!NamedNodeImpl(stmt.GetRule_bind_parameter3(), actionName, *this)) { - return false; - } - if (IsAnonymousName(actionName)) { - Error() << "Can not use anonymous name '" << actionName << "' as " << to_upper(kind) << " name"; - return false; - } - TPosition actionNamePos = Ctx.Pos(); - - TVector<TSymbolNameWithPos> argNames; + TString actionName; + if (!NamedNodeImpl(stmt.GetRule_bind_parameter3(), actionName, *this)) { + return false; + } + if (IsAnonymousName(actionName)) { + Error() << "Can not use anonymous name '" << actionName << "' as " << to_upper(kind) << " name"; + return false; + } + TPosition actionNamePos = Ctx.Pos(); + + TVector<TSymbolNameWithPos> argNames; ui32 optionalArgumentsCount = 0; - if (stmt.HasBlock5() && !ActionOrSubqueryArgs(stmt.GetBlock5().GetRule_action_or_subquery_args1(), argNames, optionalArgumentsCount)) { + if (stmt.HasBlock5() && !ActionOrSubqueryArgs(stmt.GetBlock5().GetRule_action_or_subquery_args1(), argNames, optionalArgumentsCount)) { return false; } @@ -9927,10 +9927,10 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ Ctx.AllScopes.push_back(Ctx.Scoped); *Ctx.Scoped = *saveScoped; Ctx.Scoped->Local = TScopedState::TLocal{}; - Ctx.ScopeLevel++; + Ctx.ScopeLevel++; - for (auto& arg : argNames) { - arg.Name = PushNamedAtom(arg.Pos, arg.Name); + for (auto& arg : argNames) { + arg.Name = PushNamedAtom(arg.Pos, arg.Name); } auto saveMode = Ctx.Settings.Mode; @@ -9940,11 +9940,11 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ TSqlQuery query(Ctx, Ctx.Settings.Mode, false); TBlocks innerBlocks; - const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, stmt.GetRule_define_action_or_subquery_body8()); + const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, stmt.GetRule_define_action_or_subquery_body8()); ui32 topLevelSelects = 0; for (auto& block : innerBlocks) { - if (block->IsSelect()) { + if (block->IsSelect()) { ++topLevelSelects; } } @@ -9954,10 +9954,10 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ return false; } - auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; - WarnUnusedNodes(); + auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr; + WarnUnusedNodes(); Ctx.Scoped = saveScoped; - Ctx.ScopeLevel--; + Ctx.ScopeLevel--; Ctx.Settings.Mode = saveMode; if (!ret) { @@ -9971,7 +9971,7 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ TNodePtr params = new TAstListNodeImpl(Ctx.Pos()); params->Add("world"); for (const auto& arg : argNames) { - params->Add(BuildAtom(arg.Pos, arg.Name)); + params->Add(BuildAtom(arg.Pos, arg.Name)); } auto lambda = BuildLambda(Ctx.Pos(), params, blockNode); @@ -9982,7 +9982,7 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_ }); } - PushNamedNode(actionNamePos, actionName, lambda); + PushNamedNode(actionNamePos, actionName, lambda); return true; } @@ -10013,18 +10013,18 @@ TNodePtr TSqlTranslation::IfStatement(const TRule_if_stmt& stmt) { TNodePtr TSqlTranslation::ForStatement(const TRule_for_stmt& stmt) { bool isEvaluate = stmt.HasBlock1(); TSqlExpression expr(Ctx, Mode); - TString itemArgName; - if (!NamedNodeImpl(stmt.GetRule_bind_parameter3(), itemArgName, *this)) { - return {}; - } - TPosition itemArgNamePos = Ctx.Pos(); + TString itemArgName; + if (!NamedNodeImpl(stmt.GetRule_bind_parameter3(), itemArgName, *this)) { + return {}; + } + TPosition itemArgNamePos = Ctx.Pos(); auto exprNode = expr.Build(stmt.GetRule_expr5()); if (!exprNode) { return{}; } - itemArgName = PushNamedAtom(itemArgNamePos, itemArgName); + itemArgName = PushNamedAtom(itemArgNamePos, itemArgName); auto bodyNode = DoStatement(stmt.GetRule_do_stmt6(), true, { itemArgName }); PopNamedNode(itemArgName); if (!bodyNode) { @@ -10042,33 +10042,33 @@ TNodePtr TSqlTranslation::ForStatement(const TRule_for_stmt& stmt) { return BuildWorldForNode(Ctx.Pos(), exprNode, bodyNode, elseNode, isEvaluate); } -google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err, size_t maxErrors, bool ansiLexer, google::protobuf::Arena* arena) { +google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err, size_t maxErrors, bool ansiLexer, google::protobuf::Arena* arena) { YQL_ENSURE(arena); #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, ""); - if (ansiLexer) { - NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); - return builder.BuildAST(collector); - } else { - NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); - return builder.BuildAST(collector); - } + NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, ""); + if (ansiLexer) { + NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(collector); + } else { + NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(collector); + } } -google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, bool ansiLexer, google::protobuf::Arena* arena) { +google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, bool ansiLexer, google::protobuf::Arena* arena) { YQL_ENSURE(arena); #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - if (ansiLexer) { - NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); - return builder.BuildAST(err); - } else { - NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); - return builder.BuildAST(err); - } + if (ansiLexer) { + NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } else { + NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } } TAstNode* SqlASTToYql(const google::protobuf::Message& protoAst, TContext& ctx) { @@ -10120,17 +10120,17 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra { TAstParseResult res; TContext ctx(settings, res.Issues); - NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); + NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); - google::protobuf::Message* ast(SqlAST(query, "query", collector, settings.AnsiLexer, settings.Arena)); + google::protobuf::Message* ast(SqlAST(query, "query", collector, settings.AnsiLexer, settings.Arena)); if (ast) { SqlASTToYqlImpl(res, *ast, ctx); } else { ctx.IncrementMonCounter("sql_errors", "AstError"); } - if (warningRules) { - *warningRules = ctx.WarningPolicy.GetRules(); - ctx.WarningPolicy.Clear(); + if (warningRules) { + *warningRules = ctx.WarningPolicy.GetRules(); + ctx.WarningPolicy.Clear(); } return res; } diff --git a/ydb/library/yql/sql/v1/sql.h b/ydb/library/yql/sql/v1/sql.h index c4359c59b4..9c7dc676c6 100644 --- a/ydb/library/yql/sql/v1/sql.h +++ b/ydb/library/yql/sql/v1/sql.h @@ -15,7 +15,7 @@ namespace NSQLTranslation { namespace NSQLTranslationV1 { NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr); - google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& err, size_t maxErrors, bool ansiLexer, google::protobuf::Arena* arena = nullptr); + google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& err, size_t maxErrors, bool ansiLexer, google::protobuf::Arena* arena = nullptr); NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TTranslationSettings& settings); } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index bdd4fb9e2c..85b6e0dfcd 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -7,7 +7,7 @@ #include <util/string/split.h> -using namespace NSQLTranslation; +using namespace NSQLTranslation; enum class EDebugOutput { None, @@ -24,9 +24,9 @@ TString Err2Str(NYql::TAstParseResult& res, EDebugOutput debug = EDebugOutput::N return s.Str(); } -NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ESqlMode mode = NSQLTranslation::ESqlMode::QUERY, size_t maxErrors = 10, const TString& provider = {}, - EDebugOutput debug = EDebugOutput::None, bool ansiLexer = false, NSQLTranslation::TTranslationSettings settings = {}) -{ +NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ESqlMode mode = NSQLTranslation::ESqlMode::QUERY, size_t maxErrors = 10, const TString& provider = {}, + EDebugOutput debug = EDebugOutput::None, bool ansiLexer = false, NSQLTranslation::TTranslationSettings settings = {}) +{ google::protobuf::Arena arena; const auto service = provider ? provider : TString(NYql::YtProviderName); const TString cluster = "plato"; @@ -35,8 +35,8 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES settings.MaxErrors = maxErrors; settings.Mode = mode; settings.Arena = &arena; - settings.AnsiLexer = ansiLexer; - settings.SyntaxVersion = 1; + settings.AnsiLexer = ansiLexer; + settings.SyntaxVersion = 1; auto res = SqlToYql(query, settings); if (debug == EDebugOutput::ToCerr) { Err2Str(res, debug); @@ -48,29 +48,29 @@ NYql::TAstParseResult SqlToYql(const TString& query, size_t maxErrors = 10, cons return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug); } -NYql::TAstParseResult SqlToYqlWithSettings(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { - return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, 10, {}, EDebugOutput::None, false, settings); -} - -void ExpectFailWithError(const TString& query, const TString& error) { - NYql::TAstParseResult res = SqlToYql(query); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), error); -} - -NYql::TAstParseResult SqlToYqlWithAnsiLexer(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { - bool ansiLexer = true; - return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug, ansiLexer); -} - -void ExpectFailWithErrorForAnsiLexer(const TString& query, const TString& error) { - NYql::TAstParseResult res = SqlToYqlWithAnsiLexer(query); - - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), error); -} - +NYql::TAstParseResult SqlToYqlWithSettings(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { + return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, 10, {}, EDebugOutput::None, false, settings); +} + +void ExpectFailWithError(const TString& query, const TString& error) { + NYql::TAstParseResult res = SqlToYql(query); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), error); +} + +NYql::TAstParseResult SqlToYqlWithAnsiLexer(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { + bool ansiLexer = true; + return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug, ansiLexer); +} + +void ExpectFailWithErrorForAnsiLexer(const TString& query, const TString& error) { + NYql::TAstParseResult res = SqlToYqlWithAnsiLexer(query); + + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), error); +} + TString GetPrettyPrint(const NYql::TAstParseResult& res) { TStringStream yqlProgram; res.Root->PrettyPrintTo(yqlProgram, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); @@ -104,49 +104,49 @@ TString VerifyProgram(const NYql::TAstParseResult& res, TWordCountHive& wordCoun for (const auto& line: yqlProgram) { for (auto& counterIter: wordCounter) { const auto& word = counterIter.first; - auto pos = line.find(word); - while (pos != TString::npos) { + auto pos = line.find(word); + while (pos != TString::npos) { ++counterIter.second; if (verifyLine) { verifyLine(word, line); } - pos = line.find(word, pos + word.length()); + pos = line.find(word, pos + word.length()); } } } return programm; } -void VerifySqlInHints(const TString& query, const THashSet<TString>& expectedHints, TMaybe<bool> ansi) { - TString pragma; - if (ansi.Defined()) { - pragma = *ansi ? "PRAGMA AnsiInForEmptyOrNullableItemsCollections;" : - "PRAGMA DisableAnsiInForEmptyOrNullableItemsCollections;"; - } - - NYql::TAstParseResult res = SqlToYql(pragma + query); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { - Y_UNUSED(word); - if (!ansi.Defined()) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('warnNoAnsi)")); - } else if (*ansi) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('ansi)")); - } - for (auto& hint : expectedHints) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(hint)); - } - }; - TWordCountHive elementStat = {{TString("SqlIn"), 0}}; - VerifyProgram(res, elementStat, verifyLine); -} - -void VerifySqlInHints(const TString& query, const THashSet<TString>& expectedHints) { - VerifySqlInHints(query, expectedHints, false); - VerifySqlInHints(query, expectedHints, true); -} - +void VerifySqlInHints(const TString& query, const THashSet<TString>& expectedHints, TMaybe<bool> ansi) { + TString pragma; + if (ansi.Defined()) { + pragma = *ansi ? "PRAGMA AnsiInForEmptyOrNullableItemsCollections;" : + "PRAGMA DisableAnsiInForEmptyOrNullableItemsCollections;"; + } + + NYql::TAstParseResult res = SqlToYql(pragma + query); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { + Y_UNUSED(word); + if (!ansi.Defined()) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('warnNoAnsi)")); + } else if (*ansi) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('ansi)")); + } + for (auto& hint : expectedHints) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(hint)); + } + }; + TWordCountHive elementStat = {{TString("SqlIn"), 0}}; + VerifyProgram(res, elementStat, verifyLine); +} + +void VerifySqlInHints(const TString& query, const THashSet<TString>& expectedHints) { + VerifySqlInHints(query, expectedHints, false); + VerifySqlInHints(query, expectedHints, true); +} + Y_UNIT_TEST_SUITE(AnsiMode) { Y_UNIT_TEST(PragmaAnsi) { UNIT_ASSERT(SqlToYql("PRAGMA ANSI 2016;").IsOk()); @@ -163,44 +163,44 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input WITH (INFER_SCHEMA)").IsOk()); } - Y_UNIT_TEST(InNoHints) { - TString query = "SELECT * FROM plato.Input WHERE key IN (1,2,3)"; - - VerifySqlInHints(query, { "'('('warnNoAnsi))" }, {}); - VerifySqlInHints(query, { "'()" }, false); - VerifySqlInHints(query, { "'('('ansi))" }, true); - } - - Y_UNIT_TEST(InHintCompact) { - // should parse COMPACT as hint - TString query = "SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"; - - VerifySqlInHints(query, { "'('isCompact)" }); - } - - Y_UNIT_TEST(InHintSubquery) { - // should parse tableSource as hint - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"; - - VerifySqlInHints(query, { "'('tableSource)" }); - } - + Y_UNIT_TEST(InNoHints) { + TString query = "SELECT * FROM plato.Input WHERE key IN (1,2,3)"; + + VerifySqlInHints(query, { "'('('warnNoAnsi))" }, {}); + VerifySqlInHints(query, { "'()" }, false); + VerifySqlInHints(query, { "'('('ansi))" }, true); + } + + Y_UNIT_TEST(InHintCompact) { + // should parse COMPACT as hint + TString query = "SELECT * FROM plato.Input WHERE key IN COMPACT(1, 2, 3)"; + + VerifySqlInHints(query, { "'('isCompact)" }); + } + + Y_UNIT_TEST(InHintSubquery) { + // should parse tableSource as hint + TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN $subq"; + + VerifySqlInHints(query, { "'('tableSource)" }); + } + Y_UNIT_TEST(InHintCompactSubquery) { - TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"; - - VerifySqlInHints(query, { "'('isCompact)", "'('tableSource)" }); - } - - Y_UNIT_TEST(CompactKeywordNotReservedForNames) { - UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); - } - + TString query = "$subq = (SELECT key FROM plato.Input); SELECT * FROM plato.Input WHERE key IN COMPACT $subq"; + + VerifySqlInHints(query, { "'('isCompact)", "'('tableSource)" }); + } + + Y_UNIT_TEST(CompactKeywordNotReservedForNames) { + UNIT_ASSERT(SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT(1, 2, 3)").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT * FROM COMPACT").IsOk()); + } + Y_UNIT_TEST(FamilyKeywordNotReservedForNames) { - // FIXME: check if we can get old behaviour - //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); - //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); + // FIXME: check if we can get old behaviour + //UNIT_ASSERT(SqlToYql("USE plato; CREATE TABLE FAMILY (FAMILY Uint32, PRIMARY KEY (FAMILY));").IsOk()); + //UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM FAMILY").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT FAMILY FROM Input").IsOk()); } Y_UNIT_TEST(ResetKeywordNotReservedForNames) { @@ -231,21 +231,21 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(res.Root); TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; + static bool seenStar = false; if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, true); - } else if (word == "SqlProjectStarItem") { - seenStar = true; + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); + UNIT_ASSERT_VALUES_EQUAL(seenStar, true); + } else if (word == "SqlProjectStarItem") { + seenStar = true; } }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; + TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); } Y_UNIT_TEST(QualifiedAsteriskAfter) { @@ -256,21 +256,21 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(res.Root); TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - static bool seenStar = false; + static bool seenStar = false; if (word == "FlattenMembers") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); - UNIT_ASSERT_VALUES_EQUAL(seenStar, false); - } else if (word == "SqlProjectStarItem") { - seenStar = true; + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("interested_table.")); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megahelpful_len"))); + UNIT_ASSERT_VALUES_EQUAL(seenStar, false); + } else if (word == "SqlProjectStarItem") { + seenStar = true; } }; - TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; + TWordCountHive elementStat = {{TString("FlattenMembers"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["FlattenMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); } Y_UNIT_TEST(QualifiedMembers) { @@ -281,17 +281,17 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { const bool fieldKey = TString::npos != line.find(Quote("key")); const bool fieldValue = TString::npos != line.find(Quote("value")); const bool refOnTable = TString::npos != line.find("interested_table."); - if (word == "SqlProjectItem") { + if (word == "SqlProjectItem") { UNIT_ASSERT(fieldKey || fieldValue); UNIT_ASSERT(!refOnTable); } else if (word == "Write!") { UNIT_ASSERT(fieldKey && fieldValue && !refOnTable); } }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("Write!"), 0}}; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("Write!"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); } @@ -309,19 +309,19 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { if (word == "SelectMembers") { UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("table_bb.")); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megakey"))); - } else if (word == "SqlColumn") { + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("megakey"))); + } else if (word == "SqlColumn") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("table_aa"))); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); } }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; + TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectStarItem"]); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); } Y_UNIT_TEST(Join3Table) { @@ -338,22 +338,22 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "SelectMembers") { UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa.")); - UNIT_ASSERT(line.find("table_bb.") != TString::npos || line.find("table_cc.") != TString::npos); - } else if (word == "SqlProjectItem") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("gigakey"))); - } else if (word == "SqlColumn") { + UNIT_ASSERT(line.find("table_bb.") != TString::npos || line.find("table_cc.") != TString::npos); + } else if (word == "SqlProjectItem") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("gigakey"))); + } else if (word == "SqlColumn") { const auto posTableAA = line.find(Quote("table_aa")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableAA); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("key"))); UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("table_aa", posTableAA + 3)); } }; - TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; + TWordCountHive elementStat = {{TString("SqlProjectItem"), 0}, {TString("SqlProjectStarItem"), 0}, {TString("SelectMembers"), 0}, {TString("SqlColumn"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SelectMembers"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SelectMembers"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["SqlColumn"]); } Y_UNIT_TEST(JoinWithoutConcreteColumns) { @@ -367,24 +367,24 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(res.Root); TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { - UNIT_ASSERT(line.find(Quote("a.v")) != TString::npos || line.find(Quote("b.value")) != TString::npos); - } else if (word == "SqlColumn") { + if (word == "SqlProjectItem") { + UNIT_ASSERT(line.find(Quote("a.v")) != TString::npos || line.find(Quote("b.value")) != TString::npos); + } else if (word == "SqlColumn") { const auto posTableA = line.find(Quote("a")); const auto posTableB = line.find(Quote("b")); - if (posTableA != TString::npos) { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("v"))); - } else { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableB); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("value"))); - } + if (posTableA != TString::npos) { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("v"))); + } else { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, posTableB); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(Quote("value"))); + } } }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlColumn"), 0}}; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {TString("SqlColumn"), 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlColumn"]); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlColumn"]); } Y_UNIT_TEST(JoinWithSameValues) { @@ -392,19 +392,19 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(res.Root); TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "SqlProjectItem") { + if (word == "SqlProjectItem") { const bool isValueFromA = TString::npos != line.find(Quote("a.value")); const bool isValueFromB = TString::npos != line.find(Quote("b.value")); UNIT_ASSERT(isValueFromA || isValueFromB); } if (word == "Write!") { - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("a.a.")); - UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("b.b.")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("a.a.")); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("b.b.")); } }; - TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {"Write!", 0}}; + TWordCountHive elementStat = {{TString("SqlProjectStarItem"), 0}, {TString("SqlProjectItem"), 0}, {"Write!", 0}}; VerifyProgram(res, elementStat, verifyLine); - UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); + UNIT_ASSERT_VALUES_EQUAL(0, elementStat["SqlProjectStarItem"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["SqlProjectItem"]); UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write!"]); } @@ -438,13 +438,13 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(res.Root); } - Y_UNIT_TEST(SelectSupportsResultColumnsWithTrailingComma) { - NYql::TAstParseResult res = SqlToYql("select a, b, c, from plato.Input;"); - UNIT_ASSERT(res.Root); - } - + Y_UNIT_TEST(SelectSupportsResultColumnsWithTrailingComma) { + NYql::TAstParseResult res = SqlToYql("select a, b, c, from plato.Input;"); + UNIT_ASSERT(res.Root); + } + Y_UNIT_TEST(SelectOrderByLabeledColumn) { - NYql::TAstParseResult res = SqlToYql("pragma DisableOrderedColumns; select key as goal from plato.Input order by goal"); + NYql::TAstParseResult res = SqlToYql("pragma DisableOrderedColumns; select key as goal from plato.Input order by goal"); UNIT_ASSERT(res.Root); TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "DataSource") { @@ -953,12 +953,12 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { )", 10, TString(NYql::KikimrProviderName)); UNIT_ASSERT(res.Root); } - - Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { - NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); - } + + Y_UNIT_TEST(WarnMissingIsBeforeNotNull) { + NYql::TAstParseResult res = SqlToYql("select 1 NOT NULL"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Missing IS keyword before NOT NULL, code: 4507\n"); + } Y_UNIT_TEST(Subqueries) { NYql::TAstParseResult res = SqlToYql(R"( @@ -1010,276 +1010,276 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { Cerr << Err2Str(res) << Endl; UNIT_ASSERT(res.Root); } - - Y_UNIT_TEST(AnyInBackticksAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from `any`;"); - UNIT_ASSERT(res.Root); - } - - Y_UNIT_TEST(AnyJoinForTableAndSubQuery) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = SELECT * FROM plato.Input2; - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY $r AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinForTableAndTableSource) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - $r = AsList( - AsStruct("aaa" as key, "bbb" as subkey, "ccc" as value) - ); - - SELECT * FROM ANY plato.Input1 AS l - LEFT JOIN ANY AS_TABLE($r) AS r - USING (key); - )"); - - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "EquiJoin") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); - } - }; - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); - } - - Y_UNIT_TEST(AnyJoinNested) { - NYql::TAstParseResult res = SqlToYql(R"( - USE plato; - - FROM ANY Input1 as a - JOIN Input2 as b ON a.key = b.key - LEFT JOIN ANY Input3 as c ON a.key = c.key - RIGHT JOIN ANY Input4 as d ON d.key = b.key - CROSS JOIN Input5 - SELECT *; - )"); - - UNIT_ASSERT(res.Root); - - TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; - VerifyProgram(res, elementStat); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["right"]); - } - - Y_UNIT_TEST(InlineAction) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end do\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(FlattenByCorrelationName) { - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk()); - UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk()); - } + + Y_UNIT_TEST(AnyInBackticksAsTableName) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from `any`;"); + UNIT_ASSERT(res.Root); + } + + Y_UNIT_TEST(AnyJoinForTableAndSubQuery) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + $r = SELECT * FROM plato.Input2; + + SELECT * FROM ANY plato.Input1 AS l + LEFT JOIN ANY $r AS r + USING (key); + )"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "EquiJoin") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); + } + }; + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); + } + + Y_UNIT_TEST(AnyJoinForTableAndTableSource) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + $r = AsList( + AsStruct("aaa" as key, "bbb" as subkey, "ccc" as value) + ); + + SELECT * FROM ANY plato.Input1 AS l + LEFT JOIN ANY AS_TABLE($r) AS r + USING (key); + )"); + + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "EquiJoin") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('left 'any)")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("'('right 'any)")); + } + }; + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["right"]); + } + + Y_UNIT_TEST(AnyJoinNested) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + + FROM ANY Input1 as a + JOIN Input2 as b ON a.key = b.key + LEFT JOIN ANY Input3 as c ON a.key = c.key + RIGHT JOIN ANY Input4 as d ON d.key = b.key + CROSS JOIN Input5 + SELECT *; + )"); + + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("left"), 0}, {TString("right"), 0}}; + VerifyProgram(res, elementStat); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["left"]); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["right"]); + } + + Y_UNIT_TEST(InlineAction) { + NYql::TAstParseResult res = SqlToYql( + "do begin\n" + " select 1\n" + "; end do\n"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); + } + + Y_UNIT_TEST(FlattenByCorrelationName) { + UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk()); + UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk()); + } Y_UNIT_TEST(DiscoveryMode) { UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk()); } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk()); - } - - Y_UNIT_TEST(CubeWithAutoGeneratedLikeAlias) { - UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,value as group)").IsOk()); - } - - Y_UNIT_TEST(FilterCanBeUsedAsColumnIdOrBind) { - UNIT_ASSERT(SqlToYql("select filter from plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("select 1 as filter").IsOk()); - UNIT_ASSERT(SqlToYql("$filter = 1; select $filter").IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenTopLevelStatements) { - UNIT_ASSERT(SqlToYql(";;select 1; ; select 2;/*comment*/;select 3;;--comment\n;select 4;;").IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenActionStatements) { - TString req = - "define action $action($b,$c) as\n" - " ;;$d = $b + $c;\n" - " select $b;\n" - " select $c;;\n" - " select $d,\n" - "end define;\n" - "\n" - "do $action(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenInlineActionStatements) { - TString req = - "do begin\n" - " ;select 1,\n" - "end do;\n" - "evaluate for $i in AsList(1,2,3) do begin\n" - " select $i;;\n" - " select $i + $i;;\n" - "end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenLambdaStatements) { - TString req = - "$x=1;\n" - "$foo = ($a, $b)->{\n" - " ;;$v = $a + $b;\n" - " $bar = ($c) -> {; return $c << $x};;\n" - " return $bar($v);;\n" - "};\n" - "select $foo(1,2);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); - UNIT_ASSERT(res1.Root); - UNIT_ASSERT(res2.Root); - - TWordCountHive elementStat = {{TString("a\\"), 0}}; - - VerifyProgram(res1, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); - - VerifyProgram(res2, elementStat); - UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); - } - - Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { - UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { - UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); - - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); - - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); - UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); - UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); - } - - Y_UNIT_TEST(ConstnessForListDictSetCreate) { - auto req = "$foo = ($x, $y) -> (\"aaaa\");\n" - "\n" - "select\n" - " $foo(sum(key), ListCreate(String)),\n" - " $foo(sum(key), DictCreate(String, String)),\n" - " $foo(sum(key), SetCreate(String)),\n" - "from (select 1 as key);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CanUseEmptyTupleInWindowPartitionBy) { - auto req = "select sum(key) over w\n" - "from plato.Input\n" - "window w as (partition compact by (), (subkey), (), value || value as dvalue);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(NoWarnUnionAllWithOrderByWithExplicitLegacyMode) { - auto req = "pragma DisableAnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key limit 10\n" - "union all\n" - "select * from Input order by key limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(WarnUnionAllWithDiscardIntoResultWithExplicitLegacyMode) { - auto req = "use plato;\n" - "pragma DisableAnsiOrderByLimitInUnionAll;\n" - "\n" - "select * from Input into result aaa\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Warning: INTO RESULT will be ignored here. Please use INTO RESULT after last subquery in UNION ALL if you want label entire UNION ALL result, code: 4522\n" - "<main>:6:1: Warning: DISCARD will be ignored here. Please use DISCARD before first subquery in UNION ALL if you want to discard entire UNION ALL result, code: 4522\n"); - } - - Y_UNIT_TEST(WarnUnionAllWithIgnoredOrderByLegacyMode) { - auto req = "use plato;\n" - "pragma DisableAnsiOrderByLimitInUnionAll;\n" - "\n" - "SELECT * FROM (\n" - " SELECT * FROM Input\n" - " UNION ALL\n" - " SELECT t.* FROM Input AS t ORDER BY t.key\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:7:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n"); - } - - Y_UNIT_TEST(ReduceUsingUdfWithShortcutsWorks) { - auto req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using $func(subkey);\n" - "REDUCE Input ON key using $func(UUU::VVV(TableRow()));\n"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - req = "use plato;\n" - "\n" - "$arg = 'foo';\n" - "$func = XXX::YYY($arg);\n" - "\n" - "REDUCE Input ON key using all $func(subkey);\n" - "REDUCE Input ON key using all $func(UUU::VVV(TableRow()));"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } + + Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) { + UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk()); + } + + Y_UNIT_TEST(CubeWithAutoGeneratedLikeAlias) { + UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,value as group)").IsOk()); + } + + Y_UNIT_TEST(FilterCanBeUsedAsColumnIdOrBind) { + UNIT_ASSERT(SqlToYql("select filter from plato.Input").IsOk()); + UNIT_ASSERT(SqlToYql("select 1 as filter").IsOk()); + UNIT_ASSERT(SqlToYql("$filter = 1; select $filter").IsOk()); + } + + Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenTopLevelStatements) { + UNIT_ASSERT(SqlToYql(";;select 1; ; select 2;/*comment*/;select 3;;--comment\n;select 4;;").IsOk()); + } + + Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenActionStatements) { + TString req = + "define action $action($b,$c) as\n" + " ;;$d = $b + $c;\n" + " select $b;\n" + " select $c;;\n" + " select $d,\n" + "end define;\n" + "\n" + "do $action(1,2);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(DuplicateAndMissingTrailingSemicolonsAreAllowedBetweenInlineActionStatements) { + TString req = + "do begin\n" + " ;select 1,\n" + "end do;\n" + "evaluate for $i in AsList(1,2,3) do begin\n" + " select $i;;\n" + " select $i + $i;;\n" + "end do;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(DuplicateSemicolonsAreAllowedBetweenLambdaStatements) { + TString req = + "$x=1;\n" + "$foo = ($a, $b)->{\n" + " ;;$v = $a + $b;\n" + " $bar = ($c) -> {; return $c << $x};;\n" + " return $bar($v);;\n" + "};\n" + "select $foo(1,2);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(StringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo(SELECT 'a\\';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo(SELECT "a\\";)foo"); + UNIT_ASSERT(res1.Root); + UNIT_ASSERT(res2.Root); + + TWordCountHive elementStat = {{TString("a\\"), 0}}; + + VerifyProgram(res1, elementStat); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["a\\"]); + + VerifyProgram(res2, elementStat); + UNIT_ASSERT_VALUES_EQUAL(2, elementStat["a\\"]); + } + + Y_UNIT_TEST(StringMultiLineLiteralWithEscapes) { + UNIT_ASSERT(SqlToYql("SELECT @@@foo@@@@bar@@@").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT @@@@@@@@@").IsOk()); + } + + Y_UNIT_TEST(StringMultiLineLiteralConsequitiveAt) { + UNIT_ASSERT(!SqlToYql("SELECT @").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@").IsOk()); + + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@").IsOk()); + + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@").IsOk()); + UNIT_ASSERT( SqlToYql("SELECT @@@@@@@@@").IsOk()); + UNIT_ASSERT(!SqlToYql("SELECT @@@@@@@@@@").IsOk()); + } + + Y_UNIT_TEST(ConstnessForListDictSetCreate) { + auto req = "$foo = ($x, $y) -> (\"aaaa\");\n" + "\n" + "select\n" + " $foo(sum(key), ListCreate(String)),\n" + " $foo(sum(key), DictCreate(String, String)),\n" + " $foo(sum(key), SetCreate(String)),\n" + "from (select 1 as key);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(CanUseEmptyTupleInWindowPartitionBy) { + auto req = "select sum(key) over w\n" + "from plato.Input\n" + "window w as (partition compact by (), (subkey), (), value || value as dvalue);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(NoWarnUnionAllWithOrderByWithExplicitLegacyMode) { + auto req = "pragma DisableAnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input order by key limit 10\n" + "union all\n" + "select * from Input order by key limit 1;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); + } + + Y_UNIT_TEST(WarnUnionAllWithDiscardIntoResultWithExplicitLegacyMode) { + auto req = "use plato;\n" + "pragma DisableAnsiOrderByLimitInUnionAll;\n" + "\n" + "select * from Input into result aaa\n" + "union all\n" + "discard select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Warning: INTO RESULT will be ignored here. Please use INTO RESULT after last subquery in UNION ALL if you want label entire UNION ALL result, code: 4522\n" + "<main>:6:1: Warning: DISCARD will be ignored here. Please use DISCARD before first subquery in UNION ALL if you want to discard entire UNION ALL result, code: 4522\n"); + } + + Y_UNIT_TEST(WarnUnionAllWithIgnoredOrderByLegacyMode) { + auto req = "use plato;\n" + "pragma DisableAnsiOrderByLimitInUnionAll;\n" + "\n" + "SELECT * FROM (\n" + " SELECT * FROM Input\n" + " UNION ALL\n" + " SELECT t.* FROM Input AS t ORDER BY t.key\n" + ");"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:7:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n"); + } + + Y_UNIT_TEST(ReduceUsingUdfWithShortcutsWorks) { + auto req = "use plato;\n" + "\n" + "$arg = 'foo';\n" + "$func = XXX::YYY($arg);\n" + "\n" + "REDUCE Input ON key using $func(subkey);\n" + "REDUCE Input ON key using $func(UUU::VVV(TableRow()));\n"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + req = "use plato;\n" + "\n" + "$arg = 'foo';\n" + "$func = XXX::YYY($arg);\n" + "\n" + "REDUCE Input ON key using all $func(subkey);\n" + "REDUCE Input ON key using all $func(UUU::VVV(TableRow()));"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } Y_UNIT_TEST(YsonDisableStrict) { UNIT_ASSERT(SqlToYql("pragma yson.DisableStrict = \"false\";").IsOk()); @@ -1290,45 +1290,45 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("pragma yson.Strict = \"false\";").IsOk()); UNIT_ASSERT(SqlToYql("pragma yson.Strict;").IsOk()); } - - Y_UNIT_TEST(JoinByTuple) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsTuple(a.key, a.subkey) = AsTuple(b.key, b.subkey);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByStruct) { - auto req = "use plato;\n" - "\n" - "select * from T1 as a\n" - "join T2 as b\n" - "on AsStruct(a.key as k, a.subkey as sk) = AsStruct(b.key as k, b.subkey as sk);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(JoinByUdf) { - auto req = "use plato;\n" - "\n" - "select a.align\n" - "from T1 as a\n" - "join T2 as b\n" - "on Yson::SerializeJsonEncodeUtf8(a.align)=b.align;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(EscapedIdentifierAsLambdaArg) { - auto req = "$f = ($`foo bar`, $x) -> { return $`foo bar` + $x; };\n" - "\n" - "select $f(1, 2);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - const auto programm = GetPrettyPrint(res); - auto expected = "(Apply (lambda '(\"$foo bar\" \"$x\")"; - UNIT_ASSERT(programm.find(expected) != TString::npos); - } + + Y_UNIT_TEST(JoinByTuple) { + auto req = "use plato;\n" + "\n" + "select * from T1 as a\n" + "join T2 as b\n" + "on AsTuple(a.key, a.subkey) = AsTuple(b.key, b.subkey);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(JoinByStruct) { + auto req = "use plato;\n" + "\n" + "select * from T1 as a\n" + "join T2 as b\n" + "on AsStruct(a.key as k, a.subkey as sk) = AsStruct(b.key as k, b.subkey as sk);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(JoinByUdf) { + auto req = "use plato;\n" + "\n" + "select a.align\n" + "from T1 as a\n" + "join T2 as b\n" + "on Yson::SerializeJsonEncodeUtf8(a.align)=b.align;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(EscapedIdentifierAsLambdaArg) { + auto req = "$f = ($`foo bar`, $x) -> { return $`foo bar` + $x; };\n" + "\n" + "select $f(1, 2);"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + const auto programm = GetPrettyPrint(res); + auto expected = "(Apply (lambda '(\"$foo bar\" \"$x\")"; + UNIT_ASSERT(programm.find(expected) != TString::npos); + } Y_UNIT_TEST(CompactionPolicyParseCorrect) { NYql::TAstParseResult res = SqlToYql( @@ -1414,165 +1414,165 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } - - Y_UNIT_TEST(CloneForAsTableWorksWithCube) { - UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); - } - - Y_UNIT_TEST(WindowPartitionByColumnProperlyEscaped) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input WINDOW w AS (PARTITION BY `column with space`);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"column with space\"")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - } - - Y_UNIT_TEST(WindowPartitionByExpressionWithoutAliasesAreAllowed) { - NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input as i WINDOW w AS (PARTITION BY ii.subkey);"); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "AddMember") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("AddMember row 'group_w_0 (SqlAccess 'struct (Member row '\"ii\")")); - } - if (word == "CalcOverWindow") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("CalcOverWindow core '('\"group_w_0\")")); - } - }; - - TWordCountHive elementStat = { {TString("CalcOverWindow"), 0}, {TString("AddMember"), 0} }; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AddMember"]); - } - - Y_UNIT_TEST(PqReadByAfterUse) { - ExpectFailWithError("use plato; pragma PqReadBy='plato2';", + + Y_UNIT_TEST(CloneForAsTableWorksWithCube) { + UNIT_ASSERT(SqlToYql("SELECT * FROM AS_TABLE([<|k1:1, k2:1|>]) GROUP BY CUBE(k1, k2);").IsOk()); + } + + Y_UNIT_TEST(WindowPartitionByColumnProperlyEscaped) { + NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input WINDOW w AS (PARTITION BY `column with space`);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "CalcOverWindow") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("\"column with space\"")); + } + }; + + TWordCountHive elementStat = { {TString("CalcOverWindow"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); + } + + Y_UNIT_TEST(WindowPartitionByExpressionWithoutAliasesAreAllowed) { + NYql::TAstParseResult res = SqlToYql("SELECT SUM(key) OVER w FROM plato.Input as i WINDOW w AS (PARTITION BY ii.subkey);"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "AddMember") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("AddMember row 'group_w_0 (SqlAccess 'struct (Member row '\"ii\")")); + } + if (word == "CalcOverWindow") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("CalcOverWindow core '('\"group_w_0\")")); + } + }; + + TWordCountHive elementStat = { {TString("CalcOverWindow"), 0}, {TString("AddMember"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["CalcOverWindow"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["AddMember"]); + } + + Y_UNIT_TEST(PqReadByAfterUse) { + ExpectFailWithError("use plato; pragma PqReadBy='plato2';", "<main>:1:28: Error: Cluster in PqReadPqBy pragma differs from cluster specified in USE statement: plato2 != plato\n"); - - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); - UNIT_ASSERT(SqlToYql("$x='plato'; use rtmr:$x; pragma PqReadBy='plato2';").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; pragma PqReadBy='dq';").IsOk()); - } - - Y_UNIT_TEST(MrObject) { - NYql::TAstParseResult res = SqlToYql( - "declare $path as String;\n" - "select * from plato.object($path, `format`, \"comp\" || \"ression\" as compression, 1 as bar) with schema (Int32 as y, String as x)" - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((MrObject (EvaluateAtom "$path") '"format" '('('"compression" (Concat (String '"comp") (String '"ression"))) '('"bar" (Int32 '"1")))))__")); - } else if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (StructType '('"y" (DataType 'Int32)) '('"x" (DataType 'String))) '('"y" '"x"))))__")); - } - }; - - TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(TableBindings) { - NSQLTranslation::TTranslationSettings settings; - NSQLTranslation::TTableBindingSettings bindSettings; - bindSettings.ClusterType = "s3"; - bindSettings.Settings["cluster"] = "cluster"; - bindSettings.Settings["path"] = "path"; - bindSettings.Settings["format"] = "format"; - bindSettings.Settings["compression"] = "ccompression"; - bindSettings.Settings["bar"] = "1"; + + UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2';").IsOk()); + UNIT_ASSERT(SqlToYql("pragma PqReadBy='plato2'; use plato;").IsOk()); + UNIT_ASSERT(SqlToYql("$x='plato'; use rtmr:$x; pragma PqReadBy='plato2';").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; pragma PqReadBy='dq';").IsOk()); + } + + Y_UNIT_TEST(MrObject) { + NYql::TAstParseResult res = SqlToYql( + "declare $path as String;\n" + "select * from plato.object($path, `format`, \"comp\" || \"ression\" as compression, 1 as bar) with schema (Int32 as y, String as x)" + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "MrObject") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__((MrObject (EvaluateAtom "$path") '"format" '('('"compression" (Concat (String '"comp") (String '"ression"))) '('"bar" (Int32 '"1")))))__")); + } else if (word == "userschema") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__('('('"userschema" (StructType '('"y" (DataType 'Int32)) '('"x" (DataType 'String))) '('"y" '"x"))))__")); + } + }; + + TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); + } + + Y_UNIT_TEST(TableBindings) { + NSQLTranslation::TTranslationSettings settings; + NSQLTranslation::TTableBindingSettings bindSettings; + bindSettings.ClusterType = "s3"; + bindSettings.Settings["cluster"] = "cluster"; + bindSettings.Settings["path"] = "path"; + bindSettings.Settings["format"] = "format"; + bindSettings.Settings["compression"] = "ccompression"; + bindSettings.Settings["bar"] = "1"; // schema is not validated in this test but should be valid YSON text bindSettings.Settings["schema"] = R"__("[ - "StructType"; - [ - [ - "key"; - [ - "DataType"; - "String" - ] - ]; - [ - "subkey"; - [ - "DataType"; - "String" - ] - ]; - [ - "value"; - [ - "DataType"; - "String" - ] - ] + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] ]])__"; - - settings.PrivateBindings["foo"] = bindSettings; - - NYql::TAstParseResult res = SqlToYqlWithSettings( - "select * from bindings.foo", - settings - ); - UNIT_ASSERT(res.Root); - - TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { - if (word == "MrObject") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + + settings.PrivateBindings["foo"] = bindSettings; + + NYql::TAstParseResult res = SqlToYqlWithSettings( + "select * from bindings.foo", + settings + ); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "MrObject") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((MrObject '"path" '"format" '('('"bar" (String '"1")) '('"compression" (String '"ccompression")))))__")); - } else if (word == "userschema") { - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__('('('"userschema" (SqlTypeFromYson)__")); - } - }; - - TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; - VerifyProgram(res, elementStat, verifyLine); - - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); - } - - Y_UNIT_TEST(TrailingCommaInWithout) { - UNIT_ASSERT(SqlToYql("SELECT * WITHOUT stream, FROM plato.Input").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT a.intersect, FROM plato.Input AS a").IsOk()); - UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT col1, col2, a.col3, FROM plato.Input AS a").IsOk()); - } - - Y_UNIT_TEST(NoStackOverflowOnBigCaseStatement) { - TStringBuilder req; - req << "select case 1 + 123"; - for (size_t i = 0; i < 20000; ++i) { - req << " when " << i << " then " << i + 1; - } - req << " else 100500 end;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(CollectPreaggregatedInListLiteral) { - UNIT_ASSERT(SqlToYql("SELECT [COUNT(DISTINCT a+b)] FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(SmartParenInGroupByClause) { - UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input GROUP BY (k, v)").IsOk()); - } + } else if (word == "userschema") { + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, + line.find(R"__('('('"userschema" (SqlTypeFromYson)__")); + } + }; + + TWordCountHive elementStat = {{TString("MrObject"), 0}, {TString("userschema"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["MrObject"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["userschema"]); + } + + Y_UNIT_TEST(TrailingCommaInWithout) { + UNIT_ASSERT(SqlToYql("SELECT * WITHOUT stream, FROM plato.Input").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT a.intersect, FROM plato.Input AS a").IsOk()); + UNIT_ASSERT(SqlToYql("SELECT a.* WITHOUT col1, col2, a.col3, FROM plato.Input AS a").IsOk()); + } + + Y_UNIT_TEST(NoStackOverflowOnBigCaseStatement) { + TStringBuilder req; + req << "select case 1 + 123"; + for (size_t i = 0; i < 20000; ++i) { + req << " when " << i << " then " << i + 1; + } + req << " else 100500 end;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(CollectPreaggregatedInListLiteral) { + UNIT_ASSERT(SqlToYql("SELECT [COUNT(DISTINCT a+b)] FROM plato.Input").IsOk()); + } + + Y_UNIT_TEST(SmartParenInGroupByClause) { + UNIT_ASSERT(SqlToYql("SELECT * FROM plato.Input GROUP BY (k, v)").IsOk()); + } Y_UNIT_TEST(AlterTableRenameToIsCorrect) { UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table RENAME TO moved").IsOk()); @@ -1589,19 +1589,19 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { Y_UNIT_TEST(AlterTableSetPartitioningIsCorrect) { UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (AUTO_PARTITIONING_BY_SIZE = DISABLED)").IsOk()); } - - Y_UNIT_TEST(OptionalAliases) { - UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); - UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM (VALUES (1,2), (3,4)) a(x,key) JOIN Input b ON a.key = b.key;").IsOk()); - } - - Y_UNIT_TEST(TableNameConstness) { - UNIT_ASSERT(SqlToYql("USE plato; $path = 'foo'; SELECT TableName($path), count(*) FROM Input;").IsOk()); - UNIT_ASSERT(SqlToYql("$path = 'foo'; SELECT TableName($path, 'yt'), count(*) FROM plato.Input;").IsOk()); - ExpectFailWithError("USE plato; SELECT TableName(), count(*) FROM plato.Input;", - "<main>:1:19: Error: Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery\n"); - } + + Y_UNIT_TEST(OptionalAliases) { + UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); + UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM (VALUES (1,2), (3,4)) a(x,key) JOIN Input b ON a.key = b.key;").IsOk()); + } + + Y_UNIT_TEST(TableNameConstness) { + UNIT_ASSERT(SqlToYql("USE plato; $path = 'foo'; SELECT TableName($path), count(*) FROM Input;").IsOk()); + UNIT_ASSERT(SqlToYql("$path = 'foo'; SELECT TableName($path, 'yt'), count(*) FROM plato.Input;").IsOk()); + ExpectFailWithError("USE plato; SELECT TableName(), count(*) FROM plato.Input;", + "<main>:1:19: Error: Expression has to be an aggregation function or key column, because aggregation is used elsewhere in this subquery\n"); + } } Y_UNIT_TEST_SUITE(ExternalFunction) { @@ -1653,14 +1653,14 @@ Y_UNIT_TEST_SUITE(ExternalFunction) { ExpectFailWithError("PROCESS plato.Input USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', 'bar', 'baz')", "<main>:1:15: Error: EXTERNAL FUNCTION requires from 2 to 3 arguments, but got: 4\n"); - ExpectFailWithError("PROCESS plato.Input\n" - " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|field_1: a1, field_b: b1|>)\n" - " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>,\n" - " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con',\n" - " CONCURRENCY=5, INPUT_TYPE=Struct<b:Bool>,\n" - " INIT=[0, 900]\n", - "<main>:5:2: Error: WITH \"CONCURRENCY\" clause should be specified only once\n" - "<main>:5:17: Error: WITH \"INPUT_TYPE\" clause should be specified only once\n"); + ExpectFailWithError("PROCESS plato.Input\n" + " USING EXTERNAL FUNCTION('YANDEX-CLOUD', 'foo', <|field_1: a1, field_b: b1|>)\n" + " WITH INPUT_TYPE=Struct<a:Int32>, OUTPUT_TYPE=Struct<b:Int32>,\n" + " CONCURRENCY=3, BATCH_SIZE=1000000, CONNECTION='yc-folder34fse-con',\n" + " CONCURRENCY=5, INPUT_TYPE=Struct<b:Bool>,\n" + " INIT=[0, 900]\n", + "<main>:5:2: Error: WITH \"CONCURRENCY\" clause should be specified only once\n" + "<main>:5:17: Error: WITH \"INPUT_TYPE\" clause should be specified only once\n"); } } @@ -1680,51 +1680,51 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(a1, a2); } - Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { - NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); - NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); - UNIT_ASSERT(!res1.Root); - UNIT_ASSERT(!res2.Root); - - UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n" - "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n"); - UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n" - "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n"); - } - - Y_UNIT_TEST(InvalidHexInStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - - Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { - NYql::TAstParseResult res = SqlToYql("select \"foo\n" - "bar\n" - "\\01\""); - UNIT_ASSERT(!res.Root); - TString a1 = Err2Str(res); - TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; - - UNIT_ASSERT_NO_DIFF(a1, a2); - } - + Y_UNIT_TEST(IvalidStringLiteralWithEscapedBackslash) { + NYql::TAstParseResult res1 = SqlToYql(R"foo($bar = 'a\\'b';)foo"); + NYql::TAstParseResult res2 = SqlToYql(R"foo($bar = "a\\"b";)foo"); + UNIT_ASSERT(!res1.Root); + UNIT_ASSERT(!res2.Root); + + UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n" + "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n" + "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n"); + } + + Y_UNIT_TEST(InvalidHexInStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\\x1\\xfe\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:1:15: Error: Failed to parse string literal: Invalid hexadecimal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + + Y_UNIT_TEST(InvalidOctalInMultilineStringLiteral) { + NYql::TAstParseResult res = SqlToYql("select \"foo\n" + "bar\n" + "\\01\""); + UNIT_ASSERT(!res.Root); + TString a1 = Err2Str(res); + TString a2 = "<main>:3:4: Error: Failed to parse string literal: Invalid octal value\n"; + + UNIT_ASSERT_NO_DIFF(a1, a2); + } + Y_UNIT_TEST(InvalidDoubleAtString) { NYql::TAstParseResult res = SqlToYql("select @@@@@@"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + } + + Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { + NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); } - Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) { - NYql::TAstParseResult res = SqlToYql("SELECT @@foo@@ @ @@bar@@"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:7: Error: Unexpected token '@@foo@@' : cannot match to any predicted input...\n\n"); - } - Y_UNIT_TEST(InvalidStringFromTable) { NYql::TAstParseResult res = SqlToYql("select \"FOO\"\"BAR from plato.foo"); UNIT_ASSERT(!res.Root); @@ -1734,14 +1734,14 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(InvalidDoubleAtStringFromTable) { NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); } Y_UNIT_TEST(SelectInvalidSyntax) { NYql::TAstParseResult res = SqlToYql("select 1 form Wat"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unexpected token 'Wat' : cannot match to any predicted input...\n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:14: Error: Unexpected token 'Wat' : cannot match to any predicted input...\n\n"); } Y_UNIT_TEST(SelectNoCluster) { @@ -1777,7 +1777,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { "FROM plato.Input AS a\n" "WHERE CASE WHEN a.key = \"foo\" a.subkey ELSE a.value END\n" ); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: Unexpected token absence : Missing THEN \n\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:30: Error: Unexpected token absence : Missing THEN \n\n"); } Y_UNIT_TEST(SelectCaseWithoutEnd) { @@ -1790,144 +1790,144 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { NYql::TAstParseResult res = SqlToYql("select a, Min(b), c"); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" "<main>:1:8: Error: Column reference 'a'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" "<main>:1:15: Error: Column reference 'b'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" "<main>:1:19: Error: Column reference 'c'\n" ); } Y_UNIT_TEST(SelectWithBadAggregation) { - ExpectFailWithError("select count(*), 1 + key from plato.Input", - "<main>:1:22: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select count(*), 1 + key from plato.Input", + "<main>:1:22: Error: Column `key` must either be a key column in GROUP BY or it should be used in aggregation function\n"); } Y_UNIT_TEST(SelectWithBadAggregatedTerms) { - ExpectFailWithError("select key, 2 * subkey from plato.Input group by key", - "<main>:1:17: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select key, 2 * subkey from plato.Input group by key", + "<main>:1:17: Error: Column `subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); } Y_UNIT_TEST(SelectWithBadAggregationInHaving) { - ExpectFailWithError("select key from plato.Input group by key\n" - "having \"f\" || value == \"foo\"", - "<main>:2:15: Error: Column `value` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(JoinWithNonAggregatedColumnInProjection) { - ExpectFailWithError("select a.key, 1 + b.subkey\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column `b.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select a.key, 1 + b.subkey.x\n" - "from plato.Input1 as a join plato.Input2 as b using(key)\n" - "group by a.key;", - "<main>:1:19: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(SelectWithBadAggregatedTermsWithSources) { - ExpectFailWithError("select key, 1 + a.subkey\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - ExpectFailWithError("select key, 1 + a.subkey.x\n" - "from plato.Input1 as a\n" - "group by a.key;", - "<main>:1:17: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(WarnForAggregationBySelectAlias) { - NYql::TAstParseResult res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by c"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:11: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - - res = SqlToYql("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2) as c;"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenAggrFunctionsAreUsedInAlias) { - NYql::TAstParseResult res = SqlToYql("select\n" - " cast(avg(val) as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select\n" - " cast(avg(val) over w as int) as value,\n" - " value as key\n" - "from\n" - " plato.Input\n" - "group by value\n" - "window w as ()"); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenQualifiedNameIsUsed) { - NYql::TAstParseResult res = SqlToYql("select\n" - " Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "join plato.Input2 as b using(k)\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select Unwrap(a.key) as key\n" - "from plato.Input as a\n" - "group by a.key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenTrivialRenamingIsUsed) { - NYql::TAstParseResult res = SqlToYql("select a.key as key\n" - "from plato.Input as a\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - - res = SqlToYql("select key as key\n" - "from plato.Input\n" - "group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(ErrorByAggregatingByExpressionWithSameExpressionInSelect) { - ExpectFailWithError("select k * 2 from plato.Input group by k * 2", - "<main>:1:8: Error: Column `k` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - - Y_UNIT_TEST(ErrorForAggregationBySelectAlias) { - ExpectFailWithError("select key, Math::Floor(1.1 + a.subkey) as foo\n" - "from plato.Input as a\n" - "group by a.key, foo;", - "<main>:3:17: Warning: GROUP BY will aggregate by column `foo` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:19: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:31: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - - ExpectFailWithError("select c + 1 as c from plato.Input\n" - "group by Math::Floor(c + 2);", - "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" - "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" - "<main>:1:8: Error: Column `c` must either be a key column in GROUP BY or it should be used in aggregation function\n"); - } - + ExpectFailWithError("select key from plato.Input group by key\n" + "having \"f\" || value == \"foo\"", + "<main>:2:15: Error: Column `value` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(JoinWithNonAggregatedColumnInProjection) { + ExpectFailWithError("select a.key, 1 + b.subkey\n" + "from plato.Input1 as a join plato.Input2 as b using(key)\n" + "group by a.key;", + "<main>:1:19: Error: Column `b.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + + ExpectFailWithError("select a.key, 1 + b.subkey.x\n" + "from plato.Input1 as a join plato.Input2 as b using(key)\n" + "group by a.key;", + "<main>:1:19: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(SelectWithBadAggregatedTermsWithSources) { + ExpectFailWithError("select key, 1 + a.subkey\n" + "from plato.Input1 as a\n" + "group by a.key;", + "<main>:1:17: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select key, 1 + a.subkey.x\n" + "from plato.Input1 as a\n" + "group by a.key;", + "<main>:1:17: Error: Column must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(WarnForAggregationBySelectAlias) { + NYql::TAstParseResult res = SqlToYql("select c + 1 as c from plato.Input\n" + "group by c"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:2:11: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); + + res = SqlToYql("select c + 1 as c from plato.Input\n" + "group by Math::Floor(c + 2) as c;"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n"); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenAggrFunctionsAreUsedInAlias) { + NYql::TAstParseResult res = SqlToYql("select\n" + " cast(avg(val) as int) as value,\n" + " value as key\n" + "from\n" + " plato.Input\n" + "group by value"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select\n" + " cast(avg(val) over w as int) as value,\n" + " value as key\n" + "from\n" + " plato.Input\n" + "group by value\n" + "window w as ()"); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenQualifiedNameIsUsed) { + NYql::TAstParseResult res = SqlToYql("select\n" + " Unwrap(a.key) as key\n" + "from plato.Input as a\n" + "join plato.Input2 as b using(k)\n" + "group by a.key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select Unwrap(a.key) as key\n" + "from plato.Input as a\n" + "group by a.key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(NoWarnForAggregationBySelectAliasWhenTrivialRenamingIsUsed) { + NYql::TAstParseResult res = SqlToYql("select a.key as key\n" + "from plato.Input as a\n" + "group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + + res = SqlToYql("select key as key\n" + "from plato.Input\n" + "group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(ErrorByAggregatingByExpressionWithSameExpressionInSelect) { + ExpectFailWithError("select k * 2 from plato.Input group by k * 2", + "<main>:1:8: Error: Column `k` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + + Y_UNIT_TEST(ErrorForAggregationBySelectAlias) { + ExpectFailWithError("select key, Math::Floor(1.1 + a.subkey) as foo\n" + "from plato.Input as a\n" + "group by a.key, foo;", + "<main>:3:17: Warning: GROUP BY will aggregate by column `foo` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:19: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" + "<main>:1:31: Error: Column `a.subkey` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + + ExpectFailWithError("select c + 1 as c from plato.Input\n" + "group by Math::Floor(c + 2);", + "<main>:2:22: Warning: GROUP BY will aggregate by column `c` instead of aggregating by SELECT expression with same alias, code: 4532\n" + "<main>:1:10: Warning: You should probably use alias in GROUP BY instead of using it here. Please consult documentation for more details, code: 4532\n" + "<main>:1:8: Error: Column `c` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + } + Y_UNIT_TEST(SelectWithDuplicateGroupingColumns) { NYql::TAstParseResult res = SqlToYql("select c from plato.Input group by c, c"); UNIT_ASSERT(!res.Root); @@ -1937,13 +1937,13 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(SelectWithBadAggregationInGrouping) { NYql::TAstParseResult res = SqlToYql("select a, Min(b), c group by c"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:30: Error: Column reference 'c'\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:30: Error: Column reference 'c'\n"); } Y_UNIT_TEST(SelectWithOpOnBadAggregation) { - ExpectFailWithError("select 1 + a + Min(b) from plato.Input", - "<main>:1:12: Error: Column `a` must either be a key column in GROUP BY or it should be used in aggregation function\n"); + ExpectFailWithError("select 1 + a + Min(b) from plato.Input", + "<main>:1:12: Error: Column `a` must either be a key column in GROUP BY or it should be used in aggregation function\n"); } Y_UNIT_TEST(SelectOrderByConstantNum) { @@ -1994,26 +1994,26 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:65: Error: JOIN: column requires correlation name\n"); } - Y_UNIT_TEST(SelectJoinMissingCorrName1) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 join $foo USING(key);\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:27: Error: JOIN: missing correlation name for source\n"); - } - - Y_UNIT_TEST(SelectJoinMissingCorrName2) { - NYql::TAstParseResult res = SqlToYql( - "use plato;\n" - "$foo = select * from Input1;\n" - "select * from Input2 cross join $foo;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:33: Error: JOIN: missing correlation name for source\n"); - } - + Y_UNIT_TEST(SelectJoinMissingCorrName1) { + NYql::TAstParseResult res = SqlToYql( + "use plato;\n" + "$foo = select * from Input1;\n" + "select * from Input2 join $foo USING(key);\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:27: Error: JOIN: missing correlation name for source\n"); + } + + Y_UNIT_TEST(SelectJoinMissingCorrName2) { + NYql::TAstParseResult res = SqlToYql( + "use plato;\n" + "$foo = select * from Input1;\n" + "select * from Input2 cross join $foo;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:33: Error: JOIN: missing correlation name for source\n"); + } + Y_UNIT_TEST(SelectJoinEmptyCorrNames) { NYql::TAstParseResult res = SqlToYql( "$left = (SELECT * FROM plato.Input1 LIMIT 2);\n" @@ -2033,7 +2033,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(SelectJoinConstPredicateArg) { NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input1 as A JOIN plato.Input2 as B ON A.key == B.key AND A.subkey == \"wtf\"\n"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN: each equality predicate argument must depend on exactly one JOIN input\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:87: Error: JOIN: each equality predicate argument must depend on exactly one JOIN input\n"); } Y_UNIT_TEST(SelectJoinNonEqualityPredicate) { @@ -2060,22 +2060,22 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:79: Error: JOIN ON equality predicate must have one of its arguments from the rightmost source\n"); } - Y_UNIT_TEST(SelectEquiJoinOuterWithoutType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A Outer JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Invalid join type: OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - - Y_UNIT_TEST(SelectEquiJoinOuterWithWrongType) { - NYql::TAstParseResult res = SqlToYql( - "SELECT * FROM plato.A LEFT semi OUTER JOIN plato.B ON A.key == B.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Invalid join type: LEFT SEMI OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); - } - + Y_UNIT_TEST(SelectEquiJoinOuterWithoutType) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.A Outer JOIN plato.B ON A.key == B.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Invalid join type: OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); + } + + Y_UNIT_TEST(SelectEquiJoinOuterWithWrongType) { + NYql::TAstParseResult res = SqlToYql( + "SELECT * FROM plato.A LEFT semi OUTER JOIN plato.B ON A.key == B.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:33: Error: Invalid join type: LEFT SEMI OUTER JOIN. OUTER keyword is optional and can only be used after LEFT, RIGHT or FULL\n"); + } + Y_UNIT_TEST(InsertNoCluster) { NYql::TAstParseResult res = SqlToYql("insert into Output (foo) values (1)"); UNIT_ASSERT(!res.Root); @@ -2127,7 +2127,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(ReduceDistinct) { NYql::TAstParseResult res = SqlToYql("reduce plato.Input on key using some::udf(distinct value)"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: DISTINCT can not be used in PROCESS/REDUCE\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: DISTINCT can not be used in PROCESS/REDUCE\n"); } Y_UNIT_TEST(CreateTableWithView) { @@ -2139,13 +2139,13 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(AsteriskWithSomethingAfter) { NYql::TAstParseResult res = SqlToYql("select *, LENGTH(value) from plato.Input;"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); } Y_UNIT_TEST(AsteriskWithSomethingBefore) { NYql::TAstParseResult res = SqlToYql("select LENGTH(value), * from plato.Input;"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:23: Error: Unable to use plain '*' with other projection items. Please use qualified asterisk instead: '<table>.*' (<table> can be either table name or table alias).\n"); } Y_UNIT_TEST(DuplicatedQualifiedAsterisk) { @@ -2250,12 +2250,12 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate alias found: kk in FlattenBy section\n"); } - Y_UNIT_TEST(SelectFlattenByExprSameAliases) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as kk);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: kk in FlattenBy section\n"); - } - + Y_UNIT_TEST(SelectFlattenByExprSameAliases) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as kk);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: kk in FlattenBy section\n"); + } + Y_UNIT_TEST(SelectFlattenByConflictNameAndAlias0) { NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, subkey as key);"); UNIT_ASSERT(!res.Root); @@ -2268,39 +2268,39 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Collision between alias and column name: key in FlattenBy section\n"); } - Y_UNIT_TEST(SelectFlattenByExprConflictNameAndAlias1) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as key);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate column name found: key in FlattenBy section\n"); - } - - Y_UNIT_TEST(SelectFlattenByUnnamedExpr) { - NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, ListSkip(key, 1))"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Unnamed expression after FLATTEN BY is not allowed\n"); - } - + Y_UNIT_TEST(SelectFlattenByExprConflictNameAndAlias1) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key as kk, ListSkip(subkey,1) as key);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Error: Duplicate column name found: key in FlattenBy section\n"); + } + + Y_UNIT_TEST(SelectFlattenByUnnamedExpr) { + NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, ListSkip(key, 1))"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:46: Error: Unnamed expression after FLATTEN BY is not allowed\n"); + } + Y_UNIT_TEST(UseInOnStrings) { NYql::TAstParseResult res = SqlToYql("select * from plato.Input where \"foo\" in \"foovalue\";"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " - "expecting tuple, list, dict or single column table source\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:42: Error: Unable to use IN predicate with string argument, it won't search substring - " + "expecting tuple, list, dict or single column table source\n"); } - Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { - NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); - UNIT_ASSERT(res.Root); + Y_UNIT_TEST(UseSubqueryInScalarContextInsideIn) { + NYql::TAstParseResult res = SqlToYql("$q = (select key from plato.Input); select * from plato.Input where subkey in ($q);"); + UNIT_ASSERT(res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:79: Warning: Using subrequest in scalar context after IN, " "perhaps you should remove parenthesis here, code: 4501\n"); - } - - Y_UNIT_TEST(InHintsWithKeywordClash) { + } + + Y_UNIT_TEST(InHintsWithKeywordClash) { NYql::TAstParseResult res = SqlToYql("SELECT COMPACT FROM plato.Input WHERE COMPACT IN COMPACT `COMPACT`(1,2,3)"); - UNIT_ASSERT(!res.Root); - // should try to parse last compact as call expression + UNIT_ASSERT(!res.Root); + // should try to parse last compact as call expression UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:58: Error: Unknown builtin: COMPACT\n"); - } - + } + Y_UNIT_TEST(ErrorColumnPosition) { NYql::TAstParseResult res = SqlToYql( "USE plato;\n" @@ -2382,26 +2382,26 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(GroupingSetByExprWithoutAlias) { NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY GROUPING SETS (cast(key as uint32), subkey);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); + } + + Y_UNIT_TEST(GroupingSetByExprWithoutAlias2) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY subkey || subkey, GROUPING SETS (\n" + "cast(key as uint32), subkey);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:1: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); } - Y_UNIT_TEST(GroupingSetByExprWithoutAlias2) { - NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY subkey || subkey, GROUPING SETS (\n" - "cast(key as uint32), subkey);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:1: Error: Unnamed expressions are not supported in GROUPING SETS. Please use '<expr> AS <name>'.\n"); - } - Y_UNIT_TEST(CubeByExprWithoutAlias) { NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY CUBE (key, subkey / key);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: Unnamed expressions are not supported in CUBE. Please use '<expr> AS <name>'.\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:56: Error: Unnamed expressions are not supported in CUBE. Please use '<expr> AS <name>'.\n"); } Y_UNIT_TEST(RollupByExprWithoutAlias) { NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input GROUP BY ROLLUP (subkey / key);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in ROLLUP. Please use '<expr> AS <name>'.\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:53: Error: Unnamed expressions are not supported in ROLLUP. Please use '<expr> AS <name>'.\n"); } Y_UNIT_TEST(GroupByHugeCubeDeniedNoPragma) { @@ -2413,7 +2413,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(GroupByInvalidPragma) { NYql::TAstParseResult res = SqlToYql("PRAGMA GroupByCubeLimit = '-4';"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Expected unsigned integer literal as a single argument for: GroupByCubeLimit\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:27: Error: Expected unsigned integer literal as a single argument for: GroupByCubeLimit\n"); } Y_UNIT_TEST(GroupByHugeCubeDeniedPragme) { @@ -2439,31 +2439,31 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { "select count(1), key_first, val_first, grouping(key_first, val_first, nomind) as group\n" "from plato.Input group by grouping sets (cast(key as uint32) /100 as key_first, Substring(value, 1, 1) as val_first);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:71: Error: Column 'nomind' is not a grouping column\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:71: Error: Column 'nomind' is not a grouping column\n"); } Y_UNIT_TEST(NoGroupingColumn1) { NYql::TAstParseResult res = SqlToYql("select count(1), grouping(key, value) as group_duo from plato.Input group by cube (key, subkey);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Column 'value' is not a grouping column\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:32: Error: Column 'value' is not a grouping column\n"); } Y_UNIT_TEST(EmptyAccess0) { NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(``));"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"\" is not allowed in current scope\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"\" is not allowed in current scope\n"); } Y_UNIT_TEST(EmptyAccess1) { NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), ``);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: Column reference \"\" is not allowed in current scope\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:66: Error: Column reference \"\" is not allowed in current scope\n"); } Y_UNIT_TEST(UseUnknownColumnInInsert) { NYql::TAstParseResult res = SqlToYql("insert into plato.Output (list0, list1) values (AsList(0, 1, 2), AsList(`test`));"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"test\" is not allowed in current scope\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:73: Error: Column reference \"test\" is not allowed in current scope\n"); } Y_UNIT_TEST(GroupByEmptyColumn) { @@ -2472,35 +2472,35 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:43: Error: Column name can not be empty\n"); } - Y_UNIT_TEST(ConvertNumberOutOfBase) { + Y_UNIT_TEST(ConvertNumberOutOfBase) { NYql::TAstParseResult res = SqlToYql("select 0o80l;"); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0o80l, char: '8' is out of base: 8\n"); } - Y_UNIT_TEST(ConvertNumberOutOfRangeForInt64ButFitsInUint64) { + Y_UNIT_TEST(ConvertNumberOutOfRangeForInt64ButFitsInUint64) { NYql::TAstParseResult res = SqlToYql("select 0xc000000000000000l;"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse 13835058055282163712 as integer literal of Int64 type: value out of range for Int64\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse 13835058055282163712 as integer literal of Int64 type: value out of range for Int64\n"); } - Y_UNIT_TEST(ConvertNumberOutOfRangeUint64) { + Y_UNIT_TEST(ConvertNumberOutOfRangeUint64) { NYql::TAstParseResult res = SqlToYql("select 0xc0000000000000000l;"); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 0xc0000000000000000l, number limit overflow\n"); - - res = SqlToYql("select 1234234543563435151456;\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); - } - - Y_UNIT_TEST(ConvertNumberNegativeOutOfRange) { - NYql::TAstParseResult res = SqlToYql("select -9223372036854775808;\n" - "select -9223372036854775809;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Error: Failed to parse negative integer: -9223372036854775809, number limit overflow\n"); - } - + + res = SqlToYql("select 1234234543563435151456;\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to parse number from string: 1234234543563435151456, number limit overflow\n"); + } + + Y_UNIT_TEST(ConvertNumberNegativeOutOfRange) { + NYql::TAstParseResult res = SqlToYql("select -9223372036854775808;\n" + "select -9223372036854775809;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Error: Failed to parse negative integer: -9223372036854775809, number limit overflow\n"); + } + Y_UNIT_TEST(InvaildUsageReal0) { NYql::TAstParseResult res = SqlToYql("select .0;"); UNIT_ASSERT(!res.Root); @@ -2516,7 +2516,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { Y_UNIT_TEST(InvaildUsageWinFunctionWithoutWindow) { NYql::TAstParseResult res = SqlToYql("select lead(key, 2) from plato.Input;"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to use window function Lead without window specification\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Failed to use window function Lead without window specification\n"); } Y_UNIT_TEST(DropTableWithIfExists) { @@ -2565,55 +2565,55 @@ select FormatType($f()); UNIT_ASSERT(res.Root); } - Y_UNIT_TEST(BlockedInvalidFrameBounds) { - auto check = [](const TString& frame, const TString& err) { - const TString prefix = "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n"; - NYql::TAstParseResult res = SqlToYql(prefix + frame + ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), err); - }; - - check("ROWS UNBOUNDED FOLLOWING", "<main>:2:5: Error: Frame cannot start from UNBOUNDED FOLLOWING\n"); - check("ROWS BETWEEN 5 PRECEDING AND UNBOUNDED PRECEDING", "<main>:2:29: Error: Frame cannot end with UNBOUNDED PRECEDING\n"); - check("ROWS BETWEEN CURRENT ROW AND 5 PRECEDING", "<main>:2:13: Error: Frame cannot start from CURRENT ROW and end with PRECEDING\n"); - check("ROWS BETWEEN 5 FOLLOWING AND CURRENT ROW", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with CURRENT ROW\n"); - check("ROWS BETWEEN 5 FOLLOWING AND 5 PRECEDING", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with PRECEDING\n"); - } - - Y_UNIT_TEST(NoColumnsInFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (ROWS BETWEEN\n" - " 1 + key PRECEDING AND 2 + key FOLLOWING);"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:6: Error: Column reference \"key\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(WarnOnEmptyFrameBounds) { - NYql::TAstParseResult res = SqlToYql( - "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n" - "ROWS BETWEEN 10 FOLLOWING AND 5 FOLLOWING)"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:14: Warning: Used frame specification implies empty window frame, code: 4520\n"); - } - - Y_UNIT_TEST(WarnOnRankWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK() OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank() is used with unordered window - all rows will be considered equal to each other, code: 4521\n"); - } - - Y_UNIT_TEST(WarnOnRankExprWithUnorderedWindow) { - NYql::TAstParseResult res = SqlToYql("SELECT RANK(key) OVER w FROM plato.Input WINDOW w AS ()"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank(<expression>) is used with unordered window - the result is likely to be undefined, code: 4521\n"); - } - - Y_UNIT_TEST(AnyAsTableName) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected token ';' : syntax error...\n\n"); - } - + Y_UNIT_TEST(BlockedInvalidFrameBounds) { + auto check = [](const TString& frame, const TString& err) { + const TString prefix = "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n"; + NYql::TAstParseResult res = SqlToYql(prefix + frame + ")"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), err); + }; + + check("ROWS UNBOUNDED FOLLOWING", "<main>:2:5: Error: Frame cannot start from UNBOUNDED FOLLOWING\n"); + check("ROWS BETWEEN 5 PRECEDING AND UNBOUNDED PRECEDING", "<main>:2:29: Error: Frame cannot end with UNBOUNDED PRECEDING\n"); + check("ROWS BETWEEN CURRENT ROW AND 5 PRECEDING", "<main>:2:13: Error: Frame cannot start from CURRENT ROW and end with PRECEDING\n"); + check("ROWS BETWEEN 5 FOLLOWING AND CURRENT ROW", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with CURRENT ROW\n"); + check("ROWS BETWEEN 5 FOLLOWING AND 5 PRECEDING", "<main>:2:14: Error: Frame cannot start from FOLLOWING and end with PRECEDING\n"); + } + + Y_UNIT_TEST(NoColumnsInFrameBounds) { + NYql::TAstParseResult res = SqlToYql( + "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (ROWS BETWEEN\n" + " 1 + key PRECEDING AND 2 + key FOLLOWING);"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:6: Error: Column reference \"key\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(WarnOnEmptyFrameBounds) { + NYql::TAstParseResult res = SqlToYql( + "SELECT SUM(x) OVER w FROM plato.Input WINDOW w AS (PARTITION BY key ORDER BY subkey\n" + "ROWS BETWEEN 10 FOLLOWING AND 5 FOLLOWING)"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:14: Warning: Used frame specification implies empty window frame, code: 4520\n"); + } + + Y_UNIT_TEST(WarnOnRankWithUnorderedWindow) { + NYql::TAstParseResult res = SqlToYql("SELECT RANK() OVER w FROM plato.Input WINDOW w AS ()"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank() is used with unordered window - all rows will be considered equal to each other, code: 4521\n"); + } + + Y_UNIT_TEST(WarnOnRankExprWithUnorderedWindow) { + NYql::TAstParseResult res = SqlToYql("SELECT RANK(key) OVER w FROM plato.Input WINDOW w AS ()"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Warning: Rank(<expression>) is used with unordered window - the result is likely to be undefined, code: 4521\n"); + } + + Y_UNIT_TEST(AnyAsTableName) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from any;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected token ';' : syntax error...\n\n"); + } + Y_UNIT_TEST(IncorrectOrderOfLambdaOptionalArgs) { NYql::TAstParseResult res = SqlToYql("$f = ($x?, $y)->($x + $y); select $f(1);"); UNIT_ASSERT(!res.Root); @@ -2629,59 +2629,59 @@ select FormatType($f()); Y_UNIT_TEST(NotAllowedQuestionOnNamedNode) { NYql::TAstParseResult res = SqlToYql("$f = 1; select $f?;"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(AnyAndCrossJoin) { - NYql::TAstParseResult res = SqlToYql("use plato; select * from any Input1 cross join Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:26: Error: ANY should not be used with Cross JOIN\n"); - - res = SqlToYql("use plato; select * from Input1 cross join any Input2"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:44: Error: ANY should not be used with Cross JOIN\n"); - } - - Y_UNIT_TEST(ErrorPlainEndAsInlineActionTerminator) { - NYql::TAstParseResult res = SqlToYql( - "do begin\n" - " select 1\n" - "; end\n"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token absence : Missing DO \n\n"); - } - - Y_UNIT_TEST(ErrorMultiWayJoinWithUsing) { - NYql::TAstParseResult res = SqlToYql( - "USE plato;\n" - "PRAGMA DisableSimpleColumns;\n" - "SELECT *\n" - "FROM Input1 AS a\n" - "JOIN Input2 AS b USING(key)\n" - "JOIN Input3 AS c ON a.key = c.key;\n" - ); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:5:24: Error: Multi-way JOINs should be connected with ON clause instead of USING clause\n" - ); - } - - Y_UNIT_TEST(RequireLabelInFlattenByWithDot) { - NYql::TAstParseResult res = SqlToYql("select * from plato.Input flatten by x.y"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), - "<main>:1:40: Error: Unnamed expression after FLATTEN BY is not allowed\n" - ); - } - - Y_UNIT_TEST(WarnUnnamedColumns) { - NYql::TAstParseResult res = SqlToYql( - "PRAGMA WarnUnnamedColumns;\n" - "\n" - "SELECT key, subkey, key || subkey FROM plato.Input ORDER BY subkey;\n"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); - } + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(AnyAndCrossJoin) { + NYql::TAstParseResult res = SqlToYql("use plato; select * from any Input1 cross join Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:26: Error: ANY should not be used with Cross JOIN\n"); + + res = SqlToYql("use plato; select * from Input1 cross join any Input2"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:44: Error: ANY should not be used with Cross JOIN\n"); + } + + Y_UNIT_TEST(ErrorPlainEndAsInlineActionTerminator) { + NYql::TAstParseResult res = SqlToYql( + "do begin\n" + " select 1\n" + "; end\n"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token absence : Missing DO \n\n"); + } + + Y_UNIT_TEST(ErrorMultiWayJoinWithUsing) { + NYql::TAstParseResult res = SqlToYql( + "USE plato;\n" + "PRAGMA DisableSimpleColumns;\n" + "SELECT *\n" + "FROM Input1 AS a\n" + "JOIN Input2 AS b USING(key)\n" + "JOIN Input3 AS c ON a.key = c.key;\n" + ); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:5:24: Error: Multi-way JOINs should be connected with ON clause instead of USING clause\n" + ); + } + + Y_UNIT_TEST(RequireLabelInFlattenByWithDot) { + NYql::TAstParseResult res = SqlToYql("select * from plato.Input flatten by x.y"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), + "<main>:1:40: Error: Unnamed expression after FLATTEN BY is not allowed\n" + ); + } + + Y_UNIT_TEST(WarnUnnamedColumns) { + NYql::TAstParseResult res = SqlToYql( + "PRAGMA WarnUnnamedColumns;\n" + "\n" + "SELECT key, subkey, key || subkey FROM plato.Input ORDER BY subkey;\n"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:28: Warning: Autogenerated column name column2 will be used for expression, code: 4516\n"); + } Y_UNIT_TEST(WarnSourceColumnMismatch) { NYql::TAstParseResult res = SqlToYql( @@ -2729,220 +2729,220 @@ select FormatType($f()); UNIT_ASSERT(!res.Root); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n"); } - - Y_UNIT_TEST(YsonFuncWithoutArgs) { - UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk()); - } - - Y_UNIT_TEST(CanNotUseOrderByInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseLimitInNonLastSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input limit 1\n" - "union all\n" - "select * from Input order by key limit 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(CanNotUseDiscardInNonFirstSelectInUnionAllChain) { - auto req = "pragma AnsiOrderByLimitInUnionAll;\n" - "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(CanNotUseIntoResultInNonLastSelectInUnionAllChain) { - auto req = "use plato;\n" - "pragma AnsiOrderByLimitInUnionAll;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } + + Y_UNIT_TEST(YsonFuncWithoutArgs) { + UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk()); + } + + Y_UNIT_TEST(CanNotUseOrderByInNonLastSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input order by key\n" + "union all\n" + "select * from Input order by key limit 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(CanNotUseLimitInNonLastSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input limit 1\n" + "union all\n" + "select * from Input order by key limit 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(CanNotUseDiscardInNonFirstSelectInUnionAllChain) { + auto req = "pragma AnsiOrderByLimitInUnionAll;\n" + "use plato;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } + + Y_UNIT_TEST(CanNotUseIntoResultInNonLastSelectInUnionAllChain) { + auto req = "use plato;\n" + "pragma AnsiOrderByLimitInUnionAll;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:6:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } Y_UNIT_TEST(YsonStrictInvalidPragma) { auto res = SqlToYql("pragma yson.Strict = \"wrong\";"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Expected 'true', 'false' or no parameter for: Strict\n"); - } - - Y_UNIT_TEST(WarnTableNameInSomeContexts) { - UNIT_ASSERT(SqlToYql("use plato; select TableName() from Input;").IsOk()); - UNIT_ASSERT(SqlToYql("use plato; select TableName(\"aaaa\");").IsOk()); - UNIT_ASSERT(SqlToYql("select TableName(\"aaaa\", \"yt\");").IsOk()); - - auto res = SqlToYql("select TableName() from plato.Input;"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: TableName requires either one of \"yt\"/\"kikimr\"/\"rtmr\" as second argument or current cluster name\n"); - - res = SqlToYql("use plato;\n" - "select TableName() from Input1 as a join Input2 as b using(key);"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Warning: TableName() may produce empty result when used in ambiguous context (with JOIN), code: 4525\n"); - - res = SqlToYql("use plato;\n" - "select SOME(TableName()), key from Input group by key;"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Warning: TableName() will produce empty result when used with aggregation.\n" - "Please consult https://yql.yandex-team.ru/docs/yt/builtins/basic/#tablepath for possible workaround, code: 4525\n"); - } - - Y_UNIT_TEST(WarnOnDistincWithHavingWithoutAggregations) { - auto res = SqlToYql("select distinct key from plato.Input having key != '0';"); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Warning: The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead., code: 4526\n"); - } - - Y_UNIT_TEST(FlattenByExprWithNestedNull) { - auto res = SqlToYql("USE plato;\n" - "\n" - "SELECT * FROM (SELECT 1 AS region_id)\n" - "FLATTEN BY (\n" - " CAST($unknown(region_id) AS List<String>) AS region\n" - ")"); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:10: Error: Unknown name: $unknown\n"); - } - - Y_UNIT_TEST(EmptySymbolNameIsForbidden) { - auto req = " $`` = 1; select $``;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Empty symbol name is not allowed\n"); - } - - Y_UNIT_TEST(WarnOnBinaryOpWithNullArg) { - auto req = "select * from plato.Input where cast(key as Int32) != NULL"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Warning: Binary operation != will return NULL here, code: 4529\n"); - - req = "select 1 or null"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); - } - - Y_UNIT_TEST(ErrorIfTableSampleArgUsesColumns) { - auto req = "SELECT key FROM plato.Input TABLESAMPLE BERNOULLI(MIN_OF(100.0, CAST(subkey as Int32)));"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:70: Error: Column reference \"subkey\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(DerivedColumnListForSelectIsNotSupportedYet) { - auto req = "SELECT a,b,c FROM plato.Input as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:35: Error: Derived column list is only supported for VALUES\n"); - } - - Y_UNIT_TEST(ErrorIfValuesHasDifferentCountOfColumns) { - auto req = "VALUES (1,2,3), (4,5);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: All VALUES items should have same size: expecting 3, got 2\n"); - } - - Y_UNIT_TEST(ErrorIfDerivedColumnSizeExceedValuesColumnCount) { - auto req = "SELECT * FROM(VALUES (1,2), (3,4)) as t(x,y,z);"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: Derived column list size exceeds column count in VALUES\n"); - } - - Y_UNIT_TEST(WarnoOnAutogeneratedNamesForValues) { - auto req = "PRAGMA WarnUnnamedColumns;\n" - "SELECT * FROM (VALUES (1,2,3,4), (5,6,7,8)) as t(x,y);"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:16: Warning: Autogenerated column names column2...column3 will be used here, code: 4516\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithOrderByWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input order by key\n" - "union all\n" - "select * from Input order by key;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithLimitWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input limit 10\n" - "union all\n" - "select * from Input limit 1;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithIntoResultWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input into result aaa\n" - "union all\n" - "select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: INTO RESULT within UNION ALL is only allowed after last subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllWithDiscardWithoutExplicitLegacyMode) { - auto req = "use plato;\n" - "\n" - "select * from Input\n" - "union all\n" - "discard select * from Input;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); - } - - Y_UNIT_TEST(ErrUnionAllKeepsIgnoredOrderByWarning) { - auto req = "use plato;\n" - "\n" - "SELECT * FROM (\n" - " SELECT * FROM Input\n" - " UNION ALL\n" - " SELECT t.* FROM Input AS t ORDER BY t.key\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" - "<main>:6:39: Error: Unknown correlation name: t\n"); - } + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:22: Error: Expected 'true', 'false' or no parameter for: Strict\n"); + } + + Y_UNIT_TEST(WarnTableNameInSomeContexts) { + UNIT_ASSERT(SqlToYql("use plato; select TableName() from Input;").IsOk()); + UNIT_ASSERT(SqlToYql("use plato; select TableName(\"aaaa\");").IsOk()); + UNIT_ASSERT(SqlToYql("select TableName(\"aaaa\", \"yt\");").IsOk()); + + auto res = SqlToYql("select TableName() from plato.Input;"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: TableName requires either one of \"yt\"/\"kikimr\"/\"rtmr\" as second argument or current cluster name\n"); + + res = SqlToYql("use plato;\n" + "select TableName() from Input1 as a join Input2 as b using(key);"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:8: Warning: TableName() may produce empty result when used in ambiguous context (with JOIN), code: 4525\n"); + + res = SqlToYql("use plato;\n" + "select SOME(TableName()), key from Input group by key;"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Warning: TableName() will produce empty result when used with aggregation.\n" + "Please consult https://yql.yandex-team.ru/docs/yt/builtins/basic/#tablepath for possible workaround, code: 4525\n"); + } + + Y_UNIT_TEST(WarnOnDistincWithHavingWithoutAggregations) { + auto res = SqlToYql("select distinct key from plato.Input having key != '0';"); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:49: Warning: The usage of HAVING without aggregations with SELECT DISTINCT is non-standard and will stop working soon. Please use WHERE instead., code: 4526\n"); + } + + Y_UNIT_TEST(FlattenByExprWithNestedNull) { + auto res = SqlToYql("USE plato;\n" + "\n" + "SELECT * FROM (SELECT 1 AS region_id)\n" + "FLATTEN BY (\n" + " CAST($unknown(region_id) AS List<String>) AS region\n" + ")"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:10: Error: Unknown name: $unknown\n"); + } + + Y_UNIT_TEST(EmptySymbolNameIsForbidden) { + auto req = " $`` = 1; select $``;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Empty symbol name is not allowed\n"); + } + + Y_UNIT_TEST(WarnOnBinaryOpWithNullArg) { + auto req = "select * from plato.Input where cast(key as Int32) != NULL"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:52: Warning: Binary operation != will return NULL here, code: 4529\n"); + + req = "select 1 or null"; + res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), ""); + } + + Y_UNIT_TEST(ErrorIfTableSampleArgUsesColumns) { + auto req = "SELECT key FROM plato.Input TABLESAMPLE BERNOULLI(MIN_OF(100.0, CAST(subkey as Int32)));"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:70: Error: Column reference \"subkey\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(DerivedColumnListForSelectIsNotSupportedYet) { + auto req = "SELECT a,b,c FROM plato.Input as t(x,y,z);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:35: Error: Derived column list is only supported for VALUES\n"); + } + + Y_UNIT_TEST(ErrorIfValuesHasDifferentCountOfColumns) { + auto req = "VALUES (1,2,3), (4,5);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: All VALUES items should have same size: expecting 3, got 2\n"); + } + + Y_UNIT_TEST(ErrorIfDerivedColumnSizeExceedValuesColumnCount) { + auto req = "SELECT * FROM(VALUES (1,2), (3,4)) as t(x,y,z);"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: Derived column list size exceeds column count in VALUES\n"); + } + + Y_UNIT_TEST(WarnoOnAutogeneratedNamesForValues) { + auto req = "PRAGMA WarnUnnamedColumns;\n" + "SELECT * FROM (VALUES (1,2,3,4), (5,6,7,8)) as t(x,y);"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:16: Warning: Autogenerated column names column2...column3 will be used here, code: 4516\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithOrderByWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input order by key\n" + "union all\n" + "select * from Input order by key;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: ORDER BY within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithLimitWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input limit 10\n" + "union all\n" + "select * from Input limit 1;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: LIMIT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithIntoResultWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input into result aaa\n" + "union all\n" + "select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:21: Error: INTO RESULT within UNION ALL is only allowed after last subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllWithDiscardWithoutExplicitLegacyMode) { + auto req = "use plato;\n" + "\n" + "select * from Input\n" + "union all\n" + "discard select * from Input;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:5:1: Error: DISCARD within UNION ALL is only allowed before first subquery\n"); + } + + Y_UNIT_TEST(ErrUnionAllKeepsIgnoredOrderByWarning) { + auto req = "use plato;\n" + "\n" + "SELECT * FROM (\n" + " SELECT * FROM Input\n" + " UNION ALL\n" + " SELECT t.* FROM Input AS t ORDER BY t.key\n" + ");"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:3: Warning: ORDER BY without LIMIT in subquery will be ignored, code: 4504\n" + "<main>:6:39: Error: Unknown correlation name: t\n"); + } Y_UNIT_TEST(InvalidTtl) { auto req = R"( @@ -2955,455 +2955,455 @@ select FormatType($f()); UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:26: Error: Literal of Interval type is expected for TTL\n" "<main>:4:26: Error: Invalid TTL settings\n"); } - - Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2, subkey\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY GROUPING SETS(\n" - " (a.key as k1, b.subkey as k2),\n" - " (k1),\n" - " (subkey)\n" - ");"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:8:4: Error: Columns in grouping sets should have correlation name, error in key: subkey\n"); - } - - Y_UNIT_TEST(ErrJoinWithGroupByWithoutCorrelationName) { - auto req = "USE plato;\n" - "\n" - "SELECT k1, k2,\n" - " value\n" - "FROM T1 AS a JOIN T2 AS b USING (key)\n" - "GROUP BY a.key as k1, b.subkey as k2,\n" - " value;"; - ExpectFailWithError(req, - "<main>:7:5: Error: Columns in GROUP BY should have correlation name, error in key: value\n"); - } - - Y_UNIT_TEST(ErrWithMissingFrom) { - auto req = "select 1 as key where 1 > 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:25: Error: Filtering is not allowed without FROM\n"); - - req = "select 1 + count(*);"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Aggregation is not allowed without FROM\n"); - - req = "select 1 as key, subkey + value;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:18: Error: Column reference 'subkey'\n" - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:27: Error: Column reference 'value'\n"); - - req = "select count(1) group by key;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:26: Error: Column reference 'key'\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForWindow) { - auto req = "$c = () -> (1 + count(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:9: Error: Window and aggregation functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "$c = () -> (1 + lead(1) over w);\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:17: Error: Window functions are not allowed in this context\n" - "<main>:1:17: Error: Failed to use window function Lead without window specification or in wrong place\n"); - - req = "select 1 + count(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over w window w as ();"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrWithMissingFromForInplaceWindow) { - auto req = "$c = () -> (1 + count(1) over ());\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:26: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "$c = () -> (1 + lead(1) over (rows between unbounded preceding and current row));\n" - "select $c();"; - ExpectFailWithError(req, - "<main>:1:25: Error: Window and aggregation functions are not allowed in this context\n"); - - req = "select 1 + count(1) over ();"; - ExpectFailWithError(req, - "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); - - req = "select 1 + lead(1) over (rows between current row and unbounded following);"; - ExpectFailWithError(req, - "<main>:1:12: Error: Window functions are not allowed without FROM\n" - "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); - } - - Y_UNIT_TEST(ErrDistinctInWrongPlace) { - auto req = "select Some::Udf(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:18: Error: DISTINCT can only be used in aggregation functions\n"); - req = "select sum(key)(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:17: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "select len(distinct foo) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:8: Error: DISTINCT can only be used in aggregation functions\n"); - - req = "$foo = ($x) -> ($x); select $foo(distinct key) from plato.Input;"; - ExpectFailWithError(req, - "<main>:1:34: Error: DISTINCT can only be used in aggregation functions\n"); - } - - Y_UNIT_TEST(ErrForNotSingleChildInInlineAST) { - ExpectFailWithError("select YQL::\"\"", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - ExpectFailWithError("select YQL::@@ \t@@", - "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); - auto req = "$lambda = YQL::@@(lambda '(x)(+ x x)) (lambda '(y)(+ y y))@@;\n" - "select ListMap([1, 2, 3], $lambda);"; - ExpectFailWithError(req, - "<main>:1:11: Error: Failed to parse YQL: expecting AST root node with single child, but got 2\n"); - } - - Y_UNIT_TEST(ErrEmptyColumnName) { - ExpectFailWithError("select * without \"\" from plato.Input", - "<main>:1:18: Error: String literal can not be used here\n"); - - ExpectFailWithError("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without \"\" from plato.Input", - "<main>:1:18: Error: Empty column name is not allowed\n"); - - ExpectFailWithErrorForAnsiLexer("select * without `` from plato.Input;", - "<main>:1:18: Error: Empty column name is not allowed\n"); - } - - Y_UNIT_TEST(ErrOnNonZeroArgumentsForTableRows) { - ExpectFailWithError("$udf=\"\";process plato.Input using $udf(TableRows(k))", - "<main>:1:40: Error: TableRows requires exactly 0 arguments\n"); - } - - Y_UNIT_TEST(ErrGroupByWithAggregationFunctionAndDistinctExpr) { - ExpectFailWithError("select * from plato.Input group by count(distinct key|key)", - "<main>:1:36: Error: Unable to GROUP BY aggregated values\n"); - } - - // FIXME: check if we can get old behaviour -#if 0 - Y_UNIT_TEST(ErrWithSchemaWithColumnsWithoutType) { - ExpectFailWithError("select * from plato.Input with COLUMNs", - "<main>:1:32: Error: Expected type after COLUMNS\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - - ExpectFailWithError("select * from plato.Input with scheMa", - "<main>:1:32: Error: Expected type after SCHEMA\n" - "<main>:1:32: Error: Failed to parse table hints\n"); - } -#endif - - Y_UNIT_TEST(ErrCollectPreaggregatedInListLiteralWithoutFrom) { - ExpectFailWithError("SELECT([VARIANCE(DISTINCT[])])", - "<main>:1:1: Error: Column references are not allowed without FROM\n" - "<main>:1:9: Error: Column reference '_yql_preagg_Variance0'\n"); - } - - Y_UNIT_TEST(ErrGroupBySmartParenAsTuple) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (k, v,)", - "<main>:1:36: Error: Unable to use tuple in group by clause\n"); - } - - Y_UNIT_TEST(HandleNestedSmartParensInGroupBy) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (+() as k)", - "<main>:1:37: Error: Unable to GROUP BY constant expression\n"); - } - + + Y_UNIT_TEST(ErrJoinWithGroupingSetsWithoutCorrelationName) { + auto req = "USE plato;\n" + "\n" + "SELECT k1, k2, subkey\n" + "FROM T1 AS a JOIN T2 AS b USING (key)\n" + "GROUP BY GROUPING SETS(\n" + " (a.key as k1, b.subkey as k2),\n" + " (k1),\n" + " (subkey)\n" + ");"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:8:4: Error: Columns in grouping sets should have correlation name, error in key: subkey\n"); + } + + Y_UNIT_TEST(ErrJoinWithGroupByWithoutCorrelationName) { + auto req = "USE plato;\n" + "\n" + "SELECT k1, k2,\n" + " value\n" + "FROM T1 AS a JOIN T2 AS b USING (key)\n" + "GROUP BY a.key as k1, b.subkey as k2,\n" + " value;"; + ExpectFailWithError(req, + "<main>:7:5: Error: Columns in GROUP BY should have correlation name, error in key: value\n"); + } + + Y_UNIT_TEST(ErrWithMissingFrom) { + auto req = "select 1 as key where 1 > 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:25: Error: Filtering is not allowed without FROM\n"); + + req = "select 1 + count(*);"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:12: Error: Aggregation is not allowed without FROM\n"); + + req = "select 1 as key, subkey + value;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:18: Error: Column reference 'subkey'\n" + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:27: Error: Column reference 'value'\n"); + + req = "select count(1) group by key;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:26: Error: Column reference 'key'\n"); + } + + Y_UNIT_TEST(ErrWithMissingFromForWindow) { + auto req = "$c = () -> (1 + count(1) over w);\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:9: Error: Window and aggregation functions are not allowed in this context\n" + "<main>:1:17: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "$c = () -> (1 + lead(1) over w);\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:17: Error: Window functions are not allowed in this context\n" + "<main>:1:17: Error: Failed to use window function Lead without window specification or in wrong place\n"); + + req = "select 1 + count(1) over w window w as ();"; + ExpectFailWithError(req, + "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "select 1 + lead(1) over w window w as ();"; + ExpectFailWithError(req, + "<main>:1:12: Error: Window functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); + } + + Y_UNIT_TEST(ErrWithMissingFromForInplaceWindow) { + auto req = "$c = () -> (1 + count(1) over ());\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:26: Error: Window and aggregation functions are not allowed in this context\n"); + + req = "$c = () -> (1 + lead(1) over (rows between unbounded preceding and current row));\n" + "select $c();"; + ExpectFailWithError(req, + "<main>:1:25: Error: Window and aggregation functions are not allowed in this context\n"); + + req = "select 1 + count(1) over ();"; + ExpectFailWithError(req, + "<main>:1:1: Error: Window and aggregation functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use aggregation function Count without window specification or in wrong place\n"); + + req = "select 1 + lead(1) over (rows between current row and unbounded following);"; + ExpectFailWithError(req, + "<main>:1:12: Error: Window functions are not allowed without FROM\n" + "<main>:1:12: Error: Failed to use window function Lead without window specification or in wrong place\n"); + } + + Y_UNIT_TEST(ErrDistinctInWrongPlace) { + auto req = "select Some::Udf(distinct key) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:18: Error: DISTINCT can only be used in aggregation functions\n"); + req = "select sum(key)(distinct foo) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:17: Error: DISTINCT can only be used in aggregation functions\n"); + + req = "select len(distinct foo) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:8: Error: DISTINCT can only be used in aggregation functions\n"); + + req = "$foo = ($x) -> ($x); select $foo(distinct key) from plato.Input;"; + ExpectFailWithError(req, + "<main>:1:34: Error: DISTINCT can only be used in aggregation functions\n"); + } + + Y_UNIT_TEST(ErrForNotSingleChildInInlineAST) { + ExpectFailWithError("select YQL::\"\"", + "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); + ExpectFailWithError("select YQL::@@ \t@@", + "<main>:1:8: Error: Failed to parse YQL: expecting AST root node with single child, but got 0\n"); + auto req = "$lambda = YQL::@@(lambda '(x)(+ x x)) (lambda '(y)(+ y y))@@;\n" + "select ListMap([1, 2, 3], $lambda);"; + ExpectFailWithError(req, + "<main>:1:11: Error: Failed to parse YQL: expecting AST root node with single child, but got 2\n"); + } + + Y_UNIT_TEST(ErrEmptyColumnName) { + ExpectFailWithError("select * without \"\" from plato.Input", + "<main>:1:18: Error: String literal can not be used here\n"); + + ExpectFailWithError("select * without `` from plato.Input;", + "<main>:1:18: Error: Empty column name is not allowed\n"); + + ExpectFailWithErrorForAnsiLexer("select * without \"\" from plato.Input", + "<main>:1:18: Error: Empty column name is not allowed\n"); + + ExpectFailWithErrorForAnsiLexer("select * without `` from plato.Input;", + "<main>:1:18: Error: Empty column name is not allowed\n"); + } + + Y_UNIT_TEST(ErrOnNonZeroArgumentsForTableRows) { + ExpectFailWithError("$udf=\"\";process plato.Input using $udf(TableRows(k))", + "<main>:1:40: Error: TableRows requires exactly 0 arguments\n"); + } + + Y_UNIT_TEST(ErrGroupByWithAggregationFunctionAndDistinctExpr) { + ExpectFailWithError("select * from plato.Input group by count(distinct key|key)", + "<main>:1:36: Error: Unable to GROUP BY aggregated values\n"); + } + + // FIXME: check if we can get old behaviour +#if 0 + Y_UNIT_TEST(ErrWithSchemaWithColumnsWithoutType) { + ExpectFailWithError("select * from plato.Input with COLUMNs", + "<main>:1:32: Error: Expected type after COLUMNS\n" + "<main>:1:32: Error: Failed to parse table hints\n"); + + ExpectFailWithError("select * from plato.Input with scheMa", + "<main>:1:32: Error: Expected type after SCHEMA\n" + "<main>:1:32: Error: Failed to parse table hints\n"); + } +#endif + + Y_UNIT_TEST(ErrCollectPreaggregatedInListLiteralWithoutFrom) { + ExpectFailWithError("SELECT([VARIANCE(DISTINCT[])])", + "<main>:1:1: Error: Column references are not allowed without FROM\n" + "<main>:1:9: Error: Column reference '_yql_preagg_Variance0'\n"); + } + + Y_UNIT_TEST(ErrGroupBySmartParenAsTuple) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (k, v,)", + "<main>:1:36: Error: Unable to use tuple in group by clause\n"); + } + + Y_UNIT_TEST(HandleNestedSmartParensInGroupBy) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY (+() as k)", + "<main>:1:37: Error: Unable to GROUP BY constant expression\n"); + } + Y_UNIT_TEST(ErrRenameWithAddColumn) { ExpectFailWithError("USE plato; ALTER TABLE table RENAME TO moved, ADD COLUMN addc uint64", "<main>:1:40: Error: RENAME TO can not be used together with another table action\n"); } Y_UNIT_TEST(ErrAddColumnAndRename) { - // FIXME: fix positions in ALTER TABLE + // FIXME: fix positions in ALTER TABLE ExpectFailWithError("USE plato; ALTER TABLE table ADD COLUMN addc uint64, RENAME TO moved", - "<main>:1:46: Error: RENAME TO can not be used together with another table action\n"); - } - - Y_UNIT_TEST(InvalidUuidValue) { - ExpectFailWithError("SELECT Uuid('123e4567ae89ba12d3aa456a426614174ab0')", - "<main>:1:8: Error: Invalid value \"123e4567ae89ba12d3aa456a426614174ab0\" for type Uuid\n"); - ExpectFailWithError("SELECT Uuid('123e4567ae89b-12d3-a456-426614174000')", - "<main>:1:8: Error: Invalid value \"123e4567ae89b-12d3-a456-426614174000\" for type Uuid\n"); - } - - Y_UNIT_TEST(WindowFunctionWithoutOver) { - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input GROUP BY key", - "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); - } - - Y_UNIT_TEST(CreateAlterUserWithoutCluster) { - ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); - ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); - } - - Y_UNIT_TEST(ReservedRoleNames) { - ExpectFailWithError("USE plato; CREATE USER current_User;", "<main>:1:24: Error: System role CURRENT_USER can not be used here\n"); - ExpectFailWithError("USE plato; ALTER USER current_User RENAME TO Current_role", "<main>:1:46: Error: System role CURRENT_ROLE can not be used here\n"); - UNIT_ASSERT(SqlToYql("USE plato; DROP GROUP IF EXISTS a, b, c, current_User;").IsOk()); - } - - Y_UNIT_TEST(DisableClassicDivisionWithError) { - ExpectFailWithError("pragma ClassicDivision = 'false'; select $foo / 30;", "<main>:1:42: Error: Unknown name: $foo\n"); - } + "<main>:1:46: Error: RENAME TO can not be used together with another table action\n"); + } + + Y_UNIT_TEST(InvalidUuidValue) { + ExpectFailWithError("SELECT Uuid('123e4567ae89ba12d3aa456a426614174ab0')", + "<main>:1:8: Error: Invalid value \"123e4567ae89ba12d3aa456a426614174ab0\" for type Uuid\n"); + ExpectFailWithError("SELECT Uuid('123e4567ae89b-12d3-a456-426614174000')", + "<main>:1:8: Error: Invalid value \"123e4567ae89b-12d3-a456-426614174000\" for type Uuid\n"); + } + + Y_UNIT_TEST(WindowFunctionWithoutOver) { + ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input", + "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); + ExpectFailWithError("SELECT LAST_VALUE(foo) FROM plato.Input GROUP BY key", + "<main>:1:8: Error: Can't use window function LastValue without window specification (OVER keyword is missing)\n"); + } + + Y_UNIT_TEST(CreateAlterUserWithoutCluster) { + ExpectFailWithError("\n CREATE USER user ENCRYPTED PASSWORD 'foobar';", "<main>:2:2: Error: USE statement is missing - no default cluster is selected\n"); + ExpectFailWithError("ALTER USER CURRENT_USER RENAME TO $foo;", "<main>:1:1: Error: USE statement is missing - no default cluster is selected\n"); + } + + Y_UNIT_TEST(ReservedRoleNames) { + ExpectFailWithError("USE plato; CREATE USER current_User;", "<main>:1:24: Error: System role CURRENT_USER can not be used here\n"); + ExpectFailWithError("USE plato; ALTER USER current_User RENAME TO Current_role", "<main>:1:46: Error: System role CURRENT_ROLE can not be used here\n"); + UNIT_ASSERT(SqlToYql("USE plato; DROP GROUP IF EXISTS a, b, c, current_User;").IsOk()); + } + + Y_UNIT_TEST(DisableClassicDivisionWithError) { + ExpectFailWithError("pragma ClassicDivision = 'false'; select $foo / 30;", "<main>:1:42: Error: Unknown name: $foo\n"); + } +} + +void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) { + auto res = SqlToYql(req); + + UNIT_ASSERT(res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), TStringBuilder() << "<main>:" << row << ":" << col << ": Warning: Symbol " << symbol << " is not used, code: 4527\n"); +} + +Y_UNIT_TEST_SUITE(WarnUnused) { + Y_UNIT_TEST(ActionOrSubquery) { + TString req = " $a()\n" + "as select 1;\n" + "end define;\n" + "\n" + "select 1;"; + CheckUnused("define action\n" + req, "$a", 2, 3); + CheckUnused("define subquery\n" + req, "$a", 2, 3); + } + + Y_UNIT_TEST(Import) { + TString req = "import lib1 symbols\n" + " $sqr;\n" + "select 1;"; + CheckUnused(req, "$sqr", 2, 3); + + req = "import lib1 symbols\n" + " $sqr as\n" + " $sq;\n" + "select 1;"; + CheckUnused(req, "$sq", 3, 5); + } + + Y_UNIT_TEST(NamedNodeStatement) { + TString req = " $a, $a = AsTuple(1, 2);\n" + "select $a;"; + CheckUnused(req, "$a", 1, 2); + req = "$a, $b = AsTuple(1, 2);\n" + "select $a;"; + CheckUnused(req, "$b", 1, 6); + CheckUnused(" $a = 1; $a = 2; select $a;", "$a", 1, 2); + } + + Y_UNIT_TEST(Declare) { + CheckUnused("declare $a as String;select 1;", "$a", 1, 9); + } + + Y_UNIT_TEST(ActionParams) { + TString req = "define action $a($x, $y) as\n" + " select $x;\n" + "end define;\n" + "\n" + "do $a(1,2);"; + CheckUnused(req, "$y", 1, 22); + } + + Y_UNIT_TEST(SubqueryParams) { + TString req = "use plato;\n" + "define subquery $q($name, $x) as\n" + " select * from $name;\n" + "end define;\n" + "\n" + "select * from $q(\"Input\", 1);"; + CheckUnused(req, "$x", 2, 27); + } + + Y_UNIT_TEST(For) { + TString req = "define action $a() as\n" + " select 1;\n" + "end define;\n" + "\n" + "for $i in ListFromRange(1, 10)\n" + "do $a();"; + CheckUnused(req, "$i", 5, 5); + } + + Y_UNIT_TEST(LambdaParams) { + TString req = "$lambda = ($x, $y) -> ($x);\n" + "select $lambda(1, 2);"; + CheckUnused(req, "$y", 1, 16); + } + + Y_UNIT_TEST(InsideLambdaBody) { + TString req = "$lambda = () -> {\n" + " $x = 1; return 1;\n" + "};\n" + "select $lambda();"; + CheckUnused(req, "$x", 2, 3); + req = "$lambda = () -> {\n" + " $x = 1; $x = 2; return $x;\n" + "};\n" + "select $lambda();"; + CheckUnused(req, "$x", 2, 3); + } + + Y_UNIT_TEST(InsideAction) { + TString req = "define action $a() as\n" + " $x = 1; select 1;\n" + "end define;\n" + "\n" + "do $a();"; + CheckUnused(req, "$x", 2, 3); + req = "define action $a() as\n" + " $x = 1; $x = 2; select $x;\n" + "end define;\n" + "\n" + "do $a();"; + CheckUnused(req, "$x", 2, 3); + } + + Y_UNIT_TEST(NoWarnOnNestedActions) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "define action $action($b) as\n" + " define action $aaa() as\n" + " select $b;\n" + " end define;\n" + " do $aaa();\n" + "end define;\n" + "\n" + "do $action(1);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(NoWarnForUsageAfterSubquery) { + auto req = "use plato;\n" + "pragma warning(\"error\", \"4527\");\n" + "\n" + "$a = 1;\n" + "\n" + "define subquery $q($table) as\n" + " select * from $table;\n" + "end define;\n" + "\n" + "select * from $q(\"Input\");\n" + "select $a;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } +} + +Y_UNIT_TEST_SUITE(AnonymousNames) { + Y_UNIT_TEST(ReferenceAnonymousVariableIsForbidden) { + auto req = "$_ = 1; select $_;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unable to reference anonymous name $_\n"); + + req = "$`_` = 1; select $`_`;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to reference anonymous name $_\n"); + } + + Y_UNIT_TEST(Declare) { + auto req = "declare $_ as String;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Error: Can not use anonymous name '$_' in DECLARE statement\n"); + } + + Y_UNIT_TEST(ActionSubquery) { + auto req = "define action $_() as select 1; end define;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Can not use anonymous name '$_' as ACTION name\n"); + + req = "define subquery $_() as select 1; end define;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Can not use anonymous name '$_' as SUBQUERY name\n"); + } + + Y_UNIT_TEST(Import) { + auto req = "import lib symbols $sqr as $_;"; + auto res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Can not import anonymous name $_\n"); + } + + Y_UNIT_TEST(Export) { + auto req = "export $_;"; + auto res = SqlToYqlWithMode(req, NSQLTranslation::ESqlMode::LIBRARY); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Can not export anonymous name $_\n"); + } + + Y_UNIT_TEST(AnonymousInActionArgs) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "define action $a($_, $y, $_) as\n" + " select $y;\n" + "end define;\n" + "\n" + "do $a(1,2,3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInSubqueryArgs) { + auto req = "use plato;\n" + "pragma warning(\"error\", \"4527\");\n" + "define subquery $q($_, $y, $_) as\n" + " select * from $y;\n" + "end define;\n" + "\n" + "select * from $q(1,\"Input\",3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInLambdaArgs) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "$lambda = ($_, $x, $_) -> ($x);\n" + "select $lambda(1,2,3);"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(AnonymousInFor) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "evaluate for $_ in ListFromRange(1, 10) do begin select 1; end do;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } + + Y_UNIT_TEST(Assignment) { + auto req = "pragma warning(\"error\", \"4527\");\n" + "$_ = 1;\n" + "$_, $x, $_ = AsTuple(1,2,3);\n" + "select $x;"; + UNIT_ASSERT(SqlToYql(req).IsOk()); + } } -void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) { - auto res = SqlToYql(req); - - UNIT_ASSERT(res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), TStringBuilder() << "<main>:" << row << ":" << col << ": Warning: Symbol " << symbol << " is not used, code: 4527\n"); -} - -Y_UNIT_TEST_SUITE(WarnUnused) { - Y_UNIT_TEST(ActionOrSubquery) { - TString req = " $a()\n" - "as select 1;\n" - "end define;\n" - "\n" - "select 1;"; - CheckUnused("define action\n" + req, "$a", 2, 3); - CheckUnused("define subquery\n" + req, "$a", 2, 3); - } - - Y_UNIT_TEST(Import) { - TString req = "import lib1 symbols\n" - " $sqr;\n" - "select 1;"; - CheckUnused(req, "$sqr", 2, 3); - - req = "import lib1 symbols\n" - " $sqr as\n" - " $sq;\n" - "select 1;"; - CheckUnused(req, "$sq", 3, 5); - } - - Y_UNIT_TEST(NamedNodeStatement) { - TString req = " $a, $a = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$a", 1, 2); - req = "$a, $b = AsTuple(1, 2);\n" - "select $a;"; - CheckUnused(req, "$b", 1, 6); - CheckUnused(" $a = 1; $a = 2; select $a;", "$a", 1, 2); - } - - Y_UNIT_TEST(Declare) { - CheckUnused("declare $a as String;select 1;", "$a", 1, 9); - } - - Y_UNIT_TEST(ActionParams) { - TString req = "define action $a($x, $y) as\n" - " select $x;\n" - "end define;\n" - "\n" - "do $a(1,2);"; - CheckUnused(req, "$y", 1, 22); - } - - Y_UNIT_TEST(SubqueryParams) { - TString req = "use plato;\n" - "define subquery $q($name, $x) as\n" - " select * from $name;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\", 1);"; - CheckUnused(req, "$x", 2, 27); - } - - Y_UNIT_TEST(For) { - TString req = "define action $a() as\n" - " select 1;\n" - "end define;\n" - "\n" - "for $i in ListFromRange(1, 10)\n" - "do $a();"; - CheckUnused(req, "$i", 5, 5); - } - - Y_UNIT_TEST(LambdaParams) { - TString req = "$lambda = ($x, $y) -> ($x);\n" - "select $lambda(1, 2);"; - CheckUnused(req, "$y", 1, 16); - } - - Y_UNIT_TEST(InsideLambdaBody) { - TString req = "$lambda = () -> {\n" - " $x = 1; return 1;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - req = "$lambda = () -> {\n" - " $x = 1; $x = 2; return $x;\n" - "};\n" - "select $lambda();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(InsideAction) { - TString req = "define action $a() as\n" - " $x = 1; select 1;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - req = "define action $a() as\n" - " $x = 1; $x = 2; select $x;\n" - "end define;\n" - "\n" - "do $a();"; - CheckUnused(req, "$x", 2, 3); - } - - Y_UNIT_TEST(NoWarnOnNestedActions) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $action($b) as\n" - " define action $aaa() as\n" - " select $b;\n" - " end define;\n" - " do $aaa();\n" - "end define;\n" - "\n" - "do $action(1);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(NoWarnForUsageAfterSubquery) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "\n" - "$a = 1;\n" - "\n" - "define subquery $q($table) as\n" - " select * from $table;\n" - "end define;\n" - "\n" - "select * from $q(\"Input\");\n" - "select $a;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } -} - -Y_UNIT_TEST_SUITE(AnonymousNames) { - Y_UNIT_TEST(ReferenceAnonymousVariableIsForbidden) { - auto req = "$_ = 1; select $_;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unable to reference anonymous name $_\n"); - - req = "$`_` = 1; select $`_`;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: Unable to reference anonymous name $_\n"); - } - - Y_UNIT_TEST(Declare) { - auto req = "declare $_ as String;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Error: Can not use anonymous name '$_' in DECLARE statement\n"); - } - - Y_UNIT_TEST(ActionSubquery) { - auto req = "define action $_() as select 1; end define;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Can not use anonymous name '$_' as ACTION name\n"); - - req = "define subquery $_() as select 1; end define;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Can not use anonymous name '$_' as SUBQUERY name\n"); - } - - Y_UNIT_TEST(Import) { - auto req = "import lib symbols $sqr as $_;"; - auto res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Can not import anonymous name $_\n"); - } - - Y_UNIT_TEST(Export) { - auto req = "export $_;"; - auto res = SqlToYqlWithMode(req, NSQLTranslation::ESqlMode::LIBRARY); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:8: Error: Can not export anonymous name $_\n"); - } - - Y_UNIT_TEST(AnonymousInActionArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "define action $a($_, $y, $_) as\n" - " select $y;\n" - "end define;\n" - "\n" - "do $a(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInSubqueryArgs) { - auto req = "use plato;\n" - "pragma warning(\"error\", \"4527\");\n" - "define subquery $q($_, $y, $_) as\n" - " select * from $y;\n" - "end define;\n" - "\n" - "select * from $q(1,\"Input\",3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInLambdaArgs) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$lambda = ($_, $x, $_) -> ($x);\n" - "select $lambda(1,2,3);"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(AnonymousInFor) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "evaluate for $_ in ListFromRange(1, 10) do begin select 1; end do;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } - - Y_UNIT_TEST(Assignment) { - auto req = "pragma warning(\"error\", \"4527\");\n" - "$_ = 1;\n" - "$_, $x, $_ = AsTuple(1,2,3);\n" - "select $x;"; - UNIT_ASSERT(SqlToYql(req).IsOk()); - } -} - Y_UNIT_TEST_SUITE(JsonValue) { Y_UNIT_TEST(JsonValueArgumentCount) { NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{\"key\": 1238}@@ as Json));"); @@ -3459,10 +3459,10 @@ Y_UNIT_TEST_SUITE(JsonValue) { } Y_UNIT_TEST(JsonValueInvalidReturningType) { - NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{'key': 1238}@@ as Json), 'strict $.key' RETURNING invalid);"); + NYql::TAstParseResult res = SqlToYql("select JSON_VALUE(CAST(@@{'key': 1238}@@ as Json), 'strict $.key' RETURNING invalid);"); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Unknown simple type 'invalid'\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:77: Error: Unknown simple type 'invalid'\n"); } Y_UNIT_TEST(JsonValueAndReturningInExpressions) { @@ -3726,7 +3726,7 @@ Y_UNIT_TEST_SUITE(JsonQuery) { VerifyProgram(res, elementStat, verifyLine); UNIT_ASSERT(elementStat["JsonQuery"] > 0); } -} +} Y_UNIT_TEST_SUITE(JsonPassing) { Y_UNIT_TEST(SupportedVariableTypes) { @@ -3800,7 +3800,7 @@ Y_UNIT_TEST_SUITE(JsonPassing) { UNIT_ASSERT(elementStat["JsonVariables"] > 0); } } -} +} Y_UNIT_TEST_SUITE(MigrationToJsonApi) { Y_UNIT_TEST(WarningOnDeprecatedJsonUdf) { @@ -3813,195 +3813,195 @@ Y_UNIT_TEST_SUITE(MigrationToJsonApi) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:26: Warning: Json UDF is deprecated. Please use JSON API instead, code: 4506\n"); } } - -Y_UNIT_TEST_SUITE(AnsiIdentsNegative) { - Y_UNIT_TEST(EnableAnsiLexerFromRequestSpecialComments) { - auto req = "\n" - "\t --!ansi_lexer \n" - "-- Some comment\n" - "-- another comment\n" - "pragma SimpleColumns;\n" - "\n" - "select 1, '''' as empty;"; - - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(AnsiLexerShouldNotBeEnabledHere) { - auto req = "$str = '\n" - "--!ansi_lexer\n" - "--!syntax_v1\n" - "';\n" - "\n" - "select 1, $str, \"\" as empty;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Issues.Size() == 0); - } - - Y_UNIT_TEST(DoubleQuotesInDictsTuplesOrLists) { - auto req = "$d = { 'a': 1, \"b\": 2, 'c': 3,};"; - - auto res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Column reference \"b\" is not allowed in current scope\n"); - - req = "$t = (1, 2, \"a\");"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Column reference \"a\" is not allowed in current scope\n"); - - req = "$l = ['a', 'b', \"c\"];"; - - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Column reference \"c\" is not allowed in current scope\n"); - } - - Y_UNIT_TEST(MultilineComments) { - auto req = "/*/**/ select 1;"; - auto res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n" - "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); - - req = "/*\n" - "--/*\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(res.Root); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n" - "<main>:3:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); - - req = "/*\n" - "/*\n" - "--*/\n" - "*/ select 1;"; - res = SqlToYql(req); - UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token '*' : cannot match to any predicted input...\n\n"); - res = SqlToYqlWithAnsiLexer(req); - UNIT_ASSERT(res.Root); - } -} - -Y_UNIT_TEST_SUITE(AnsiOptionalAs) { - Y_UNIT_TEST(OptionalAsInProjection) { - UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a b, c FROM plato.Input;").IsOk()); - ExpectFailWithError("PRAGMA DisableAnsiOptionalAs;\n" - "SELECT a b, c FROM plato.Input;", - "<main>:2:10: Error: Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility\n"); - } -} - -Y_UNIT_TEST_SUITE(SessionWindowNegative) { - Y_UNIT_TEST(SessionWindowWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32);", - "<main>:1:12: Error: SessionWindow requires data source\n"); - } - - Y_UNIT_TEST(SessionWindowInProjection) { - ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32) from plato.Input;", - "<main>:1:12: Error: SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression\n"); - } - - Y_UNIT_TEST(SessionWindowWithNonConstSecondArg) { - ExpectFailWithError( - "SELECT key, session_start FROM plato.Input\n" - "GROUP BY SessionWindow(ts, 32 + subkey) as session_start, key;", - - "<main>:2:10: Error: Source does not allow column references\n" - "<main>:2:33: Error: Column reference 'subkey'\n"); - } - - Y_UNIT_TEST(SessionWindowWithWrongNumberOfArgs) { - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow()", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow(key, subkey, 100)", - "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); - } - - Y_UNIT_TEST(DuplicateSessionWindow) { - ExpectFailWithError( - "SELECT\n" - " *\n" - "FROM plato.Input\n" - "GROUP BY\n" - " SessionWindow(ts, 10),\n" - " user,\n" - " SessionWindow(ts, 20)\n" - ";", - - "<main>:7:5: Error: Duplicate session window specification:\n" + +Y_UNIT_TEST_SUITE(AnsiIdentsNegative) { + Y_UNIT_TEST(EnableAnsiLexerFromRequestSpecialComments) { + auto req = "\n" + "\t --!ansi_lexer \n" + "-- Some comment\n" + "-- another comment\n" + "pragma SimpleColumns;\n" + "\n" + "select 1, '''' as empty;"; + + auto res = SqlToYql(req); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(AnsiLexerShouldNotBeEnabledHere) { + auto req = "$str = '\n" + "--!ansi_lexer\n" + "--!syntax_v1\n" + "';\n" + "\n" + "select 1, $str, \"\" as empty;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Issues.Size() == 0); + } + + Y_UNIT_TEST(DoubleQuotesInDictsTuplesOrLists) { + auto req = "$d = { 'a': 1, \"b\": 2, 'c': 3,};"; + + auto res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Column reference \"b\" is not allowed in current scope\n"); + + req = "$t = (1, 2, \"a\");"; + + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Column reference \"a\" is not allowed in current scope\n"); + + req = "$l = ['a', 'b', \"c\"];"; + + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Column reference \"c\" is not allowed in current scope\n"); + } + + Y_UNIT_TEST(MultilineComments) { + auto req = "/*/**/ select 1;"; + auto res = SqlToYql(req); + UNIT_ASSERT(res.Root); + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n" + "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + + req = "/*\n" + "--/*\n" + "*/ select 1;"; + res = SqlToYql(req); + UNIT_ASSERT(res.Root); + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n" + "<main>:3:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n"); + + req = "/*\n" + "/*\n" + "--*/\n" + "*/ select 1;"; + res = SqlToYql(req); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:4:0: Error: Unexpected token '*' : cannot match to any predicted input...\n\n"); + res = SqlToYqlWithAnsiLexer(req); + UNIT_ASSERT(res.Root); + } +} + +Y_UNIT_TEST_SUITE(AnsiOptionalAs) { + Y_UNIT_TEST(OptionalAsInProjection) { + UNIT_ASSERT(SqlToYql("PRAGMA AnsiOptionalAs; SELECT a b, c FROM plato.Input;").IsOk()); + ExpectFailWithError("PRAGMA DisableAnsiOptionalAs;\n" + "SELECT a b, c FROM plato.Input;", + "<main>:2:10: Error: Expecting mandatory AS here. Did you miss comma? Please add PRAGMA AnsiOptionalAs; for ANSI compatibility\n"); + } +} + +Y_UNIT_TEST_SUITE(SessionWindowNegative) { + Y_UNIT_TEST(SessionWindowWithoutSource) { + ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32);", + "<main>:1:12: Error: SessionWindow requires data source\n"); + } + + Y_UNIT_TEST(SessionWindowInProjection) { + ExpectFailWithError("SELECT 1 + SessionWindow(ts, 32) from plato.Input;", + "<main>:1:12: Error: SessionWindow can only be used as a top-level GROUP BY / PARTITION BY expression\n"); + } + + Y_UNIT_TEST(SessionWindowWithNonConstSecondArg) { + ExpectFailWithError( + "SELECT key, session_start FROM plato.Input\n" + "GROUP BY SessionWindow(ts, 32 + subkey) as session_start, key;", + + "<main>:2:10: Error: Source does not allow column references\n" + "<main>:2:33: Error: Column reference 'subkey'\n"); + } + + Y_UNIT_TEST(SessionWindowWithWrongNumberOfArgs) { + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow()", + "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); + ExpectFailWithError("SELECT * FROM plato.Input GROUP BY SessionWindow(key, subkey, 100)", + "<main>:1:36: Error: SessionWindow requires either two or four arguments\n"); + } + + Y_UNIT_TEST(DuplicateSessionWindow) { + ExpectFailWithError( + "SELECT\n" + " *\n" + "FROM plato.Input\n" + "GROUP BY\n" + " SessionWindow(ts, 10),\n" + " user,\n" + " SessionWindow(ts, 20)\n" + ";", + + "<main>:7:5: Error: Duplicate session window specification:\n" "<main>:5:5: Error: Previous session window is declared here\n"); - - ExpectFailWithError( - "SELECT\n" - " MIN(key) over w\n" - "FROM plato.Input\n" - "WINDOW w AS (\n" - " PARTITION BY SessionWindow(ts, 10), user,\n" - " SessionWindow(ts, 20)\n" - ");", - - "<main>:6:5: Error: Duplicate session window specification:\n" - "<main>:5:18: Error: Previous session window is declared here\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutSource) { - ExpectFailWithError("SELECT 1 + SessionStart();", - "<main>:1:12: Error: SessionStart requires data source\n"); - ExpectFailWithError("SELECT 1 + SessionState();", - "<main>:1:12: Error: SessionState requires data source\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutGroupByOrWindow) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input;", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input;", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow\n"); - } - - Y_UNIT_TEST(SessionStartStateWithGroupByWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input group by user;", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() from plato.Input group by user;", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithoutOverWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(SessionStartStateWithWindowWithoutSession) { - ExpectFailWithError("SELECT 1 + SessionStart() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionStart can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - ExpectFailWithError("SELECT 1 + SessionState() over w, MIN(key) over w from plato.Input window w as ()", - "<main>:1:12: Error: SessionState can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); - } - - Y_UNIT_TEST(SessionStartStateWithSessionedWindow) { - ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", - "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); - } - - Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { - ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", - "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); - } - + + ExpectFailWithError( + "SELECT\n" + " MIN(key) over w\n" + "FROM plato.Input\n" + "WINDOW w AS (\n" + " PARTITION BY SessionWindow(ts, 10), user,\n" + " SessionWindow(ts, 20)\n" + ");", + + "<main>:6:5: Error: Duplicate session window specification:\n" + "<main>:5:18: Error: Previous session window is declared here\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutSource) { + ExpectFailWithError("SELECT 1 + SessionStart();", + "<main>:1:12: Error: SessionStart requires data source\n"); + ExpectFailWithError("SELECT 1 + SessionState();", + "<main>:1:12: Error: SessionState requires data source\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutGroupByOrWindow) { + ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input;", + "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow\n"); + ExpectFailWithError("SELECT 1 + SessionState() from plato.Input;", + "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow\n"); + } + + Y_UNIT_TEST(SessionStartStateWithGroupByWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart() from plato.Input group by user;", + "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY\n"); + ExpectFailWithError("SELECT 1 + SessionState() from plato.Input group by user;", + "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY\n"); + } + + Y_UNIT_TEST(SessionStartStateWithoutOverWithWindowWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionStart can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); + ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionState can not be used without aggregation by SessionWindow. Maybe you forgot to add OVER `window_name`?\n"); + } + + Y_UNIT_TEST(SessionStartStateWithWindowWithoutSession) { + ExpectFailWithError("SELECT 1 + SessionStart() over w, MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionStart can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); + ExpectFailWithError("SELECT 1 + SessionState() over w, MIN(key) over w from plato.Input window w as ()", + "<main>:1:12: Error: SessionState can not be used with window w: SessionWindow specification is missing in PARTITION BY\n"); + } + + Y_UNIT_TEST(SessionStartStateWithSessionedWindow) { + ExpectFailWithError("SELECT 1 + SessionStart(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", + "<main>:1:12: Error: SessionStart can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); + ExpectFailWithError("SELECT 1 + SessionState(), MIN(key) over w from plato.Input group by key window w as (partition by SessionWindow(ts, 1)) ", + "<main>:1:12: Error: SessionState can not be used here: SessionWindow specification is missing in GROUP BY. Maybe you forgot to add OVER `window_name`?\n"); + } + + Y_UNIT_TEST(AggregationBySessionStateIsNotSupportedYet) { + ExpectFailWithError("SELECT SOME(1 + SessionState()), key from plato.Input group by key, SessionWindow(ts, 1);", + "<main>:1:17: Error: SessionState with GROUP BY is not supported yet\n"); + } + Y_UNIT_TEST(SessionWindowInRtmr) { NYql::TAstParseResult res = SqlToYql( "SELECT * FROM plato.Input GROUP BY SessionWindow(ts, 10);", @@ -4015,9 +4015,9 @@ Y_UNIT_TEST_SUITE(SessionWindowNegative) { )", 10, TString(NYql::RtmrProviderName)); UNIT_ASSERT(!res.Root); - UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:2:13: Error: SessionWindow is unsupported for streaming sources\n"); } -} +} Y_UNIT_TEST_SUITE(LibraSqlSugar) { auto makeResult = [](TStringBuf settings) { @@ -4088,53 +4088,53 @@ Y_UNIT_TEST_SUITE(LibraSqlSugar) { UNIT_ASSERT_STRING_CONTAINS(Err2Str(res), "Error: MakeLibraPreprocessor entity must be string literal"); } } - -Y_UNIT_TEST_SUITE(TrailingQuestionsNegative) { - Y_UNIT_TEST(Basic) { - ExpectFailWithError("SELECT 1?;", "<main>:1:9: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: Unexpected token '+' : cannot match to any predicted input...\n\n"); - ExpectFailWithError("SELECT 1 + 1??? < 2", "<main>:1:13: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT 1? > 2? > 3?", - "<main>:1:11: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:16: Error: Unexpected token '?' at the end of expression\n" - "<main>:1:21: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(SmartParen) { - ExpectFailWithError("$x = 1; SELECT (Int32?, $x?)", "<main>:1:27: Error: Unexpected token '?' at the end of expression\n"); - ExpectFailWithError("SELECT (Int32, foo?)", "<main>:1:19: Error: Unexpected token '?' at the end of expression\n"); - } - - Y_UNIT_TEST(LambdaOptArgs) { - ExpectFailWithError("$l = ($x, $y?, $z??, $t?) -> ($x);", "<main>:1:18: Error: Expecting at most one '?' token here (for optional lambda parameters), but got 2\n"); - } -} - -Y_UNIT_TEST_SUITE(FlexibleTypes) { - Y_UNIT_TEST(AssumeOrderByType) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT 1 AS int32 ASSUME ORDER BY int32").IsOk()); - } - - Y_UNIT_TEST(GroupingSets) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT COUNT(*) AS cnt, text, uuid FROM plato.Input GROUP BY GROUPING SETS((uuid), (uuid, text));").IsOk()); - } - - Y_UNIT_TEST(WeakField) { - UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT WeakField(text, string) as text FROM plato.Input").IsOk()); - } - - Y_UNIT_TEST(Aggregation1) { - TString q = - "PRAGMA FlexibleTypes;\n" - "$foo = ($x, $const, $type) -> ($x || $const || FormatType($type));\n" - "SELECT $foo(SOME(x), 'aaa', String) FROM plato.Input GROUP BY y;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } - - Y_UNIT_TEST(Aggregation2) { - TString q = - "PRAGMA FlexibleTypes;\n" - "SELECT 1 + String + MAX(key) FROM plato.Input;"; - UNIT_ASSERT(SqlToYql(q).IsOk()); - } -} + +Y_UNIT_TEST_SUITE(TrailingQuestionsNegative) { + Y_UNIT_TEST(Basic) { + ExpectFailWithError("SELECT 1?;", "<main>:1:9: Error: Unexpected token '?' at the end of expression\n"); + ExpectFailWithError("SELECT 1? + 1;", "<main>:1:10: Error: Unexpected token '+' : cannot match to any predicted input...\n\n"); + ExpectFailWithError("SELECT 1 + 1??? < 2", "<main>:1:13: Error: Unexpected token '?' at the end of expression\n"); + ExpectFailWithError("SELECT 1? > 2? > 3?", + "<main>:1:11: Error: Unexpected token '?' at the end of expression\n" + "<main>:1:16: Error: Unexpected token '?' at the end of expression\n" + "<main>:1:21: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(SmartParen) { + ExpectFailWithError("$x = 1; SELECT (Int32?, $x?)", "<main>:1:27: Error: Unexpected token '?' at the end of expression\n"); + ExpectFailWithError("SELECT (Int32, foo?)", "<main>:1:19: Error: Unexpected token '?' at the end of expression\n"); + } + + Y_UNIT_TEST(LambdaOptArgs) { + ExpectFailWithError("$l = ($x, $y?, $z??, $t?) -> ($x);", "<main>:1:18: Error: Expecting at most one '?' token here (for optional lambda parameters), but got 2\n"); + } +} + +Y_UNIT_TEST_SUITE(FlexibleTypes) { + Y_UNIT_TEST(AssumeOrderByType) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT 1 AS int32 ASSUME ORDER BY int32").IsOk()); + } + + Y_UNIT_TEST(GroupingSets) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT COUNT(*) AS cnt, text, uuid FROM plato.Input GROUP BY GROUPING SETS((uuid), (uuid, text));").IsOk()); + } + + Y_UNIT_TEST(WeakField) { + UNIT_ASSERT(SqlToYql("PRAGMA FlexibleTypes; SELECT WeakField(text, string) as text FROM plato.Input").IsOk()); + } + + Y_UNIT_TEST(Aggregation1) { + TString q = + "PRAGMA FlexibleTypes;\n" + "$foo = ($x, $const, $type) -> ($x || $const || FormatType($type));\n" + "SELECT $foo(SOME(x), 'aaa', String) FROM plato.Input GROUP BY y;"; + UNIT_ASSERT(SqlToYql(q).IsOk()); + } + + Y_UNIT_TEST(Aggregation2) { + TString q = + "PRAGMA FlexibleTypes;\n" + "SELECT 1 + String + MAX(key) FROM plato.Input;"; + UNIT_ASSERT(SqlToYql(q).IsOk()); + } +} diff --git a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp index 0f9eadfe96..763fea8024 100644 --- a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp +++ b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp @@ -221,7 +221,7 @@ namespace { } inline bool ValidateMonthShortName(const std::string_view& monthName, ui8& month) { - static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) { + static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) { int cmp = strnicmp(a.data(), b.data(), std::min(a.size(), b.size())); if (cmp == 0) return a.size() < b.size(); @@ -250,7 +250,7 @@ namespace { } inline bool ValidateMonthFullName(const std::string_view& monthName, ui8& month) { - static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) { + static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) { int cmp = strnicmp(a.data(), b.data(), std::min(a.size(), b.size())); if (cmp == 0) return a.size() < b.size(); @@ -311,7 +311,7 @@ namespace { bool typesOnly) { builder.UserType(userType); - builder.Args()->Add<TUserDataType>().Flags(ICallablePayload::TArgumentFlags::AutoMap); + builder.Args()->Add<TUserDataType>().Flags(ICallablePayload::TArgumentFlags::AutoMap); builder.Returns(builder.Resource(TMResourceName)); if (!typesOnly) { @@ -662,7 +662,7 @@ namespace { .Add(builder.Optional()->Item<ui8>().Build()).Name("Minute") .Add(builder.Optional()->Item<ui8>().Build()).Name("Second") .Add(builder.Optional()->Item<ui32>().Build()).Name("Microsecond") - .Add(builder.Optional()->Item<ui16>().Build()).Name("TimezoneId"); + .Add(builder.Optional()->Item<ui16>().Build()).Name("TimezoneId"); builder.Returns(optionalResourceType); @@ -1052,7 +1052,7 @@ namespace { auto resourceType = builder.Resource(TMResourceName); - builder.Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap); + builder.Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap); builder.RunConfig<char*>().Returns<char*>(); if (!typesOnly) { @@ -1239,8 +1239,8 @@ namespace { static constexpr std::string_view mp[] { "Jan", "Feb", - "Mar", - "Apr", + "Mar", + "Apr", "May", "Jun", "Jul", @@ -1250,9 +1250,9 @@ namespace { "Nov", "Dec" }; - auto month = GetMonth(value); - Y_ENSURE(month > 0 && month <= sizeof(mp) / sizeof(mp[0]), "Invalid month value"); - std::memcpy(out, mp[month - 1].data(), size); + auto month = GetMonth(value); + Y_ENSURE(month > 0 && month <= sizeof(mp) / sizeof(mp[0]), "Invalid month value"); + std::memcpy(out, mp[month - 1].data(), size); return size; }); ReservedSize_ += size; @@ -1263,8 +1263,8 @@ namespace { static constexpr std::string_view mp[] { "January", "February", - "March", - "April", + "March", + "April", "May", "June", "July", @@ -1274,9 +1274,9 @@ namespace { "November", "December" }; - auto month = GetMonth(value); - Y_ENSURE(month > 0 && month <= sizeof(mp) / sizeof(mp[0]), "Invalid month value"); - const std::string_view monthFullName = mp[month - 1]; + auto month = GetMonth(value); + Y_ENSURE(month > 0 && month <= sizeof(mp) / sizeof(mp[0]), "Invalid month value"); + const std::string_view monthFullName = mp[month - 1]; std::memcpy(out, monthFullName.data(), monthFullName.size()); return monthFullName.size(); }); diff --git a/ydb/library/yql/udfs/common/digest/digest_udf.cpp b/ydb/library/yql/udfs/common/digest/digest_udf.cpp index 31b3dbded0..7487a078dd 100644 --- a/ydb/library/yql/udfs/common/digest/digest_udf.cpp +++ b/ydb/library/yql/udfs/common/digest/digest_udf.cpp @@ -91,7 +91,7 @@ namespace { Y_UNUSED(userType); if (Name() == name) { auto type = builder.Tuple(2)->Add<ui64>().Add<ui64>().Build(); - builder.Args(1)->Add<TAutoMap<char*>>(); + builder.Args(1)->Add<TAutoMap<char*>>(); builder.Returns(type); if (!typesOnly) { builder.Implementation(new TCityHash128); @@ -202,21 +202,21 @@ namespace { return TUnboxedValuePod(hash); } - SIMPLE_UDF(TFarmHashFingerprint, ui64(TAutoMap<ui64>)) { - Y_UNUSED(valueBuilder); - ui64 input = args[0].Get<ui64>(); - ui64 hash = util::Fingerprint(input); - return TUnboxedValuePod(hash); - } - - SIMPLE_UDF(TFarmHashFingerprint2, ui64(TAutoMap<ui64>, TAutoMap<ui64>)) { - Y_UNUSED(valueBuilder); - ui64 low = args[0].Get<ui64>(); - ui64 high = args[1].Get<ui64>(); - ui64 hash = util::Fingerprint(util::Uint128(low, high)); - return TUnboxedValuePod(hash); - } - + SIMPLE_UDF(TFarmHashFingerprint, ui64(TAutoMap<ui64>)) { + Y_UNUSED(valueBuilder); + ui64 input = args[0].Get<ui64>(); + ui64 hash = util::Fingerprint(input); + return TUnboxedValuePod(hash); + } + + SIMPLE_UDF(TFarmHashFingerprint2, ui64(TAutoMap<ui64>, TAutoMap<ui64>)) { + Y_UNUSED(valueBuilder); + ui64 low = args[0].Get<ui64>(); + ui64 high = args[1].Get<ui64>(); + ui64 hash = util::Fingerprint(util::Uint128(low, high)); + return TUnboxedValuePod(hash); + } + SIMPLE_UDF(TFarmHashFingerprint32, ui32(TAutoMap<char*>)) { Y_UNUSED(valueBuilder); const auto& inputRef = args[0].AsStringRef(); @@ -246,7 +246,7 @@ namespace { Y_UNUSED(userType); if (Name() == name) { auto type = builder.Tuple(2)->Add<ui64>().Add<ui64>().Build(); - builder.Args(1)->Add<TAutoMap<char*>>(); + builder.Args(1)->Add<TAutoMap<char*>>(); builder.Returns(type); if (!typesOnly) { builder.Implementation(new TFarmHashFingerprint128); @@ -327,7 +327,7 @@ namespace { static bool DeclareSignature(const TStringRef& name, TType*, IFunctionTypeInfoBuilder& builder, bool typesOnly) { if (Name() == name) { const auto type = builder.Tuple(2)->Add<ui64>().Add<ui64>().Build(); - builder.Args(1)->Add<TAutoMap<char*>>(); + builder.Args(1)->Add<TAutoMap<char*>>(); builder.Returns(type); if (!typesOnly) { builder.Implementation(new TXXH3_128); @@ -367,8 +367,8 @@ namespace { TBlake2B, TSipHash, THighwayHash, - TFarmHashFingerprint, - TFarmHashFingerprint2, + TFarmHashFingerprint, + TFarmHashFingerprint2, TFarmHashFingerprint32, TFarmHashFingerprint64, TFarmHashFingerprint128, diff --git a/ydb/library/yql/udfs/common/top/top_udf.cpp b/ydb/library/yql/udfs/common/top/top_udf.cpp index ada0890be3..748ffe489b 100644 --- a/ydb/library/yql/udfs/common/top/top_udf.cpp +++ b/ydb/library/yql/udfs/common/top/top_udf.cpp @@ -3,11 +3,11 @@ #include <library/cpp/containers/top_keeper/top_keeper.h> -#include <util/generic/set.h> - -#include <algorithm> -#include <iterator> - +#include <util/generic/set.h> + +#include <algorithm> +#include <iterator> + using namespace NKikimr; using namespace NUdf; @@ -17,7 +17,7 @@ using TUnboxedValuePair = std::pair<TUnboxedValue, TUnboxedValue>; template <EDataSlot Slot, bool IsTop> struct TDataCompare { - bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { + bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { if (IsTop) { return CompareValues<Slot>(left, right) > 0; } else { @@ -28,7 +28,7 @@ struct TDataCompare { template <EDataSlot Slot, bool IsTop> struct TDataPairCompare { - bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { + bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { if (IsTop) { return CompareValues<Slot>(left.first, right.first) > 0; } else { @@ -41,7 +41,7 @@ template <bool IsTop> struct TGenericCompare { ICompare::TPtr Compare; - bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { + bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const { if (IsTop) { return Compare->Less(right, left); } else { @@ -54,7 +54,7 @@ template <bool IsTop> struct TGenericPairCompare { ICompare::TPtr Compare; - bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { + bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const { if (IsTop) { return Compare->Less(right.first, left.first); } else { @@ -63,95 +63,95 @@ struct TGenericPairCompare { } }; -template <typename TValue, typename TCompare, typename TAllocator> -class TTopKeeperContainer { - TTopKeeper<TValue, TCompare, true, TAllocator> Keeper; - using TOrderedSet = TSet<TValue, TCompare, TAllocator>; - TMaybe<TOrderedSet> OrderedSet; - size_t MaxSize = 0; - bool Finalized = false; - TCompare Compare; -public: - explicit TTopKeeperContainer(TCompare compare) - : Keeper(0, compare) - , Compare(compare) - {} - - TVector<TValue, TAllocator> GetInternal() { - if (OrderedSet) { - TVector<TValue, TAllocator> result; - std::copy(OrderedSet->begin(), OrderedSet->end(), std::back_inserter(result)); - return result; - } - Finalized = true; - return Keeper.GetInternal(); - } - - void Insert(const TValue& value) { - if (MaxSize == 0) { - return; - } - if (Finalized && !OrderedSet) { - const auto& items = Keeper.Extract(); - OrderedSet = TOrderedSet{items.begin(), items.end(), Compare}; - } - if (OrderedSet) { - if (OrderedSet->size() < MaxSize) { - OrderedSet->insert(value); - return; - } - Y_ENSURE(OrderedSet->size() == MaxSize); - Y_ENSURE(!OrderedSet->empty()); - auto last = --OrderedSet->end(); - if (Compare(value, *last)) { - OrderedSet->erase(last); - OrderedSet->insert(value); - } - return; - } - Keeper.Insert(value); - } - - bool IsEmpty() const { - return OrderedSet ? OrderedSet->empty() : Keeper.IsEmpty(); - } - - size_t GetSize() const { - return OrderedSet ? OrderedSet->size() : Keeper.GetSize(); - } - - size_t GetMaxSize() const { - return MaxSize; - } - - void SetMaxSize(size_t newMaxSize) { - MaxSize = newMaxSize; - if (Finalized && !OrderedSet) { - auto items = Keeper.Extract(); - auto begin = items.begin(); - auto end = begin + Min(MaxSize, items.size()); - OrderedSet = TOrderedSet{begin, end, Compare}; - } - if (OrderedSet) { - while (OrderedSet->size() > MaxSize) { - auto last = --OrderedSet->end(); - OrderedSet->erase(last); - } - return; - } - - Keeper.SetMaxSize(MaxSize); - } -}; - +template <typename TValue, typename TCompare, typename TAllocator> +class TTopKeeperContainer { + TTopKeeper<TValue, TCompare, true, TAllocator> Keeper; + using TOrderedSet = TSet<TValue, TCompare, TAllocator>; + TMaybe<TOrderedSet> OrderedSet; + size_t MaxSize = 0; + bool Finalized = false; + TCompare Compare; +public: + explicit TTopKeeperContainer(TCompare compare) + : Keeper(0, compare) + , Compare(compare) + {} + + TVector<TValue, TAllocator> GetInternal() { + if (OrderedSet) { + TVector<TValue, TAllocator> result; + std::copy(OrderedSet->begin(), OrderedSet->end(), std::back_inserter(result)); + return result; + } + Finalized = true; + return Keeper.GetInternal(); + } + + void Insert(const TValue& value) { + if (MaxSize == 0) { + return; + } + if (Finalized && !OrderedSet) { + const auto& items = Keeper.Extract(); + OrderedSet = TOrderedSet{items.begin(), items.end(), Compare}; + } + if (OrderedSet) { + if (OrderedSet->size() < MaxSize) { + OrderedSet->insert(value); + return; + } + Y_ENSURE(OrderedSet->size() == MaxSize); + Y_ENSURE(!OrderedSet->empty()); + auto last = --OrderedSet->end(); + if (Compare(value, *last)) { + OrderedSet->erase(last); + OrderedSet->insert(value); + } + return; + } + Keeper.Insert(value); + } + + bool IsEmpty() const { + return OrderedSet ? OrderedSet->empty() : Keeper.IsEmpty(); + } + + size_t GetSize() const { + return OrderedSet ? OrderedSet->size() : Keeper.GetSize(); + } + + size_t GetMaxSize() const { + return MaxSize; + } + + void SetMaxSize(size_t newMaxSize) { + MaxSize = newMaxSize; + if (Finalized && !OrderedSet) { + auto items = Keeper.Extract(); + auto begin = items.begin(); + auto end = begin + Min(MaxSize, items.size()); + OrderedSet = TOrderedSet{begin, end, Compare}; + } + if (OrderedSet) { + while (OrderedSet->size() > MaxSize) { + auto last = --OrderedSet->end(); + OrderedSet->erase(last); + } + return; + } + + Keeper.SetMaxSize(MaxSize); + } +}; + template <typename TCompare> class TTopKeeperWrapperBase { protected: - TTopKeeperContainer<TUnboxedValue, TCompare, TUnboxedValue::TAllocator> Keeper; + TTopKeeperContainer<TUnboxedValue, TCompare, TUnboxedValue::TAllocator> Keeper; protected: explicit TTopKeeperWrapperBase(TCompare compare) - : Keeper(compare) + : Keeper(compare) {} void Init(const TUnboxedValuePod& value, ui32 maxSize) { @@ -215,11 +215,11 @@ public: template <typename TCompare> class TTopKeeperPairWrapperBase { protected: - TTopKeeperContainer<TUnboxedValuePair, TCompare, TStdAllocatorForUdf<TUnboxedValuePair>> Keeper; + TTopKeeperContainer<TUnboxedValuePair, TCompare, TStdAllocatorForUdf<TUnboxedValuePair>> Keeper; protected: explicit TTopKeeperPairWrapperBase(TCompare compare) - : Keeper(compare) + : Keeper(compare) {} void Init(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) { diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h index 6c2363987c..411190d257 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h @@ -4,8 +4,8 @@ #include <ydb/library/yql/public/udf/udf_helpers.h> #include <ydb/library/yql/public/udf/udf_type_ops.h> -#include <unordered_map> - +#include <unordered_map> + template <typename THash, typename TEquals> class TTopFreqBase { protected: @@ -14,12 +14,12 @@ protected: using IValueBuilder = NKikimr::NUdf::IValueBuilder; using TVectorElement = std::pair<TUnboxedValue, ui64>; - using TVectorType = std::vector<TVectorElement, NKikimr::NUdf::TStdAllocatorForUdf<TVectorElement>>; + using TVectorType = std::vector<TVectorElement, NKikimr::NUdf::TStdAllocatorForUdf<TVectorElement>>; TVectorType Freqs_; - std::unordered_map<TUnboxedValue, ui32, THash, TEquals, NKikimr::NUdf::TStdAllocatorForUdf<std::pair<const TUnboxedValue, ui32>>> Indices_; - ui32 MinSize_ = 0; - ui32 MaxSize_ = 0; + std::unordered_map<TUnboxedValue, ui32, THash, TEquals, NKikimr::NUdf::TStdAllocatorForUdf<std::pair<const TUnboxedValue, ui32>>> Indices_; + ui32 MinSize_ = 0; + ui32 MaxSize_ = 0; void Add(const TTopFreqBase& otherCalc); void Update(const TUnboxedValuePod& key, const ui64 value); diff --git a/ydb/library/yql/udfs/common/url_base/lib/url_parse.cpp b/ydb/library/yql/udfs/common/url_base/lib/url_parse.cpp index 8c35cc1626..63015dadc6 100644 --- a/ydb/library/yql/udfs/common/url_base/lib/url_parse.cpp +++ b/ydb/library/yql/udfs/common/url_base/lib/url_parse.cpp @@ -35,7 +35,7 @@ namespace NUrlUdf { if (Name() == name) { TUrlParseIndexes urlParseIndexes; - builder.Args(1)->Add<TAutoMap<char*>>(); + builder.Args(1)->Add<TAutoMap<char*>>(); const auto optionalStringType = builder.Optional()->Item<char*>().Build(); const auto structBuilder = builder.Struct(FieldsCount); structBuilder->AddField("ParseError", optionalStringType, &urlParseIndexes.ParseError); diff --git a/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp b/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp index 4eb94e0167..1278ec565c 100644 --- a/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp +++ b/ydb/library/yql/udfs/common/yson2/yson2_udf.cpp @@ -985,7 +985,7 @@ public: return true; } - builder.Args()->Add(builder.Resource(NodeResourceName)).Flags(ICallablePayload::TArgumentFlags::AutoMap).Add(optionsType); + builder.Args()->Add(builder.Resource(NodeResourceName)).Flags(ICallablePayload::TArgumentFlags::AutoMap).Add(optionsType); builder.Returns(resultType); if (!typesOnly) { diff --git a/ydb/library/yql/utils/backtrace/backtrace.cpp b/ydb/library/yql/utils/backtrace/backtrace.cpp index 3d3050fd3a..f9487f6671 100644 --- a/ydb/library/yql/utils/backtrace/backtrace.cpp +++ b/ydb/library/yql/utils/backtrace/backtrace.cpp @@ -5,7 +5,7 @@ #include <llvm/Support/raw_ostream.h> #include <library/cpp/malloc/api/malloc.h> - + #include <util/generic/hash.h> #include <util/generic/xrange.h> #include <util/generic/yexception.h> @@ -147,10 +147,10 @@ void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t st void PrintBacktraceToStderr(int signum) { - if (!NMalloc::IsAllocatorCorrupted) { - /* we want memory allocation for backtrace printing */ - KikimrBacktraceFormatImpl(&Cerr); - } + if (!NMalloc::IsAllocatorCorrupted) { + /* we want memory allocation for backtrace printing */ + KikimrBacktraceFormatImpl(&Cerr); + } /* Now reraise the signal. We reactivate the signal’s default handling, which is to terminate the process. We could just call exit or abort, but reraising the signal sets the return status from the process diff --git a/ydb/library/yql/utils/cast.h b/ydb/library/yql/utils/cast.h index 5c0452a6e1..dac8508092 100644 --- a/ydb/library/yql/utils/cast.h +++ b/ydb/library/yql/utils/cast.h @@ -1,15 +1,15 @@ -#pragma once -#include "yql_panic.h" - -namespace NYql { - -template<class T, class F> -[[nodiscard]] -inline T EnsureDynamicCast(F from) { - YQL_ENSURE(from, "source should not be null"); - T result = dynamic_cast<T>(from); - YQL_ENSURE(result, "dynamic_cast failed"); - return result; -} - -} // namespace NYql +#pragma once +#include "yql_panic.h" + +namespace NYql { + +template<class T, class F> +[[nodiscard]] +inline T EnsureDynamicCast(F from) { + YQL_ENSURE(from, "source should not be null"); + T result = dynamic_cast<T>(from); + YQL_ENSURE(result, "dynamic_cast failed"); + return result; +} + +} // namespace NYql diff --git a/ydb/library/yql/utils/utf8.cpp b/ydb/library/yql/utils/utf8.cpp index 48a90233c3..af284849a8 100644 --- a/ydb/library/yql/utils/utf8.cpp +++ b/ydb/library/yql/utils/utf8.cpp @@ -1,9 +1,9 @@ #include "utf8.h" - -#include <util/charset/wide.h> - + +#include <util/charset/wide.h> + #include <ctype.h> -#include <vector> +#include <vector> namespace NYql { @@ -27,105 +27,105 @@ unsigned char GetRange(unsigned char c) { return type[c]; } -struct TByteRange { - ui8 First = 0; - ui8 Last = 0; -}; - -struct TUtf8Ranges { - size_t BytesCount = 0; - TByteRange Bytes[4] = {}; -}; - -// see https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8 -inline static const std::vector<TUtf8Ranges> Utf8Ranges = { - { 1, { {0x00, 0x7f}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, } }, - { 2, { {0xc2, 0xdf}, {0x80, 0xbf}, {0x00, 0x00}, {0x00, 0x00}, } }, - { 3, { {0xe0, 0xe0}, {0xa0, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xe1, 0xec}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xed, 0xed}, {0x80, 0x9f}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xee, 0xef}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 4, { {0xf0, 0xf0}, {0x90, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, - { 4, { {0xf1, 0xf3}, {0x80, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, - { 4, { {0xf4, 0xf4}, {0x80, 0x8f}, {0x80, 0xbf}, {0x80, 0xbf}, } }, -}; - -std::optional<std::string> RoundBadUtf8(size_t range, std::string_view inputString, size_t pos, - bool roundDown) -{ - Y_ENSURE(range > 0); - Y_ENSURE(range < Utf8Ranges.size()); - - const std::string prefix{inputString.substr(0, pos)}; - std::string_view suffix = inputString.substr(pos, Utf8Ranges[range].BytesCount); - - std::string newSuffix; - if (roundDown) { - std::optional<size_t> lastNonMin; - for (size_t i = 0; i < suffix.size(); ++i) { - if (Utf8Ranges[range].Bytes[i].First < ui8(suffix[i])) { - lastNonMin = i; - } - } - - if (!lastNonMin) { - for (size_t i = 0; i < Utf8Ranges[range - 1].BytesCount; ++i) { - newSuffix.push_back(Utf8Ranges[range - 1].Bytes[i].Last); - } - } else { - for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { - if (i < *lastNonMin) { - ui8 c = suffix[i]; - newSuffix.push_back(c); - } else if (i == *lastNonMin) { - ui8 c = suffix[i]; - newSuffix.push_back(std::min<ui8>(c - 1, Utf8Ranges[range].Bytes[i].Last)); - } else { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].Last); - } - } - } - } else { - std::optional<size_t> lastNonMax; - bool valid = true; - for (size_t i = 0; i < suffix.size(); ++i) { - ui8 last = Utf8Ranges[range].Bytes[i].Last; - ui8 first = Utf8Ranges[range].Bytes[i].First; - ui8 curr = ui8(suffix[i]); - - valid = valid && curr <= last && curr >= first; - if (curr < last) { - lastNonMax = i; - } - } - - if (valid) { - newSuffix = suffix; - for (size_t i = suffix.size(); i < Utf8Ranges[range].BytesCount; ++i) { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); - } - } else if (!lastNonMax) { - return NextValidUtf8(prefix); - } else { - for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { - if (i < *lastNonMax) { - ui8 c = suffix[i]; - newSuffix.push_back(c); - } else if (i == *lastNonMax) { - ui8 c = suffix[i]; - newSuffix.push_back(std::max<ui8>(c + 1, Utf8Ranges[range].Bytes[i].First)); - } else { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); - } - } - } - - } - return prefix + newSuffix; -} - -} - +struct TByteRange { + ui8 First = 0; + ui8 Last = 0; +}; + +struct TUtf8Ranges { + size_t BytesCount = 0; + TByteRange Bytes[4] = {}; +}; + +// see https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8 +inline static const std::vector<TUtf8Ranges> Utf8Ranges = { + { 1, { {0x00, 0x7f}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, } }, + { 2, { {0xc2, 0xdf}, {0x80, 0xbf}, {0x00, 0x00}, {0x00, 0x00}, } }, + { 3, { {0xe0, 0xe0}, {0xa0, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, + { 3, { {0xe1, 0xec}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, + { 3, { {0xed, 0xed}, {0x80, 0x9f}, {0x80, 0xbf}, {0x00, 0x00}, } }, + { 3, { {0xee, 0xef}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, + { 4, { {0xf0, 0xf0}, {0x90, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, + { 4, { {0xf1, 0xf3}, {0x80, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, + { 4, { {0xf4, 0xf4}, {0x80, 0x8f}, {0x80, 0xbf}, {0x80, 0xbf}, } }, +}; + +std::optional<std::string> RoundBadUtf8(size_t range, std::string_view inputString, size_t pos, + bool roundDown) +{ + Y_ENSURE(range > 0); + Y_ENSURE(range < Utf8Ranges.size()); + + const std::string prefix{inputString.substr(0, pos)}; + std::string_view suffix = inputString.substr(pos, Utf8Ranges[range].BytesCount); + + std::string newSuffix; + if (roundDown) { + std::optional<size_t> lastNonMin; + for (size_t i = 0; i < suffix.size(); ++i) { + if (Utf8Ranges[range].Bytes[i].First < ui8(suffix[i])) { + lastNonMin = i; + } + } + + if (!lastNonMin) { + for (size_t i = 0; i < Utf8Ranges[range - 1].BytesCount; ++i) { + newSuffix.push_back(Utf8Ranges[range - 1].Bytes[i].Last); + } + } else { + for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { + if (i < *lastNonMin) { + ui8 c = suffix[i]; + newSuffix.push_back(c); + } else if (i == *lastNonMin) { + ui8 c = suffix[i]; + newSuffix.push_back(std::min<ui8>(c - 1, Utf8Ranges[range].Bytes[i].Last)); + } else { + newSuffix.push_back(Utf8Ranges[range].Bytes[i].Last); + } + } + } + } else { + std::optional<size_t> lastNonMax; + bool valid = true; + for (size_t i = 0; i < suffix.size(); ++i) { + ui8 last = Utf8Ranges[range].Bytes[i].Last; + ui8 first = Utf8Ranges[range].Bytes[i].First; + ui8 curr = ui8(suffix[i]); + + valid = valid && curr <= last && curr >= first; + if (curr < last) { + lastNonMax = i; + } + } + + if (valid) { + newSuffix = suffix; + for (size_t i = suffix.size(); i < Utf8Ranges[range].BytesCount; ++i) { + newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); + } + } else if (!lastNonMax) { + return NextValidUtf8(prefix); + } else { + for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { + if (i < *lastNonMax) { + ui8 c = suffix[i]; + newSuffix.push_back(c); + } else if (i == *lastNonMax) { + ui8 c = suffix[i]; + newSuffix.push_back(std::max<ui8>(c + 1, Utf8Ranges[range].Bytes[i].First)); + } else { + newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); + } + } + } + + } + return prefix + newSuffix; +} + +} + bool IsUtf8(const std::string_view& str) { for (auto it = str.cbegin(); str.cend() != it;) { #define COPY() if (str.cend() != it) { c = *it++; } else { return false; } @@ -169,92 +169,92 @@ unsigned char WideCharSize(char head) { } } -std::optional<std::string> RoundToNearestValidUtf8(const std::string_view& str, bool roundDown) { - const size_t ss = str.size(); - for (size_t pos = 0; pos < ss; ) { - ui8 c = str[pos]; - - for (size_t i = 0; i < Utf8Ranges.size(); ++i) { - auto& range = Utf8Ranges[i]; - - if (c < range.Bytes[0].First) { - return RoundBadUtf8(i, str, pos, roundDown); - } - - if (c <= range.Bytes[0].Last) { - // valid UTF8 code point start - for (size_t j = 1; j < range.BytesCount; ++j) { - if (pos + j >= ss) { - return RoundBadUtf8(i, str, pos, roundDown); - } - ui8 cur = str[pos + j]; - if (!(cur >= range.Bytes[j].First && cur <= range.Bytes[j].Last)) { - return RoundBadUtf8(i, str, pos, roundDown); - } - } - - pos += range.BytesCount; - break; - } else if (i + 1 == Utf8Ranges.size()) { - if (!roundDown) { - return NextValidUtf8(str.substr(0, pos)); - } - return RoundBadUtf8(i, str, pos, roundDown); - } - } - } - return std::string(str); +std::optional<std::string> RoundToNearestValidUtf8(const std::string_view& str, bool roundDown) { + const size_t ss = str.size(); + for (size_t pos = 0; pos < ss; ) { + ui8 c = str[pos]; + + for (size_t i = 0; i < Utf8Ranges.size(); ++i) { + auto& range = Utf8Ranges[i]; + + if (c < range.Bytes[0].First) { + return RoundBadUtf8(i, str, pos, roundDown); + } + + if (c <= range.Bytes[0].Last) { + // valid UTF8 code point start + for (size_t j = 1; j < range.BytesCount; ++j) { + if (pos + j >= ss) { + return RoundBadUtf8(i, str, pos, roundDown); + } + ui8 cur = str[pos + j]; + if (!(cur >= range.Bytes[j].First && cur <= range.Bytes[j].Last)) { + return RoundBadUtf8(i, str, pos, roundDown); + } + } + + pos += range.BytesCount; + break; + } else if (i + 1 == Utf8Ranges.size()) { + if (!roundDown) { + return NextValidUtf8(str.substr(0, pos)); + } + return RoundBadUtf8(i, str, pos, roundDown); + } + } + } + return std::string(str); +} + +std::optional<std::string> NextValidUtf8(const std::string_view& str) { + Y_ENSURE(IsUtf8(str)); + TUtf32String wide = UTF8ToUTF32<false>(str); + bool incremented = false; + size_t toDrop = 0; + for (auto it = wide.rbegin(); it != wide.rend(); ++it) { + auto& c = *it; + if (c < 0x10ffff) { + c = (c == 0xd7ff) ? 0xe000 : (c + 1); + incremented = true; + break; + } else { + ++toDrop; + } + } + + if (!incremented) { + return {}; + } + + Y_ENSURE(toDrop < wide.size()); + wide.resize(wide.size() - toDrop); + + TString result = WideToUTF8(wide); + return std::string(result.data(), result.size()); +} + +std::optional<std::string> NextLexicographicString(const std::string_view& str) { + bool incremented = false; + size_t toDrop = 0; + std::string result{str}; + for (auto it = result.rbegin(); it != result.rend(); ++it) { + auto& c = *it; + if (ui8(c) < 0xff) { + ++c; + incremented = true; + break; + } else { + ++toDrop; + } + } + + if (!incremented) { + return {}; + } + + Y_ENSURE(toDrop < result.size()); + result.resize(result.size() - toDrop); + return result; +} + } - -std::optional<std::string> NextValidUtf8(const std::string_view& str) { - Y_ENSURE(IsUtf8(str)); - TUtf32String wide = UTF8ToUTF32<false>(str); - bool incremented = false; - size_t toDrop = 0; - for (auto it = wide.rbegin(); it != wide.rend(); ++it) { - auto& c = *it; - if (c < 0x10ffff) { - c = (c == 0xd7ff) ? 0xe000 : (c + 1); - incremented = true; - break; - } else { - ++toDrop; - } - } - - if (!incremented) { - return {}; - } - - Y_ENSURE(toDrop < wide.size()); - wide.resize(wide.size() - toDrop); - - TString result = WideToUTF8(wide); - return std::string(result.data(), result.size()); -} - -std::optional<std::string> NextLexicographicString(const std::string_view& str) { - bool incremented = false; - size_t toDrop = 0; - std::string result{str}; - for (auto it = result.rbegin(); it != result.rend(); ++it) { - auto& c = *it; - if (ui8(c) < 0xff) { - ++c; - incremented = true; - break; - } else { - ++toDrop; - } - } - - if (!incremented) { - return {}; - } - - Y_ENSURE(toDrop < result.size()); - result.resize(result.size() - toDrop); - return result; -} - -} diff --git a/ydb/library/yql/utils/utf8.h b/ydb/library/yql/utils/utf8.h index ac8da7c3f0..5c28353416 100644 --- a/ydb/library/yql/utils/utf8.h +++ b/ydb/library/yql/utils/utf8.h @@ -1,6 +1,6 @@ #pragma once -#include <optional> +#include <optional> #include <string_view> namespace NYql { @@ -9,8 +9,8 @@ bool IsUtf8(const std::string_view& str); unsigned char WideCharSize(char head); -std::optional<std::string> RoundToNearestValidUtf8(const std::string_view& str, bool roundDown); -std::optional<std::string> NextValidUtf8(const std::string_view& str); -std::optional<std::string> NextLexicographicString(const std::string_view& str); - +std::optional<std::string> RoundToNearestValidUtf8(const std::string_view& str, bool roundDown); +std::optional<std::string> NextValidUtf8(const std::string_view& str); +std::optional<std::string> NextLexicographicString(const std::string_view& str); + } diff --git a/ydb/library/yql/utils/utf8_ut.cpp b/ydb/library/yql/utils/utf8_ut.cpp index 0003103f21..7479acd7a1 100644 --- a/ydb/library/yql/utils/utf8_ut.cpp +++ b/ydb/library/yql/utils/utf8_ut.cpp @@ -24,76 +24,76 @@ Y_UNIT_TEST_SUITE(TUtf8Tests) { UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\xF0'), 4); UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\xFF'), 0); } - - Y_UNIT_TEST(RoundingDown) { - auto checkDown = [](std::string_view in, std::string_view out) { - auto res = NYql::RoundToNearestValidUtf8(in, true); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res <= in); - }; - checkDown("привет", "привет"); - checkDown("тест\x80", "тест\x7f"); - checkDown("привет\xf5", "привет\xf4\x8f\xbf\xbf"); - checkDown("тест2\xee\x80\x7f", "тест2\xed\x9f\xbf"); - checkDown("ага\xf0\xaa\xaa\xff", "ага\xf0\xaa\xaa\xbf"); - } - - Y_UNIT_TEST(RoundingUp) { - auto checkUp = [](std::string_view in, std::string_view out) { - auto res = NYql::RoundToNearestValidUtf8(in, false); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res >= in); - }; - - checkUp("", ""); - checkUp("привет", "привет"); - checkUp("а\xf6", "б"); - checkUp("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf\xf5", "\xf4\x8f\xbf\xbfб"); - UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf4\x8f\xbf\xbf\xf5", false)); - UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf5", false)); - checkUp("тест\x80", "тест\xc2\x80"); - checkUp("тест\xdf", "тест\xdf\x80"); - checkUp("тест\xf0\x90\xff", "тест\xf0\x91\x80\x80"); - checkUp("ааа\xff", "ааб"); - } - - Y_UNIT_TEST(NextValid) { - auto checkNext = [](std::string_view in, std::string_view out) { - auto res = NYql::NextValidUtf8(in); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res > in); - }; - - UNIT_ASSERT(!NYql::NextValidUtf8("")); - checkNext("привет", "привеу"); - checkNext("а", "б"); - checkNext(std::string_view("\x00", 1), "\x01"); - checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfб"); - UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf")); - UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbf")); - } - - Y_UNIT_TEST(NextValidString) { - auto checkNext = [](std::string_view in, std::string_view out) { - auto res = NYql::NextLexicographicString(in); - UNIT_ASSERT(res); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res > in); - }; - - UNIT_ASSERT(!NYql::NextLexicographicString("")); - checkNext("привет", "привеу"); - checkNext("а", "б"); - checkNext(std::string_view("\x00", 1), "\x01"); - checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xc0"); - UNIT_ASSERT(!NYql::NextLexicographicString("\xff")); - UNIT_ASSERT(!NYql::NextLexicographicString("\xff\xff")); - checkNext(std::string_view("x\x00\xff\xff", 4), "x\x01"); - } + + Y_UNIT_TEST(RoundingDown) { + auto checkDown = [](std::string_view in, std::string_view out) { + auto res = NYql::RoundToNearestValidUtf8(in, true); + UNIT_ASSERT(res); + UNIT_ASSERT(NYql::IsUtf8(*res)); + UNIT_ASSERT_VALUES_EQUAL(*res, out); + UNIT_ASSERT(*res <= in); + }; + checkDown("привет", "привет"); + checkDown("тест\x80", "тест\x7f"); + checkDown("привет\xf5", "привет\xf4\x8f\xbf\xbf"); + checkDown("тест2\xee\x80\x7f", "тест2\xed\x9f\xbf"); + checkDown("ага\xf0\xaa\xaa\xff", "ага\xf0\xaa\xaa\xbf"); + } + + Y_UNIT_TEST(RoundingUp) { + auto checkUp = [](std::string_view in, std::string_view out) { + auto res = NYql::RoundToNearestValidUtf8(in, false); + UNIT_ASSERT(res); + UNIT_ASSERT(NYql::IsUtf8(*res)); + UNIT_ASSERT_VALUES_EQUAL(*res, out); + UNIT_ASSERT(*res >= in); + }; + + checkUp("", ""); + checkUp("привет", "привет"); + checkUp("а\xf6", "б"); + checkUp("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf\xf5", "\xf4\x8f\xbf\xbfб"); + UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf4\x8f\xbf\xbf\xf5", false)); + UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf5", false)); + checkUp("тест\x80", "тест\xc2\x80"); + checkUp("тест\xdf", "тест\xdf\x80"); + checkUp("тест\xf0\x90\xff", "тест\xf0\x91\x80\x80"); + checkUp("ааа\xff", "ааб"); + } + + Y_UNIT_TEST(NextValid) { + auto checkNext = [](std::string_view in, std::string_view out) { + auto res = NYql::NextValidUtf8(in); + UNIT_ASSERT(res); + UNIT_ASSERT(NYql::IsUtf8(*res)); + UNIT_ASSERT_VALUES_EQUAL(*res, out); + UNIT_ASSERT(*res > in); + }; + + UNIT_ASSERT(!NYql::NextValidUtf8("")); + checkNext("привет", "привеу"); + checkNext("а", "б"); + checkNext(std::string_view("\x00", 1), "\x01"); + checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfб"); + UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf")); + UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbf")); + } + + Y_UNIT_TEST(NextValidString) { + auto checkNext = [](std::string_view in, std::string_view out) { + auto res = NYql::NextLexicographicString(in); + UNIT_ASSERT(res); + UNIT_ASSERT_VALUES_EQUAL(*res, out); + UNIT_ASSERT(*res > in); + }; + + UNIT_ASSERT(!NYql::NextLexicographicString("")); + checkNext("привет", "привеу"); + checkNext("а", "б"); + checkNext(std::string_view("\x00", 1), "\x01"); + checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xc0"); + UNIT_ASSERT(!NYql::NextLexicographicString("\xff")); + UNIT_ASSERT(!NYql::NextLexicographicString("\xff\xff")); + checkNext(std::string_view("x\x00\xff\xff", 4), "x\x01"); + } } diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make index 2c0ea0d2fa..afba70be53 100644 --- a/ydb/library/yql/utils/ya.make +++ b/ydb/library/yql/utils/ya.make @@ -8,7 +8,7 @@ OWNER( SRCS( bind_in_range.cpp bind_in_range.h - cast.h + cast.h debug_info.cpp debug_info.h future_action.cpp @@ -32,8 +32,8 @@ SRCS( swap_bytes.h yql_panic.cpp yql_panic.h - yql_paths.cpp - yql_paths.h + yql_paths.cpp + yql_paths.h utf8.cpp ) diff --git a/ydb/library/yql/utils/yql_paths.cpp b/ydb/library/yql/utils/yql_paths.cpp index 40f63f9099..cc7dd29bfb 100644 --- a/ydb/library/yql/utils/yql_paths.cpp +++ b/ydb/library/yql/utils/yql_paths.cpp @@ -1,23 +1,23 @@ -#include "yql_paths.h" - -#include <util/folder/pathsplit.h> - -namespace NYql { - -TString BuildTablePath(TStringBuf prefixPath, TStringBuf path) { - if (prefixPath.empty()) { - return TString(path); - } - prefixPath.SkipPrefix("//"); - - TPathSplitUnix prefixPathSplit(prefixPath); - TPathSplitUnix pathSplit(path); - - if (pathSplit.IsAbsolute) { - return TString(path); - } - - return prefixPathSplit.AppendMany(pathSplit.begin(), pathSplit.end()).Reconstruct(); -} - -} +#include "yql_paths.h" + +#include <util/folder/pathsplit.h> + +namespace NYql { + +TString BuildTablePath(TStringBuf prefixPath, TStringBuf path) { + if (prefixPath.empty()) { + return TString(path); + } + prefixPath.SkipPrefix("//"); + + TPathSplitUnix prefixPathSplit(prefixPath); + TPathSplitUnix pathSplit(path); + + if (pathSplit.IsAbsolute) { + return TString(path); + } + + return prefixPathSplit.AppendMany(pathSplit.begin(), pathSplit.end()).Reconstruct(); +} + +} diff --git a/ydb/library/yql/utils/yql_paths.h b/ydb/library/yql/utils/yql_paths.h index 5ba94e9e95..3ec05e7df8 100644 --- a/ydb/library/yql/utils/yql_paths.h +++ b/ydb/library/yql/utils/yql_paths.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NYql { - -TString BuildTablePath(TStringBuf prefixPath, TStringBuf path); - -} +#pragma once + +#include <util/generic/strbuf.h> + +namespace NYql { + +TString BuildTablePath(TStringBuf prefixPath, TStringBuf path); + +} |