diff options
author | vartyukh <vartyukh@yandex-team.ru> | 2022-02-10 16:50:16 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:16 +0300 |
commit | 97df5ca7413550bf233fc6c7210e292fca0a51af (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 18f9abdade80b500b7ea1ef2d286e2465a1c8466 (diff) | |
download | ydb-97df5ca7413550bf233fc6c7210e292fca0a51af.tar.gz |
Restoring authorship annotation for <vartyukh@yandex-team.ru>. Commit 2 of 2.
81 files changed, 4940 insertions, 4940 deletions
diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index be1a85ff70..6fa25b9965 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -1702,18 +1702,18 @@ namespace NActors { } TActorSystem* TTestActorRuntimeBase::GetAnyNodeActorSystem() { - for (auto& x : Nodes) { - return x.second->ActorSystem.Get(); - } + for (auto& x : Nodes) { + return x.second->ActorSystem.Get(); + } Y_FAIL("Don't use this method."); - } - + } + TActorSystem* TTestActorRuntimeBase::GetActorSystem(ui32 nodeId) { auto it = Nodes.find(GetNodeId(nodeId)); Y_VERIFY(it != Nodes.end()); return it->second->ActorSystem.Get(); } - + TEventMailBox& TTestActorRuntimeBase::GetMailbox(ui32 nodeId, ui32 hint) { TGuard<TMutex> guard(Mutex); diff --git a/library/cpp/actors/testlib/test_runtime.h b/library/cpp/actors/testlib/test_runtime.h index 5e339698f2..26e3b45c98 100644 --- a/library/cpp/actors/testlib/test_runtime.h +++ b/library/cpp/actors/testlib/test_runtime.h @@ -285,7 +285,7 @@ namespace NActors { } TActorSystem* SingleSys() const; - TActorSystem* GetAnyNodeActorSystem(); + TActorSystem* GetAnyNodeActorSystem(); TActorSystem* GetActorSystem(ui32 nodeId); template <typename TEvent> TEvent* GrabEdgeEventIf(TAutoPtr<IEventHandle>& handle, std::function<bool(const TEvent&)> predicate, TDuration simTimeout = TDuration::Max()) { diff --git a/library/cpp/grpc/client/ya.make b/library/cpp/grpc/client/ya.make index 607bd474cb..a4e74b067c 100644 --- a/library/cpp/grpc/client/ya.make +++ b/library/cpp/grpc/client/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER( ddoarn g:kikimr ) - -SRCS( + +SRCS( grpc_client_low.cpp -) - -PEERDIR( +) + +PEERDIR( contrib/libs/grpc -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/library/cpp/ipv6_address/ipv6_address.cpp b/library/cpp/ipv6_address/ipv6_address.cpp index 5f65aa37d1..be8fcbae13 100644 --- a/library/cpp/ipv6_address/ipv6_address.cpp +++ b/library/cpp/ipv6_address/ipv6_address.cpp @@ -116,31 +116,31 @@ TString TIpv6Address::ToString(bool* ok) const noexcept { TString TIpv6Address::ToString(bool PrintScopeId, bool* ok) const noexcept { TString result; bool isOk = true; - + if (Type_ == TIpv6Address::Ipv4) { - result.resize(INET_ADDRSTRLEN + 2); - in_addr addr; + result.resize(INET_ADDRSTRLEN + 2); + in_addr addr; ToInAddr(addr); - isOk = inet_ntop(AF_INET, &addr, result.begin(), INET_ADDRSTRLEN); + isOk = inet_ntop(AF_INET, &addr, result.begin(), INET_ADDRSTRLEN); result.resize(result.find('\0')); } else if (Type_ == TIpv6Address::Ipv6) { - result.resize(INET6_ADDRSTRLEN + 2); - in6_addr addr; + result.resize(INET6_ADDRSTRLEN + 2); + in6_addr addr; ToIn6Addr(addr); - isOk = inet_ntop(AF_INET6, &addr, result.begin(), INET6_ADDRSTRLEN); + isOk = inet_ntop(AF_INET6, &addr, result.begin(), INET6_ADDRSTRLEN); result.resize(result.find('\0')); if (PrintScopeId) result += "%" + ::ToString(ScopeId_); } else { result = "null"; isOk = true; - } - + } + if (ok) { - *ok = isOk; - } - - return result; + *ok = isOk; + } + + return result; } void TIpv6Address::ToSockaddrAndSocklen(sockaddr_in& sockAddrIPv4, @@ -259,9 +259,9 @@ IOutputStream& operator<<(IOutputStream& out, const TIpv6Address& ipv6Address) n bool ok; const TString& strIp = ipv6Address.ToString(&ok); if (!ok) { - return out << "Can not convert ip to string"; + return out << "Can not convert ip to string"; } else { - return out << strIp; + return out << strIp; } } diff --git a/library/cpp/messagebus/rain_check/http/client.cpp b/library/cpp/messagebus/rain_check/http/client.cpp index eec4b619af..5ef5ceeece 100644 --- a/library/cpp/messagebus/rain_check/http/client.cpp +++ b/library/cpp/messagebus/rain_check/http/client.cpp @@ -1,7 +1,7 @@ #include "client.h" -#include "http_code_extractor.h" - +#include "http_code_extractor.h" + #include <library/cpp/http/io/stream.h> #include <library/cpp/neh/factory.h> #include <library/cpp/neh/http_common.h> @@ -9,7 +9,7 @@ #include <library/cpp/neh/neh.h> #include <util/generic/ptr.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/network/socket.h> #include <util/stream/str.h> @@ -129,7 +129,7 @@ namespace NRainCheck { return Response->Data; } - + void THttpFuture::SetDoneAndSchedule(TAutoPtr<NNeh::TResponse> response) { if (!response->IsError()) { ErrorCode = THttpFuture::NoError; diff --git a/library/cpp/messagebus/rain_check/http/client.h b/library/cpp/messagebus/rain_check/http/client.h index 343205ade4..d4199c4c98 100644 --- a/library/cpp/messagebus/rain_check/http/client.h +++ b/library/cpp/messagebus/rain_check/http/client.h @@ -4,7 +4,7 @@ #include <library/cpp/http/misc/httpcodes.h> -#include <util/generic/maybe.h> +#include <util/generic/maybe.h> #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/system/defaults.h> @@ -45,7 +45,7 @@ namespace NRainCheck { ui32 GetHttpCode() const; TString GetResponseBody() const; - + bool HasError() const; EError GetErrorCode() const; TString GetErrorDescription() const; diff --git a/library/cpp/messagebus/rain_check/http/client_ut.cpp b/library/cpp/messagebus/rain_check/http/client_ut.cpp index ef1712a03d..1628114391 100644 --- a/library/cpp/messagebus/rain_check/http/client_ut.cpp +++ b/library/cpp/messagebus/rain_check/http/client_ut.cpp @@ -1,7 +1,7 @@ #include <library/cpp/testing/unittest/registar.h> #include "client.h" -#include "http_code_extractor.h" +#include "http_code_extractor.h" #include <library/cpp/messagebus/rain_check/test/ut/test.h> @@ -173,26 +173,26 @@ Y_UNIT_TEST_SUITE(RainCheckHttpClient) { env.TestSync.WaitForAndIncrement(1); } - + Y_UNIT_TEST(HttpCodeExtraction) { // Find "request failed(" string, then copy len("HTTP/1.X NNN") chars and try to convert NNN to HTTP code. - + #define CHECK_VALID_LINE(line, code) \ UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ UNIT_ASSERT(!!TryGetHttpCodeFromErrorDescription(line)); \ UNIT_ASSERT_EQUAL(*TryGetHttpCodeFromErrorDescription(line), code) - + CHECK_VALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.0 200 Some random message"), 200); CHECK_VALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.0 404 Some random message"), 404); CHECK_VALID_LINE(TStringBuf("request failed(HTTP/1.0 100 Some random message"), 100); CHECK_VALID_LINE(TStringBuf("request failed(HTTP/1.0 105)"), 105); CHECK_VALID_LINE(TStringBuf("request failed(HTTP/1.1 2004 Some random message"), 200); -#undef CHECK_VALID_LINE - +#undef CHECK_VALID_LINE + #define CHECK_INVALID_LINE(line) \ UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ UNIT_ASSERT(!TryGetHttpCodeFromErrorDescription(line)) - + CHECK_INVALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.1 1 Some random message")); CHECK_INVALID_LINE(TStringBuf("request failed(HTTP/1.0 asdf Some random message")); CHECK_INVALID_LINE(TStringBuf("HTTP/1.0 200 Some random message")); @@ -200,6 +200,6 @@ Y_UNIT_TEST_SUITE(RainCheckHttpClient) { CHECK_INVALID_LINE(TStringBuf("HTTP/1.0 200 Some random message")); CHECK_INVALID_LINE(TStringBuf("HTTP/1.0 200")); CHECK_INVALID_LINE(TStringBuf("request failed(HTTP/1.1 3334 Some random message")); -#undef CHECK_INVALID_LINE - } +#undef CHECK_INVALID_LINE + } } diff --git a/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp b/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp index 3d97899d3b..51d75762f6 100644 --- a/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp +++ b/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp @@ -1,13 +1,13 @@ -#include "http_code_extractor.h" - +#include "http_code_extractor.h" + #include <library/cpp/http/io/stream.h> #include <library/cpp/http/misc/httpcodes.h> #include <util/generic/maybe.h> -#include <util/generic/strbuf.h> -#include <util/string/cast.h> - -namespace NRainCheck { +#include <util/generic/strbuf.h> +#include <util/string/cast.h> + +namespace NRainCheck { TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage) { // Try to get HttpCode from library/cpp/neh response. // If response has HttpCode and it is not 200 OK, library/cpp/neh will send a message @@ -17,9 +17,9 @@ namespace NRainCheck { const TStringBuf SUBSTR = "request failed("; const size_t SUBSTR_LEN = SUBSTR.size(); const size_t FIRST_LINE_LEN = TStringBuf("HTTP/1.X NNN").size(); - + TMaybe<HttpCodes> httpCode; - + const size_t substrPos = errorMessage.find(SUBSTR); if (substrPos != TStringBuf::npos) { const TStringBuf firstLineStart = errorMessage.SubStr(substrPos + SUBSTR_LEN, FIRST_LINE_LEN); @@ -30,10 +30,10 @@ namespace NRainCheck { } } catch (const TFromStringException& ex) { // Can't parse HttpCode: it is OK, because ErrorDescription can be random string. - } - } + } + } return httpCode; - } - -} + } + +} diff --git a/library/cpp/messagebus/rain_check/http/http_code_extractor.h b/library/cpp/messagebus/rain_check/http/http_code_extractor.h index 553a1017ec..33b565fa1c 100644 --- a/library/cpp/messagebus/rain_check/http/http_code_extractor.h +++ b/library/cpp/messagebus/rain_check/http/http_code_extractor.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <library/cpp/http/misc/httpcodes.h> -#include <util/generic/maybe.h> -#include <util/generic/strbuf.h> - -namespace NRainCheck { +#include <util/generic/maybe.h> +#include <util/generic/strbuf.h> + +namespace NRainCheck { // Try to get HttpCode from library/cpp/neh response. // If response has HttpCode and it is not 200 OK, library/cpp/neh will send a message // "library/cpp/neh/http.cpp:<LINE>: request failed(<FIRST-HTTP-RESPONSE-LINE>)" // (see library/cpp/neh/http.cpp:625). So, we will try to parse this message and // find out HttpCode in it. It is bad temporary solution, but we have no choice. TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage); - + } diff --git a/library/cpp/messagebus/rain_check/http/ya.make b/library/cpp/messagebus/rain_check/http/ya.make index 3a5b1b1c50..ef13329df3 100644 --- a/library/cpp/messagebus/rain_check/http/ya.make +++ b/library/cpp/messagebus/rain_check/http/ya.make @@ -4,7 +4,7 @@ OWNER(g:messagebus) SRCS( client.cpp - http_code_extractor.cpp + http_code_extractor.cpp ) PEERDIR( diff --git a/util/folder/filelist_ut.cpp b/util/folder/filelist_ut.cpp index 295c358c8a..0cdcdf3d00 100644 --- a/util/folder/filelist_ut.cpp +++ b/util/folder/filelist_ut.cpp @@ -10,12 +10,12 @@ class TFileListTest: public TTestBase { UNIT_TEST_SUITE(TFileListTest); UNIT_TEST(TestSimple); - UNIT_TEST(TestPrefix); + UNIT_TEST(TestPrefix); UNIT_TEST_SUITE_END(); public: void TestSimple(); - void TestPrefix(); + void TestPrefix(); }; void TFileListTest::TestSimple() { @@ -30,27 +30,27 @@ void TFileListTest::TestSimple() { UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); } -void TFileListTest::TestPrefix() { - TTempDir tempDir("nonexistingdir"); +void TFileListTest::TestPrefix() { + TTempDir tempDir("nonexistingdir"); TFile((tempDir() + LOCSLASH_S "good_file1").data(), CreateAlways); TFile((tempDir() + LOCSLASH_S "good_file2").data(), CreateAlways); TFile((tempDir() + LOCSLASH_S "bad_file1").data(), CreateAlways); TFile((tempDir() + LOCSLASH_S "bad_file2").data(), CreateAlways); - - const bool SORT = true; - TFileList fileList; - { + + const bool SORT = true; + TFileList fileList; + { fileList.Fill(tempDir().data(), "good_file", SORT); UNIT_ASSERT_EQUAL(TString(fileList.Next()), "good_file1"); UNIT_ASSERT_EQUAL(TString(fileList.Next()), "good_file2"); - UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); - } - { + UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); + } + { fileList.Fill(tempDir().data(), "bad_file", SORT); UNIT_ASSERT_EQUAL(TString(fileList.Next()), "bad_file1"); UNIT_ASSERT_EQUAL(TString(fileList.Next()), "bad_file2"); - UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); - } -} - + UNIT_ASSERT_EQUAL(fileList.Next(), nullptr); + } +} + UNIT_TEST_SUITE_REGISTRATION(TFileListTest); diff --git a/ydb/core/client/client_ut.cpp b/ydb/core/client/client_ut.cpp index 8b279bb704..b94c722588 100644 --- a/ydb/core/client/client_ut.cpp +++ b/ydb/core/client/client_ut.cpp @@ -12,7 +12,7 @@ #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/diff/diff.h> - + #include <ydb/library/yql/ast/yql_ast.h> #include <ydb/library/yql/ast/yql_expr.h> @@ -30,7 +30,7 @@ namespace Tests { // const ui32 TestDomain = 1; static const TString TablePlacement = "/dc-1/Berkanavt/tables"; - + namespace { void SetupLogging(TServer& server) { @@ -43,7 +43,7 @@ void SetupLogging(TServer& server) { server.GetRuntime()->SetLogPriority(NKikimrServices::HIVE, NActors::NLog::PRI_DEBUG); #endif } - + // Creates test tables on ctor and deletes on dtor struct TTestTables { enum EVariant { @@ -56,7 +56,7 @@ struct TTestTables { Sharded_SoftUpdates, Sharded_OutOfOrder_SoftUpdates, }; - + struct TOpts { bool Sharded = false; bool OutOfOrder = false; @@ -85,7 +85,7 @@ struct TTestTables { default: break; } - + switch (var) { case Sharded_NoOpts: case Sharded_OutOfOrder: @@ -943,7 +943,7 @@ Y_UNIT_TEST_SUITE(TClientTest) { )___"; UNIT_ASSERT(client.FlatQuery(writeQuery, writeRes)); - + NKikimrMiniKQL::TResult updateRes; const TString updateQuery = R"___( ( @@ -1178,7 +1178,7 @@ Y_UNIT_TEST_SUITE(TClientTest) { (return params) ) )___"; - + UNIT_ASSERT(client.FlatQueryParams(readQuery, readParams, false, readRes)); { @@ -1218,7 +1218,7 @@ Y_UNIT_TEST_SUITE(TClientTest) { )___"; UNIT_ASSERT(client.Compile(readQuery, binQuery)); - + const TString readParams = R"___( ( (let params (Parameters)) @@ -1259,7 +1259,7 @@ Y_UNIT_TEST_SUITE(TClientTest) { )) ) )___"; - + UNIT_ASSERT(client.FlatQuery(readQuery, readRes)); { @@ -2868,6 +2868,6 @@ Y_UNIT_TEST_SUITE(TClientTest) { } -} // namespace Tests -} // namespace NKikimr - +} // namespace Tests +} // namespace NKikimr + diff --git a/ydb/core/client/minikql_compile/compile_context.cpp b/ydb/core/client/minikql_compile/compile_context.cpp index 3689ec705c..44526ee5aa 100644 --- a/ydb/core/client/minikql_compile/compile_context.cpp +++ b/ydb/core/client/minikql_compile/compile_context.cpp @@ -1,34 +1,34 @@ -#include "compile_context.h" - +#include "compile_context.h" + #include <ydb/core/base/domain.h> #include <ydb/core/engine/kikimr_program_builder.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> - -namespace NYql { - + +namespace NYql { + TContext::TContext(const IFunctionRegistry* funcRegistry, - const TTypeEnvironment* typeEnv) + const TTypeEnvironment* typeEnv) : FuncRegistry(funcRegistry) , TypeEnv(typeEnv) , TypeInfoHelper(new NKikimr::NMiniKQL::TTypeInfoHelper) , PgmBuilder(new TKikimrProgramBuilder(*typeEnv, *funcRegistry)) - , WasParams(false) -{ -} - -bool TContext::NewParamsBuilder() { - if (!ParamsBuilder) { - ParamsBuilder = PgmBuilder->GetParametersBuilder(); - return true; - } - return false; -} - -TRuntimeNode TContext::NewParam(TStringBuf name, TType* type) { - WasParams = true; + , WasParams(false) +{ +} + +bool TContext::NewParamsBuilder() { + if (!ParamsBuilder) { + ParamsBuilder = PgmBuilder->GetParametersBuilder(); + return true; + } + return false; +} + +TRuntimeNode TContext::NewParam(TStringBuf name, TType* type) { + WasParams = true; return PgmBuilder->Parameter(TString(name), type); -} - +} + void TContext::AddTableLookup(const IDbSchemeResolver::TTable& request) { auto state = Tables.FindPtr(request.TableName); if (state) { @@ -61,20 +61,20 @@ TContext::TTableMap& TContext::GetTablesToResolve() { return Tables; } -TConvertResult TContext::Finish(TRuntimeNode convertedNode) { - TConvertResult convRes; - - if (!ParamsBuilder) { - // Program. - convRes.Node = PgmBuilder->Prepare(convertedNode); - } else if (!WasParams && ParamsBuilder) { - // Params without params. - convRes.Node = convertedNode; - } else { - // Params with params. +TConvertResult TContext::Finish(TRuntimeNode convertedNode) { + TConvertResult convRes; + + if (!ParamsBuilder) { + // Program. + convRes.Node = PgmBuilder->Prepare(convertedNode); + } else if (!WasParams && ParamsBuilder) { + // Params without params. + convRes.Node = convertedNode; + } else { + // Params with params. convRes.Errors.AddIssue(TPosition(1, 1), "Params program can't contains params."); - } - return convRes; -} - -} // namespace NYql + } + return convRes; +} + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/compile_context.h b/ydb/core/client/minikql_compile/compile_context.h index 389bfb3c22..22a9ebdf37 100644 --- a/ydb/core/client/minikql_compile/compile_context.h +++ b/ydb/core/client/minikql_compile/compile_context.h @@ -1,52 +1,52 @@ -#pragma once - -#include "compile_result.h" - +#pragma once + +#include "compile_result.h" + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/core/client/minikql_compile/db_key_resolver.h> #include <ydb/core/engine/kikimr_program_builder.h> - -#include <util/generic/hash.h> -#include <util/generic/list.h> -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> - - -namespace NYql { - -using NKikimr::NMiniKQL::IFunctionRegistry; -using NKikimr::NMiniKQL::TParametersBuilder; -using NKikimr::NMiniKQL::TTypeEnvironment; -using NKikimr::NMiniKQL::TType; -using NKikimr::NMiniKQL::TRuntimeNode; + +#include <util/generic/hash.h> +#include <util/generic/list.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> + + +namespace NYql { + +using NKikimr::NMiniKQL::IFunctionRegistry; +using NKikimr::NMiniKQL::TParametersBuilder; +using NKikimr::NMiniKQL::TTypeEnvironment; +using NKikimr::NMiniKQL::TType; +using NKikimr::NMiniKQL::TRuntimeNode; using NKikimr::NMiniKQL::TKikimrProgramBuilder; - - -struct TContext : public TAtomicRefCount<TContext> { - using TPtr = TIntrusivePtr<TContext>; + + +struct TContext : public TAtomicRefCount<TContext> { + using TPtr = TIntrusivePtr<TContext>; struct TTableState { IDbSchemeResolver::TTable Request; TMaybe<IDbSchemeResolver::TTableResult> Response; }; - + using TTableMap = THashMap<TString, TTableState>; TContext(const IFunctionRegistry* funcRegistry, - const TTypeEnvironment* typeEnv); - - bool NewParamsBuilder(); - TRuntimeNode NewParam(TStringBuf name, TType* type); - + const TTypeEnvironment* typeEnv); + + bool NewParamsBuilder(); + TRuntimeNode NewParam(TStringBuf name, TType* type); + void AddTableLookup(const IDbSchemeResolver::TTable& request); template<typename TStringType> IDbSchemeResolver::TTableResult* GetTableLookup(const TExprNode& node, const TStringType& tableName); TTableMap& GetTablesToResolve(); - - TConvertResult Finish(TRuntimeNode convertedNode); - -public: + + TConvertResult Finish(TRuntimeNode convertedNode); + +public: const IFunctionRegistry* FuncRegistry; const TTypeEnvironment* TypeEnv; NKikimr::NUdf::ITypeInfoHelper::TPtr TypeInfoHelper; @@ -54,8 +54,8 @@ public: TMaybe<TParametersBuilder> ParamsBuilder; private: - bool WasParams; + bool WasParams; TTableMap Tables; -}; - -} // namespace NYql +}; + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/compile_result.cpp b/ydb/core/client/minikql_compile/compile_result.cpp index 28156cf318..9891b7fc6f 100644 --- a/ydb/core/client/minikql_compile/compile_result.cpp +++ b/ydb/core/client/minikql_compile/compile_result.cpp @@ -1,20 +1,20 @@ -#include "compile_result.h" - - -namespace NYql { - +#include "compile_result.h" + + +namespace NYql { + TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssue& error) { Errors.AddIssue(error); -} - +} + TMiniKQLCompileResult::TMiniKQLCompileResult(const TIssues& errors) - : Errors(errors) -{ -} - + : Errors(errors) +{ +} + TMiniKQLCompileResult::TMiniKQLCompileResult(const TString& compiledProgram) - : CompiledProgram(compiledProgram) -{ -} - -} // namespace NYql + : CompiledProgram(compiledProgram) +{ +} + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/compile_result.h b/ydb/core/client/minikql_compile/compile_result.h index f244e8f3cd..d92fd02603 100644 --- a/ydb/core/client/minikql_compile/compile_result.h +++ b/ydb/core/client/minikql_compile/compile_result.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/ast/yql_errors.h> #include <ydb/library/yql/minikql/mkql_node.h> - -namespace NYql { - -using NKikimr::NMiniKQL::TRuntimeNode; - -struct TConvertResult { - TRuntimeNode Node; + +namespace NYql { + +using NKikimr::NMiniKQL::TRuntimeNode; + +struct TConvertResult { + TRuntimeNode Node; TIssues Errors; -}; - - -struct TMiniKQLCompileResult { - TMiniKQLCompileResult() = default; +}; + + +struct TMiniKQLCompileResult { + TMiniKQLCompileResult() = default; explicit TMiniKQLCompileResult(const TIssue& error); explicit TMiniKQLCompileResult(const TIssues& errors); explicit TMiniKQLCompileResult(const TString& compiledProgram); TIssues Errors; TString CompiledProgram; -}; - -} // namespace NYql +}; + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/db_key_resolver.cpp b/ydb/core/client/minikql_compile/db_key_resolver.cpp index e098277fe9..4e5cb34cf0 100644 --- a/ydb/core/client/minikql_compile/db_key_resolver.cpp +++ b/ydb/core/client/minikql_compile/db_key_resolver.cpp @@ -1,13 +1,13 @@ -#include "db_key_resolver.h" - -#include <util/stream/str.h> - - -namespace NYql { - +#include "db_key_resolver.h" + +#include <util/stream/str.h> + + +namespace NYql { + IDbSchemeResolver::TEvents::TEvResolveTablesResult::TEvResolveTablesResult(TTableResults&& result) : Result(std::move(result)) { } -} // namespace NYql +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/db_key_resolver.h b/ydb/core/client/minikql_compile/db_key_resolver.h index 42a858705d..e748525fa8 100644 --- a/ydb/core/client/minikql_compile/db_key_resolver.h +++ b/ydb/core/client/minikql_compile/db_key_resolver.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actorid.h> #include <library/cpp/actors/core/event.h> #include <library/cpp/actors/core/event_local.h> #include <library/cpp/actors/core/events.h> #include <ydb/core/scheme/tablet_scheme_defs.h> #include <ydb/core/scheme/scheme_tabledefs.h> - + #include <util/generic/map.h> #include <util/generic/set.h> #include <util/generic/string.h> - + #include <library/cpp/threading/future/future.h> - - -namespace NYql { - -using NKikimr::TTagDetails; - -class IDbSchemeResolver { -public: + + +namespace NYql { + +using NKikimr::TTagDetails; + +class IDbSchemeResolver { +public: struct TTable { TString TableName; TSet<TString> ColumnNames; @@ -28,11 +28,11 @@ public: }; struct TTableResult { - enum EStatus { + enum EStatus { Ok = 0, Error = 1, LookupError = 2 - }; + }; TTableResult(EStatus status, const TString& reason = TString()) : Status(status) @@ -46,38 +46,38 @@ public: ui32 AllowInplaceMode; }; - EStatus Status; + EStatus Status; TString Reason; TTable Table; TAutoPtr<NKikimr::TTableId> TableId; ui32 KeyColumnCount = 0; TMap<TString, TColumn> Columns; ui64 CacheGeneration = 0; - }; - + }; + using TTableResults = TVector<TTableResult>; - + virtual ~IDbSchemeResolver() {} - - // Future-based API. + + // Future-based API. virtual NThreading::TFuture<TTableResults> ResolveTables(const TVector<TTable>& tables) = 0; - - // MessagePassing-based API. - struct TEvents { - enum { + + // MessagePassing-based API. + struct TEvents { + enum { EvResolveTablesResult = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - End - }; - static_assert(End < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect End < EventSpaceEnd(TEvents::ES_PRIVATE)"); + End + }; + static_assert(End < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect End < EventSpaceEnd(TEvents::ES_PRIVATE)"); struct TEvResolveTablesResult : public NActors::TEventLocal<TEvResolveTablesResult, EvResolveTablesResult> { TEvResolveTablesResult(TTableResults&& result); TTableResults Result; }; - }; - + }; + virtual void ResolveTables(const TVector<TTable>& tables, NActors::TActorId responseTo) = 0; // TEvResolveTablesResult. -}; - -} // namespace NYql +}; + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/mkql_compile_service.cpp b/ydb/core/client/minikql_compile/mkql_compile_service.cpp index f25b62842f..e01d61a22c 100644 --- a/ydb/core/client/minikql_compile/mkql_compile_service.cpp +++ b/ydb/core/client/minikql_compile/mkql_compile_service.cpp @@ -1,5 +1,5 @@ -#include "mkql_compile_service.h" - +#include "mkql_compile_service.h" + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/executor_thread.h> #include <library/cpp/actors/core/hfunc.h> @@ -10,32 +10,32 @@ #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> - + #include <util/generic/queue.h> - -namespace NKikimr { - + +namespace NKikimr { + TMiniKQLCompileServiceEvents::TEvCompile::TEvCompile(const TString& program) - : Program(program) + : Program(program) {} - + TMiniKQLCompileServiceEvents::TEvCompileStatus::TEvCompileStatus(const TString& pgm, const NYql::TMiniKQLCompileResult& result) - : Program(pgm) - , Result(result) + : Program(pgm) + , Result(result) {} - -// Actor with queue of programs to compile. Creates TMiniKQLCompileActor for each program. -class TMiniKQLCompileService : public NActors::TActorBootstrapped<TMiniKQLCompileService> { -public: - struct TCompileContext : public TSimpleRefCount<TCompileContext>{ - using TPtr = TIntrusivePtr<TCompileContext>; - + +// Actor with queue of programs to compile. Creates TMiniKQLCompileActor for each program. +class TMiniKQLCompileService : public NActors::TActorBootstrapped<TMiniKQLCompileService> { +public: + struct TCompileContext : public TSimpleRefCount<TCompileContext>{ + using TPtr = TIntrusivePtr<TCompileContext>; + TCompileContext(const TString& pgm, TActorId sender, const TAlignedPagePoolCounters& allocPoolCounters, const NMiniKQL::IFunctionRegistry* functionRegistry) - : Program(pgm) - , ResponseTo(sender) + : Program(pgm) + , ResponseTo(sender) , Alloc(allocPoolCounters, functionRegistry->SupportsSizedAllocators()) , TypeEnv(Alloc) , Cookie(0) @@ -43,7 +43,7 @@ public: { Alloc.Release(); } - + ~TCompileContext() { Alloc.Acquire(); @@ -52,13 +52,13 @@ public: TString Program; TActorId ResponseTo; NMiniKQL::TScopedAlloc Alloc; - NMiniKQL::TTypeEnvironment TypeEnv; + NMiniKQL::TTypeEnvironment TypeEnv; ui64 Cookie; bool Retried; THashMap<TString, ui64> CompileResolveCookies; bool ForceRefresh; - }; - + }; + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::MINIKQL_COMPILE_SERVICE; } @@ -68,8 +68,8 @@ public: , DbSchemeResolver(std::move(dbSchemeResolver)) { } - - void Bootstrap(const TActorContext& ctx) { + + void Bootstrap(const TActorContext& ctx) { Counters = GetServiceCounters(AppData(ctx)->Counters, "compile")->GetSubgroup("subsystem", "cache"); AllocPoolCounters = TAlignedPagePoolCounters(AppData(ctx)->Counters, "compile"); @@ -85,36 +85,36 @@ public: auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), Counters); SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); return NSchCache::CreateDbSchemeResolver(ctx.ExecutorThread.ActorSystem, SchemeCache); - } - - STFUNC(StateWork) { - switch (ev->GetTypeRewrite()) { - HFunc(TMiniKQLCompileServiceEvents::TEvCompile, Handle); - HFunc(NYql::TMiniKQLCompileActorEvents::TEvCompileResult, Handle); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + HFunc(TMiniKQLCompileServiceEvents::TEvCompile, Handle); + HFunc(NYql::TMiniKQLCompileActorEvents::TEvCompileResult, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); - default: + default: Y_FAIL(""); - } - } - -private: - void Handle(TMiniKQLCompileServiceEvents::TEvCompile::TPtr& ev, const TActorContext& ctx) { + } + } + +private: + void Handle(TMiniKQLCompileServiceEvents::TEvCompile::TPtr& ev, const TActorContext& ctx) { TMiniKQLCompileServiceEvents::TEvCompile *msg = ev->Get(); TCompileContext::TPtr c(new TCompileContext(msg->Program, ev->Sender, AllocPoolCounters, AppData(ctx)->FunctionRegistry)); c->Cookie = ev->Cookie; c->CompileResolveCookies = std::move(msg->CompileResolveCookies); c->ForceRefresh = msg->ForceRefresh; - CompileQueue.push(c); - Compile(ctx); - } - - void Handle(NYql::TMiniKQLCompileActorEvents::TEvCompileResult::TPtr& ev, const TActorContext& ctx) { + CompileQueue.push(c); + Compile(ctx); + } + + void Handle(NYql::TMiniKQLCompileActorEvents::TEvCompileResult::TPtr& ev, const TActorContext& ctx) { auto *msg = ev->Get(); - auto it = Compiling.find(ev->Sender); + auto it = Compiling.find(ev->Sender); Y_VERIFY(it != Compiling.end()); TCompileContext::TPtr cptr = it->second; - Compiling.erase(it); + Compiling.erase(it); const bool hasErrors = !msg->Result.Errors.Empty(); if (!hasErrors || cptr->Retried || cptr->ForceRefresh) { @@ -134,13 +134,13 @@ private: const TActorId actId = ctx.ExecutorThread.RegisterActor(compileActor, TMailboxType::HTSwap, appData->UserPoolId); Compiling.insert(TCompilingMap::value_type(actId, c)); } - } - - void Compile(const TActorContext& ctx) { - if (IsCompileLimitReached() || CompileQueue.empty()) { - return; - } - auto next = CompileQueue.front(); + } + + void Compile(const TActorContext& ctx) { + if (IsCompileLimitReached() || CompileQueue.empty()) { + return; + } + auto next = CompileQueue.front(); auto* act = NYql::CreateCompileActor( next->Program, &next->TypeEnv, @@ -150,32 +150,32 @@ private: next->ForceRefresh); auto *appData = AppData(ctx); auto actId = ctx.ExecutorThread.RegisterActor(act, TMailboxType::HTSwap, appData->UserPoolId); - Compiling.insert(TCompilingMap::value_type(actId, next)); - CompileQueue.pop(); - } - - bool IsCompileLimitReached() const { - return Compiling.size() >= COMPILE_INFLIGHT_LIMIT; - } - -private: - const size_t COMPILE_INFLIGHT_LIMIT; + Compiling.insert(TCompilingMap::value_type(actId, next)); + CompileQueue.pop(); + } + + bool IsCompileLimitReached() const { + return Compiling.size() >= COMPILE_INFLIGHT_LIMIT; + } + +private: + const size_t COMPILE_INFLIGHT_LIMIT; TQueue<TCompileContext::TPtr> CompileQueue; using TCompilingMap = THashMap<TActorId, TCompileContext::TPtr>; - TCompilingMap Compiling; - + TCompilingMap Compiling; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; TAlignedPagePoolCounters AllocPoolCounters; TActorId SchemeCache; - THolder<NYql::IDbSchemeResolver> DbSchemeResolver; -}; - - + THolder<NYql::IDbSchemeResolver> DbSchemeResolver; +}; + + TActorId MakeMiniKQLCompileServiceID() { - const char x[12] = "MKQLCompile"; + const char x[12] = "MKQLCompile"; return TActorId(0, TStringBuf(x, 12)); -} - +} + const TActorId& GetMiniKQLCompileServiceID() { static TActorId miniKQLCompileServiceID = MakeMiniKQLCompileServiceID(); return miniKQLCompileServiceID; @@ -184,10 +184,10 @@ const TActorId& GetMiniKQLCompileServiceID() { IActor* CreateMiniKQLCompileService(size_t compileInflightLimit) { THolder<NYql::IDbSchemeResolver> resolver; return new TMiniKQLCompileService(compileInflightLimit, std::move(resolver)); -} - +} + IActor* CreateMiniKQLCompileService(size_t compileInflightLimit, THolder<NYql::IDbSchemeResolver>&& dbSchemeResolver) { return new TMiniKQLCompileService(compileInflightLimit, std::move(dbSchemeResolver)); } -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/client/minikql_compile/mkql_compile_service.h b/ydb/core/client/minikql_compile/mkql_compile_service.h index 2165e8515b..4b0aca1491 100644 --- a/ydb/core/client/minikql_compile/mkql_compile_service.h +++ b/ydb/core/client/minikql_compile/mkql_compile_service.h @@ -1,48 +1,48 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/event_local.h> #include <ydb/core/client/minikql_compile/yql_expr_minikql.h> #include <ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.h> - + #include <util/generic/ptr.h> -namespace NActors { -class IActor; -} // namespace NActors - -namespace NKikimr { - -struct TMiniKQLCompileServiceEvents { - enum { - Compile = EventSpaceBegin(NActors::TEvents::ES_USERSPACE), - CompileStatus, - End - }; - - static_assert(End < EventSpaceEnd(NActors::TEvents::ES_USERSPACE), "expect End < EventSpaceEnd(TEvents::ES_USERSPACE)"); - +namespace NActors { +class IActor; +} // namespace NActors + +namespace NKikimr { + +struct TMiniKQLCompileServiceEvents { + enum { + Compile = EventSpaceBegin(NActors::TEvents::ES_USERSPACE), + CompileStatus, + End + }; + + static_assert(End < EventSpaceEnd(NActors::TEvents::ES_USERSPACE), "expect End < EventSpaceEnd(TEvents::ES_USERSPACE)"); + struct TEvCompile : public NActors::TEventLocal<TEvCompile, Compile> { explicit TEvCompile(const TString& program); TString Program; bool ForceRefresh = false; THashMap<TString, ui64> CompileResolveCookies; - }; - + }; + struct TEvCompileStatus : public NActors::TEventLocal<TEvCompileStatus, CompileStatus> { TEvCompileStatus(const TString& program, const NYql::TMiniKQLCompileResult& result); TString Program; - NYql::TMiniKQLCompileResult Result; + NYql::TMiniKQLCompileResult Result; THashMap<TString, ui64> CompileResolveCookies; - }; -}; - + }; +}; + NActors::TActorId MakeMiniKQLCompileServiceID(); const NActors::TActorId& GetMiniKQLCompileServiceID(); NActors::IActor* CreateMiniKQLCompileService(size_t compileInflightLimit); NActors::IActor* CreateMiniKQLCompileService(size_t compileInflightLimit, THolder<NYql::IDbSchemeResolver>&& dbSchemeResolver); - -} // namespace NKikimr + +} // namespace NKikimr diff --git a/ydb/core/client/minikql_compile/ut/ya.make b/ydb/core/client/minikql_compile/ut/ya.make index 58444c62ff..f9fb7ce498 100644 --- a/ydb/core/client/minikql_compile/ut/ya.make +++ b/ydb/core/client/minikql_compile/ut/ya.make @@ -1,14 +1,14 @@ UNITTEST_FOR(ydb/core/client/minikql_compile) - -OWNER(vvvv) - + +OWNER(vvvv) + ALLOCATOR(J) -SRCS( +SRCS( yql_expr_minikql_compile_ut.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/testing/unittest library/cpp/threading/future ydb/core/client/scheme_cache_lib @@ -17,8 +17,8 @@ PEERDIR( ydb/core/testlib ydb/library/yql/minikql ydb/library/yql/public/udf/service/exception_policy -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/client/minikql_compile/ya.make b/ydb/core/client/minikql_compile/ya.make index af4a075459..a8bd3ca518 100644 --- a/ydb/core/client/minikql_compile/ya.make +++ b/ydb/core/client/minikql_compile/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER(vvvv) - -SRCS( - compile_context.cpp - compile_context.h - compile_result.cpp - compile_result.h - db_key_resolver.cpp - db_key_resolver.h +LIBRARY() + +OWNER(vvvv) + +SRCS( + compile_context.cpp + compile_context.h + compile_result.cpp + compile_result.h + db_key_resolver.cpp + db_key_resolver.h mkql_compile_service.cpp - yql_expr_minikql.cpp - yql_expr_minikql.h -) - -PEERDIR( + yql_expr_minikql.cpp + yql_expr_minikql.h +) + +PEERDIR( library/cpp/actors/core library/cpp/threading/future ydb/core/base @@ -25,8 +25,8 @@ PEERDIR( ydb/library/yql/core ydb/library/yql/minikql ydb/library/yql/providers/common/mkql -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index f4135d8ff0..76f38bf35c 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -1,8 +1,8 @@ -#include "yql_expr_minikql.h" - -#include "compile_context.h" -#include "db_key_resolver.h" - +#include "yql_expr_minikql.h" + +#include "compile_context.h" +#include "db_key_resolver.h" + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/executor_thread.h> #include <library/cpp/actors/core/hfunc.h> @@ -31,37 +31,37 @@ #include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h> #include <library/cpp/threading/future/async.h> - -#include <util/generic/algorithm.h> -#include <util/generic/bt_exception.h> -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/list.h> -#include <util/generic/mapfindptr.h> -#include <util/generic/stack.h> -#include <util/generic/vector.h> -#include <util/string/cast.h> -#include <util/string/hex.h> + +#include <util/generic/algorithm.h> +#include <util/generic/bt_exception.h> +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/list.h> +#include <util/generic/mapfindptr.h> +#include <util/generic/stack.h> +#include <util/generic/vector.h> +#include <util/string/cast.h> +#include <util/string/hex.h> #include <util/string/builder.h> #include <utility> - -namespace NYql { - -using namespace NKikimr; -using namespace NKikimr::NMiniKQL; - -using namespace NThreading; - + +namespace NYql { + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +using namespace NThreading; + using namespace NYql::NCommon; -namespace { - +namespace { + TReadTarget GetReadTarget(TExprNode* node) { Y_ENSURE_EX(node->IsAtom(), TNodeException(node) << "Expected atom"); auto modeStr = node->Content(); if (modeStr == "head") { return TReadTarget::Head(); - } + } else if (modeStr == "online") { return TReadTarget::Online(); } @@ -88,7 +88,7 @@ EInplaceUpdateMode ParseUpdateMode(const TStringBuf& name) { else if (name == "IfNotExistOrEmpty") { mode = EInplaceUpdateMode::IfNotExistOrEmpty; } - + return mode; } @@ -146,11 +146,11 @@ void CollectEraseRowKey(const TExprNode* child, TContext::TPtr ctx) { } ctx->AddTableLookup(request); -} - +} + void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { IDbSchemeResolver::TTable request; - + Y_ENSURE_EX(child->ChildrenSize() == 3, TNodeException(child) << child->Content() << " takes 3 args."); @@ -158,17 +158,17 @@ void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { auto rowTuple = child->Child(1); Y_ENSURE_EX(rowTuple->IsList() && rowTuple->ChildrenSize() > 0, TNodeException(rowTuple) << child->Content() << "Expected non-empty tuple"); - + for (auto& tupleItem : rowTuple->Children()) { Y_ENSURE_EX(tupleItem->IsList() && tupleItem->ChildrenSize() == 2, TNodeException(*tupleItem) << child->Content() << "Expected pair"); - + Y_ENSURE_EX(tupleItem->Child(0)->IsAtom() && !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); } - + auto updateTuple = child->Child(2); Y_ENSURE_EX(updateTuple->IsList(), TNodeException(updateTuple) << child->Content() << "Expected tuple"); @@ -179,11 +179,11 @@ void CollectUpdateRowKey(const TExprNode* child, TContext::TPtr ctx) { !tupleItem->Child(0)->Content().empty(), TNodeException(tupleItem->Child(0)) << "Expected column name as non-empty atom."); request.ColumnNames.insert(TString(tupleItem->Child(0)->Content())); - } - + } + ctx->AddTableLookup(request); -} - +} + void CollectSelectRowKey(const TExprNode* child, TContext::TPtr ctx) { IDbSchemeResolver::TTable request; @@ -466,7 +466,7 @@ private: } else { finishedFrom = true; } - + if (!rangeItem->Child(2)->IsCallable("Void")) { Y_ENSURE_EX(!finishedTo, TNodeException(rangeItem->Child(2)) << "To tuple is already set to infinity"); @@ -475,30 +475,30 @@ private: finishedTo = true; } } - + Y_ENSURE_EX(keyCount <= lookup->KeyColumnCount, TNodeException(node) << "Too many key columns specified, table [" << lookup->Table.TableName << "] has only: " << lookup->KeyColumnCount << ", but got " << keyCount << "."); - + Y_ENSURE_EX(fromComponents > 0, TNodeException(node) << "Expected at least one component of key in the 'from' section of the range"); - + ui32 keyIndex = 0; for (auto rangeItem : rangeTuple->Children()) { if (rangeItem->IsAtom()) { continue; } - + auto columnName = rangeItem->Child(0)->Content(); if (!CheckKeyColumn(columnName, keyIndex, lookup, node, ctx)) { return TStatus::Error; } ++keyIndex; } - + auto selectTuple = node.Child(2); - + // Check that all selected columns are present in table schema ui32 selectIndex = 0; for (auto selectItem : selectTuple->Children()) { @@ -519,7 +519,7 @@ private: for (auto optionsItem : optionsNode->Children()) { Y_ENSURE_EX(optionsItem->IsList() && optionsItem->ChildrenSize() == 2 && optionsItem->Child(0)->IsAtom(), TNodeException(*optionsItem) << "Expected pair of atom and value"); - + auto optionName = optionsItem->Child(0)->Content(); if (optionName != "ItemsLimit" && optionName != "BytesLimit" && @@ -531,25 +531,25 @@ private: ythrow TNodeException(node) << "Unsupported option: " << optionName; } } - + auto selectType = GetSelectType(lookup, *selectTuple, ctx); auto listSelectType = ctx.MakeType<TListExprType>(selectType); - + TVector<const TItemExprType*> resultItems; resultItems.reserve(2); resultItems.push_back(ctx.MakeType<TItemExprType>("List", listSelectType)); auto boolType = ctx.MakeType<TDataExprType>(EDataSlot::Bool); resultItems.push_back(ctx.MakeType<TItemExprType>("Truncated", boolType)); auto resultType = ctx.MakeType<TStructExprType>(resultItems); - + node.SetTypeAnn(resultType); return TStatus::Ok; } - + IGraphTransformer::TStatus UpdateRowWrapper(TExprNode& node, TExprContext& ctx) { const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rowTuple = node.Child(1); - + if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { return TStatus::Error; } @@ -582,11 +582,11 @@ private: node.SetTypeAnn(ctx.MakeType<TVoidExprType>()); return TStatus::Ok; } - + IGraphTransformer::TStatus EraseRowWrapper(TExprNode& node, TExprContext& ctx) { const auto lookup = MkqlCtx->GetTableLookup(node, GetTableName(&node)); auto rowTuple = node.Child(1); - + if (!CheckRowTuple(lookup, node, *rowTuple, ctx)) { return TStatus::Error; } @@ -594,20 +594,20 @@ private: node.SetTypeAnn(ctx.MakeType<TVoidExprType>()); return TStatus::Ok; } - + IGraphTransformer::TStatus SetResultWrapper(TExprNode& node, TExprContext& ctx) { Y_ENSURE_EX(node.ChildrenSize() == 2, TNodeException(node) << "SetResult expects 2 args."); Y_ENSURE_EX(node.Child(0)->Type() == TExprNode::Atom, TNodeException(node) << "First SetResult argument should be Atom."); - + node.SetTypeAnn(ctx.MakeType<TVoidExprType>()); return TStatus::Ok; } - + IGraphTransformer::TStatus IncrementWrapper(TExprNode& node, TExprContext& ctx) { if (!EnsureArgsCount(node, 1, ctx)) { return IGraphTransformer::TStatus::Error; } - + node.SetTypeAnn(node.Child(0)->GetTypeAnn()); return TStatus::Ok; } @@ -622,16 +622,16 @@ private: items.reserve(2); items.push_back(ui64Type); items.push_back(ui64Type); - + auto tupleType = ctx.MakeType<TTupleExprType>(items); - + node.SetTypeAnn(tupleType); return TStatus::Ok; } - + IGraphTransformer::TStatus ParameterWrapper(TExprNode& node, TExprContext& ctx) { Y_UNUSED(ctx); - + Y_ENSURE_EX(node.ChildrenSize() == 2, TNodeException(node) << "Parameter expects 2 args."); if (!EnsureType(*node.Child(1), ctx)) { @@ -642,20 +642,20 @@ private: node.SetTypeAnn(parameterType); return TStatus::Ok; - } - + } + IGraphTransformer::TStatus ParametersWrapper(TExprNode& node, TExprContext& ctx) { Y_ENSURE_EX(node.ChildrenSize() == 0, TNodeException(node) << "Parameters expects 0 args."); - + auto structType = ctx.MakeType<TStructExprType>(TVector<const TItemExprType*>()); - + node.SetTypeAnn(structType); return TStatus::Ok; } - + IGraphTransformer::TStatus AsParametersWrapper(TExprNode& node, TExprContext& ctx) { Y_ENSURE_EX(node.ChildrenSize() > 0, TNodeException(node) << "AsParameters expects > 0 args."); - + // Use AsStruct type annotation auto tmpNode = ctx.RenameNode(node, "AsStruct"); auto output = tmpNode; @@ -664,19 +664,19 @@ private: node.SetTypeAnn(tmpNode->GetTypeAnn()); return TStatus::Ok; } - + IGraphTransformer::TStatus AddParameterWrapper(TExprNode& node, TExprContext& ctx) { Y_ENSURE_EX(node.ChildrenSize() == 3, TNodeException(node) << "AddParameter expects 3 args."); - + // Use AddMember type annotation auto tmpNode = ctx.RenameNode(node, "AddMember"); auto output = tmpNode; YQL_ENSURE(CallableTransformer->Transform(tmpNode, output, ctx) == TStatus::Ok); - + node.SetTypeAnn(tmpNode->GetTypeAnn()); return TStatus::Ok; } - + IGraphTransformer::TStatus MapParameterWrapper(TExprNode& node, TExprContext& ctx) { Y_ENSURE_EX(node.ChildrenSize() == 2, TNodeException(node) << "MapParameter expects 2 args."); Y_ENSURE_EX(node.Child(0)->IsCallable("Parameter"), TNodeException(node.Child(0)) << "Expected Parameter as 1 arg."); @@ -762,7 +762,7 @@ private: IGraphTransformer::TStatus DataConstructorWrapper(TExprNode& node, TExprContext& ctx) { Y_UNUSED(ctx); - + if (node.Content() == "ByteString") { node.SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::String)); } else if (node.Content() == "Utf8String") { @@ -770,10 +770,10 @@ private: } else { node.SetTypeAnn(ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(node.Content()))); } - + return TStatus::Ok; } - + IGraphTransformer::TStatus DataTypeWrapper(TExprNode& node, TExprContext& ctx) { switch (node.ChildrenSize()) { @@ -790,12 +790,12 @@ private: } return TStatus::Ok; } - + private: TContext::TPtr MkqlCtx; TAutoPtr<IGraphTransformer> CallableTransformer; }; - + bool PerformTypeAnnotation(TExprNode::TPtr& exprRoot, TExprContext& ctx, TContext::TPtr mkqlContext) { TTypeAnnotationContext types; types.DeprecatedSQL = true; @@ -803,15 +803,15 @@ bool PerformTypeAnnotation(TExprNode::TPtr& exprRoot, TExprContext& ctx, TContex TAutoPtr<IGraphTransformer> callableTransformer = CreateExtCallableTypeAnnotationTransformer(types); types.TimeProvider = CreateDefaultTimeProvider(); types.RandomProvider = CreateDefaultRandomProvider(); - + TAutoPtr<IGraphTransformer> kikimrTransformer = new TKikimrCallableTypeAnnotationTransformer( mkqlContext, callableTransformer); - + auto typeTransformer = CreateTypeAnnotationTransformer(kikimrTransformer, types); - + return InstantTransform(*typeTransformer, exprRoot, ctx) == IGraphTransformer::TStatus::Ok; } - + TRuntimeNode GetReadTargetNode(const TExprNode& callable, ui32 index, TMkqlBuildContext& ctx, TContext::TPtr mkqlContext) { @@ -891,16 +891,16 @@ static void FillKeyPosition(TVector<bool>& arr, const TExprNode::TPtr& listNode, TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr mkqlContext) { auto compiler = MakeIntrusive<NCommon::TMkqlCommonCallableCompiler>(); - + compiler->AddCallable("SetResult", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { auto label = node.Child(0)->Content(); auto payload = MkqlBuildExpr(*node.Child(1), ctx); TRuntimeNode result = mkqlContext->PgmBuilder->SetResult(label, payload); - + return result; }); - + compiler->AddCallable("SelectRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); @@ -913,7 +913,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (legacy) { lookup->TableId->SchemaVersion = 0; } - + auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); TVector<ui32> keyTypes(rowTuple->ChildrenSize()); @@ -925,17 +925,17 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr row[i] = keyValue; keyTypes[i] = column->Type; } - + TVector<TSelectColumn> columnsToRead; FillColumnsToRead(lookup, node.Child(2), columnsToRead); - + auto readTargetNode = GetReadTargetNode(node, 3, ctx, mkqlContext); - + auto result = mkqlContext->PgmBuilder->SelectRow(*lookup->TableId, keyTypes, columnsToRead, row, readTargetNode); return result; }); - + compiler->AddCallable("SelectRange", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); @@ -972,20 +972,20 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr else { YQL_ENSURE(false, "Unexpected SelectRange range bound"); } - + continue; } - + ++keyCount; if (!rangeItem->Child(1)->IsCallable("Void")) { ++fromComponents; } - + if (!rangeItem->Child(2)->IsCallable("Void")) { ++toComponents; } } - + TTableRangeOptions options = mkqlContext->PgmBuilder->GetDefaultTableRangeOptions(); TVector<ui32> keyTypes(Max(fromComponents, toComponents)); TVector<TRuntimeNode> from(fromComponents); @@ -995,7 +995,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (rangeItem->IsAtom()) { continue; } - + if (keyIndex >= fromComponents && keyIndex >= toComponents) { break; } @@ -1003,17 +1003,17 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto columnName = rangeItem->Child(0)->Content(); auto column = lookup->Columns.FindPtr(columnName); YQL_ENSURE(column); - + if (keyIndex < fromComponents) { auto fromKeyValue = MkqlBuildExpr(*rangeItem->Child(1), ctx); from[keyIndex] = fromKeyValue; } - + if (keyIndex < toComponents) { auto toKeyValue = MkqlBuildExpr(*rangeItem->Child(2), ctx); to[keyIndex] = toKeyValue; } - + keyTypes[keyIndex] = column->Type; ++keyIndex; } @@ -1023,7 +1023,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr options.Flags = ctx.ProgramBuilder.NewDataLiteral(flags); options.FromColumns = from; options.ToColumns = to; - + TVector<bool> skipNullKeys(lookup->KeyColumnCount, false); options.SkipNullKeys = skipNullKeys; TVector<bool> forbidNullArgsFrom(lookup->KeyColumnCount, false); @@ -1033,7 +1033,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr TVector<TSelectColumn> columnsToRead; FillColumnsToRead(lookup, node.Child(2), columnsToRead); - + auto optionsNode = node.Child(3); for (auto optionsItem : optionsNode->Children()) { if (optionsItem->Child(0)->Content() == "ItemsLimit") { @@ -1052,14 +1052,14 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr YQL_ENSURE(false, "Unexpected SelectRange option."); } } - + auto readTargetNode = GetReadTargetNode(node, 4, ctx, mkqlContext); - + TRuntimeNode result = mkqlContext->PgmBuilder->SelectRange(*lookup->TableId, keyTypes, columnsToRead, options, readTargetNode); return result; }); - + compiler->AddCallable("UpdateRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); @@ -1072,7 +1072,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (legacy) { lookup->TableId->SchemaVersion = 0; } - + auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); TVector<ui32> keyTypes(rowTuple->ChildrenSize()); @@ -1084,7 +1084,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr row[i] = keyValue; keyTypes[i] = column->Type; } - + auto update = mkqlContext->PgmBuilder->GetUpdateRowBuilder(); auto updateTuple = node.Child(2); for (ui32 i = 0; i < updateTuple->ChildrenSize(); ++i) { @@ -1092,7 +1092,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto columnName = child->Child(0)->Content(); auto column = lookup->Columns.FindPtr(columnName); YQL_ENSURE(column); - + if (child->ChildrenSize() == 1) { update.EraseColumn(column->Column); } @@ -1106,11 +1106,11 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr update.InplaceUpdateColumn(column->Column, column->Type, mergeValue, mode); } } - + TRuntimeNode result = mkqlContext->PgmBuilder->UpdateRow(*lookup->TableId, keyTypes, row, update); return result; }); - + compiler->AddCallable("EraseRow", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { YQL_ENSURE(node.Child(0)->IsList() || HasUnversionedTable(node), "expected list or atom as table"); @@ -1123,7 +1123,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (legacy) { lookup->TableId->SchemaVersion = 0; } - + auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); TVector<ui32> keyTypes(rowTuple->ChildrenSize()); @@ -1135,37 +1135,37 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr row[i] = keyValue; keyTypes[i] = column->Type; } - + TRuntimeNode result = mkqlContext->PgmBuilder->EraseRow(*lookup->TableId, keyTypes, row); return result; }); - + compiler->AddCallable("Increment", [](const TExprNode& node, TMkqlBuildContext& ctx) { auto arg = MkqlBuildExpr(*node.Child(0), ctx); - + return ctx.ProgramBuilder.Increment(arg); }); - + compiler->AddCallable("StepTxId", [mkqlContext](const TExprNode&, TMkqlBuildContext&) { return mkqlContext->PgmBuilder->StepTxId(); }); - + compiler->OverrideCallable("Parameter", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { const auto& name = node.Child(0)->Content(); const auto type = BuildType(*node.Child(1), *node.Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); return mkqlContext->NewParam(name, type); }); - + compiler->AddCallable("Parameters", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { Y_UNUSED(ctx); - + if (!mkqlContext->NewParamsBuilder()) { throw TNodeException(node) << "You can create only one Parameters object."; } - + return TRuntimeNode(); }); - + compiler->AddCallable("AsParameters", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { if (!mkqlContext->NewParamsBuilder()) { throw TNodeException(node) << "You can create only one Parameters object."; @@ -1175,25 +1175,25 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto value = MkqlBuildExpr(*child->Child(1), ctx); mkqlContext->ParamsBuilder->Add(name, value); } - + return mkqlContext->ParamsBuilder->Build(); }); - + compiler->AddCallable("AddParameter", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { if (!mkqlContext->NewParamsBuilder()) { throw TNodeException(node) << "You can create only one Parameters object."; } - + for (auto curNode = &node; curNode->Content() != "Parameters"; curNode = curNode->Child(0)) { Y_ENSURE_EX(curNode->Content() == "AddParameter", TNodeException(*curNode) << "Only AddParameter func can be in AddParameter func."); const auto& name = curNode->Child(1)->Content(); auto value = MkqlBuildExpr(*curNode->Child(2), ctx); mkqlContext->ParamsBuilder->Add(name, value); } - + return mkqlContext->ParamsBuilder->Build(); }); - + compiler->AddCallable("MapParameter", [mkqlContext](const TExprNode& node, TMkqlBuildContext& ctx) { auto list = MkqlBuildExpr(*node.Child(0), ctx); @@ -1267,7 +1267,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr if (!slot) { throw TNodeException(node) << "Unknown type '" << typeName << "'."; } - + NUdf::TDataTypeId typeId = NKikimr::NUdf::GetDataTypeInfo(*slot).TypeId; if (typeId == NUdf::TDataType<NUdf::TDecimal>::Id) { auto precision = node.Child(1)->Content(); @@ -1279,10 +1279,10 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr return TRuntimeNode(type, true); } }); - + return compiler; } - + TRuntimeNode CompileNode(const TExprNode& node, TExprContext& exprCtx, TContext::TPtr ctx, const IMkqlCallableCompiler* compiler) { const auto guard = ctx->TypeEnv->BindAllocator(); TMkqlBuildContext mkqlCtx(*compiler, *ctx->PgmBuilder, exprCtx); @@ -1291,7 +1291,7 @@ TRuntimeNode CompileNode(const TExprNode& node, TExprContext& exprCtx, TContext: } // anonymous namespace -void CollectKeys(const TExprNode* root, TContext::TPtr ctx) { +void CollectKeys(const TExprNode* root, TContext::TPtr ctx) { TStack<const TExprNode*> activeNodes; activeNodes.push(root); THashSet<const TExprNode*> visited; @@ -1302,7 +1302,7 @@ void CollectKeys(const TExprNode* root, TContext::TPtr ctx) { { if (current->IsCallable("EraseRow")) { CollectEraseRowKey(current, ctx); - } + } else if (current->IsCallable("UpdateRow")) { CollectUpdateRowKey(current, ctx); } @@ -1312,29 +1312,29 @@ void CollectKeys(const TExprNode* root, TContext::TPtr ctx) { else if (current->IsCallable("SelectRange")) { CollectSelectRangeKey(current, ctx); } - } + } for (auto& child : current->Children()) { if (visited.insert(child.Get()).second) { activeNodes.push(child.Get()); } } - } -} - -TFuture<TConvertResult> -ConvertToMiniKQL(TExprContainer::TPtr expr, + } +} + +TFuture<TConvertResult> +ConvertToMiniKQL(TExprContainer::TPtr expr, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, - IDbSchemeResolver* dbSchemeResolver) -{ - if (expr->Root == nullptr) { - return MakeFuture<TConvertResult>(TConvertResult()); - } - - TPromise<TConvertResult> promise = NewPromise<TConvertResult>(); - - try { + const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, + IDbSchemeResolver* dbSchemeResolver) +{ + if (expr->Root == nullptr) { + return MakeFuture<TConvertResult>(TConvertResult()); + } + + TPromise<TConvertResult> promise = NewPromise<TConvertResult>(); + + try { TContext::TPtr ctx(new TContext(functionRegistry, typeEnv)); auto compiler = CreateMkqlCompiler(ctx); CollectKeys(expr->Root.Get(), ctx); @@ -1399,56 +1399,56 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, TConvertResult convRes = ctx->Finish(convertedNode); promise.SetValue(convRes); } - } catch (const TNodeException& ex) { + } catch (const TNodeException& ex) { // TODO: pass backtrace - TConvertResult convRes; + TConvertResult convRes; convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); - promise.SetValue(convRes); - } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. + promise.SetValue(convRes); + } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. // TODO: pass backtrace - TConvertResult convRes; + TConvertResult convRes; convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); - promise.SetValue(convRes); - } - - return promise.GetFuture(); -} - - + promise.SetValue(convRes); + } + + return promise.GetFuture(); +} + + TMiniKQLCompileActorEvents::TEvCompileResult::TEvCompileResult(const TMiniKQLCompileResult& result, THashMap<TString, ui64> &&resolveCookies) - : Result(result) + : Result(result) , CompileResolveCookies(std::move(resolveCookies)) {} - -class TMiniKQLCompileActor : public TActorBootstrapped<TMiniKQLCompileActor> { -public: + +class TMiniKQLCompileActor : public TActorBootstrapped<TMiniKQLCompileActor> { +public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::MINIKQL_COMPILE_ACTOR; } TMiniKQLCompileActor(const TString& program, - const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, - IDbSchemeResolver* dbSchemeResolver, + const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, + IDbSchemeResolver* dbSchemeResolver, TActorId responseTo, THashMap<TString, ui64> &&resolveRefreshCookies, bool forceCacheRefresh) - : TypeEnv(typeEnv) - , Program(program) - , DbSchemeResolver(dbSchemeResolver) - , ResponseTo(responseTo) + : TypeEnv(typeEnv) + , Program(program) + , DbSchemeResolver(dbSchemeResolver) + , ResponseTo(responseTo) , ResolveRefreshCookies(std::move(resolveRefreshCookies)) { Y_UNUSED(forceCacheRefresh); } - - void Bootstrap(const TActorContext& ctx) { - auto* appData = AppData(ctx); + + void Bootstrap(const TActorContext& ctx) { + auto* appData = AppData(ctx); CompileCtx.Reset(new TContext(appData->FunctionRegistry, TypeEnv)); - try { - TMiniKQLCompileResult result; - if (!ParseProgram(result.Errors)) { + try { + TMiniKQLCompileResult result; + if (!ParseProgram(result.Errors)) { return SendResponseAndDie(result, {}, ctx); - } + } CollectKeys(Expr->Root.Get(), CompileCtx); @@ -1463,35 +1463,35 @@ public: DbSchemeResolver->ResolveTables(requests, ctx.SelfID); Become(&TThis::StateCompileProgram); - } else { + } else { if (!PerformTypeAnnotation(Expr->Root, Expr->Context, CompileCtx)) { result.Errors.AddIssues(Expr->Context.IssueManager.GetIssues()); return SendResponseAndDie(result, {}, ctx); } - result.CompiledProgram = CompileProgram(); + result.CompiledProgram = CompileProgram(); return SendResponseAndDie(result, {}, ctx); - } - } catch (const TNodeException& ex) { - // TODO: pass backtrace + } + } catch (const TNodeException& ex) { + // TODO: pass backtrace TMiniKQLCompileResult res(NYql::TIssue(Expr->Context.GetPosition(ex.Pos()), ex.what())); return SendResponseAndDie(res, {}, ctx); - } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. - // TODO: pass backtrace + } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. + // TODO: pass backtrace TMiniKQLCompileResult res(NYql::ExceptionToIssue(ex)); return SendResponseAndDie(res, {}, ctx); - } - } - - STFUNC(StateCompileProgram) { - switch (ev->GetTypeRewrite()) { + } + } + + STFUNC(StateCompileProgram) { + switch (ev->GetTypeRewrite()) { HFunc(IDbSchemeResolver::TEvents::TEvResolveTablesResult, Handle) - default: + default: Y_FAIL("Unknown event"); - } - } - -private: + } + } + +private: void Handle(IDbSchemeResolver::TEvents::TEvResolveTablesResult::TPtr& ev, const TActorContext& ctx) { THashMap<TString, ui64> compileResolveCookies; @@ -1560,19 +1560,19 @@ private: } bool ParseProgram(TIssues& errors) { - Expr.Reset(new NYql::TExprContainer()); - NYql::TAstParseResult astRes = NYql::ParseAst(Program); + Expr.Reset(new NYql::TExprContainer()); + NYql::TAstParseResult astRes = NYql::ParseAst(Program); auto root = astRes.Root; astRes.Root = nullptr; // don't cleanup library nodes if (!root) { errors = astRes.Issues; - return false; - } + return false; + } Expr->Context.IssueManager.AddIssues(std::move(astRes.Issues)); if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr)) { errors = Expr->Context.IssueManager.GetIssues(); - return false; - } + return false; + } IGraphTransformer::TStatus status(IGraphTransformer::TStatus::Ok); do { status = ExpandApply(Expr->Root, Expr->Root, Expr->Context); @@ -1583,15 +1583,15 @@ private: errors = Expr->Context.IssueManager.GetIssues(); return false; } - return true; - } - + return true; + } + TString CompileProgram() { TRuntimeNode convertedNode = CompileNode(*Expr->Root, Expr->Context, CompileCtx, Compiler.Get()); - TConvertResult convRes = CompileCtx->Finish(convertedNode); - return NMiniKQL::SerializeRuntimeNode(convRes.Node, CompileCtx->PgmBuilder->GetTypeEnvironment()); - } - + TConvertResult convRes = CompileCtx->Finish(convertedNode); + return NMiniKQL::SerializeRuntimeNode(convRes.Node, CompileCtx->PgmBuilder->GetTypeEnvironment()); + } + void SendResponseAndDie(const TMiniKQLCompileResult& result, THashMap<TString, ui64> &&resolveCookies, const TActorContext& ctx) { ctx.ExecutorThread.Send( new IEventHandle( @@ -1599,35 +1599,35 @@ private: ctx.SelfID, new TMiniKQLCompileActorEvents::TEvCompileResult(result, std::move(resolveCookies)) )); - Die(ctx); - } - -private: + Die(ctx); + } + +private: void Cleanup() { CompileCtx.Drop(); Expr.Drop(); Compiler.Drop(); } - const NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv; + const NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv; TString Program; - TContext::TPtr CompileCtx; - IDbSchemeResolver* DbSchemeResolver; + TContext::TPtr CompileCtx; + IDbSchemeResolver* DbSchemeResolver; TActorId ResponseTo; - TExprContainer::TPtr Expr; + TExprContainer::TPtr Expr; TIntrusivePtr<NCommon::IMkqlCallableCompiler> Compiler; THashMap<TString, ui64> ResolveRefreshCookies; -}; - -NActors::IActor* +}; + +NActors::IActor* CreateCompileActor(const TString& program, - const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, - IDbSchemeResolver* dbSchemeResolver, + const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, + IDbSchemeResolver* dbSchemeResolver, TActorId responseTo, THashMap<TString, ui64> &&resolveRefreshCookies, bool forceCacheRefresh) -{ +{ return new TMiniKQLCompileActor(program, typeEnv, dbSchemeResolver, responseTo, std::move(resolveRefreshCookies), forceCacheRefresh); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.h b/ydb/core/client/minikql_compile/yql_expr_minikql.h index 7b66cf2360..ab6011c105 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.h +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.h @@ -1,63 +1,63 @@ -#pragma once - -#include "compile_result.h" +#pragma once + +#include "compile_result.h" #include "compile_context.h" -#include "db_key_resolver.h" - +#include "db_key_resolver.h" + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/events.h> #include <ydb/library/yql/ast/yql_expr.h> - + #include <library/cpp/threading/future/future.h> - -#include <util/generic/bt_exception.h> - - -namespace NKikimr { - -namespace NMiniKQL { -class TProgramBuilder; + +#include <util/generic/bt_exception.h> + + +namespace NKikimr { + +namespace NMiniKQL { +class TProgramBuilder; class IFunctionRegistry; -} // namespace NMiniKQL - -} // namespace NKikimr - -namespace NYql { - -struct TExprContainer : public TAtomicRefCount<TExprContainer> { - using TPtr = TIntrusivePtr<TExprContainer>; - - TExprContext Context; +} // namespace NMiniKQL + +} // namespace NKikimr + +namespace NYql { + +struct TExprContainer : public TAtomicRefCount<TExprContainer> { + using TPtr = TIntrusivePtr<TExprContainer>; + + TExprContext Context; TExprNode::TPtr Root; -}; - -NThreading::TFuture<TConvertResult> -ConvertToMiniKQL(TExprContainer::TPtr expr, +}; + +NThreading::TFuture<TConvertResult> +ConvertToMiniKQL(TExprContainer::TPtr expr, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, - IDbSchemeResolver* dbSchemeResolver); - -struct TMiniKQLCompileActorEvents { - enum { - CompileResult = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - End - }; - static_assert(End < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect End < EventSpaceEnd(TEvents::ES_PRIVATE)"); - struct TEvCompileResult : public NActors::TEventBase<TEvCompileResult, CompileResult> { + const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, + IDbSchemeResolver* dbSchemeResolver); + +struct TMiniKQLCompileActorEvents { + enum { + CompileResult = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + End + }; + static_assert(End < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect End < EventSpaceEnd(TEvents::ES_PRIVATE)"); + struct TEvCompileResult : public NActors::TEventBase<TEvCompileResult, CompileResult> { explicit TEvCompileResult(const TMiniKQLCompileResult& result, THashMap<TString, ui64> &&resolveCookies); - DEFINE_SIMPLE_LOCAL_EVENT(TEvCompileResult, "EvCompileResult"); + DEFINE_SIMPLE_LOCAL_EVENT(TEvCompileResult, "EvCompileResult"); - TMiniKQLCompileResult Result; + TMiniKQLCompileResult Result; THashMap<TString, ui64> CompileResolveCookies; - }; -}; - -NActors::IActor* + }; +}; + +NActors::IActor* CreateCompileActor(const TString& program, - const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, - IDbSchemeResolver* dbSchemeResolver, + const NKikimr::NMiniKQL::TTypeEnvironment* typeEnv, + IDbSchemeResolver* dbSchemeResolver, NActors::TActorId responseTo, THashMap<TString, ui64> &&resolveRefreshCookies, bool forceCacheRefresh); // TEvCompileResult. - -} // namespace NYql + +} // namespace NYql diff --git a/ydb/core/client/minikql_result_lib/converter.cpp b/ydb/core/client/minikql_result_lib/converter.cpp index c9634f75a0..bfb1ea6db3 100644 --- a/ydb/core/client/minikql_result_lib/converter.cpp +++ b/ydb/core/client/minikql_result_lib/converter.cpp @@ -1,14 +1,14 @@ -#include "converter.h" - +#include "converter.h" + #include <ydb/library/mkql_proto/protos/minikql.pb.h> - - -namespace NKikimr { + + +namespace NKikimr { namespace NResultLib { - -TStruct ConvertResult(const NKikimrMiniKQL::TValue& value, const NKikimrMiniKQL::TType& type) { - return TStruct(value, type); -} - + +TStruct ConvertResult(const NKikimrMiniKQL::TValue& value, const NKikimrMiniKQL::TType& type) { + return TStruct(value, type); +} + } // namespace NResultLib -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/converter.h b/ydb/core/client/minikql_result_lib/converter.h index 156751fe5b..108b098cfe 100644 --- a/ydb/core/client/minikql_result_lib/converter.h +++ b/ydb/core/client/minikql_result_lib/converter.h @@ -1,21 +1,21 @@ -#pragma once - -#include "objects.h" - +#pragma once + +#include "objects.h" + #include <ydb/public/lib/deprecated/kicli/kicli.h> - - -namespace NKikimrMiniKQL { -class TValue; -class TType; -} // namespace NKikimrMiniKQL - - -namespace NKikimr { + + +namespace NKikimrMiniKQL { +class TValue; +class TType; +} // namespace NKikimrMiniKQL + + +namespace NKikimr { namespace NResultLib { - -TStruct ConvertResult(const NKikimrMiniKQL::TValue& value, const NKikimrMiniKQL::TType& type); - + +TStruct ConvertResult(const NKikimrMiniKQL::TValue& value, const NKikimrMiniKQL::TType& type); + // convert C++ API result inline TStruct ConvertResult(const NKikimr::NClient::TQueryResult& apiResult) { const NKikimrClient::TResponse& response = apiResult.GetResult<NKikimrClient::TResponse>(); @@ -26,4 +26,4 @@ inline TStruct ConvertResult(const NKikimr::NClient::TQueryResult& apiResult) { } } // namespace NResultLib -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/converter_ut.cpp b/ydb/core/client/minikql_result_lib/converter_ut.cpp index 5f1638c199..0fb5eff621 100644 --- a/ydb/core/client/minikql_result_lib/converter_ut.cpp +++ b/ydb/core/client/minikql_result_lib/converter_ut.cpp @@ -1,17 +1,17 @@ -#include "converter.h" - +#include "converter.h" + #include <ydb/core/testlib/test_client.h> - + #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> - - -namespace NKikimr { - + + +namespace NKikimr { + using namespace NResultLib; Y_UNIT_TEST_SUITE(TMiniKQLResultConverterTest) { - + Y_UNIT_TEST(TTestWithSimpleProgram) { TPortManager pm; ui16 port = pm.GetPort(2134); @@ -19,52 +19,52 @@ Y_UNIT_TEST(TTestWithSimpleProgram) { Tests::TServer server = Tests::TServer(settings); Tests::TClient client(settings); client.InitRootScheme(); - + const TString pgmText = R"___( -( -(let list (AsList (Uint32 '20) (Uint32 '10) (Uint32 '0))) +( +(let list (AsList (Uint32 '20) (Uint32 '10) (Uint32 '0))) (let opt (Just (String 'i_am_opt))) (let tuple '((Double '12) (Float '22))) -(return (AsList - (SetResult 'list list) - (SetResult 'opt opt) +(return (AsList + (SetResult 'list list) + (SetResult 'opt opt) (SetResult 'emptyOpt (Nothing (OptionalType (DataType 'Int32)))) - (SetResult 'tuple tuple) -)) -) -)___"; - - NKikimrMiniKQL::TResult result; + (SetResult 'tuple tuple) +)) +) +)___"; + + NKikimrMiniKQL::TResult result; UNIT_ASSERT(client.FlatQuery(pgmText, result)); - - TStruct s = ConvertResult(result.GetValue(), result.GetType()); - { + + TStruct s = ConvertResult(result.GetValue(), result.GetType()); + { TListType l = s.GetMember<TOptional>("list").GetItem<TListType>(); TVector<ui32> v; - for (ui32 item : l.MakeIterable<ui32>()) { - v.push_back(item); - } - UNIT_ASSERT_EQUAL(v.size(), 3); - UNIT_ASSERT_EQUAL(v[0], 20); - UNIT_ASSERT_EQUAL(v[1], 10); - UNIT_ASSERT_EQUAL(v[2], 0); - } - { - TOptional opt = s.GetMember<TOptional>("opt").GetItem<TOptional>(); - UNIT_ASSERT_EQUAL(opt.GetItem<TStringBuf>(), "i_am_opt"); - } - { - TOptional opt = s.GetMember<TOptional>("emptyOpt").GetItem<TOptional>(); - UNIT_ASSERT(!opt.HasItem()); - } - { - TTuple tuple = s.GetMember<TOptional>("tuple").GetItem<TTuple>(); - UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement<double>(0), 12.0, 1e-3); - UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement<float>(1), 22.0, 1e-3); - } -} - -} - - -} // namespace NKikimr + for (ui32 item : l.MakeIterable<ui32>()) { + v.push_back(item); + } + UNIT_ASSERT_EQUAL(v.size(), 3); + UNIT_ASSERT_EQUAL(v[0], 20); + UNIT_ASSERT_EQUAL(v[1], 10); + UNIT_ASSERT_EQUAL(v[2], 0); + } + { + TOptional opt = s.GetMember<TOptional>("opt").GetItem<TOptional>(); + UNIT_ASSERT_EQUAL(opt.GetItem<TStringBuf>(), "i_am_opt"); + } + { + TOptional opt = s.GetMember<TOptional>("emptyOpt").GetItem<TOptional>(); + UNIT_ASSERT(!opt.HasItem()); + } + { + TTuple tuple = s.GetMember<TOptional>("tuple").GetItem<TTuple>(); + UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement<double>(0), 12.0, 1e-3); + UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement<float>(1), 22.0, 1e-3); + } +} + +} + + +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/data_funcs.inl b/ydb/core/client/minikql_result_lib/data_funcs.inl index 52ba6da5b9..a152c054a1 100644 --- a/ydb/core/client/minikql_result_lib/data_funcs.inl +++ b/ydb/core/client/minikql_result_lib/data_funcs.inl @@ -1,120 +1,120 @@ -#pragma once - - -#define ENSURE_SCHEME_TYPE(expected, gotName) \ - do { \ +#pragma once + + +#define ENSURE_SCHEME_TYPE(expected, gotName) \ + do { \ Y_ENSURE(expected == NScheme::NTypeIds::gotName, "Data scheme type mismatch: expected " << expected << ", but got " #gotName << "."); \ - } while (0); - - -template <> + } while (0); + + +template <> inline bool HasData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Bool); - return value.HasBool(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Bool); + return value.HasBool(); +} + +template <> inline bool HasData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Int32); - return value.HasInt32(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Int32); + return value.HasInt32(); +} + +template <> inline bool HasData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Uint32); - return value.HasUint32(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Uint32); + return value.HasUint32(); +} + +template <> inline bool HasData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Int64); - return value.HasInt64(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Int64); + return value.HasInt64(); +} + +template <> inline bool HasData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Uint64); - return value.HasUint64(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Uint64); + return value.HasUint64(); +} + +template <> inline bool HasData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Float); - return value.HasFloat(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Float); + return value.HasFloat(); +} + +template <> inline bool HasData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_SCHEME_TYPE(schemeType, Double); - return value.HasDouble(); -} - -template <> + ENSURE_SCHEME_TYPE(schemeType, Double); + return value.HasDouble(); +} + +template <> inline bool HasData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { if (schemeType == NScheme::NTypeIds::Utf8) { - return value.HasText(); - } else { - return value.HasBytes(); - } -} - -#undef ENSURE_SCHEME_TYPE - -#define ENSURE_HAS_DATA(type, value, schemeType) \ - do { \ + return value.HasText(); + } else { + return value.HasBytes(); + } +} + +#undef ENSURE_SCHEME_TYPE + +#define ENSURE_HAS_DATA(type, value, schemeType) \ + do { \ Y_ENSURE(HasData<type>(value, schemeType), "No data of type " #type "."); \ - } while (0); - -template <> + } while (0); + +template <> inline bool GetData<bool>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(bool, value, schemeType); - return value.GetBool(); -} - -template <> + ENSURE_HAS_DATA(bool, value, schemeType); + return value.GetBool(); +} + +template <> inline i32 GetData<i32>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(i32, value, schemeType); - return value.GetInt32(); -} - -template <> + ENSURE_HAS_DATA(i32, value, schemeType); + return value.GetInt32(); +} + +template <> inline ui32 GetData<ui32>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(ui32, value, schemeType); - return value.GetUint32(); -} - -template <> + ENSURE_HAS_DATA(ui32, value, schemeType); + return value.GetUint32(); +} + +template <> inline i64 GetData<i64>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(i64, value, schemeType); - return value.GetInt64(); -} - -template <> + ENSURE_HAS_DATA(i64, value, schemeType); + return value.GetInt64(); +} + +template <> inline ui64 GetData<ui64>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(ui64, value, schemeType); - return value.GetUint64(); -} - -template <> + ENSURE_HAS_DATA(ui64, value, schemeType); + return value.GetUint64(); +} + +template <> inline float GetData<float>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(float, value, schemeType); - return value.GetFloat(); -} - -template <> + ENSURE_HAS_DATA(float, value, schemeType); + return value.GetFloat(); +} + +template <> inline double GetData<double>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(double, value, schemeType); - return value.GetDouble(); -} - -template <> + ENSURE_HAS_DATA(double, value, schemeType); + return value.GetDouble(); +} + +template <> inline TStringBuf GetData<TStringBuf>(const TProtoValue& value, NScheme::TTypeId schemeType) { - ENSURE_HAS_DATA(TStringBuf, value, schemeType); + ENSURE_HAS_DATA(TStringBuf, value, schemeType); if (schemeType == NScheme::NTypeIds::Utf8) { - return value.GetText(); - } else { - return value.GetBytes(); - } -} - -#undef ENSURE_HAS_DATA + return value.GetText(); + } else { + return value.GetBytes(); + } +} + +#undef ENSURE_HAS_DATA diff --git a/ydb/core/client/minikql_result_lib/list_funcs.inl b/ydb/core/client/minikql_result_lib/list_funcs.inl index 46f49aa40f..6fb2d46ab8 100644 --- a/ydb/core/client/minikql_result_lib/list_funcs.inl +++ b/ydb/core/client/minikql_result_lib/list_funcs.inl @@ -1,81 +1,81 @@ -#pragma once - - -template <typename T> +#pragma once + + +template <typename T> inline TListType::iterator<T>::iterator(google::protobuf::RepeatedPtrField<TProtoValue>::const_iterator item, const TProtoType& itemType) - : Item(item) - , ItemType(itemType) -{ -} - -template <typename T> + : Item(item) + , ItemType(itemType) +{ +} + +template <typename T> inline TListType::iterator<T>::iterator(const iterator& it) - : Item(it.Item) - , ItemType(it.ItemType) -{ -} - -template <typename T> + : Item(it.Item) + , ItemType(it.ItemType) +{ +} + +template <typename T> inline TListType::iterator<T>& TListType::iterator<T>::operator++() { - ++Item; - return *this; -} - -template <typename T> + ++Item; + return *this; +} + +template <typename T> inline TListType::iterator<T> TListType::iterator<T>::operator++(int) { - iterator it(*this); - ++(*this); - return it; -} - -template <typename T> + iterator it(*this); + ++(*this); + return it; +} + +template <typename T> inline bool TListType::iterator<T>::operator!=(const iterator& it) const { - return Item != it.Item; -} - -template <typename T> + return Item != it.Item; +} + +template <typename T> inline bool TListType::iterator<T>::operator==(const iterator& it) const { - return Item == it.Item; -} - -template <typename T> + return Item == it.Item; +} + +template <typename T> inline T TListType::iterator<T>::operator*() const { - return Get(); -} - -template <typename T> + return Get(); +} + +template <typename T> inline T TListType::iterator<T>::Get() const { - ENSURE_KIND(ItemType, Data); + ENSURE_KIND(ItemType, Data); auto schemeType = ItemType.GetData().GetScheme(); - return NPrivate::GetData<T>(*Item, schemeType); -} - -template <> + return NPrivate::GetData<T>(*Item, schemeType); +} + +template <> inline void TListType::iterator<void>::Get() const { - return; -} - -template <> + return; +} + +template <> inline TOptional TListType::iterator<TOptional>::Get() const { - return TOptional(*Item, ItemType); -} - -template <> + return TOptional(*Item, ItemType); +} + +template <> inline TListType TListType::iterator<TListType>::Get() const { return TListType(*Item, ItemType); -} - -template <> +} + +template <> inline TTuple TListType::iterator<TTuple>::Get() const { - return TTuple(*Item, ItemType); -} - -template <> + return TTuple(*Item, ItemType); +} + +template <> inline TStruct TListType::iterator<TStruct>::Get() const { - return TStruct(*Item, ItemType); -} - -template <> + return TStruct(*Item, ItemType); +} + +template <> inline TDict TListType::iterator<TDict>::Get() const { - return TDict(*Item, ItemType); -} + return TDict(*Item, ItemType); +} diff --git a/ydb/core/client/minikql_result_lib/objects.cpp b/ydb/core/client/minikql_result_lib/objects.cpp index 374c8591f0..eaaa2e4551 100644 --- a/ydb/core/client/minikql_result_lib/objects.cpp +++ b/ydb/core/client/minikql_result_lib/objects.cpp @@ -1,6 +1,6 @@ -#include "objects.h" - - -namespace NKikimr { - -} // namespace NKikimr +#include "objects.h" + + +namespace NKikimr { + +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/objects.h b/ydb/core/client/minikql_result_lib/objects.h index 17066c20c5..4b5ae0da45 100644 --- a/ydb/core/client/minikql_result_lib/objects.h +++ b/ydb/core/client/minikql_result_lib/objects.h @@ -1,318 +1,318 @@ -#pragma once - +#pragma once + #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/core/scheme/scheme_type_id.h> - + #include <google/protobuf/repeated_field.h> - -#include <util/generic/hash.h> -#include <util/generic/yexception.h> - - -namespace NKikimr { + +#include <util/generic/hash.h> +#include <util/generic/yexception.h> + + +namespace NKikimr { namespace NResultLib { - -using TProtoValue = NKikimrMiniKQL::TValue; -using TProtoType = NKikimrMiniKQL::TType; -using EProtoTypeKind = NKikimrMiniKQL::ETypeKind; - -class TOptional; + +using TProtoValue = NKikimrMiniKQL::TValue; +using TProtoType = NKikimrMiniKQL::TType; +using EProtoTypeKind = NKikimrMiniKQL::ETypeKind; + +class TOptional; class TListType; -class TTuple; -class TStruct; -class TDict; - - -namespace NPrivate { - -template <typename T> +class TTuple; +class TStruct; +class TDict; + + +namespace NPrivate { + +template <typename T> inline bool HasData(const TProtoValue& value, NScheme::TTypeId schemeType) { Y_UNUSED(value); Y_UNUSED(schemeType); Y_FAIL("Not scpecified type."); -} - -template <typename T> +} + +template <typename T> inline T GetData(const TProtoValue& value, NScheme::TTypeId schemeType) { Y_UNUSED(value); Y_UNUSED(schemeType); Y_FAIL("Not scpecified type."); -} - -#include "data_funcs.inl" - -} // namespace NPrivate - - -#define ENSURE_KIND(type, expectedKind) \ - do { \ - auto kind = (EProtoTypeKind) type.GetKind(); \ +} + +#include "data_funcs.inl" + +} // namespace NPrivate + + +#define ENSURE_KIND(type, expectedKind) \ + do { \ + auto kind = (EProtoTypeKind) type.GetKind(); \ Y_ENSURE(kind == EProtoTypeKind::expectedKind, \ - "Expected " #expectedKind " instead of " << NKikimrMiniKQL::ETypeKind_Name(kind)); \ + "Expected " #expectedKind " instead of " << NKikimrMiniKQL::ETypeKind_Name(kind)); \ Y_ENSURE(type.Has ## expectedKind(), "No " #expectedKind " in type, seems like an error."); \ - } while (false); - - -class TOptional { -public: - explicit TOptional(const TProtoValue& value, const TProtoType& type) - : RootValue(value) - , RootType(type) - { - ENSURE_KIND(RootType, Optional); - } - - template <typename T> - T GetItem() const; - - bool HasItem() const { - return RootValue.HasOptional(); - } - -private: - const TProtoValue& RootValue; - const TProtoType& RootType; -}; - - + } while (false); + + +class TOptional { +public: + explicit TOptional(const TProtoValue& value, const TProtoType& type) + : RootValue(value) + , RootType(type) + { + ENSURE_KIND(RootType, Optional); + } + + template <typename T> + T GetItem() const; + + bool HasItem() const { + return RootValue.HasOptional(); + } + +private: + const TProtoValue& RootValue; + const TProtoType& RootType; +}; + + class TListType { -public: - template <typename T> - class iterator { - public: - iterator(google::protobuf::RepeatedPtrField<TProtoValue>::const_iterator item, const TProtoType& itemType); - iterator(const iterator& it); - - iterator& operator++(); - iterator operator++(int); - bool operator!=(const iterator& it) const; - bool operator==(const iterator& it) const; - T operator*() const; - T Get() const; - - private: - google::protobuf::RepeatedPtrField<TProtoValue>::const_iterator Item; - const TProtoType& ItemType; - }; - - - template <typename T> - class TIterableList { - public: +public: + template <typename T> + class iterator { + public: + iterator(google::protobuf::RepeatedPtrField<TProtoValue>::const_iterator item, const TProtoType& itemType); + iterator(const iterator& it); + + iterator& operator++(); + iterator operator++(int); + bool operator!=(const iterator& it) const; + bool operator==(const iterator& it) const; + T operator*() const; + T Get() const; + + private: + google::protobuf::RepeatedPtrField<TProtoValue>::const_iterator Item; + const TProtoType& ItemType; + }; + + + template <typename T> + class TIterableList { + public: TIterableList(const TListType& l) - : List(l) - { - } - - iterator<T> begin() { - return iterator<T>(List.RootValue.GetList().begin(), List.RootType.GetList().GetItem()); - } - - iterator<T> end() { - return iterator<T>(List.RootValue.GetList().end(), List.RootType.GetList().GetItem()); - } - - private: + : List(l) + { + } + + iterator<T> begin() { + return iterator<T>(List.RootValue.GetList().begin(), List.RootType.GetList().GetItem()); + } + + iterator<T> end() { + return iterator<T>(List.RootValue.GetList().end(), List.RootType.GetList().GetItem()); + } + + private: const TListType& List; - }; - - + }; + + explicit TListType(const TProtoValue& value, const TProtoType& type) - : RootValue(value) - , RootType(type) - , Size(RootValue.ListSize()) - { - ENSURE_KIND(RootType, List); - } - - template <typename T> - TIterableList<T> MakeIterable() { - return TIterableList<T>(*this); - } - - template <typename T> - T GetItem(size_t index) { + : RootValue(value) + , RootType(type) + , Size(RootValue.ListSize()) + { + ENSURE_KIND(RootType, List); + } + + template <typename T> + TIterableList<T> MakeIterable() { + return TIterableList<T>(*this); + } + + template <typename T> + T GetItem(size_t index) { Y_ENSURE(CheckIndex(index), "List item index" << index << " is out of bounds."); - iterator<T> it(RootValue.GetList().begin() + index, RootType.GetList().GetItem()); - return it.Get(); - } - - size_t GetSize() const { - return Size; - } - -private: - bool CheckIndex(size_t index) { - return index < Size; - } - -private: - const TProtoValue& RootValue; - const TProtoType& RootType; - const size_t Size; -}; - - -class TTuple { -public: - explicit TTuple(const TProtoValue& value, const TProtoType& type) - : RootValue(value) - , RootType(type) - , Size(RootValue.TupleSize()) - { - ENSURE_KIND(RootType, Tuple); + iterator<T> it(RootValue.GetList().begin() + index, RootType.GetList().GetItem()); + return it.Get(); + } + + size_t GetSize() const { + return Size; + } + +private: + bool CheckIndex(size_t index) { + return index < Size; + } + +private: + const TProtoValue& RootValue; + const TProtoType& RootType; + const size_t Size; +}; + + +class TTuple { +public: + explicit TTuple(const TProtoValue& value, const TProtoType& type) + : RootValue(value) + , RootType(type) + , Size(RootValue.TupleSize()) + { + ENSURE_KIND(RootType, Tuple); Y_ENSURE(RootType.GetTuple().ElementSize() == RootValue.TupleSize(), "Size mismatch."); - } - - template <typename T> - T GetElement(size_t index) const; - - size_t GetSize() const { - return Size; - } - -private: - bool CheckIndex(size_t index) const { - return index < Size; - } - -private: - const TProtoValue& RootValue; - const TProtoType& RootType; - const size_t Size; -}; - - -class TStruct { -public: - explicit TStruct(const TProtoValue& value, const TProtoType& type) - : RootValue(value) - , RootType(type) - , Size(RootValue.StructSize()) - { - ENSURE_KIND(RootType, Struct); + } + + template <typename T> + T GetElement(size_t index) const; + + size_t GetSize() const { + return Size; + } + +private: + bool CheckIndex(size_t index) const { + return index < Size; + } + +private: + const TProtoValue& RootValue; + const TProtoType& RootType; + const size_t Size; +}; + + +class TStruct { +public: + explicit TStruct(const TProtoValue& value, const TProtoType& type) + : RootValue(value) + , RootType(type) + , Size(RootValue.StructSize()) + { + ENSURE_KIND(RootType, Struct); Y_ENSURE(RootType.GetStruct().MemberSize() == RootValue.StructSize(), "Size mismatch."); - } - - template <typename T> - T GetMember(const TStringBuf& memberName); - - template <typename T> - T GetMember(size_t memberIndex); - - size_t GetMemberIndex(const TStringBuf& memberName) const { - for (size_t i = 0, end = RootType.GetStruct().MemberSize(); i < end; ++i) { - const TStringBuf& name = RootType.GetStruct().GetMember(i).GetName(); - if (name == memberName) { - return i; - } - } - ythrow yexception() << "Unknown Struct member name: " << memberName << "."; - } - - size_t GetSize() const { - return Size; - } - -private: - bool CheckIndex(size_t index) const { - return index < Size; - } - -private: - const TProtoValue& RootValue; - const TProtoType& RootType; - const size_t Size; -}; - - -class TDict { -public: - explicit TDict(const TProtoValue& value, const TProtoType& type) - : RootValue(value) - , RootType(type) - { - ENSURE_KIND(RootType, Dict); - ENSURE_KIND(RootType.GetDict().GetKey(), Data); - - } - - template <typename K, typename V> + } + + template <typename T> + T GetMember(const TStringBuf& memberName); + + template <typename T> + T GetMember(size_t memberIndex); + + size_t GetMemberIndex(const TStringBuf& memberName) const { + for (size_t i = 0, end = RootType.GetStruct().MemberSize(); i < end; ++i) { + const TStringBuf& name = RootType.GetStruct().GetMember(i).GetName(); + if (name == memberName) { + return i; + } + } + ythrow yexception() << "Unknown Struct member name: " << memberName << "."; + } + + size_t GetSize() const { + return Size; + } + +private: + bool CheckIndex(size_t index) const { + return index < Size; + } + +private: + const TProtoValue& RootValue; + const TProtoType& RootType; + const size_t Size; +}; + + +class TDict { +public: + explicit TDict(const TProtoValue& value, const TProtoType& type) + : RootValue(value) + , RootType(type) + { + ENSURE_KIND(RootType, Dict); + ENSURE_KIND(RootType.GetDict().GetKey(), Data); + + } + + template <typename K, typename V> THashMap<K, V> GetHashMap() const; - -private: - const TProtoValue& RootValue; - const TProtoType& RootType; -}; - - -// TOptional. -template <typename T> -T TOptional::GetItem() const { + +private: + const TProtoValue& RootValue; + const TProtoType& RootType; +}; + + +// TOptional. +template <typename T> +T TOptional::GetItem() const { Y_ENSURE(HasItem(), "Optional is empty!"); - const auto& itemType = RootType.GetOptional().GetItem(); - ENSURE_KIND(itemType, Data); + const auto& itemType = RootType.GetOptional().GetItem(); + ENSURE_KIND(itemType, Data); auto schemeType = itemType.GetData().GetScheme(); - return NPrivate::GetData<T>(RootValue.GetOptional(), schemeType); -} - -#include "optional_funcs.inl" - - + return NPrivate::GetData<T>(RootValue.GetOptional(), schemeType); +} + +#include "optional_funcs.inl" + + // TListType. -#include "list_funcs.inl" - - -// TTuple. -template <typename T> -T TTuple::GetElement(size_t index) const { +#include "list_funcs.inl" + + +// TTuple. +template <typename T> +T TTuple::GetElement(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); - ENSURE_KIND(elementType, Data); + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); + ENSURE_KIND(elementType, Data); auto schemeType = elementType.GetData().GetScheme(); - return NPrivate::GetData<T>(element, schemeType); -} - -#include "tuple_funcs.inl" - - -// TStruct. -template <typename T> -T TStruct::GetMember(const TStringBuf& memberName) { - size_t memberIndex = GetMemberIndex(memberName); - return GetMember<T>(memberIndex); -} - -template <typename T> -T TStruct::GetMember(size_t memberIndex) { + return NPrivate::GetData<T>(element, schemeType); +} + +#include "tuple_funcs.inl" + + +// TStruct. +template <typename T> +T TStruct::GetMember(const TStringBuf& memberName) { + size_t memberIndex = GetMemberIndex(memberName); + return GetMember<T>(memberIndex); +} + +template <typename T> +T TStruct::GetMember(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); - ENSURE_KIND(memberType, Data); + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); + ENSURE_KIND(memberType, Data); auto schemeType = memberType.GetData().GetScheme(); - return NPrivate::GetData<T>(member, schemeType); -} - -#include "struct_funcs.inl" - - -// TDict. -template <typename K, typename V> + return NPrivate::GetData<T>(member, schemeType); +} + +#include "struct_funcs.inl" + + +// TDict. +template <typename K, typename V> THashMap<K, V> TDict::GetHashMap() const { THashMap<K, V> m; - ui32 keySchemeType = RootType.GetDict().GetKey().GetData().GetScheme(); - for (const auto& kvPair : RootValue.GetDict()) { - auto dictKey = NPrivate::GetData<K>(kvPair.GetKey(), keySchemeType); - const auto& dictValue = kvPair.GetPayload(); - const auto& dictValueType = RootType.GetDict().GetPayload(); - m[dictKey] = V(dictValue, dictValueType); - } - return m; -} - - -#undef ENSURE_KIND - + ui32 keySchemeType = RootType.GetDict().GetKey().GetData().GetScheme(); + for (const auto& kvPair : RootValue.GetDict()) { + auto dictKey = NPrivate::GetData<K>(kvPair.GetKey(), keySchemeType); + const auto& dictValue = kvPair.GetPayload(); + const auto& dictValueType = RootType.GetDict().GetPayload(); + m[dictKey] = V(dictValue, dictValueType); + } + return m; +} + + +#undef ENSURE_KIND + } // namespace NResultLib -} // namespace NKikimr +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/objects_ut.cpp b/ydb/core/client/minikql_result_lib/objects_ut.cpp index c0b6c1c091..49b3d300dc 100644 --- a/ydb/core/client/minikql_result_lib/objects_ut.cpp +++ b/ydb/core/client/minikql_result_lib/objects_ut.cpp @@ -1,158 +1,158 @@ -#include "objects.h" - +#include "objects.h" + #include <ydb/core/scheme/scheme_type_id.h> - + #include <library/cpp/testing/unittest/registar.h> - - -namespace NKikimr { - -using namespace NKikimrMiniKQL; + + +namespace NKikimr { + +using namespace NKikimrMiniKQL; using namespace NResultLib; - + Y_UNIT_TEST_SUITE(TMiniKQLResultTest) { - + Y_UNIT_TEST(TOptionalTest) { - { // Optional of Bool. - TType type; - type.SetKind(ETypeKind::Optional); - type.MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); + { // Optional of Bool. + TType type; + type.SetKind(ETypeKind::Optional); + type.MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); type.MutableOptional()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Bool); - TValue value; - value.MutableOptional()->SetBool(true); - TOptional opt(value, type); - UNIT_ASSERT_EQUAL(opt.GetItem<bool>(), true); - } - { // Optional of Uint64. - TType type; - type.SetKind(ETypeKind::Optional); - type.MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); + TValue value; + value.MutableOptional()->SetBool(true); + TOptional opt(value, type); + UNIT_ASSERT_EQUAL(opt.GetItem<bool>(), true); + } + { // Optional of Uint64. + TType type; + type.SetKind(ETypeKind::Optional); + type.MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); type.MutableOptional()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Uint64); - TValue value; - value.MutableOptional()->SetUint64(200100); - TOptional opt(value, type); - UNIT_ASSERT_EQUAL(opt.GetItem<ui64>(), 200100); - } -} - + TValue value; + value.MutableOptional()->SetUint64(200100); + TOptional opt(value, type); + UNIT_ASSERT_EQUAL(opt.GetItem<ui64>(), 200100); + } +} + Y_UNIT_TEST(TListTest) { - { // List of Int32. - TType type; - type.SetKind(ETypeKind::List); - type.MutableList()->MutableItem()->SetKind(ETypeKind::Data); + { // List of Int32. + TType type; + type.SetKind(ETypeKind::List); + type.MutableList()->MutableItem()->SetKind(ETypeKind::Data); type.MutableList()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Int32); - TValue value; - value.AddList()->SetInt32(-100); - value.AddList()->SetInt32(0); - value.AddList()->SetInt32(100); - + TValue value; + value.AddList()->SetInt32(-100); + value.AddList()->SetInt32(0); + value.AddList()->SetInt32(100); + TListType l(value, type); - - UNIT_ASSERT_EQUAL(l.GetSize(), 3); - UNIT_ASSERT_EQUAL(l.GetItem<i32>(0), -100); - UNIT_ASSERT_EQUAL(l.GetItem<i32>(1), 0); - UNIT_ASSERT_EQUAL(l.GetItem<i32>(2), 100); - + + UNIT_ASSERT_EQUAL(l.GetSize(), 3); + UNIT_ASSERT_EQUAL(l.GetItem<i32>(0), -100); + UNIT_ASSERT_EQUAL(l.GetItem<i32>(1), 0); + UNIT_ASSERT_EQUAL(l.GetItem<i32>(2), 100); + TVector<i32> v; - for (const auto& listItem : l.MakeIterable<i32>()) { - v.push_back(listItem); - } - UNIT_ASSERT_EQUAL(v.size(), 3); - UNIT_ASSERT_EQUAL(v[0], -100); - UNIT_ASSERT_EQUAL(v[1], 0); - UNIT_ASSERT_EQUAL(v[2], 100); - } - { // List of Optional of Int32. - TType type; - type.SetKind(ETypeKind::List); - type.MutableList()->MutableItem()->SetKind(ETypeKind::Optional); - type.MutableList()->MutableItem()->MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); + for (const auto& listItem : l.MakeIterable<i32>()) { + v.push_back(listItem); + } + UNIT_ASSERT_EQUAL(v.size(), 3); + UNIT_ASSERT_EQUAL(v[0], -100); + UNIT_ASSERT_EQUAL(v[1], 0); + UNIT_ASSERT_EQUAL(v[2], 100); + } + { // List of Optional of Int32. + TType type; + type.SetKind(ETypeKind::List); + type.MutableList()->MutableItem()->SetKind(ETypeKind::Optional); + type.MutableList()->MutableItem()->MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); type.MutableList()->MutableItem()->MutableOptional()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Int32); - TValue value; - value.AddList()->MutableOptional()->SetInt32(-100); - value.AddList()->MutableOptional()->SetInt32(0); - value.AddList()->MutableOptional()->SetInt32(100); + TValue value; + value.AddList()->MutableOptional()->SetInt32(-100); + value.AddList()->MutableOptional()->SetInt32(0); + value.AddList()->MutableOptional()->SetInt32(100); TListType l(value, type); - + TVector<i32> v; - for (const TOptional& opt : l.MakeIterable<TOptional>()) { - v.push_back(opt.GetItem<i32>()); - } - UNIT_ASSERT_EQUAL(v.size(), 3); - UNIT_ASSERT_EQUAL(v[0], -100); - UNIT_ASSERT_EQUAL(v[1], 0); - UNIT_ASSERT_EQUAL(v[2], 100); - } -} - - + for (const TOptional& opt : l.MakeIterable<TOptional>()) { + v.push_back(opt.GetItem<i32>()); + } + UNIT_ASSERT_EQUAL(v.size(), 3); + UNIT_ASSERT_EQUAL(v[0], -100); + UNIT_ASSERT_EQUAL(v[1], 0); + UNIT_ASSERT_EQUAL(v[2], 100); + } +} + + Y_UNIT_TEST(TTupleTest) { - { // Tuple of (Optional of Int32, List of Uint64). - TType type; - type.SetKind(ETypeKind::Tuple); - auto el0Type = type.MutableTuple()->AddElement(); - el0Type->SetKind(ETypeKind::Optional); - el0Type->MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); + { // Tuple of (Optional of Int32, List of Uint64). + TType type; + type.SetKind(ETypeKind::Tuple); + auto el0Type = type.MutableTuple()->AddElement(); + el0Type->SetKind(ETypeKind::Optional); + el0Type->MutableOptional()->MutableItem()->SetKind(ETypeKind::Data); el0Type->MutableOptional()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Int32); - - auto el1Type = type.MutableTuple()->AddElement(); - el1Type->SetKind(ETypeKind::List); - el1Type->MutableList()->MutableItem()->SetKind(ETypeKind::Data); + + auto el1Type = type.MutableTuple()->AddElement(); + el1Type->SetKind(ETypeKind::List); + el1Type->MutableList()->MutableItem()->SetKind(ETypeKind::Data); el1Type->MutableList()->MutableItem()->MutableData()->SetScheme(NScheme::NTypeIds::Uint64); - - TValue value; - value.AddTuple()->MutableOptional()->SetInt32(911); - auto el1 = value.AddTuple(); - el1->AddList()->SetUint64(0); - el1->AddList()->SetUint64(1); - el1->AddList()->SetUint64(2); - - TTuple t(value, type); - - TOptional opt = t.GetElement<TOptional>(0); - UNIT_ASSERT_EQUAL(opt.GetItem<i32>(), 911); + + TValue value; + value.AddTuple()->MutableOptional()->SetInt32(911); + auto el1 = value.AddTuple(); + el1->AddList()->SetUint64(0); + el1->AddList()->SetUint64(1); + el1->AddList()->SetUint64(2); + + TTuple t(value, type); + + TOptional opt = t.GetElement<TOptional>(0); + UNIT_ASSERT_EQUAL(opt.GetItem<i32>(), 911); TListType l = t.GetElement<TListType>(1); - + TVector<ui64> v; - for (ui64 item : l.MakeIterable<ui64>()) { - v.push_back(item); - } - UNIT_ASSERT_EQUAL(v.size(), 3); - UNIT_ASSERT_EQUAL(v[0], 0); - UNIT_ASSERT_EQUAL(v[1], 1); - UNIT_ASSERT_EQUAL(v[2], 2); - } -} - + for (ui64 item : l.MakeIterable<ui64>()) { + v.push_back(item); + } + UNIT_ASSERT_EQUAL(v.size(), 3); + UNIT_ASSERT_EQUAL(v[0], 0); + UNIT_ASSERT_EQUAL(v[1], 1); + UNIT_ASSERT_EQUAL(v[2], 2); + } +} + Y_UNIT_TEST(TStructTest) { - { // Struct of {"a" : Int64, "b" : Int32}. - TType type; - type.SetKind(ETypeKind::Struct); - auto mem1Type = type.MutableStruct()->AddMember(); - mem1Type->SetName("a"); - mem1Type->MutableType()->SetKind(ETypeKind::Data); + { // Struct of {"a" : Int64, "b" : Int32}. + TType type; + type.SetKind(ETypeKind::Struct); + auto mem1Type = type.MutableStruct()->AddMember(); + mem1Type->SetName("a"); + mem1Type->MutableType()->SetKind(ETypeKind::Data); mem1Type->MutableType()->MutableData()->SetScheme(NScheme::NTypeIds::Int64); - auto mem2Type = type.MutableStruct()->AddMember(); - mem2Type->SetName("b"); - mem2Type->MutableType()->SetKind(ETypeKind::Data); + auto mem2Type = type.MutableStruct()->AddMember(); + mem2Type->SetName("b"); + mem2Type->MutableType()->SetKind(ETypeKind::Data); mem2Type->MutableType()->MutableData()->SetScheme(NScheme::NTypeIds::Int32); - - TValue value; - value.AddStruct()->SetInt64(-1000); - value.AddStruct()->SetInt32(1000); - - TStruct s(value, type); - - UNIT_ASSERT_EQUAL(s.GetMember<i64>("a"), -1000); - UNIT_ASSERT_EQUAL(s.GetMember<i32>("b"), 1000); - - auto aIndex = s.GetMemberIndex("a"); - auto bIndex = s.GetMemberIndex("b"); - UNIT_ASSERT_EQUAL(s.GetMember<i64>(aIndex), -1000); - UNIT_ASSERT_EQUAL(s.GetMember<i32>(bIndex), 1000); - } -} - -} - -} // namespace NKikimr + + TValue value; + value.AddStruct()->SetInt64(-1000); + value.AddStruct()->SetInt32(1000); + + TStruct s(value, type); + + UNIT_ASSERT_EQUAL(s.GetMember<i64>("a"), -1000); + UNIT_ASSERT_EQUAL(s.GetMember<i32>("b"), 1000); + + auto aIndex = s.GetMemberIndex("a"); + auto bIndex = s.GetMemberIndex("b"); + UNIT_ASSERT_EQUAL(s.GetMember<i64>(aIndex), -1000); + UNIT_ASSERT_EQUAL(s.GetMember<i32>(bIndex), 1000); + } +} + +} + +} // namespace NKikimr diff --git a/ydb/core/client/minikql_result_lib/optional_funcs.inl b/ydb/core/client/minikql_result_lib/optional_funcs.inl index e457f5421d..56cc00dcc7 100644 --- a/ydb/core/client/minikql_result_lib/optional_funcs.inl +++ b/ydb/core/client/minikql_result_lib/optional_funcs.inl @@ -1,49 +1,49 @@ -#pragma once - - -#define ENSURE_HAS_ITEM() \ - do { \ +#pragma once + + +#define ENSURE_HAS_ITEM() \ + do { \ Y_ENSURE(HasItem(), "Optional is empty!"); \ - } while (0); - -template <> -inline void TOptional::GetItem<void>() const { - return; -} - -template <> -inline TOptional TOptional::GetItem<TOptional>() const { - ENSURE_HAS_ITEM(); - const auto& itemType = RootType.GetOptional().GetItem(); - return TOptional(RootValue.GetOptional(), itemType); -} - -template <> + } while (0); + +template <> +inline void TOptional::GetItem<void>() const { + return; +} + +template <> +inline TOptional TOptional::GetItem<TOptional>() const { + ENSURE_HAS_ITEM(); + const auto& itemType = RootType.GetOptional().GetItem(); + return TOptional(RootValue.GetOptional(), itemType); +} + +template <> inline TListType TOptional::GetItem<TListType>() const { - ENSURE_HAS_ITEM(); - const auto& itemType = RootType.GetOptional().GetItem(); + ENSURE_HAS_ITEM(); + const auto& itemType = RootType.GetOptional().GetItem(); return TListType(RootValue.GetOptional(), itemType); -} - -template <> -inline TTuple TOptional::GetItem<TTuple>() const { - ENSURE_HAS_ITEM(); - const auto& itemType = RootType.GetOptional().GetItem(); - return TTuple(RootValue.GetOptional(), itemType); -} - -template <> -inline TStruct TOptional::GetItem<TStruct>() const { - ENSURE_HAS_ITEM(); - const auto& itemType = RootType.GetOptional().GetItem(); - return TStruct(RootValue.GetOptional(), itemType); -} - -template <> -inline TDict TOptional::GetItem<TDict>() const { - ENSURE_HAS_ITEM(); - const auto& itemType = RootType.GetOptional().GetItem(); - return TDict(RootValue.GetOptional(), itemType); -} - -#undef ENSURE_HAS_ITEM +} + +template <> +inline TTuple TOptional::GetItem<TTuple>() const { + ENSURE_HAS_ITEM(); + const auto& itemType = RootType.GetOptional().GetItem(); + return TTuple(RootValue.GetOptional(), itemType); +} + +template <> +inline TStruct TOptional::GetItem<TStruct>() const { + ENSURE_HAS_ITEM(); + const auto& itemType = RootType.GetOptional().GetItem(); + return TStruct(RootValue.GetOptional(), itemType); +} + +template <> +inline TDict TOptional::GetItem<TDict>() const { + ENSURE_HAS_ITEM(); + const auto& itemType = RootType.GetOptional().GetItem(); + return TDict(RootValue.GetOptional(), itemType); +} + +#undef ENSURE_HAS_ITEM diff --git a/ydb/core/client/minikql_result_lib/struct_funcs.inl b/ydb/core/client/minikql_result_lib/struct_funcs.inl index aacb8ba6ec..4c95387842 100644 --- a/ydb/core/client/minikql_result_lib/struct_funcs.inl +++ b/ydb/core/client/minikql_result_lib/struct_funcs.inl @@ -1,48 +1,48 @@ -#pragma once - - -template <> -inline void TStruct::GetMember<void>(size_t memberIndex) { +#pragma once + + +template <> +inline void TStruct::GetMember<void>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - return; -} - -template <> -inline TOptional TStruct::GetMember<TOptional>(size_t memberIndex) { + return; +} + +template <> +inline TOptional TStruct::GetMember<TOptional>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); - return TOptional(member, memberType); -} - -template <> + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); + return TOptional(member, memberType); +} + +template <> inline TListType TStruct::GetMember<TListType>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); return TListType(member, memberType); -} - -template <> -inline TTuple TStruct::GetMember<TTuple>(size_t memberIndex) { +} + +template <> +inline TTuple TStruct::GetMember<TTuple>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); - return TTuple(member, memberType); -} - -template <> -inline TStruct TStruct::GetMember<TStruct>(size_t memberIndex) { + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); + return TTuple(member, memberType); +} + +template <> +inline TStruct TStruct::GetMember<TStruct>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); - return TStruct(member, memberType); -} - -template <> -inline TDict TStruct::GetMember<TDict>(size_t memberIndex) { + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); + return TStruct(member, memberType); +} + +template <> +inline TDict TStruct::GetMember<TDict>(size_t memberIndex) { Y_ENSURE(CheckIndex(memberIndex), "Struct member index" << memberIndex << " is out of bounds."); - const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); - const auto& member = RootValue.GetStruct(memberIndex); - return TDict(member, memberType); -} + const auto& memberType = RootType.GetStruct().GetMember(memberIndex).GetType(); + const auto& member = RootValue.GetStruct(memberIndex); + return TDict(member, memberType); +} diff --git a/ydb/core/client/minikql_result_lib/tuple_funcs.inl b/ydb/core/client/minikql_result_lib/tuple_funcs.inl index 4e7a84ebf9..9c11f2f61f 100644 --- a/ydb/core/client/minikql_result_lib/tuple_funcs.inl +++ b/ydb/core/client/minikql_result_lib/tuple_funcs.inl @@ -1,48 +1,48 @@ -#pragma once - - -template <> -inline void TTuple::GetElement<void>(size_t index) const { +#pragma once + + +template <> +inline void TTuple::GetElement<void>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - return; -} - -template <> -inline TOptional TTuple::GetElement<TOptional>(size_t index) const { + return; +} + +template <> +inline TOptional TTuple::GetElement<TOptional>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); - return TOptional(element, elementType); -} - -template <> + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); + return TOptional(element, elementType); +} + +template <> inline TListType TTuple::GetElement<TListType>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); return TListType(element, elementType); -} - -template <> -inline TTuple TTuple::GetElement<TTuple>(size_t index) const { +} + +template <> +inline TTuple TTuple::GetElement<TTuple>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); - return TTuple(element, elementType); -} - -template <> -inline TStruct TTuple::GetElement<TStruct>(size_t index) const { + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); + return TTuple(element, elementType); +} + +template <> +inline TStruct TTuple::GetElement<TStruct>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); - return TStruct(element, elementType); -} - -template <> -inline TDict TTuple::GetElement<TDict>(size_t index) const { + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); + return TStruct(element, elementType); +} + +template <> +inline TDict TTuple::GetElement<TDict>(size_t index) const { Y_ENSURE(CheckIndex(index), "Tuple element index" << index << " is out of bounds."); - const auto& elementType = RootType.GetTuple().GetElement(index); - const auto& element = RootValue.GetTuple(index); - return TDict(element, elementType); -} + const auto& elementType = RootType.GetTuple().GetElement(index); + const auto& element = RootValue.GetTuple(index); + return TDict(element, elementType); +} diff --git a/ydb/core/client/minikql_result_lib/ut/ya.make b/ydb/core/client/minikql_result_lib/ut/ya.make index bbd78fb9b5..977bac4c9a 100644 --- a/ydb/core/client/minikql_result_lib/ut/ya.make +++ b/ydb/core/client/minikql_result_lib/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/core/client/minikql_result_lib) - + OWNER(g:kikimr) - + FORK_SUBTESTS() TIMEOUT(300) SIZE(MEDIUM) -SRCS( - converter_ut.cpp - objects_ut.cpp -) - -PEERDIR( +SRCS( + converter_ut.cpp + objects_ut.cpp +) + +PEERDIR( library/cpp/testing/unittest ydb/core/testlib -) - +) + YQL_LAST_ABI_VERSION() REQUIREMENTS(network:full ram:13) -END() +END() diff --git a/ydb/core/client/minikql_result_lib/ya.make b/ydb/core/client/minikql_result_lib/ya.make index ac62583443..d726bfeea4 100644 --- a/ydb/core/client/minikql_result_lib/ya.make +++ b/ydb/core/client/minikql_result_lib/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - objects.h - objects.cpp - converter.h - converter.cpp -) - -PEERDIR( - contrib/libs/protobuf + +SRCS( + objects.h + objects.cpp + converter.h + converter.cpp +) + +PEERDIR( + contrib/libs/protobuf ydb/core/protos ydb/core/scheme ydb/library/mkql_proto/protos ydb/public/lib/deprecated/kicli -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/client/scheme_cache_lib/ya.make b/ydb/core/client/scheme_cache_lib/ya.make index ce4a740709..3140bad5a2 100644 --- a/ydb/core/client/scheme_cache_lib/ya.make +++ b/ydb/core/client/scheme_cache_lib/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - yql_db_scheme_resolver.h - yql_db_scheme_resolver.cpp -) - -PEERDIR( - contrib/libs/protobuf + +SRCS( + yql_db_scheme_resolver.h + yql_db_scheme_resolver.cpp +) + +PEERDIR( + contrib/libs/protobuf library/cpp/actors/core library/cpp/grpc/client library/cpp/threading/future @@ -18,8 +18,8 @@ PEERDIR( ydb/core/scheme ydb/core/tablet ydb/core/tx -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp index 060e5b6ec1..eb74fa36e9 100644 --- a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp +++ b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp @@ -1,19 +1,19 @@ -#include "yql_db_scheme_resolver.h" - +#include "yql_db_scheme_resolver.h" + #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/client/minikql_compile/yql_expr_minikql.h> - + #include <library/cpp/actors/core/actorsystem.h> - -namespace NKikimr { -namespace NSchCache { - -using namespace NYql; -using namespace NThreading; - -namespace { - + +namespace NKikimr { +namespace NSchCache { + +using namespace NYql; +using namespace NThreading; + +namespace { + class TTableProxyActor : public TActorBootstrapped<TTableProxyActor> { using TTable = NYql::IDbSchemeResolver::TTable; using TTableResult = NYql::IDbSchemeResolver::TTableResult; @@ -130,22 +130,22 @@ public: } }; -} // anonymous namespace - -class TDbSchemeResolver : public IDbSchemeResolver { -public: +} // anonymous namespace + +class TDbSchemeResolver : public IDbSchemeResolver { +public: TDbSchemeResolver(TActorSystem *actorSystem) : HostActorSystem(actorSystem) - { - } - + { + } + TDbSchemeResolver(TActorSystem *actorSystem, const TActorId &schemeCacheActor) : HostActorSystem(actorSystem) , SchemeCacheActor(schemeCacheActor) {} - + virtual ~TDbSchemeResolver() {} - + virtual NThreading::TFuture<TTableResults> ResolveTables(const TVector<TTable>& tables) override { TTableResults results; for (auto& table : tables) { @@ -162,16 +162,16 @@ public: HostActorSystem->Register(proxyActor.Release(), TMailboxType::HTSwap, HostActorSystem->AppData<TAppData>()->UserPoolId); } -private: +private: TActorSystem *HostActorSystem; TActorId SchemeCacheActor; -}; - +}; + NYql::IDbSchemeResolver* CreateDbSchemeResolver(TActorSystem *actorSystem, const TActorId &schemeCacheActor) { TAutoPtr<NYql::IDbSchemeResolver> resolver(new TDbSchemeResolver(actorSystem, schemeCacheActor)); - return resolver.Release(); -} - - -} // namespace NSchCache -} // namespace NKikimr + return resolver.Release(); +} + + +} // namespace NSchCache +} // namespace NKikimr diff --git a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.h b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.h index 3ef8bcc26e..b1d2eaae8e 100644 --- a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.h +++ b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.h @@ -1,19 +1,19 @@ -#pragma once +#pragma once #include <ydb/core/base/defs.h> - -namespace NYql { -class IDbSchemeResolver; -} // namespace NYql - + +namespace NYql { +class IDbSchemeResolver; +} // namespace NYql + namespace NActors { class TActorSystem; struct TActorId; } -namespace NKikimr { -namespace NSchCache { - +namespace NKikimr { +namespace NSchCache { + NYql::IDbSchemeResolver* CreateDbSchemeResolver(TActorSystem *actorSystem, const TActorId &schemeCacheActor); - -} // namespace NSchCache -} // namespace NKikimr + +} // namespace NSchCache +} // namespace NKikimr diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 7d2dbc7dd1..7f3146d21a 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -2,7 +2,7 @@ #include "msgbus_server_request.h" #include "msgbus_server_proxy.h" #include "msgbus_securereq.h" - + #include <library/cpp/actors/core/hfunc.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> @@ -120,8 +120,8 @@ TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, response->Record.SetExecutionEngineStatus(result.GetExecutionEngineStatus()); if (result.HasExecutionEngineResponseStatus()) response->Record.SetExecutionEngineResponseStatus(result.GetExecutionEngineResponseStatus()); - if (result.HasMiniKQLCompileResults()) - response->Record.MutableMiniKQLCompileResults()->CopyFrom(result.GetMiniKQLCompileResults()); + if (result.HasMiniKQLCompileResults()) + response->Record.MutableMiniKQLCompileResults()->CopyFrom(result.GetMiniKQLCompileResults()); if (result.HasMiniKQLErrors()) response->Record.SetMiniKQLErrors(result.GetMiniKQLErrors()); if (result.HasDataShardErrors()) diff --git a/ydb/core/client/server/msgbus_server_request.cpp b/ydb/core/client/server/msgbus_server_request.cpp index b2fd6d1321..9223bea9f1 100644 --- a/ydb/core/client/server/msgbus_server_request.cpp +++ b/ydb/core/client/server/msgbus_server_request.cpp @@ -19,8 +19,8 @@ class TMessageBusServerRequest : public TMessageBusSecureRequest<TMessageBusServ using TBase = TMessageBusSecureRequest<TMessageBusServerRequestBase<TMessageBusServerRequest>>; THolder<TBusRequest> Request; TVector<TString*> WriteResolvedKeysTo; - TAutoPtr<TEvTxUserProxy::TEvProposeTransaction> Proposal; - TAutoPtr<NKikimrTxUserProxy::TEvProposeTransactionStatus> ProposalStatus; + TAutoPtr<TEvTxUserProxy::TEvProposeTransaction> Proposal; + TAutoPtr<NKikimrTxUserProxy::TEvProposeTransactionStatus> ProposalStatus; size_t InFlyRequests; THashMap<TString, ui64> CompileResolveCookies; @@ -33,10 +33,10 @@ class TMessageBusServerRequest : public TMessageBusSecureRequest<TMessageBusServ void FinishReply(const TActorContext &ctx); void TryToAllocateQuota(const TActorContext &ctx); - void Handle(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr &ev, const TActorContext &ctx); + void Handle(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr &ev, const TActorContext &ctx); void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr &ev, const TActorContext &ctx); void Handle(TEvDataShard::TEvGetReadTableStreamStateRequest::TPtr &ev, const TActorContext &ctx); - + bool AllRequestsCompleted(const TActorContext& ctx); bool AllRequestsCompletedMKQL(const TActorContext& ctx); bool AllRequestsCompletedReadTable(const TActorContext& ctx); @@ -59,7 +59,7 @@ public: //STFUNC(StateWork) void StateWork(TAutoPtr<NActors::IEventHandle> &ev, const NActors::TActorContext &ctx) { switch (ev->GetTypeRewrite()) { - HFunc(TMiniKQLCompileServiceEvents::TEvCompileStatus, Handle); + HFunc(TMiniKQLCompileServiceEvents::TEvCompileStatus, Handle); HFunc(TEvTxUserProxy::TEvProposeTransactionStatus, Handle); HFunc(TEvDataShard::TEvGetReadTableStreamStateRequest, Handle); } @@ -136,11 +136,11 @@ public: } else if (transaction->HasReadTableTransaction()) { NKikimrTxUserProxy::TEvProposeTransactionStatus status; return ReplyWithResult(EResponseStatus::MSTATUS_ERROR, status, ctx); - } + } ctx.Send(MakeTxProxyID(), Proposal.Release()); - } + } }; - + bool TMessageBusServerRequest::RetryResolve(const TActorContext &ctx) { if (CompilationRetried || !TextProgramForCompilation) return false; @@ -161,9 +161,9 @@ void TMessageBusServerRequest::ReplyWithResult(EResponseStatus status, { TAutoPtr<TBusResponse> response(ProposeTransactionStatusToResponse(status, result)); - if (result.HasExecutionEngineEvaluatedResponse()) { + if (result.HasExecutionEngineEvaluatedResponse()) { response->Record.MutableExecutionEngineEvaluatedResponse()->Swap(result.MutableExecutionEngineEvaluatedResponse()); - } + } if (result.HasSerializedReadTableResponse()) { response->Record.SetSerializedReadTableResponse(result.GetSerializedReadTableResponse()); } @@ -188,49 +188,49 @@ void TMessageBusServerRequest::FinishReply(const TActorContext &ctx) Die(ctx); } -void TMessageBusServerRequest::Handle(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr &ev, const TActorContext &ctx) { - auto* mkqlTx = Proposal->Record.MutableTransaction()->MutableMiniKQLTransaction(); - - const auto& result = ev->Get()->Result; - +void TMessageBusServerRequest::Handle(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr &ev, const TActorContext &ctx) { + auto* mkqlTx = Proposal->Record.MutableTransaction()->MutableMiniKQLTransaction(); + + const auto& result = ev->Get()->Result; + const bool need2CompileProgram = (bool)TextProgramForCompilation; - const bool need2CompileParams = mkqlTx->HasParams() && mkqlTx->GetParams().HasText(); + const bool need2CompileParams = mkqlTx->HasParams() && mkqlTx->GetParams().HasText(); const TString& pgm = ev->Get()->Program; Y_VERIFY((need2CompileProgram && TextProgramForCompilation == pgm) // TODO: do not check texts, trust cookies - || (need2CompileParams && mkqlTx->GetParams().GetText() == pgm)); - + || (need2CompileParams && mkqlTx->GetParams().GetText() == pgm)); + if (need2CompileProgram && TextProgramForCompilation == pgm) { - auto* compileResults = ProposalStatus->MutableMiniKQLCompileResults(); - auto* pgm = mkqlTx->MutableProgram(); - if (result.Errors.Empty()) { - pgm->ClearText(); - pgm->SetBin(result.CompiledProgram); - compileResults->SetCompiledProgram(result.CompiledProgram); - } else { + auto* compileResults = ProposalStatus->MutableMiniKQLCompileResults(); + auto* pgm = mkqlTx->MutableProgram(); + if (result.Errors.Empty()) { + pgm->ClearText(); + pgm->SetBin(result.CompiledProgram); + compileResults->SetCompiledProgram(result.CompiledProgram); + } else { NYql::IssuesToMessage(result.Errors, compileResults->MutableProgramCompileErrors()); - } + } --InFlyRequests; CompileResolveCookies = std::move(ev->Get()->CompileResolveCookies); - } - - if (need2CompileParams && mkqlTx->GetParams().GetText() == pgm) { - auto* compileResults = ProposalStatus->MutableMiniKQLCompileResults(); - auto* params = mkqlTx->MutableParams(); - if (result.Errors.Empty()) { - params->ClearText(); - params->SetBin(result.CompiledProgram); - compileResults->SetCompiledParams(result.CompiledProgram); - } else { + } + + if (need2CompileParams && mkqlTx->GetParams().GetText() == pgm) { + auto* compileResults = ProposalStatus->MutableMiniKQLCompileResults(); + auto* params = mkqlTx->MutableParams(); + if (result.Errors.Empty()) { + params->ClearText(); + params->SetBin(result.CompiledProgram); + compileResults->SetCompiledParams(result.CompiledProgram); + } else { NYql::IssuesToMessage(result.Errors, compileResults->MutableParamsCompileErrors()); - } + } --InFlyRequests; - } - + } + if (InFlyRequests == 0) { AllRequestsCompleted(ctx); - } -} - + } +} + bool TMessageBusServerRequest::AllRequestsCompleted(const TActorContext& ctx) { auto &transaction = Proposal->Record.GetTransaction(); if (transaction.HasMiniKQLTransaction()) @@ -240,41 +240,41 @@ bool TMessageBusServerRequest::AllRequestsCompleted(const TActorContext& ctx) { } bool TMessageBusServerRequest::AllRequestsCompletedMKQL(const TActorContext& ctx) { - auto* mkqlTx = Proposal->Record.MutableTransaction()->MutableMiniKQLTransaction(); - const bool need2CompileProgram = mkqlTx->HasProgram() && mkqlTx->GetProgram().HasText(); - const bool need2CompileParams = mkqlTx->HasParams() && mkqlTx->GetParams().HasText(); - const bool programCompiled = need2CompileProgram - ? (ProposalStatus->MutableMiniKQLCompileResults()->ProgramCompileErrorsSize() == 0) - : true; - const bool paramsCompiled = need2CompileParams - ? (ProposalStatus->MutableMiniKQLCompileResults()->ParamsCompileErrorsSize() == 0) - : true; - - const bool allCompiledOk = programCompiled && paramsCompiled; - if (!allCompiledOk) { - ReplyWithResult(MSTATUS_ERROR, *ProposalStatus.Get(), ctx); - return false; - } - + auto* mkqlTx = Proposal->Record.MutableTransaction()->MutableMiniKQLTransaction(); + const bool need2CompileProgram = mkqlTx->HasProgram() && mkqlTx->GetProgram().HasText(); + const bool need2CompileParams = mkqlTx->HasParams() && mkqlTx->GetParams().HasText(); + const bool programCompiled = need2CompileProgram + ? (ProposalStatus->MutableMiniKQLCompileResults()->ProgramCompileErrorsSize() == 0) + : true; + const bool paramsCompiled = need2CompileParams + ? (ProposalStatus->MutableMiniKQLCompileResults()->ParamsCompileErrorsSize() == 0) + : true; + + const bool allCompiledOk = programCompiled && paramsCompiled; + if (!allCompiledOk) { + ReplyWithResult(MSTATUS_ERROR, *ProposalStatus.Get(), ctx); + return false; + } + NKikimrTxUserProxy::TEvProposeTransaction &record = Proposal->Record; record.SetUserToken(TBase::GetSerializedToken()); - switch (mkqlTx->GetMode()) { - case NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC: { + switch (mkqlTx->GetMode()) { + case NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC: { TAutoPtr<TEvTxUserProxy::TEvProposeTransaction> ev = new TEvTxUserProxy::TEvProposeTransaction(); ev->Record.CopyFrom(Proposal->Record); ctx.Send(MakeTxProxyID(), ev.Release()); - return true; - } - case NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE: { - ReplyWithResult(MSTATUS_OK, *ProposalStatus.Get(), ctx); - return true; - } - default: + return true; + } + case NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE: { + ReplyWithResult(MSTATUS_OK, *ProposalStatus.Get(), ctx); + return true; + } + default: Y_FAIL("Unknown mkqlTxMode."); - } -} - + } +} + void TMessageBusServerRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr &ev, const TActorContext &ctx) { TEvTxUserProxy::TEvProposeTransactionStatus *msg = ev->Get(); diff --git a/ydb/core/client/server/ya.make b/ydb/core/client/server/ya.make index 3b9e6aa16d..8cbeec21b7 100644 --- a/ydb/core/client/server/ya.make +++ b/ydb/core/client/server/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER( ddoarn g:kikimr g:logbroker ) - -SRCS( - document_conversion.h + +SRCS( + document_conversion.h http_ping.cpp http_ping.h msgbus_blobstorage_config.cpp - msgbus_bsadm.cpp + msgbus_bsadm.cpp msgbus_http_server.h msgbus_http_server.cpp - msgbus_server.cpp - msgbus_server.h + msgbus_server.cpp + msgbus_server.h msgbus_server_cms.cpp - msgbus_server_configdummy.cpp + msgbus_server_configdummy.cpp msgbus_server_console.cpp msgbus_server_db.cpp msgbus_server_drain_node.cpp @@ -39,12 +39,12 @@ SRCS( msgbus_server_local_minikql.cpp msgbus_server_local_scheme_tx.cpp msgbus_server_node_registration.cpp - msgbus_server_proxy.cpp + msgbus_server_proxy.cpp msgbus_server_proxy.h msgbus_server_request.cpp msgbus_server_request.h msgbus_server_s3_listing.cpp - msgbus_server_scheme_initroot.cpp + msgbus_server_scheme_initroot.cpp msgbus_server_scheme_request.cpp msgbus_server_sqs.cpp msgbus_server_tablet_counters.cpp @@ -54,17 +54,17 @@ SRCS( msgbus_server_tracer.cpp msgbus_server_tracer.h msgbus_server_tx_request.cpp - msgbus_server_types.cpp + msgbus_server_types.cpp msgbus_server_whoami.cpp msgbus_servicereq.h - msgbus_tabletreq.h + msgbus_tabletreq.h grpc_server.cpp grpc_server.h grpc_proxy_status.h grpc_proxy_status.cpp -) - -PEERDIR( +) + +PEERDIR( library/cpp/actors/helpers library/cpp/json library/cpp/messagebus @@ -96,8 +96,8 @@ PEERDIR( ydb/public/lib/deprecated/client ydb/public/lib/deprecated/kicli ydb/services/persqueue_v1 -) - +) + YQL_LAST_ABI_VERSION() END() diff --git a/ydb/core/client/ya.make b/ydb/core/client/ya.make index a1fe24023a..cd39222ad1 100644 --- a/ydb/core/client/ya.make +++ b/ydb/core/client/ya.make @@ -12,10 +12,10 @@ PEERDIR( ydb/core/client/server ydb/core/engine ydb/public/lib/deprecated/kicli -) +) END() - + RECURSE( metadata minikql_compile diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h index cbc9013d51..099664792e 100644 --- a/ydb/core/driver_lib/cli_utils/cli.h +++ b/ydb/core/driver_lib/cli_utils/cli.h @@ -29,7 +29,7 @@ namespace NDriverClient { int SchemeInitRoot(TCommandConfig &cmdConf, int argc, char** argv); int BSAdmCreateVSlots(TCommandConfig &cmdConf, int argc, char **argv); int BSAdmCreateGroup(TCommandConfig &cmdConf, int argc, char **argv); - int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv); + int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv); int MessageBusTrace(TCommandConfig &cmdConf, int argc, char** argv); int KeyValueRequest(TCommandConfig &cmdConf, int argc, char **argv); int PersQueueRequest(TCommandConfig &cmdConf, int argc, char **argv); diff --git a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp index 1bb98fc3d9..b4429d5721 100644 --- a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp @@ -1,5 +1,5 @@ -#include "cli.h" - +#include "cli.h" + #include <ydb/library/yql/ast/yql_ast.h> #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/core/engine/mkql_engine_flat.h> @@ -8,159 +8,159 @@ #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/client/minikql_compile/yql_expr_minikql.h> - - -namespace NKikimr { -namespace NDriverClient { - -using namespace NYql; - -struct TCmdCompileAndExecMiniKQLConfig : public TCliCmdConfig { - TCmdCompileAndExecMiniKQLConfig(); - - void Parse(int argc, char **argv); - + + +namespace NKikimr { +namespace NDriverClient { + +using namespace NYql; + +struct TCmdCompileAndExecMiniKQLConfig : public TCliCmdConfig { + TCmdCompileAndExecMiniKQLConfig(); + + void Parse(int argc, char **argv); + TString PathToTextPgm; TString PathToBinPgm; - + TString PathToTextParams; TString PathToBinParams; -}; - -int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv) { +}; + +int CompileAndExecMiniKQL(TCommandConfig &cmdConf, int argc, char **argv) { Y_UNUSED(cmdConf); - -#ifdef _win32_ - WSADATA dummy; - WSAStartup(MAKEWORD(2, 2), &dummy); -#endif - - TCmdCompileAndExecMiniKQLConfig config; - config.Parse(argc, argv); - + +#ifdef _win32_ + WSADATA dummy; + WSAStartup(MAKEWORD(2, 2), &dummy); +#endif + + TCmdCompileAndExecMiniKQLConfig config; + config.Parse(argc, argv); + auto functionRegistry = NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry()); TAlignedPagePoolCounters countersStub; NMiniKQL::TScopedAlloc alloc(countersStub); NMiniKQL::TTypeEnvironment TypeEnv(alloc); - + TAutoPtr<NMsgBusProxy::TBusRequest> request(new NMsgBusProxy::TBusRequest()); - - auto* mkqlTx = request->Record.MutableTransaction()->MutableMiniKQLTransaction(); - if (config.PathToBinPgm) { + + auto* mkqlTx = request->Record.MutableTransaction()->MutableMiniKQLTransaction(); + if (config.PathToBinPgm) { TString pgmBin = TFileInput(config.PathToBinPgm).ReadAll(); - mkqlTx->MutableProgram()->SetBin(pgmBin); - } else if (config.PathToTextPgm) { + mkqlTx->MutableProgram()->SetBin(pgmBin); + } else if (config.PathToTextPgm) { TString pgmText = TFileInput(config.PathToTextPgm).ReadAll(); mkqlTx->MutableProgram()->SetText(pgmText); - } - - if (config.PathToBinParams) { + } + + if (config.PathToBinParams) { TString paramsBin = TFileInput(config.PathToBinParams).ReadAll(); - mkqlTx->MutableParams()->SetBin(paramsBin); - } else if (config.PathToTextParams) { + mkqlTx->MutableParams()->SetBin(paramsBin); + } else if (config.PathToTextParams) { TString paramsText = TFileInput(config.PathToTextParams).ReadAll(); mkqlTx->MutableParams()->SetText(paramsText); - } - + } + mkqlTx->SetFlatMKQL(true); - TAutoPtr<NBus::TBusMessage> reply; + TAutoPtr<NBus::TBusMessage> reply; NBus::EMessageStatus msgStatus = config.SyncCall(request, reply); - - if (msgStatus != NBus::MESSAGE_OK) { - Cerr << "Can't send request, msgstatus=" << msgStatus << ".\n"; - return 1; - } + + if (msgStatus != NBus::MESSAGE_OK) { + Cerr << "Can't send request, msgstatus=" << msgStatus << ".\n"; + return 1; + } const NKikimrClient::TResponse& response = static_cast<NMsgBusProxy::TBusResponse*>(reply.Get())->Record; - - auto txRes = response.GetMiniKQLCompileResults(); - - if (txRes.ProgramCompileErrorsSize() > 0) { + + auto txRes = response.GetMiniKQLCompileResults(); + + if (txRes.ProgramCompileErrorsSize() > 0) { TIssues errors; NYql::IssuesFromMessage(txRes.GetProgramCompileErrors(), errors); - Cerr << "Program compile errors:\n"; - if (config.PathToBinPgm) { - errors.PrintTo(Cerr); - } else { + Cerr << "Program compile errors:\n"; + if (config.PathToBinPgm) { + errors.PrintTo(Cerr); + } else { const TString pgmText = TFileInput(config.PathToTextPgm).ReadAll(); - errors.PrintWithProgramTo(Cerr, config.PathToTextPgm, pgmText); - } - } - - if (txRes.ParamsCompileErrorsSize() > 0) { + errors.PrintWithProgramTo(Cerr, config.PathToTextPgm, pgmText); + } + } + + if (txRes.ParamsCompileErrorsSize() > 0) { TIssues errors; NYql::IssuesFromMessage(txRes.GetParamsCompileErrors(), errors); - Cerr << "Params compile errors:\n"; - if (config.PathToBinParams) { - errors.PrintTo(Cerr); - } else { + Cerr << "Params compile errors:\n"; + if (config.PathToBinParams) { + errors.PrintTo(Cerr); + } else { const TString paramsText = TFileInput(config.PathToTextParams).ReadAll(); - errors.PrintWithProgramTo(Cerr, config.PathToTextParams, paramsText); - } - } - - + errors.PrintWithProgramTo(Cerr, config.PathToTextParams, paramsText); + } + } + + if (!response.GetDataShardErrors().empty()) { - Cerr << "DataShard errors:\n " << response.GetDataShardErrors() << Endl; - } + Cerr << "DataShard errors:\n " << response.GetDataShardErrors() << Endl; + } if (!response.GetMiniKQLErrors().empty()) { - Cerr << "MiniKQL errors:\n" << response.GetMiniKQLErrors() << Endl; - } - if (response.HasProxyErrorCode()) { - Cerr << "Proxy status: " << static_cast<TEvTxUserProxy::TResultStatus::EStatus>(response.GetProxyErrorCode()) << Endl; - } - if (response.HasProxyErrors()) { - Cerr << "Proxy errors: " << response.GetProxyErrors() << Endl; - } - if (response.UnresolvedKeysSize() > 0) { - for (size_t i = 0, end = response.UnresolvedKeysSize(); i < end; ++i) { - Cerr << response.GetUnresolvedKeys(i) << Endl; - } - } - if ((ui32) NMsgBusProxy::MSTATUS_OK != response.GetStatus()) { - Cerr << "Bad Response status: " << (NMsgBusProxy::EResponseStatus) response.GetStatus() << Endl; - return 1; - } - + Cerr << "MiniKQL errors:\n" << response.GetMiniKQLErrors() << Endl; + } + if (response.HasProxyErrorCode()) { + Cerr << "Proxy status: " << static_cast<TEvTxUserProxy::TResultStatus::EStatus>(response.GetProxyErrorCode()) << Endl; + } + if (response.HasProxyErrors()) { + Cerr << "Proxy errors: " << response.GetProxyErrors() << Endl; + } + if (response.UnresolvedKeysSize() > 0) { + for (size_t i = 0, end = response.UnresolvedKeysSize(); i < end; ++i) { + Cerr << response.GetUnresolvedKeys(i) << Endl; + } + } + if ((ui32) NMsgBusProxy::MSTATUS_OK != response.GetStatus()) { + Cerr << "Bad Response status: " << (NMsgBusProxy::EResponseStatus) response.GetStatus() << Endl; + return 1; + } + auto execEngineStatus = static_cast<NMiniKQL::IEngineFlat::EStatus>(response.GetExecutionEngineResponseStatus()); - Cerr << "ExecutionEngineResponse status: " << (ui32) execEngineStatus << Endl; + Cerr << "ExecutionEngineResponse status: " << (ui32) execEngineStatus << Endl; if (execEngineStatus != NMiniKQL::IEngineFlat::EStatus::Complete && execEngineStatus != NMiniKQL::IEngineFlat::EStatus::Aborted) - { - return 1; - } + { + return 1; + } const auto& evaluatedResp = response.GetExecutionEngineEvaluatedResponse(); Cout << evaluatedResp.DebugString() << Endl; - - return 0; -} - - -TCmdCompileAndExecMiniKQLConfig::TCmdCompileAndExecMiniKQLConfig() -{} - -void TCmdCompileAndExecMiniKQLConfig::Parse(int argc, char **argv) { - using namespace NLastGetopt; - - TOpts opts = TOpts::Default(); + + return 0; +} + + +TCmdCompileAndExecMiniKQLConfig::TCmdCompileAndExecMiniKQLConfig() +{} + +void TCmdCompileAndExecMiniKQLConfig::Parse(int argc, char **argv) { + using namespace NLastGetopt; + + TOpts opts = TOpts::Default(); opts.AddLongOption('f', "pgm-txt", "path to file with text program").RequiredArgument("PATH").StoreResult(&PathToTextPgm); opts.AddLongOption('b', "pgm-bin", "path to file with serialized program").RequiredArgument("PATH").StoreResult(&PathToBinPgm); opts.AddLongOption("params-txt", "path to file with text program params").RequiredArgument("PATH").StoreResult(&PathToTextParams); opts.AddLongOption("params-bin", "path to file with serialized program params").RequiredArgument("PATH").StoreResult(&PathToBinParams); - - ConfigureBaseLastGetopt(opts); - - TOptsParseResult res(&opts, argc, argv); - + + ConfigureBaseLastGetopt(opts); + + TOptsParseResult res(&opts, argc, argv); + ConfigureMsgBusLastGetopt(res, argc, argv); - if (!PathToBinPgm && !PathToTextPgm) { - ythrow yexception() << "One of the ['pgm-txt', 'pgm-bin'] options must be set."; - } - if (PathToBinPgm && PathToTextPgm) { - ythrow yexception() << "Only one of the ['pgm-txt', 'pgm-bin'] options must be set."; - } -} - -} // namespace NDriverClient -} // namespace NKikimr + if (!PathToBinPgm && !PathToTextPgm) { + ythrow yexception() << "One of the ['pgm-txt', 'pgm-bin'] options must be set."; + } + if (PathToBinPgm && PathToTextPgm) { + ythrow yexception() << "Only one of the ['pgm-txt', 'pgm-bin'] options must be set."; + } +} + +} // namespace NDriverClient +} // namespace NKikimr diff --git a/ydb/core/driver_lib/cli_utils/cli_scheme_cache_append.cpp b/ydb/core/driver_lib/cli_utils/cli_scheme_cache_append.cpp index 2a99cfa03c..00c4820330 100644 --- a/ydb/core/driver_lib/cli_utils/cli_scheme_cache_append.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_scheme_cache_append.cpp @@ -1 +1 @@ -#include "cli.h" +#include "cli.h" diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp index ff4351d8e4..f0a9315685 100644 --- a/ydb/core/driver_lib/run/main.cpp +++ b/ydb/core/driver_lib/run/main.cpp @@ -128,8 +128,8 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> return MainNodeByHost(cmdConf, argc, argv); case EDM_SCHEME_INITROOT: return NDriverClient::SchemeInitRoot(cmdConf, argc, argv); - case EDM_COMPILE_AND_EXEC_MINIKQL: - return NDriverClient::CompileAndExecMiniKQL(cmdConf, argc, argv); + case EDM_COMPILE_AND_EXEC_MINIKQL: + return NDriverClient::CompileAndExecMiniKQL(cmdConf, argc, argv); case EDM_TRACE: return NDriverClient::MessageBusTrace(cmdConf, argc, argv); case EDM_KEYVALUE_REQUEST: diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 4bbbbd9bd1..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -323,7 +323,7 @@ TKikimrRunner::TKikimrRunner(std::shared_ptr<TModuleFactories> factories) , PollerThreads(new NInterconnect::TPollerThreads) { } - + TKikimrRunner::~TKikimrRunner() { if (!!ActorSystem) { // Stop ActorSystem first, so no one actor can call any grpc stuff. @@ -1240,7 +1240,7 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers new NNodeTabletMonitor::TTabletStateClassifier(), new NNodeTabletMonitor::TTabletListRenderer())); } - + if (serviceMask.EnableViewerService) { sil->AddServiceInitializer(new TViewerInitializer(runConfig)); } @@ -1351,18 +1351,18 @@ void TKikimrRunner::KikimrStart() { if (!!Monitoring) { Monitoring->Start(); - } - + } + if (!!PollerThreads) { PollerThreads->Start(); } - ThreadSigmask(SIG_BLOCK); + ThreadSigmask(SIG_BLOCK); if (ActorSystem) { RegisterBaseTagForMemoryProfiling(ActorSystem.Get()); ActorSystem->Start(); } - + for (auto& server : GRpcServers) { if (server.second) { server.second->Start(); diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index 50f4d8d477..df7cda5980 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -157,7 +157,7 @@ message TResponse { ]} ]; // Program "exit code": see NMiniKQL::IEngine::EStatus. - optional NKikimrMiniKQL.TResult ExecutionEngineEvaluatedResponse = 103; + optional NKikimrMiniKQL.TResult ExecutionEngineEvaluatedResponse = 103; optional bytes MiniKQLErrors = 104; optional bytes DataShardErrors = 105; @@ -165,10 +165,10 @@ message TResponse { repeated bytes UnresolvedKeys = 107; // text for each key optional NKikimrQueryStats.TTxStats TxStats = 111; - optional NKikimrTxUserProxy.TMiniKQLCompileResults MiniKQLCompileResults = 108; + optional NKikimrTxUserProxy.TMiniKQLCompileResults MiniKQLCompileResults = 108; optional bool HadFollowerReads = 109; optional bytes SerializedReadTableResponse = 110; // Serialized Ydb.Db.TResultSet - + optional uint32 BSAssignedID = 40; optional NKikimrTxUserProxy.TTxProxyTimings ProxyTimings = 50; diff --git a/ydb/core/protos/tx_proxy.proto b/ydb/core/protos/tx_proxy.proto index 725de01865..dedd2f9cbb 100644 --- a/ydb/core/protos/tx_proxy.proto +++ b/ydb/core/protos/tx_proxy.proto @@ -118,26 +118,26 @@ message TCommitWritesTx { optional uint64 WriteTxId = 2; } -message TMiniKQLTransaction { - message TMiniKQLProgram { - // Server will use Text, if not set -- Bin. - optional string Text = 1; // Text program. - optional bytes Bin = 2; // Serialized program. - optional bytes ResultDocScheme = 3; // [opt] Will be applied to result document for program. - }; - - message TMiniKQLParams { - // Server will use Text, if not set -- Bin. - optional string Text = 1; // Text params. - optional bytes Bin = 2; // Serialized params. +message TMiniKQLTransaction { + message TMiniKQLProgram { + // Server will use Text, if not set -- Bin. + optional string Text = 1; // Text program. + optional bytes Bin = 2; // Serialized program. + optional bytes ResultDocScheme = 3; // [opt] Will be applied to result document for program. + }; + + message TMiniKQLParams { + // Server will use Text, if not set -- Bin. + optional string Text = 1; // Text params. + optional bytes Bin = 2; // Serialized params. optional NKikimrMiniKQL.TParams Proto = 3; - }; - - enum EMode { - COMPILE_AND_EXEC = 1; - COMPILE = 2; - }; - + }; + + enum EMode { + COMPILE_AND_EXEC = 1; + COMPILE = 2; + }; + message TLimits { optional uint32 AffectedShardsLimit = 1; optional uint32 ReadsetCountLimit = 2; @@ -145,12 +145,12 @@ message TMiniKQLTransaction { optional uint64 TotalReadSizeLimitBytes = 4; }; - optional EMode Mode = 1 [default = COMPILE_AND_EXEC]; - optional TMiniKQLProgram Program = 2; - optional TMiniKQLParams Params = 3; - - optional bool EvaluateResultType = 50 [default = true]; - optional bool EvaluateResultValue = 51 [default = true]; + optional EMode Mode = 1 [default = COMPILE_AND_EXEC]; + optional TMiniKQLProgram Program = 2; + optional TMiniKQLParams Params = 3; + + optional bool EvaluateResultType = 50 [default = true]; + optional bool EvaluateResultValue = 51 [default = true]; optional bool FlatMKQL = 52 [default = true]; optional bool LlvmRuntime = 53; optional uint64 PerShardKeysSizeLimitBytes = 54; @@ -158,17 +158,17 @@ message TMiniKQLTransaction { optional TLimits Limits = 56; optional uint64 SnapshotStep = 9; optional uint64 SnapshotTxId = 10; -}; - -message TMiniKQLCompileResults { - optional bytes CompiledProgram = 1; +}; + +message TMiniKQLCompileResults { + optional bytes CompiledProgram = 1; repeated Ydb.Issue.IssueMessage ProgramCompileErrors = 2; - optional bytes CompiledParams = 20; + optional bytes CompiledParams = 20; repeated Ydb.Issue.IssueMessage ParamsCompileErrors = 21; -}; - +}; + message TTransaction { - optional TMiniKQLTransaction MiniKQLTransaction = 4; + optional TMiniKQLTransaction MiniKQLTransaction = 4; optional uint32 Flags = 6; // See ydb/core/tx/tx_datashard.h NKikimr::NTxDataShard::TTxFlags optional string UserRequestId = 8; @@ -224,7 +224,7 @@ message TEvProposeTransactionStatus { repeated fixed64 ComplainingDataShards = 15; repeated bytes UnresolvedKeys = 16; // text - optional TMiniKQLCompileResults MiniKQLCompileResults = 17; + optional TMiniKQLCompileResults MiniKQLCompileResults = 17; optional NKikimrMiniKQL.TResult ExecutionEngineEvaluatedResponse = 18; optional NKikimrQueryStats.TTxStats TxStats = 19; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 3beb6f6536..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -623,7 +623,7 @@ namespace Tests { TActorId compileServiceId = Runtime->Register(compileService, nodeIdx, Runtime->GetAppData(nodeIdx).SystemPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(MakeMiniKQLCompileServiceID(), compileServiceId, nodeIdx); } - + { IActor* longTxService = NLongTxService::CreateLongTxService(); TActorId longTxServiceId = Runtime->Register(longTxService, nodeIdx); @@ -913,7 +913,7 @@ namespace Tests { ClientConfig.Port = serverSetup.Port; ClientConfig.BusSessionConfig.TotalTimeout = Max<int>() / 2; ClientConfig.BusSessionConfig.ConnectTimeout = ConnectTimeoutMilliSeconds; - ClientConfig.BusSessionConfig.NumRetries = 10; + ClientConfig.BusSessionConfig.NumRetries = 10; Client.reset(new NMsgBusProxy::TMsgBusClient(ClientConfig)); Client->Init(); } diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 3098798ea3..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -1,6 +1,6 @@ #pragma once -#include "tablet_helpers.h" - +#include "tablet_helpers.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/subdomain.h> #include <ydb/core/base/tablet_types.h> @@ -284,8 +284,8 @@ namespace Tests { const NScheme::TTypeRegistry& GetTypeRegistry() const; const NScheme::TTypeMetadataRegistry& GetTypeMetadataRegistry() const; const NMiniKQL::IFunctionRegistry& GetFunctionRegistry() const; - - template <typename T> + + template <typename T> void PrepareRequest(TAutoPtr<T>&) {} void PrepareRequest(TAutoPtr<NMsgBusProxy::TBusRequest>& request) { @@ -319,19 +319,19 @@ namespace Tests { const ui64 finishTimeMs = TInstant::Now().MilliSeconds() + TIME_LIMIT_MS; PrepareRequest(msgHolder); while (TInstant::Now().MilliSeconds() < finishTimeMs) { - T* msgCopy(new T()); - msgCopy->Record = msgHolder->Record; - msgbusStatus = Client->SyncCall(msgCopy, reply); - if (msgbusStatus == NBus::MESSAGE_CONNECT_FAILED) { + T* msgCopy(new T()); + msgCopy->Record = msgHolder->Record; + msgbusStatus = Client->SyncCall(msgCopy, reply); + if (msgbusStatus == NBus::MESSAGE_CONNECT_FAILED) { Sleep(ITERATION_DURATION); - continue; - } else { - break; - } - } - return msgbusStatus; - } - + continue; + } else { + break; + } + } + return msgbusStatus; + } + static ui64 GetPatchedSchemeRoot(ui64 schemeRoot, ui32 domain, bool supportsRedirect); void WaitRootIsUp(const TString& root); TAutoPtr<NBus::TBusMessage> InitRootSchemeWithReply(const TString& root); @@ -341,7 +341,7 @@ namespace Tests { void ExecuteTraceCommand(NKikimrClient::TMessageBusTraceRequest::ECommand command, const TString &path = TString()); TString StartTrace(const TString &path); void StopTrace(); - + // Flat DB operations NMsgBusProxy::EResponseStatus WaitCreateTx(TTestActorRuntime* runtime, const TString& path, TDuration timeout); NMsgBusProxy::EResponseStatus MkDir(const TString& parent, const TString& name, const TApplyIf& applyIf = {}); diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 9b63b7ea60..af48b42d60 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -1,10 +1,10 @@ #include "proxy.h" - + #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/balance_coverage/balance_coverage_builder.h> #include <ydb/core/tx/tx_processing.h> - + #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/base/appdata.h> #include <ydb/core/base/blobstorage.h> @@ -19,7 +19,7 @@ #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/scheme/scheme_types_defs.h> #include <ydb/core/base/row_version.h> - + #include <ydb/library/yql/minikql/mkql_type_ops.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/yql_issue_manager.h> @@ -1274,8 +1274,8 @@ void TDataReq::Handle(TEvTxProxyReq::TEvMakeRequest::TPtr &ev, const TActorConte NCpuTime::TCpuTimer timer(CpuTime); if (txbody.HasMiniKQLTransaction()) { - const auto& mkqlTxBody = txbody.GetMiniKQLTransaction(); - + const auto& mkqlTxBody = txbody.GetMiniKQLTransaction(); + const TAppData* appData = AppData(ctx); const auto functionRegistry = appData->FunctionRegistry; diff --git a/ydb/library/yql/ast/ut/ya.make b/ydb/library/yql/ast/ut/ya.make index 827dc46120..0a9b5944f2 100644 --- a/ydb/library/yql/ast/ut/ya.make +++ b/ydb/library/yql/ast/ut/ya.make @@ -1,18 +1,18 @@ UNITTEST_FOR(ydb/library/yql/ast) - + OWNER(vvvv g:kikimr) - + FORK_SUBTESTS() -SRCS( - yql_ast_ut.cpp +SRCS( + yql_ast_ut.cpp yql_expr_check_args_ut.cpp yql_expr_builder_ut.cpp - yql_expr_ut.cpp - yql_type_string_ut.cpp -) - -PEERDIR( -) - -END() + yql_expr_ut.cpp + yql_type_string_ut.cpp +) + +PEERDIR( +) + +END() diff --git a/ydb/library/yql/ast/ya.make b/ydb/library/yql/ast/ya.make index 6fdf7bf9c3..2d55860c89 100644 --- a/ydb/library/yql/ast/ya.make +++ b/ydb/library/yql/ast/ya.make @@ -1,36 +1,36 @@ -LIBRARY() - +LIBRARY() + OWNER( vvvv g:kikimr g:yql g:yql_ydb_core ) - -SRCS( - yql_ast.cpp - yql_ast.h + +SRCS( + yql_ast.cpp + yql_ast.h yql_constraint.cpp yql_constraint.h - yql_ast_annotation.cpp - yql_ast_annotation.h + yql_ast_annotation.cpp + yql_ast_annotation.h yql_ast_escaping.cpp yql_ast_escaping.h - yql_errors.cpp - yql_errors.h - yql_expr.cpp - yql_expr.h + yql_errors.cpp + yql_errors.h + yql_expr.cpp + yql_expr.h yql_expr_builder.cpp yql_expr_builder.h - yql_expr_types.cpp - yql_expr_types.h - yql_gc_nodes.cpp - yql_gc_nodes.h - yql_type_string.cpp - yql_type_string.h -) - -PEERDIR( + yql_expr_types.cpp + yql_expr_types.h + yql_gc_nodes.cpp + yql_gc_nodes.h + yql_type_string.cpp + yql_type_string.h +) + +PEERDIR( library/cpp/colorizer library/cpp/containers/sorted_vector library/cpp/containers/stack_vector @@ -41,9 +41,9 @@ PEERDIR( ydb/library/yql/public/udf ydb/library/yql/utils ydb/library/yql/core/issue -) - -END() +) + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/yql/ast/yql_ast.cpp b/ydb/library/yql/ast/yql_ast.cpp index 74d08b6424..16e521d901 100644 --- a/ydb/library/yql/ast/yql_ast.cpp +++ b/ydb/library/yql/ast/yql_ast.cpp @@ -1,318 +1,318 @@ -#include "yql_ast.h" +#include "yql_ast.h" #include "yql_ast_escaping.h" - -#include <util/string/builder.h> -#include <util/system/compiler.h> + +#include <util/string/builder.h> +#include <util/system/compiler.h> #include <library/cpp/containers/stack_vector/stack_vec.h> #include <ydb/library/yql/utils/utf8.h> - -#include <cstdlib> - -namespace NYql { - -namespace { - - inline bool IsWhitespace(char c) { - return c == ' ' || c == '\n' || c == '\r' || c == '\t'; - } - - inline bool IsListStart(char c) { return c == '('; } - inline bool IsListEnd(char c) { return c == ')'; } - inline bool IsCommentStart(char c) { return c == '#'; } - inline bool IsQuoteStart(char c) { return c == '\''; } - inline bool IsStringStart(char c) { return c == '"'; } - inline bool IsHexStringStart(char c) { return c == 'x'; } - inline bool IsMultilineStringStart(char c) { return c == '@'; } - - inline bool NeedEscaping(const TStringBuf& str) { - for (char ch: str) { - if (IsWhitespace(ch) || IsListStart(ch) || - IsListEnd(ch) || IsCommentStart(ch) || - IsQuoteStart(ch) || IsStringStart(ch) || - !isprint(ch & 0xff)) - { - return true; - } - } - + +#include <cstdlib> + +namespace NYql { + +namespace { + + inline bool IsWhitespace(char c) { + return c == ' ' || c == '\n' || c == '\r' || c == '\t'; + } + + inline bool IsListStart(char c) { return c == '('; } + inline bool IsListEnd(char c) { return c == ')'; } + inline bool IsCommentStart(char c) { return c == '#'; } + inline bool IsQuoteStart(char c) { return c == '\''; } + inline bool IsStringStart(char c) { return c == '"'; } + inline bool IsHexStringStart(char c) { return c == 'x'; } + inline bool IsMultilineStringStart(char c) { return c == '@'; } + + inline bool NeedEscaping(const TStringBuf& str) { + for (char ch: str) { + if (IsWhitespace(ch) || IsListStart(ch) || + IsListEnd(ch) || IsCommentStart(ch) || + IsQuoteStart(ch) || IsStringStart(ch) || + !isprint(ch & 0xff)) + { + return true; + } + } + return str.empty(); - } - - /////////////////////////////////////////////////////////////////////////// - // TAstParser - /////////////////////////////////////////////////////////////////////////// - class TAstParserContext { - public: + } + + /////////////////////////////////////////////////////////////////////////// + // TAstParser + /////////////////////////////////////////////////////////////////////////// + class TAstParserContext { + public: inline TAstParserContext(const TStringBuf& str, TMemoryPool* externalPool, const TString& file) - : Str_(str) + : Str_(str) , Position_(1, 1, file) - , Offset_(0) - , Pool_(externalPool) - { - if (!Pool_) { + , Offset_(0) + , Pool_(externalPool) + { + if (!Pool_) { InnerPool_ = std::make_unique<TMemoryPool>(4096); Pool_ = InnerPool_.get(); - } - } - - inline char Peek() const { + } + } + + inline char Peek() const { Y_VERIFY(!AtEnd()); - return Str_[Offset_]; - } - - inline bool AtEnd() const { + return Str_[Offset_]; + } + + inline bool AtEnd() const { return Str_.size() == Offset_; - } - - inline char Next() { + } + + inline char Next() { Y_VERIFY(!AtEnd()); - char ch = Str_[Offset_]; - if (ch == '\n') { - ++Position_.Row; - Position_.Column = 1; - } else { - ++Position_.Column; - } - - ++Offset_; - return ch; - } - - // stops right afetr stopChar - inline void SeekTo(char stopChar) { - while (!AtEnd() && Next() != stopChar) { - // empty loop - } - } - - inline TStringBuf GetToken(ui32 begin, ui32 end) { + char ch = Str_[Offset_]; + if (ch == '\n') { + ++Position_.Row; + Position_.Column = 1; + } else { + ++Position_.Column; + } + + ++Offset_; + return ch; + } + + // stops right afetr stopChar + inline void SeekTo(char stopChar) { + while (!AtEnd() && Next() != stopChar) { + // empty loop + } + } + + inline TStringBuf GetToken(ui32 begin, ui32 end) { Y_VERIFY(end >= begin); - return Str_.SubString(begin, end - begin); - } - - inline bool IsAtomEnded() { - if (AtEnd()) { - return true; - } - char c = Peek(); - return IsWhitespace(c) || IsListStart(c) || IsListEnd(c); - } - - inline const TStringBuf& Str() const { return Str_; } - inline ui32 Offset() const { return Offset_; } - inline const TPosition& Position() const { return Position_; } - inline TMemoryPool& Pool() { return *Pool_; } + return Str_.SubString(begin, end - begin); + } + + inline bool IsAtomEnded() { + if (AtEnd()) { + return true; + } + char c = Peek(); + return IsWhitespace(c) || IsListStart(c) || IsListEnd(c); + } + + inline const TStringBuf& Str() const { return Str_; } + inline ui32 Offset() const { return Offset_; } + inline const TPosition& Position() const { return Position_; } + inline TMemoryPool& Pool() { return *Pool_; } inline std::unique_ptr<TMemoryPool>&& InnerPool() { return std::move(InnerPool_); } - - private: - TStringBuf Str_; - TPosition Position_; - ui32 Offset_; - TMemoryPool* Pool_; + + private: + TStringBuf Str_; + TPosition Position_; + ui32 Offset_; + TMemoryPool* Pool_; std::unique_ptr<TMemoryPool> InnerPool_; - }; - - /////////////////////////////////////////////////////////////////////////// - // TAstParser - /////////////////////////////////////////////////////////////////////////// - class TAstParser { - public: + }; + + /////////////////////////////////////////////////////////////////////////// + // TAstParser + /////////////////////////////////////////////////////////////////////////// + class TAstParser { + public: TAstParser(const TStringBuf& str, TMemoryPool* externalPool, const TString& file) : Ctx_(str, externalPool, file) - { - } - - TAstParseResult Parse() { + { + } + + TAstParseResult Parse() { TAstNode* root = nullptr; if (!IsUtf8(Ctx_.Str())) { AddError("Invalid UTF8 input"); } else { root = ParseList(0U); - + SkipSpace(); if (!Ctx_.AtEnd()) { AddError("Unexpected symbols after end of root list"); } - } - - TAstParseResult result; + } + + TAstParseResult result; if (!Issues_.Empty()) { result.Issues = std::move(Issues_); - } else { - result.Root = root; - result.Pool = Ctx_.InnerPool(); - } - return result; - } - - private: + } else { + result.Root = root; + result.Pool = Ctx_.InnerPool(); + } + return result; + } + + private: inline void AddError(const TString& message) { Issues_.AddIssue(Ctx_.Position(), message); - } - - inline void SkipComment() { - Ctx_.SeekTo('\n'); - } - - void SkipSpace() { - while (!Ctx_.AtEnd()) { - char c = Ctx_.Peek(); - if (IsWhitespace(c)) { - Ctx_.Next(); - continue; - } - - if (IsCommentStart(c)) { - SkipComment(); - continue; - } - - break; - } - } - + } + + inline void SkipComment() { + Ctx_.SeekTo('\n'); + } + + void SkipSpace() { + while (!Ctx_.AtEnd()) { + char c = Ctx_.Peek(); + if (IsWhitespace(c)) { + Ctx_.Next(); + continue; + } + + if (IsCommentStart(c)) { + SkipComment(); + continue; + } + + break; + } + } + TAstNode* ParseList(size_t level) { if (level >= 1000U) { AddError("Too deep graph!"); return nullptr; } - SkipSpace(); - - if (Ctx_.AtEnd()) { - AddError("Unexpected end"); - return nullptr; - } - - if (!IsListStart(Ctx_.Peek())) { - AddError("Expected ("); - return nullptr; - } - - Ctx_.Next(); - - TSmallVec<TAstNode*> children; - auto listPos = Ctx_.Position(); - while (true) { - SkipSpace(); - - if (Ctx_.AtEnd()) { - AddError("Expected )"); - return nullptr; - } - - if (IsListEnd(Ctx_.Peek())) { - Ctx_.Next(); - return TAstNode::NewList(listPos, children.data(), children.size(), Ctx_.Pool()); - } - + SkipSpace(); + + if (Ctx_.AtEnd()) { + AddError("Unexpected end"); + return nullptr; + } + + if (!IsListStart(Ctx_.Peek())) { + AddError("Expected ("); + return nullptr; + } + + Ctx_.Next(); + + TSmallVec<TAstNode*> children; + auto listPos = Ctx_.Position(); + while (true) { + SkipSpace(); + + if (Ctx_.AtEnd()) { + AddError("Expected )"); + return nullptr; + } + + if (IsListEnd(Ctx_.Peek())) { + Ctx_.Next(); + return TAstNode::NewList(listPos, children.data(), children.size(), Ctx_.Pool()); + } + TAstNode* elem = ParseElement(level); - if (!elem) - return nullptr; - - children.push_back(elem); - } - } - + if (!elem) + return nullptr; + + children.push_back(elem); + } + } + TAstNode* ParseElement(size_t level) { - if (Ctx_.AtEnd()) { - AddError("Expected element"); - return nullptr; - } - - char c = Ctx_.Peek(); - if (IsQuoteStart(c)) { - auto resPosition = Ctx_.Position(); - Ctx_.Next(); - - char ch = Ctx_.Peek(); - if (Ctx_.AtEnd() || IsWhitespace(ch) || IsCommentStart(ch) || - IsListEnd(ch)) - { - AddError("Expected quotation"); - return nullptr; - } - - TAstNode* content = IsListStart(ch) + if (Ctx_.AtEnd()) { + AddError("Expected element"); + return nullptr; + } + + char c = Ctx_.Peek(); + if (IsQuoteStart(c)) { + auto resPosition = Ctx_.Position(); + Ctx_.Next(); + + char ch = Ctx_.Peek(); + if (Ctx_.AtEnd() || IsWhitespace(ch) || IsCommentStart(ch) || + IsListEnd(ch)) + { + AddError("Expected quotation"); + return nullptr; + } + + TAstNode* content = IsListStart(ch) ? ParseList(++level) - : ParseAtom(); - if (!content) - return nullptr; - - return TAstNode::NewList(resPosition, Ctx_.Pool(), &TAstNode::QuoteAtom, content); - } - - if (IsListStart(c)) + : ParseAtom(); + if (!content) + return nullptr; + + return TAstNode::NewList(resPosition, Ctx_.Pool(), &TAstNode::QuoteAtom, content); + } + + if (IsListStart(c)) return ParseList(++level); - - return ParseAtom(); - } - - TAstNode* ParseAtom() { - if (Ctx_.AtEnd()) { - AddError("Expected atom"); - return nullptr; - } - - auto resPosition = Ctx_.Position(); - ui32 atomStart = Ctx_.Offset(); - - while (true) { - char c = Ctx_.Peek(); - // special symbols = 0x20, 0x0a, 0x0d, 0x09 space - // 0x22, 0x23, 0x28, 0x29 "#() - // 0x27 ' - // special symbols = 0x40, 0x78 @x - // &0x3f = 0x00,0x38 -#define MASK(x) (1ull << ui64(x)) - const ui64 mask1 = MASK(0x20) | MASK(0x0a) | MASK(0x0d) - | MASK(0x09) | MASK(0x22) | MASK(0x23) | MASK(0x28) | MASK(0x29) | MASK(0x27); - const ui64 mask2 = MASK(0x00) | MASK(0x38); -#undef MASK - if (!(c & 0x80) && ((1ull << (c & 0x3f)) & (c <= 0x3f ? mask1 : mask2))) { - if (IsWhitespace(c) || IsListStart(c) || IsListEnd(c)) - break; - - if (IsCommentStart(c)) { - AddError("Unexpected comment"); - return nullptr; - } - - if (IsQuoteStart(c)) { - AddError("Unexpected quotation"); - return nullptr; - } - - // multiline starts with '@@' - if (IsMultilineStringStart(c)) { - Ctx_.Next(); - if (Ctx_.AtEnd()) break; - - if (!IsMultilineStringStart(Ctx_.Peek())) { - continue; - } - + + return ParseAtom(); + } + + TAstNode* ParseAtom() { + if (Ctx_.AtEnd()) { + AddError("Expected atom"); + return nullptr; + } + + auto resPosition = Ctx_.Position(); + ui32 atomStart = Ctx_.Offset(); + + while (true) { + char c = Ctx_.Peek(); + // special symbols = 0x20, 0x0a, 0x0d, 0x09 space + // 0x22, 0x23, 0x28, 0x29 "#() + // 0x27 ' + // special symbols = 0x40, 0x78 @x + // &0x3f = 0x00,0x38 +#define MASK(x) (1ull << ui64(x)) + const ui64 mask1 = MASK(0x20) | MASK(0x0a) | MASK(0x0d) + | MASK(0x09) | MASK(0x22) | MASK(0x23) | MASK(0x28) | MASK(0x29) | MASK(0x27); + const ui64 mask2 = MASK(0x00) | MASK(0x38); +#undef MASK + if (!(c & 0x80) && ((1ull << (c & 0x3f)) & (c <= 0x3f ? mask1 : mask2))) { + if (IsWhitespace(c) || IsListStart(c) || IsListEnd(c)) + break; + + if (IsCommentStart(c)) { + AddError("Unexpected comment"); + return nullptr; + } + + if (IsQuoteStart(c)) { + AddError("Unexpected quotation"); + return nullptr; + } + + // multiline starts with '@@' + if (IsMultilineStringStart(c)) { + Ctx_.Next(); + if (Ctx_.AtEnd()) break; + + if (!IsMultilineStringStart(Ctx_.Peek())) { + continue; + } + TString token; - if (!TryParseMultilineToken(token)) { - return nullptr; - } - - if (!Ctx_.IsAtomEnded()) { - AddError("Unexpected end of @@"); - return nullptr; - } - - return TAstNode::NewAtom(resPosition, token, Ctx_.Pool(), TNodeFlags::MultilineContent); - } - // hex string starts with 'x"' - else if (IsHexStringStart(c)) { + if (!TryParseMultilineToken(token)) { + return nullptr; + } + + if (!Ctx_.IsAtomEnded()) { + AddError("Unexpected end of @@"); + return nullptr; + } + + return TAstNode::NewAtom(resPosition, token, Ctx_.Pool(), TNodeFlags::MultilineContent); + } + // hex string starts with 'x"' + else if (IsHexStringStart(c)) { Ctx_.Next(); // skip 'x' - if (Ctx_.AtEnd()) break; - - if (!IsStringStart(Ctx_.Peek())) { - continue; - } - + if (Ctx_.AtEnd()) break; + + if (!IsStringStart(Ctx_.Peek())) { + continue; + } + Ctx_.Next(); // skip first '"' size_t readBytes = 0; @@ -328,23 +328,23 @@ namespace { if (unescapeResult != EUnescapeResult::OK) { AddError(TString(UnescapeResultToString(unescapeResult))); - return nullptr; - } - + return nullptr; + } + Ctx_.Next(); // skip last '"' - if (!Ctx_.IsAtomEnded()) { - AddError("Unexpected end of \""); - return nullptr; - } - + if (!Ctx_.IsAtomEnded()) { + AddError("Unexpected end of \""); + return nullptr; + } + return TAstNode::NewAtom(resPosition, ss.Str(), Ctx_.Pool(), TNodeFlags::BinaryContent); - } - else if (IsStringStart(c)) { - if (Ctx_.Offset() != atomStart) { - AddError("Unexpected \""); - return nullptr; - } - + } + else if (IsStringStart(c)) { + if (Ctx_.Offset() != atomStart) { + AddError("Unexpected \""); + return nullptr; + } + Ctx_.Next(); // skip first '"' size_t readBytes = 0; @@ -360,231 +360,231 @@ namespace { if (unescapeResult != EUnescapeResult::OK) { AddError(TString(UnescapeResultToString(unescapeResult))); - return nullptr; - } - - if (!Ctx_.IsAtomEnded()) { - AddError("Unexpected end of \""); - return nullptr; - } - + return nullptr; + } + + if (!Ctx_.IsAtomEnded()) { + AddError("Unexpected end of \""); + return nullptr; + } + return TAstNode::NewAtom(resPosition, ss.Str(), Ctx_.Pool(), TNodeFlags::ArbitraryContent); - } - } - - Ctx_.Next(); - if (Ctx_.AtEnd()) { - break; - } - } - - return TAstNode::NewAtom(resPosition, Ctx_.GetToken(atomStart, Ctx_.Offset()), Ctx_.Pool()); - } - + } + } + + Ctx_.Next(); + if (Ctx_.AtEnd()) { + break; + } + } + + return TAstNode::NewAtom(resPosition, Ctx_.GetToken(atomStart, Ctx_.Offset()), Ctx_.Pool()); + } + bool TryParseMultilineToken(TString& token) { - Ctx_.Next(); // skip second '@' - - ui32 start = Ctx_.Offset(); - while (true) { - Ctx_.SeekTo('@'); - - if (Ctx_.AtEnd()) { - AddError("Unexpected multiline atom end"); - return false; - } - - ui32 count = 1; // we seek to first '@' - while (!Ctx_.AtEnd() && Ctx_.Peek() == '@') { - count++; - Ctx_.Next(); - if (count == 4) { - // Reduce each four '@' to two - token.append(Ctx_.GetToken(start, Ctx_.Offset() - 2)); - start = Ctx_.Offset(); - count = 0; - } - } - if (count >= 2) { - break; - } - } - - // two '@@' at the end - token.append(Ctx_.GetToken(start, Ctx_.Offset() - 2)); - return true; - } - - private: - TAstParserContext Ctx_; + Ctx_.Next(); // skip second '@' + + ui32 start = Ctx_.Offset(); + while (true) { + Ctx_.SeekTo('@'); + + if (Ctx_.AtEnd()) { + AddError("Unexpected multiline atom end"); + return false; + } + + ui32 count = 1; // we seek to first '@' + while (!Ctx_.AtEnd() && Ctx_.Peek() == '@') { + count++; + Ctx_.Next(); + if (count == 4) { + // Reduce each four '@' to two + token.append(Ctx_.GetToken(start, Ctx_.Offset() - 2)); + start = Ctx_.Offset(); + count = 0; + } + } + if (count >= 2) { + break; + } + } + + // two '@@' at the end + token.append(Ctx_.GetToken(start, Ctx_.Offset() - 2)); + return true; + } + + private: + TAstParserContext Ctx_; TIssues Issues_; - }; - - /////////////////////////////////////////////////////////////////////////// - // ast node printing functions - /////////////////////////////////////////////////////////////////////////// - - inline bool IsQuoteNode(const TAstNode& node) { - return node.GetChildrenCount() == 2 - && node.GetChild(0)->GetType() == TAstNode::Atom + }; + + /////////////////////////////////////////////////////////////////////////// + // ast node printing functions + /////////////////////////////////////////////////////////////////////////// + + inline bool IsQuoteNode(const TAstNode& node) { + return node.GetChildrenCount() == 2 + && node.GetChild(0)->GetType() == TAstNode::Atom && node.GetChild(0)->GetContent() == TStringBuf("quote"); - } - - inline bool IsBlockNode(const TAstNode& node) { - return node.GetChildrenCount() == 2 - && node.GetChild(0)->GetType() == TAstNode::Atom + } + + inline bool IsBlockNode(const TAstNode& node) { + return node.GetChildrenCount() == 2 + && node.GetChild(0)->GetType() == TAstNode::Atom && node.GetChild(0)->GetContent() == TStringBuf("block"); - } - + } + Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { char* whitespaces = reinterpret_cast<char*>(alloca(indentation)); - memset(whitespaces, ' ', indentation); - out.Write(whitespaces, indentation); - } - + memset(whitespaces, ' ', indentation); + out.Write(whitespaces, indentation); + } + void MultilineAtomPrint(IOutputStream& out, const TStringBuf& str) { out << TStringBuf("@@"); - size_t idx = str.find('@'); + size_t idx = str.find('@'); if (idx == TString::npos) { - out << str; - } else { + out << str; + } else { const char* begin = str.data(); - do { - ui32 count = 0; + do { + ui32 count = 0; for (; idx < str.length() && str[idx] == '@'; ++idx) { ++count; } - - if (count % 2 == 0) { + + if (count % 2 == 0) { out.Write(begin, idx - (begin - str.data()) - count); begin = str.data() + idx; - + while (count--) { out.Write(TStringBuf("@@")); } - } - idx = str.find('@', idx); + } + idx = str.find('@', idx); } while (idx != TString::npos); - out.Write(begin, str.end() - begin); - } + out.Write(begin, str.end() - begin); + } out << TStringBuf("@@"); - } - + } + void PrintNode(IOutputStream& out, const TAstNode& node) { - if (node.GetType() == TAstNode::Atom) { - if (node.GetFlags() & TNodeFlags::ArbitraryContent) { + if (node.GetType() == TAstNode::Atom) { + if (node.GetFlags() & TNodeFlags::ArbitraryContent) { EscapeArbitraryAtom(node.GetContent(), '"', &out); - } else if (node.GetFlags() & TNodeFlags::BinaryContent) { + } else if (node.GetFlags() & TNodeFlags::BinaryContent) { EscapeBinaryAtom(node.GetContent(), '"', &out); - } else if (node.GetFlags() & TNodeFlags::MultilineContent) { - MultilineAtomPrint(out, node.GetContent()); + } else if (node.GetFlags() & TNodeFlags::MultilineContent) { + MultilineAtomPrint(out, node.GetContent()); } else if (node.GetContent().empty()) { EscapeArbitraryAtom(node.GetContent(), '"', &out); - } else { - out << node.GetContent(); - } - } else if (node.GetType() == TAstNode::List) { - if (!node.GetChildrenCount()) { + } else { + out << node.GetContent(); + } + } else if (node.GetType() == TAstNode::List) { + if (!node.GetChildrenCount()) { out << TStringBuf("()"); - } else if (IsQuoteNode(node)) { - out << '\''; - PrintNode(out, *node.GetChild(1)); - } else { - out << '('; - ui32 index = 0; - while (true) { - PrintNode(out, *node.GetChild(index)); - ++index; - if (index == node.GetChildrenCount()) break; - out << ' '; - } - out << ')'; - } - } - } - - void PrettyPrintNode( + } else if (IsQuoteNode(node)) { + out << '\''; + PrintNode(out, *node.GetChild(1)); + } else { + out << '('; + ui32 index = 0; + while (true) { + PrintNode(out, *node.GetChild(index)); + ++index; + if (index == node.GetChildrenCount()) break; + out << ' '; + } + out << ')'; + } + } + } + + void PrettyPrintNode( IOutputStream& out, const TAstNode& node, - i32 indent, i32 blockLevel, i32 localIndent, ui32 flags) - { - if (!(flags & TAstPrintFlags::PerLine)) { - Indent(out, indent * 2); - } else if (localIndent == 1) { - Indent(out, blockLevel * 2); - } - - bool isQuote = false; - if (node.GetType() == TAstNode::Atom) { - if (node.GetFlags() & TNodeFlags::ArbitraryContent) { - if ((flags & TAstPrintFlags::AdaptArbitraryContent) && - !NeedEscaping(node.GetContent())) - { - out << node.GetContent(); - } else { + i32 indent, i32 blockLevel, i32 localIndent, ui32 flags) + { + if (!(flags & TAstPrintFlags::PerLine)) { + Indent(out, indent * 2); + } else if (localIndent == 1) { + Indent(out, blockLevel * 2); + } + + bool isQuote = false; + if (node.GetType() == TAstNode::Atom) { + if (node.GetFlags() & TNodeFlags::ArbitraryContent) { + if ((flags & TAstPrintFlags::AdaptArbitraryContent) && + !NeedEscaping(node.GetContent())) + { + out << node.GetContent(); + } else { EscapeArbitraryAtom(node.GetContent(), '"', &out); - } - } else if (node.GetFlags() & TNodeFlags::BinaryContent) { + } + } else if (node.GetFlags() & TNodeFlags::BinaryContent) { EscapeBinaryAtom(node.GetContent(), '"', &out); - } else if (node.GetFlags() & TNodeFlags::MultilineContent) { - MultilineAtomPrint(out, node.GetContent()); - } else { + } else if (node.GetFlags() & TNodeFlags::MultilineContent) { + MultilineAtomPrint(out, node.GetContent()); + } else { if (((flags & TAstPrintFlags::AdaptArbitraryContent) && NeedEscaping(node.GetContent())) || node.GetContent().empty()) - { + { EscapeArbitraryAtom(node.GetContent(), '"', &out); - } else { - out << node.GetContent(); - } - } - } else if (node.GetType() == TAstNode::List) { - isQuote = IsQuoteNode(node); - if (isQuote && (flags & TAstPrintFlags::ShortQuote)) { - out << '\''; - if (localIndent == 0 || !(flags & TAstPrintFlags::PerLine)) { - out << Endl; - } - - PrettyPrintNode(out, *node.GetChild(1), indent + 1, blockLevel, localIndent + 1, flags); - } else { - out << '('; - if (localIndent == 0 || !(flags & TAstPrintFlags::PerLine)) { - out << Endl; - } - - bool isBlock = IsBlockNode(node); - for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { - if (localIndent > 0 && (index > 0) && (flags & TAstPrintFlags::PerLine)) { - out << ' '; - } - - if (isBlock && (index > 0)) { - PrettyPrintNode(out, *node.GetChild(index), indent + 1, blockLevel + 1, -1, flags); - } else { - PrettyPrintNode(out, *node.GetChild(index), indent + 1, blockLevel, localIndent + 1, flags); - } - } - } - - if (!isQuote || !(flags & TAstPrintFlags::ShortQuote)) { - if (!(flags & TAstPrintFlags::PerLine)) { - Indent(out, indent * 2); - } - + } else { + out << node.GetContent(); + } + } + } else if (node.GetType() == TAstNode::List) { + isQuote = IsQuoteNode(node); + if (isQuote && (flags & TAstPrintFlags::ShortQuote)) { + out << '\''; + if (localIndent == 0 || !(flags & TAstPrintFlags::PerLine)) { + out << Endl; + } + + PrettyPrintNode(out, *node.GetChild(1), indent + 1, blockLevel, localIndent + 1, flags); + } else { + out << '('; + if (localIndent == 0 || !(flags & TAstPrintFlags::PerLine)) { + out << Endl; + } + + bool isBlock = IsBlockNode(node); + for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { + if (localIndent > 0 && (index > 0) && (flags & TAstPrintFlags::PerLine)) { + out << ' '; + } + + if (isBlock && (index > 0)) { + PrettyPrintNode(out, *node.GetChild(index), indent + 1, blockLevel + 1, -1, flags); + } else { + PrettyPrintNode(out, *node.GetChild(index), indent + 1, blockLevel, localIndent + 1, flags); + } + } + } + + if (!isQuote || !(flags & TAstPrintFlags::ShortQuote)) { + if (!(flags & TAstPrintFlags::PerLine)) { + Indent(out, indent * 2); + } + if (localIndent == 0 && blockLevel > 0) { Indent(out, (blockLevel - 1) * 2); } - out << ')'; - } - } - - if (!isQuote || !(flags & TAstPrintFlags::ShortQuote)) { + out << ')'; + } + } + + if (!isQuote || !(flags & TAstPrintFlags::ShortQuote)) { if (localIndent > 0 || blockLevel == 0) { if (localIndent <= 1 || !(flags & TAstPrintFlags::PerLine)) { out << Endl; } - } - } - } + } + } + } void DestroyNode(TAstNode* node) { if (node->IsList()) { @@ -597,8 +597,8 @@ namespace { node->Destroy(); } } -} // namespace - +} // namespace + TAstParseResult::~TAstParseResult() { Destroy(); } @@ -628,34 +628,34 @@ void TAstParseResult::Destroy() { } TAstParseResult ParseAst(const TStringBuf& str, TMemoryPool* externalPool, const TString& file) -{ +{ TAstParser parser(str, externalPool, file); - return parser.Parse(); -} - + return parser.Parse(); +} + void TAstNode::PrintTo(IOutputStream& out) const { - PrintNode(out, *this); -} - + PrintNode(out, *this); +} + void TAstNode::PrettyPrintTo(IOutputStream& out, ui32 flags) const { - PrettyPrintNode(out, *this, 0, 0, 0, flags); -} - + PrettyPrintNode(out, *this, 0, 0, 0, flags); +} + TAstNode TAstNode::QuoteAtom(TPosition(0, 0), TStringBuf("quote"), TNodeFlags::Default); - -} // namespace NYql - -template<> + +} // namespace NYql + +template<> void Out<NYql::TAstNode::EType>(class IOutputStream &o, NYql::TAstNode::EType x) { -#define YQL_AST_NODE_TYPE_MAP_TO_STRING_IMPL(name, ...) \ - case ::NYql::TAstNode::name: \ - o << #name; \ - return; - - switch (x) { - YQL_AST_NODE_TYPE_MAP(YQL_AST_NODE_TYPE_MAP_TO_STRING_IMPL) - default: - o << static_cast<int>(x); - return; - } -} +#define YQL_AST_NODE_TYPE_MAP_TO_STRING_IMPL(name, ...) \ + case ::NYql::TAstNode::name: \ + o << #name; \ + return; + + switch (x) { + YQL_AST_NODE_TYPE_MAP(YQL_AST_NODE_TYPE_MAP_TO_STRING_IMPL) + default: + o << static_cast<int>(x); + return; + } +} diff --git a/ydb/library/yql/ast/yql_ast.h b/ydb/library/yql/ast/yql_ast.h index ed8f59880a..887b53dbc5 100644 --- a/ydb/library/yql/ast/yql_ast.h +++ b/ydb/library/yql/ast/yql_ast.h @@ -1,247 +1,247 @@ -#pragma once - -#include "yql_errors.h" - +#pragma once + +#include "yql_errors.h" + #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/stream/output.h> -#include <util/stream/str.h> -#include <util/memory/pool.h> - -namespace NYql { - -struct TNodeFlags { +#include <util/generic/vector.h> +#include <util/stream/output.h> +#include <util/stream/str.h> +#include <util/memory/pool.h> + +namespace NYql { + +struct TNodeFlags { enum : ui16 { - Default = 0, - ArbitraryContent = 0x01, - BinaryContent = 0x02, - MultilineContent = 0x04, - }; + Default = 0, + ArbitraryContent = 0x01, + BinaryContent = 0x02, + MultilineContent = 0x04, + }; static constexpr ui32 FlagsMask = 0x07; // all flags should fit here -}; - -struct TAstNode { -#define YQL_AST_NODE_TYPE_MAP(xx) \ - xx(List, 0) \ - xx(Atom, 1) \ - - enum EType : ui32 { - YQL_AST_NODE_TYPE_MAP(ENUM_VALUE_GEN) - }; - - - static const ui32 SmallListCount = 2; - +}; + +struct TAstNode { +#define YQL_AST_NODE_TYPE_MAP(xx) \ + xx(List, 0) \ + xx(Atom, 1) \ + + enum EType : ui32 { + YQL_AST_NODE_TYPE_MAP(ENUM_VALUE_GEN) + }; + + + static const ui32 SmallListCount = 2; + void PrintTo(IOutputStream& out) const; void PrettyPrintTo(IOutputStream& out, ui32 prettyFlags) const; - + inline TString ToString() const { - TStringStream str; - PrintTo(str); - return str.Str(); - } - + TStringStream str; + PrintTo(str); + return str.Str(); + } + inline TString ToString(ui32 prettyFlags) const { - TStringStream str; - PrettyPrintTo(str, prettyFlags); - return str.Str(); - } - - inline EType GetType() const { - return Type; - } - - inline bool IsAtom() const { - return Type == Atom; - } - - inline bool IsList() const { - return Type == List; - } - + TStringStream str; + PrettyPrintTo(str, prettyFlags); + return str.Str(); + } + + inline EType GetType() const { + return Type; + } + + inline bool IsAtom() const { + return Type == Atom; + } + + inline bool IsList() const { + return Type == List; + } + inline bool IsListOfSize(ui32 len) const { return Type == List && ListCount == len; } - inline TPosition GetPosition() const { - return Position; - } - - inline void SetPosition(TPosition position) { - Position = position; - } - - inline TStringBuf GetContent() const { + inline TPosition GetPosition() const { + return Position; + } + + inline void SetPosition(TPosition position) { + Position = position; + } + + inline TStringBuf GetContent() const { Y_VERIFY(IsAtom()); - return TStringBuf(Data.A.Content, Data.A.Size); - } - - inline void SetContent(TStringBuf newContent, TMemoryPool& pool) { + return TStringBuf(Data.A.Content, Data.A.Size); + } + + inline void SetContent(TStringBuf newContent, TMemoryPool& pool) { Y_VERIFY(IsAtom()); - auto poolContent = pool.AppendString(newContent); + auto poolContent = pool.AppendString(newContent); Data.A.Content = poolContent.data(); Data.A.Size = poolContent.size(); - } - - inline void SetLiteralContent(TStringBuf newContent) { + } + + inline void SetLiteralContent(TStringBuf newContent) { Y_VERIFY(IsAtom()); Data.A.Content = newContent.data(); Data.A.Size = newContent.size(); - } - - inline ui32 GetFlags() const { + } + + inline ui32 GetFlags() const { Y_VERIFY(IsAtom()); - return Data.A.Flags; - } - - inline void SetFlags(ui32 flags) { + return Data.A.Flags; + } + + inline void SetFlags(ui32 flags) { Y_VERIFY(IsAtom()); - Data.A.Flags = flags; - } - - inline ui32 GetChildrenCount() const { + Data.A.Flags = flags; + } + + inline ui32 GetChildrenCount() const { Y_VERIFY(IsList()); - return ListCount; - } - - inline const TAstNode* GetChild(ui32 index) const { + return ListCount; + } + + inline const TAstNode* GetChild(ui32 index) const { Y_VERIFY(IsList()); Y_VERIFY(index < ListCount); - if (ListCount <= SmallListCount) { - return Data.S.Children[index]; - } else { - return Data.L.Children[index]; - } - } - - inline TAstNode* GetChild(ui32 index) { + if (ListCount <= SmallListCount) { + return Data.S.Children[index]; + } else { + return Data.L.Children[index]; + } + } + + inline TAstNode* GetChild(ui32 index) { Y_VERIFY(IsList()); Y_VERIFY(index < ListCount); - if (ListCount <= SmallListCount) { - return Data.S.Children[index]; - } else { - return Data.L.Children[index]; - } - } - - static inline TAstNode* NewAtom(TPosition position, TStringBuf content, TMemoryPool& pool, ui32 flags = TNodeFlags::Default) { - auto poolContent = pool.AppendString(content); - auto ret = pool.Allocate<TAstNode>(); - ::new(ret) TAstNode(position, poolContent, flags); - return ret; - } - - // atom with non-owning content, useful for literal strings - static inline TAstNode* NewLiteralAtom(TPosition position, TStringBuf content, TMemoryPool& pool, ui32 flags = TNodeFlags::Default) { - auto ret = pool.Allocate<TAstNode>(); - ::new(ret) TAstNode(position, content, flags); - return ret; - } - - static inline TAstNode* NewList(TPosition position, TAstNode** children, ui32 childrenCount, TMemoryPool& pool) { - TAstNode** poolChildren = nullptr; - if (childrenCount) { - if (childrenCount > SmallListCount) { - poolChildren = pool.AllocateArray<TAstNode*>(childrenCount); - memcpy(poolChildren, children, sizeof(TAstNode*) * childrenCount); - } else { - poolChildren = children; - } - - for (ui32 index = 0; index < childrenCount; ++index) { + if (ListCount <= SmallListCount) { + return Data.S.Children[index]; + } else { + return Data.L.Children[index]; + } + } + + static inline TAstNode* NewAtom(TPosition position, TStringBuf content, TMemoryPool& pool, ui32 flags = TNodeFlags::Default) { + auto poolContent = pool.AppendString(content); + auto ret = pool.Allocate<TAstNode>(); + ::new(ret) TAstNode(position, poolContent, flags); + return ret; + } + + // atom with non-owning content, useful for literal strings + static inline TAstNode* NewLiteralAtom(TPosition position, TStringBuf content, TMemoryPool& pool, ui32 flags = TNodeFlags::Default) { + auto ret = pool.Allocate<TAstNode>(); + ::new(ret) TAstNode(position, content, flags); + return ret; + } + + static inline TAstNode* NewList(TPosition position, TAstNode** children, ui32 childrenCount, TMemoryPool& pool) { + TAstNode** poolChildren = nullptr; + if (childrenCount) { + if (childrenCount > SmallListCount) { + poolChildren = pool.AllocateArray<TAstNode*>(childrenCount); + memcpy(poolChildren, children, sizeof(TAstNode*) * childrenCount); + } else { + poolChildren = children; + } + + for (ui32 index = 0; index < childrenCount; ++index) { Y_VERIFY(poolChildren[index]); - } - } - - auto ret = pool.Allocate<TAstNode>(); - ::new(ret) TAstNode(position, poolChildren, childrenCount); - return ret; - } - - template <typename... TNodes> - static inline TAstNode* NewList(TPosition position, TMemoryPool& pool, TNodes... nodes) { - TAstNode* children[] = { nodes... }; - return NewList(position, children, sizeof...(nodes), pool); - } - - static inline TAstNode* NewList(TPosition position, TMemoryPool& pool) { - return NewList(position, nullptr, 0, pool); - } - - static TAstNode QuoteAtom; - + } + } + + auto ret = pool.Allocate<TAstNode>(); + ::new(ret) TAstNode(position, poolChildren, childrenCount); + return ret; + } + + template <typename... TNodes> + static inline TAstNode* NewList(TPosition position, TMemoryPool& pool, TNodes... nodes) { + TAstNode* children[] = { nodes... }; + return NewList(position, children, sizeof...(nodes), pool); + } + + static inline TAstNode* NewList(TPosition position, TMemoryPool& pool) { + return NewList(position, nullptr, 0, pool); + } + + static TAstNode QuoteAtom; + static inline TAstNode* Quote(TPosition position, TMemoryPool& pool, TAstNode* node) { return NewList(position, pool, &QuoteAtom, node); } - inline ~TAstNode() {} - + inline ~TAstNode() {} + void Destroy() { TString().swap(Position.File); } private: - inline TAstNode(TPosition position, TStringBuf content, ui32 flags) - : Position(position) - , Type(Atom) + inline TAstNode(TPosition position, TStringBuf content, ui32 flags) + : Position(position) + , Type(Atom) , ListCount(0) - { + { Data.A.Content = content.data(); Data.A.Size = content.size(); - Data.A.Flags = flags; - } - - inline TAstNode(TPosition position, TAstNode** children, ui32 childrenCount) - : Position(position) - , Type(List) + Data.A.Flags = flags; + } + + inline TAstNode(TPosition position, TAstNode** children, ui32 childrenCount) + : Position(position) + , Type(List) , ListCount(childrenCount) - { - if (childrenCount <= SmallListCount) { - for (ui32 index = 0; index < childrenCount; ++index) { - Data.S.Children[index] = children[index]; - } - } else { - Data.L.Children = children; - } - } - - TPosition Position; - const EType Type; + { + if (childrenCount <= SmallListCount) { + for (ui32 index = 0; index < childrenCount; ++index) { + Data.S.Children[index] = children[index]; + } + } else { + Data.L.Children = children; + } + } + + TPosition Position; + const EType Type; const ui32 ListCount; - - struct TAtom { - const char* Content; - ui32 Size; - ui32 Flags; - }; - + + struct TAtom { + const char* Content; + ui32 Size; + ui32 Flags; + }; + struct TListType { - TAstNode** Children; - }; - - struct TSmallList { - TAstNode* Children[SmallListCount]; - }; - - union { - TAtom A; + TAstNode** Children; + }; + + struct TSmallList { + TAstNode* Children[SmallListCount]; + }; + + union { + TAtom A; TListType L; - TSmallList S; - } Data; -}; - -struct TAstParseResult { + TSmallList S; + } Data; +}; + +struct TAstParseResult { std::unique_ptr<TMemoryPool> Pool; - TAstNode* Root = nullptr; + TAstNode* Root = nullptr; TIssues Issues; - - inline bool IsOk() const { - return !!Root; - } + + inline bool IsOk() const { + return !!Root; + } TAstParseResult() = default; ~TAstParseResult(); @@ -252,20 +252,20 @@ struct TAstParseResult { TAstParseResult& operator=(TAstParseResult&&); void Destroy(); -}; - -struct TAstPrintFlags { - enum { - Default = 0, - PerLine = 0x01, - ShortQuote = 0x02, - AdaptArbitraryContent = 0x04, - }; -}; - +}; + +struct TAstPrintFlags { + enum { + Default = 0, + PerLine = 0x01, + ShortQuote = 0x02, + AdaptArbitraryContent = 0x04, + }; +}; + TAstParseResult ParseAst(const TStringBuf& str, TMemoryPool* externalPool = nullptr, const TString& file = {}); - -} // namespace NYql - -template<> + +} // namespace NYql + +template<> void Out<NYql::TAstNode::EType>(class IOutputStream &o, NYql::TAstNode::EType x); diff --git a/ydb/library/yql/ast/yql_ast_annotation.cpp b/ydb/library/yql/ast/yql_ast_annotation.cpp index f44f6f53b4..fc97b879c8 100644 --- a/ydb/library/yql/ast/yql_ast_annotation.cpp +++ b/ydb/library/yql/ast/yql_ast_annotation.cpp @@ -1,178 +1,178 @@ -#include "yql_ast_annotation.h" -#include <util/string/printf.h> -#include <util/string/split.h> -#include <util/string/cast.h> +#include "yql_ast_annotation.h" +#include <util/string/printf.h> +#include <util/string/split.h> +#include <util/string/cast.h> #include <util/string/builder.h> #include <library/cpp/containers/stack_vector/stack_vec.h> - -namespace NYql { - -namespace { - -TAstNode* AnnotateNodePosition(TAstNode& node, TMemoryPool& pool) { - auto newPosition = node.GetPosition(); - TAstNode* pos = PositionAsNode(node.GetPosition(), pool); - TAstNode* shallowClone = &node; - if (node.IsList()) { - TSmallVec<TAstNode*> listChildren(node.GetChildrenCount()); - for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { - listChildren[index] = AnnotateNodePosition(*node.GetChild(index), pool); - } - - shallowClone = TAstNode::NewList(node.GetPosition(), listChildren.data(), listChildren.size(), pool); - } - - return TAstNode::NewList(newPosition, pool, pos, shallowClone); -} - -TAstNode* RemoveNodeAnnotations(TAstNode& node, TMemoryPool& pool) { - if (!node.IsList()) - return nullptr; - - if (node.GetChildrenCount() == 0) - return nullptr; - - auto lastNode = node.GetChild(node.GetChildrenCount() - 1); - auto res = lastNode; - if (lastNode->IsList()) { - TSmallVec<TAstNode*> listChildren(lastNode->GetChildrenCount()); - for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { - auto item = RemoveNodeAnnotations(*lastNode->GetChild(index), pool); - if (!item) - return nullptr; - - listChildren[index] = item; - } - - res = TAstNode::NewList(lastNode->GetPosition(), listChildren.data(), listChildren.size(), pool); - } - - return res; -} - -TAstNode* ExtractNodeAnnotations(TAstNode& node, TAnnotationNodeMap& annotations, TMemoryPool& pool) { - if (!node.IsList()) - return nullptr; - - if (node.GetChildrenCount() == 0) - return nullptr; - - auto lastNode = node.GetChild(node.GetChildrenCount() - 1); - auto res = lastNode; - if (lastNode->IsList()) { - TSmallVec<TAstNode*> listChildren(lastNode->GetChildrenCount()); - for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { - auto item = ExtractNodeAnnotations(*lastNode->GetChild(index), annotations, pool); - if (!item) - return nullptr; - - listChildren[index] = item; - } - - res = TAstNode::NewList(lastNode->GetPosition(), listChildren.data(), listChildren.size(), pool); - } - - auto& v = annotations[res]; - v.resize(node.GetChildrenCount() - 1); - for (ui32 index = 0; index + 1 < node.GetChildrenCount(); ++index) { - v[index] = node.GetChild(index); - } - - return res; -} - -TAstNode* ApplyNodePositionAnnotations(TAstNode& node, ui32 annotationIndex, TMemoryPool& pool) { - if (!node.IsList()) - return nullptr; - - if (node.GetChildrenCount() < annotationIndex + 2) - return nullptr; - - auto annotation = node.GetChild(annotationIndex); + +namespace NYql { + +namespace { + +TAstNode* AnnotateNodePosition(TAstNode& node, TMemoryPool& pool) { + auto newPosition = node.GetPosition(); + TAstNode* pos = PositionAsNode(node.GetPosition(), pool); + TAstNode* shallowClone = &node; + if (node.IsList()) { + TSmallVec<TAstNode*> listChildren(node.GetChildrenCount()); + for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { + listChildren[index] = AnnotateNodePosition(*node.GetChild(index), pool); + } + + shallowClone = TAstNode::NewList(node.GetPosition(), listChildren.data(), listChildren.size(), pool); + } + + return TAstNode::NewList(newPosition, pool, pos, shallowClone); +} + +TAstNode* RemoveNodeAnnotations(TAstNode& node, TMemoryPool& pool) { + if (!node.IsList()) + return nullptr; + + if (node.GetChildrenCount() == 0) + return nullptr; + + auto lastNode = node.GetChild(node.GetChildrenCount() - 1); + auto res = lastNode; + if (lastNode->IsList()) { + TSmallVec<TAstNode*> listChildren(lastNode->GetChildrenCount()); + for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { + auto item = RemoveNodeAnnotations(*lastNode->GetChild(index), pool); + if (!item) + return nullptr; + + listChildren[index] = item; + } + + res = TAstNode::NewList(lastNode->GetPosition(), listChildren.data(), listChildren.size(), pool); + } + + return res; +} + +TAstNode* ExtractNodeAnnotations(TAstNode& node, TAnnotationNodeMap& annotations, TMemoryPool& pool) { + if (!node.IsList()) + return nullptr; + + if (node.GetChildrenCount() == 0) + return nullptr; + + auto lastNode = node.GetChild(node.GetChildrenCount() - 1); + auto res = lastNode; + if (lastNode->IsList()) { + TSmallVec<TAstNode*> listChildren(lastNode->GetChildrenCount()); + for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { + auto item = ExtractNodeAnnotations(*lastNode->GetChild(index), annotations, pool); + if (!item) + return nullptr; + + listChildren[index] = item; + } + + res = TAstNode::NewList(lastNode->GetPosition(), listChildren.data(), listChildren.size(), pool); + } + + auto& v = annotations[res]; + v.resize(node.GetChildrenCount() - 1); + for (ui32 index = 0; index + 1 < node.GetChildrenCount(); ++index) { + v[index] = node.GetChild(index); + } + + return res; +} + +TAstNode* ApplyNodePositionAnnotations(TAstNode& node, ui32 annotationIndex, TMemoryPool& pool) { + if (!node.IsList()) + return nullptr; + + if (node.GetChildrenCount() < annotationIndex + 2) + return nullptr; + + auto annotation = node.GetChild(annotationIndex); auto str = annotation->GetContent(); - TStringBuf rowPart; - TStringBuf colPart; + TStringBuf rowPart; + TStringBuf colPart; TString filePart; GetNext(str, ':', rowPart); GetNext(str, ':', colPart); filePart = str; - ui32 row = 0, col = 0; - if (!TryFromString(rowPart, row) || !TryFromString(colPart, col)) - return nullptr; - - TSmallVec<TAstNode*> listChildren(node.GetChildrenCount()); - for (ui32 index = 0; index < node.GetChildrenCount() - 1; ++index) { - listChildren[index] = node.GetChild(index); - } - - auto lastNode = node.GetChild(node.GetChildrenCount() - 1); - TAstNode* lastResNode; - if (lastNode->IsAtom()) { + ui32 row = 0, col = 0; + if (!TryFromString(rowPart, row) || !TryFromString(colPart, col)) + return nullptr; + + TSmallVec<TAstNode*> listChildren(node.GetChildrenCount()); + for (ui32 index = 0; index < node.GetChildrenCount() - 1; ++index) { + listChildren[index] = node.GetChild(index); + } + + auto lastNode = node.GetChild(node.GetChildrenCount() - 1); + TAstNode* lastResNode; + if (lastNode->IsAtom()) { lastResNode = TAstNode::NewAtom(TPosition(col, row, filePart), lastNode->GetContent(), pool, lastNode->GetFlags()); - } else { - TSmallVec<TAstNode*> lastNodeChildren(lastNode->GetChildrenCount()); - for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { - lastNodeChildren[index] = ApplyNodePositionAnnotations(*lastNode->GetChild(index), annotationIndex, pool); - } - + } else { + TSmallVec<TAstNode*> lastNodeChildren(lastNode->GetChildrenCount()); + for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { + lastNodeChildren[index] = ApplyNodePositionAnnotations(*lastNode->GetChild(index), annotationIndex, pool); + } + lastResNode = TAstNode::NewList(TPosition(col, row, filePart), lastNodeChildren.data(), lastNodeChildren.size(), pool); - } - - listChildren[node.GetChildrenCount() - 1] = lastResNode; - return TAstNode::NewList(node.GetPosition(), listChildren.data(), listChildren.size(), pool); -} - -bool ApplyNodePositionAnnotationsInplace(TAstNode& node, ui32 annotationIndex) { - if (!node.IsList()) - return false; - - if (node.GetChildrenCount() < annotationIndex + 2) - return false; - - auto annotation = node.GetChild(annotationIndex); + } + + listChildren[node.GetChildrenCount() - 1] = lastResNode; + return TAstNode::NewList(node.GetPosition(), listChildren.data(), listChildren.size(), pool); +} + +bool ApplyNodePositionAnnotationsInplace(TAstNode& node, ui32 annotationIndex) { + if (!node.IsList()) + return false; + + if (node.GetChildrenCount() < annotationIndex + 2) + return false; + + auto annotation = node.GetChild(annotationIndex); TStringBuf str = annotation->GetContent(); - TStringBuf rowPart; - TStringBuf colPart; + TStringBuf rowPart; + TStringBuf colPart; TString filePart; GetNext(str, ':', rowPart); GetNext(str, ':', colPart); filePart = str; - ui32 row = 0, col = 0; - if (!TryFromString(rowPart, row) || !TryFromString(colPart, col)) - return false; - - auto lastNode = node.GetChild(node.GetChildrenCount() - 1); + ui32 row = 0, col = 0; + if (!TryFromString(rowPart, row) || !TryFromString(colPart, col)) + return false; + + auto lastNode = node.GetChild(node.GetChildrenCount() - 1); lastNode->SetPosition(TPosition(col, row, filePart)); - if (lastNode->IsList()) { - for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { - if (!ApplyNodePositionAnnotationsInplace(*lastNode->GetChild(index), annotationIndex)) - return false; - } - } - - return true; -} - -} - -TAstNode* AnnotatePositions(TAstNode& root, TMemoryPool& pool) { - return AnnotateNodePosition(root, pool); -} - -TAstNode* RemoveAnnotations(TAstNode& root, TMemoryPool& pool) { - return RemoveNodeAnnotations(root, pool); -} - -TAstNode* ApplyPositionAnnotations(TAstNode& root, ui32 annotationIndex, TMemoryPool& pool) { - return ApplyNodePositionAnnotations(root, annotationIndex, pool); -} - -bool ApplyPositionAnnotationsInplace(TAstNode& root, ui32 annotationIndex) { - return ApplyNodePositionAnnotationsInplace(root, annotationIndex); -} - -TAstNode* PositionAsNode(TPosition position, TMemoryPool& pool) { + if (lastNode->IsList()) { + for (ui32 index = 0; index < lastNode->GetChildrenCount(); ++index) { + if (!ApplyNodePositionAnnotationsInplace(*lastNode->GetChild(index), annotationIndex)) + return false; + } + } + + return true; +} + +} + +TAstNode* AnnotatePositions(TAstNode& root, TMemoryPool& pool) { + return AnnotateNodePosition(root, pool); +} + +TAstNode* RemoveAnnotations(TAstNode& root, TMemoryPool& pool) { + return RemoveNodeAnnotations(root, pool); +} + +TAstNode* ApplyPositionAnnotations(TAstNode& root, ui32 annotationIndex, TMemoryPool& pool) { + return ApplyNodePositionAnnotations(root, annotationIndex, pool); +} + +bool ApplyPositionAnnotationsInplace(TAstNode& root, ui32 annotationIndex) { + return ApplyNodePositionAnnotationsInplace(root, annotationIndex); +} + +TAstNode* PositionAsNode(TPosition position, TMemoryPool& pool) { TStringBuilder str; str << position.Row << ':' << position.Column; if (!position.File.empty()) { @@ -180,10 +180,10 @@ TAstNode* PositionAsNode(TPosition position, TMemoryPool& pool) { } return TAstNode::NewAtom(position, str, pool); -} - -TAstNode* ExtractAnnotations(TAstNode& root, TAnnotationNodeMap& annotations, TMemoryPool& pool) { - return ExtractNodeAnnotations(root, annotations, pool); -} - -} +} + +TAstNode* ExtractAnnotations(TAstNode& root, TAnnotationNodeMap& annotations, TMemoryPool& pool) { + return ExtractNodeAnnotations(root, annotations, pool); +} + +} diff --git a/ydb/library/yql/ast/yql_ast_annotation.h b/ydb/library/yql/ast/yql_ast_annotation.h index c4fbccffac..c909ce9df4 100644 --- a/ydb/library/yql/ast/yql_ast_annotation.h +++ b/ydb/library/yql/ast/yql_ast_annotation.h @@ -1,22 +1,22 @@ -#pragma once -#include "yql_ast.h" -#include <util/generic/hash.h> - -namespace NYql { - -TAstNode* PositionAsNode(TPosition position, TMemoryPool& pool); - -TAstNode* AnnotatePositions(TAstNode& root, TMemoryPool& pool); -// returns nullptr in case of error -TAstNode* RemoveAnnotations(TAstNode& root, TMemoryPool& pool); -// returns nullptr in case of error -TAstNode* ApplyPositionAnnotations(TAstNode& root, ui32 annotationIndex, TMemoryPool& pool); -// returns false in case of error -bool ApplyPositionAnnotationsInplace(TAstNode& root, ui32 annotationIndex); - +#pragma once +#include "yql_ast.h" +#include <util/generic/hash.h> + +namespace NYql { + +TAstNode* PositionAsNode(TPosition position, TMemoryPool& pool); + +TAstNode* AnnotatePositions(TAstNode& root, TMemoryPool& pool); +// returns nullptr in case of error +TAstNode* RemoveAnnotations(TAstNode& root, TMemoryPool& pool); +// returns nullptr in case of error +TAstNode* ApplyPositionAnnotations(TAstNode& root, ui32 annotationIndex, TMemoryPool& pool); +// returns false in case of error +bool ApplyPositionAnnotationsInplace(TAstNode& root, ui32 annotationIndex); + typedef THashMap<const TAstNode*, TVector<const TAstNode*>> TAnnotationNodeMap; - -// returns nullptr in case of error -TAstNode* ExtractAnnotations(TAstNode& root, TAnnotationNodeMap& annotations, TMemoryPool& pool); - -} + +// returns nullptr in case of error +TAstNode* ExtractAnnotations(TAstNode& root, TAnnotationNodeMap& annotations, TMemoryPool& pool); + +} diff --git a/ydb/library/yql/ast/yql_ast_ut.cpp b/ydb/library/yql/ast/yql_ast_ut.cpp index 7c2e7b04be..23e140653d 100644 --- a/ydb/library/yql/ast/yql_ast_ut.cpp +++ b/ydb/library/yql/ast/yql_ast_ut.cpp @@ -1,112 +1,112 @@ -#include "yql_ast.h" -#include "yql_ast_annotation.h" - +#include "yql_ast.h" +#include "yql_ast_annotation.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/string/util.h> + +#include <util/string/util.h> #include <util/system/sanitizers.h> - -namespace NYql { - + +namespace NYql { + Y_UNIT_TEST_SUITE(TParseYqlAst) { constexpr TStringBuf TEST_PROGRAM = - "(\n" - "#comment\n" - "(let mr_source (DataSource 'yamr 'cedar))\n" - "(let x (Read! world mr_source (Key '('table (KeyString 'Input))) '('key 'value) '()))\n" - "(let world (Left! x))\n" - "(let table1 (Right! x))\n" - "(let tresh (Int32 '100))\n" - "(let table1low (Filter table1 (lambda '(item) (< (member item 'key) tresh))))\n" - "(let mr_sink (DataSink 'yamr (quote cedar)))\n" - "(let world (Write! world mr_sink (Key '('table (KeyString 'Output))) table1low '('('mode 'append))))\n" - "(let world (Commit! world mr_sink))\n" - "(return world)\n" + "(\n" + "#comment\n" + "(let mr_source (DataSource 'yamr 'cedar))\n" + "(let x (Read! world mr_source (Key '('table (KeyString 'Input))) '('key 'value) '()))\n" + "(let world (Left! x))\n" + "(let table1 (Right! x))\n" + "(let tresh (Int32 '100))\n" + "(let table1low (Filter table1 (lambda '(item) (< (member item 'key) tresh))))\n" + "(let mr_sink (DataSink 'yamr (quote cedar)))\n" + "(let world (Write! world mr_sink (Key '('table (KeyString 'Output))) table1low '('('mode 'append))))\n" + "(let world (Commit! world mr_sink))\n" + "(return world)\n" ")"; - + Y_UNIT_TEST(ParseAstTest) { - TAstParseResult res = ParseAst(TEST_PROGRAM); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Root->IsList()); + TAstParseResult res = ParseAst(TEST_PROGRAM); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Root->IsList()); UNIT_ASSERT(res.Issues.Empty()); - } - + } + Y_UNIT_TEST(ParseAstTestPerf) { #ifdef WITH_VALGRIND const ui32 n = 1000; #else const ui32 n = NSan::PlainOrUnderSanitizer(100000, 1000); #endif - auto t1 = TInstant::Now(); - for (ui32 i = 0; i < n; ++i) { - TAstParseResult res = ParseAst(TEST_PROGRAM); - UNIT_ASSERT(res.IsOk()); - UNIT_ASSERT(res.Root->IsList()); + auto t1 = TInstant::Now(); + for (ui32 i = 0; i < n; ++i) { + TAstParseResult res = ParseAst(TEST_PROGRAM); + UNIT_ASSERT(res.IsOk()); + UNIT_ASSERT(res.Root->IsList()); UNIT_ASSERT(res.Issues.Empty()); - } - auto t2 = TInstant::Now(); - Cout << t2 - t1 << Endl; - } - + } + auto t2 = TInstant::Now(); + Cout << t2 - t1 << Endl; + } + Y_UNIT_TEST(PrintAstTest) { - TAstParseResult ast = ParseAst(TEST_PROGRAM); - UNIT_ASSERT(ast.IsOk()); - + TAstParseResult ast = ParseAst(TEST_PROGRAM); + UNIT_ASSERT(ast.IsOk()); + TString printedProgram = ast.Root->ToString(); UNIT_ASSERT(printedProgram.find('\n') == TString::npos); - - TAstParseResult parsedAst = ParseAst(printedProgram); - UNIT_ASSERT(parsedAst.IsOk()); - } - + + TAstParseResult parsedAst = ParseAst(printedProgram); + UNIT_ASSERT(parsedAst.IsOk()); + } + Y_UNIT_TEST(PrettyPrintAst) { - const ui32 testFlags[] = { - TAstPrintFlags::Default, - TAstPrintFlags::PerLine, - //TAstPrintFlags::ShortQuote, //-- generates invalid AST - TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote - }; - - TAstParseResult ast = ParseAst(TEST_PROGRAM); - UNIT_ASSERT(ast.IsOk()); - + const ui32 testFlags[] = { + TAstPrintFlags::Default, + TAstPrintFlags::PerLine, + //TAstPrintFlags::ShortQuote, //-- generates invalid AST + TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote + }; + + TAstParseResult ast = ParseAst(TEST_PROGRAM); + UNIT_ASSERT(ast.IsOk()); + for (ui32 i = 0; i < Y_ARRAY_SIZE(testFlags); ++i) { - ui32 prettyFlags = testFlags[i]; - + ui32 prettyFlags = testFlags[i]; + TString printedProgram1 = ast.Root->ToString(prettyFlags); - TAstParseResult parsedAst = ParseAst(printedProgram1); - UNIT_ASSERT(parsedAst.IsOk()); - + TAstParseResult parsedAst = ParseAst(printedProgram1); + UNIT_ASSERT(parsedAst.IsOk()); + TString printedProgram2 = parsedAst.Root->ToString(prettyFlags); - UNIT_ASSERT_STRINGS_EQUAL(printedProgram1, printedProgram2); - } - } - + UNIT_ASSERT_STRINGS_EQUAL(printedProgram1, printedProgram2); + } + } + Y_UNIT_TEST(AnnotatedAstPrint) { TMemoryPool pool(4096); TAstParseResult ast = ParseAst(TEST_PROGRAM, &pool); - UNIT_ASSERT(ast.IsOk()); - + UNIT_ASSERT(ast.IsOk()); + TAstParseResult astWithPositions; astWithPositions.Root = AnnotatePositions(*ast.Root, pool); UNIT_ASSERT(!!astWithPositions.Root); - + TString sAnn = astWithPositions.Root->ToString(); UNIT_ASSERT(false == sAnn.empty()); - - TAstParseResult annRes = ParseAst(sAnn); - UNIT_ASSERT(annRes.IsOk()); - + + TAstParseResult annRes = ParseAst(sAnn); + UNIT_ASSERT(annRes.IsOk()); + TAstParseResult removedAnn; removedAnn.Root = RemoveAnnotations(*annRes.Root, pool); UNIT_ASSERT(!!removedAnn.Root); - + TString strOriginal = ast.Root->ToString(); TString strAnnRemoved = removedAnn.Root->ToString(); - UNIT_ASSERT_VALUES_EQUAL(strOriginal, strAnnRemoved); - + UNIT_ASSERT_VALUES_EQUAL(strOriginal, strAnnRemoved); + astWithPositions.Root->GetChild(0)->SetContent("100:100", pool); - + TAstParseResult appliedPositionsAnn; appliedPositionsAnn.Root = ApplyPositionAnnotations(*astWithPositions.Root, 0, pool); UNIT_ASSERT(appliedPositionsAnn.Root); @@ -115,59 +115,59 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { removedAnn2.Root = RemoveAnnotations(*appliedPositionsAnn.Root, pool); UNIT_ASSERT(removedAnn2.Root); UNIT_ASSERT_VALUES_EQUAL(removedAnn2.Root->GetPosition().Row, 100); - } - - template <typename TCharType> - void TestGoodArbitraryAtom( + } + + template <typename TCharType> + void TestGoodArbitraryAtom( const TString& program, const TBasicStringBuf<TCharType>& expectedValue) - { - TAstParseResult ast = ParseAst(program); - UNIT_ASSERT(ast.IsOk()); - UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); - - TAstNode* atom = ast.Root->GetChild(0); - UNIT_ASSERT(atom->IsAtom()); - UNIT_ASSERT_STRINGS_EQUAL_C( - atom->GetContent(), + { + TAstParseResult ast = ParseAst(program); + UNIT_ASSERT(ast.IsOk()); + UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); + + TAstNode* atom = ast.Root->GetChild(0); + UNIT_ASSERT(atom->IsAtom()); + UNIT_ASSERT_STRINGS_EQUAL_C( + atom->GetContent(), TString((char*)expectedValue.data(), expectedValue.size()), - program); - } - + program); + } + Y_UNIT_TEST(GoodArbitraryAtom) { TestGoodArbitraryAtom("(\"\")", TStringBuf()); TestGoodArbitraryAtom("(\" 1 a 3 b \")", TStringBuf(" 1 a 3 b ")); - - ui8 expectedHex[] = { 0xab, 'c', 'd', 0x00 }; + + ui8 expectedHex[] = { 0xab, 'c', 'd', 0x00 }; TestGoodArbitraryAtom("(\"\\xabcd\")", TBasicStringBuf<ui8>(expectedHex)); TestGoodArbitraryAtom("(\" \\x3d \")", TStringBuf(" \x3d ")); - - ui8 expectedOctal[] = { 056, '7', '8', 0x00 }; + + ui8 expectedOctal[] = { 056, '7', '8', 0x00 }; TestGoodArbitraryAtom("(\"\\05678\")", TBasicStringBuf<ui8>(expectedOctal)); TestGoodArbitraryAtom("(\" \\056 \")", TStringBuf(" \056 ")); TestGoodArbitraryAtom("(\" \\177 \")", TStringBuf(" \177 ")); TestGoodArbitraryAtom("(\" \\377 \")", TStringBuf(" \377 ")); TestGoodArbitraryAtom("(\" \\477 \")", TStringBuf(" 477 ")); - - { - ui8 expected1[] = { 0x01, 0x00 }; + + { + ui8 expected1[] = { 0x01, 0x00 }; TestGoodArbitraryAtom("(\"\\u0001\")", TBasicStringBuf<ui8>(expected1)); - - ui8 expected2[] = { 0xE1, 0x88, 0xB4, 0x00 }; + + ui8 expected2[] = { 0xE1, 0x88, 0xB4, 0x00 }; TestGoodArbitraryAtom("(\"\\u1234\")", TBasicStringBuf<ui8>(expected2)); - - ui8 expected3[] = { 0xef, 0xbf, 0xbf, 0x00 }; + + ui8 expected3[] = { 0xef, 0xbf, 0xbf, 0x00 }; TestGoodArbitraryAtom("(\"\\uffff\")", TBasicStringBuf<ui8>(expected3)); - } - - { - ui8 expected1[] = { 0x01, 0x00 }; + } + + { + ui8 expected1[] = { 0x01, 0x00 }; TestGoodArbitraryAtom("(\"\\U00000001\")", TBasicStringBuf<ui8>(expected1)); - - ui8 expected2[] = { 0xf4, 0x8f, 0xbf, 0xbf, 0x00 }; + + ui8 expected2[] = { 0xf4, 0x8f, 0xbf, 0xbf, 0x00 }; TestGoodArbitraryAtom("(\"\\U0010ffff\")", TBasicStringBuf<ui8>(expected2)); - } - + } + TestGoodArbitraryAtom("(\"\\t\")", TStringBuf("\t")); TestGoodArbitraryAtom("(\"\\n\")", TStringBuf("\n")); TestGoodArbitraryAtom("(\"\\r\")", TStringBuf("\r")); @@ -175,180 +175,180 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { TestGoodArbitraryAtom("(\"\\f\")", TStringBuf("\f")); TestGoodArbitraryAtom("(\"\\a\")", TStringBuf("\a")); TestGoodArbitraryAtom("(\"\\v\")", TStringBuf("\v")); - } - - void TestBadArbitraryAtom( + } + + void TestBadArbitraryAtom( const TString& program, const TString& expectedError) - { - TAstParseResult ast = ParseAst(program); - UNIT_ASSERT(false == ast.IsOk()); - UNIT_ASSERT(false == !!ast.Root); + { + TAstParseResult ast = ParseAst(program); + UNIT_ASSERT(false == ast.IsOk()); + UNIT_ASSERT(false == !!ast.Root); UNIT_ASSERT(false == ast.Issues.Empty()); UNIT_ASSERT_STRINGS_EQUAL(ast.Issues.begin()->Message, expectedError); - } - + } + Y_UNIT_TEST(BadArbitraryAtom) { - TestBadArbitraryAtom("(a\")", "Unexpected \""); - TestBadArbitraryAtom("(\"++++\"11111)", "Unexpected end of \""); - TestBadArbitraryAtom("(\"\\", "Expected escape sequence"); + TestBadArbitraryAtom("(a\")", "Unexpected \""); + TestBadArbitraryAtom("(\"++++\"11111)", "Unexpected end of \""); + TestBadArbitraryAtom("(\"\\", "Expected escape sequence"); TestBadArbitraryAtom("(\"\\\")", "Unexpected end of atom"); TestBadArbitraryAtom("(\"abc)", "Unexpected end of atom"); - - TestBadArbitraryAtom("(\"\\018\")", "Invalid octal value"); - TestBadArbitraryAtom("(\"\\01\")", "Invalid octal value"); - TestBadArbitraryAtom("(\"\\378\")", "Invalid octal value"); - - TestBadArbitraryAtom("(\"\\x1g\")", "Invalid hexadecimal value"); - TestBadArbitraryAtom("(\"\\xf\")", "Invalid hexadecimal value"); - - TestBadArbitraryAtom("(\"\\u\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\u1\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\u12\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\u123\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\ughij\")", "Invalid unicode value"); - - TestBadArbitraryAtom("(\"\\U\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U11\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U1122\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U112233\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\Ughijklmn\")", "Invalid unicode value"); + + TestBadArbitraryAtom("(\"\\018\")", "Invalid octal value"); + TestBadArbitraryAtom("(\"\\01\")", "Invalid octal value"); + TestBadArbitraryAtom("(\"\\378\")", "Invalid octal value"); + + TestBadArbitraryAtom("(\"\\x1g\")", "Invalid hexadecimal value"); + TestBadArbitraryAtom("(\"\\xf\")", "Invalid hexadecimal value"); + + TestBadArbitraryAtom("(\"\\u\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\u1\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\u12\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\u123\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\ughij\")", "Invalid unicode value"); + + TestBadArbitraryAtom("(\"\\U\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\U11\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\U1122\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\U112233\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\Ughijklmn\")", "Invalid unicode value"); TestBadArbitraryAtom("(\"\\U00110000\")", "Invalid unicode value"); TestBadArbitraryAtom("(\"\\U00123456\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U00200000\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\Uffffffff\")", "Invalid unicode value"); - - // surrogate range - TestBadArbitraryAtom("(\"\\ud800\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\udfff\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U0000d800\")", "Invalid unicode value"); - TestBadArbitraryAtom("(\"\\U0000dfff\")", "Invalid unicode value"); - + TestBadArbitraryAtom("(\"\\U00200000\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\Uffffffff\")", "Invalid unicode value"); + + // surrogate range + TestBadArbitraryAtom("(\"\\ud800\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\udfff\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\U0000d800\")", "Invalid unicode value"); + TestBadArbitraryAtom("(\"\\U0000dfff\")", "Invalid unicode value"); + TestBadArbitraryAtom("(x\"ag\")", "Invalid binary value"); TestBadArbitraryAtom("(x\"abc\")", "Invalid binary value"); TestBadArbitraryAtom("(x\"abcd)", "Invalid binary value"); TestBadArbitraryAtom("(x\"abcd", "Unexpected end of atom"); - } - + } + void ParseAndPrint(const TString& program, const TString& expected) { - TAstParseResult ast = ParseAst(program); - UNIT_ASSERT_C(ast.IsOk(), program); - + TAstParseResult ast = ParseAst(program); + UNIT_ASSERT_C(ast.IsOk(), program); + TString result = ast.Root->ToString(); - UNIT_ASSERT_STRINGS_EQUAL_C(result, expected, program); - } - + UNIT_ASSERT_STRINGS_EQUAL_C(result, expected, program); + } + Y_UNIT_TEST(ArbitraryAtomEscaping) { - ParseAndPrint( - "(\"\\t\\n\\r\\b\\a\\f\\v\")", - "(\"\\t\\n\\r\\b\\a\\f\\v\")"); - - ParseAndPrint("(\"\\u1234\")", "(\"\\u1234\")"); - ParseAndPrint("(\"\\u1234abcd\")", "(\"\\u1234abcd\")"); - ParseAndPrint("(\"\\177\")", "(\"\\x7F\")"); - ParseAndPrint("(\"\\377\")", "(\"\\xFF\")"); - - ParseAndPrint( - "(\"тестовая строка\")", - "(\"\\u0442\\u0435\\u0441\\u0442\\u043E\\u0432\\u0430" - "\\u044F \\u0441\\u0442\\u0440\\u043E\\u043A\\u0430\")"); + ParseAndPrint( + "(\"\\t\\n\\r\\b\\a\\f\\v\")", + "(\"\\t\\n\\r\\b\\a\\f\\v\")"); + + ParseAndPrint("(\"\\u1234\")", "(\"\\u1234\")"); + ParseAndPrint("(\"\\u1234abcd\")", "(\"\\u1234abcd\")"); + ParseAndPrint("(\"\\177\")", "(\"\\x7F\")"); + ParseAndPrint("(\"\\377\")", "(\"\\xFF\")"); + + ParseAndPrint( + "(\"тестовая строка\")", + "(\"\\u0442\\u0435\\u0441\\u0442\\u043E\\u0432\\u0430" + "\\u044F \\u0441\\u0442\\u0440\\u043E\\u043A\\u0430\")"); ParseAndPrint("(\"\")", "(\"\")"); - } - + } + Y_UNIT_TEST(BinaryAtom) { - ParseAndPrint("(x\"abcdef\")", "(x\"ABCDEF\")"); - ParseAndPrint("(x\"aBcDeF\")", "(x\"ABCDEF\")"); - ParseAndPrint("(x)", "(x)"); - ParseAndPrint("(x x)", "(x x)"); - ParseAndPrint("(x\"\" x)", "(x\"\" x)"); - ParseAndPrint("(x\"ab12cd\" x)", "(x\"AB12CD\" x)"); - } - + ParseAndPrint("(x\"abcdef\")", "(x\"ABCDEF\")"); + ParseAndPrint("(x\"aBcDeF\")", "(x\"ABCDEF\")"); + ParseAndPrint("(x)", "(x)"); + ParseAndPrint("(x x)", "(x x)"); + ParseAndPrint("(x\"\" x)", "(x\"\" x)"); + ParseAndPrint("(x\"ab12cd\" x)", "(x\"AB12CD\" x)"); + } + void ParseAndAdaptPrint(const TString& program, const TString& expected) { - TAstParseResult ast = ParseAst(program); - UNIT_ASSERT_C(ast.IsOk(), program); - + TAstParseResult ast = ParseAst(program); + UNIT_ASSERT_C(ast.IsOk(), program); + TString result = ast.Root->ToString( - TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine | - TAstPrintFlags::AdaptArbitraryContent); - - RemoveAll(result, '\n'); // for simplify expected string - UNIT_ASSERT_STRINGS_EQUAL_C(result, expected, program); - } - + TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine | + TAstPrintFlags::AdaptArbitraryContent); + + RemoveAll(result, '\n'); // for simplify expected string + UNIT_ASSERT_STRINGS_EQUAL_C(result, expected, program); + } + Y_UNIT_TEST(AdaptArbitraryAtom) { - ParseAndAdaptPrint("(\"test\")", "(test)"); - ParseAndAdaptPrint("(\"another test\")", "(\"another test\")"); - ParseAndAdaptPrint("(\"braces(in)test\")", "(\"braces(in)test\")"); - ParseAndAdaptPrint("(\"escaped\\u1234sequence\")", "(\"escaped\\u1234sequence\")"); - ParseAndAdaptPrint("(\"escaped\\x41sequence\")", "(escapedAsequence)"); + ParseAndAdaptPrint("(\"test\")", "(test)"); + ParseAndAdaptPrint("(\"another test\")", "(\"another test\")"); + ParseAndAdaptPrint("(\"braces(in)test\")", "(\"braces(in)test\")"); + ParseAndAdaptPrint("(\"escaped\\u1234sequence\")", "(\"escaped\\u1234sequence\")"); + ParseAndAdaptPrint("(\"escaped\\x41sequence\")", "(escapedAsequence)"); ParseAndAdaptPrint("(\"\")", "(\"\")"); - } - + } + void ParseError(const TString& program) { - TAstParseResult ast = ParseAst(program); - UNIT_ASSERT_C(!ast.IsOk(), program); - } - + TAstParseResult ast = ParseAst(program); + UNIT_ASSERT_C(!ast.IsOk(), program); + } + Y_UNIT_TEST(MultilineAtomTrivial) { - TStringStream s; - for (ui32 i = 4; i < 13; ++i) { - TStringStream prog; - prog << "("; - for (ui32 j = 0; j < i; ++j) { - prog << "@"; - } - prog << ")"; - TAstParseResult ast = ParseAst(prog.Str()); - s << prog.Str() << " --> "; - if (ast.IsOk()) { - UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); - - TAstNode* atom = ast.Root->GetChild(0); - UNIT_ASSERT(atom->IsAtom()); - UNIT_ASSERT(atom->GetFlags() & TNodeFlags::MultilineContent); - s << "'" << atom->GetContent() << "'" << Endl; - } else { - s << "Error" << Endl; - } - } - //~ Cerr << s.Str() << Endl; - UNIT_ASSERT_NO_DIFF( - "(@@@@) --> ''\n" - "(@@@@@) --> '@'\n" - "(@@@@@@) --> Error\n" - "(@@@@@@@) --> Error\n" - "(@@@@@@@@) --> '@@'\n" - "(@@@@@@@@@) --> '@@@'\n" - "(@@@@@@@@@@) --> Error\n" - "(@@@@@@@@@@@) --> Error\n" - "(@@@@@@@@@@@@) --> '@@@@'\n", - s.Str() - ); - } - + TStringStream s; + for (ui32 i = 4; i < 13; ++i) { + TStringStream prog; + prog << "("; + for (ui32 j = 0; j < i; ++j) { + prog << "@"; + } + prog << ")"; + TAstParseResult ast = ParseAst(prog.Str()); + s << prog.Str() << " --> "; + if (ast.IsOk()) { + UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); + + TAstNode* atom = ast.Root->GetChild(0); + UNIT_ASSERT(atom->IsAtom()); + UNIT_ASSERT(atom->GetFlags() & TNodeFlags::MultilineContent); + s << "'" << atom->GetContent() << "'" << Endl; + } else { + s << "Error" << Endl; + } + } + //~ Cerr << s.Str() << Endl; + UNIT_ASSERT_NO_DIFF( + "(@@@@) --> ''\n" + "(@@@@@) --> '@'\n" + "(@@@@@@) --> Error\n" + "(@@@@@@@) --> Error\n" + "(@@@@@@@@) --> '@@'\n" + "(@@@@@@@@@) --> '@@@'\n" + "(@@@@@@@@@@) --> Error\n" + "(@@@@@@@@@@@) --> Error\n" + "(@@@@@@@@@@@@) --> '@@@@'\n", + s.Str() + ); + } + Y_UNIT_TEST(MultilineAtom) { TString s1 = "(@@multi \n" - "line \n" - "string@@)"; - ParseAndPrint(s1, s1); - + "line \n" + "string@@)"; + ParseAndPrint(s1, s1); + TString s2 = "(@@multi \n" - "l@ine \n" - "string@@)"; - ParseAndPrint(s2, s2); - + "l@ine \n" + "string@@)"; + ParseAndPrint(s2, s2); + TString s3 = "(@@multi \n" - "l@@@ine \n" - "string@@)"; - ParseError(s3); - + "l@@@ine \n" + "string@@)"; + ParseError(s3); + TString s4 = "(@@multi \n" - "l@@@@ine \n" - "string@@)"; - ParseAndPrint(s4, s4); - + "l@@@@ine \n" + "string@@)"; + ParseAndPrint(s4, s4); + TString s5 = "(@@\n" "one@\n" "two@@@@\n" @@ -358,24 +358,24 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { "@@)"; TAstParseResult ast = ParseAst(s5); - UNIT_ASSERT(ast.IsOk()); - UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); - - TAstNode* atom = ast.Root->GetChild(0); - UNIT_ASSERT(atom->IsAtom()); - UNIT_ASSERT(atom->GetFlags() & TNodeFlags::MultilineContent); - + UNIT_ASSERT(ast.IsOk()); + UNIT_ASSERT_VALUES_EQUAL(ast.Root->GetChildrenCount(), 1); + + TAstNode* atom = ast.Root->GetChild(0); + UNIT_ASSERT(atom->IsAtom()); + UNIT_ASSERT(atom->GetFlags() & TNodeFlags::MultilineContent); + TString expected = "\n" - "one@\n" - "two@@\n" - "four@@@@\n" - "@@two\n" - "@one\n"; - UNIT_ASSERT_STRINGS_EQUAL(atom->GetContent(), expected); + "one@\n" + "two@@\n" + "four@@@@\n" + "@@two\n" + "@one\n"; + UNIT_ASSERT_STRINGS_EQUAL(atom->GetContent(), expected); TString printResult = ast.Root->ToString(); UNIT_ASSERT_STRINGS_EQUAL(s5, printResult); - } + } Y_UNIT_TEST(UnicodePrettyPrint) { ParseAndAdaptPrint("(\"абв αβγ ﬡ\")", "(\"\\u0430\\u0431\\u0432 \\u03B1\\u03B2\\u03B3 \\uFB21\")"); @@ -398,6 +398,6 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) { RemoveAll(pretty, '\n'); UNIT_ASSERT_EQUAL(pretty, expected); } -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/ast/yql_errors.cpp b/ydb/library/yql/ast/yql_errors.cpp index e72303fad6..c16c3650e1 100644 --- a/ydb/library/yql/ast/yql_errors.cpp +++ b/ydb/library/yql/ast/yql_errors.cpp @@ -1 +1 @@ -#include "yql_errors.h" +#include "yql_errors.h" diff --git a/ydb/library/yql/ast/yql_errors.h b/ydb/library/yql/ast/yql_errors.h index fee1768e30..cb3cac387c 100644 --- a/ydb/library/yql/ast/yql_errors.h +++ b/ydb/library/yql/ast/yql_errors.h @@ -1,3 +1,3 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/yql_issue.h> diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 727f9b6c8d..82f6c5812a 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -1,15 +1,15 @@ -#include "yql_expr.h" -#include "yql_ast_annotation.h" -#include "yql_gc_nodes.h" - +#include "yql_expr.h" +#include "yql_ast_annotation.h" +#include "yql_gc_nodes.h" + #include <ydb/library/yql/utils/utf8.h> #include <library/cpp/containers/stack_vector/stack_vec.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/size_literals.h> -#include <util/string/cast.h> +#include <util/string/cast.h> #include <util/string/join.h> - + #include <util/digest/fnv.h> #include <util/digest/murmur.h> #include <util/digest/city.h> @@ -17,15 +17,15 @@ #include <map> -namespace NYql { - +namespace NYql { + const TStringBuf ZeroString = ""; void ReportError(TExprContext& ctx, const TIssue& issue) { ctx.AddError(issue); } -namespace { +namespace { template <typename T> const T* FindType(const T& sample, TExprContext& ctx) { const auto it = ctx.TypeSet.find(&sample); @@ -87,60 +87,60 @@ namespace { } } - struct TContext { - struct TFrame { + struct TContext { + struct TFrame { THashMap<TString, TExprNode::TListType> Bindings; THashMap<TString, TString> Imports; TExprNode::TListType Return; - }; - - TExprContext& Expr; + }; + + TExprContext& Expr; TVector<TFrame> Frames; TLibraryCohesion Cohesion; TNodeOnNodeOwnedMap DeepClones; - const TAnnotationNodeMap* Annotations = nullptr; + const TAnnotationNodeMap* Annotations = nullptr; IModuleResolver* ModuleResolver = nullptr; - ui32 TypeAnnotationIndex = Max<ui32>(); + ui32 TypeAnnotationIndex = Max<ui32>(); TString File; ui16 SyntaxVersion = 0; - - TContext(TExprContext& expr) - : Expr(expr) - { - } - + + TContext(TExprContext& expr) + : Expr(expr) + { + } + void AddError(const TAstNode& node, const TString& message) { Expr.AddError(TIssue(node.GetPosition(), message)); - } - + } + TExprNode::TPtr&& ProcessNode(const TAstNode& node, TExprNode::TPtr&& exprNode) { - if (TypeAnnotationIndex != Max<ui32>()) { + if (TypeAnnotationIndex != Max<ui32>()) { exprNode->SetTypeAnn(CompileTypeAnnotation(node)); - } - + } + return std::move(exprNode); - } - - void PushFrame() { - Frames.push_back(TFrame()); - } - - void PopFrame() { - Frames.pop_back(); - } - + } + + void PushFrame() { + Frames.push_back(TFrame()); + } + + void PopFrame() { + Frames.pop_back(); + } + TExprNode::TListType FindBinding(const TStringBuf& name) const { for (auto it = Frames.crbegin(); it != Frames.crend(); ++it) { const auto r = it->Bindings.find(name); if (it->Bindings.cend() != r) return r->second; - } - + } + return {}; - } - + } + TString FindImport(const TStringBuf& name) const { for (auto it = Frames.crbegin(); it != Frames.crend(); ++it) { const auto r = it->Imports.find(name); @@ -152,64 +152,64 @@ namespace { } const TTypeAnnotationNode* CompileTypeAnnotation(const TAstNode& node) { - auto ptr = Annotations->FindPtr(&node); - if (!ptr || TypeAnnotationIndex >= ptr->size()) { - AddError(node, "Failed to load type annotation"); - return nullptr; - } - - return CompileTypeAnnotationNode(*(*ptr)[TypeAnnotationIndex]); - } - + auto ptr = Annotations->FindPtr(&node); + if (!ptr || TypeAnnotationIndex >= ptr->size()) { + AddError(node, "Failed to load type annotation"); + return nullptr; + } + + return CompileTypeAnnotationNode(*(*ptr)[TypeAnnotationIndex]); + } + const TTypeAnnotationNode* CompileTypeAnnotationNode(const TAstNode& node) { - if (node.IsAtom()) { + if (node.IsAtom()) { if (node.GetContent() == TStringBuf(".")) { - return nullptr; - } + return nullptr; + } else if (node.GetContent() == TStringBuf("Unit")) { return Expr.MakeType<TUnitExprType>(); - } + } else if (node.GetContent() == TStringBuf("World")) { return Expr.MakeType<TWorldExprType>(); - } + } else if (node.GetContent() == TStringBuf("Void")) { return Expr.MakeType<TVoidExprType>(); - } + } else if (node.GetContent() == TStringBuf("Null")) { return Expr.MakeType<TNullExprType>(); } else if (node.GetContent() == TStringBuf("Generic")) { return Expr.MakeType<TGenericExprType>(); - } + } else if (node.GetContent() == TStringBuf("EmptyList")) { return Expr.MakeType<TEmptyListExprType>(); } else if (node.GetContent() == TStringBuf("EmptyDict")) { return Expr.MakeType<TEmptyDictExprType>(); } - else { - AddError(node, TStringBuilder() << "Unknown type annotation: " << node.GetContent()); - return nullptr; - } - } else { - if (node.GetChildrenCount() == 0) { - AddError(node, "Bad type annotation, expected not empty list"); - return nullptr; - } - - if (!node.GetChild(0)->IsAtom()) { - AddError(node, "Bad type annotation, first list item must be an atom"); - return nullptr; - } - - auto content = node.GetChild(0)->GetContent(); + else { + AddError(node, TStringBuilder() << "Unknown type annotation: " << node.GetContent()); + return nullptr; + } + } else { + if (node.GetChildrenCount() == 0) { + AddError(node, "Bad type annotation, expected not empty list"); + return nullptr; + } + + if (!node.GetChild(0)->IsAtom()) { + AddError(node, "Bad type annotation, first list item must be an atom"); + return nullptr; + } + + auto content = node.GetChild(0)->GetContent(); if (content == TStringBuf("Data")) { const auto count = node.GetChildrenCount(); if (!(count == 2 || count == 4) || !node.GetChild(1)->IsAtom()) { - AddError(node, "Bad data type annotation"); - return nullptr; - } - + AddError(node, "Bad data type annotation"); + return nullptr; + } + auto slot = NUdf::FindDataSlot(node.GetChild(1)->GetContent()); if (!slot) { AddError(node, "Bad data type annotation"); @@ -231,15 +231,15 @@ namespace { return ann; } } else if (content == TStringBuf("List")) { - if (node.GetChildrenCount() != 2) { - AddError(node, "Bad list type annotation"); - return nullptr; - } - - auto r = CompileTypeAnnotationNode(*node.GetChild(1)); - if (!r) - return nullptr; - + if (node.GetChildrenCount() != 2) { + AddError(node, "Bad list type annotation"); + return nullptr; + } + + auto r = CompileTypeAnnotationNode(*node.GetChild(1)); + if (!r) + return nullptr; + return Expr.MakeType<TListExprType>(r); } else if (content == TStringBuf("Stream")) { if (node.GetChildrenCount() != 2) { @@ -254,25 +254,25 @@ namespace { return Expr.MakeType<TStreamExprType>(r); } else if (content == TStringBuf("Struct")) { TVector<const TItemExprType*> children; - for (size_t index = 1; index < node.GetChildrenCount(); ++index) { - auto r = CompileTypeAnnotationNode(*node.GetChild(index)); - if (!r) - return nullptr; - - if (r->GetKind() != ETypeAnnotationKind::Item) { - AddError(node, "Expected item type annotation"); - return nullptr; - } - - children.push_back(r->Cast<TItemExprType>()); - } - + for (size_t index = 1; index < node.GetChildrenCount(); ++index) { + auto r = CompileTypeAnnotationNode(*node.GetChild(index)); + if (!r) + return nullptr; + + if (r->GetKind() != ETypeAnnotationKind::Item) { + AddError(node, "Expected item type annotation"); + return nullptr; + } + + children.push_back(r->Cast<TItemExprType>()); + } + auto ann = Expr.MakeType<TStructExprType>(children); if (!ann->Validate(node.GetPosition(), Expr)) { - return nullptr; - } - - return ann; + return nullptr; + } + + return ann; } else if (content == TStringBuf("Multi")) { TTypeAnnotationNode::TListType children; for (size_t index = 1; index < node.GetChildrenCount(); ++index) { @@ -291,14 +291,14 @@ namespace { return ann; } else if (content == TStringBuf("Tuple")) { TTypeAnnotationNode::TListType children; - for (size_t index = 1; index < node.GetChildrenCount(); ++index) { - auto r = CompileTypeAnnotationNode(*node.GetChild(index)); - if (!r) - return nullptr; - - children.push_back(r); - } - + for (size_t index = 1; index < node.GetChildrenCount(); ++index) { + auto r = CompileTypeAnnotationNode(*node.GetChild(index)); + if (!r) + return nullptr; + + children.push_back(r); + } + auto ann = Expr.MakeType<TTupleExprType>(children); if (!ann->Validate(node.GetPosition(), Expr)) { return nullptr; @@ -306,74 +306,74 @@ namespace { return ann; } else if (content == TStringBuf("Item")) { - if (node.GetChildrenCount() != 3 || !node.GetChild(1)->IsAtom()) { - AddError(node, "Bad item type annotation"); - return nullptr; - } - - auto r = CompileTypeAnnotationNode(*node.GetChild(2)); - if (!r) - return nullptr; - + if (node.GetChildrenCount() != 3 || !node.GetChild(1)->IsAtom()) { + AddError(node, "Bad item type annotation"); + return nullptr; + } + + auto r = CompileTypeAnnotationNode(*node.GetChild(2)); + if (!r) + return nullptr; + return Expr.MakeType<TItemExprType>(TString(node.GetChild(1)->GetContent()), r); } else if (content == TStringBuf("Optional")) { - if (node.GetChildrenCount() != 2) { + if (node.GetChildrenCount() != 2) { AddError(node, "Bad optional type annotation"); - return nullptr; - } - - auto r = CompileTypeAnnotationNode(*node.GetChild(1)); - if (!r) - return nullptr; - + return nullptr; + } + + auto r = CompileTypeAnnotationNode(*node.GetChild(1)); + if (!r) + return nullptr; + return Expr.MakeType<TOptionalExprType>(r); } else if (content == TStringBuf("Type")) { - if (node.GetChildrenCount() != 2) { - AddError(node, "Bad type annotation"); - return nullptr; - } - - auto r = CompileTypeAnnotationNode(*node.GetChild(1)); - if (!r) - return nullptr; - + if (node.GetChildrenCount() != 2) { + AddError(node, "Bad type annotation"); + return nullptr; + } + + auto r = CompileTypeAnnotationNode(*node.GetChild(1)); + if (!r) + return nullptr; + return Expr.MakeType<TTypeExprType>(r); - } + } else if (content == TStringBuf("Dict")) { - if (node.GetChildrenCount() != 3) { - AddError(node, "Bad dict annotation"); - return nullptr; - } - - auto r1 = CompileTypeAnnotationNode(*node.GetChild(1)); - if (!r1) - return nullptr; - - auto r2 = CompileTypeAnnotationNode(*node.GetChild(2)); - if (!r2) - return nullptr; - + if (node.GetChildrenCount() != 3) { + AddError(node, "Bad dict annotation"); + return nullptr; + } + + auto r1 = CompileTypeAnnotationNode(*node.GetChild(1)); + if (!r1) + return nullptr; + + auto r2 = CompileTypeAnnotationNode(*node.GetChild(2)); + if (!r2) + return nullptr; + auto ann = Expr.MakeType<TDictExprType>(r1, r2); if (!ann->Validate(node.GetPosition(), Expr)) { - return nullptr; - } - - return ann; + return nullptr; + } + + return ann; } else if (content == TStringBuf("Callable")) { if (node.GetChildrenCount() <= 2) { - AddError(node, "Bad callable annotation"); - return nullptr; - } - + AddError(node, "Bad callable annotation"); + return nullptr; + } + TVector<TCallableExprType::TArgumentInfo> args; - size_t optCount = 0; + size_t optCount = 0; TString payload; if (!node.GetChild(1)->IsList()) { AddError(node, "Bad callable annotation - expected list"); - return nullptr; - } - + return nullptr; + } + if (node.GetChild(1)->GetChildrenCount() > 2) { AddError(node, "Bad callable annotation - too many settings nodes"); return nullptr; @@ -412,7 +412,7 @@ namespace { if (!retType) return nullptr; - for (size_t index = 3; index < node.GetChildrenCount(); ++index) { + for (size_t index = 3; index < node.GetChildrenCount(); ++index) { auto argSettings = node.GetChild(index); if (!argSettings->IsList() || argSettings->GetChildrenCount() < 1 || argSettings->GetChildrenCount() > 3) { @@ -421,9 +421,9 @@ namespace { } auto r = CompileTypeAnnotationNode(*argSettings->GetChild(0)); - if (!r) - return nullptr; - + if (!r) + return nullptr; + TCallableExprType::TArgumentInfo arg; arg.Type = r; @@ -451,8 +451,8 @@ namespace { } args.push_back(arg); - } - + } + auto ann = Expr.MakeType<TCallableExprType>(retType, args, optCount, payload); if (!ann->Validate(node.GetPosition(), Expr)) { return nullptr; @@ -542,74 +542,74 @@ namespace { return nullptr; return Expr.MakeType<TFlowExprType>(r); - } else { + } else { AddError(node, TStringBuilder() << "Unknown type annotation"); - return nullptr; - } - } - } - }; - - TAstNode* ConvertTypeAnnotationToAst(const TTypeAnnotationNode& annotation, TMemoryPool& pool, bool refAtoms) { - switch (annotation.GetKind()) { + return nullptr; + } + } + } + }; + + TAstNode* ConvertTypeAnnotationToAst(const TTypeAnnotationNode& annotation, TMemoryPool& pool, bool refAtoms) { + switch (annotation.GetKind()) { case ETypeAnnotationKind::Unit: - { + { return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Unit"), pool); - } - - case ETypeAnnotationKind::Tuple: - { + } + + case ETypeAnnotationKind::Tuple: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Tuple"), pool); - TSmallVec<TAstNode*> children; - children.push_back(self); - for (auto& child : annotation.Cast<TTupleExprType>()->GetItems()) { - children.push_back(ConvertTypeAnnotationToAst(*child, pool, refAtoms)); - } - - return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); - } - - case ETypeAnnotationKind::Struct: - { + TSmallVec<TAstNode*> children; + children.push_back(self); + for (auto& child : annotation.Cast<TTupleExprType>()->GetItems()) { + children.push_back(ConvertTypeAnnotationToAst(*child, pool, refAtoms)); + } + + return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); + } + + case ETypeAnnotationKind::Struct: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Struct"), pool); - TSmallVec<TAstNode*> children; - children.push_back(self); - for (auto& child : annotation.Cast<TStructExprType>()->GetItems()) { - children.push_back(ConvertTypeAnnotationToAst(*child, pool, refAtoms)); - } - - return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); - } - - case ETypeAnnotationKind::List: - { + TSmallVec<TAstNode*> children; + children.push_back(self); + for (auto& child : annotation.Cast<TStructExprType>()->GetItems()) { + children.push_back(ConvertTypeAnnotationToAst(*child, pool, refAtoms)); + } + + return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); + } + + case ETypeAnnotationKind::List: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("List"), pool); - auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TListExprType>()->GetItemType(), pool, refAtoms); - return TAstNode::NewList(TPosition(), pool, self, itemType); - } - - case ETypeAnnotationKind::Optional: - { + auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TListExprType>()->GetItemType(), pool, refAtoms); + return TAstNode::NewList(TPosition(), pool, self, itemType); + } + + case ETypeAnnotationKind::Optional: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Optional"), pool); - auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TOptionalExprType>()->GetItemType(), pool, refAtoms); - return TAstNode::NewList(TPosition(), pool, self, itemType); - } - - case ETypeAnnotationKind::Item: - { - auto casted = annotation.Cast<TItemExprType>(); + auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TOptionalExprType>()->GetItemType(), pool, refAtoms); + return TAstNode::NewList(TPosition(), pool, self, itemType); + } + + case ETypeAnnotationKind::Item: + { + auto casted = annotation.Cast<TItemExprType>(); auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Item"), pool); - auto name = refAtoms ? - TAstNode::NewLiteralAtom(TPosition(), casted->GetName(), pool, TNodeFlags::ArbitraryContent) : - TAstNode::NewAtom(TPosition(), casted->GetName(), pool, TNodeFlags::ArbitraryContent); - auto itemType = ConvertTypeAnnotationToAst(*casted->GetItemType(), pool, refAtoms); - return TAstNode::NewList(TPosition(), pool, self, name, itemType); - } - - case ETypeAnnotationKind::Data: - { + auto name = refAtoms ? + TAstNode::NewLiteralAtom(TPosition(), casted->GetName(), pool, TNodeFlags::ArbitraryContent) : + TAstNode::NewAtom(TPosition(), casted->GetName(), pool, TNodeFlags::ArbitraryContent); + auto itemType = ConvertTypeAnnotationToAst(*casted->GetItemType(), pool, refAtoms); + return TAstNode::NewList(TPosition(), pool, self, name, itemType); + } + + case ETypeAnnotationKind::Data: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Data"), pool); - auto datatype = refAtoms ? + auto datatype = refAtoms ? TAstNode::NewLiteralAtom(TPosition(), annotation.Cast<TDataExprType>()->GetName(), pool) : TAstNode::NewAtom(TPosition(), annotation.Cast<TDataExprType>()->GetName(), pool); if (auto params = dynamic_cast<const TDataExprParamsType*>(&annotation)) { @@ -624,44 +624,44 @@ namespace { return TAstNode::NewList(TPosition(), pool, self, datatype, param1, param2); } - return TAstNode::NewList(TPosition(), pool, self, datatype); - } - - case ETypeAnnotationKind::World: - { + return TAstNode::NewList(TPosition(), pool, self, datatype); + } + + case ETypeAnnotationKind::World: + { return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("World"), pool); - } - - case ETypeAnnotationKind::Type: - { + } + + case ETypeAnnotationKind::Type: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Type"), pool); - auto type = ConvertTypeAnnotationToAst(*annotation.Cast<TTypeExprType>()->GetType(), pool, refAtoms); - return TAstNode::NewList(TPosition(), pool, self, type); - } - - case ETypeAnnotationKind::Dict: - { + auto type = ConvertTypeAnnotationToAst(*annotation.Cast<TTypeExprType>()->GetType(), pool, refAtoms); + return TAstNode::NewList(TPosition(), pool, self, type); + } + + case ETypeAnnotationKind::Dict: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Dict"), pool); - auto dictType = annotation.Cast<TDictExprType>(); - auto keyType = ConvertTypeAnnotationToAst(*dictType->GetKeyType(), pool, refAtoms); - auto payloadType = ConvertTypeAnnotationToAst(*dictType->GetPayloadType(), pool, refAtoms); - return TAstNode::NewList(TPosition(), pool, self, keyType, payloadType); - } - - case ETypeAnnotationKind::Void: - { + auto dictType = annotation.Cast<TDictExprType>(); + auto keyType = ConvertTypeAnnotationToAst(*dictType->GetKeyType(), pool, refAtoms); + auto payloadType = ConvertTypeAnnotationToAst(*dictType->GetPayloadType(), pool, refAtoms); + return TAstNode::NewList(TPosition(), pool, self, keyType, payloadType); + } + + case ETypeAnnotationKind::Void: + { return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Void"), pool); - } - + } + case ETypeAnnotationKind::Null: { return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Null"), pool); } - case ETypeAnnotationKind::Callable: - { + case ETypeAnnotationKind::Callable: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Callable"), pool); - auto callable = annotation.Cast<TCallableExprType>(); + auto callable = annotation.Cast<TCallableExprType>(); TSmallVec<TAstNode*> mainSettings; if (callable->GetOptionalArgumentsCount() > 0 || !callable->GetPayload().empty()) { auto optArgs = TAstNode::NewAtom(TPosition(), ToString(callable->GetOptionalArgumentsCount()), pool); @@ -674,8 +674,8 @@ namespace { mainSettings.push_back(payload); } - TSmallVec<TAstNode*> children; - children.push_back(self); + TSmallVec<TAstNode*> children; + children.push_back(self); children.push_back(TAstNode::NewList(TPosition(), mainSettings.data(), mainSettings.size(), pool)); @@ -697,25 +697,25 @@ namespace { } children.push_back(TAstNode::NewList(TPosition(), argSettings.data(), argSettings.size(), pool)); - } - - return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); - } - + } + + return TAstNode::NewList(TPosition(), children.data(), children.size(), pool); + } + case ETypeAnnotationKind::Generic: - { + { return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Generic"), pool); - } - - case ETypeAnnotationKind::Resource: - { + } + + case ETypeAnnotationKind::Resource: + { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Resource"), pool); auto restype = refAtoms ? TAstNode::NewLiteralAtom(TPosition(), annotation.Cast<TResourceExprType>()->GetTag(), pool, TNodeFlags::ArbitraryContent) : TAstNode::NewAtom(TPosition(), annotation.Cast<TResourceExprType>()->GetTag(), pool, TNodeFlags::ArbitraryContent); return TAstNode::NewList(TPosition(), pool, self, restype); - } - + } + case ETypeAnnotationKind::Tagged: { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Tagged"), pool); @@ -760,7 +760,7 @@ namespace { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Flow"), pool); auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TFlowExprType>()->GetItemType(), pool, refAtoms); return TAstNode::NewList(TPosition(), pool, self, itemType); - } + } case ETypeAnnotationKind::Multi: { @@ -786,90 +786,90 @@ namespace { YQL_ENSURE(false, "Unknown kind: " << annotation.GetKind()); } - } - - TAstNode* AnnotateAstNode(TAstNode* node, const TExprNode* exprNode, ui32 flags, TMemoryPool& pool, bool refAtoms) { - if (!flags) - return node; - - TSmallVec<TAstNode*> children; - if (flags & TExprAnnotationFlags::Position) { - children.push_back(PositionAsNode(node->GetPosition(), pool)); - } - - if ((flags & TExprAnnotationFlags::Types)) { - TAstNode* typeAnn = nullptr; - if (exprNode) { + } + + TAstNode* AnnotateAstNode(TAstNode* node, const TExprNode* exprNode, ui32 flags, TMemoryPool& pool, bool refAtoms) { + if (!flags) + return node; + + TSmallVec<TAstNode*> children; + if (flags & TExprAnnotationFlags::Position) { + children.push_back(PositionAsNode(node->GetPosition(), pool)); + } + + if ((flags & TExprAnnotationFlags::Types)) { + TAstNode* typeAnn = nullptr; + if (exprNode) { YQL_ENSURE(exprNode->GetTypeAnn()); typeAnn = ConvertTypeAnnotationToAst(*exprNode->GetTypeAnn(), pool, refAtoms); - } else { + } else { typeAnn = TAstNode::NewLiteralAtom(node->GetPosition(), TStringBuf("."), pool); - } - - children.push_back(typeAnn); - } - - children.push_back(node); - return TAstNode::NewList(node->GetPosition(), children.data(), children.size(), pool); - } - + } + + children.push_back(typeAnn); + } + + children.push_back(node); + return TAstNode::NewList(node->GetPosition(), children.data(), children.size(), pool); + } + TExprNode::TListType Compile(const TAstNode& node, TContext& ctx); - + TExprNode::TPtr CompileQuote(const TAstNode& node, TContext& ctx) { - if (node.IsAtom()) { + if (node.IsAtom()) { return ctx.ProcessNode(node, ctx.Expr.NewAtom(node.GetPosition(), TString(node.GetContent()), node.GetFlags())); - } else { + } else { TExprNode::TListType children; children.reserve(node.GetChildrenCount()); - for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { - auto r = Compile(*node.GetChild(index), ctx); + for (ui32 index = 0; index < node.GetChildrenCount(); ++index) { + auto r = Compile(*node.GetChild(index), ctx); if (r.empty()) return {}; - + std::move(r.begin(), r.end(), std::back_inserter(children)); - } - + } + return ctx.ProcessNode(node, ctx.Expr.NewList(node.GetPosition(), std::move(children))); - } - } - + } + } + TExprNode::TListType CompileLambda(const TAstNode& node, TContext& ctx) { if (node.GetChildrenCount() < 2) { ctx.AddError(node, "Expected size of list at least 3."); return {}; - } - + } + const auto args = node.GetChild(1); - if (!args->IsList() || args->GetChildrenCount() != 2 || !args->GetChild(0)->IsAtom() || + if (!args->IsList() || args->GetChildrenCount() != 2 || !args->GetChild(0)->IsAtom() || args->GetChild(0)->GetContent() != TStringBuf("quote") || !args->GetChild(1)->IsList()) { - ctx.AddError(node, "Lambda arguments must be a quoted list of atoms"); + ctx.AddError(node, "Lambda arguments must be a quoted list of atoms"); return {}; - } - + } + const auto params = args->GetChild(1); - for (ui32 index = 0; index < params->GetChildrenCount(); ++index) { - if (!params->GetChild(index)->IsAtom()) { - ctx.AddError(node, "Lambda arguments must be a quoted list of atoms"); + for (ui32 index = 0; index < params->GetChildrenCount(); ++index) { + if (!params->GetChild(index)->IsAtom()) { + ctx.AddError(node, "Lambda arguments must be a quoted list of atoms"); return {}; - } - } - - ctx.PushFrame(); + } + } + + ctx.PushFrame(); TExprNode::TListType argNodes; - for (ui32 index = 0; index < params->GetChildrenCount(); ++index) { - auto arg = params->GetChild(index); + for (ui32 index = 0; index < params->GetChildrenCount(); ++index) { + auto arg = params->GetChild(index); auto lambdaArg = ctx.ProcessNode(*arg, ctx.Expr.NewArgument(arg->GetPosition(), TString(arg->GetContent()))); argNodes.push_back(lambdaArg); - auto& binding = ctx.Frames.back().Bindings[arg->GetContent()]; + auto& binding = ctx.Frames.back().Bindings[arg->GetContent()]; if (!binding.empty()) { - ctx.PopFrame(); - ctx.AddError(*arg, TStringBuilder() << "Duplicated name of lambda parameter: " << arg->GetContent()); + ctx.PopFrame(); + ctx.AddError(*arg, TStringBuilder() << "Duplicated name of lambda parameter: " << arg->GetContent()); return {}; - } - + } + binding = {lambdaArg}; - } - + } + TExprNode::TListType body; body.reserve(node.GetChildrenCount() - 2U); for (auto i = 2U; i < node.GetChildrenCount(); ++i) { @@ -879,12 +879,12 @@ namespace { std::move(r.begin(), r.end(), std::back_inserter(body)); } ctx.PopFrame(); - + auto arguments = ctx.ProcessNode(*args, ctx.Expr.NewArguments(args->GetPosition(), std::move(argNodes))); auto lambda = ctx.ProcessNode(node, ctx.Expr.NewLambda(node.GetPosition(), std::move(arguments), std::move(body))); return {lambda}; - } - + } + bool CompileSetPackageVersion(const TAstNode& node, TContext& ctx) { if (node.GetChildrenCount() != 3) { ctx.AddError(node, "Expected list of size 3"); @@ -1143,17 +1143,17 @@ namespace { } TExprNode::TListType CompileFunction(const TAstNode& root, TContext& ctx, bool topLevel = false) { - if (!root.IsList()) { - ctx.AddError(root, "Expected list"); + if (!root.IsList()) { + ctx.AddError(root, "Expected list"); return {}; - } - + } + if (ctx.Frames.size() > 1000U) { ctx.AddError(root, "Too deep graph!"); return {}; } - ctx.PushFrame(); + ctx.PushFrame(); if (topLevel) { for (ui32 index = 0; index < root.GetChildrenCount(); ++index) { const auto node = root.GetChild(index); @@ -1191,30 +1191,30 @@ namespace { } } - for (ui32 index = 0; index < root.GetChildrenCount(); ++index) { + for (ui32 index = 0; index < root.GetChildrenCount(); ++index) { const auto node = root.GetChild(index); if (!ctx.Frames.back().Return.empty()) { ctx.Frames.back().Return.clear(); - ctx.AddError(*node, "Return is already exist"); + ctx.AddError(*node, "Return is already exist"); return {}; - } - - if (!node->IsList()) { - ctx.AddError(*node, "Expected list"); + } + + if (!node->IsList()) { + ctx.AddError(*node, "Expected list"); return {}; - } - - if (node->GetChildrenCount() == 0) { - ctx.AddError(*node, "Expected not empty list"); + } + + if (node->GetChildrenCount() == 0) { + ctx.AddError(*node, "Expected not empty list"); return {}; - } - - auto firstChild = node->GetChild(0); - if (!firstChild->IsAtom()) { - ctx.AddError(*firstChild, "Expected atom"); + } + + auto firstChild = node->GetChild(0); + if (!firstChild->IsAtom()) { + ctx.AddError(*firstChild, "Expected atom"); return {}; - } - + } + if (firstChild->GetContent() == TStringBuf("let")) { if (!CompileLet(*node, ctx)) return {}; @@ -1246,21 +1246,21 @@ namespace { } continue; - } else { + } else { ctx.AddError(*firstChild, ToString("expected either let, return or import, but have ") + firstChild->GetContent()); return {}; - } - } - + } + } + auto ret = std::move(ctx.Frames.back().Return); - ctx.PopFrame(); + ctx.PopFrame(); if (ret.empty()) { - ctx.AddError(root, "No return found"); - } - - return ret; - } - + ctx.AddError(root, "No return found"); + } + + return ret; + } + bool CompileLibrary(const TAstNode& root, TContext& ctx) { if (!root.IsList()) { ctx.AddError(root, "Expected list"); @@ -1307,48 +1307,48 @@ namespace { } TExprNode::TListType Compile(const TAstNode& node, TContext& ctx) { - if (node.IsAtom()) { + if (node.IsAtom()) { const auto foundNode = ctx.FindBinding(node.GetContent()); if (foundNode.empty()) { - ctx.AddError(node, TStringBuilder() << "Name not found: " << node.GetContent()); + ctx.AddError(node, TStringBuilder() << "Name not found: " << node.GetContent()); return {}; - } - - return foundNode; - } - - if (node.GetChildrenCount() == 0) { - ctx.AddError(node, "Empty list, did you forget quote?"); + } + + return foundNode; + } + + if (node.GetChildrenCount() == 0) { + ctx.AddError(node, "Empty list, did you forget quote?"); return {}; - } - - if (!node.GetChild(0)->IsAtom()) { - ctx.AddError(node, "First item in list is not an atom, did you forget quote?"); + } + + if (!node.GetChild(0)->IsAtom()) { + ctx.AddError(node, "First item in list is not an atom, did you forget quote?"); return {}; - } - - auto function = node.GetChild(0)->GetContent(); + } + + auto function = node.GetChild(0)->GetContent(); if (function == TStringBuf("quote")) { - if (node.GetChildrenCount() != 2) { - ctx.AddError(node, "Quote should have one argument"); + if (node.GetChildrenCount() != 2) { + ctx.AddError(node, "Quote should have one argument"); return {}; - } - + } + if (auto quote = CompileQuote(*node.GetChild(1), ctx)) return {std::move(quote)}; return {}; - } - + } + if (function == TStringBuf("let") || function == TStringBuf("return")) { - ctx.AddError(node, "Let and return should be used only at first level or inside def"); + ctx.AddError(node, "Let and return should be used only at first level or inside def"); return {}; - } - + } + if (function == TStringBuf("lambda")) { - return CompileLambda(node, ctx); - } - + return CompileLambda(node, ctx); + } + if (function == TStringBuf("bind")) { if (auto bind = CompileBind(node, ctx)) return {std::move(bind)}; @@ -1356,53 +1356,53 @@ namespace { } if (function == TStringBuf("block")) { - if (node.GetChildrenCount() != 2) { - ctx.AddError(node, "Block should have one argument"); + if (node.GetChildrenCount() != 2) { + ctx.AddError(node, "Block should have one argument"); return {}; - } - + } + const auto quotedList = node.GetChild(1); - if (quotedList->GetChildrenCount() != 2 || !quotedList->GetChild(0)->IsAtom() || + if (quotedList->GetChildrenCount() != 2 || !quotedList->GetChild(0)->IsAtom() || quotedList->GetChild(0)->GetContent() != TStringBuf("quote")) { - ctx.AddError(node, "Expected quoted list"); + ctx.AddError(node, "Expected quoted list"); return {}; - } - - return CompileFunction(*quotedList->GetChild(1), ctx); - } - + } + + return CompileFunction(*quotedList->GetChild(1), ctx); + } + TExprNode::TListType children; children.reserve(node.GetChildrenCount() - 1U); for (auto index = 1U; index < node.GetChildrenCount(); ++index) { - auto r = Compile(*node.GetChild(index), ctx); + auto r = Compile(*node.GetChild(index), ctx); if (r.empty()) return {}; - + std::move(r.begin(), r.end(), std::back_inserter(children)); - } - + } + return {ctx.ProcessNode(node, ctx.Expr.NewCallable(node.GetPosition(), TString(function), std::move(children)))}; - } - - struct TFrameContext { - size_t Index = 0; - size_t Parent = 0; + } + + struct TFrameContext { + size_t Index = 0; + size_t Parent = 0; std::map<size_t, const TExprNode*> Nodes; std::vector<const TExprNode*> TopoSortedNodes; TNodeMap<TString> Bindings; - }; - - struct TVisitNodeContext { + }; + + struct TVisitNodeContext { explicit TVisitNodeContext(TExprContext& expr) : Expr(expr) {} TExprContext& Expr; size_t Order = 0ULL; - bool RefAtoms = false; + bool RefAtoms = false; std::unique_ptr<TMemoryPool> Pool; std::vector<TFrameContext> Frames; - TFrameContext* CurrentFrame = nullptr; + TFrameContext* CurrentFrame = nullptr; TNodeMap<size_t> LambdaFrames; std::map<TStringBuf, std::pair<const TExprNode*, TAstNode*>> Parameters; @@ -1435,8 +1435,8 @@ namespace { return 0ULL; } - }; - + }; + void VisitArguments(const TExprNode& node, TVisitNodeContext& ctx) { YQL_ENSURE(node.Type() == TExprNode::Arguments); for (const auto& arg : node.Children()) { @@ -1472,7 +1472,7 @@ namespace { void VisitNode(const TExprNode& node, size_t neighbors, TVisitNodeContext& ctx) { if (TExprNode::Argument == node.Type()) return; - + auto& counts = ctx.References[&node]; counts.Neighbors += neighbors; if (counts.References++) { @@ -1506,15 +1506,15 @@ namespace { counts.Order = ++ctx.Order; ctx.CurrentFrame->Nodes.emplace(counts.Order, &node); - } - } - + } + } + using TRoots = TSmallVec<const TExprNode*>; - + TAstNode* ConvertFunction(TPositionHandle position, const TRoots& roots, TVisitNodeContext& ctx, ui32 annotationFlags, TMemoryPool& pool); TAstNode* BuildValueNode(const TExprNode& node, TVisitNodeContext& ctx, const TString& topLevelName, ui32 annotationFlags, TMemoryPool& pool, bool useBindings) { - TAstNode* res = nullptr; + TAstNode* res = nullptr; const auto& name = ctx.FindBinding(&node); if (!name.empty() && name != topLevelName && useBindings) { res = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); @@ -1528,26 +1528,26 @@ namespace { TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()) : TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), node.Content(), pool, node.Flags()), &node, annotationFlags, pool, ctx.RefAtoms); - + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, content); break; - } - + } + case TExprNode::List: { TSmallVec<TAstNode*> values; for (const auto& child : node.Children()) { values.push_back(BuildValueNode(*child, ctx, topLevelName, annotationFlags, pool, useBindings)); } - + auto quote = AnnotateAstNode(&TAstNode::QuoteAtom, nullptr, annotationFlags, pool, ctx.RefAtoms); auto list = AnnotateAstNode(TAstNode::NewList( ctx.Expr.GetPosition(node.Pos()), values.data(), values.size(), pool), &node, annotationFlags, pool, ctx.RefAtoms); - + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), pool, quote, list); break; - } - + } + case TExprNode::Callable: { if (node.Content() == "Parameter") { @@ -1588,11 +1588,11 @@ namespace { for (const auto& child : node.Children()) { children.push_back(BuildValueNode(*child, ctx, topLevelName, annotationFlags, pool, useBindings)); } - + res = TAstNode::NewList(ctx.Expr.GetPosition(node.Pos()), children.data(), children.size(), pool); break; - } - + } + case TExprNode::Lambda: { const auto prevFrame = ctx.CurrentFrame; @@ -1605,16 +1605,16 @@ namespace { const auto atom = TAstNode::NewAtom(ctx.Expr.GetPosition(node.Pos()), name, pool); argsChildren.emplace_back(AnnotateAstNode(atom, arg.Get(), annotationFlags, pool, ctx.RefAtoms)); } - + auto argsNode = TAstNode::NewList(ctx.Expr.GetPosition(args.Pos()), argsChildren.data(), argsChildren.size(), pool); auto argsContainer = TAstNode::NewList(ctx.Expr.GetPosition(args.Pos()), pool, AnnotateAstNode(&TAstNode::QuoteAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), AnnotateAstNode(argsNode, nullptr, annotationFlags, pool, ctx.RefAtoms)); - + const bool block = ctx.CurrentFrame->Bindings.cend() != std::find_if(ctx.CurrentFrame->Bindings.cbegin(), ctx.CurrentFrame->Bindings.cend(), [](const auto& bind) { return bind.first->Type() != TExprNode::Argument; } ); - + if (block) { TSmallVec<const TExprNode*> body(node.ChildrenSize() - 1U); for (ui32 i = 0U; i < body.size(); ++i) @@ -1655,32 +1655,32 @@ namespace { break; default: YQL_ENSURE(false, "Unknown type: " << static_cast<ui32>(node.Type())); - } - } - + } + } + return AnnotateAstNode(res, &node, annotationFlags, pool, ctx.RefAtoms); - } - + } + TAstNode* ConvertFunction(TPositionHandle position, const TRoots& roots, TVisitNodeContext& ctx, ui32 annotationFlags, TMemoryPool& pool) { YQL_ENSURE(!roots.empty(), "Missed roots."); - TSmallVec<TAstNode*> children; + TSmallVec<TAstNode*> children; for (const auto& node : ctx.CurrentFrame->TopoSortedNodes) { const auto& name = ctx.FindBinding(node); if (name.empty() || node->Type() == TExprNode::Arguments || node->Type() == TExprNode::Argument) { - continue; - } - + continue; + } + const auto letAtom = TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(node->Pos()), TStringBuf("let"), pool); const auto nameAtom = TAstNode::NewAtom(ctx.Expr.GetPosition(node->Pos()), name, pool); const auto valueNode = BuildValueNode(*node, ctx, name, annotationFlags, pool, true); - + const auto letNode = TAstNode::NewList(ctx.Expr.GetPosition(node->Pos()), pool, - AnnotateAstNode(letAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), - AnnotateAstNode(nameAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), - valueNode); - children.push_back(AnnotateAstNode(letNode, nullptr, annotationFlags, pool, ctx.RefAtoms)); - } - + AnnotateAstNode(letAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), + AnnotateAstNode(nameAtom, nullptr, annotationFlags, pool, ctx.RefAtoms), + valueNode); + children.push_back(AnnotateAstNode(letNode, nullptr, annotationFlags, pool, ctx.RefAtoms)); + } + const auto returnAtom = TAstNode::NewLiteralAtom(ctx.Expr.GetPosition(position), TStringBuf("return"), pool); TSmallVec<TAstNode*> returnChildren; returnChildren.reserve(roots.size() + 1U); @@ -1703,8 +1703,8 @@ namespace { } const auto res = TAstNode::NewList(ctx.Expr.GetPosition(position), children.data(), children.size(), pool); - return AnnotateAstNode(res, nullptr, annotationFlags, pool, ctx.RefAtoms); - } + return AnnotateAstNode(res, nullptr, annotationFlags, pool, ctx.RefAtoms); + } bool InlineNode(const TExprNode& node, size_t references, size_t neighbors, const TConvertToAstSettings& settings) { if (settings.NoInlineFunc) { @@ -1935,55 +1935,55 @@ namespace { return true; } -} // namespace - +} // namespace + bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx, IModuleResolver* resolver, bool hasAnnotations, ui32 typeAnnotationIndex, ui16 syntaxVersion) { exprRoot.Reset(); TAstNode* cleanRoot = nullptr; - TAnnotationNodeMap annotations; + TAnnotationNodeMap annotations; const TAnnotationNodeMap* currentAnnotations = nullptr; TAstParseResult cleanupRes; - if (!hasAnnotations) { - typeAnnotationIndex = Max<ui32>(); - cleanRoot = &astRoot; - currentAnnotations = nullptr; - } else if (typeAnnotationIndex != Max<ui32>()) { + if (!hasAnnotations) { + typeAnnotationIndex = Max<ui32>(); + cleanRoot = &astRoot; + currentAnnotations = nullptr; + } else if (typeAnnotationIndex != Max<ui32>()) { cleanupRes.Pool = std::make_unique<TMemoryPool>(4096); cleanRoot = ExtractAnnotations(astRoot, annotations, *cleanupRes.Pool); cleanupRes.Root = cleanRoot; - currentAnnotations = &annotations; - } else { + currentAnnotations = &annotations; + } else { cleanupRes.Pool = std::make_unique<TMemoryPool>(4096); cleanRoot = RemoveAnnotations(astRoot, *cleanupRes.Pool); cleanupRes.Root = cleanRoot; - currentAnnotations = nullptr; - } - + currentAnnotations = nullptr; + } + if (!cleanRoot) { return false; } - TContext compileCtx(ctx); + TContext compileCtx(ctx); compileCtx.SyntaxVersion = syntaxVersion; - compileCtx.Annotations = currentAnnotations; - compileCtx.TypeAnnotationIndex = typeAnnotationIndex; + compileCtx.Annotations = currentAnnotations; + compileCtx.TypeAnnotationIndex = typeAnnotationIndex; compileCtx.ModuleResolver = resolver; - compileCtx.PushFrame(); + compileCtx.PushFrame(); auto world = compileCtx.Expr.NewWorld(astRoot.GetPosition()); - if (typeAnnotationIndex != Max<ui32>()) { + if (typeAnnotationIndex != Max<ui32>()) { world->SetTypeAnn(compileCtx.Expr.MakeType<TWorldExprType>()); - } - + } + compileCtx.Frames.back().Bindings[TStringBuf("world")] = {std::move(world)}; auto ret = CompileFunction(*cleanRoot, compileCtx, true); if (1U != ret.size()) return false; exprRoot = std::move(ret.front()); - compileCtx.PopFrame(); + compileCtx.PopFrame(); return bool(exprRoot); -} - +} + bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx, IModuleResolver* resolver, ui32 annotationFlags, ui16 syntaxVersion) { @@ -2009,10 +2009,10 @@ bool CompileExpr(TAstNode& astRoot, TLibraryCohesion& library, TExprContext& ctx } const TTypeAnnotationNode* CompileTypeAnnotation(const TAstNode& node, TExprContext& ctx) { - TContext compileCtx(ctx); - return compileCtx.CompileTypeAnnotationNode(node); -} - + TContext compileCtx(ctx); + return compileCtx.CompileTypeAnnotationNode(node); +} + template<class Set> bool IsDependedImpl(const TExprNode& node, const Set& dependences, TNodeSet& visited) { if (!visited.emplace(&node).second) @@ -2377,12 +2377,12 @@ TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& exprContext, c TVisitNodeContext ctx(exprContext); ctx.RefAtoms = settings.RefAtoms; ctx.Pool = std::make_unique<TMemoryPool>(4096); - ctx.Frames.push_back(TFrameContext()); - ctx.CurrentFrame = &ctx.Frames.front(); + ctx.Frames.push_back(TFrameContext()); + ctx.CurrentFrame = &ctx.Frames.front(); VisitNode(root, 0ULL, ctx); - ui32 uniqueNum = 0; + ui32 uniqueNum = 0; - for (auto& frame : ctx.Frames) { + for (auto& frame : ctx.Frames) { ctx.CurrentFrame = &frame; frame.TopoSortedNodes.reserve(frame.Nodes.size()); for (const auto& node : frame.Nodes) { @@ -2401,18 +2401,18 @@ TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& exprContext, c YQL_ENSURE(frame.Bindings.emplace(node.second, buffer).second); } frame.TopoSortedNodes.emplace_back(node.second); - } - } - } - } - - ctx.CurrentFrame = &ctx.Frames.front(); - TAstParseResult result; + } + } + } + } + + ctx.CurrentFrame = &ctx.Frames.front(); + TAstParseResult result; result.Root = ConvertFunction(exprContext.AppendPosition(TPosition(1, 1)), {&root}, ctx, settings.AnnotationFlags, *ctx.Pool); result.Pool = std::move(ctx.Pool); - return result; -} - + return result; +} + TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& exprContext, ui32 annotationFlags, bool refAtoms) { TConvertToAstSettings settings; settings.AnnotationFlags = annotationFlags; @@ -2442,8 +2442,8 @@ TExprNode::TPtr TExprContext::ShallowCopy(const TExprNode& node) { const auto newNode = node.Clone(AllocateNextUniqueId()); ExprNodes.emplace_back(newNode.Get()); return newNode; -} - +} + TExprNode::TPtr TExprContext::ChangeChildren(const TExprNode& node, TExprNode::TListType&& children) { const auto newNode = node.ChangeChildren(AllocateNextUniqueId(), std::move(children)); ExprNodes.emplace_back(newNode.Get()); @@ -2475,8 +2475,8 @@ TExprNode::TPtr TExprContext::ExactShallowCopy(const TExprNode& node) { newNode->Result = node.Result; ExprNodes.emplace_back(newNode.Get()); return newNode; -} - +} + TExprNode::TListType GetLambdaBody(const TExprNode& node) { switch (node.ChildrenSize()) { case 1U: return {}; @@ -2611,21 +2611,21 @@ TExprNode::TPtr TExprContext::SwapWithHead(const TExprNode& node) { return ChangeChild(node.Head(), 0U, ChangeChild(node, 0U, node.Head().HeadPtr())); } -TNodeException::TNodeException() +TNodeException::TNodeException() : Pos_() -{ -} - -TNodeException::TNodeException(const TExprNode& node) +{ +} + +TNodeException::TNodeException(const TExprNode& node) : Pos_(node.Pos()) -{ -} - -TNodeException::TNodeException(const TExprNode* node) +{ +} + +TNodeException::TNodeException(const TExprNode* node) : Pos_(node ? node->Pos() : TPositionHandle()) -{ -} - +{ +} + TNodeException::TNodeException(const TPositionHandle& pos) : Pos_(pos) { @@ -2635,9 +2635,9 @@ bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprCo if (name.empty()) { ctx.AddError(TIssue(position, TStringBuilder() << "Empty " << descr << " name is not allowed")); - return false; - } - + return false; + } + if (!IsUtf8(name)) { ctx.AddError(TIssue(position, TStringBuilder() << TString(descr).to_title() << " name must be a valid utf-8 byte sequence: " << TString{name}.Quote())); @@ -2650,9 +2650,9 @@ bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprCo return false; } - return true; -} - + return true; +} + bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx) { return ValidateName(ctx.GetPosition(position), name, descr, ctx); } @@ -2718,29 +2718,29 @@ bool TTupleExprType::Validate(TPositionHandle position, TExprContext& ctx) const return Validate(ctx.GetPosition(position), ctx); } -bool TStructExprType::Validate(TPosition position, TExprContext& ctx) const { +bool TStructExprType::Validate(TPosition position, TExprContext& ctx) const { if (Items.size() > Max<ui16>()) { ctx.AddError(TIssue(position, TStringBuilder() << "Too many struct members: " << Items.size())); return false; } TString lastName; - for (auto& item : Items) { - if (!item->Validate(position, ctx)) { - return false; - } - - if (item->GetName() == lastName) { + for (auto& item : Items) { + if (!item->Validate(position, ctx)) { + return false; + } + + if (item->GetName() == lastName) { ctx.AddError(TIssue(position, TStringBuilder() << "Duplicated member: " << lastName)); - return false; - } - - lastName = item->GetName(); - } - - return true; -} - + return false; + } + + lastName = item->GetName(); + } + + return true; +} + bool TStructExprType::Validate(TPositionHandle position, TExprContext& ctx) const { return Validate(ctx.GetPosition(position), ctx); } @@ -2795,15 +2795,15 @@ ui32 TVariantExprType::MakeFlags(const TTypeAnnotationNode* underlyingType) { } -bool TDictExprType::Validate(TPosition position, TExprContext& ctx) const { +bool TDictExprType::Validate(TPosition position, TExprContext& ctx) const { if (!KeyType->IsHashable() || !KeyType->IsEquatable()) { ctx.AddError(TIssue(position, TStringBuilder() << "Expected hashable and equatable type as dict key type, but got: " << *KeyType)); - return false; - } - - return true; -} - + return false; + } + + return true; +} + bool TDictExprType::Validate(TPositionHandle position, TExprContext& ctx) const { return Validate(ctx.GetPosition(position), ctx); } @@ -2878,7 +2878,7 @@ TExprContext::TExprContext(ui64 nextUniqueId) "To disable it use \"pragma warning(\"default\", <code>);\""); IssueManager.SetIssueCountLimit(100); } - + TPositionHandle TExprContext::AppendPosition(const TPosition& pos) { YQL_ENSURE(Positions.size() <= Max<ui32>(), "Too many positions"); @@ -2903,7 +2903,7 @@ TPosition TExprContext::GetPosition(TPositionHandle handle) const { TExprContext::~TExprContext() { UnFreeze(); } - + void TExprContext::Freeze() { for (auto& node : ExprNodes) { node->MarkFrozen(); @@ -3196,34 +3196,34 @@ void CheckCounts(const TExprNode& root) { CheckReferences(root, refCounts, visited); } -} // namespace NYql - -template<> +} // namespace NYql + +template<> void Out<NYql::TExprNode::EType>(class IOutputStream &o, NYql::TExprNode::EType x) { -#define YQL_EXPR_NODE_TYPE_MAP_TO_STRING_IMPL(name, ...) \ - case NYql::TExprNode::name: \ - o << #name; \ - return; - - switch (x) { - YQL_EXPR_NODE_TYPE_MAP(YQL_EXPR_NODE_TYPE_MAP_TO_STRING_IMPL) - default: - o << static_cast<int>(x); - return; - } -} - -template<> +#define YQL_EXPR_NODE_TYPE_MAP_TO_STRING_IMPL(name, ...) \ + case NYql::TExprNode::name: \ + o << #name; \ + return; + + switch (x) { + YQL_EXPR_NODE_TYPE_MAP(YQL_EXPR_NODE_TYPE_MAP_TO_STRING_IMPL) + default: + o << static_cast<int>(x); + return; + } +} + +template<> void Out<NYql::TExprNode::EState>(class IOutputStream &o, NYql::TExprNode::EState x) { -#define YQL_EXPR_NODE_STATE_MAP_TO_STRING_IMPL(name, ...) \ - case NYql::TExprNode::EState::name: \ - o << #name; \ - return; - - switch (x) { - YQL_EXPR_NODE_STATE_MAP(YQL_EXPR_NODE_STATE_MAP_TO_STRING_IMPL) - default: - o << static_cast<int>(x); - return; - } -} +#define YQL_EXPR_NODE_STATE_MAP_TO_STRING_IMPL(name, ...) \ + case NYql::TExprNode::EState::name: \ + o << #name; \ + return; + + switch (x) { + YQL_EXPR_NODE_STATE_MAP(YQL_EXPR_NODE_STATE_MAP_TO_STRING_IMPL) + default: + o << static_cast<int>(x); + return; + } +} diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index a48ce157c9..d6eb544f9a 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -1,13 +1,13 @@ -#pragma once - -#include "yql_ast.h" -#include "yql_expr_types.h" -#include "yql_type_string.h" +#pragma once + +#include "yql_ast.h" +#include "yql_expr_types.h" +#include "yql_type_string.h" #include "yql_expr_builder.h" #include "yql_gc_nodes.h" #include "yql_constraint.h" #include "yql_pos_handle.h" - + #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/library/yql/public/issue/yql_issue_manager.h> #include <ydb/library/yql/public/udf/udf_data_type.h> @@ -17,16 +17,16 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/generic/array_ref.h> -#include <util/generic/deque.h> -#include <util/generic/cast.h> -#include <util/generic/hash.h> -#include <util/generic/maybe.h> -#include <util/generic/bt_exception.h> -#include <util/generic/algorithm.h> +#include <util/generic/deque.h> +#include <util/generic/cast.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/bt_exception.h> +#include <util/generic/algorithm.h> #include <util/digest/murmur.h> - + #include <algorithm> #include <unordered_set> #include <unordered_map> @@ -47,27 +47,27 @@ #define ENSURE_NOT_FROZEN_CTX Y_VERIFY_DEBUG(!Frozen); #endif -namespace NYql { +namespace NYql { using NUdf::EDataSlot; class TUnitExprType; class TMultiExprType; -class TTupleExprType; -class TStructExprType; -class TItemExprType; -class TListExprType; +class TTupleExprType; +class TStructExprType; +class TItemExprType; +class TListExprType; class TStreamExprType; -class TDataExprType; -class TWorldExprType; -class TOptionalExprType; -class TCallableExprType; -class TResourceExprType; -class TTypeExprType; -class TDictExprType; -class TVoidExprType; +class TDataExprType; +class TWorldExprType; +class TOptionalExprType; +class TCallableExprType; +class TResourceExprType; +class TTypeExprType; +class TDictExprType; +class TVoidExprType; class TNullExprType; -class TGenericExprType; +class TGenericExprType; class TTaggedExprType; class TErrorExprType; class TVariantExprType; @@ -75,44 +75,44 @@ class TStreamExprType; class TFlowExprType; class TEmptyListExprType; class TEmptyDictExprType; - + const size_t DefaultMistypeDistance = 3; extern const TStringBuf ZeroString; -struct TTypeAnnotationVisitor { - virtual ~TTypeAnnotationVisitor() = default; - +struct TTypeAnnotationVisitor { + virtual ~TTypeAnnotationVisitor() = default; + virtual void Visit(const TUnitExprType& type) = 0; virtual void Visit(const TMultiExprType& type) = 0; - virtual void Visit(const TTupleExprType& type) = 0; - virtual void Visit(const TStructExprType& type) = 0; - virtual void Visit(const TItemExprType& type) = 0; - virtual void Visit(const TListExprType& type) = 0; + virtual void Visit(const TTupleExprType& type) = 0; + virtual void Visit(const TStructExprType& type) = 0; + virtual void Visit(const TItemExprType& type) = 0; + virtual void Visit(const TListExprType& type) = 0; virtual void Visit(const TStreamExprType& type) = 0; virtual void Visit(const TFlowExprType& type) = 0; - virtual void Visit(const TDataExprType& type) = 0; - virtual void Visit(const TWorldExprType& type) = 0; - virtual void Visit(const TOptionalExprType& type) = 0; - virtual void Visit(const TCallableExprType& type) = 0; - virtual void Visit(const TResourceExprType& type) = 0; - virtual void Visit(const TTypeExprType& type) = 0; - virtual void Visit(const TDictExprType& type) = 0; - virtual void Visit(const TVoidExprType& type) = 0; + virtual void Visit(const TDataExprType& type) = 0; + virtual void Visit(const TWorldExprType& type) = 0; + virtual void Visit(const TOptionalExprType& type) = 0; + virtual void Visit(const TCallableExprType& type) = 0; + virtual void Visit(const TResourceExprType& type) = 0; + virtual void Visit(const TTypeExprType& type) = 0; + virtual void Visit(const TDictExprType& type) = 0; + virtual void Visit(const TVoidExprType& type) = 0; virtual void Visit(const TNullExprType& type) = 0; - virtual void Visit(const TGenericExprType& type) = 0; + virtual void Visit(const TGenericExprType& type) = 0; virtual void Visit(const TTaggedExprType& type) = 0; virtual void Visit(const TErrorExprType& type) = 0; virtual void Visit(const TVariantExprType& type) = 0; virtual void Visit(const TEmptyListExprType& type) = 0; virtual void Visit(const TEmptyDictExprType& type) = 0; -}; - -enum ETypeAnnotationFlags { - TypeNonComposable = 0x01, - TypeNonPersistable = 0x02, - TypeNonComputable = 0x04, - TypeNonInspectable = 0x08, +}; + +enum ETypeAnnotationFlags { + TypeNonComposable = 0x01, + TypeNonPersistable = 0x02, + TypeNonComputable = 0x04, + TypeNonInspectable = 0x08, TypeNonHashable = 0x10, TypeNonEquatable = 0x20, TypeNonComparable = 0x40, @@ -120,8 +120,8 @@ enum ETypeAnnotationFlags { TypeHasOptional = 0x100, TypeHasManyValues = 0x200, TypeHasBareYson = 0x400, -}; - +}; + const ui64 TypeHashMagic = 0x10000; inline ui64 StreamHash(const void* buffer, size_t size, ui64 seed) { @@ -135,27 +135,27 @@ inline ui64 StreamHash(ui64 value, ui64 seed) { void ReportError(TExprContext& ctx, const TIssue& issue); class TTypeAnnotationNode { -protected: +protected: TTypeAnnotationNode(ETypeAnnotationKind kind, ui32 flags, ui64 hash) - : Kind(kind) + : Kind(kind) , Flags(flags) , Hash(hash) - { - } - -public: + { + } + +public: virtual ~TTypeAnnotationNode() = default; - template <typename T> - const T* Cast() const { + template <typename T> + const T* Cast() const { static_assert(std::is_base_of<TTypeAnnotationNode, T>::value, "Should be derived from TTypeAnnotationNode"); const auto ret = dynamic_cast<const T*>(this); YQL_ENSURE(ret, "Cannot cast type " << *this << " to " << ETypeAnnotationKind(T::KindValue)); return ret; - } - + } + template <typename T> const T* UserCast(TPosition pos, TExprContext& ctx) const { static_assert(std::is_base_of<TTypeAnnotationNode, T>::value, @@ -169,26 +169,26 @@ public: return ret; } - ETypeAnnotationKind GetKind() const { - return Kind; - } - - bool IsComposable() const { - return (GetFlags() & TypeNonComposable) == 0; - } - - bool IsPersistable() const { - return (GetFlags() & TypeNonPersistable) == 0; - } - - bool IsComputable() const { - return (GetFlags() & TypeNonComputable) == 0; - } - - bool IsInspectable() const { - return (GetFlags() & TypeNonInspectable) == 0; - } - + ETypeAnnotationKind GetKind() const { + return Kind; + } + + bool IsComposable() const { + return (GetFlags() & TypeNonComposable) == 0; + } + + bool IsPersistable() const { + return (GetFlags() & TypeNonPersistable) == 0; + } + + bool IsComputable() const { + return (GetFlags() & TypeNonComputable) == 0; + } + + bool IsInspectable() const { + return (GetFlags() & TypeNonInspectable) == 0; + } + bool IsHashable() const { return IsPersistable() && (GetFlags() & TypeNonHashable) == 0; } @@ -229,24 +229,24 @@ public: ui32 GetFlags() const { return Flags; } - + ui64 GetHash() const { return Hash; } - bool Equals(const TTypeAnnotationNode& node) const; - void Accept(TTypeAnnotationVisitor& visitor) const; - + bool Equals(const TTypeAnnotationNode& node) const; + void Accept(TTypeAnnotationVisitor& visitor) const; + void Out(IOutputStream& out) const { - out << FormatType(this); - } - + out << FormatType(this); + } + struct THash { size_t operator()(const TTypeAnnotationNode* node) const { return node->GetHash(); } }; - + struct TEqual { bool operator()(const TTypeAnnotationNode* one, const TTypeAnnotationNode* two) const { return one->Equals(*two); @@ -266,12 +266,12 @@ protected: return flags; } -private: - const ETypeAnnotationKind Kind; +private: + const ETypeAnnotationKind Kind; const ui32 Flags; const ui64 Hash; -}; - +}; + class TUnitExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Unit; @@ -281,25 +281,25 @@ public: TypeNonComputable | TypeNonPersistable, hash) { } - + static ui64 MakeHash() { return TypeHashMagic | (ui64)ETypeAnnotationKind::Unit; - } - + } + bool operator==(const TUnitExprType& other) const { Y_UNUSED(other); - return true; - } -}; - -class TTupleExprType : public TTypeAnnotationNode { -public: + return true; + } +}; + +class TTupleExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Tuple; TTupleExprType(ui64 hash, const TTypeAnnotationNode::TListType& items) : TTypeAnnotationNode(KindValue, CombineFlags(items), hash) - , Items(items) - { + , Items(items) + { } static ui64 MakeHash(const TTypeAnnotationNode::TListType& items) { @@ -307,40 +307,40 @@ public: hash = StreamHash(items.size(), hash); for (const auto& item : items) { hash = StreamHash(item->GetHash(), hash); - } + } return hash; - } - - size_t GetSize() const { - return Items.size(); - } - + } + + size_t GetSize() const { + return Items.size(); + } + const TTypeAnnotationNode::TListType& GetItems() const { - return Items; - } - - bool operator==(const TTupleExprType& other) const { - if (GetSize() != other.GetSize()) { - return false; - } - - for (ui32 i = 0, e = GetSize(); i < e; ++i) { + return Items; + } + + bool operator==(const TTupleExprType& other) const { + if (GetSize() != other.GetSize()) { + return false; + } + + for (ui32 i = 0, e = GetSize(); i < e; ++i) { if (GetItems()[i] != other.GetItems()[i]) { - return false; - } - } - - return true; - } - + return false; + } + } + + return true; + } + bool Validate(TPosition position, TExprContext& ctx) const; bool Validate(TPositionHandle position, TExprContext& ctx) const; -private: +private: TTypeAnnotationNode::TListType Items; -}; - +}; + class TMultiExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Multi; @@ -396,17 +396,17 @@ struct TExprContext; bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx); bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx); -class TItemExprType : public TTypeAnnotationNode { -public: +class TItemExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Item; TItemExprType(ui64 hash, const TStringBuf& name, const TTypeAnnotationNode* itemType) : TTypeAnnotationNode(KindValue, itemType->GetFlags(), hash) - , Name(name) - , ItemType(itemType) - { - } - + , Name(name) + , ItemType(itemType) + { + } + static ui64 MakeHash(const TStringBuf& name, const TTypeAnnotationNode* itemType) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::Item; hash = StreamHash(name.size(), hash); @@ -414,48 +414,48 @@ public: return StreamHash(itemType->GetHash(), hash); } - bool Validate(TPosition position, TExprContext& ctx) const; + bool Validate(TPosition position, TExprContext& ctx) const; bool Validate(TPositionHandle position, TExprContext& ctx) const; - + const TStringBuf& GetName() const { - return Name; - } - - const TTypeAnnotationNode* GetItemType() const { - return ItemType; - } - - bool operator==(const TItemExprType& other) const { + return Name; + } + + const TTypeAnnotationNode* GetItemType() const { + return ItemType; + } + + bool operator==(const TItemExprType& other) const { return GetName() == other.GetName() && GetItemType() == other.GetItemType(); - } - -private: + } + +private: const TStringBuf Name; const TTypeAnnotationNode* ItemType; -}; - -class TStructExprType : public TTypeAnnotationNode { -public: +}; + +class TStructExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Struct; - struct TItemLess { - bool operator()(const TItemExprType* x, const TItemExprType* y) const { - return x->GetName() < y->GetName(); - }; - + struct TItemLess { + bool operator()(const TItemExprType* x, const TItemExprType* y) const { + return x->GetName() < y->GetName(); + }; + bool operator()(const TItemExprType* x, const TStringBuf& y) const { - return x->GetName() < y; - }; - + return x->GetName() < y; + }; + bool operator()(const TStringBuf& x, const TItemExprType* y) const { - return x < y->GetName(); - }; - }; - + return x < y->GetName(); + }; + }; + TStructExprType(ui64 hash, const TVector<const TItemExprType*>& items) : TTypeAnnotationNode(KindValue, TypeNonComparable | CombineFlags(items), hash) - , Items(items) - { + , Items(items) + { } static ui64 MakeHash(const TVector<const TItemExprType*>& items) { @@ -464,31 +464,31 @@ public: hash = StreamHash(items.size(), hash); for (const auto& item : items) { hash = StreamHash(item->GetHash(), hash); - } - + } + return hash; - } - - bool Validate(TPosition position, TExprContext& ctx) const; + } + + bool Validate(TPosition position, TExprContext& ctx) const; bool Validate(TPositionHandle position, TExprContext& ctx) const; - - size_t GetSize() const { - return Items.size(); - } - + + size_t GetSize() const { + return Items.size(); + } + const TVector<const TItemExprType*>& GetItems() const { - return Items; - } - + return Items; + } + TMaybe<ui32> FindItem(const TStringBuf& name) const { - auto it = LowerBound(Items.begin(), Items.end(), name, TItemLess()); - if (it == Items.end() || (*it)->GetName() != name) { - return TMaybe<ui32>(); - } - - return it - Items.begin(); - } - + auto it = LowerBound(Items.begin(), Items.end(), name, TItemLess()); + if (it == Items.end() || (*it)->GetName() != name) { + return TMaybe<ui32>(); + } + + return it - Items.begin(); + } + const TTypeAnnotationNode* FindItemType(const TStringBuf& name) const { const auto it = LowerBound(Items.begin(), Items.end(), name, TItemLess()); if (it == Items.end() || (*it)->GetName() != name) { @@ -507,51 +507,51 @@ public: return TMaybe<TStringBuf>(); } - bool operator==(const TStructExprType& other) const { - if (GetSize() != other.GetSize()) { - return false; - } - - for (ui32 i = 0, e = GetSize(); i < e; ++i) { + bool operator==(const TStructExprType& other) const { + if (GetSize() != other.GetSize()) { + return false; + } + + for (ui32 i = 0, e = GetSize(); i < e; ++i) { if (GetItems()[i] != other.GetItems()[i]) { - return false; - } - } - - return true; - } - -private: + return false; + } + } + + return true; + } + +private: TVector<const TItemExprType*> Items; -}; - -class TListExprType : public TTypeAnnotationNode { -public: +}; + +class TListExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::List; TListExprType(ui64 hash, const TTypeAnnotationNode* itemType) : TTypeAnnotationNode(KindValue, itemType->GetFlags(), hash) - , ItemType(itemType) - { - } - + , ItemType(itemType) + { + } + static ui64 MakeHash(const TTypeAnnotationNode* itemType) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::List; return StreamHash(itemType->GetHash(), hash); } - const TTypeAnnotationNode* GetItemType() const { - return ItemType; - } - - bool operator==(const TListExprType& other) const { + const TTypeAnnotationNode* GetItemType() const { + return ItemType; + } + + bool operator==(const TListExprType& other) const { return GetItemType() == other.GetItemType(); - } - -private: + } + +private: const TTypeAnnotationNode* ItemType; -}; - +}; + class TStreamExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Stream; @@ -606,8 +606,8 @@ private: const TTypeAnnotationNode* ItemType; }; -class TDataExprType : public TTypeAnnotationNode { -public: +class TDataExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Data; TDataExprType(ui64 hash, EDataSlot slot) @@ -615,7 +615,7 @@ public: , Slot(slot) { } - + static ui32 GetFlags(EDataSlot slot) { ui32 ret = TypeHasManyValues; auto props = NUdf::GetDataTypeInfo(slot).Features; @@ -647,20 +647,20 @@ public: EDataSlot GetSlot() const { return Slot; - } - + } + TStringBuf GetName() const { return NUdf::GetDataTypeInfo(Slot).Name; } - bool operator==(const TDataExprType& other) const { + bool operator==(const TDataExprType& other) const { return Slot == other.Slot; - } - -private: + } + +private: EDataSlot Slot; -}; - +}; + class TDataExprParamsType : public TDataExprType { public: TDataExprParamsType(ui64 hash, EDataSlot slot, const TStringBuf& one, const TStringBuf& two) @@ -695,7 +695,7 @@ private: const TStringBuf One, Two; }; -class TWorldExprType : public TTypeAnnotationNode { +class TWorldExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::World; @@ -704,27 +704,27 @@ public: TypeNonComposable | TypeNonComputable | TypeNonPersistable | TypeNonInspectable, hash) { } - + static ui64 MakeHash() { return TypeHashMagic | (ui64)ETypeAnnotationKind::World; - } - - bool operator==(const TWorldExprType& other) const { + } + + bool operator==(const TWorldExprType& other) const { Y_UNUSED(other); - return true; - } -}; - -class TOptionalExprType : public TTypeAnnotationNode { -public: + return true; + } +}; + +class TOptionalExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Optional; TOptionalExprType(ui64 hash, const TTypeAnnotationNode* itemType) : TTypeAnnotationNode(KindValue, GetFlags(itemType), hash) - , ItemType(itemType) - { - } - + , ItemType(itemType) + { + } + static ui32 GetFlags(const TTypeAnnotationNode* itemType) { auto ret = TypeHasOptional | itemType->GetFlags(); if (itemType->GetKind() == ETypeAnnotationKind::Data && @@ -740,18 +740,18 @@ public: return StreamHash(itemType->GetHash(), hash); } - const TTypeAnnotationNode* GetItemType() const { - return ItemType; - } - - bool operator==(const TOptionalExprType& other) const { + const TTypeAnnotationNode* GetItemType() const { + return ItemType; + } + + bool operator==(const TOptionalExprType& other) const { return GetItemType() == other.GetItemType(); - } - -private: + } + +private: const TTypeAnnotationNode* ItemType; -}; - +}; + class TVariantExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Variant; @@ -784,44 +784,44 @@ private: const TTypeAnnotationNode* UnderlyingType; }; -class TTypeExprType : public TTypeAnnotationNode { -public: +class TTypeExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Type; TTypeExprType(ui64 hash, const TTypeAnnotationNode* type) : TTypeAnnotationNode(KindValue, TypeNonPersistable | TypeNonComputable, hash) - , Type(type) + , Type(type) { } - + static ui64 MakeHash(const TTypeAnnotationNode* type) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::Type; return StreamHash(type->GetHash(), hash); - } - - const TTypeAnnotationNode* GetType() const { - return Type; - } - - bool operator==(const TTypeExprType& other) const { + } + + const TTypeAnnotationNode* GetType() const { + return Type; + } + + bool operator==(const TTypeExprType& other) const { return GetType() == other.GetType(); - } - -private: + } + +private: const TTypeAnnotationNode* Type; -}; - -class TDictExprType : public TTypeAnnotationNode { -public: +}; + +class TDictExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Dict; TDictExprType(ui64 hash, const TTypeAnnotationNode* keyType, const TTypeAnnotationNode* payloadType) : TTypeAnnotationNode(KindValue, TypeNonComparable | keyType->GetFlags() | payloadType->GetFlags(), hash) - , KeyType(keyType) - , PayloadType(payloadType) - { - } - + , KeyType(keyType) + , PayloadType(payloadType) + { + } + static ui64 MakeHash(const TTypeAnnotationNode* keyType, const TTypeAnnotationNode* payloadType) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::Dict; return StreamHash(StreamHash(keyType->GetHash(), hash), payloadType->GetHash()); @@ -830,25 +830,25 @@ public: bool Validate(TPosition position, TExprContext& ctx) const; bool Validate(TPositionHandle position, TExprContext& ctx) const; - const TTypeAnnotationNode* GetKeyType() const { - return KeyType; - } - - const TTypeAnnotationNode* GetPayloadType() const { - return PayloadType; - } - - bool operator==(const TDictExprType& other) const { + const TTypeAnnotationNode* GetKeyType() const { + return KeyType; + } + + const TTypeAnnotationNode* GetPayloadType() const { + return PayloadType; + } + + bool operator==(const TDictExprType& other) const { return GetKeyType() == other.GetKeyType() && GetPayloadType() == other.GetPayloadType(); - } - -private: + } + +private: const TTypeAnnotationNode* KeyType; const TTypeAnnotationNode* PayloadType; -}; - -class TVoidExprType : public TTypeAnnotationNode { +}; + +class TVoidExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Void; @@ -856,17 +856,17 @@ public: : TTypeAnnotationNode(KindValue, 0, hash) { } - + static ui64 MakeHash() { return TypeHashMagic | (ui64)ETypeAnnotationKind::Void; - } - - bool operator==(const TVoidExprType& other) const { + } + + bool operator==(const TVoidExprType& other) const { Y_UNUSED(other); - return true; - } -}; - + return true; + } +}; + class TNullExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Null; @@ -892,8 +892,8 @@ struct TArgumentFlags { }; }; -class TCallableExprType : public TTypeAnnotationNode { -public: +class TCallableExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Callable; struct TArgumentInfo { @@ -913,19 +913,19 @@ public: TCallableExprType(ui64 hash, const TTypeAnnotationNode* returnType, const TVector<TArgumentInfo>& arguments , size_t optionalArgumentsCount, const TStringBuf& payload) : TTypeAnnotationNode(KindValue, MakeFlags(returnType), hash) - , ReturnType(returnType) - , Arguments(arguments) - , OptionalArgumentsCount(optionalArgumentsCount) + , ReturnType(returnType) + , Arguments(arguments) + , OptionalArgumentsCount(optionalArgumentsCount) , Payload(payload) - { + { for (ui32 i = 0; i < Arguments.size(); ++i) { const auto& arg = Arguments[i]; if (!arg.Name.empty()) { IndexByName.insert({ arg.Name, i }); } - } + } } - + static ui64 MakeHash(const TTypeAnnotationNode* returnType, const TVector<TArgumentInfo>& arguments , size_t optionalArgumentsCount, const TStringBuf& payload) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::Callable; @@ -942,50 +942,50 @@ public: hash = StreamHash(payload.size(), hash); hash = StreamHash(payload.data(), payload.size(), hash); return hash; - } - - const TTypeAnnotationNode* GetReturnType() const { - return ReturnType; - } - - size_t GetOptionalArgumentsCount() const { - return OptionalArgumentsCount; - } - + } + + const TTypeAnnotationNode* GetReturnType() const { + return ReturnType; + } + + size_t GetOptionalArgumentsCount() const { + return OptionalArgumentsCount; + } + const TStringBuf& GetPayload() const { return Payload; } - size_t GetArgumentsSize() const { - return Arguments.size(); - } - + size_t GetArgumentsSize() const { + return Arguments.size(); + } + const TVector<TArgumentInfo>& GetArguments() const { - return Arguments; - } - - bool operator==(const TCallableExprType& other) const { - if (GetArgumentsSize() != other.GetArgumentsSize()) { - return false; - } - - if (GetOptionalArgumentsCount() != other.GetOptionalArgumentsCount()) { - return false; - } - + return Arguments; + } + + bool operator==(const TCallableExprType& other) const { + if (GetArgumentsSize() != other.GetArgumentsSize()) { + return false; + } + + if (GetOptionalArgumentsCount() != other.GetOptionalArgumentsCount()) { + return false; + } + if (GetReturnType() != other.GetReturnType()) { - return false; - } - - for (ui32 i = 0, e = GetArgumentsSize(); i < e; ++i) { + return false; + } + + for (ui32 i = 0, e = GetArgumentsSize(); i < e; ++i) { if (GetArguments()[i] != other.GetArguments()[i]) { - return false; - } - } - - return true; - } - + return false; + } + } + + return true; + } + bool Validate(TPosition position, TExprContext& ctx) const; bool Validate(TPositionHandle position, TExprContext& ctx) const; @@ -998,7 +998,7 @@ public: return it->second; } -private: +private: static ui32 MakeFlags(const TTypeAnnotationNode* returnType) { ui32 flags = TypeNonPersistable; flags |= returnType->GetFlags(); @@ -1008,12 +1008,12 @@ private: private: const TTypeAnnotationNode* ReturnType; TVector<TArgumentInfo> Arguments; - const size_t OptionalArgumentsCount; + const size_t OptionalArgumentsCount; const TStringBuf Payload; THashMap<TStringBuf, ui32> IndexByName; -}; - -class TGenericExprType : public TTypeAnnotationNode { +}; + +class TGenericExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Generic; @@ -1021,26 +1021,26 @@ public: : TTypeAnnotationNode(KindValue, TypeNonComputable, hash) { } - + static ui64 MakeHash() { return TypeHashMagic | (ui64)ETypeAnnotationKind::Generic; - } - - bool operator==(const TGenericExprType& other) const { + } + + bool operator==(const TGenericExprType& other) const { Y_UNUSED(other); - return true; - } -}; - -class TResourceExprType : public TTypeAnnotationNode { -public: + return true; + } +}; + +class TResourceExprType : public TTypeAnnotationNode { +public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Resource; TResourceExprType(ui64 hash, const TStringBuf& tag) : TTypeAnnotationNode(KindValue, TypeNonPersistable | TypeHasManyValues, hash) , Tag(tag) - {} - + {} + static ui64 MakeHash(const TStringBuf& tag) { ui64 hash = TypeHashMagic | (ui64)ETypeAnnotationKind::Resource; hash = StreamHash(tag.size(), hash); @@ -1051,14 +1051,14 @@ public: return Tag; } - bool operator==(const TResourceExprType& other) const { + bool operator==(const TResourceExprType& other) const { return Tag == other.Tag; - } - + } + private: const TStringBuf Tag; -}; - +}; + class TTaggedExprType : public TTypeAnnotationNode { public: static constexpr ETypeAnnotationKind KindValue = ETypeAnnotationKind::Tagged; @@ -1159,7 +1159,7 @@ public: } }; -inline bool TTypeAnnotationNode::Equals(const TTypeAnnotationNode& node) const { +inline bool TTypeAnnotationNode::Equals(const TTypeAnnotationNode& node) const { if (this == &node) { return true; } @@ -1168,56 +1168,56 @@ inline bool TTypeAnnotationNode::Equals(const TTypeAnnotationNode& node) const { return false; } - if (Kind != node.GetKind()) { - return false; - } - - switch (Kind) { + if (Kind != node.GetKind()) { + return false; + } + + switch (Kind) { case ETypeAnnotationKind::Unit: return static_cast<const TUnitExprType&>(*this) == static_cast<const TUnitExprType&>(node); - - case ETypeAnnotationKind::Tuple: - return static_cast<const TTupleExprType&>(*this) == static_cast<const TTupleExprType&>(node); - - case ETypeAnnotationKind::Struct: - return static_cast<const TStructExprType&>(*this) == static_cast<const TStructExprType&>(node); - - case ETypeAnnotationKind::Item: - return static_cast<const TItemExprType&>(*this) == static_cast<const TItemExprType&>(node); - - case ETypeAnnotationKind::List: - return static_cast<const TListExprType&>(*this) == static_cast<const TListExprType&>(node); - - case ETypeAnnotationKind::Data: - return static_cast<const TDataExprType&>(*this) == static_cast<const TDataExprType&>(node); - - case ETypeAnnotationKind::World: - return static_cast<const TWorldExprType&>(*this) == static_cast<const TWorldExprType&>(node); - - case ETypeAnnotationKind::Optional: - return static_cast<const TOptionalExprType&>(*this) == static_cast<const TOptionalExprType&>(node); - - case ETypeAnnotationKind::Type: - return static_cast<const TTypeExprType&>(*this) == static_cast<const TTypeExprType&>(node); - - case ETypeAnnotationKind::Dict: - return static_cast<const TDictExprType&>(*this) == static_cast<const TDictExprType&>(node); - - case ETypeAnnotationKind::Void: - return static_cast<const TVoidExprType&>(*this) == static_cast<const TVoidExprType&>(node); - + + case ETypeAnnotationKind::Tuple: + return static_cast<const TTupleExprType&>(*this) == static_cast<const TTupleExprType&>(node); + + case ETypeAnnotationKind::Struct: + return static_cast<const TStructExprType&>(*this) == static_cast<const TStructExprType&>(node); + + case ETypeAnnotationKind::Item: + return static_cast<const TItemExprType&>(*this) == static_cast<const TItemExprType&>(node); + + case ETypeAnnotationKind::List: + return static_cast<const TListExprType&>(*this) == static_cast<const TListExprType&>(node); + + case ETypeAnnotationKind::Data: + return static_cast<const TDataExprType&>(*this) == static_cast<const TDataExprType&>(node); + + case ETypeAnnotationKind::World: + return static_cast<const TWorldExprType&>(*this) == static_cast<const TWorldExprType&>(node); + + case ETypeAnnotationKind::Optional: + return static_cast<const TOptionalExprType&>(*this) == static_cast<const TOptionalExprType&>(node); + + case ETypeAnnotationKind::Type: + return static_cast<const TTypeExprType&>(*this) == static_cast<const TTypeExprType&>(node); + + case ETypeAnnotationKind::Dict: + return static_cast<const TDictExprType&>(*this) == static_cast<const TDictExprType&>(node); + + case ETypeAnnotationKind::Void: + return static_cast<const TVoidExprType&>(*this) == static_cast<const TVoidExprType&>(node); + case ETypeAnnotationKind::Null: return static_cast<const TNullExprType&>(*this) == static_cast<const TNullExprType&>(node); - case ETypeAnnotationKind::Callable: - return static_cast<const TCallableExprType&>(*this) == static_cast<const TCallableExprType&>(node); - + case ETypeAnnotationKind::Callable: + return static_cast<const TCallableExprType&>(*this) == static_cast<const TCallableExprType&>(node); + case ETypeAnnotationKind::Generic: - return static_cast<const TGenericExprType&>(*this) == static_cast<const TGenericExprType&>(node); - - case ETypeAnnotationKind::Resource: - return static_cast<const TResourceExprType&>(*this) == static_cast<const TResourceExprType&>(node); - + return static_cast<const TGenericExprType&>(*this) == static_cast<const TGenericExprType&>(node); + + case ETypeAnnotationKind::Resource: + return static_cast<const TResourceExprType&>(*this) == static_cast<const TResourceExprType&>(node); + case ETypeAnnotationKind::Tagged: return static_cast<const TTaggedExprType&>(*this) == static_cast<const TTaggedExprType&>(node); @@ -1245,41 +1245,41 @@ inline bool TTypeAnnotationNode::Equals(const TTypeAnnotationNode& node) const { case ETypeAnnotationKind::LastType: YQL_ENSURE(false, "Incorrect type"); - } + } return false; -} - -inline void TTypeAnnotationNode::Accept(TTypeAnnotationVisitor& visitor) const { - switch (Kind) { +} + +inline void TTypeAnnotationNode::Accept(TTypeAnnotationVisitor& visitor) const { + switch (Kind) { case ETypeAnnotationKind::Unit: return visitor.Visit(static_cast<const TUnitExprType&>(*this)); - case ETypeAnnotationKind::Tuple: + case ETypeAnnotationKind::Tuple: return visitor.Visit(static_cast<const TTupleExprType&>(*this)); - case ETypeAnnotationKind::Struct: + case ETypeAnnotationKind::Struct: return visitor.Visit(static_cast<const TStructExprType&>(*this)); - case ETypeAnnotationKind::Item: + case ETypeAnnotationKind::Item: return visitor.Visit(static_cast<const TItemExprType&>(*this)); - case ETypeAnnotationKind::List: + case ETypeAnnotationKind::List: return visitor.Visit(static_cast<const TListExprType&>(*this)); - case ETypeAnnotationKind::Data: + case ETypeAnnotationKind::Data: return visitor.Visit(static_cast<const TDataExprType&>(*this)); - case ETypeAnnotationKind::World: + case ETypeAnnotationKind::World: return visitor.Visit(static_cast<const TWorldExprType&>(*this)); - case ETypeAnnotationKind::Optional: + case ETypeAnnotationKind::Optional: return visitor.Visit(static_cast<const TOptionalExprType&>(*this)); - case ETypeAnnotationKind::Type: + case ETypeAnnotationKind::Type: return visitor.Visit(static_cast<const TTypeExprType&>(*this)); - case ETypeAnnotationKind::Dict: + case ETypeAnnotationKind::Dict: return visitor.Visit(static_cast<const TDictExprType&>(*this)); - case ETypeAnnotationKind::Void: + case ETypeAnnotationKind::Void: return visitor.Visit(static_cast<const TVoidExprType&>(*this)); case ETypeAnnotationKind::Null: return visitor.Visit(static_cast<const TNullExprType&>(*this)); - case ETypeAnnotationKind::Callable: + case ETypeAnnotationKind::Callable: return visitor.Visit(static_cast<const TCallableExprType&>(*this)); case ETypeAnnotationKind::Generic: return visitor.Visit(static_cast<const TGenericExprType&>(*this)); - case ETypeAnnotationKind::Resource: + case ETypeAnnotationKind::Resource: return visitor.Visit(static_cast<const TResourceExprType&>(*this)); case ETypeAnnotationKind::Tagged: return visitor.Visit(static_cast<const TTaggedExprType&>(*this)); @@ -1299,9 +1299,9 @@ inline void TTypeAnnotationNode::Accept(TTypeAnnotationVisitor& visitor) const { return visitor.Visit(static_cast<const TMultiExprType&>(*this)); case ETypeAnnotationKind::LastType: YQL_ENSURE(false, "Incorrect type"); - } -} - + } +} + class TExprNode { friend class TExprNodeBuilder; friend class TExprNodeReplaceBuilder; @@ -1317,7 +1317,7 @@ private: static constexpr ui32 FlagsMask = 0x03; // all flags should fit here }; -public: +public: typedef TIntrusivePtr<TExprNode> TPtr; typedef std::vector<TPtr> TListType; typedef TArrayRef<const TPtr> TChildrenType; @@ -1328,26 +1328,26 @@ public: } }; -#define YQL_EXPR_NODE_TYPE_MAP(xx) \ - xx(List, 0) \ - xx(Atom, 1) \ - xx(Callable, 2) \ - xx(Lambda, 3) \ - xx(Argument, 4) \ - xx(Arguments, 5) \ - xx(World, 7) - +#define YQL_EXPR_NODE_TYPE_MAP(xx) \ + xx(List, 0) \ + xx(Atom, 1) \ + xx(Callable, 2) \ + xx(Lambda, 3) \ + xx(Argument, 4) \ + xx(Arguments, 5) \ + xx(World, 7) + enum EType : ui8 { - YQL_EXPR_NODE_TYPE_MAP(ENUM_VALUE_GEN) - }; - + YQL_EXPR_NODE_TYPE_MAP(ENUM_VALUE_GEN) + }; + static constexpr ui32 TypeMask = 0x07; // all types should fit here -#define YQL_EXPR_NODE_STATE_MAP(xx) \ - xx(Initial, 0) \ - xx(TypeInProgress, 1) \ - xx(TypePending, 2) \ - xx(TypeComplete, 3) \ +#define YQL_EXPR_NODE_STATE_MAP(xx) \ + xx(Initial, 0) \ + xx(TypeInProgress, 1) \ + xx(TypePending, 2) \ + xx(TypeComplete, 3) \ xx(ConstrInProgress, 4) \ xx(ConstrPending, 5) \ xx(ConstrComplete, 6) \ @@ -1357,11 +1357,11 @@ public: xx(ExecutionComplete, 10) \ xx(Error, 11) \ xx(Last, 12) - + enum class EState : ui8 { - YQL_EXPR_NODE_STATE_MAP(ENUM_VALUE_GEN) - }; - + YQL_EXPR_NODE_STATE_MAP(ENUM_VALUE_GEN) + }; + static TPtr GetResult(const TPtr& node) { return node->Type() == Callable ? node->Result : node; } @@ -1369,24 +1369,24 @@ public: const TExprNode& GetResult() const { ENSURE_NOT_DELETED return Type() == Callable ? *Result : *this; - } - - bool HasResult() const { + } + + bool HasResult() const { ENSURE_NOT_DELETED return Type() != Callable || bool(Result); - } - + } + void SetResult(TPtr&& result) { ENSURE_NOT_DELETED ENSURE_NOT_FROZEN Result = std::move(result); - } - - bool IsCallable(const TStringBuf& name) const { + } + + bool IsCallable(const TStringBuf& name) const { ENSURE_NOT_DELETED return Type() == TExprNode::Callable && Content() == name; - } - + } + bool IsCallable(const std::initializer_list<std::string_view>& names) const { ENSURE_NOT_DELETED return Type() == TExprNode::Callable && names.end() != std::find(names.begin(), names.end(), Content()); @@ -1413,11 +1413,11 @@ public: return Type() == TExprNode::World; } - bool IsAtom(const TStringBuf& content) const { + bool IsAtom(const TStringBuf& content) const { ENSURE_NOT_DELETED return Type() == TExprNode::Atom && Content() == content; - } - + } + bool IsList() const { ENSURE_NOT_DELETED return Type() == TExprNode::List; @@ -1433,26 +1433,26 @@ public: return Type() == TExprNode::Argument; } - bool IsComposable() const { + bool IsComposable() const { ENSURE_NOT_DELETED return !IsLambda() && TypeAnnotation_->IsComposable(); - } - - bool IsPersistable() const { + } + + bool IsPersistable() const { ENSURE_NOT_DELETED return !IsLambda() && TypeAnnotation_->IsPersistable(); - } - - bool IsComputable() const { + } + + bool IsComputable() const { ENSURE_NOT_DELETED return !IsLambda() && TypeAnnotation_->IsComputable(); - } - - bool IsInspectable() const { + } + + bool IsInspectable() const { ENSURE_NOT_DELETED return !IsLambda() && TypeAnnotation_->IsInspectable(); - } - + } + bool ForDisclosing() const { ENSURE_NOT_DELETED return Type() == TExprNode::List && ShallBeDisclosed; @@ -1464,16 +1464,16 @@ public: ShallBeDisclosed = true; } - ui32 GetFlagsToCompare() const { + ui32 GetFlagsToCompare() const { ENSURE_NOT_DELETED ui32 ret = Flags(); - if ((ret & TNodeFlags::BinaryContent) == 0) { - ret |= TNodeFlags::ArbitraryContent | TNodeFlags::MultilineContent; - } - - return ret; - } - + if ((ret & TNodeFlags::BinaryContent) == 0) { + ret |= TNodeFlags::ArbitraryContent | TNodeFlags::MultilineContent; + } + + return ret; + } + TString Dump() const; bool StartsExecution() const { @@ -1968,8 +1968,8 @@ private: ui8 UnordChildren : 1; ui8 ShallBeDisclosed: 1; }; -}; - +}; + class TExportTable { public: using TSymbols = THashMap<TString, TExprNode::TPtr>; @@ -2025,46 +2025,46 @@ public: virtual ~IModuleResolver() = default; }; -struct TExprStep { - enum ELevel { - ExpandApplyForLambdas, +struct TExprStep { + enum ELevel { + ExpandApplyForLambdas, ValidateProviders, Configure, ExprEval, DiscoveryIO, Epochs, Intents, - LoadTablesMetadata, + LoadTablesMetadata, RewriteIO, Recapture, LastLevel - }; - + }; + TExprStep() - { - } - - void Done(ELevel level) { + { + } + + void Done(ELevel level) { Steps_.Set(level); - } - + } + void Reset() { Steps_.Reset(); - } - + } + TExprStep& Repeat(ELevel level) { Steps_.Reset(level); return *this; } - bool IsDone(ELevel level) { + bool IsDone(ELevel level) { return Steps_.Test(level); - } - -private: + } + +private: TEnumBitSet<ELevel, ExpandApplyForLambdas, LastLevel> Steps_; -}; - +}; + template <typename T> struct TMakeTypeImpl; @@ -2213,7 +2213,7 @@ using TSingletonTypeCache = std::tuple< const TMultiExprType* >; -struct TExprContext : private TNonCopyable { +struct TExprContext : private TNonCopyable { class TFreezeGuard { public: TFreezeGuard(const TFreezeGuard&) = delete; @@ -2250,7 +2250,7 @@ struct TExprContext : private TNonCopyable { std::unordered_set<TStringBuf, THash<TStringBuf>> DisabledConstraints; ui64 NextUniqueId = 0; - ui64 NodeAllocationCounter = 0; + ui64 NodeAllocationCounter = 0; ui64 NodesAllocationLimit = 3000000; ui64 StringsAllocationLimit = 100000000; ui64 RepeatTransformLimit = 1000000; @@ -2260,13 +2260,13 @@ struct TExprContext : private TNonCopyable { std::unordered_multimap<ui64, TExprNode*> UniqueNodes; - TExprStep Step; - + TExprStep Step; + bool Frozen; explicit TExprContext(ui64 nextUniqueId = 0ULL); - ~TExprContext(); - + ~TExprContext(); + ui64 AllocateNextUniqueId() { ENSURE_NOT_FROZEN_CTX const auto ret = ++NextUniqueId; @@ -2314,7 +2314,7 @@ struct TExprContext : private TNonCopyable { TExprNode::TPtr DeepCopyLambda(const TExprNode& node, TExprNode::TPtr&& body = TExprNode::TPtr()); [[nodiscard]] TExprNode::TPtr FuseLambdas(const TExprNode& outer, const TExprNode& inner); - + using TCustomDeepCopier = std::function<bool(const TExprNode& node, TExprNode::TListType& newChildren)>; [[nodiscard]] @@ -2333,22 +2333,22 @@ struct TExprContext : private TNonCopyable { const auto node = TExprNode::NewAtom(AllocateNextUniqueId(), pos, AppendString(content), flags); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewArgument(TPositionHandle pos, const TStringBuf& name) { - ++NodeAllocationCounter; + ++NodeAllocationCounter; const auto node = TExprNode::NewArgument(AllocateNextUniqueId(), pos, AppendString(name)); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewArguments(TPositionHandle pos, TExprNode::TListType&& argNodes) { ++NodeAllocationCounter; const auto node = TExprNode::NewArguments(AllocateNextUniqueId(), pos, std::move(argNodes)); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewLambda(TPositionHandle pos, TExprNode::TListType&& lambda) { ++NodeAllocationCounter; const auto node = TExprNode::NewLambda(AllocateNextUniqueId(), pos, std::move(lambda)); @@ -2368,8 +2368,8 @@ struct TExprContext : private TNonCopyable { const auto node = TExprNode::NewLambda(AllocateNextUniqueId(), pos, std::move(args), std::move(body)); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewWorld(TPositionHandle pos) { ++NodeAllocationCounter; const auto node = TExprNode::NewWorld(AllocateNextUniqueId(), pos); @@ -2382,15 +2382,15 @@ struct TExprContext : private TNonCopyable { const auto node = TExprNode::NewList(AllocateNextUniqueId(), pos, std::move(children)); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewCallable(TPositionHandle pos, const TStringBuf& name, TExprNode::TListType&& children) { ++NodeAllocationCounter; const auto node = TExprNode::NewCallable(AllocateNextUniqueId(), pos, AppendString(name), std::move(children)); ExprNodes.emplace_back(node.Get()); return node; - } - + } + TExprNode::TPtr NewAtom(TPosition pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent) { return NewAtom(AppendPosition(pos), content, flags); } @@ -2438,7 +2438,7 @@ struct TExprContext : private TNonCopyable { void AddError(const TIssue& error) { ENSURE_NOT_FROZEN_CTX IssueManager.RaiseIssue(error); - } + } bool AddWarning(const TIssue& warning) { ENSURE_NOT_FROZEN_CTX @@ -2464,8 +2464,8 @@ private: std::unordered_set<TPositionHandle, TPositionHandleHasher, TPositionHandleEqualPred> PositionSet; std::deque<TPosition> Positions; -}; - +}; + template <typename T, typename... Args> inline const T* TExprContext::MakeConstraint(Args&&... args) { ENSURE_NOT_FROZEN_CTX @@ -2497,35 +2497,35 @@ const T* TExprContext::MakeType(Args&&... args) { return TMakeTypeImpl<T>::Make(*this, std::forward<Args>(args)...); } -struct TExprAnnotationFlags { - enum { - None = 0x00, - Position = 0x01, - Types = 0x02 - }; -}; - -/////////////////////////////////////////////////////////////////////////////// -// TNodeException -/////////////////////////////////////////////////////////////////////////////// -class TNodeException: public yexception { -public: - TNodeException(); - explicit TNodeException(const TExprNode& node); - explicit TNodeException(const TExprNode* node); +struct TExprAnnotationFlags { + enum { + None = 0x00, + Position = 0x01, + Types = 0x02 + }; +}; + +/////////////////////////////////////////////////////////////////////////////// +// TNodeException +/////////////////////////////////////////////////////////////////////////////// +class TNodeException: public yexception { +public: + TNodeException(); + explicit TNodeException(const TExprNode& node); + explicit TNodeException(const TExprNode* node); explicit TNodeException(const TPositionHandle& pos); - + inline const TPositionHandle& Pos() const { - return Pos_; - } - -private: + return Pos_; + } + +private: const TPositionHandle Pos_; -}; - +}; + bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx, IModuleResolver* resolver, bool hasAnnotations = false, ui32 typeAnnotationIndex = Max<ui32>(), ui16 syntaxVersion = 0); - + bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx, IModuleResolver* resolver, ui32 annotationFlags, ui16 syntaxVersion = 0); @@ -2537,10 +2537,10 @@ struct TLibraryCohesion { bool CompileExpr(TAstNode& astRoot, TLibraryCohesion& cohesion, TExprContext& ctx, ui16 syntaxVersion = 0); const TTypeAnnotationNode* CompileTypeAnnotation(const TAstNode& node, TExprContext& ctx); - + // validate consistency of arguments and lambdas void CheckArguments(const TExprNode& root); - + void CheckCounts(const TExprNode& root); // Compare expression trees and return first diffrent nodes. @@ -2565,13 +2565,13 @@ TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& ctx, ui32 anno TExprNode::TListType GetLambdaBody(const TExprNode& lambda); -} // namespace NYql - -template<> -inline void Out<NYql::TTypeAnnotationNode>( +} // namespace NYql + +template<> +inline void Out<NYql::TTypeAnnotationNode>( IOutputStream &out, const NYql::TTypeAnnotationNode& type) -{ - type.Out(out); -} +{ + type.Out(out); +} #include "yql_expr_builder.inl" diff --git a/ydb/library/yql/ast/yql_expr_types.cpp b/ydb/library/yql/ast/yql_expr_types.cpp index 6d21d5c550..0e44a620f3 100644 --- a/ydb/library/yql/ast/yql_expr_types.cpp +++ b/ydb/library/yql/ast/yql_expr_types.cpp @@ -1,19 +1,19 @@ -#include "yql_expr_types.h" - -namespace NYql { -} - -template<> +#include "yql_expr_types.h" + +namespace NYql { +} + +template<> void Out<NYql::ETypeAnnotationKind>(class IOutputStream &o, NYql::ETypeAnnotationKind x) { -#define YQL_TYPE_ANN_KIND_MAP_TO_STRING_IMPL(name, ...) \ - case NYql::ETypeAnnotationKind::name: \ - o << #name; \ - return; - - switch (x) { - YQL_TYPE_ANN_KIND_MAP(YQL_TYPE_ANN_KIND_MAP_TO_STRING_IMPL) - default: - o << static_cast<int>(x); - return; - } -} +#define YQL_TYPE_ANN_KIND_MAP_TO_STRING_IMPL(name, ...) \ + case NYql::ETypeAnnotationKind::name: \ + o << #name; \ + return; + + switch (x) { + YQL_TYPE_ANN_KIND_MAP(YQL_TYPE_ANN_KIND_MAP_TO_STRING_IMPL) + default: + o << static_cast<int>(x); + return; + } +} diff --git a/ydb/library/yql/ast/yql_expr_types.h b/ydb/library/yql/ast/yql_expr_types.h index 26473268d0..391fa1ca97 100644 --- a/ydb/library/yql/ast/yql_expr_types.h +++ b/ydb/library/yql/ast/yql_expr_types.h @@ -1,12 +1,12 @@ -#pragma once +#pragma once #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/stream/output.h> - -namespace NYql { - -#define YQL_TYPE_ANN_KIND_MAP(xx) \ +#include <util/stream/output.h> + +namespace NYql { + +#define YQL_TYPE_ANN_KIND_MAP(xx) \ xx(Unit, 1) \ - xx(Tuple, 2) \ + xx(Tuple, 2) \ xx(Struct, 3) \ xx(Item, 4) \ xx(List, 5) \ @@ -28,13 +28,13 @@ namespace NYql { xx(EmptyList, 21) \ xx(EmptyDict, 22) \ xx(Multi, 23) - + enum class ETypeAnnotationKind : ui64 { - YQL_TYPE_ANN_KIND_MAP(ENUM_VALUE_GEN) + YQL_TYPE_ANN_KIND_MAP(ENUM_VALUE_GEN) LastType -}; - -} - -template<> +}; + +} + +template<> void Out<NYql::ETypeAnnotationKind>(class IOutputStream &o, NYql::ETypeAnnotationKind x); diff --git a/ydb/library/yql/ast/yql_expr_ut.cpp b/ydb/library/yql/ast/yql_expr_ut.cpp index bcbc508464..06550f2507 100644 --- a/ydb/library/yql/ast/yql_expr_ut.cpp +++ b/ydb/library/yql/ast/yql_expr_ut.cpp @@ -1,27 +1,27 @@ -#include "yql_expr.h" +#include "yql_expr.h" #include <library/cpp/testing/unittest/registar.h> - -#include <util/string/hex.h> - -namespace NYql { - + +#include <util/string/hex.h> + +namespace NYql { + Y_UNIT_TEST_SUITE(TCompileYqlExpr) { - static TAstParseResult ParseAstWithCheck(const TStringBuf& s) { - TAstParseResult res = ParseAst(s); + static TAstParseResult ParseAstWithCheck(const TStringBuf& s) { + TAstParseResult res = ParseAst(s); res.Issues.PrintTo(Cout); - UNIT_ASSERT(res.IsOk()); - return res; - } - + UNIT_ASSERT(res.IsOk()); + return res; + } + static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, ui32 typeAnnotationIndex = Max<ui32>()) { const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex); exprCtx.IssueManager.GetIssues().PrintTo(Cout); - - UNIT_ASSERT(success); + + UNIT_ASSERT(success); UNIT_ASSERT_VALUES_EQUAL(exprRoot->GetState(), typeAnnotationIndex != Max<ui32>() ? TExprNode::EState::TypeComplete : TExprNode::EState::Initial); - } - + } + static void CompileExprWithCheck(TAstNode& root, TLibraryCohesion& cohesion, TExprContext& exprCtx) { const bool success = CompileExpr(root, cohesion, exprCtx); exprCtx.IssueManager.GetIssues().PrintTo(Cout); @@ -30,27 +30,27 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { } static bool ParseAndCompile(const TString& program) { - TAstParseResult astRes = ParseAstWithCheck(program); - TExprContext exprCtx; + TAstParseResult astRes = ParseAstWithCheck(program); + TExprContext exprCtx; TExprNode::TPtr exprRoot; bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr); exprCtx.IssueManager.GetIssues().PrintTo(Cout); return result; - } - + } + Y_UNIT_TEST(TestNoReturn1) { - auto s = "(\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } - + auto s = "(\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } + Y_UNIT_TEST(TestNoReturn2) { - auto s = "(\n" - "(let x 'y)\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } - + auto s = "(\n" + "(let x 'y)\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } + Y_UNIT_TEST(TestExportInsteadOfReturn) { const auto s = "# library\n" @@ -63,25 +63,25 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { } Y_UNIT_TEST(TestLeftAfterReturn) { - auto s = "(\n" - "(return 'x)\n" - "(let x 'y)\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } - + auto s = "(\n" + "(return 'x)\n" + "(let x 'y)\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } + Y_UNIT_TEST(TestReturn) { - auto s = "(\n" - "(return world)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + auto s = "(\n" + "(return world)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::World); - } - + } + Y_UNIT_TEST(TestExport) { auto s = "(\n" "(let X 'Y)\n" @@ -115,104 +115,104 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { } Y_UNIT_TEST(TestArbitraryAtom) { - auto s = "(\n" + auto s = "(\n" "(let x '\"\\x01\\x23\\x45\\x67\\x89\\xAB\\xCD\\xEF\")" - "(return x)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + "(return x)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::Atom); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(exprRoot->Content()), "0123456789ABCDEF"); UNIT_ASSERT(exprRoot->Flags() & TNodeFlags::ArbitraryContent); - + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); - TAstNode* xValue = ast.Root->GetChild(0)->GetChild(1)->GetChild(1); + TAstNode* xValue = ast.Root->GetChild(0)->GetChild(1)->GetChild(1); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(TString(xValue->GetContent())), "0123456789ABCDEF"); - UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::ArbitraryContent); - } - + UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::ArbitraryContent); + } + Y_UNIT_TEST(TestBinaryAtom) { - auto s = "(\n" - "(let x 'x\"FEDCBA9876543210\")" - "(return x)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + auto s = "(\n" + "(let x 'x\"FEDCBA9876543210\")" + "(return x)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::Atom); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(exprRoot->Content()), "FEDCBA9876543210"); UNIT_ASSERT(exprRoot->Flags() & TNodeFlags::BinaryContent); - + auto ast = ConvertToAst(*exprRoot, exprCtx, TExprAnnotationFlags::None, true); TAstNode* xValue = ast.Root->GetChild(0)->GetChild(2)->GetChild(1); UNIT_ASSERT_STRINGS_EQUAL(HexEncode(TString(xValue->GetContent())), "FEDCBA9876543210"); - UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::BinaryContent); - } - + UNIT_ASSERT(xValue->GetFlags() & TNodeFlags::BinaryContent); + } + Y_UNIT_TEST(TestLet) { - auto s = "(\n" - "(let x 'y)\n" - "(return x)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + auto s = "(\n" + "(let x 'y)\n" + "(return x)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::Atom); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Content(), "y"); - } - + } + Y_UNIT_TEST(TestComplexQuote) { - auto s = "(\n" - "(let x 'a)\n" - "(let y 'b)\n" - "(let z (quote (x y)))\n" - "(return z)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + auto s = "(\n" + "(let x 'a)\n" + "(let y 'b)\n" + "(let z (quote (x y)))\n" + "(return z)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::List); UNIT_ASSERT_VALUES_EQUAL(exprRoot->ChildrenSize(), 2); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(0)->Type(), TExprNode::Atom); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(0)->Content(), "a"); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Type(), TExprNode::Atom); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Content(), "b"); - } - + } + Y_UNIT_TEST(TestEmptyReturn) { - auto s = "(\n" - "(return)\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } - + auto s = "(\n" + "(return)\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } + Y_UNIT_TEST(TestManyReturn) { - auto s = "(\n" - "(return world world)\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } - + auto s = "(\n" + "(return world world)\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } + Y_UNIT_TEST(TestUnknownFunction) { - auto s = "(\n" - "(let a '2)\n" - "(let x (+ a '3))\n" - "(return x)\n" - ")\n"; - - TAstParseResult astRes = ParseAstWithCheck(s); - TExprContext exprCtx; + auto s = "(\n" + "(let a '2)\n" + "(let x (+ a '3))\n" + "(return x)\n" + ")\n"; + + TAstParseResult astRes = ParseAstWithCheck(s); + TExprContext exprCtx; TExprNode::TPtr exprRoot; - CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); + CompileExprWithCheck(*astRes.Root, exprRoot, exprCtx); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Type(), TExprNode::Callable); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Content(), "+"); UNIT_ASSERT_VALUES_EQUAL(exprRoot->ChildrenSize(), 2); @@ -220,15 +220,15 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(0)->Content(), "2"); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Type(), TExprNode::Atom); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Content(), "3"); - } - + } + Y_UNIT_TEST(TestReturnTwice) { - auto s = "(\n" - "(return)\n" - "(return)\n" - ")\n"; - UNIT_ASSERT(false == ParseAndCompile(s)); - } + auto s = "(\n" + "(return)\n" + "(return)\n" + ")\n"; + UNIT_ASSERT(false == ParseAndCompile(s)); + } Y_UNIT_TEST(TestDeclareNonTop) { const auto s = R"( @@ -303,8 +303,8 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) { UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Type(), TExprNode::Callable); UNIT_ASSERT_VALUES_EQUAL(exprRoot->Child(1)->Content(), "DataType"); } -} - +} + Y_UNIT_TEST_SUITE(TCompareExprTrees) { void CompileAndCompare(const TString& one, const TString& two, const std::pair<TPosition, TPosition> *const diffPositions = nullptr) { const auto progOne(ParseAst(one)), progTwo(ParseAst(two)); @@ -1215,4 +1215,4 @@ Y_UNIT_TEST_SUITE(TConvertToAst) { } } -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/ast/yql_gc_nodes.cpp b/ydb/library/yql/ast/yql_gc_nodes.cpp index eff26a4e7b..6259e9babe 100644 --- a/ydb/library/yql/ast/yql_gc_nodes.cpp +++ b/ydb/library/yql/ast/yql_gc_nodes.cpp @@ -1,5 +1,5 @@ -#include "yql_gc_nodes.h" - -namespace NYql { - -} +#include "yql_gc_nodes.h" + +namespace NYql { + +} diff --git a/ydb/library/yql/ast/yql_gc_nodes.h b/ydb/library/yql/ast/yql_gc_nodes.h index fc84bfb529..59c35d5a51 100644 --- a/ydb/library/yql/ast/yql_gc_nodes.h +++ b/ydb/library/yql/ast/yql_gc_nodes.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <util/system/types.h> - -namespace NYql { - -struct TGcNodeSettings { - ui64 NodeCountThreshold = 1000; - double CollectRatio = 0.8; -}; - -struct TGcNodeStatistics { - ui64 CollectCount = 0; - ui64 TotalCollectedNodes = 0; -}; - + +namespace NYql { + +struct TGcNodeSettings { + ui64 NodeCountThreshold = 1000; + double CollectRatio = 0.8; +}; + +struct TGcNodeStatistics { + ui64 CollectCount = 0; + ui64 TotalCollectedNodes = 0; +}; + struct TGcNodeConfig { - TGcNodeSettings Settings; - TGcNodeStatistics Statistics; -}; - -} + TGcNodeSettings Settings; + TGcNodeStatistics Statistics; +}; + +} diff --git a/ydb/library/yql/ast/yql_type_string.cpp b/ydb/library/yql/ast/yql_type_string.cpp index cd46a14e65..af9af93259 100644 --- a/ydb/library/yql/ast/yql_type_string.cpp +++ b/ydb/library/yql/ast/yql_type_string.cpp @@ -1,36 +1,36 @@ -#include "yql_type_string.h" -#include "yql_expr.h" +#include "yql_type_string.h" +#include "yql_expr.h" #include "yql_ast_escaping.h" - + #include <library/cpp/containers/stack_vector/stack_vec.h> - -#include <util/string/cast.h> -#include <util/generic/map.h> -#include <util/generic/utility.h> + +#include <util/string/cast.h> +#include <util/generic/map.h> +#include <util/generic/utility.h> #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> - - + + #define EXPECT_AND_SKIP_TOKEN_IMPL(token, message, result) \ - do { \ + do { \ if (Y_LIKELY(Token == token)) { \ GetNextToken(); \ } else { \ AddError(message); \ return result; \ - } \ - } while (0); - + } \ + } while (0); + #define EXPECT_AND_SKIP_TOKEN(token, result) \ EXPECT_AND_SKIP_TOKEN_IMPL(token, "Expected " #token, result) - -namespace NYql { -namespace { - + +namespace NYql { +namespace { + enum EToken { TOKEN_EOF = -1, - + // type keywords TOKEN_TYPE_MIN = -2, TOKEN_STRING = -3, @@ -147,29 +147,29 @@ EToken TokenTypeFromStr(TStringBuf str) if (it != map.end()) { return it->second; } - + return TOKEN_IDENTIFIER; } -////////////////////////////////////////////////////////////////////////////// -// TTypeParser -////////////////////////////////////////////////////////////////////////////// -class TTypeParser -{ -public: - TTypeParser( +////////////////////////////////////////////////////////////////////////////// +// TTypeParser +////////////////////////////////////////////////////////////////////////////// +class TTypeParser +{ +public: + TTypeParser( TStringBuf str, TIssues& issues, - TPosition position, TMemoryPool& pool) - : Str(str) + TPosition position, TMemoryPool& pool) + : Str(str) , Issues(issues) - , Position(position) - , Index(0) - , Pool(pool) + , Position(position) + , Index(0) + , Pool(pool) { GetNextToken(); } - + TAstNode* ParseTopLevelType() { TAstNode* type = ParseType(); if (type) { @@ -180,12 +180,12 @@ public: } private: - TAstNode* ParseType() { - TAstNode* type = nullptr; + TAstNode* ParseType() { + TAstNode* type = nullptr; switch (Token) { case '(': return ParseCallableType(); - + case TOKEN_STRING: case TOKEN_BOOL: case TOKEN_INT8: @@ -214,7 +214,7 @@ private: type = MakeDataType(Identifier); GetNextToken(); break; - + case TOKEN_DECIMAL: type = ParseDecimalType(); break; @@ -222,7 +222,7 @@ private: case TOKEN_LIST: type = ParseListType(); break; - + case TOKEN_OPTIONAL: type = ParseOptionalType(); break; @@ -307,16 +307,16 @@ private: } return AddError(TString("Unknown type: '") + Identifier + "\'"); } - + if (type) { while (Token == '?') { type = MakeOptionalType(type); GetNextToken(); } } - return type; - } - + return type; + } + char LookaheadNonSpaceChar() { size_t i = Index; while (i < Str.size() && isspace(Str[i])) { @@ -327,8 +327,8 @@ private: int GetNextToken() { return Token = ReadNextToken(); - } - + } + int ReadNextToken() { // skip spaces while (!AtEnd() && isspace(Get())) { @@ -387,38 +387,38 @@ private: } } - TAstNode* ParseCallableType() { + TAstNode* ParseCallableType() { EXPECT_AND_SKIP_TOKEN('(', nullptr); - - TSmallVec<TAstNode*> args; + + TSmallVec<TAstNode*> args; args.push_back(nullptr); // CallableType Atom + settings + return type - args.push_back(nullptr); - args.push_back(nullptr); - bool optArgsStarted = false; + args.push_back(nullptr); + args.push_back(nullptr); + bool optArgsStarted = false; bool namedArgsStarted = false; - ui32 optArgsCount = 0; + ui32 optArgsCount = 0; bool lastWasTypeStatement = false; - - // (1) parse argements - for (;;) { + + // (1) parse argements + for (;;) { if (Token == TOKEN_EOF) { - if (optArgsStarted) { - return AddError("Expected ']'"); - } - return AddError("Expected ')'"); - } - + if (optArgsStarted) { + return AddError("Expected ']'"); + } + return AddError("Expected ')'"); + } + if (Token == ']' || Token == ')') { - break; - } - + break; + } + if (lastWasTypeStatement) { EXPECT_AND_SKIP_TOKEN(',', nullptr); lastWasTypeStatement = false; - } - + } + if (Token == '[') { - optArgsStarted = true; + optArgsStarted = true; GetNextToken(); // eat '[' } else if (Token == ':') { return AddError("Expected non empty argument name"); @@ -453,9 +453,9 @@ private: auto argType = ParseType(); if (!argType) { return nullptr; - } + } lastWasTypeStatement = true; - + if (optArgsStarted) { if (!argType->IsList() || argType->GetChildrenCount() == 0 || !argType->GetChild(0)->IsAtom() || @@ -469,7 +469,7 @@ private: ui32 argFlags = 0; if (Token == '{') { if (!ParseCallableArgFlags(argFlags)) return nullptr; - } + } TSmallVec<TAstNode*> argSettings; argSettings.push_back(argType); @@ -488,24 +488,24 @@ private: } else { return AddError("Expected type or argument name"); } - } - - if (optArgsStarted) { + } + + if (optArgsStarted) { EXPECT_AND_SKIP_TOKEN(']', nullptr); - } - + } + EXPECT_AND_SKIP_TOKEN(')', nullptr); - - // (2) expect '->' after arguments + + // (2) expect '->' after arguments EXPECT_AND_SKIP_TOKEN_IMPL( TOKEN_ARROW, "Expected '->' after arguments", nullptr); - - // (3) parse return type - TAstNode* returnType = ParseType(); - if (!returnType) { - return nullptr; - } - + + // (3) parse return type + TAstNode* returnType = ParseType(); + if (!returnType) { + return nullptr; + } + // (4) parse payload TStringBuf payload; if (Token == '{') { @@ -513,8 +513,8 @@ private: } return MakeCallableType(args, optArgsCount, returnType, payload); - } - + } + // { Flags: f1 | f2 | f3 } bool ParseCallableArgFlags(ui32& argFlags) { GetNextToken(); // eat '{' @@ -577,28 +577,28 @@ private: return true; } - TAstNode* ParseCallableTypeWithKeyword() { + TAstNode* ParseCallableTypeWithKeyword() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); - - auto type = ParseCallableType(); - if (!type) return nullptr; - + + auto type = ParseCallableType(); + if (!type) return nullptr; + EXPECT_AND_SKIP_TOKEN('>', nullptr); - return type; - } - - TAstNode* ParseListType() { + return type; + } + + TAstNode* ParseListType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); - - auto itemType = ParseType(); - if (!itemType) return nullptr; - + + auto itemType = ParseType(); + if (!itemType) return nullptr; + EXPECT_AND_SKIP_TOKEN('>', nullptr); - return MakeListType(itemType); - } - + return MakeListType(itemType); + } + TAstNode* ParseStreamType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -638,33 +638,33 @@ private: return MakeDecimalType(precision, scale); } - TAstNode* ParseOptionalType() { + TAstNode* ParseOptionalType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); - - auto itemType = ParseType(); - if (!itemType) return nullptr; - + + auto itemType = ParseType(); + if (!itemType) return nullptr; + EXPECT_AND_SKIP_TOKEN('>', nullptr); - return MakeOptionalType(itemType); - } - - TAstNode* ParseDictType() { + return MakeOptionalType(itemType); + } + + TAstNode* ParseDictType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); - - auto keyType = ParseType(); - if (!keyType) return nullptr; - + + auto keyType = ParseType(); + if (!keyType) return nullptr; + EXPECT_AND_SKIP_TOKEN(',', nullptr); - - auto valueType = ParseType(); - if (!valueType) return nullptr; - + + auto valueType = ParseType(); + if (!valueType) return nullptr; + EXPECT_AND_SKIP_TOKEN('>', nullptr); - return MakeDictType(keyType, valueType); - } - + return MakeDictType(keyType, valueType); + } + TAstNode* ParseSetType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -677,16 +677,16 @@ private: } TAstNode* ParseTupleTypeImpl() { - TSmallVec<TAstNode*> items; - items.push_back(nullptr); // reserve for TupleType - + TSmallVec<TAstNode*> items; + items.push_back(nullptr); // reserve for TupleType + if (Token != '>') { for (;;) { auto itemType = ParseType(); if (!itemType) return nullptr; - + items.push_back(itemType); - + if (Token == '>') { break; } else if (Token == ',') { @@ -694,12 +694,12 @@ private: } else { return AddError("Expected '>' or ','"); } - } - } + } + } + + return MakeTupleType(items); + } - return MakeTupleType(items); - } - TAstNode* ParseTupleType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -709,7 +709,7 @@ private: } return tupleType; } - + TAstNode* ParseStructTypeImpl() { TMap<TString, TAstNode*> members; if (Token != '>') { @@ -722,7 +722,7 @@ private: } else { return AddError("Expected struct member name"); } - + if (name.empty()) { return AddError("Empty name is not allowed"); } else if (members.contains(name)) { @@ -731,12 +731,12 @@ private: GetNextToken(); // eat member name EXPECT_AND_SKIP_TOKEN(':', nullptr); - + auto type = ParseType(); if (!type) return nullptr; - + members.emplace(std::move(name), type); - + if (Token == '>') { break; } else if (Token == ',') { @@ -744,12 +744,12 @@ private: } else { return AddError("Expected '>' or ','"); } - } - } + } + } return MakeStructType(members); } - + TAstNode* ParseStructType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -777,8 +777,8 @@ private: EXPECT_AND_SKIP_TOKEN('>', nullptr); return MakeVariantType(underlyingType); - } - + } + TAstNode* ParseEnumType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -817,15 +817,15 @@ private: return MakeVariantType(underlyingType); } - TAstNode* MakeCallableType( - TSmallVec<TAstNode*>& args, size_t optionalArgsCount, + TAstNode* MakeCallableType( + TSmallVec<TAstNode*>& args, size_t optionalArgsCount, TAstNode* returnType, TStringBuf payload) - { + { args[0] = MakeLiteralAtom(TStringBuf("CallableType")); TSmallVec<TAstNode*> mainSettings; if (optionalArgsCount || !payload.empty()) { mainSettings.push_back(optionalArgsCount - ? MakeQuotedAtom(ToString(optionalArgsCount)) + ? MakeQuotedAtom(ToString(optionalArgsCount)) : MakeQuotedLiteralAtom(TStringBuf("0"))); } @@ -839,17 +839,17 @@ private: returnSettings.push_back(returnType); args[2] = MakeQuote(MakeList(returnSettings.data(), returnSettings.size())); - return MakeList(args.data(), args.size()); - } - - TAstNode* MakeListType(TAstNode* itemType) { - TAstNode* items[] = { + return MakeList(args.data(), args.size()); + } + + TAstNode* MakeListType(TAstNode* itemType) { + TAstNode* items[] = { MakeLiteralAtom(TStringBuf("ListType")), - itemType, - }; + itemType, + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - + } + TAstNode* MakeStreamType(TAstNode* itemType) { TAstNode* items[] = { MakeLiteralAtom(TStringBuf("StreamType")), @@ -874,35 +874,35 @@ private: return MakeList(items, Y_ARRAY_SIZE(items)); } - TAstNode* MakeDictType(TAstNode* keyType, TAstNode* valueType) { - TAstNode* items[] = { + TAstNode* MakeDictType(TAstNode* keyType, TAstNode* valueType) { + TAstNode* items[] = { MakeLiteralAtom(TStringBuf("DictType")), - keyType, - valueType, - }; + keyType, + valueType, + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - - TAstNode* MakeTupleType(TSmallVec<TAstNode*>& items) { + } + + TAstNode* MakeTupleType(TSmallVec<TAstNode*>& items) { items[0] = MakeLiteralAtom(TStringBuf("TupleType")); - return MakeList(items.data(), items.size()); - } - + return MakeList(items.data(), items.size()); + } + TAstNode* MakeStructType(const TMap<TString, TAstNode*>& members) { - TSmallVec<TAstNode*> items; + TSmallVec<TAstNode*> items; items.push_back(MakeLiteralAtom(TStringBuf("StructType"))); - - for (const auto& member: members) { - TAstNode* memberType[] = { - MakeQuotedAtom(member.first, TNodeFlags::ArbitraryContent), // name - member.second, // type - }; + + for (const auto& member: members) { + TAstNode* memberType[] = { + MakeQuotedAtom(member.first, TNodeFlags::ArbitraryContent), // name + member.second, // type + }; items.push_back(MakeQuote(MakeList(memberType, Y_ARRAY_SIZE(memberType)))); - } - - return MakeList(items.data(), items.size()); - } - + } + + return MakeList(items.data(), items.size()); + } + TAstNode* ParseResourceType() { GetNextToken(); // eat keyword EXPECT_AND_SKIP_TOKEN('<', nullptr); @@ -945,13 +945,13 @@ private: } TAstNode* MakeResourceType(TStringBuf tag) { - TAstNode* items[] = { + TAstNode* items[] = { MakeLiteralAtom(TStringBuf("ResourceType")), MakeQuotedAtom(tag), - }; + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - + } + TAstNode* MakeVoidType() { TAstNode* items[] = { MakeLiteralAtom(TStringBuf("VoidType")) @@ -1005,13 +1005,13 @@ private: } TAstNode* MakeDataType(TStringBuf type) { - TAstNode* items[] = { + TAstNode* items[] = { MakeLiteralAtom(TStringBuf("DataType")), MakeQuotedAtom(type), - }; + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - + } + TAstNode* MakeDecimalType(TStringBuf precision, TStringBuf scale) { TAstNode* items[] = { MakeLiteralAtom(TStringBuf("DataType")), @@ -1022,95 +1022,95 @@ private: return MakeList(items, Y_ARRAY_SIZE(items)); } - TAstNode* MakeOptionalType(TAstNode* type) { - TAstNode* items[] = { + TAstNode* MakeOptionalType(TAstNode* type) { + TAstNode* items[] = { MakeLiteralAtom(TStringBuf("OptionalType")), - type, - }; + type, + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - + } + TAstNode* MakeAtom(TStringBuf content, ui32 flags = TNodeFlags::Default) { - return TAstNode::NewAtom(Position, content, Pool, flags); - } - + return TAstNode::NewAtom(Position, content, Pool, flags); + } + TAstNode* MakeLiteralAtom(TStringBuf content, ui32 flags = TNodeFlags::Default) { - return TAstNode::NewLiteralAtom(Position, content, Pool, flags); - } - - TAstNode* MakeQuote(TAstNode* node) { - TAstNode* items[] = { - &TAstNode::QuoteAtom, - node, - }; + return TAstNode::NewLiteralAtom(Position, content, Pool, flags); + } + + TAstNode* MakeQuote(TAstNode* node) { + TAstNode* items[] = { + &TAstNode::QuoteAtom, + node, + }; return MakeList(items, Y_ARRAY_SIZE(items)); - } - + } + TAstNode* MakeQuotedAtom(TStringBuf content, ui32 flags = TNodeFlags::Default) { - return MakeQuote(MakeAtom(content, flags)); - } - + return MakeQuote(MakeAtom(content, flags)); + } + TAstNode* MakeQuotedLiteralAtom(TStringBuf content, ui32 flags = TNodeFlags::Default) { - return MakeQuote(MakeLiteralAtom(content, flags)); - } - - TAstNode* MakeList(TAstNode** children, ui32 count) { - return TAstNode::NewList(Position, children, count, Pool); - } - - char Get() const { - return Str[Index]; - } - - bool AtEnd() const { + return MakeQuote(MakeLiteralAtom(content, flags)); + } + + TAstNode* MakeList(TAstNode** children, ui32 count) { + return TAstNode::NewList(Position, children, count, Pool); + } + + char Get() const { + return Str[Index]; + } + + bool AtEnd() const { return Index >= Str.size(); - } - - void Move() { + } + + void Move() { if (AtEnd()) return; - ++Index; - ++Position.Column; + ++Index; + ++Position.Column; if (!AtEnd() && Str[Index] == '\n') { Position.Row++; - Position.Column = 1; - } - } - + Position.Column = 1; + } + } + TAstNode* AddError(const TString& message) { Issues.AddIssue(TIssue(TokenBegin, message)); - return nullptr; - } - -private: - TStringBuf Str; + return nullptr; + } + +private: + TStringBuf Str; TIssues& Issues; TPosition TokenBegin, Position; - size_t Index; + size_t Index; int Token; TString UnescapedIdentifier; TStringBuf Identifier; - TMemoryPool& Pool; -}; - -////////////////////////////////////////////////////////////////////////////// -// TTypePrinter -////////////////////////////////////////////////////////////////////////////// -class TTypePrinter: public TTypeAnnotationVisitor -{ -public: + TMemoryPool& Pool; +}; + +////////////////////////////////////////////////////////////////////////////// +// TTypePrinter +////////////////////////////////////////////////////////////////////////////// +class TTypePrinter: public TTypeAnnotationVisitor +{ +public: TTypePrinter(IOutputStream& out) - : Out_(out) - { - } - -private: + : Out_(out) + { + } + +private: void Visit(const TUnitExprType& type) final { Y_UNUSED(type); Out_ << TStringBuf("Unit"); - } - + } + void Visit(const TMultiExprType& type) final { Out_ << TStringBuf("Multi<"); const auto& items = type.GetItems(); @@ -1125,40 +1125,40 @@ private: void Visit(const TTupleExprType& type) final { Out_ << TStringBuf("Tuple<"); - const auto& items = type.GetItems(); - for (ui32 i = 0; i < items.size(); ++i) { - if (i) { - Out_ << ','; - } - items[i]->Accept(*this); - } - Out_ << '>'; - } - + const auto& items = type.GetItems(); + for (ui32 i = 0; i < items.size(); ++i) { + if (i) { + Out_ << ','; + } + items[i]->Accept(*this); + } + Out_ << '>'; + } + void Visit(const TStructExprType& type) final { Out_ << TStringBuf("Struct<"); - const auto& items = type.GetItems(); - for (ui32 i = 0; i < items.size(); ++i) { - if (i) { - Out_ << ','; - } - items[i]->Accept(*this); - } - Out_ << '>'; - } - + const auto& items = type.GetItems(); + for (ui32 i = 0; i < items.size(); ++i) { + if (i) { + Out_ << ','; + } + items[i]->Accept(*this); + } + Out_ << '>'; + } + void Visit(const TItemExprType& type) final { EscapeArbitraryAtom(type.GetName(), '\'', &Out_); Out_ << ':'; - type.GetItemType()->Accept(*this); - } - + type.GetItemType()->Accept(*this); + } + void Visit(const TListExprType& type) final { Out_ << TStringBuf("List<"); - type.GetItemType()->Accept(*this); - Out_ << '>'; - } - + type.GetItemType()->Accept(*this); + Out_ << '>'; + } + void Visit(const TStreamExprType& type) final { Out_ << TStringBuf("Stream<"); type.GetItemType()->Accept(*this); @@ -1176,39 +1176,39 @@ private: if (const auto dataExprParamsType = dynamic_cast<const TDataExprParamsType*>(&type)) { Out_ << '(' << dataExprParamsType->GetParamOne() << ',' << dataExprParamsType->GetParamTwo() << ')'; } - } - + } + void Visit(const TWorldExprType& type) final { Y_UNUSED(type); Out_ << TStringBuf("World"); - } - + } + void Visit(const TOptionalExprType& type) final { - const TTypeAnnotationNode* itemType = type.GetItemType(); - if (itemType->GetKind() == ETypeAnnotationKind::Callable) { + const TTypeAnnotationNode* itemType = type.GetItemType(); + if (itemType->GetKind() == ETypeAnnotationKind::Callable) { Out_ << TStringBuf("Optional<"); - itemType->Accept(*this); - Out_ << '>'; - } else { - itemType->Accept(*this); - Out_ << '?'; - } - } - + itemType->Accept(*this); + Out_ << '>'; + } else { + itemType->Accept(*this); + Out_ << '?'; + } + } + void Visit(const TCallableExprType& type) final { - const auto& args = type.GetArguments(); - ui32 argsCount = type.GetArgumentsSize(); - ui32 optArgsCount = - Min<ui32>(type.GetOptionalArgumentsCount(), argsCount); - + const auto& args = type.GetArguments(); + ui32 argsCount = type.GetArgumentsSize(); + ui32 optArgsCount = + Min<ui32>(type.GetOptionalArgumentsCount(), argsCount); + Out_ << TStringBuf("Callable<("); - for (ui32 i = 0; i < argsCount; ++i) { - if (i) { - Out_ << ','; - } - if (i == argsCount - optArgsCount) { - Out_ << '['; - } + for (ui32 i = 0; i < argsCount; ++i) { + if (i) { + Out_ << ','; + } + if (i == argsCount - optArgsCount) { + Out_ << '['; + } const TCallableExprType::TArgumentInfo& argInfo = args[i]; if (!argInfo.Name.empty()) { EscapeArbitraryAtom(argInfo.Name, '\'', &Out_); @@ -1222,32 +1222,32 @@ private: } Out_ << '}'; } - } - - if (optArgsCount > 0) { - Out_ << ']'; - } - + } + + if (optArgsCount > 0) { + Out_ << ']'; + } + Out_ << TStringBuf(")->"); - type.GetReturnType()->Accept(*this); + type.GetReturnType()->Accept(*this); if (!type.GetPayload().empty()) { Out_ << TStringBuf("{Payload:") << type.GetPayload() << '}'; } - Out_ << '>'; - } - + Out_ << '>'; + } + void Visit(const TResourceExprType& type) final { Out_ << TStringBuf("Resource<"); EscapeArbitraryAtom(type.GetTag(), '\'', &Out_); Out_ << '>'; - } - + } + void Visit(const TTypeExprType& type) final { Out_ << TStringBuf("Type<"); - type.GetType()->Accept(*this); - Out_ << '>'; - } - + type.GetType()->Accept(*this); + Out_ << '>'; + } + void Visit(const TDictExprType& type) final { if (type.GetPayloadType()->GetKind() == ETypeAnnotationKind::Void) { Out_ << TStringBuf("Set<"); @@ -1260,13 +1260,13 @@ private: type.GetPayloadType()->Accept(*this); Out_ << '>'; } - } - + } + void Visit(const TVoidExprType& type) final { Y_UNUSED(type); Out_ << TStringBuf("Void"); - } - + } + void Visit(const TNullExprType& type) final { Y_UNUSED(type); Out_ << TStringBuf("Null"); @@ -1285,8 +1285,8 @@ private: void Visit(const TGenericExprType& type) final { Y_UNUSED(type); Out_ << TStringBuf("Generic"); - } - + } + void Visit(const TTaggedExprType& type) final { Out_ << TStringBuf("Tagged<"); type.GetBaseType()->Accept(*this); @@ -1345,26 +1345,26 @@ private: Out_ << '>'; } -private: +private: IOutputStream& Out_; -}; - -} // namespace - - +}; + +} // namespace + + TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, - TPosition position /* = TPosition(1, 1) */) -{ + TPosition position /* = TPosition(1, 1) */) +{ TTypeParser parser(str, issues, position, pool); return parser.ParseTopLevelType(); -} - +} + TString FormatType(const TTypeAnnotationNode* typeNode) -{ - TStringStream ss; - TTypePrinter printer(ss); - typeNode->Accept(printer); - return ss.Str(); -} - -} // namespace NYql +{ + TStringStream ss; + TTypePrinter printer(ss); + typeNode->Accept(printer); + return ss.Str(); +} + +} // namespace NYql diff --git a/ydb/library/yql/ast/yql_type_string.h b/ydb/library/yql/ast/yql_type_string.h index 7b15ee0ef3..95c1908b9f 100644 --- a/ydb/library/yql/ast/yql_type_string.h +++ b/ydb/library/yql/ast/yql_type_string.h @@ -1,15 +1,15 @@ -#pragma once - -#include "yql_ast.h" +#pragma once + +#include "yql_ast.h" #include <ydb/library/yql/public/issue/yql_issue_manager.h> - -namespace NYql { - -class TTypeAnnotationNode; - + +namespace NYql { + +class TTypeAnnotationNode; + TAstNode* ParseType(TStringBuf str, TMemoryPool& pool, TIssues& issues, - TPosition position = {1, 1}); - + TPosition position = {1, 1}); + TString FormatType(const TTypeAnnotationNode* typeNode); - -} // namespace NYql + +} // namespace NYql diff --git a/ydb/library/yql/ast/yql_type_string_ut.cpp b/ydb/library/yql/ast/yql_type_string_ut.cpp index 6120a3535c..2f22c42258 100644 --- a/ydb/library/yql/ast/yql_type_string_ut.cpp +++ b/ydb/library/yql/ast/yql_type_string_ut.cpp @@ -1,39 +1,39 @@ -#include "yql_type_string.h" -#include "yql_expr.h" - +#include "yql_type_string.h" +#include "yql_expr.h" + #include <library/cpp/testing/unittest/registar.h> - - -using namespace NYql; - + + +using namespace NYql; + Y_UNIT_TEST_SUITE(TTypeString) -{ - void TestFail(const TStringBuf& prog, ui32 column, const TStringBuf& expectedError) { - TMemoryPool pool(4096); +{ + void TestFail(const TStringBuf& prog, ui32 column, const TStringBuf& expectedError) { + TMemoryPool pool(4096); TIssues errors; - auto res = ParseType(prog, pool, errors); - UNIT_ASSERT(res == nullptr); - UNIT_ASSERT(!errors.Empty()); + auto res = ParseType(prog, pool, errors); + UNIT_ASSERT(res == nullptr); + UNIT_ASSERT(!errors.Empty()); errors.PrintWithProgramTo(Cerr, "-memory-", TString(prog)); - UNIT_ASSERT_STRINGS_EQUAL(errors.begin()->Message, expectedError); + UNIT_ASSERT_STRINGS_EQUAL(errors.begin()->Message, expectedError); UNIT_ASSERT_VALUES_EQUAL(errors.begin()->Position.Column, column); - } - - void TestOk(const TStringBuf& prog, const TStringBuf& expectedType) { - TMemoryPool pool(4096); + } + + void TestOk(const TStringBuf& prog, const TStringBuf& expectedType) { + TMemoryPool pool(4096); TIssues errors; - auto res = ParseType(prog, pool, errors); - if (!res) { + auto res = ParseType(prog, pool, errors); + if (!res) { errors.PrintWithProgramTo(Cerr, "-memory-", TString(prog)); UNIT_FAIL(TStringBuilder() << "Parsing failed:" << Endl << prog); - } - UNIT_ASSERT_STRINGS_EQUAL(res->ToString(), expectedType); - } - + } + UNIT_ASSERT_STRINGS_EQUAL(res->ToString(), expectedType); + } + Y_UNIT_TEST(ParseEmpty) { - TestFail("", 1, "Expected type"); - } - + TestFail("", 1, "Expected type"); + } + Y_UNIT_TEST(ParseDataTypes) { TestOk("String", "(DataType 'String)"); TestOk("Bool", "(DataType 'Bool)"); @@ -75,20 +75,20 @@ Y_UNIT_TEST_SUITE(TTypeString) Y_UNIT_TEST(ParseNoArgsWithStringResult) { TestOk("()->String", "(CallableType '() '((DataType 'String)))"); TestOk("()->Utf8", "(CallableType '() '((DataType 'Utf8)))"); - } - + } + Y_UNIT_TEST(ParseNoArgsWithOptionalStringResult) { - TestOk("()->String?", + TestOk("()->String?", "(CallableType '() '((OptionalType (DataType 'String))))"); TestOk("()->Yson?", "(CallableType '() '((OptionalType (DataType 'Yson))))"); - } - + } + Y_UNIT_TEST(ParseOneArgWithDoubleResult) { - TestOk("(Int32)->Double", + TestOk("(Int32)->Double", "(CallableType '() '((DataType 'Double)) " "'((DataType 'Int32))" - ")"); + ")"); TestOk("(Yson)->Double", "(CallableType '() '((DataType 'Double)) " "'((DataType 'Yson))" @@ -97,65 +97,65 @@ Y_UNIT_TEST_SUITE(TTypeString) "(CallableType '() '((DataType 'Double)) " "'((DataType 'Utf8))" ")"); - } - + } + Y_UNIT_TEST(ParseTwoArgsWithOptionalByteResult) { TestOk("(Int32?, String)->Uint8?", "(CallableType '() '((OptionalType (DataType 'Uint8))) " "'((OptionalType (DataType 'Int32))) " "'((DataType 'String))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseWithEmptyOptionalArgsStringResult) { TestOk("([])->String", "(CallableType '() '((DataType 'String)))"); - } - + } + Y_UNIT_TEST(ParseWithOneOptionalArgDoubleResult) { TestOk("([Int32?])->Double", "(CallableType '('1) '((DataType 'Double)) " "'((OptionalType (DataType 'Int32)))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseOneReqAndOneOptionalArgsWithDoubleResult) { TestOk("(String,[Int32?])->Double", "(CallableType '('1) '((DataType 'Double)) " "'((DataType 'String)) " "'((OptionalType (DataType 'Int32)))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseOneReqAndTwoOptionalArgsWithDoubleResult) { TestOk("(String,[Int32?, Uint8?])->Double", "(CallableType '('2) '((DataType 'Double)) " "'((DataType 'String)) " "'((OptionalType (DataType 'Int32))) " "'((OptionalType (DataType 'Uint8)))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseCallableArgWithDoubleResult) { TestOk("(()->Uint8)->Double", "(CallableType '() '((DataType 'Double)) " "'((CallableType '() '((DataType 'Uint8))))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseCallableOptionalArgWithDoubleResult) { TestOk("([Optional<()->Uint8>])->Double", "(CallableType '('1) '((DataType 'Double)) " "'((OptionalType (CallableType '() '((DataType 'Uint8)))))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseOptionalCallableArgWithDoubleResult) { TestOk("(Optional<()->Uint8>)->Double", "(CallableType '() '((DataType 'Double)) " "'((OptionalType (CallableType '() '((DataType 'Uint8)))))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseCallableWithNamedArgs) { TestOk("(a:Uint8)->Double", "(CallableType '() '((DataType 'Double)) " @@ -213,29 +213,29 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(ParseOptional) { - TestOk("Uint32?", "(OptionalType (DataType 'Uint32))"); - TestOk("Optional<Uint32>", "(OptionalType (DataType 'Uint32))"); - TestOk("Uint32??", "(OptionalType (OptionalType (DataType 'Uint32)))"); - TestOk("Optional<Uint32>?", "(OptionalType (OptionalType (DataType 'Uint32)))"); - TestOk("Optional<Uint32?>", "(OptionalType (OptionalType (DataType 'Uint32)))"); - TestOk("Optional<Optional<Uint32>>", "(OptionalType (OptionalType (DataType 'Uint32)))"); - } - + TestOk("Uint32?", "(OptionalType (DataType 'Uint32))"); + TestOk("Optional<Uint32>", "(OptionalType (DataType 'Uint32))"); + TestOk("Uint32??", "(OptionalType (OptionalType (DataType 'Uint32)))"); + TestOk("Optional<Uint32>?", "(OptionalType (OptionalType (DataType 'Uint32)))"); + TestOk("Optional<Uint32?>", "(OptionalType (OptionalType (DataType 'Uint32)))"); + TestOk("Optional<Optional<Uint32>>", "(OptionalType (OptionalType (DataType 'Uint32)))"); + } + Y_UNIT_TEST(ParseCallableComplete) { - TestFail("(Uint32)->", 11, "Expected type"); + TestFail("(Uint32)->", 11, "Expected type"); TestFail("(,)->", 2, "Expected type or argument name"); TestFail("(Int32 Int32)->Int32", 8, "Expected ','"); - TestFail("([],)->Uint32", 4, "Expected ')'"); - TestFail("([)->Uint32", 3, "Expected ']'"); - TestFail("(])->Uint32", 2, "Expected ')'"); + TestFail("([],)->Uint32", 4, "Expected ')'"); + TestFail("([)->Uint32", 3, "Expected ']'"); + TestFail("(])->Uint32", 2, "Expected ')'"); TestFail("([,)->Uint32", 3, "Expected type or argument name"); TestFail("([,])->Uint32", 3, "Expected type or argument name"); TestFail("(->Uint32", 2, "Expected type or argument name"); TestFail("([Uint32],Uint8)->Uint32", 9, "Optionals are only allowed in the optional arguments"); TestFail("([Uint32?],Uint8)->Uint32", 11, "Expected ')'"); TestFail("Callable<()>", 12, "Expected '->' after arguments"); - TestFail("Callable<()->", 14, "Expected type"); - TestFail("Callable<()->Uint32", 20, "Expected '>'"); + TestFail("Callable<()->", 14, "Expected type"); + TestFail("Callable<()->Uint32", 20, "Expected '>'"); TestFail("(:Uint32)->Uint32", 2, "Expected non empty argument name"); TestFail("(a:)->Uint32", 4, "Expected type"); TestFail("(:)->Uint32", 2, "Expected non empty argument name"); @@ -254,22 +254,22 @@ Y_UNIT_TEST_SUITE(TTypeString) TestFail("(Uint32)->Uint32}", 17, "Expected end of string"); TestFail("(Uint32)->Uint32{Payload}", 25, "Expected ':'"); TestFail("(Uint32)->Uint32{Payload:}", 26, "Expected payload data"); - } - + } + Y_UNIT_TEST(ParseCallableWithKeyword) { - TestOk("(Callable<()->String>) -> Callable<()->Uint32>", + TestOk("(Callable<()->String>) -> Callable<()->Uint32>", "(CallableType '() '((CallableType '() '((DataType 'Uint32)))) " "'((CallableType '() '((DataType 'String))))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseListOfDataType) { - TestOk("(List<String>)->String", + TestOk("(List<String>)->String", "(CallableType '() '((DataType 'String)) " "'((ListType (DataType 'String)))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseStreamOfDataType) { TestOk("(Stream<String>)->String", "(CallableType '() '((DataType 'String)) " @@ -320,50 +320,50 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(ParseListAsReturnType) { - TestOk("(String, String)->List<String>", + TestOk("(String, String)->List<String>", "(CallableType '() '((ListType (DataType 'String))) " "'((DataType 'String)) " "'((DataType 'String))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseListOfOptionalDataType) { - TestOk("(List<String?>)->String", + TestOk("(List<String?>)->String", "(CallableType '() '((DataType 'String)) " "'((ListType " - "(OptionalType (DataType 'String))" + "(OptionalType (DataType 'String))" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseOptionalListOfDataType) { - TestOk("(List<String>?)->String", + TestOk("(List<String>?)->String", "(CallableType '() '((DataType 'String)) " "'((OptionalType " - "(ListType (DataType 'String))" + "(ListType (DataType 'String))" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseListOfListType) { - TestOk("(List<List<Uint32>>)->Uint32", + TestOk("(List<List<Uint32>>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " "'((ListType " - "(ListType (DataType 'Uint32))" + "(ListType (DataType 'Uint32))" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseDictOfDataTypes) { - TestOk("(Dict<String, Uint32>)->Uint32", + TestOk("(Dict<String, Uint32>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " "'((DictType " - "(DataType 'String) " - "(DataType 'Uint32)" + "(DataType 'String) " + "(DataType 'Uint32)" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseSetOfDataTypes) { TestOk("(Set<String>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " @@ -375,10 +375,10 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(ParseListComplete) { - TestFail("(List<>)->Uint32", 7, "Expected type"); - TestFail("(List<Uint32,>)->Uint32", 13, "Expected '>'"); - } - + TestFail("(List<>)->Uint32", 7, "Expected type"); + TestFail("(List<Uint32,>)->Uint32", 13, "Expected '>'"); + } + Y_UNIT_TEST(ParseVariantComplete) { TestFail("Variant<>", 9, "Expected type"); TestFail("Variant<Uint32,>", 16, "Expected type"); @@ -390,72 +390,72 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(ParseDictOfDictTypes) { - TestOk("(Dict<String, Dict<Uint32, Uint32>>)->Uint32", + TestOk("(Dict<String, Dict<Uint32, Uint32>>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " "'((DictType " - "(DataType 'String) " - "(DictType (DataType 'Uint32) (DataType 'Uint32))" + "(DataType 'String) " + "(DictType (DataType 'Uint32) (DataType 'Uint32))" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseDictComplete) { - TestFail("(Dict<>)->Uint32", 7, "Expected type"); - TestFail("(Dict<Uint32>)->Uint32", 13, "Expected ','"); - TestFail("(Dict<Uint32,>)->Uint32", 14, "Expected type"); - TestFail("(Dict<Uint32, String)->Uint32", 21, "Expected '>'"); - } - + TestFail("(Dict<>)->Uint32", 7, "Expected type"); + TestFail("(Dict<Uint32>)->Uint32", 13, "Expected ','"); + TestFail("(Dict<Uint32,>)->Uint32", 14, "Expected type"); + TestFail("(Dict<Uint32, String)->Uint32", 21, "Expected '>'"); + } + Y_UNIT_TEST(ParseTupleOfDataTypes) { TestOk("(Tuple<String, Uint32, Uint8>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " "'((TupleType " - "(DataType 'String) " - "(DataType 'Uint32) " + "(DataType 'String) " + "(DataType 'Uint32) " "(DataType 'Uint8)" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseTupleComplete) { - TestFail("(Tuple<Uint32,>)->Uint32", 15, "Expected type"); - TestFail("(Tuple<Uint32)->Uint32", 14, "Expected '>' or ','"); - } - + TestFail("(Tuple<Uint32,>)->Uint32", 15, "Expected type"); + TestFail("(Tuple<Uint32)->Uint32", 14, "Expected '>' or ','"); + } + Y_UNIT_TEST(ParseStructOfDataTypes) { - TestOk("(Struct<Name: String, Age: Uint32, Male: Bool>)->Uint32", + TestOk("(Struct<Name: String, Age: Uint32, Male: Bool>)->Uint32", "(CallableType '() '((DataType 'Uint32)) " "'((StructType " - "'('\"Age\" (DataType 'Uint32)) " - "'('\"Male\" (DataType 'Bool)) " - "'('\"Name\" (DataType 'String))" + "'('\"Age\" (DataType 'Uint32)) " + "'('\"Male\" (DataType 'Bool)) " + "'('\"Name\" (DataType 'String))" "))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseStructWithEscaping) { - TestOk("Struct<'My\\tName': String, 'My Age': Uint32>", - "(StructType " - "'('\"My\\tName\" (DataType 'String)) " - "'('\"My Age\" (DataType 'Uint32))" - ")"); - } - + TestOk("Struct<'My\\tName': String, 'My Age': Uint32>", + "(StructType " + "'('\"My\\tName\" (DataType 'String)) " + "'('\"My Age\" (DataType 'Uint32))" + ")"); + } + Y_UNIT_TEST(ParseStructComplete) { - TestFail("(Struct<name>)->Uint32", 13, "Expected ':'"); - TestFail("(Struct<name:>)->Uint32", 14, "Expected type"); + TestFail("(Struct<name>)->Uint32", 13, "Expected ':'"); + TestFail("(Struct<name:>)->Uint32", 14, "Expected type"); TestFail("(Struct<name:String,>)->Uint32", 21, "Expected struct member name"); - TestFail("(Struct<name:String)->Uint32", 20, "Expected '>' or ','"); - } - + TestFail("(Struct<name:String)->Uint32", 20, "Expected '>' or ','"); + } + Y_UNIT_TEST(ParseResource) { TestOk("Resource<aaa>", "(ResourceType 'aaa)"); TestOk("(Resource<aaa>?)->Resource<bbb>", "(CallableType '() '((ResourceType 'bbb)) " "'((OptionalType (ResourceType 'aaa)))" - ")"); - } - + ")"); + } + Y_UNIT_TEST(ParseVoid) { TestOk("Void", "(VoidType)"); TestOk("Void?", "(OptionalType (VoidType))"); @@ -510,55 +510,55 @@ Y_UNIT_TEST_SUITE(TTypeString) } void TestFormat(const TString& yql, const TString& expectedTypeStr) { - TMemoryPool pool(4096); - - TAstParseResult astRes = ParseAst(yql, &pool); - if (!astRes.IsOk()) { + TMemoryPool pool(4096); + + TAstParseResult astRes = ParseAst(yql, &pool); + if (!astRes.IsOk()) { astRes.Issues.PrintWithProgramTo(Cerr, "-memory-", yql); - UNIT_FAIL("Can't parse yql"); - } - - TExprContext ctx; + UNIT_FAIL("Can't parse yql"); + } + + TExprContext ctx; const TTypeAnnotationNode* type = CompileTypeAnnotation(*astRes.Root->GetChild(0), ctx); if (!type) { ctx.IssueManager.GetIssues().PrintWithProgramTo(Cerr, "-memory-", yql); - UNIT_FAIL("Can't compile types"); - } - + UNIT_FAIL("Can't compile types"); + } + TString typeStr = FormatType(type); - UNIT_ASSERT_STRINGS_EQUAL(typeStr, expectedTypeStr); - } - + UNIT_ASSERT_STRINGS_EQUAL(typeStr, expectedTypeStr); + } + Y_UNIT_TEST(FormatUnit) { TestFormat("(Unit)", "Unit"); - } - + } + Y_UNIT_TEST(FormatTuple) { - TestFormat("((Tuple " - " (Data Int32) " - " (Data Bool) " - " (Data String)" - "))", - "Tuple<Int32,Bool,String>"); - } - + TestFormat("((Tuple " + " (Data Int32) " + " (Data Bool) " + " (Data String)" + "))", + "Tuple<Int32,Bool,String>"); + } + Y_UNIT_TEST(FormatDataStruct) { - TestFormat("((Struct " - " (Item Name (Data String))" - " (Item Age (Data Uint32))" - " (Item Male (Data Bool))" - "))", - "Struct<'Age':Uint32,'Male':Bool,'Name':String>"); - } - + TestFormat("((Struct " + " (Item Name (Data String))" + " (Item Age (Data Uint32))" + " (Item Male (Data Bool))" + "))", + "Struct<'Age':Uint32,'Male':Bool,'Name':String>"); + } + Y_UNIT_TEST(FormatDecimal) { TestFormat("((Data Decimal 10 3))", "Decimal(10,3)"); } Y_UNIT_TEST(FormatList) { - TestFormat("((List (Data String)))", "List<String>"); - } - + TestFormat("((List (Data String)))", "List<String>"); + } + Y_UNIT_TEST(FormatStream) { TestFormat("((Stream (Data String)))", "Stream<String>"); } @@ -568,10 +568,10 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(FormatOptional) { - TestFormat("((Optional (Data Uint32)))", "Uint32?"); - TestFormat("((List (Optional (Data Uint32))))", "List<Uint32?>"); - } - + TestFormat("((Optional (Data Uint32)))", "Uint32?"); + TestFormat("((List (Optional (Data Uint32))))", "List<Uint32?>"); + } + Y_UNIT_TEST(FormatVariant) { TestFormat("((Variant (Tuple (Data String))))", "Variant<String>"); } @@ -581,13 +581,13 @@ Y_UNIT_TEST_SUITE(TTypeString) } Y_UNIT_TEST(FormatDict) { - TestFormat("((Dict " - " (Data String)" - " (Data Uint32)" - "))", - "Dict<String,Uint32>"); - } - + TestFormat("((Dict " + " (Data String)" + " (Data Uint32)" + "))", + "Dict<String,Uint32>"); + } + Y_UNIT_TEST(FormatSet) { TestFormat("((Dict " " (Data String)" @@ -601,36 +601,36 @@ Y_UNIT_TEST_SUITE(TTypeString) " ((Data String))" " ((Data Uint32))" " ((Optional (Data Uint8)))" - "))", + "))", "Callable<(Uint32,Uint8?)->String>"); TestFormat("((Callable (1) " " ((Data String))" " ((Data Uint32))" " ((Optional (Data Uint8)))" - "))", + "))", "Callable<(Uint32,[Uint8?])->String>"); TestFormat("((Callable (2) " " ((Data String))" " ((Optional (Data Uint32)))" " ((Optional (Data Uint8)))" - "))", + "))", "Callable<([Uint32?,Uint8?])->String>"); - } - + } + Y_UNIT_TEST(FormatOptionalCallable) { TestFormat("((Optional (Callable () " " ((Data String))" " ((Optional (Data Uint8)))" - ")))", + ")))", "Optional<Callable<(Uint8?)->String>>"); - + TestFormat("((Optional (Optional (Callable () " " ((Data String))" " ((Optional (Data Uint8)))" - "))))", + "))))", "Optional<Callable<(Uint8?)->String>>?"); - } - + } + Y_UNIT_TEST(FormatCallableWithNamedArgs) { TestFormat("((Callable () " " ((Data String))" @@ -658,7 +658,7 @@ Y_UNIT_TEST_SUITE(TTypeString) TestFormat("((Resource \"a b\"))", "Resource<'a b'>"); TestFormat("((Resource \"a\\t\\n\\x01b\"))", "Resource<'a\\t\\n\\x01b'>"); TestFormat("((Optional (Resource aaa)))", "Resource<'aaa'>?"); - } + } Y_UNIT_TEST(FormatTagged) { TestFormat("((Tagged (Data String) aaa))", "Tagged<String,'aaa'>"); @@ -666,4 +666,4 @@ Y_UNIT_TEST_SUITE(TTypeString) TestFormat("((Tagged (Data String) \"a\\t\\n\\x01b\"))", "Tagged<String,'a\\t\\n\\x01b'>"); } -} +} diff --git a/ydb/library/yql/minikql/mkql_node_visitor.cpp b/ydb/library/yql/minikql/mkql_node_visitor.cpp index e8543f2f05..5a4ac8e479 100644 --- a/ydb/library/yql/minikql/mkql_node_visitor.cpp +++ b/ydb/library/yql/minikql/mkql_node_visitor.cpp @@ -559,7 +559,7 @@ TRuntimeNode SinglePassVisitCallablesImpl(TRuntimeNode root, TExploringNodeVisit { auto& nodes = explorer.GetNodes(); - wereChanges = false; + wereChanges = false; for (TNode* exploredNode : nodes) { TNode* node; @@ -582,7 +582,7 @@ TRuntimeNode SinglePassVisitCallablesImpl(TRuntimeNode root, TExploringNodeVisit if (result.GetNode() != node) { if (InPlace) { callable.SetResult(result, env); - wereChanges = true; + wereChanges = true; } else { TNode* wrappedResult = TCallable::Create(result, callable.GetType(), env); exploredNode->SetCookie((ui64)wrappedResult); @@ -603,7 +603,7 @@ TRuntimeNode SinglePassVisitCallablesImpl(TRuntimeNode root, TExploringNodeVisit auto newRoot = (TNode*)root.GetNode()->GetCookie(); if (newRoot) { root = TRuntimeNode(newRoot, root.IsImmediate()); - wereChanges = true; + wereChanges = true; } } diff --git a/ydb/library/yql/minikql/mkql_node_visitor.h b/ydb/library/yql/minikql/mkql_node_visitor.h index 2a919c715f..f5bf76d520 100644 --- a/ydb/library/yql/minikql/mkql_node_visitor.h +++ b/ydb/library/yql/minikql/mkql_node_visitor.h @@ -181,7 +181,7 @@ typedef std::function<TRuntimeNode (TCallable& callable, const TTypeEnvironment& typedef std::function<TCallableVisitFunc (const TInternName& name)> TCallableVisitFuncProvider; TRuntimeNode SinglePassVisitCallables(TRuntimeNode root, TExploringNodeVisitor& explorer, - const TCallableVisitFuncProvider& funcProvider, const TTypeEnvironment& env, bool inPlace, bool& wereChanges); + const TCallableVisitFuncProvider& funcProvider, const TTypeEnvironment& env, bool inPlace, bool& wereChanges); } } diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 08603025a5..47ae6e4c8f 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -231,7 +231,7 @@ std::string_view ScriptTypeAsStr(EScriptType type) { switch (type) { #define MKQL_SCRIPT_TYPE_CASE(name, value, ...) \ case EScriptType::name: return std::string_view(#name); - + MKQL_SCRIPT_TYPES(MKQL_SCRIPT_TYPE_CASE) #undef MKQL_SCRIPT_TYPE_CASE @@ -1999,8 +1999,8 @@ TRuntimeNode TProgramBuilder::NewStruct(TType* structType, const TArrayRef<const TRuntimeNode TProgramBuilder::NewEmptyList(TType* itemType) { TListLiteralBuilder builder(Env, itemType); return TRuntimeNode(builder.Build(), true); -} - +} + TRuntimeNode TProgramBuilder::NewList(TType* itemType, const TArrayRef<const TRuntimeNode>& items) { TListLiteralBuilder builder(Env, itemType); for (auto item : items) { diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 08a797c096..10f1ad7ccf 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -32,7 +32,7 @@ enum class EJoinKind { SemiSide = 16, Cross = 32 }; - + enum class EDictItems { Both = 0, Keys = 1, @@ -128,12 +128,12 @@ public: //-- literal functions TRuntimeNode NewVoid(); TRuntimeNode NewNull(); - + TType* NewDataType(NUdf::TDataTypeId schemeType, bool optional = false); TType* NewDataType(NUdf::EDataSlot slot, bool optional = false) { return NewDataType(NUdf::GetDataTypeInfo(slot).TypeId, optional); } - + TType* NewDecimalType(ui8 precision, ui8 scale); template <typename T, typename = std::enable_if_t<NUdf::TKnownDataType<T>::Result>> @@ -154,28 +154,28 @@ public: TRuntimeNode NewDecimalLiteral(NYql::NDecimal::TInt128 data, ui8 precision, ui8 scale) const; - TType* NewOptionalType(TType* itemType); + TType* NewOptionalType(TType* itemType); TRuntimeNode NewEmptyOptional(TType* optionalType); TRuntimeNode NewEmptyOptionalDataLiteral(NUdf::TDataTypeId schemeType); - TRuntimeNode NewOptional(TRuntimeNode data); - TRuntimeNode NewOptional(TType* optionalType, TRuntimeNode data); - + TRuntimeNode NewOptional(TRuntimeNode data); + TRuntimeNode NewOptional(TType* optionalType, TRuntimeNode data); + TType* NewEmptyStructType(); TType* NewStructType(TType* baseStructType, const std::string_view& memberName, TType* memberType); TType* NewStructType(const TArrayRef<const std::pair<std::string_view, TType*>>& memberTypes); TType* NewArrayType(const TArrayRef<const std::pair<std::string_view, TType*>>& memberTypes); - TRuntimeNode NewEmptyStruct(); + TRuntimeNode NewEmptyStruct(); TRuntimeNode NewStruct(const TArrayRef<const std::pair<std::string_view, TRuntimeNode>>& members); TRuntimeNode NewStruct(TType* structType, const TArrayRef<const std::pair<std::string_view, TRuntimeNode>>& members); - + TType* NewListType(TType* itemType); TRuntimeNode NewEmptyList(TType* itemType); TRuntimeNode NewEmptyListOfVoid(); TRuntimeNode NewList(TType* itemType, const TArrayRef<const TRuntimeNode>& items); - + TType* NewDictType(TType* keyType, TType* payloadType, bool multi); TRuntimeNode NewDict(TType* dictType, const TArrayRef<const std::pair<TRuntimeNode, TRuntimeNode>>& items); - + TType* NewStreamType(TType* itemType); TType* NewFlowType(TType* itemType); TType* NewTaggedType(TType* baseType, const std::string_view& tag); diff --git a/ydb/public/lib/base/msgbus.cpp b/ydb/public/lib/base/msgbus.cpp index e924fde0b0..3de1861098 100644 --- a/ydb/public/lib/base/msgbus.cpp +++ b/ydb/public/lib/base/msgbus.cpp @@ -1,9 +1,9 @@ -#include "msgbus.h" +#include "msgbus.h" #include "msgbus_status.h" - -namespace NKikimr { -namespace NMsgBusProxy { - + +namespace NKikimr { +namespace NMsgBusProxy { + void ExplainProposeTransactionStatus(ui32 status, TString& name, TString& description) { name = ToString(status); description = "Unknown status"; @@ -59,5 +59,5 @@ void ExplainResponseStatus(ui32 status, TString& name, TString& description) { } } -} // namespace NKikimr -} // namespace NMsgBusProxy +} // namespace NKikimr +} // namespace NMsgBusProxy diff --git a/ydb/public/lib/base/ya.make b/ydb/public/lib/base/ya.make index ef6cdb5a92..ace9d9f06e 100644 --- a/ydb/public/lib/base/ya.make +++ b/ydb/public/lib/base/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - +LIBRARY() + OWNER( ddoarn g:kikimr ) - -SRCS( - defs.h + +SRCS( + defs.h msgbus_status.h - msgbus.h - msgbus.cpp -) - -PEERDIR( + msgbus.h + msgbus.cpp +) + +PEERDIR( library/cpp/deprecated/enum_codegen library/cpp/messagebus library/cpp/messagebus/protobuf ydb/core/protos -) - -END() +) + +END() diff --git a/ydb/public/lib/deprecated/client/msgbus_client.cpp b/ydb/public/lib/deprecated/client/msgbus_client.cpp index 29ad03a7fa..4d5f2427eb 100644 --- a/ydb/public/lib/deprecated/client/msgbus_client.cpp +++ b/ydb/public/lib/deprecated/client/msgbus_client.cpp @@ -185,12 +185,12 @@ NBus::EMessageStatus TMsgBusClient::AsyncCall(TAutoPtr<NBus::TBusMessage> msg, T return status; } -void TMsgBusClient::OnResult(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status, TAutoPtr<NBus::TBusMessage> pReply) { +void TMsgBusClient::OnResult(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status, TAutoPtr<NBus::TBusMessage> pReply) { static_cast<TMessageCookie*>(pMessage->Data)->Signal(pMessage, status, pReply); } void TMsgBusClient::OnReply(TAutoPtr<NBus::TBusMessage> pMessage, TAutoPtr<NBus::TBusMessage> pReply) { - OnResult(pMessage, NBus::MESSAGE_OK, pReply); + OnResult(pMessage, NBus::MESSAGE_OK, pReply); } void TMsgBusClient::OnError(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status) { diff --git a/ydb/public/lib/deprecated/client/msgbus_client.h b/ydb/public/lib/deprecated/client/msgbus_client.h index 05d424bf28..7d721946d8 100644 --- a/ydb/public/lib/deprecated/client/msgbus_client.h +++ b/ydb/public/lib/deprecated/client/msgbus_client.h @@ -4,7 +4,7 @@ #include <ydb/public/lib/base/defs.h> #include <ydb/public/lib/base/msgbus.h> - + #include <functional> #include <util/system/info.h> #include <library/cpp/messagebus/ybus.h> |