diff options
| author | vmakeev <vmakeev@yandex-team.ru> | 2022-02-10 16:52:18 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:52:18 +0300 |
| commit | b37ebaa2170f7cc5bdd379a79f65260af40b99ee (patch) | |
| tree | f773b7f67c7ca1f04bee6e95fe57231a637358a9 | |
| parent | 260a73f9a01b57317f9a61c7ed906ca443bfd7e1 (diff) | |
| download | ydb-b37ebaa2170f7cc5bdd379a79f65260af40b99ee.tar.gz | |
Restoring authorship annotation for <vmakeev@yandex-team.ru>. Commit 1 of 2.
28 files changed, 1684 insertions, 1684 deletions
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 82f6c5812ac..d5e1bf6e8cf 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -730,8 +730,8 @@ namespace { { auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Error"), pool); const auto& err = annotation.Cast<TErrorExprType>()->GetError(); - auto row = TAstNode::NewAtom(TPosition(), ToString(err.Position.Row), pool); - auto column = TAstNode::NewAtom(TPosition(), ToString(err.Position.Column), pool); + auto row = TAstNode::NewAtom(TPosition(), ToString(err.Position.Row), pool); + auto column = TAstNode::NewAtom(TPosition(), ToString(err.Position.Column), pool); auto file = refAtoms ? TAstNode::NewLiteralAtom(TPosition(), err.Position.File, pool, TNodeFlags::ArbitraryContent) : TAstNode::NewAtom(TPosition(), err.Position.File, pool, TNodeFlags::ArbitraryContent); diff --git a/ydb/library/yql/ast/yql_type_string.cpp b/ydb/library/yql/ast/yql_type_string.cpp index af9af93259d..4ef37ab32a2 100644 --- a/ydb/library/yql/ast/yql_type_string.cpp +++ b/ydb/library/yql/ast/yql_type_string.cpp @@ -1297,7 +1297,7 @@ private: void Visit(const TErrorExprType& type) final { Out_ << TStringBuf("Error<"); - auto pos = type.GetError().Position; + auto pos = type.GetError().Position; EscapeArbitraryAtom(pos.File.empty() ? "<main>" : pos.File, '\'', &Out_); Out_ << ':'; if (pos) { diff --git a/ydb/library/yql/core/yql_execution.h b/ydb/library/yql/core/yql_execution.h index cb38d913fa6..a12f843a15d 100644 --- a/ydb/library/yql/core/yql_execution.h +++ b/ydb/library/yql/core/yql_execution.h @@ -6,28 +6,28 @@ #include <util/system/mutex.h> namespace NYql { - struct TOperationProgress { + struct TOperationProgress { #define YQL_OPERATION_PROGRESS_STATE_MAP(xx) \ - xx(Started, 0) \ - xx(InProgress, 1) \ - xx(Finished, 2) \ - xx(Failed, 3) \ + xx(Started, 0) \ + xx(InProgress, 1) \ + xx(Finished, 2) \ + xx(Failed, 3) \ xx(Aborted, 4) - enum class EState { - YQL_OPERATION_PROGRESS_STATE_MAP(ENUM_VALUE_GEN) - }; + enum class EState { + YQL_OPERATION_PROGRESS_STATE_MAP(ENUM_VALUE_GEN) + }; TString Category; - ui32 Id; - EState State; + ui32 Id; + EState State; using TStage = std::pair<TString, TInstant>; TStage Stage; TString RemoteId; - struct TCounters { + struct TCounters { ui64 Completed = 0ULL; ui64 Running = 0ULL; ui64 Total = 0ULL; @@ -35,32 +35,32 @@ namespace NYql { ui64 Failed = 0ULL; ui64 Lost = 0ULL; ui64 Pending = 0ULL; - bool operator==(const TCounters& rhs) const noexcept { - return Completed == rhs.Completed && - Running == rhs.Running && + bool operator==(const TCounters& rhs) const noexcept { + return Completed == rhs.Completed && + Running == rhs.Running && Total == rhs.Total && Aborted == rhs.Aborted && Failed == rhs.Failed && Lost == rhs.Lost && Pending == rhs.Pending; - } + } - bool operator!=(const TCounters& rhs) const noexcept { - return !operator==(rhs); - } - }; + bool operator!=(const TCounters& rhs) const noexcept { + return !operator==(rhs); + } + }; - TMaybe<TCounters> Counters; + TMaybe<TCounters> Counters; TOperationProgress(const TString& category, ui32 id, EState state, const TString& stage = "") - : Category(category) - , Id(id) - , State(state) + : Category(category) + , Id(id) + , State(state) , Stage(stage, TInstant::Now()) - { - } - }; + { + } + }; struct TOperationStatistics { struct TEntry { @@ -94,7 +94,7 @@ namespace NYql { }; using TStatWriter = std::function<void(ui32, const TVector<TOperationStatistics::TEntry>&)>; - using TOperationProgressWriter = std::function<void(const TOperationProgress&)>; + using TOperationProgressWriter = std::function<void(const TOperationProgress&)>; inline TStatWriter ThreadSafeStatWriter(TStatWriter base) { struct TState : public TThrRefBase { @@ -111,34 +111,34 @@ namespace NYql { }; } - inline void NullProgressWriter(const TOperationProgress& progress) { - Y_UNUSED(progress); - } - - inline TOperationProgressWriter ChainProgressWriters(TOperationProgressWriter left, TOperationProgressWriter right) { - return [=](const TOperationProgress& progress) { - left(progress); - right(progress); - }; - } - - inline TOperationProgressWriter ThreadSafeProgressWriter(TOperationProgressWriter base) { - struct TState : public TThrRefBase { - TOperationProgressWriter Base; - TMutex Mutex; - }; - - auto state = MakeIntrusive<TState>(); - state->Base = base; - return [state](const TOperationProgress& progress) { - with_lock(state->Mutex) { - state->Base(progress); - } - }; - } + inline void NullProgressWriter(const TOperationProgress& progress) { + Y_UNUSED(progress); + } + + inline TOperationProgressWriter ChainProgressWriters(TOperationProgressWriter left, TOperationProgressWriter right) { + return [=](const TOperationProgress& progress) { + left(progress); + right(progress); + }; + } + + inline TOperationProgressWriter ThreadSafeProgressWriter(TOperationProgressWriter base) { + struct TState : public TThrRefBase { + TOperationProgressWriter Base; + TMutex Mutex; + }; + + auto state = MakeIntrusive<TState>(); + state->Base = base; + return [state](const TOperationProgress& progress) { + with_lock(state->Mutex) { + state->Base(progress); + } + }; + } TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotationContext& types, bool checkWorld = true); TAutoPtr<IGraphTransformer> CreateExecutionTransformer(TTypeAnnotationContext& types, TOperationProgressWriter writer, bool withFinalize = true); - IGraphTransformer::TStatus RequireChild(const TExprNode& node, ui32 index); + IGraphTransformer::TStatus RequireChild(const TExprNode& node, ui32 index); } diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_list.h b/ydb/library/yql/minikql/computation/mkql_computation_node_list.h index d87fe871bf0..2020c050adc 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_list.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_list.h @@ -7,430 +7,430 @@ namespace NKikimr { - namespace NMiniKQL { + namespace NMiniKQL { template <typename T> - struct TListChunk { - private: + struct TListChunk { + private: using TSelf = TListChunk<T>; - static void PlacementNew(T* ptr, ui64 count) { - T* end = ptr + count; - for (; ptr != end; ptr++) { - new (ptr) T; - } + static void PlacementNew(T* ptr, ui64 count) { + T* end = ptr + count; + for (; ptr != end; ptr++) { + new (ptr) T; + } } - public: + public: - void CheckMagic() { - Y_VERIFY_DEBUG(Magic_ == TListChunkMagic); + void CheckMagic() { + Y_VERIFY_DEBUG(Magic_ == TListChunkMagic); } - static TSelf* AllocateChunk(ui64 length) { + static TSelf* AllocateChunk(ui64 length) { const auto block = MKQLAllocWithSize(length * sizeof(T) + sizeof(TListChunk)); const auto ptr = new (block) TListChunk(length); - PlacementNew(reinterpret_cast<T*>(ptr + 1), length); - return ptr; + PlacementNew(reinterpret_cast<T*>(ptr + 1), length); + return ptr; } - TListChunk(ui64 size) - : Magic_(TListChunkMagic) - , Refs_(1) - , DataEnd_(DataBegin() + size) - { + TListChunk(ui64 size) + : Magic_(TListChunkMagic) + , Refs_(1) + , DataEnd_(DataBegin() + size) + { } - ~TListChunk() { - CheckMagic(); - for (auto it = DataBegin(); it != DataEnd(); it++) { - it->~T(); - } + ~TListChunk() { + CheckMagic(); + for (auto it = DataBegin(); it != DataEnd(); it++) { + it->~T(); + } MKQLFreeWithSize(static_cast<void*>(this), sizeof(TListChunk) + sizeof(T) * (DataEnd() - DataBegin())); } - inline T* DataBegin() { - CheckMagic(); - return reinterpret_cast<T*>(this + 1); + inline T* DataBegin() { + CheckMagic(); + return reinterpret_cast<T*>(this + 1); } - inline const T* DataEnd() { - CheckMagic(); - return DataEnd_; + inline const T* DataEnd() { + CheckMagic(); + return DataEnd_; } - ui64 Size() { - CheckMagic(); - return DataEnd() - DataBegin(); + ui64 Size() { + CheckMagic(); + return DataEnd() - DataBegin(); } - void Ref() { - CheckMagic(); - Refs_++; + void Ref() { + CheckMagic(); + Refs_++; } - void UnRef() { - CheckMagic(); - if (--Refs_ == 0) { - this->~TListChunk(); - } + void UnRef() { + CheckMagic(); + if (--Refs_ == 0) { + this->~TListChunk(); + } } - private: - static const ui32 TListChunkMagic = 12322846; - const ui32 Magic_; - ui32 Refs_; - const T* DataEnd_; + private: + static const ui32 TListChunkMagic = 12322846; + const ui32 Magic_; + ui32 Refs_; + const T* DataEnd_; - }; + }; template <typename T, ui64 MinChunkSize = 48> - class TListRepresentation { - public: + class TListRepresentation { + public: using TSelf = TListRepresentation<T, MinChunkSize>; using TChunk = TListChunk<T>; - private: - enum Type { - Normal, - Freezed - }; - - void NewNormal(const T* begin1, const T* end1, const T* begin2, const T* end2) { - Type_ = Type::Normal; - ui64 oldLength = (end1 - begin1) + (end2 - begin2); - ui64 newLength = std::max((oldLength << 1) - 1, (MinChunkSize + sizeof(T) - 1) / sizeof(T) + 1); - Chunk_ = TChunk::AllocateChunk(newLength); - Begin_ = Chunk_->DataBegin() + ((newLength - oldLength) >> 2); - End_ = std::copy(begin2, end2, std::copy(begin1, end1, Begin_)); - } - - TListRepresentation(TChunk* chunk, T* begin, T* end, Type type) - : Chunk_(chunk) - , Begin_(begin) - , End_(end) - , Type_(type) - { - if (Chunk_) { - Chunk_->Ref(); - } - } - - TListRepresentation(T* begin1, T* end1, T* begin2, T* end2) - { - NewNormal(begin1, end1, begin2, end2); - } - - public: - - struct TIterator { - TIterator() - : Owner_(nullptr) - , Position_(nullptr) - {} - - TIterator(const TListRepresentation& owner) - : Owner_(&owner) - , Position_(owner.Begin_) - { - } - - TIterator(const TIterator& other) - : Owner_(other.Owner_) - , Position_(other.Position_) - {} - - TIterator& operator=(const TIterator& other) - { - Owner_ = other.Owner_; - Position_ = other.Position_; - return *this; - } - - bool AtEnd() const { - return Position_ == Owner_->End_; - } - - const T& Current() const { - return *Position_; - } - - // use with care, list may be shared - T& MutableCurrent() { - return *Position_; - } - - void Next() { - Position_++; - } - - const TListRepresentation* Owner_; - T* Position_; - }; - - struct TReverseIterator { - TReverseIterator() - : Owner_(nullptr) - , Position_(nullptr) - { - } - - TReverseIterator(const TListRepresentation& owner) - : Owner_(&owner) - , Position_(owner.End_) - { - } - - TReverseIterator(const TIterator& other) - : Owner_(other.Owner_) - , Position_(other.Position_) - { - } - - TReverseIterator& operator=(const TReverseIterator& other) - { - Owner_ = other.Owner_; - Position_ = other.Position_; - return *this; - } - - bool AtEnd() const { - return Position_ == Owner_->Begin_; - } - - const T& Current() const { - return *(Position_ - 1); - } - - // use with care, list may be shared - T& MutableCurrent() { - return *(Position_ - 1); - } - - void Next() { - Position_--; - } - - private: - const TListRepresentation* Owner_; - T* Position_; - }; - - TListRepresentation() - : Chunk_(nullptr) - , Begin_(nullptr) - , End_(nullptr) - , Type_(Type::Freezed) - { - } - - ~TListRepresentation() { - if (Chunk_) { - Chunk_->UnRef(); - } - } - - TListRepresentation(const TSelf& other) - : Chunk_(other.Chunk_) - , Begin_(other.Begin_) - , End_(other.End_) - , Type_(other.Type_) - { - other.Type_ = Type::Freezed; - if (Chunk_) { - Chunk_->Ref(); - } - } - - TListRepresentation(TSelf&& other) - : Chunk_(other.Chunk_) - , Begin_(other.Begin_) - , End_(other.End_) - , Type_(other.Type_) - { - other.Chunk_ = nullptr; - other.Begin_ = nullptr; - other.End_ = nullptr; - other.Type_ = Type::Freezed; - } - - void operator=(const TSelf& other) { - if (this != &other) { - - if (other.Chunk_) { - other.Chunk_->Ref(); - } - - if (Chunk_) { - Chunk_->UnRef(); - } - - Chunk_ = other.Chunk_; - Begin_ = other.Begin_; - End_ = other.End_; - Type_ = other.Type_; - - other.Type_ = Type::Freezed; - } - } - - void operator=(TSelf&& other) { - if (Chunk_) { - Chunk_->UnRef(); - } - - Chunk_ = other.Chunk_; - Begin_ = other.Begin_; - End_ = other.End_; - Type_ = other.Type_; - - other.Chunk_ = nullptr; - other.Begin_ = nullptr; - other.End_ = nullptr; - other.Type_ = Type::Freezed; - } + private: + enum Type { + Normal, + Freezed + }; + + void NewNormal(const T* begin1, const T* end1, const T* begin2, const T* end2) { + Type_ = Type::Normal; + ui64 oldLength = (end1 - begin1) + (end2 - begin2); + ui64 newLength = std::max((oldLength << 1) - 1, (MinChunkSize + sizeof(T) - 1) / sizeof(T) + 1); + Chunk_ = TChunk::AllocateChunk(newLength); + Begin_ = Chunk_->DataBegin() + ((newLength - oldLength) >> 2); + End_ = std::copy(begin2, end2, std::copy(begin1, end1, Begin_)); + } + + TListRepresentation(TChunk* chunk, T* begin, T* end, Type type) + : Chunk_(chunk) + , Begin_(begin) + , End_(end) + , Type_(type) + { + if (Chunk_) { + Chunk_->Ref(); + } + } + + TListRepresentation(T* begin1, T* end1, T* begin2, T* end2) + { + NewNormal(begin1, end1, begin2, end2); + } + + public: + + struct TIterator { + TIterator() + : Owner_(nullptr) + , Position_(nullptr) + {} + + TIterator(const TListRepresentation& owner) + : Owner_(&owner) + , Position_(owner.Begin_) + { + } + + TIterator(const TIterator& other) + : Owner_(other.Owner_) + , Position_(other.Position_) + {} + + TIterator& operator=(const TIterator& other) + { + Owner_ = other.Owner_; + Position_ = other.Position_; + return *this; + } + + bool AtEnd() const { + return Position_ == Owner_->End_; + } + + const T& Current() const { + return *Position_; + } + + // use with care, list may be shared + T& MutableCurrent() { + return *Position_; + } + + void Next() { + Position_++; + } + + const TListRepresentation* Owner_; + T* Position_; + }; + + struct TReverseIterator { + TReverseIterator() + : Owner_(nullptr) + , Position_(nullptr) + { + } + + TReverseIterator(const TListRepresentation& owner) + : Owner_(&owner) + , Position_(owner.End_) + { + } + + TReverseIterator(const TIterator& other) + : Owner_(other.Owner_) + , Position_(other.Position_) + { + } + + TReverseIterator& operator=(const TReverseIterator& other) + { + Owner_ = other.Owner_; + Position_ = other.Position_; + return *this; + } + + bool AtEnd() const { + return Position_ == Owner_->Begin_; + } + + const T& Current() const { + return *(Position_ - 1); + } + + // use with care, list may be shared + T& MutableCurrent() { + return *(Position_ - 1); + } + + void Next() { + Position_--; + } + + private: + const TListRepresentation* Owner_; + T* Position_; + }; + + TListRepresentation() + : Chunk_(nullptr) + , Begin_(nullptr) + , End_(nullptr) + , Type_(Type::Freezed) + { + } + + ~TListRepresentation() { + if (Chunk_) { + Chunk_->UnRef(); + } + } + + TListRepresentation(const TSelf& other) + : Chunk_(other.Chunk_) + , Begin_(other.Begin_) + , End_(other.End_) + , Type_(other.Type_) + { + other.Type_ = Type::Freezed; + if (Chunk_) { + Chunk_->Ref(); + } + } + + TListRepresentation(TSelf&& other) + : Chunk_(other.Chunk_) + , Begin_(other.Begin_) + , End_(other.End_) + , Type_(other.Type_) + { + other.Chunk_ = nullptr; + other.Begin_ = nullptr; + other.End_ = nullptr; + other.Type_ = Type::Freezed; + } + + void operator=(const TSelf& other) { + if (this != &other) { + + if (other.Chunk_) { + other.Chunk_->Ref(); + } + + if (Chunk_) { + Chunk_->UnRef(); + } + + Chunk_ = other.Chunk_; + Begin_ = other.Begin_; + End_ = other.End_; + Type_ = other.Type_; + + other.Type_ = Type::Freezed; + } + } + + void operator=(TSelf&& other) { + if (Chunk_) { + Chunk_->UnRef(); + } + + Chunk_ = other.Chunk_; + Begin_ = other.Begin_; + End_ = other.End_; + Type_ = other.Type_; + + other.Chunk_ = nullptr; + other.Begin_ = nullptr; + other.End_ = nullptr; + other.Type_ = Type::Freezed; + } inline void FromSingleElement(T&& element) { - Type_ = Type::Normal; - ui64 chunkLength = (MinChunkSize + sizeof(T) - 1) / sizeof(T); - Chunk_ = TChunk::AllocateChunk(chunkLength); - Begin_ = Chunk_->DataBegin() + (chunkLength >> 2); - End_ = Begin_ + 1; + Type_ = Type::Normal; + ui64 chunkLength = (MinChunkSize + sizeof(T) - 1) / sizeof(T); + Chunk_ = TChunk::AllocateChunk(chunkLength); + Begin_ = Chunk_->DataBegin() + (chunkLength >> 2); + End_ = Begin_ + 1; *Begin_ = std::move(element); - } + } TListRepresentation(T&& element) - { + { FromSingleElement(std::move(element)); } TListRepresentation(T&& element, const TSelf& that) - { - if (!that.Chunk_) { + { + if (!that.Chunk_) { FromSingleElement(std::move(element)); - return; - } - if ((that.Type_ == Type::Normal) && (that.Begin_ != that.Chunk_->DataBegin())) { - Type_ = Type::Normal; - that.Type_ = Type::Freezed; - Chunk_ = that.Chunk_; - Chunk_->Ref(); - Begin_ = that.Begin_; - End_ = that.End_; + return; + } + if ((that.Type_ == Type::Normal) && (that.Begin_ != that.Chunk_->DataBegin())) { + Type_ = Type::Normal; + that.Type_ = Type::Freezed; + Chunk_ = that.Chunk_; + Chunk_->Ref(); + Begin_ = that.Begin_; + End_ = that.End_; *(--Begin_) = std::move(element); - } else { - NewNormal(&element, &element + 1, that.Begin_, that.End_); - } - } + } else { + NewNormal(&element, &element + 1, that.Begin_, that.End_); + } + } TListRepresentation(const TSelf& that, T&& element) - { - if (!that.Chunk_) { + { + if (!that.Chunk_) { FromSingleElement(std::move(element)); - return; + return; } - if ((that.Type_ == Type::Normal) && (that.End_ != that.Chunk_->DataEnd())) { - Type_ = Type::Normal; - that.Type_ = Type::Freezed; - Chunk_ = that.Chunk_; - Chunk_->Ref(); - Begin_ = that.Begin_; - End_ = that.End_ + 1; + if ((that.Type_ == Type::Normal) && (that.End_ != that.Chunk_->DataEnd())) { + Type_ = Type::Normal; + that.Type_ = Type::Freezed; + Chunk_ = that.Chunk_; + Chunk_->Ref(); + Begin_ = that.Begin_; + End_ = that.End_ + 1; *(that.End_) = std::move(element); - } else { - NewNormal(that.Begin_, that.End_, &element, &element + 1); - } - } + } else { + NewNormal(that.Begin_, that.End_, &element, &element + 1); + } + } - ui64 GetLength() const { - return End_ - Begin_; + ui64 GetLength() const { + return End_ - Begin_; } - TIterator GetIterator() const { - return TIterator(*this); - } + TIterator GetIterator() const { + return TIterator(*this); + } - TReverseIterator GetReverseIterator() const { - return TReverseIterator(*this); - } + TReverseIterator GetReverseIterator() const { + return TReverseIterator(*this); + } TSelf Append(T&& right) const { return TSelf(*this, std::move(right)); - } + } TSelf Prepend(T&& left) const { return TSelf(std::move(left), *this); - } - - TSelf MassPrepend(T* begin, T* end) const { - if ((Type_ == Type::Normal) && (Chunk_->DataBegin() + (end - begin) <= Begin_)) { - Type_ = Type::Freezed; - return TSelf(Chunk_, std::copy_backward(begin, end, Begin_), End_, Type::Normal); - } else { - return TSelf(begin, end, Begin_, End_); - } - } - - TSelf MassAppend(T* begin, T* end) const { - if ((Type_ == Type::Normal) && (End_ + (end - begin) <= Chunk_->DataEnd())) { - Type_ = Type::Freezed; - return TSelf(Chunk_, Begin_, std::copy(begin, end, End_), Type::Normal); - } else { - return TSelf(Begin_, End_, begin, end); - } - } - - TSelf Extend(const TSelf& right) const { - ui64 thisLength = GetLength(); - ui64 rightLength = right.GetLength(); - - if (!thisLength) - return TSelf(right); - - if (!rightLength) - return TSelf(*this); - - if (Type_ == Type::Freezed) { - if (right.Type_ == Type::Freezed) { - return TSelf(Begin_, End_, right.Begin_, right.End_); - } else { - return right.MassPrepend(Begin_, End_); - } - } else if ((right.Type_ == Type::Freezed) || (thisLength > rightLength)) { - return MassAppend(right.Begin_, right.End_); - } else { - return right.MassPrepend(Begin_, End_); - } - } - - TSelf SkipFromBegin(ui64 count) const { - Y_VERIFY_DEBUG((count > 0) && (count < GetLength())); - return TSelf(Chunk_, Begin_ + count, End_, Type::Freezed); - } - - TSelf SkipFromEnd(ui64 count) const { - Y_VERIFY_DEBUG((count > 0) && (count < GetLength())); - return TSelf(Chunk_, Begin_, End_ - count, Type::Freezed); - } - - T GetItemByIndex(ui64 index) const { - Y_VERIFY_DEBUG((index >= 0) && (index < GetLength())); - return Begin_[index]; - } + } + + TSelf MassPrepend(T* begin, T* end) const { + if ((Type_ == Type::Normal) && (Chunk_->DataBegin() + (end - begin) <= Begin_)) { + Type_ = Type::Freezed; + return TSelf(Chunk_, std::copy_backward(begin, end, Begin_), End_, Type::Normal); + } else { + return TSelf(begin, end, Begin_, End_); + } + } + + TSelf MassAppend(T* begin, T* end) const { + if ((Type_ == Type::Normal) && (End_ + (end - begin) <= Chunk_->DataEnd())) { + Type_ = Type::Freezed; + return TSelf(Chunk_, Begin_, std::copy(begin, end, End_), Type::Normal); + } else { + return TSelf(Begin_, End_, begin, end); + } + } + + TSelf Extend(const TSelf& right) const { + ui64 thisLength = GetLength(); + ui64 rightLength = right.GetLength(); + + if (!thisLength) + return TSelf(right); + + if (!rightLength) + return TSelf(*this); + + if (Type_ == Type::Freezed) { + if (right.Type_ == Type::Freezed) { + return TSelf(Begin_, End_, right.Begin_, right.End_); + } else { + return right.MassPrepend(Begin_, End_); + } + } else if ((right.Type_ == Type::Freezed) || (thisLength > rightLength)) { + return MassAppend(right.Begin_, right.End_); + } else { + return right.MassPrepend(Begin_, End_); + } + } + + TSelf SkipFromBegin(ui64 count) const { + Y_VERIFY_DEBUG((count > 0) && (count < GetLength())); + return TSelf(Chunk_, Begin_ + count, End_, Type::Freezed); + } + + TSelf SkipFromEnd(ui64 count) const { + Y_VERIFY_DEBUG((count > 0) && (count < GetLength())); + return TSelf(Chunk_, Begin_, End_ - count, Type::Freezed); + } + + T GetItemByIndex(ui64 index) const { + Y_VERIFY_DEBUG((index >= 0) && (index < GetLength())); + return Begin_[index]; + } T* GetItems() const { return Begin_; } - private: - TChunk* Chunk_; - T* Begin_; - T* End_; - mutable Type Type_; + private: + TChunk* Chunk_; + T* Begin_; + T* End_; + mutable Type Type_; - }; + }; - using TDefaultListRepresentation = TListRepresentation<NUdf::TUnboxedValue>; + using TDefaultListRepresentation = TListRepresentation<NUdf::TUnboxedValue>; - } + } } diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 47ae6e4c8fc..6dbc80be2b4 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -2056,14 +2056,14 @@ TRuntimeNode TProgramBuilder::NewDict(TType* dictType, const TArrayRef<const std return TRuntimeNode(TDictLiteral::Create(items.size(), items.data(), static_cast<TDictType*>(dictType), Env), true); } -TRuntimeNode TProgramBuilder::NewEmptyTuple() { - return TRuntimeNode(Env.GetEmptyTuple(), true); -} - -TType* TProgramBuilder::NewEmptyTupleType() { - return Env.GetEmptyTuple()->GetGenericType(); -} - +TRuntimeNode TProgramBuilder::NewEmptyTuple() { + return TRuntimeNode(Env.GetEmptyTuple(), true); +} + +TType* TProgramBuilder::NewEmptyTupleType() { + return Env.GetEmptyTuple()->GetGenericType(); +} + TType* TProgramBuilder::NewTupleType(const TArrayRef<TType* const>& elements) { return TTupleType::Create(elements.size(), elements.data(), Env); } diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 10f1ad7ccf1..8c5710e02de 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -181,10 +181,10 @@ public: TType* NewTaggedType(TType* baseType, const std::string_view& tag); TType* NewBlockType(TType* itemType, TBlockType::EShape shape); - TType* NewEmptyTupleType(); + TType* NewEmptyTupleType(); TType* NewTupleType(const TArrayRef<TType* const>& elements); TType* NewArrayType(const TArrayRef<TType* const>& elements); - TRuntimeNode NewEmptyTuple(); + TRuntimeNode NewEmptyTuple(); TRuntimeNode NewTuple(TType* tupleType, const TArrayRef<const TRuntimeNode>& elements); TRuntimeNode NewTuple(const TArrayRef<const TRuntimeNode>& elements); TType* NewResourceType(const std::string_view& tag); diff --git a/ydb/library/yql/mount/lib/yql/aggregate.yql b/ydb/library/yql/mount/lib/yql/aggregate.yql index a1c231f3392..3f8e1185933 100644 --- a/ydb/library/yql/mount/lib/yql/aggregate.yql +++ b/ydb/library/yql/mount/lib/yql/aggregate.yql @@ -163,7 +163,7 @@ (let variance_0_1_traits_factory_raw (lambda '(list_type) (Apply variance_traits_factory_raw list_type (Bool '0) (Bool '1)))) (let variance_1_1_traits_factory_raw (lambda '(list_type) (Apply variance_traits_factory_raw list_type (Bool '1) (Bool '1)))) -# list_type:type +# list_type:type # doesn't support optional values (let correlation_traits_factory_raw (lambda '(list_type) (block '( (let init (lambda '(value) '((Uint64 '1) (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)) (* (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0))) (* (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0))) (* (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)))))) @@ -176,9 +176,9 @@ (let dividend (* (Nth state '3) (Nth state '0))) (let dividend (- dividend (* (Nth state '1) (Nth state '2)))) (let divisor1 (* (Nth state '4) (Nth state '0))) - (let divisor1 (- divisor1 (* (Nth state '1) (Nth state '1)))) + (let divisor1 (- divisor1 (* (Nth state '1) (Nth state '1)))) (let divisor2 (* (Nth state '5) (Nth state '0))) - (let divisor2 (- divisor2 (* (Nth state '2) (Nth state '2)))) + (let divisor2 (- divisor2 (* (Nth state '2) (Nth state '2)))) (return (/ dividend (Apply (Udf 'Math.Sqrt) (* divisor1 divisor2)))) )))) (return (AggregationTraits (ListItemType list_type) init update save load merge finish (Null))) @@ -205,7 +205,7 @@ (let covariance_population_traits_factory_raw (lambda '(list_type) (Apply covariance_traits_factory_raw list_type (Bool '0)))) (let covariance_sample_traits_factory_raw (lambda '(list_type) (Apply covariance_traits_factory_raw list_type (Bool '1)))) -# list_type:type histogram:atom value:lambda weight:lambda intervals:integer +# list_type:type histogram:atom value:lambda weight:lambda intervals:integer # doesn't support optional values (let histogram_traits_factory_raw (lambda '(list_type histogram value weight intervals) (block '( (let init (lambda '(row parent) (NamedApply (Udf (Combine histogram '_Create)) '((Apply value row) (Apply weight row) intervals) (AsStruct) (DependsOn parent)))) @@ -243,10 +243,10 @@ (return (AggregationTraits value_type init update save load merge finish (EmptyList))) )))) -# list_type:type n:ui32 buffer:ui32 -# doesn't support optional values -(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( - (let value_type (ListItemType list_type)) +# list_type:type n:ui32 buffer:ui32 +# doesn't support optional values +(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( + (let value_type (ListItemType list_type)) (let vector_element_type (TupleType (DataType 'Uint64) value_type)) (let serialized_type (TupleType (DataType 'Uint32) (DataType 'Uint32) (ListType vector_element_type))) @@ -260,16 +260,16 @@ (let UdfTopFreqDeserialize (Udf 'TopFreq.TopFreq_Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type))) (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) - (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) - (let save (lambda '(state) (Apply UdfTopFreqSerialize state))) - (let load (lambda '(state) (Apply UdfTopFreqDeserialize state))) - (let merge (lambda '(one two) (Apply UdfTopFreqMerge one two))) - (let finish (lambda '(state) (Apply UdfTopFreqGet state n))) + (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let save (lambda '(state) (Apply UdfTopFreqSerialize state))) + (let load (lambda '(state) (Apply UdfTopFreqDeserialize state))) + (let merge (lambda '(one two) (Apply UdfTopFreqMerge one two))) + (let finish (lambda '(state) (Apply UdfTopFreqGet state n))) (return (AggregationTraits value_type init update save load merge finish (EmptyList))) -)))) - +)))) + # list_type:type extractor:lambda count:ui32 is_top:atom # doesn't support optional values (let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '( @@ -576,16 +576,16 @@ # list_type:type n:double # support optional values -(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) +(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) # list_type:type count:ui32 # support optional values (let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count))))) -# list_type:type n:ui32 buffer:ui32 -# support optional values +# list_type:type n:ui32 buffer:ui32 +# support optional values (let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer))))) - + # list_type:type # support optional values (let hyperloglog_traits_factory_opt (lambda '(list_type precision) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply hyperloglog_traits_factory_raw list_type precision))))) @@ -653,16 +653,16 @@ # list_type:type extractor:lambda n:double # support optional values -(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) +(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) # list_type:type extractor:lambda count:ui32 # support optional values (let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) -# list_type:type extractor:lambda n:ui32 buffer:ui32 -# support optional values -(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) - +# list_type:type extractor:lambda n:ui32 buffer:ui32 +# support optional values +(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) + # list_type:type extractor:lambda precision:ui32 # support optional values (let hyperloglog_traits_factory (lambda '(list_type extractor precision) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply hyperloglog_traits_factory_opt list_type precision))))) @@ -851,21 +851,21 @@ # deprecated (let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) (let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) - + (let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void)))) (let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void)))) (let min_by_traits_factory2 (lambda '(list_type first second limit) (Apply list_compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second limit))) (let max_by_traits_factory2 (lambda '(list_type first second limit) (Apply list_compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second limit))) -(let avg_if_traits_factory (lambda '(list_type payload predicate) -(Apply avg_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( - (let pl (Apply payload value)) - (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) -(let sum_if_traits_factory (lambda '(list_type payload predicate) -(Apply sum_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( - (let pl (Apply payload value)) - (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) +(let avg_if_traits_factory (lambda '(list_type payload predicate) +(Apply avg_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( + (let pl (Apply payload value)) + (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) +(let sum_if_traits_factory (lambda '(list_type payload predicate) +(Apply sum_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( + (let pl (Apply payload value)) + (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) (let top_bottom_traits_factory (lambda '(list_type extractor count is_top) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) @@ -891,7 +891,7 @@ (export min_traits_factory) (export max_traits_factory) (export sum_traits_factory) -(export sum_if_traits_factory) +(export sum_if_traits_factory) (export count_traits_factory) (export count_all_traits_factory) (export count_if_traits_factory) @@ -903,7 +903,7 @@ (export or_traits_factory) (export xor_traits_factory) (export avg_traits_factory) -(export avg_if_traits_factory) +(export avg_if_traits_factory) (export list_traits_factory) (export list2_traits_factory) (export min_by_traits_factory) @@ -929,7 +929,7 @@ (export udaf_traits_factory) (export percentile_traits_factory) (export set_traits_factory) -(export topfreq_traits_factory) +(export topfreq_traits_factory) (export hyperloglog_traits_factory) (export top_traits_factory) (export bottom_traits_factory) diff --git a/ydb/library/yql/mount/lib/yql/window.yql b/ydb/library/yql/mount/lib/yql/window.yql index 371f4c687c8..e013144a058 100644 --- a/ydb/library/yql/mount/lib/yql/window.yql +++ b/ydb/library/yql/mount/lib/yql/window.yql @@ -174,7 +174,7 @@ (let variance_1_1_traits_factory_raw (lambda '(list_type) (Apply variance_traits_factory_raw list_type (Bool '1) (Bool '1)))) -# list_type:type +# list_type:type # doesn't support optional values (let correlation_traits_factory_raw (lambda '(list_type) (block '( (let init (lambda '(value) '((Uint64 '1) (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)) (* (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0))) (* (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '0) 'Double) (Double '0.0))) (* (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)) (Coalesce (Convert (Nth value '1) 'Double) (Double '0.0)))))) @@ -185,9 +185,9 @@ (let dividend (* (Nth state '3) (Nth state '0))) (let dividend (- dividend (* (Nth state '1) (Nth state '2)))) (let divisor1 (* (Nth state '4) (Nth state '0))) - (let divisor1 (- divisor1 (* (Nth state '1) (Nth state '1)))) + (let divisor1 (- divisor1 (* (Nth state '1) (Nth state '1)))) (let divisor2 (* (Nth state '5) (Nth state '0))) - (let divisor2 (- divisor2 (* (Nth state '2) (Nth state '2)))) + (let divisor2 (- divisor2 (* (Nth state '2) (Nth state '2)))) (return (/ dividend (Apply (Udf 'Math.Sqrt) (* divisor1 divisor2)))) )))) (return (WindowTraits (ListItemType list_type) init update shift current (Null))) @@ -222,7 +222,7 @@ (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) -# list_type:type n:ui32 buffer:ui32 +# list_type:type n:ui32 buffer:ui32 # doesn't support optional values (let set_traits_factory_raw (lambda '(list_type count) (block '( (let value_type (ListItemType list_type)) @@ -244,8 +244,8 @@ # list_type:type n:ui32 buffer:ui32 # doesn't support optional values -(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( - (let value_type (ListItemType list_type)) +(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '( + (let value_type (ListItemType list_type)) (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type))) @@ -254,10 +254,10 @@ (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type))) (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type))) - (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) - (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) + (let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent)))) + (let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent)))) (let shift (lambda '(value state) (Void))) - (let current (lambda '(state) (Apply UdfTopFreqGet state n))) + (let current (lambda '(state) (Apply UdfTopFreqGet state n))) (return (WindowTraits value_type init update shift current (EmptyList))) )))) @@ -360,7 +360,7 @@ (let current (lambda '(state) (Apply (Udf (Combine histogram '_GetResult)) state))) (return (WindowTraits (ListItemType list_type) init update shift current (Null))) )))) - + # list_type:type value:lambda binsize:double minimum:double maximum:double # doesn't support optional values (let histogram_linear_traits_factory_raw (lambda '(list_type value binsize minimum maximum) (Apply linear_histogram_traits_factory_impl list_type 'Histogram.LinearHistogram value binsize minimum maximum))) @@ -548,16 +548,16 @@ # list_type:type n:double # support optional values -(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) +(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n))))) # list_type:type count:ui32 # support optional values (let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count))))) -# list_type:type n:ui32 buffer:ui32 -# support optional values +# list_type:type n:ui32 buffer:ui32 +# support optional values (let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer))))) - + # list_type:type # support optional values (let hyperloglog_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply hyperloglog_traits_factory_raw list_type n))))) @@ -628,16 +628,16 @@ # list_type:type extractor:lambda n:double # support optional values -(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) +(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n))))) # list_type:type extractor:lambda count:ui32 # support optional values (let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count))))) -# list_type:type extractor:lambda n:ui32 buffer:ui32 -# support optional values -(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) - +# list_type:type extractor:lambda n:ui32 buffer:ui32 +# support optional values +(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer))))) + # list_type:type extractor:lambda # support optional values (let hyperloglog_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply hyperloglog_traits_factory_opt list_type n))))) @@ -810,21 +810,21 @@ #deprecated (let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit))) (let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit))) - + (let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void)))) (let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void)))) (let min_by_traits_factory2 (lambda '(list_type first second limit) (Apply list_compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second limit))) (let max_by_traits_factory2 (lambda '(list_type first second limit) (Apply list_compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second limit))) -(let avg_if_traits_factory (lambda '(list_type payload predicate) -(Apply avg_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( - (let pl (Apply payload value)) - (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) -(let sum_if_traits_factory (lambda '(list_type payload predicate) -(Apply sum_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( - (let pl (Apply payload value)) - (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) +(let avg_if_traits_factory (lambda '(list_type payload predicate) +(Apply avg_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( + (let pl (Apply payload value)) + (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) +(let sum_if_traits_factory (lambda '(list_type payload predicate) +(Apply sum_traits_factory list_type (lambda '(value) (FlatOptionalIf (Coalesce (Apply predicate value) (Bool '0)) (block '( + (let pl (Apply payload value)) + (return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl))))))))))) # list_type:type init:lambda update:lambda finish:lambda # support optional values @@ -856,7 +856,7 @@ (export min_traits_factory) (export max_traits_factory) (export sum_traits_factory) -(export sum_if_traits_factory) +(export sum_if_traits_factory) (export count_traits_factory) (export count_all_traits_factory) (export count_if_traits_factory) @@ -868,7 +868,7 @@ (export or_traits_factory) (export xor_traits_factory) (export avg_traits_factory) -(export avg_if_traits_factory) +(export avg_if_traits_factory) (export list_traits_factory) (export list2_traits_factory) (export min_by_traits_factory) @@ -894,7 +894,7 @@ (export udwf_traits_factory) (export percentile_traits_factory) (export set_traits_factory) -(export topfreq_traits_factory) +(export topfreq_traits_factory) (export hyperloglog_traits_factory) (export rank_traits_factory) (export dense_rank_traits_factory) diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 8561fe25368..9e106b1847e 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -165,12 +165,12 @@ message TYdbGatewayConfig { ///////////////////////////// ClickHouse ///////////////////////////// -message TClickHouseClusterConfig { - optional string Name = 1; - optional bool Default = 2 [default = false]; - optional string Cluster = 3; - optional string CHToken = 4; - repeated TAttr Settings = 101; +message TClickHouseClusterConfig { + optional string Name = 1; + optional bool Default = 2 [default = false]; + optional string Cluster = 3; + optional string CHToken = 4; + repeated TAttr Settings = 101; optional EHostScheme HostScheme = 5; optional uint32 HostPort = 6; optional string CHTokenYavSecretId = 7; @@ -180,13 +180,13 @@ message TClickHouseClusterConfig { optional uint32 NativeHostPort = 11 [default = 9000]; optional bool NativeSecure = 12 [default = false]; optional string Id = 13; -} - -message TClickHouseGatewayConfig { - repeated TClickHouseClusterConfig ClusterMapping = 1; - repeated TAttr DefaultSettings = 2; -} - +} + +message TClickHouseGatewayConfig { + repeated TClickHouseClusterConfig ClusterMapping = 1; + repeated TAttr DefaultSettings = 2; +} + ///////////////////////////// RTMR ///////////////////////////// enum EYfArtifactType { AT_NONE = 0; diff --git a/ydb/library/yql/sql/v0/aggregation.cpp b/ydb/library/yql/sql/v0/aggregation.cpp index 64406048811..76797590101 100644 --- a/ydb/library/yql/sql/v0/aggregation.cpp +++ b/ydb/library/yql/sql/v0/aggregation.cpp @@ -13,8 +13,8 @@ using namespace NYql; namespace NSQLTranslationV0 { -class TAggregationFactory : public IAggregation { -public: +class TAggregationFactory : public IAggregation { +public: TAggregationFactory(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi = false) : IAggregation(pos, name, func, aggMode), Factory(!func.empty() ? BuildBind(Pos, aggMode == EAggregateMode::OverWindow ? "window_module" : "aggregate_module", func) : nullptr), @@ -25,7 +25,7 @@ public: } } -protected: +protected: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) override { ui32 expectedArgs = !Factory ? 2 : (isFactory ? 0 : 1); if (!Factory) { @@ -35,9 +35,9 @@ protected: if (expectedArgs != exprs.size()) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires exactly " << expectedArgs << " argument(s), given: " << exprs.size(); - return false; - } - + return false; + } + if (!Factory) { Factory = exprs[1]; } @@ -47,22 +47,22 @@ protected: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { node.Add("Member", "row", Q(Name)); } - return true; - } + return true; + } TNodePtr AggregationTraitsFactory() const override { return Factory; } - TNodePtr GetApply(const TNodePtr& type) const override { + TNodePtr GetApply(const TNodePtr& type) const override { if (!Multi) { return Y("Apply", Factory, (DynamicFactory ? Y("ListItemType", type) : type), BuildLambda(Pos, Y("row"), Y("EnsurePersistable", Expr))); @@ -72,32 +72,32 @@ protected: Y("ListItemType", type), BuildLambda(Pos, Y("row"), Y("EnsurePersistable", Expr)), Factory); - } - - bool DoInit(TContext& ctx, ISource* src) override { + } + + bool DoInit(TContext& ctx, ISource* src) override { if (!Expr) { return true; } ctx.PushBlockShortcuts(); - if (!Expr->Init(ctx, src)) { - return false; + if (!Expr->Init(ctx, src)) { + return false; } if (Expr->IsAggregated() && !Expr->IsAggregationKey() && !IsOverWindow()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden for no window functions"; - return false; - } - if (AggMode == EAggregateMode::Distinct) { - const auto column = Expr->GetColumnName(); - if (!column) { - ctx.Error(Expr->GetPos()) << "DISTINCT qualifier may only be used with column references"; + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden for no window functions"; + return false; + } + if (AggMode == EAggregateMode::Distinct) { + const auto column = Expr->GetColumnName(); + if (!column) { + ctx.Error(Expr->GetPos()) << "DISTINCT qualifier may only be used with column references"; return false; } - DistinctKey = *column; + DistinctKey = *column; YQL_ENSURE(src); if (src->GetJoin()) { - const auto sourcePtr = Expr->GetSourceName(); - if (!sourcePtr || !*sourcePtr) { + const auto sourcePtr = Expr->GetSourceName(); + if (!sourcePtr || !*sourcePtr) { if (!src->IsGroupByColumn(DistinctKey)) { ctx.Error(Expr->GetPos()) << ErrorDistinctWithoutCorrelation(DistinctKey); return false; @@ -110,7 +110,7 @@ protected: ctx.Error(Expr->GetPos()) << ErrorDistinctByGroupKey(DistinctKey); return false; } - Expr = AstNode("row"); + Expr = AstNode("row"); ctx.PopBlockShortcuts(); } else { Expr = ctx.GroundBlockShortcutsForExpr(Expr); @@ -138,27 +138,27 @@ protected: } } - return true; - } + return true; + } TNodePtr Factory; TSourcePtr FakeSource; - TNodePtr Expr; + TNodePtr Expr; bool DynamicFactory; bool Multi; -}; +}; -class TAggregationFactoryImpl final : public TAggregationFactory { -public: +class TAggregationFactoryImpl final : public TAggregationFactory { +public: TAggregationFactoryImpl(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi) : TAggregationFactory(pos, name, func, aggMode, multi) - {} + {} -private: - TNodePtr DoClone() const final { +private: + TNodePtr DoClone() const final { return new TAggregationFactoryImpl(Pos, Name, Func, AggMode, Multi); } -}; +}; class TAggregationFactoryWinAutoargImpl final : public TAggregationFactory { public: @@ -204,25 +204,25 @@ private: TAggregationPtr BuildFactoryAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi) { return new TAggregationFactoryImpl(pos, name, func, aggMode, multi); -} +} TAggregationPtr BuildFactoryAggregationWinAutoarg(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) { return new TAggregationFactoryWinAutoargImpl(pos, name, func, aggMode); } -class TKeyPayloadAggregationFactory final : public TAggregationFactory { -public: +class TKeyPayloadAggregationFactory final : public TAggregationFactory { +public: TKeyPayloadAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} + : TAggregationFactory(pos, name, factory, aggMode) + {} -private: +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() < adjustArgsCount || exprs.size() > 1 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << " or " << (1 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; + return false; } if (!isFactory) { @@ -235,27 +235,27 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { node.Add("Member", "row", Q(Name)); } - return true; - } + return true; + } - TNodePtr DoClone() const final { - return new TKeyPayloadAggregationFactory(Pos, Name, Func, AggMode); - } + TNodePtr DoClone() const final { + return new TKeyPayloadAggregationFactory(Pos, Name, Func, AggMode); + } - TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); AddFactoryArguments(apply); return apply; - } - + } + void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, Limit); } @@ -264,54 +264,54 @@ private: return {1u, 0u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Key) { return true; } ctx.PushBlockShortcuts(); - if (!Key->Init(ctx, src)) { - return false; - } + if (!Key->Init(ctx, src)) { + return false; + } Key = ctx.GroundBlockShortcutsForExpr(Key); ctx.PushBlockShortcuts(); - if (!Payload->Init(ctx, src)) { - return false; - } + if (!Payload->Init(ctx, src)) { + return false; + } Payload = ctx.GroundBlockShortcutsForExpr(Payload); - if (!Limit->Init(ctx, src)) { - return false; - } - - if (Key->IsAggregated()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } - - TNodePtr Key, Payload, Limit; -}; - + if (!Limit->Init(ctx, src)) { + return false; + } + + if (Key->IsAggregated()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + + TNodePtr Key, Payload, Limit; +}; + TAggregationPtr BuildKeyPayloadFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TKeyPayloadAggregationFactory(pos, name, factory, aggMode); -} - -class TPayloadPredicateAggregationFactory final : public TAggregationFactory { -public: + return new TKeyPayloadAggregationFactory(pos, name, factory, aggMode); +} + +class TPayloadPredicateAggregationFactory final : public TAggregationFactory { +public: TPayloadPredicateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} - -private: + : TAggregationFactory(pos, name, factory, aggMode) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() != adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << "arguments, given: " << exprs.size(); - return false; + return false; } - + if (!isFactory) { Payload = exprs.front(); Predicate = exprs.back(); @@ -319,75 +319,75 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { node.Add("Member", "row", Q(Name)); } - return true; - } + return true; + } - TNodePtr DoClone() const final { - return new TPayloadPredicateAggregationFactory(Pos, Name, Func, AggMode); - } + TNodePtr DoClone() const final { + return new TPayloadPredicateAggregationFactory(Pos, Name, Func, AggMode); + } - TNodePtr GetApply(const TNodePtr& type) const final { - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); - } + TNodePtr GetApply(const TNodePtr& type) const final { + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); + } std::vector<ui32> GetFactoryColumnIndices() const final { return {0u, 1u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Predicate) { return true; } ctx.PushBlockShortcuts(); - if (!Predicate->Init(ctx, src)) { - return false; + if (!Predicate->Init(ctx, src)) { + return false; } Predicate = ctx.GroundBlockShortcutsForExpr(Predicate); ctx.PushBlockShortcuts(); - if (!Payload->Init(ctx, src)) { - return false; - } + if (!Payload->Init(ctx, src)) { + return false; + } Payload = ctx.GroundBlockShortcutsForExpr(Payload); - if (Payload->IsAggregated()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } + if (Payload->IsAggregated()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } - return true; + return true; } - TNodePtr Payload, Predicate; -}; + TNodePtr Payload, Predicate; +}; TAggregationPtr BuildPayloadPredicateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TPayloadPredicateAggregationFactory(pos, name, factory, aggMode); -} + return new TPayloadPredicateAggregationFactory(pos, name, factory, aggMode); +} -class TTwoArgsAggregationFactory final : public TAggregationFactory { -public: +class TTwoArgsAggregationFactory final : public TAggregationFactory { +public: TTwoArgsAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} - -private: + : TAggregationFactory(pos, name, factory, aggMode) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() != adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << " arguments, given: " << exprs.size(); - return false; + return false; } - + if (!isFactory) { One = exprs.front(); Two = exprs.back(); @@ -395,66 +395,66 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { node.Add("Member", "row", Q(Name)); } - return true; - } - - TNodePtr DoClone() const final { - return new TTwoArgsAggregationFactory(Pos, Name, Func, AggMode); - } + return true; + } - TNodePtr GetApply(const TNodePtr& type) const final { - auto tuple = Q(Y(One, Two)); - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), tuple)); - } + TNodePtr DoClone() const final { + return new TTwoArgsAggregationFactory(Pos, Name, Func, AggMode); + } - bool DoInit(TContext& ctx, ISource* src) final { + TNodePtr GetApply(const TNodePtr& type) const final { + auto tuple = Q(Y(One, Two)); + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), tuple)); + } + + bool DoInit(TContext& ctx, ISource* src) final { if (!One) { return true; } ctx.PushBlockShortcuts(); - if (!One->Init(ctx, src)) { - return false; - } + if (!One->Init(ctx, src)) { + return false; + } One = ctx.GroundBlockShortcutsForExpr(One); ctx.PushBlockShortcuts(); - if (!Two->Init(ctx, src)) { - return false; - } + if (!Two->Init(ctx, src)) { + return false; + } Two = ctx.GroundBlockShortcutsForExpr(Two); - if ((One->IsAggregated() || Two->IsAggregated()) && !IsOverWindow()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } - - TNodePtr One, Two; -}; + if ((One->IsAggregated() || Two->IsAggregated()) && !IsOverWindow()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + TNodePtr One, Two; +}; + TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTwoArgsAggregationFactory(pos, name, factory, aggMode); -} - -class THistogramAggregationFactory final : public TAggregationFactory { -public: + return new TTwoArgsAggregationFactory(pos, name, factory, aggMode); +} + +class THistogramAggregationFactory final : public TAggregationFactory { +public: THistogramAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) , Weight(Y("Double", Q("1.0"))) , Intervals(Y("Uint32", Q("100"))) - {} - -private: + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { if (isFactory) { if (exprs.size() > 1) { @@ -467,7 +467,7 @@ private: return false; } } - + if (!isFactory) { /// \todo: solve it with named arguments const auto integer = exprs.back()->IsIntegerLiteral(); @@ -484,10 +484,10 @@ private: } Weight = exprs[1]; break; - } + } if (exprs.size() >= 2 && integer) { Intervals = Y("Cast", exprs.back(), Q("Uint32")); - } + } } else { if (exprs.size() >= 1) { const auto integer = exprs.back()->IsIntegerLiteral(); @@ -501,17 +501,17 @@ private: } return TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front())); + } + + TNodePtr DoClone() const final { + return new THistogramAggregationFactory(Pos, Name, Func, AggMode); } - TNodePtr DoClone() const final { - return new THistogramAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), BuildLambda(Pos, Y("row"), Weight)); AddFactoryArguments(apply); return apply; - } + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, Intervals); @@ -521,151 +521,151 @@ private: return {0u, 1u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { ctx.PushBlockShortcuts(); if (!Weight->Init(ctx, src)) { - return false; - } + return false; + } Weight = ctx.GroundBlockShortcutsForExpr(Weight); ctx.PushBlockShortcuts(); if (!Intervals->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } Intervals = ctx.GroundBlockShortcutsForExpr(Intervals); - return TAggregationFactory::DoInit(ctx, src); - } + return TAggregationFactory::DoInit(ctx, src); + } TSourcePtr FakeSource; - TNodePtr Weight, Intervals; -}; - + TNodePtr Weight, Intervals; +}; + TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new THistogramAggregationFactory(pos, name, factory, aggMode); -} - -class TLinearHistogramAggregationFactory final : public TAggregationFactory { -public: + return new THistogramAggregationFactory(pos, name, factory, aggMode); +} + +class TLinearHistogramAggregationFactory final : public TAggregationFactory { +public: TLinearHistogramAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - , BinSize(Y("Double", Q("10.0"))) - , Minimum(Y("Double", Q(ToString(-1.0 * Max<double>())))) - , Maximum(Y("Double", Q(ToString(Max<double>())))) - {} - -private: + , BinSize(Y("Double", Q("10.0"))) + , Minimum(Y("Double", Q(ToString(-1.0 * Max<double>())))) + , Maximum(Y("Double", Q(ToString(Max<double>())))) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { Y_UNUSED(isFactory); - if (exprs.empty() || exprs.size() > 4) { - ctx.Error(Pos) << "Aggregation function " << Name << " requires one to four arguments, given: " << exprs.size(); - return false; + if (exprs.empty() || exprs.size() > 4) { + ctx.Error(Pos) << "Aggregation function " << Name << " requires one to four arguments, given: " << exprs.size(); + return false; } - if (exprs.size() > 1) { - BinSize = exprs[1]; + if (exprs.size() > 1) { + BinSize = exprs[1]; } - if (exprs.size() > 2) { - Minimum = exprs[2]; + if (exprs.size() > 2) { + Minimum = exprs[2]; } - if (exprs.size() > 3) { - Maximum = exprs[3]; + if (exprs.size() > 3) { + Maximum = exprs[3]; } return TAggregationFactory::InitAggr(ctx, isFactory, src, node, { exprs.front() }); - } + } - TNodePtr DoClone() const final { - return new TLinearHistogramAggregationFactory(Pos, Name, Func, AggMode); + TNodePtr DoClone() const final { + return new TLinearHistogramAggregationFactory(Pos, Name, Func, AggMode); } - TNodePtr GetApply(const TNodePtr& type) const final { - return Y("Apply", Factory, type, - BuildLambda(Pos, Y("row"), Expr), - BinSize, Minimum, Maximum); - } + TNodePtr GetApply(const TNodePtr& type) const final { + return Y("Apply", Factory, type, + BuildLambda(Pos, Y("row"), Expr), + BinSize, Minimum, Maximum); + } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { ctx.PushBlockShortcuts(); if (!BinSize->Init(ctx, FakeSource.Get())) { - return false; + return false; } BinSize = ctx.GroundBlockShortcutsForExpr(BinSize); ctx.PushBlockShortcuts(); if (!Minimum->Init(ctx, FakeSource.Get())) { - return false; + return false; } Minimum = ctx.GroundBlockShortcutsForExpr(Minimum); ctx.PushBlockShortcuts(); if (!Maximum->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } Maximum = ctx.GroundBlockShortcutsForExpr(Maximum); - return TAggregationFactory::DoInit(ctx, src); - } + return TAggregationFactory::DoInit(ctx, src); + } TSourcePtr FakeSource; - TNodePtr BinSize, Minimum, Maximum; -}; + TNodePtr BinSize, Minimum, Maximum; +}; TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TLinearHistogramAggregationFactory(pos, name, factory, aggMode); -} + return new TLinearHistogramAggregationFactory(pos, name, factory, aggMode); +} -class TPercentileFactory final : public TAggregationFactory { -public: +class TPercentileFactory final : public TAggregationFactory { +public: TPercentileFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - {} + {} -private: +private: const TString* GetGenericKey() const final { - return Column; - } - - void Join(IAggregation* aggr) final { - const auto percentile = dynamic_cast<TPercentileFactory*>(aggr); - Y_VERIFY(percentile); - Y_VERIFY(*Column == *percentile->Column); - Y_VERIFY(AggMode == percentile->AggMode); - Percentiles.insert(percentile->Percentiles.cbegin(), percentile->Percentiles.cend()); - percentile->Percentiles.clear(); - } - + return Column; + } + + void Join(IAggregation* aggr) final { + const auto percentile = dynamic_cast<TPercentileFactory*>(aggr); + Y_VERIFY(percentile); + Y_VERIFY(*Column == *percentile->Column); + Y_VERIFY(AggMode == percentile->AggMode); + Percentiles.insert(percentile->Percentiles.cbegin(), percentile->Percentiles.cend()); + percentile->Percentiles.clear(); + } + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; if (exprs.size() < 0 + adjustArgsCount || exprs.size() > 1 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << (0 + adjustArgsCount) << " or " << (1 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } - + return false; + } + if (!isFactory) { Column = exprs.front()->GetColumnName(); if (!Column) { ctx.Error(Pos) << Name << " may only be used with column reference as first argument."; return false; } - } - + } + if (!TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front()))) - return false; - + return false; + TNodePtr x; if (1 + adjustArgsCount == exprs.size()) { x = exprs.back(); ctx.PushBlockShortcuts(); if (!x->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } x = ctx.GroundBlockShortcutsForExpr(x); - } else { + } else { x = Y("Double", Q("0.5")); - } + } if (isFactory) { FactoryPercentile = x; @@ -673,194 +673,194 @@ private: Percentiles.emplace(Name, x); } - return true; - } - - TNodePtr DoClone() const final { - return new TPercentileFactory(Pos, Name, Func, AggMode); - } + return true; + } - TNodePtr GetApply(const TNodePtr& type) const final { - TNodePtr percentiles(Percentiles.cbegin()->second); - - if (Percentiles.size() > 1U) { - percentiles = Y(); - for (const auto& percentile : Percentiles) { - percentiles = L(percentiles, percentile.second); - } - percentiles = Q(percentiles); + TNodePtr DoClone() const final { + return new TPercentileFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr percentiles(Percentiles.cbegin()->second); + + if (Percentiles.size() > 1U) { + percentiles = Y(); + for (const auto& percentile : Percentiles) { + percentiles = L(percentiles, percentile.second); + } + percentiles = Q(percentiles); } - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), percentiles); - } + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), percentiles); + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, FactoryPercentile); } - TNodePtr AggregationTraits(const TNodePtr& type) const final { - if (Percentiles.empty()) - return TNodePtr(); - - TNodePtr names(Q(Percentiles.cbegin()->first)); - - if (Percentiles.size() > 1U) { - names = Y(); - for (const auto& percentile : Percentiles) - names = L(names, Q(percentile.first)); - names = Q(names); - } - - const bool distinct = AggMode == EAggregateMode::Distinct; - const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; - return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); - } - - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& p : Percentiles) { - if (!p.second->Init(ctx, src)) { - return false; + TNodePtr AggregationTraits(const TNodePtr& type) const final { + if (Percentiles.empty()) + return TNodePtr(); + + TNodePtr names(Q(Percentiles.cbegin()->first)); + + if (Percentiles.size() > 1U) { + names = Y(); + for (const auto& percentile : Percentiles) + names = L(names, Q(percentile.first)); + names = Q(names); + } + + const bool distinct = AggMode == EAggregateMode::Distinct; + const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; + return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); + } + + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& p : Percentiles) { + if (!p.second->Init(ctx, src)) { + return false; } } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } TSourcePtr FakeSource; std::multimap<TString, TNodePtr> Percentiles; TNodePtr FactoryPercentile; const TString* Column = nullptr; -}; +}; TAggregationPtr BuildPercentileFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TPercentileFactory(pos, name, factory, aggMode); -} - -class TTopFreqFactory final : public TAggregationFactory { -public: + return new TPercentileFactory(pos, name, factory, aggMode); +} + +class TTopFreqFactory final : public TAggregationFactory { +public: TTopFreqFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} - -private: - - //first - n, second - buffer - using TPair = std::pair<TNodePtr, TNodePtr>; - + : TAggregationFactory(pos, name, factory, aggMode) + {} + +private: + + //first - n, second - buffer + using TPair = std::pair<TNodePtr, TNodePtr>; + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; - const double DefaultBufferC = 1.5; - const ui32 MinBuffer = 100; + const double DefaultBufferC = 1.5; + const ui32 MinBuffer = 100; if (exprs.size() < adjustArgsCount || exprs.size() > 2 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory? "factory " : "") << Name << " requires " << adjustArgsCount << " to " << (2 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } - + return false; + } + if (!TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front()))) - return false; - - ui32 n; - ui32 buffer; - + return false; + + ui32 n; + ui32 buffer; + if (1 + adjustArgsCount <= exprs.size()) { auto posSecondArg = exprs[adjustArgsCount]->GetPos(); if (!Parseui32(exprs[adjustArgsCount], n)) { ctx.Error(posSecondArg) << "TopFreq: invalid argument #" << (1 + adjustArgsCount) << ", numeric literal is expected"; - return false; - } - } else { - n = 1; - } - + return false; + } + } else { + n = 1; + } + if (2 + adjustArgsCount == exprs.size()) { auto posThirdArg = exprs[1 + adjustArgsCount]->GetPos(); if (Parseui32(exprs[1 + adjustArgsCount], buffer)) { - if (n > buffer) { + if (n > buffer) { ctx.Error(posThirdArg) << "TopFreq: #" << (2 + adjustArgsCount) << " argument (buffer size) must be greater or equal than previous argument "; - return false; - } - } else { + return false; + } + } else { ctx.Error(posThirdArg) << "TopFreq: invalid #" << (2 + adjustArgsCount) << " argument, numeric literal is expected"; - return false; - } - } else { - buffer = std::max(ui32(n * DefaultBufferC), MinBuffer); - } - + return false; + } + } else { + buffer = std::max(ui32(n * DefaultBufferC), MinBuffer); + } + auto x = TPair{ Y("Uint32", Q(ToString(n))), Y("Uint32", Q(ToString(buffer))) }; if (isFactory) { TopFreqFactoryParams = x; } else { TopFreqs.emplace(Name, x); } - - return true; - } - - TNodePtr DoClone() const final { - return new TTopFreqFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { - TPair topFreqs(TopFreqs.cbegin()->second); - - if (TopFreqs.size() > 1U) { - topFreqs = { Y(), Y() }; - for (const auto& topFreq : TopFreqs) { - topFreqs = { L(topFreqs.first, topFreq.second.first), L(topFreqs.second, topFreq.second.second) }; + + return true; + } + + TNodePtr DoClone() const final { + return new TTopFreqFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TPair topFreqs(TopFreqs.cbegin()->second); + + if (TopFreqs.size() > 1U) { + topFreqs = { Y(), Y() }; + for (const auto& topFreq : TopFreqs) { + topFreqs = { L(topFreqs.first, topFreq.second.first), L(topFreqs.second, topFreq.second.second) }; } - topFreqs = { Q(topFreqs.first), Q(topFreqs.second) }; + topFreqs = { Q(topFreqs.first), Q(topFreqs.second) }; } auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), topFreqs.first, topFreqs.second); return apply; - } + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, TopFreqFactoryParams.first, TopFreqFactoryParams.second); } - TNodePtr AggregationTraits(const TNodePtr& type) const final { - if (TopFreqs.empty()) - return TNodePtr(); + TNodePtr AggregationTraits(const TNodePtr& type) const final { + if (TopFreqs.empty()) + return TNodePtr(); - TNodePtr names(Q(TopFreqs.cbegin()->first)); + TNodePtr names(Q(TopFreqs.cbegin()->first)); - if (TopFreqs.size() > 1U) { - names = Y(); - for (const auto& topFreq : TopFreqs) - names = L(names, Q(topFreq.first)); - names = Q(names); + if (TopFreqs.size() > 1U) { + names = Y(); + for (const auto& topFreq : TopFreqs) + names = L(names, Q(topFreq.first)); + names = Q(names); } - const bool distinct = AggMode == EAggregateMode::Distinct; - const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; - return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); - } - - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& topFreq : TopFreqs) { - if (!topFreq.second.first->Init(ctx, src)) { - return false; - } + const bool distinct = AggMode == EAggregateMode::Distinct; + const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; + return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); + } - if (!topFreq.second.second->Init(ctx, src)) { - return false; - } + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& topFreq : TopFreqs) { + if (!topFreq.second.first->Init(ctx, src)) { + return false; + } + + if (!topFreq.second.second->Init(ctx, src)) { + return false; + } } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } std::multimap<TString, TPair> TopFreqs; TPair TopFreqFactoryParams; -}; - +}; + TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTopFreqFactory(pos, name, factory, aggMode); -} + return new TTopFreqFactory(pos, name, factory, aggMode); +} template <bool HasKey> class TTopAggregationFactory final : public TAggregationFactory { @@ -1104,101 +1104,101 @@ TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, return new TListAggregationFactory(pos, name, factory, aggMode); } -class TUserDefinedAggregationFactory final : public TAggregationFactory { -public: +class TUserDefinedAggregationFactory final : public TAggregationFactory { +public: TUserDefinedAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} + : TAggregationFactory(pos, name, factory, aggMode) + {} -private: +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; if (exprs.size() < (3 + adjustArgsCount) || exprs.size() > (7 + adjustArgsCount)) { ctx.Error(Pos) << "User defined aggregation function " << (isFactory ? "factory " : "") << " requires " << (3 + adjustArgsCount) << " to " << (7 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } + return false; + } Lambdas[0] = BuildLambda(Pos, Y("value", "parent"), Y("NamedApply", exprs[adjustArgsCount], Q(Y("value")), Y("AsStruct"), Y("DependsOn", "parent"))); Lambdas[1] = BuildLambda(Pos, Y("value", "state", "parent"), Y("NamedApply", exprs[adjustArgsCount + 1], Q(Y("state", "value")), Y("AsStruct"), Y("DependsOn", "parent"))); Lambdas[2] = BuildLambda(Pos, Y("one", "two"), Y("Apply", exprs[adjustArgsCount + 2], "one", "two")); - for (size_t i = 3U; i < Lambdas.size(); ++i) { + for (size_t i = 3U; i < Lambdas.size(); ++i) { const auto j = adjustArgsCount + i; - Lambdas[i] = BuildLambda(Pos, Y("state"), j >= exprs.size() ? AstNode("state") : Y("Apply", exprs[j], "state")); - } + Lambdas[i] = BuildLambda(Pos, Y("state"), j >= exprs.size() ? AstNode("state") : Y("Apply", exprs[j], "state")); + } DefVal = (exprs.size() == (7 + adjustArgsCount)) ? exprs[adjustArgsCount + 6] : Y("Null"); return TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front())); - } - - TNodePtr DoClone() const final { - return new TUserDefinedAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { + } + + TNodePtr DoClone() const final { + return new TUserDefinedAggregationFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr)); AddFactoryArguments(apply); return apply; - } - + } + void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, Lambdas[0], Lambdas[1], Lambdas[2], Lambdas[3], Lambdas[4], Lambdas[5], DefVal); } - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& lambda : Lambdas) { - if (!lambda->Init(ctx, src)) { - return false; - } + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& lambda : Lambdas) { + if (!lambda->Init(ctx, src)) { + return false; + } } - + if (!DefVal->Init(ctx, src)) { return false; } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } - std::array<TNodePtr, 6> Lambdas; + std::array<TNodePtr, 6> Lambdas; TNodePtr DefVal; -}; +}; TAggregationPtr BuildUserDefinedFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TUserDefinedAggregationFactory(pos, name, factory, aggMode); -} + return new TUserDefinedAggregationFactory(pos, name, factory, aggMode); +} -class TCountAggregation final : public TAggregationFactory { -public: +class TCountAggregation final : public TAggregationFactory { +public: TCountAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) - : TAggregationFactory(pos, name, func, aggMode) - {} + : TAggregationFactory(pos, name, func, aggMode) + {} -private: - TNodePtr DoClone() const final { - return new TCountAggregation(Pos, Name, Func, AggMode); - } +private: + TNodePtr DoClone() const final { + return new TCountAggregation(Pos, Name, Func, AggMode); + } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Expr) { return true; } if (Expr->IsAsterisk()) { - Expr = Y("Void"); + Expr = Y("Void"); } ctx.PushBlockShortcuts(); if (!Expr->Init(ctx, src)) { return false; } - Expr->SetCountHint(Expr->IsConstant()); + Expr->SetCountHint(Expr->IsConstant()); Expr = ctx.GroundBlockShortcutsForExpr(Expr); - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } -}; +}; TAggregationPtr BuildCountAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) { - return new TCountAggregation(pos, name, func, aggMode); -} + return new TCountAggregation(pos, name, func, aggMode); +} } // namespace NSQLTranslationV0 diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp index a0945f9d05e..0ca9332b66d 100644 --- a/ydb/library/yql/sql/v0/builtin.cpp +++ b/ydb/library/yql/sql/v0/builtin.cpp @@ -1648,13 +1648,13 @@ enum EAggrFuncTypeCallback { NORMAL, WINDOW_AUTOARGS, KEY_PAYLOAD, - PAYLOAD_PREDICATE, + PAYLOAD_PREDICATE, TWO_ARGS, COUNT, HISTOGRAM, LINEAR_HISTOGRAM, PERCENTILE, - TOPFREQ, + TOPFREQ, TOP, TOP_BY, COUNT_DISTINCT_ESTIMATE, @@ -1701,9 +1701,9 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case KEY_PAYLOAD: factory = BuildKeyPayloadFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case PAYLOAD_PREDICATE: - factory = BuildPayloadPredicateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); - break; + case PAYLOAD_PREDICATE: + factory = BuildPayloadPredicateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); + break; case TWO_ARGS: factory = BuildTwoArgsFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; @@ -1719,9 +1719,9 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case PERCENTILE: factory = BuildPercentileFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case TOPFREQ: - factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); - break; + case TOPFREQ: + factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); + break; case TOP: factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); break; @@ -2054,8 +2054,8 @@ struct TBuiltinFuncData { {"maxby", BuildAggrFuncFactoryCallback("MaxBy", "max_by_traits_factory", KEY_PAYLOAD)}, {"sum", BuildAggrFuncFactoryCallback("Sum", "sum_traits_factory")}, - {"sumif", BuildAggrFuncFactoryCallback("SumIf", "sum_if_traits_factory", PAYLOAD_PREDICATE) }, - + {"sumif", BuildAggrFuncFactoryCallback("SumIf", "sum_if_traits_factory", PAYLOAD_PREDICATE) }, + {"some", BuildAggrFuncFactoryCallback("Some", "some_traits_factory")}, {"somevalue", BuildAggrFuncFactoryCallback("SomeValue", "some_traits_factory")}, @@ -2071,8 +2071,8 @@ struct TBuiltinFuncData { {"bitxor", BuildAggrFuncFactoryCallback("BitXor", "bit_xor_traits_factory")}, {"avg", BuildAggrFuncFactoryCallback("Avg", "avg_traits_factory")}, - {"avgif", BuildAggrFuncFactoryCallback("AvgIf", "avg_if_traits_factory", PAYLOAD_PREDICATE) }, - + {"avgif", BuildAggrFuncFactoryCallback("AvgIf", "avg_if_traits_factory", PAYLOAD_PREDICATE) }, + {"list", BuildAggrFuncFactoryCallback("List", "list_traits_factory", LIST)}, {"agglist", BuildAggrFuncFactoryCallback("AggregateList", "list2_traits_factory", LIST)}, {"aggrlist", BuildAggrFuncFactoryCallback("AggregateList", "list2_traits_factory", LIST)}, @@ -2085,9 +2085,9 @@ struct TBuiltinFuncData { {"median", BuildAggrFuncFactoryCallback("Median", "percentile_traits_factory", PERCENTILE)}, {"percentile", BuildAggrFuncFactoryCallback("Percentile", "percentile_traits_factory", PERCENTILE)}, - {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, - {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, - + {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, + {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, + {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp index 7752b28b017..dcfd98fcf13 100644 --- a/ydb/library/yql/sql/v0/query.cpp +++ b/ydb/library/yql/sql/v0/query.cpp @@ -52,7 +52,7 @@ public: key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, View))))); } if (mode == ITableKeys::EBuildKeysMode::INPUT && - IsQueryMode(ctx.Settings.Mode) && + IsQueryMode(ctx.Settings.Mode) && ctx.GetClusterProvider(Cluster).GetRef() != "kikimr" && ctx.GetClusterProvider(Cluster).GetRef() != "rtmr") { @@ -1023,10 +1023,10 @@ public: Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0")))); Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1")))); } - } else { + } else { const auto& ref = block->GetLabel(); Add(Y("let", ref ? ref : "world", block)); - } + } } if (TopLevel) { diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index b028fc04398..82135ccfeca 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -4215,7 +4215,7 @@ private: bool AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alter_table_drop_column& node, const TTableRef& tr); TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success); void AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node); - + TNodePtr Build(const TRule_delete_stmt& stmt); TNodePtr Build(const TRule_update_stmt& stmt); @@ -4251,9 +4251,9 @@ private: void TSqlQuery::AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node) { blocks.emplace_back(node); -} - - +} + + bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core) { TString internalStatementName; TString humanStatementName; @@ -4279,7 +4279,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } if (nodeExpr) { - AddStatementToBlocks(blocks, nodeExpr); + AddStatementToBlocks(blocks, nodeExpr); } break; } @@ -4362,7 +4362,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } TTableRef tr(SimpleTableRefImpl(rule.GetRule_simple_table_ref4(), Mode, *this)); - AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr)); + AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore6: { @@ -4397,7 +4397,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& if (!updateNode) { return false; } - AddStatementToBlocks(blocks, updateNode); + AddStatementToBlocks(blocks, updateNode); break; } case TRule_sql_stmt_core::kAltSqlStmtCore10: { @@ -5089,8 +5089,8 @@ TNodePtr TSqlQuery::Build(const TSQLParserAST& ast) { return nullptr; } - if (!Ctx.PragmaAutoCommit && Ctx.Settings.EndOfQueryCommit && IsQueryMode(Mode)) { - AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); + if (!Ctx.PragmaAutoCommit && Ctx.Settings.EndOfQueryCommit && IsQueryMode(Mode)) { + AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); } return BuildQuery(Ctx.Pos(), blocks, true); } @@ -5286,14 +5286,14 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TTranslationSettings& settings) { - YQL_ENSURE(IsQueryMode(settings.Mode)); + YQL_ENSURE(IsQueryMode(settings.Mode)); TAstParseResult res; TContext ctx(settings, res.Issues); SqlASTToYqlImpl(res, protoAst, ctx); return res; } - + NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules) { TAstParseResult res; diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp index 3a70aef93cc..36f21925c56 100644 --- a/ydb/library/yql/sql/v0/sql_ut.cpp +++ b/ydb/library/yql/sql/v0/sql_ut.cpp @@ -31,7 +31,7 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES NSQLTranslation::TTranslationSettings settings; settings.ClusterMapping[cluster] = service; settings.MaxErrors = maxErrors; - settings.Mode = mode; + settings.Mode = mode; settings.Arena = &arena; settings.V0Behavior = NSQLTranslation::EV0Behavior::Report; settings.WarnOnV0 = false; @@ -42,10 +42,10 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES return res; } -NYql::TAstParseResult SqlToYql(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { +NYql::TAstParseResult SqlToYql(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug); -} - +} + TString GetPrettyPrint(const NYql::TAstParseResult& res) { TStringStream yqlProgram; res.Root->PrettyPrintTo(yqlProgram, NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::ShortQuote); @@ -57,19 +57,19 @@ TString Quote(const char* str) { } class TWordCountHive: public TMap<TString, unsigned> { -public: - TWordCountHive(std::initializer_list<TString> strings) { - for (auto& str: strings) { - emplace(str, 0); - } - } - - TWordCountHive(std::initializer_list<std::pair<const TString, unsigned>> list) +public: + TWordCountHive(std::initializer_list<TString> strings) { + for (auto& str: strings) { + emplace(str, 0); + } + } + + TWordCountHive(std::initializer_list<std::pair<const TString, unsigned>> list) : TMap(list) - { - } -}; - + { + } +}; + typedef std::function<void (const TString& word, const TString& line)> TVerifyLineFunc; TString VerifyProgram(const NYql::TAstParseResult& res, TWordCountHive& wordCounter, TVerifyLineFunc verifyLine = TVerifyLineFunc()) { diff --git a/ydb/library/yql/sql/v1/aggregation.cpp b/ydb/library/yql/sql/v1/aggregation.cpp index 850e0e5056e..172a4003401 100644 --- a/ydb/library/yql/sql/v1/aggregation.cpp +++ b/ydb/library/yql/sql/v1/aggregation.cpp @@ -29,8 +29,8 @@ namespace { } } -class TAggregationFactory : public IAggregation { -public: +class TAggregationFactory : public IAggregation { +public: TAggregationFactory(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi = false) : IAggregation(pos, name, func, aggMode), Factory(!func.empty() ? BuildBind(Pos, aggMode == EAggregateMode::OverWindow ? "window_module" : "aggregate_module", func) : nullptr), @@ -41,7 +41,7 @@ public: } } -protected: +protected: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) override { ui32 expectedArgs = !Factory ? 2 : (isFactory ? 0 : 1); if (!Factory) { @@ -51,9 +51,9 @@ protected: if (expectedArgs != exprs.size()) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires exactly " << expectedArgs << " argument(s), given: " << exprs.size(); - return false; - } - + return false; + } + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -67,8 +67,8 @@ protected: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { @@ -78,14 +78,14 @@ protected: } } - return true; - } + return true; + } TNodePtr AggregationTraitsFactory() const override { return Factory; } - TNodePtr GetApply(const TNodePtr& type) const override { + TNodePtr GetApply(const TNodePtr& type) const override { if (!Multi) { return Y("Apply", Factory, (DynamicFactory ? Y("ListItemType", type) : type), BuildLambda(Pos, Y("row"), Y("PersistableRepr", Expr))); @@ -95,28 +95,28 @@ protected: Y("ListItemType", type), BuildLambda(Pos, Y("row"), Y("PersistableRepr", Expr)), Factory); - } - - bool DoInit(TContext& ctx, ISource* src) override { + } + + bool DoInit(TContext& ctx, ISource* src) override { if (!Expr) { return true; } - if (!Expr->Init(ctx, src)) { - return false; + if (!Expr->Init(ctx, src)) { + return false; } if (Expr->IsAggregated() && !Expr->IsAggregationKey() && !IsOverWindow()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden for no window functions"; - return false; - } - if (AggMode == EAggregateMode::Distinct) { - const auto column = Expr->GetColumnName(); + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden for no window functions"; + return false; + } + if (AggMode == EAggregateMode::Distinct) { + const auto column = Expr->GetColumnName(); YQL_ENSURE(column); - DistinctKey = *column; + DistinctKey = *column; YQL_ENSURE(src); if (!IsGeneratedKeyColumn && src->GetJoin()) { - const auto sourcePtr = Expr->GetSourceName(); - if (!sourcePtr || !*sourcePtr) { + const auto sourcePtr = Expr->GetSourceName(); + if (!sourcePtr || !*sourcePtr) { if (!src->IsGroupByColumn(DistinctKey)) { ctx.Error(Expr->GetPos()) << ErrorDistinctWithoutCorrelation(DistinctKey); return false; @@ -129,7 +129,7 @@ protected: ctx.Error(Expr->GetPos()) << ErrorDistinctByGroupKey(DistinctKey); return false; } - Expr = AstNode("row"); + Expr = AstNode("row"); } if (FakeSource) { @@ -152,8 +152,8 @@ protected: } } - return true; - } + return true; + } TNodePtr Factory; TNodePtr Expr; @@ -162,37 +162,37 @@ protected: private: TSourcePtr FakeSource; bool DynamicFactory; -}; +}; -class TAggregationFactoryImpl final : public TAggregationFactory { -public: +class TAggregationFactoryImpl final : public TAggregationFactory { +public: TAggregationFactoryImpl(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi) : TAggregationFactory(pos, name, func, aggMode, multi) - {} + {} -private: - TNodePtr DoClone() const final { +private: + TNodePtr DoClone() const final { return new TAggregationFactoryImpl(Pos, Name, Func, AggMode, Multi); } -}; +}; TAggregationPtr BuildFactoryAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi) { return new TAggregationFactoryImpl(pos, name, func, aggMode, multi); -} +} -class TKeyPayloadAggregationFactory final : public TAggregationFactory { -public: +class TKeyPayloadAggregationFactory final : public TAggregationFactory { +public: TKeyPayloadAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} + : TAggregationFactory(pos, name, factory, aggMode) + {} -private: +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() < adjustArgsCount || exprs.size() > 1 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << " or " << (1 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; + return false; } if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; @@ -218,8 +218,8 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { @@ -229,19 +229,19 @@ private: } } - return true; - } + return true; + } - TNodePtr DoClone() const final { - return new TKeyPayloadAggregationFactory(Pos, Name, Func, AggMode); - } + TNodePtr DoClone() const final { + return new TKeyPayloadAggregationFactory(Pos, Name, Func, AggMode); + } - TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload)); AddFactoryArguments(apply); return apply; - } - + } + void AddFactoryArguments(TNodePtr& apply) const final { if (Limit) { apply = L(apply, Limit); @@ -252,52 +252,52 @@ private: return {1u, 0u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Key) { return true; } - if (!Key->Init(ctx, src)) { - return false; - } - if (!Payload->Init(ctx, src)) { - return false; - } + if (!Key->Init(ctx, src)) { + return false; + } + if (!Payload->Init(ctx, src)) { + return false; + } if (Limit) { if (!Limit->Init(ctx, src)) { return false; } - } - - if (Key->IsAggregated()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } - - TNodePtr Key, Payload, Limit; -}; - + } + + if (Key->IsAggregated()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + + TNodePtr Key, Payload, Limit; +}; + TAggregationPtr BuildKeyPayloadFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TKeyPayloadAggregationFactory(pos, name, factory, aggMode); -} - -class TPayloadPredicateAggregationFactory final : public TAggregationFactory { -public: + return new TKeyPayloadAggregationFactory(pos, name, factory, aggMode); +} + +class TPayloadPredicateAggregationFactory final : public TAggregationFactory { +public: TPayloadPredicateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} - -private: + : TAggregationFactory(pos, name, factory, aggMode) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() != adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << "arguments, given: " << exprs.size(); - return false; + return false; } - + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -308,8 +308,8 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { @@ -319,63 +319,63 @@ private: } } - return true; - } + return true; + } - TNodePtr DoClone() const final { - return new TPayloadPredicateAggregationFactory(Pos, Name, Func, AggMode); - } + TNodePtr DoClone() const final { + return new TPayloadPredicateAggregationFactory(Pos, Name, Func, AggMode); + } - TNodePtr GetApply(const TNodePtr& type) const final { - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); - } + TNodePtr GetApply(const TNodePtr& type) const final { + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate)); + } std::vector<ui32> GetFactoryColumnIndices() const final { return {0u, 1u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Predicate) { return true; } - if (!Predicate->Init(ctx, src)) { - return false; - } - if (!Payload->Init(ctx, src)) { - return false; + if (!Predicate->Init(ctx, src)) { + return false; } + if (!Payload->Init(ctx, src)) { + return false; + } - if (Payload->IsAggregated()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } + if (Payload->IsAggregated()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } - return true; + return true; } - TNodePtr Payload, Predicate; -}; + TNodePtr Payload, Predicate; +}; TAggregationPtr BuildPayloadPredicateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TPayloadPredicateAggregationFactory(pos, name, factory, aggMode); -} + return new TPayloadPredicateAggregationFactory(pos, name, factory, aggMode); +} -class TTwoArgsAggregationFactory final : public TAggregationFactory { -public: +class TTwoArgsAggregationFactory final : public TAggregationFactory { +public: TTwoArgsAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} - -private: + : TAggregationFactory(pos, name, factory, aggMode) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 2; if (exprs.size() != adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << adjustArgsCount << " arguments, given: " << exprs.size(); - return false; + return false; } - + if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) { return false; } @@ -386,8 +386,8 @@ private: Name = src->MakeLocalName(Name); } - if (!Init(ctx, src)) { - return false; + if (!Init(ctx, src)) { + return false; } if (!isFactory) { @@ -397,54 +397,54 @@ private: } } - return true; - } - - TNodePtr DoClone() const final { - return new TTwoArgsAggregationFactory(Pos, Name, Func, AggMode); - } + return true; + } - TNodePtr GetApply(const TNodePtr& type) const final { - auto tuple = Q(Y(One, Two)); - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), tuple)); - } + TNodePtr DoClone() const final { + return new TTwoArgsAggregationFactory(Pos, Name, Func, AggMode); + } - bool DoInit(TContext& ctx, ISource* src) final { + TNodePtr GetApply(const TNodePtr& type) const final { + auto tuple = Q(Y(One, Two)); + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), tuple)); + } + + bool DoInit(TContext& ctx, ISource* src) final { if (!One) { return true; } - if (!One->Init(ctx, src)) { - return false; - } - if (!Two->Init(ctx, src)) { - return false; - } - - if ((One->IsAggregated() || Two->IsAggregated()) && !IsOverWindow()) { - ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; - return false; - } - return true; - } + if (!One->Init(ctx, src)) { + return false; + } + if (!Two->Init(ctx, src)) { + return false; + } - TNodePtr One, Two; -}; + if ((One->IsAggregated() || Two->IsAggregated()) && !IsOverWindow()) { + ctx.Error(Pos) << "Aggregation of aggregated values is forbidden"; + return false; + } + return true; + } + TNodePtr One, Two; +}; + TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTwoArgsAggregationFactory(pos, name, factory, aggMode); -} - -class THistogramAggregationFactory final : public TAggregationFactory { -public: + return new TTwoArgsAggregationFactory(pos, name, factory, aggMode); +} + +class THistogramAggregationFactory final : public TAggregationFactory { +public: THistogramAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) , Weight(Y("Double", Q("1.0"))) , Intervals(Y("Uint32", Q("100"))) - {} - -private: + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { if (isFactory) { if (exprs.size() > 1) { @@ -457,7 +457,7 @@ private: return false; } } - + if (!isFactory) { /// \todo: solve it with named arguments const auto integer = exprs.back()->IsIntegerLiteral(); @@ -474,10 +474,10 @@ private: } Weight = exprs[1]; break; - } + } if (exprs.size() >= 2 && integer) { Intervals = Y("Cast", exprs.back(), Q("Uint32")); - } + } } else { if (exprs.size() >= 1) { const auto integer = exprs.back()->IsIntegerLiteral(); @@ -491,17 +491,17 @@ private: } return TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front())); + } + + TNodePtr DoClone() const final { + return new THistogramAggregationFactory(Pos, Name, Func, AggMode); } - TNodePtr DoClone() const final { - return new THistogramAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), BuildLambda(Pos, Y("row"), Weight)); AddFactoryArguments(apply); return apply; - } + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, Intervals); @@ -511,36 +511,36 @@ private: return {0u, 1u}; } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Weight->Init(ctx, src)) { - return false; - } + return false; + } if (!Intervals->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } - return TAggregationFactory::DoInit(ctx, src); - } + return TAggregationFactory::DoInit(ctx, src); + } TSourcePtr FakeSource; - TNodePtr Weight, Intervals; -}; - + TNodePtr Weight, Intervals; +}; + TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new THistogramAggregationFactory(pos, name, factory, aggMode); -} - -class TLinearHistogramAggregationFactory final : public TAggregationFactory { -public: + return new THistogramAggregationFactory(pos, name, factory, aggMode); +} + +class TLinearHistogramAggregationFactory final : public TAggregationFactory { +public: TLinearHistogramAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - , BinSize(Y("Double", Q("10.0"))) - , Minimum(Y("Double", Q(ToString(-1.0 * Max<double>())))) - , Maximum(Y("Double", Q(ToString(Max<double>())))) - {} - -private: + , BinSize(Y("Double", Q("10.0"))) + , Minimum(Y("Double", Q(ToString(-1.0 * Max<double>())))) + , Maximum(Y("Double", Q(ToString(Max<double>())))) + {} + +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { if (isFactory) { if (exprs.size() > 3) { @@ -567,93 +567,93 @@ private: } return TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front())); - } + } - TNodePtr DoClone() const final { - return new TLinearHistogramAggregationFactory(Pos, Name, Func, AggMode); + TNodePtr DoClone() const final { + return new TLinearHistogramAggregationFactory(Pos, Name, Func, AggMode); } - TNodePtr GetApply(const TNodePtr& type) const final { - return Y("Apply", Factory, type, - BuildLambda(Pos, Y("row"), Expr), - BinSize, Minimum, Maximum); - } + TNodePtr GetApply(const TNodePtr& type) const final { + return Y("Apply", Factory, type, + BuildLambda(Pos, Y("row"), Expr), + BinSize, Minimum, Maximum); + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, BinSize, Minimum, Maximum); } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!BinSize->Init(ctx, FakeSource.Get())) { - return false; + return false; } if (!Minimum->Init(ctx, FakeSource.Get())) { - return false; + return false; } if (!Maximum->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } - return TAggregationFactory::DoInit(ctx, src); - } + return TAggregationFactory::DoInit(ctx, src); + } TSourcePtr FakeSource; - TNodePtr BinSize, Minimum, Maximum; -}; + TNodePtr BinSize, Minimum, Maximum; +}; TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TLinearHistogramAggregationFactory(pos, name, factory, aggMode); -} + return new TLinearHistogramAggregationFactory(pos, name, factory, aggMode); +} -class TPercentileFactory final : public TAggregationFactory { -public: +class TPercentileFactory final : public TAggregationFactory { +public: TPercentileFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - {} + {} -private: +private: const TString* GetGenericKey() const final { - return Column; - } - - void Join(IAggregation* aggr) final { - const auto percentile = dynamic_cast<TPercentileFactory*>(aggr); - Y_VERIFY(percentile); - Y_VERIFY(*Column == *percentile->Column); - Y_VERIFY(AggMode == percentile->AggMode); - Percentiles.insert(percentile->Percentiles.cbegin(), percentile->Percentiles.cend()); - percentile->Percentiles.clear(); - } - + return Column; + } + + void Join(IAggregation* aggr) final { + const auto percentile = dynamic_cast<TPercentileFactory*>(aggr); + Y_VERIFY(percentile); + Y_VERIFY(*Column == *percentile->Column); + Y_VERIFY(AggMode == percentile->AggMode); + Percentiles.insert(percentile->Percentiles.cbegin(), percentile->Percentiles.cend()); + percentile->Percentiles.clear(); + } + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; if (exprs.size() < 0 + adjustArgsCount || exprs.size() > 1 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires " << (0 + adjustArgsCount) << " or " << (1 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } - + return false; + } + if (!isFactory) { Column = exprs.front()->GetColumnName(); if (!Column) { ctx.Error(Pos) << Name << " may only be used with column reference as first argument."; return false; } - } - + } + if (!TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front()))) - return false; - + return false; + TNodePtr x; if (1 + adjustArgsCount == exprs.size()) { x = exprs.back(); if (!x->Init(ctx, FakeSource.Get())) { - return false; - } - } else { + return false; + } + } else { x = Y("Double", Q("0.5")); - } + } if (isFactory) { FactoryPercentile = x; @@ -661,116 +661,116 @@ private: Percentiles.emplace(Name, x); } - return true; - } - - TNodePtr DoClone() const final { - return new TPercentileFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { - TNodePtr percentiles(Percentiles.cbegin()->second); + return true; + } - if (Percentiles.size() > 1U) { - percentiles = Y(); - for (const auto& percentile : Percentiles) { - percentiles = L(percentiles, percentile.second); - } - percentiles = Q(percentiles); + TNodePtr DoClone() const final { + return new TPercentileFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TNodePtr percentiles(Percentiles.cbegin()->second); + + if (Percentiles.size() > 1U) { + percentiles = Y(); + for (const auto& percentile : Percentiles) { + percentiles = L(percentiles, percentile.second); + } + percentiles = Q(percentiles); } - return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), percentiles); - } + return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), percentiles); + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, FactoryPercentile); } - TNodePtr AggregationTraits(const TNodePtr& type) const final { - if (Percentiles.empty()) - return TNodePtr(); - - TNodePtr names(Q(Percentiles.cbegin()->first)); - - if (Percentiles.size() > 1U) { - names = Y(); - for (const auto& percentile : Percentiles) - names = L(names, Q(percentile.first)); - names = Q(names); - } - - const bool distinct = AggMode == EAggregateMode::Distinct; - const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; - return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); - } - - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& p : Percentiles) { - if (!p.second->Init(ctx, src)) { - return false; + TNodePtr AggregationTraits(const TNodePtr& type) const final { + if (Percentiles.empty()) + return TNodePtr(); + + TNodePtr names(Q(Percentiles.cbegin()->first)); + + if (Percentiles.size() > 1U) { + names = Y(); + for (const auto& percentile : Percentiles) + names = L(names, Q(percentile.first)); + names = Q(names); + } + + const bool distinct = AggMode == EAggregateMode::Distinct; + const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; + return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); + } + + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& p : Percentiles) { + if (!p.second->Init(ctx, src)) { + return false; } } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } TSourcePtr FakeSource; std::multimap<TString, TNodePtr> Percentiles; TNodePtr FactoryPercentile; const TString* Column = nullptr; -}; +}; TAggregationPtr BuildPercentileFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TPercentileFactory(pos, name, factory, aggMode); -} - -class TTopFreqFactory final : public TAggregationFactory { -public: + return new TPercentileFactory(pos, name, factory, aggMode); +} + +class TTopFreqFactory final : public TAggregationFactory { +public: TTopFreqFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) + : TAggregationFactory(pos, name, factory, aggMode) , FakeSource(BuildFakeSource(pos)) - {} - -private: - - //first - n, second - buffer - using TPair = std::pair<TNodePtr, TNodePtr>; - + {} + +private: + + //first - n, second - buffer + using TPair = std::pair<TNodePtr, TNodePtr>; + bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; - const double DefaultBufferC = 1.5; - const ui32 MinBuffer = 100; + const double DefaultBufferC = 1.5; + const ui32 MinBuffer = 100; if (exprs.size() < adjustArgsCount || exprs.size() > 2 + adjustArgsCount) { ctx.Error(Pos) << "Aggregation function " << (isFactory? "factory " : "") << Name << " requires " << adjustArgsCount << " to " << (2 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } - + return false; + } + if (!TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front()))) - return false; - + return false; + TNodePtr n = Y("Null"); TNodePtr buffer = Y("Null"); - + if (1 + adjustArgsCount <= exprs.size()) { n = exprs[adjustArgsCount]; if (!n->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } n = Y("SafeCast", n, Q("Uint32")); - } - + } + n = Y("Coalesce", n, Y("Uint32", Q("1"))); if (2 + adjustArgsCount == exprs.size()) { buffer = exprs[1 + adjustArgsCount]; if (!buffer->Init(ctx, FakeSource.Get())) { - return false; - } + return false; + } buffer = Y("SafeCast", buffer, Q("Uint32")); - } - + } + buffer = Y("Coalesce", buffer, Y("SafeCast", Y("*", n, Y("Double", Q(ToString(DefaultBufferC)))), Q("Uint32"))); buffer = Y("Coalesce", buffer, Y("Uint32", Q(ToString(MinBuffer)))); buffer = Y("Max", buffer, Y("Uint32", Q(ToString(MinBuffer)))); @@ -781,73 +781,73 @@ private: } else { TopFreqs.emplace(Name, x); } - - return true; - } - - TNodePtr DoClone() const final { - return new TTopFreqFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { - TPair topFreqs(TopFreqs.cbegin()->second); - - if (TopFreqs.size() > 1U) { - topFreqs = { Y(), Y() }; - for (const auto& topFreq : TopFreqs) { - topFreqs = { L(topFreqs.first, topFreq.second.first), L(topFreqs.second, topFreq.second.second) }; + + return true; + } + + TNodePtr DoClone() const final { + return new TTopFreqFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { + TPair topFreqs(TopFreqs.cbegin()->second); + + if (TopFreqs.size() > 1U) { + topFreqs = { Y(), Y() }; + for (const auto& topFreq : TopFreqs) { + topFreqs = { L(topFreqs.first, topFreq.second.first), L(topFreqs.second, topFreq.second.second) }; } - topFreqs = { Q(topFreqs.first), Q(topFreqs.second) }; + topFreqs = { Q(topFreqs.first), Q(topFreqs.second) }; } auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr), topFreqs.first, topFreqs.second); return apply; - } + } void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, TopFreqFactoryParams.first, TopFreqFactoryParams.second); } - TNodePtr AggregationTraits(const TNodePtr& type) const final { - if (TopFreqs.empty()) - return TNodePtr(); + TNodePtr AggregationTraits(const TNodePtr& type) const final { + if (TopFreqs.empty()) + return TNodePtr(); - TNodePtr names(Q(TopFreqs.cbegin()->first)); + TNodePtr names(Q(TopFreqs.cbegin()->first)); - if (TopFreqs.size() > 1U) { - names = Y(); - for (const auto& topFreq : TopFreqs) - names = L(names, Q(topFreq.first)); - names = Q(names); + if (TopFreqs.size() > 1U) { + names = Y(); + for (const auto& topFreq : TopFreqs) + names = L(names, Q(topFreq.first)); + names = Q(names); } - const bool distinct = AggMode == EAggregateMode::Distinct; - const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; - return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); - } - - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& topFreq : TopFreqs) { - if (!topFreq.second.first->Init(ctx, src)) { - return false; - } + const bool distinct = AggMode == EAggregateMode::Distinct; + const auto listType = distinct ? Y("ListType", Y("StructMemberType", Y("ListItemType", type), BuildQuotedAtom(Pos, DistinctKey))) : type; + return distinct ? Q(Y(names, GetApply(listType), BuildQuotedAtom(Pos, DistinctKey))) : Q(Y(names, GetApply(listType))); + } - if (!topFreq.second.second->Init(ctx, src)) { - return false; - } + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& topFreq : TopFreqs) { + if (!topFreq.second.first->Init(ctx, src)) { + return false; + } + + if (!topFreq.second.second->Init(ctx, src)) { + return false; + } } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } std::multimap<TString, TPair> TopFreqs; TPair TopFreqFactoryParams; TSourcePtr FakeSource; -}; - +}; + TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TTopFreqFactory(pos, name, factory, aggMode); -} + return new TTopFreqFactory(pos, name, factory, aggMode); +} template <bool HasKey> class TTopAggregationFactory final : public TAggregationFactory { @@ -1109,99 +1109,99 @@ TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, return new TListAggregationFactory(pos, name, factory, aggMode); } -class TUserDefinedAggregationFactory final : public TAggregationFactory { -public: +class TUserDefinedAggregationFactory final : public TAggregationFactory { +public: TUserDefinedAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) - : TAggregationFactory(pos, name, factory, aggMode) - {} + : TAggregationFactory(pos, name, factory, aggMode) + {} -private: +private: bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final { ui32 adjustArgsCount = isFactory ? 0 : 1; if (exprs.size() < (3 + adjustArgsCount) || exprs.size() > (7 + adjustArgsCount)) { ctx.Error(Pos) << "User defined aggregation function " << (isFactory ? "factory " : "") << " requires " << (3 + adjustArgsCount) << " to " << (7 + adjustArgsCount) << " arguments, given: " << exprs.size(); - return false; - } + return false; + } Lambdas[0] = BuildLambda(Pos, Y("value", "parent"), Y("NamedApply", exprs[adjustArgsCount], Q(Y("value")), Y("AsStruct"), Y("DependsOn", "parent"))); Lambdas[1] = BuildLambda(Pos, Y("value", "state", "parent"), Y("NamedApply", exprs[adjustArgsCount + 1], Q(Y("state", "value")), Y("AsStruct"), Y("DependsOn", "parent"))); Lambdas[2] = BuildLambda(Pos, Y("one", "two"), Y("Apply", exprs[adjustArgsCount + 2], "one", "two")); - for (size_t i = 3U; i < Lambdas.size(); ++i) { + for (size_t i = 3U; i < Lambdas.size(); ++i) { const auto j = adjustArgsCount + i; - Lambdas[i] = BuildLambda(Pos, Y("state"), j >= exprs.size() ? AstNode("state") : Y("Apply", exprs[j], "state")); - } + Lambdas[i] = BuildLambda(Pos, Y("state"), j >= exprs.size() ? AstNode("state") : Y("Apply", exprs[j], "state")); + } DefVal = (exprs.size() == (7 + adjustArgsCount)) ? exprs[adjustArgsCount + 6] : Y("Null"); return TAggregationFactory::InitAggr(ctx, isFactory, src, node, isFactory ? TVector<TNodePtr>() : TVector<TNodePtr>(1, exprs.front())); - } - - TNodePtr DoClone() const final { - return new TUserDefinedAggregationFactory(Pos, Name, Func, AggMode); - } - - TNodePtr GetApply(const TNodePtr& type) const final { + } + + TNodePtr DoClone() const final { + return new TUserDefinedAggregationFactory(Pos, Name, Func, AggMode); + } + + TNodePtr GetApply(const TNodePtr& type) const final { auto apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Expr)); AddFactoryArguments(apply); return apply; - } - + } + void AddFactoryArguments(TNodePtr& apply) const final { apply = L(apply, Lambdas[0], Lambdas[1], Lambdas[2], Lambdas[3], Lambdas[4], Lambdas[5], DefVal); } - bool DoInit(TContext& ctx, ISource* src) final { - for (const auto& lambda : Lambdas) { - if (!lambda->Init(ctx, src)) { - return false; - } + bool DoInit(TContext& ctx, ISource* src) final { + for (const auto& lambda : Lambdas) { + if (!lambda->Init(ctx, src)) { + return false; + } } - + if (!DefVal->Init(ctx, src)) { return false; } - return TAggregationFactory::DoInit(ctx, src); + return TAggregationFactory::DoInit(ctx, src); } - std::array<TNodePtr, 6> Lambdas; + std::array<TNodePtr, 6> Lambdas; TNodePtr DefVal; -}; +}; TAggregationPtr BuildUserDefinedFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) { - return new TUserDefinedAggregationFactory(pos, name, factory, aggMode); -} + return new TUserDefinedAggregationFactory(pos, name, factory, aggMode); +} -class TCountAggregation final : public TAggregationFactory { -public: +class TCountAggregation final : public TAggregationFactory { +public: TCountAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) - : TAggregationFactory(pos, name, func, aggMode) - {} + : TAggregationFactory(pos, name, func, aggMode) + {} -private: - TNodePtr DoClone() const final { - return new TCountAggregation(Pos, Name, Func, AggMode); - } +private: + TNodePtr DoClone() const final { + return new TCountAggregation(Pos, Name, Func, AggMode); + } - bool DoInit(TContext& ctx, ISource* src) final { + bool DoInit(TContext& ctx, ISource* src) final { if (!Expr) { return true; } if (Expr->IsAsterisk()) { - Expr = Y("Void"); + Expr = Y("Void"); } if (!Expr->Init(ctx, src)) { return false; } - Expr->SetCountHint(Expr->IsConstant()); - return TAggregationFactory::DoInit(ctx, src); + Expr->SetCountHint(Expr->IsConstant()); + return TAggregationFactory::DoInit(ctx, src); } -}; +}; TAggregationPtr BuildCountAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode) { - return new TCountAggregation(pos, name, func, aggMode); -} + return new TCountAggregation(pos, name, func, aggMode); +} } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index 43ef1243f10..e9f2500e26e 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -2286,13 +2286,13 @@ private: enum EAggrFuncTypeCallback { NORMAL, KEY_PAYLOAD, - PAYLOAD_PREDICATE, + PAYLOAD_PREDICATE, TWO_ARGS, COUNT, HISTOGRAM, LINEAR_HISTOGRAM, PERCENTILE, - TOPFREQ, + TOPFREQ, TOP, TOP_BY, COUNT_DISTINCT_ESTIMATE, @@ -2342,9 +2342,9 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case KEY_PAYLOAD: factory = BuildKeyPayloadFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case PAYLOAD_PREDICATE: - factory = BuildPayloadPredicateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); - break; + case PAYLOAD_PREDICATE: + factory = BuildPayloadPredicateFactoryAggregation(pos, realFunctionName, factoryName, aggMode); + break; case TWO_ARGS: factory = BuildTwoArgsFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; @@ -2360,9 +2360,9 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback( case PERCENTILE: factory = BuildPercentileFactoryAggregation(pos, realFunctionName, factoryName, aggMode); break; - case TOPFREQ: - factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); - break; + case TOPFREQ: + factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode); + break; case TOP: factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode); break; @@ -2746,8 +2746,8 @@ struct TBuiltinFuncData { {"maxby", BuildAggrFuncFactoryCallback("MaxBy", "max_by_traits_factory", KEY_PAYLOAD)}, {"sum", BuildAggrFuncFactoryCallback("Sum", "sum_traits_factory")}, - {"sumif", BuildAggrFuncFactoryCallback("SumIf", "sum_if_traits_factory", PAYLOAD_PREDICATE) }, - + {"sumif", BuildAggrFuncFactoryCallback("SumIf", "sum_if_traits_factory", PAYLOAD_PREDICATE) }, + {"some", BuildAggrFuncFactoryCallback("Some", "some_traits_factory")}, {"somevalue", BuildAggrFuncFactoryCallback("SomeValue", "some_traits_factory")}, @@ -2764,8 +2764,8 @@ struct TBuiltinFuncData { {"bitxor", BuildAggrFuncFactoryCallback("BitXor", "bit_xor_traits_factory")}, {"avg", BuildAggrFuncFactoryCallback("Avg", "avg_traits_factory")}, - {"avgif", BuildAggrFuncFactoryCallback("AvgIf", "avg_if_traits_factory", PAYLOAD_PREDICATE) }, - + {"avgif", BuildAggrFuncFactoryCallback("AvgIf", "avg_if_traits_factory", PAYLOAD_PREDICATE) }, + {"agglist", BuildAggrFuncFactoryCallback("AggregateList", "list2_traits_factory", LIST)}, {"aggrlist", BuildAggrFuncFactoryCallback("AggregateList", "list2_traits_factory", LIST)}, {"aggregatelist", BuildAggrFuncFactoryCallback("AggregateList", "list2_traits_factory", LIST)}, @@ -2776,9 +2776,9 @@ struct TBuiltinFuncData { {"median", BuildAggrFuncFactoryCallback("Median", "percentile_traits_factory", PERCENTILE)}, {"percentile", BuildAggrFuncFactoryCallback("Percentile", "percentile_traits_factory", PERCENTILE)}, - {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, - {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, - + {"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) }, + {"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) }, + {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)}, {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)}, {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)}, diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 725356b9f57..a9c30caa6bb 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -66,7 +66,7 @@ public: key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, View))))); } if (mode == ITableKeys::EBuildKeysMode::INPUT && - IsQueryMode(ctx.Settings.Mode) && + IsQueryMode(ctx.Settings.Mode) && Service != KikimrProviderName && Service != RtmrProviderName && Service != YdbProviderName) { @@ -1740,7 +1740,7 @@ public: } else { const auto& ref = block->GetLabel(); Add(Y("let", ref ? ref : "world", block)); - } + } } if (TopLevel) { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index ab5a8647885..7602d7f6fb9 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -8165,7 +8165,7 @@ private: void AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params); TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success); void AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node); - + TNodePtr Build(const TRule_delete_stmt& stmt); TNodePtr Build(const TRule_update_stmt& stmt); @@ -8292,9 +8292,9 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda void TSqlQuery::AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node) { blocks.emplace_back(node); -} - - +} + + bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& core) { TString internalStatementName; TString humanStatementName; @@ -8321,7 +8321,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } if (nodeExpr) { - AddStatementToBlocks(blocks, nodeExpr); + AddStatementToBlocks(blocks, nodeExpr); } break; } @@ -8449,7 +8449,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& if (!updateNode) { return false; } - AddStatementToBlocks(blocks, updateNode); + AddStatementToBlocks(blocks, updateNode); break; } case TRule_sql_stmt_core::kAltSqlStmtCore10: { @@ -9872,8 +9872,8 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { return nullptr; } - if (!Ctx.PragmaAutoCommit && Ctx.Settings.EndOfQueryCommit && IsQueryMode(Mode)) { - AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); + if (!Ctx.PragmaAutoCommit && Ctx.Settings.EndOfQueryCommit && IsQueryMode(Mode)) { + AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos())); } auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped); @@ -10109,7 +10109,7 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TTranslationSettings& settings) { - YQL_ENSURE(IsQueryMode(settings.Mode)); + YQL_ENSURE(IsQueryMode(settings.Mode)); TAstParseResult res; TContext ctx(settings, res.Issues); SqlASTToYqlImpl(res, protoAst, ctx); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 85b6e0dfcd3..0acd86fd66a 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -33,7 +33,7 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES settings.ClusterMapping[cluster] = service; settings.ClusterMapping["hahn"] = NYql::YtProviderName; settings.MaxErrors = maxErrors; - settings.Mode = mode; + settings.Mode = mode; settings.Arena = &arena; settings.AnsiLexer = ansiLexer; settings.SyntaxVersion = 1; @@ -44,10 +44,10 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES return res; } -NYql::TAstParseResult SqlToYql(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { +NYql::TAstParseResult SqlToYql(const TString& query, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) { return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, maxErrors, provider, debug); -} - +} + NYql::TAstParseResult SqlToYqlWithSettings(const TString& query, const NSQLTranslation::TTranslationSettings& settings) { return SqlToYqlWithMode(query, NSQLTranslation::ESqlMode::QUERY, 10, {}, EDebugOutput::None, false, settings); } @@ -82,19 +82,19 @@ TString Quote(const char* str) { } class TWordCountHive: public TMap<TString, unsigned> { -public: - TWordCountHive(std::initializer_list<TString> strings) { - for (auto& str: strings) { - emplace(str, 0); - } - } - - TWordCountHive(std::initializer_list<std::pair<const TString, unsigned>> list) +public: + TWordCountHive(std::initializer_list<TString> strings) { + for (auto& str: strings) { + emplace(str, 0); + } + } + + TWordCountHive(std::initializer_list<std::pair<const TString, unsigned>> list) : TMap(list) - { - } -}; - + { + } +}; + typedef std::function<void (const TString& word, const TString& line)> TVerifyLineFunc; TString VerifyProgram(const NYql::TAstParseResult& res, TWordCountHive& wordCounter, TVerifyLineFunc verifyLine = TVerifyLineFunc()) { diff --git a/ydb/library/yql/udfs/common/topfreq/static/static_udf.cpp b/ydb/library/yql/udfs/common/topfreq/static/static_udf.cpp index 4075bfa9c2b..0aab31a2f43 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/static_udf.cpp +++ b/ydb/library/yql/udfs/common/topfreq/static/static_udf.cpp @@ -1,10 +1,10 @@ -#include "topfreq_udf.h" - +#include "topfreq_udf.h" + namespace NYql { - namespace NUdf { - NUdf::TUniquePtr<NUdf::IUdfModule> CreateTopFreqModule() { - return new TTopFreqModule(); - } - - } -} + namespace NUdf { + NUdf::TUniquePtr<NUdf::IUdfModule> CreateTopFreqModule() { + return new TTopFreqModule(); + } + + } +} diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp index 31fcd88b547..cd891f41b07 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp @@ -1,10 +1,10 @@ -#include "topfreq.h" -#include <cmath> -#include <algorithm> - -using namespace NKikimr; -using namespace NUdf; - +#include "topfreq.h" +#include <cmath> +#include <algorithm> + +using namespace NKikimr; +using namespace NUdf; + template <typename THash, typename TEquals> TTopFreqBase<THash, TEquals>::TTopFreqBase(THash hash, TEquals equals) : Indices_(0, hash, equals) @@ -15,12 +15,12 @@ void TTopFreqBase<THash, TEquals>::Init(const TUnboxedValuePod& value, const ui3 MinSize_ = minSize; MaxSize_ = maxSize; - Freqs_.reserve(MaxSize_ + 1); - Indices_.reserve(MaxSize_ + 1); + Freqs_.reserve(MaxSize_ + 1); + Indices_.reserve(MaxSize_ + 1); AddValue(value); -} - +} + template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::Merge(const TTopFreqBase& topFreq1, const TTopFreqBase& topFreq2) { MinSize_ = std::max(topFreq1.MinSize_, topFreq2.MinSize_); @@ -38,15 +38,15 @@ void TTopFreqBase<THash, TEquals>::Deserialize(const TUnboxedValuePod& serialize MinSize_ = serialized.GetElement(0).Get<ui32>(); MaxSize_ = serialized.GetElement(1).Get<ui32>(); - Freqs_.reserve(MaxSize_ + 1); - Indices_.reserve(MaxSize_ + 1); + Freqs_.reserve(MaxSize_ + 1); + Indices_.reserve(MaxSize_ + 1); const auto listIter = serialized.GetElement(2).GetListIterator(); for (TUnboxedValue current; listIter.Next(current);) { Update(current.GetElement(1), current.GetElement(0).Get<ui64>()); - } -} - + } +} + template <typename THash, typename TEquals> TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBuilder) const { TUnboxedValue* values = nullptr; @@ -63,77 +63,77 @@ TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBu template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) { for (auto& it : otherModeCalc.Freqs_) { - Update(it.first, it.second); - } - - TryCompress(); -} - + Update(it.first, it.second); + } + + TryCompress(); +} + template <typename THash, typename TEquals> TUnboxedValue TTopFreqBase<THash, TEquals>::Get(const IValueBuilder* builder, ui32 resultSize) { - resultSize = std::min(resultSize, ui32(Freqs_.size())); - Compress(resultSize, true); + resultSize = std::min(resultSize, ui32(Freqs_.size())); + Compress(resultSize, true); return Convert(builder); -} - +} + template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::AddValue(const TUnboxedValuePod& value) { - Update(value, 1); - TryCompress(); -} - + Update(value, 1); + TryCompress(); +} + template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 freq) { - Freqs_.emplace_back(TUnboxedValuePod(value), freq); + Freqs_.emplace_back(TUnboxedValuePod(value), freq); auto mapInsertResult = Indices_.emplace(TUnboxedValuePod(value), Freqs_.size() - 1); - - if (!mapInsertResult.second) { - Freqs_[mapInsertResult.first->second].second += freq; - Freqs_.pop_back(); - } -} - + + if (!mapInsertResult.second) { + Freqs_[mapInsertResult.first->second].second += freq; + Freqs_.pop_back(); + } +} + template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::TryCompress() { - ui32 freqSize = Freqs_.size(); - if (freqSize > MaxSize_) { - Compress(MinSize_); - } -} - + ui32 freqSize = Freqs_.size(); + if (freqSize > MaxSize_) { + Compress(MinSize_); + } +} + template <typename THash, typename TEquals> void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) { - auto compare = [](const TVectorElement& v1, const TVectorElement& v2) { - return v1.second > v2.second; - }; - - if (sort) { - std::sort(Freqs_.begin(), Freqs_.end(), compare); - } else { - std::nth_element(Freqs_.begin(), Freqs_.begin() + newSize - 1, Freqs_.end(), compare); - } - - Indices_.clear(); - Freqs_.resize(newSize); - - for (ui32 i = 0; i < newSize; i++) { + auto compare = [](const TVectorElement& v1, const TVectorElement& v2) { + return v1.second > v2.second; + }; + + if (sort) { + std::sort(Freqs_.begin(), Freqs_.end(), compare); + } else { + std::nth_element(Freqs_.begin(), Freqs_.begin() + newSize - 1, Freqs_.end(), compare); + } + + Indices_.clear(); + Freqs_.resize(newSize); + + for (ui32 i = 0; i < newSize; i++) { Indices_[Freqs_[i].first] = i; - } -} - + } +} + template <typename THash, typename TEquals> TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* builder) { - if (ui32(Freqs_.size()) > MinSize_) { - Compress(MinSize_); - } - + if (ui32(Freqs_.size()) > MinSize_) { + Compress(MinSize_); + } + TUnboxedValue* items = nullptr; auto tuple = builder->NewArray(3U, items); items[0] = TUnboxedValuePod(MinSize_); items[1] = TUnboxedValuePod(MaxSize_); items[2] = Convert(builder); return tuple; -} +} template <EDataSlot Slot> TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h index 411190d2571..57c4491e881 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <ydb/library/yql/public/udf/udf_allocator.h> #include <ydb/library/yql/public/udf/udf_helpers.h> @@ -10,23 +10,23 @@ template <typename THash, typename TEquals> class TTopFreqBase { protected: using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod; - using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; + using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; using IValueBuilder = NKikimr::NUdf::IValueBuilder; - using TVectorElement = std::pair<TUnboxedValue, ui64>; + using TVectorElement = std::pair<TUnboxedValue, ui64>; using TVectorType = std::vector<TVectorElement, NKikimr::NUdf::TStdAllocatorForUdf<TVectorElement>>; - - TVectorType Freqs_; + + TVectorType Freqs_; std::unordered_map<TUnboxedValue, ui32, THash, TEquals, NKikimr::NUdf::TStdAllocatorForUdf<std::pair<const TUnboxedValue, ui32>>> Indices_; ui32 MinSize_ = 0; ui32 MaxSize_ = 0; - + void Add(const TTopFreqBase& otherCalc); void Update(const TUnboxedValuePod& key, const ui64 value); - void TryCompress(); - void Compress(ui32 newSize, bool sort = false); + void TryCompress(); + void Compress(ui32 newSize, bool sort = false); TUnboxedValue Convert(const IValueBuilder* valueBuilder) const; - + protected: TTopFreqBase(THash hash, TEquals equals); @@ -37,7 +37,7 @@ protected: TUnboxedValue Serialize(const IValueBuilder* builder); TUnboxedValue Get(const IValueBuilder* builder, ui32 resultSize); void AddValue(const TUnboxedValuePod& value); -}; +}; template <NKikimr::NUdf::EDataSlot Slot> class TTopFreqData diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h index 5ea0a5f7294..9789d2f4c6f 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h +++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/udf/udf_helpers.h> #include <ydb/library/yql/public/udf/udf_value_builder.h> #include <ydb/library/yql/public/udf/udf_registrator.h> @@ -7,14 +7,14 @@ #include <ydb/library/yql/public/udf/udf_type_inspection.h> #include <ydb/library/yql/public/udf/udf_value.h> #include <ydb/library/yql/public/udf/udf_types.h> -#include "topfreq.h" -#include <algorithm> -#include <array> - +#include "topfreq.h" +#include <algorithm> +#include <array> + using namespace NYql; -using namespace NUdf; - -namespace { +using namespace NUdf; + +namespace { extern const char TopFreqResourceNameGeneric[] = "TopFreq.TopFreqResource.Generic"; class TTopFreqResource: public TBoxedResource<TTopFreqGeneric, TopFreqResourceNameGeneric> @@ -25,16 +25,16 @@ namespace { : TBoxedResource(std::forward<Args>(args)...) {} }; - + template <EDataSlot Slot> class TTopFreqResourceData; - + template <EDataSlot Slot> TTopFreqResourceData<Slot>* GetTopFreqResourceData(const TUnboxedValuePod& arg) { TTopFreqResourceData<Slot>::Validate(arg); return static_cast<TTopFreqResourceData<Slot>*>(arg.AsBoxed().Get()); - } - + } + TTopFreqResource* GetTopFreqResource(const TUnboxedValuePod& arg) { TTopFreqResource::Validate(arg); return static_cast<TTopFreqResource*>(arg.AsBoxed().Get()); @@ -53,9 +53,9 @@ namespace { class TTopFreq_Create: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { - ui32 minSize = args[1].Get<ui32>(); + ui32 minSize = args[1].Get<ui32>(); return TUnboxedValuePod(new TTopFreqResource(args[0], minSize, minSize * 2, Hash_, Equate_)); - } + } public: TTopFreq_Create(IHash::TPtr hash, IEquate::TPtr equate) @@ -75,16 +75,16 @@ namespace { const auto topFreq = GetTopFreqResourceData<Slot>(args[0]); topFreq->Get()->AddValue(args[1]); return TUnboxedValuePod(topFreq); - } - }; - + } + }; + class TTopFreq_AddValue: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { const auto topFreq = GetTopFreqResource(args[0]); - topFreq->Get()->AddValue(args[1]); + topFreq->Get()->AddValue(args[1]); return TUnboxedValuePod(topFreq); - } + } }; template <EDataSlot Slot> @@ -92,14 +92,14 @@ namespace { private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { return GetTopFreqResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder); - } - }; - + } + }; + class TTopFreq_Serialize: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { return GetTopFreqResource(args[0])->Get()->Serialize(valueBuilder); - } + } }; template <EDataSlot Slot> @@ -107,14 +107,14 @@ namespace { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0])); - } - }; - + } + }; + class TTopFreq_Deserialize: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { return TUnboxedValuePod(new TTopFreqResource(args[0], Hash_, Equate_)); - } + } public: TTopFreq_Deserialize(IHash::TPtr hash, IEquate::TPtr equate) @@ -134,16 +134,16 @@ namespace { const auto topFreq0 = GetTopFreqResourceData<Slot>(args[0]); const auto topFreq1 = GetTopFreqResourceData<Slot>(args[1]); return TUnboxedValuePod(new TTopFreqResourceData<Slot>(*topFreq0->Get(), *topFreq1->Get())); - } - }; - + } + }; + class TTopFreq_Merge: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const { const auto topFreq0 = GetTopFreqResource(args[0]); const auto topFreq1 = GetTopFreqResource(args[1]); return TUnboxedValuePod(new TTopFreqResource(*topFreq0->Get(), *topFreq1->Get(), Hash_, Equate_)); - } + } public: TTopFreq_Merge(IHash::TPtr hash, IEquate::TPtr equate) @@ -154,23 +154,23 @@ namespace { private: IHash::TPtr Hash_; IEquate::TPtr Equate_; - }; - + }; + template <EDataSlot Slot> class TTopFreq_GetData: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { return GetTopFreqResourceData<Slot>(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); - } + } }; class TTopFreq_Get: public TBoxedValue { private: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { return GetTopFreqResource(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>()); - } - }; - + } + }; + #define MAKE_RESOURCE(slot, ...) \ extern const char TopFreqResourceName##slot[] = "TopFreq.TopFreqResource."#slot; \ @@ -213,14 +213,14 @@ namespace { static const auto GetName = TStringRef::Of("TopFreq_Get"); class TTopFreqModule: public IUdfModule { - public: - TStringRef Name() const { - return TStringRef::Of("TopFreq"); - } - + public: + TStringRef Name() const { + return TStringRef::Of("TopFreq"); + } + void CleanupOnTerminate() const final { } - + void GetAllFunctions(IFunctionsSink& sink) const final { sink.Add(CreateName)->SetTypeAwareness(); sink.Add(AddValueName)->SetTypeAwareness(); @@ -228,21 +228,21 @@ namespace { sink.Add(DeserializeName)->SetTypeAwareness(); sink.Add(MergeName)->SetTypeAwareness(); sink.Add(GetName)->SetTypeAwareness(); - } - - void BuildFunctionTypeInfo( - const TStringRef& name, + } + + void BuildFunctionTypeInfo( + const TStringRef& name, TType* userType, - const TStringRef& typeConfig, - ui32 flags, + const TStringRef& typeConfig, + ui32 flags, IFunctionTypeInfoBuilder& builder) const final { - Y_UNUSED(typeConfig); - - try { + Y_UNUSED(typeConfig); + + try { const bool typesOnly = (flags & TFlags::TypesOnly); builder.UserType(userType); - + auto typeHelper = builder.TypeInfoHelper(); auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType); @@ -293,8 +293,8 @@ namespace { if (name == CreateName) { builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topFreqType); - - if (!typesOnly) { + + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_Create(hash, equate)); } else { @@ -302,13 +302,13 @@ namespace { UDF_TYPE_ID_MAP(MAKE_CREATE) } } - } - } - + } + } + if (name == AddValueName) { builder.Args()->Add(topFreqType).Add(valueType).Done().Returns(topFreqType); - - if (!typesOnly) { + + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_AddValue); } else { @@ -316,13 +316,13 @@ namespace { UDF_TYPE_ID_MAP(MAKE_ADD_VALUE) } } - } - } - + } + } + if (name == MergeName) { - builder.Args()->Add(topFreqType).Add(topFreqType).Done().Returns(topFreqType); + builder.Args()->Add(topFreqType).Add(topFreqType).Done().Returns(topFreqType); - if (!typesOnly) { + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_Merge(hash, equate)); } else { @@ -330,13 +330,13 @@ namespace { UDF_TYPE_ID_MAP(MAKE_MERGE) } } - } - } - + } + } + if (name == SerializeName) { builder.Args()->Add(topFreqType).Done().Returns(serializedType); - - if (!typesOnly) { + + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_Serialize); } else { @@ -344,13 +344,13 @@ namespace { UDF_TYPE_ID_MAP(MAKE_SERIALIZE) } } - } - } - + } + } + if (name == DeserializeName) { builder.Args()->Add(serializedType).Done().Returns(topFreqType); - - if (!typesOnly) { + + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_Deserialize(hash, equate)); } else { @@ -358,17 +358,17 @@ namespace { UDF_TYPE_ID_MAP(MAKE_DESERIALIZE) } } - } - } - + } + } + if (name == GetName) { ui32 indexF, indexV; auto itemType = builder.Struct()->AddField<ui64>("Frequency", &indexF).AddField("Value", valueType, &indexV).Build(); auto resultType = builder.List()->Item(itemType).Build(); - + builder.Args()->Add(topFreqType).Add<ui32>().Done().Returns(resultType); - - if (!typesOnly) { + + if (!typesOnly) { if (isGeneric) { builder.Implementation(new TTopFreq_Get); } else { @@ -376,13 +376,13 @@ namespace { UDF_TYPE_ID_MAP(MAKE_GET) } } - } - } + } + } } catch (const std::exception& e) { - builder.SetError(CurrentExceptionMessage()); - } - } - }; + builder.SetError(CurrentExceptionMessage()); + } + } + }; } // namespace diff --git a/ydb/library/yql/udfs/common/topfreq/static/ya.make b/ydb/library/yql/udfs/common/topfreq/static/ya.make index ce5d98cb6b2..2f9b6c8078a 100644 --- a/ydb/library/yql/udfs/common/topfreq/static/ya.make +++ b/ydb/library/yql/udfs/common/topfreq/static/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + YQL_ABI_VERSION( 2 10 0 -) - +) + OWNER( g:yql g:yql_ydb_core ) -SRCS( - static_udf.cpp - topfreq.cpp -) - -PEERDIR( +SRCS( + static_udf.cpp + topfreq.cpp +) + +PEERDIR( ydb/library/yql/public/udf -) - -END() +) + +END() diff --git a/ydb/library/yql/udfs/common/topfreq/topfreq_udf.cpp b/ydb/library/yql/udfs/common/topfreq/topfreq_udf.cpp index 7107f2bd006..ed95c9f6613 100644 --- a/ydb/library/yql/udfs/common/topfreq/topfreq_udf.cpp +++ b/ydb/library/yql/udfs/common/topfreq/topfreq_udf.cpp @@ -1,3 +1,3 @@ -#include "static/topfreq_udf.h" - -REGISTER_MODULES(TTopFreqModule) +#include "static/topfreq_udf.h" + +REGISTER_MODULES(TTopFreqModule) diff --git a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp index c6f6ecab72a..1c881f3c24c 100644 --- a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp +++ b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp @@ -5,16 +5,16 @@ #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> -#include <util/random/random.h> -#include <array> +#include <util/random/random.h> +#include <array> #include <ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h> - + namespace NYql { using namespace NKikimr::NMiniKQL; - namespace NUdf { - extern NUdf::TUniquePtr<NUdf::IUdfModule> CreateTopFreqModule(); - } - + namespace NUdf { + extern NUdf::TUniquePtr<NUdf::IUdfModule> CreateTopFreqModule(); + } + class TSetup { public: TSetup() @@ -27,28 +27,28 @@ namespace NYql { MutableFunctionRegistry_->AddModule("", "TopFreq", NUdf::CreateTopFreqModule()); PgmBuidler_.Reset(new TProgramBuilder(Env_, *MutableFunctionRegistry_)); } - + TProgramBuilder& GetProgramBuilder() { return *PgmBuidler_.Get(); } - + NUdf::TUnboxedValue GetValue(TRuntimeNode& node) { Explorer_.Walk(node.GetNode(), Env_); - + TComputationPatternOpts opts(Alloc_.Ref(), Env_, GetBuiltinFactory(), MutableFunctionRegistry_.Get(), NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "", EGraphPerProcess::Multi); Pattern_ = MakeComputationPattern(Explorer_, node, {}, opts); Graph_ = Pattern_->Clone(opts.ToComputationOptions(*RandomProvider_, *TimeProvider_)); - + return Graph_->GetValue(); } - + private: using IMutableFunctionRegistryPtr = TIntrusivePtr<IMutableFunctionRegistry>; using IRandomProviderPtr = TIntrusivePtr<IRandomProvider>; using ITimeProviderPtr = TIntrusivePtr<ITimeProvider>; - + IMutableFunctionRegistryPtr MutableFunctionRegistry_; IRandomProviderPtr RandomProvider_; ITimeProviderPtr TimeProvider_; @@ -59,25 +59,25 @@ namespace NYql { THolder<IComputationGraph> Graph_; TExploringNodeVisitor Explorer_; }; - + Y_UNIT_TEST_SUITE(TUDFTopFreqTest) { Y_UNIT_TEST(SimpleTopFreq) { TSetup setup; TProgramBuilder& pgmBuilder = setup.GetProgramBuilder(); - + const auto valueType = pgmBuilder.NewDataType(NUdf::TDataType<i32>::Id); const auto emptyStructType = pgmBuilder.NewEmptyStructType(); const auto resourceType = pgmBuilder.NewResourceType("TopFreq.TopFreqResource.Int32"); const auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); - + const auto createArgsType = pgmBuilder.NewTupleType({valueType, ui32Type}); const auto createUserType = pgmBuilder.NewTupleType({createArgsType, emptyStructType, valueType}); auto udfTopFreq_Create = pgmBuilder.Udf("TopFreq.TopFreq_Create", TRuntimeNode(), createUserType); - + auto addValueArgsType = pgmBuilder.NewTupleType({resourceType, valueType}); auto addValueUserType = pgmBuilder.NewTupleType({addValueArgsType, emptyStructType, valueType}); auto udfTopFreq_AddValue = pgmBuilder.Udf("TopFreq.TopFreq_AddValue", TRuntimeNode(), addValueUserType); - + auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type}); auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType}); auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType); @@ -86,34 +86,34 @@ namespace NYql { { auto val = pgmBuilder.NewDataLiteral<i32>(3); auto param = pgmBuilder.NewDataLiteral<ui32>(10); - + TVector<TRuntimeNode> params = {val, param}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_Create, params); } - + for (int n = 0; n < 9; n++) { auto value = pgmBuilder.NewDataLiteral<i32>(1); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + for (int n = 0; n < 7; n++) { auto value = pgmBuilder.NewDataLiteral<i32>(4); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + TRuntimeNode pgmReturn; { auto param = pgmBuilder.NewDataLiteral<ui32>(4); TVector<TRuntimeNode> params = {pgmTopFreq, param}; pgmReturn = pgmBuilder.Apply(udfTopFreq_Get, params); } - + auto value = setup.GetValue(pgmReturn); - + auto listIterator = value.GetListIterator(); - + TUnboxedValue item; UNIT_ASSERT(listIterator.Next(item)); @@ -130,24 +130,24 @@ namespace NYql { UNIT_ASSERT(!listIterator.Next(item)); } - + Y_UNIT_TEST(MergingTopFreq) { TSetup setup; TProgramBuilder& pgmBuilder = setup.GetProgramBuilder(); - + const auto valueType = pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); const auto emptyStructType = pgmBuilder.NewEmptyStructType(); const auto resourceType = pgmBuilder.NewResourceType("TopFreq.TopFreqResource.Uint64"); const auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); - + const auto createArgsType = pgmBuilder.NewTupleType({valueType, ui32Type}); const auto createUserType = pgmBuilder.NewTupleType({createArgsType, emptyStructType, valueType}); auto udfTopFreq_Create = pgmBuilder.Udf("TopFreq.TopFreq_Create", TRuntimeNode(), createUserType); - + auto addValueArgsType = pgmBuilder.NewTupleType({resourceType, valueType}); auto addValueUserType = pgmBuilder.NewTupleType({addValueArgsType, emptyStructType, valueType}); auto udfTopFreq_AddValue = pgmBuilder.Udf("TopFreq.TopFreq_AddValue", TRuntimeNode(), addValueUserType); - + auto mergeArgsType = pgmBuilder.NewTupleType({resourceType, resourceType}); auto mergeUserType = pgmBuilder.NewTupleType({mergeArgsType, emptyStructType, valueType}); auto udfTopFreq_Merge = pgmBuilder.Udf("TopFreq.TopFreq_Merge", TRuntimeNode(), mergeUserType); @@ -163,25 +163,25 @@ namespace NYql { TVector<TRuntimeNode> params = {value, param}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_Create, params); } - + for (int n = 0; n < 1; n++) { auto value = pgmBuilder.NewDataLiteral<ui64>(1); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + for (int n = 0; n < 4; n++) { auto value = pgmBuilder.NewDataLiteral<ui64>(5); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + for (int n = 0; n < 1; n++) { auto value = pgmBuilder.NewDataLiteral<ui64>(3); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + TRuntimeNode pgmTopFreq2; { auto value = pgmBuilder.NewDataLiteral<ui64>(1); @@ -189,36 +189,36 @@ namespace NYql { TVector<TRuntimeNode> params = {value, param}; pgmTopFreq2 = pgmBuilder.Apply(udfTopFreq_Create, params); } - + for (int n = 0; n < 5; n++) { auto value = pgmBuilder.NewDataLiteral<ui64>(1); TVector<TRuntimeNode> params = {pgmTopFreq2, value}; pgmTopFreq2 = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + for (int n = 0; n < 5; n++) { auto value = pgmBuilder.NewDataLiteral<ui64>(5); TVector<TRuntimeNode> params = {pgmTopFreq2, value}; pgmTopFreq2 = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + TRuntimeNode pgmTopFreq3; { TVector<TRuntimeNode> params = {pgmTopFreq, pgmTopFreq2}; pgmTopFreq3 = pgmBuilder.Apply(udfTopFreq_Merge, params); } - + TRuntimeNode pgmReturn; { auto param = pgmBuilder.NewDataLiteral<ui32>(1); TVector<TRuntimeNode> params = {pgmTopFreq3, param}; pgmReturn = pgmBuilder.Apply(udfTopFreq_Get, params); } - + auto value = setup.GetValue(pgmReturn); - + auto listIterator = value.GetListIterator(); - + TUnboxedValue item; UNIT_ASSERT(listIterator.Next(item)); @@ -227,17 +227,17 @@ namespace NYql { UNIT_ASSERT(!listIterator.Next(item)); } - + Y_UNIT_TEST(SerializedTopFreq) { TSetup setup; TProgramBuilder& pgmBuilder = setup.GetProgramBuilder(); - + const auto valueType = pgmBuilder.NewDataType(NUdf::TDataType<bool>::Id); const auto emptyStructType = pgmBuilder.NewEmptyStructType(); const auto resourceType = pgmBuilder.NewResourceType("TopFreq.TopFreqResource.Bool"); const auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); const auto ui64Type = pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); - + const auto createArgsType = pgmBuilder.NewTupleType({valueType, ui32Type}); const auto createUserType = pgmBuilder.NewTupleType({createArgsType, emptyStructType, valueType}); auto udfTopFreq_Create = pgmBuilder.Udf("TopFreq.TopFreq_Create", TRuntimeNode(), createUserType); @@ -268,42 +268,42 @@ namespace NYql { TVector<TRuntimeNode> params = {value, param}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_Create, params); } - + for (int n = 0; n < 7; n++) { auto value = pgmBuilder.NewDataLiteral<bool>(true); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + for (int n = 0; n < 10; n++) { auto value = pgmBuilder.NewDataLiteral<bool>(false); TVector<TRuntimeNode> params = {pgmTopFreq, value}; pgmTopFreq = pgmBuilder.Apply(udfTopFreq_AddValue, params); } - + TRuntimeNode pgmSerializedTopFreq; { TVector<TRuntimeNode> params = {pgmTopFreq}; pgmSerializedTopFreq = pgmBuilder.Apply(udfTopFreq_Serialize, params); } - + TRuntimeNode pgmDeserializedTopFreq; { TVector<TRuntimeNode> params = {pgmSerializedTopFreq}; pgmDeserializedTopFreq = pgmBuilder.Apply(udfTopFreq_Deserialize, params); } - + TRuntimeNode pgmReturn; { auto param = pgmBuilder.NewDataLiteral<ui32>(3); TVector<TRuntimeNode> params = {pgmDeserializedTopFreq, param}; pgmReturn = pgmBuilder.Apply(udfTopFreq_Get, params); } - + auto value = setup.GetValue(pgmReturn); - + auto listIterator = value.GetListIterator(); - + TUnboxedValue item; UNIT_ASSERT(listIterator.Next(item)); @@ -316,21 +316,21 @@ namespace NYql { UNIT_ASSERT(!listIterator.Next(item)); } - + Y_UNIT_TEST(ApproxTopFreq) { TSetup setup; TProgramBuilder& pgmBuilder = setup.GetProgramBuilder(); - + const auto valueType = pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); const auto emptyStructType = pgmBuilder.NewEmptyStructType(); const auto resourceType = pgmBuilder.NewResourceType("TopFreq.TopFreqResource.Uint64"); const auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); const auto ui64Type = pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); - + const auto createArgsType = pgmBuilder.NewTupleType({valueType, ui32Type}); const auto createUserType = pgmBuilder.NewTupleType({createArgsType, emptyStructType, valueType}); auto udfTopFreq_Create = pgmBuilder.Udf("TopFreq.TopFreq_Create", TRuntimeNode(), createUserType); - + auto addValueArgsType = pgmBuilder.NewTupleType({resourceType, valueType}); auto addValueUserType = pgmBuilder.NewTupleType({addValueArgsType, emptyStructType, valueType}); auto udfTopFreq_AddValue = pgmBuilder.Udf("TopFreq.TopFreq_AddValue", TRuntimeNode(), addValueUserType); @@ -363,49 +363,49 @@ namespace NYql { static const ui64 BlockSize = 200; static const ui64 BlockCount = 10; static const i32 WorksIfAtLeast = 15; - + std::array<ui64, Total> values; std::array<TRuntimeNode, BlockCount> pgmTopFreqs; - + i32 curIndex = 0; for (ui64 i = 1; i <= BigNum; i++) { for (ui64 j = 0; j < BigEach; j++) { values[curIndex++] = i; - } + } } - + for (ui64 i = BigNum + 1; i <= BigNum + SmallNum; i++) { for (ui64 j = 0; j < SmallEach; j++) { values[curIndex++] = i; - } + } } - + Shuffle(values.begin(), values.end()); - + TVector<TRuntimeNode> params; TRuntimeNode param; TRuntimeNode pgmvalue; - + for (ui64 i = 0; i < BlockCount; i++) { - { + { pgmvalue = pgmBuilder.NewDataLiteral<ui64>(values[i * BlockSize]); - param = pgmBuilder.NewDataLiteral<ui32>(AskFor); + param = pgmBuilder.NewDataLiteral<ui32>(AskFor); params = {pgmvalue, param}; pgmTopFreqs[i] = pgmBuilder.Apply(udfTopFreq_Create, params); - } - + } + for (ui64 j = i * BlockSize + 1; j < (i + 1) * BlockSize; j++) { pgmvalue = pgmBuilder.NewDataLiteral<ui64>(values[j]); params = {pgmTopFreqs[i], pgmvalue}; pgmTopFreqs[i] = pgmBuilder.Apply(udfTopFreq_AddValue, params); - } - - { + } + + { params = {pgmTopFreqs[i]}; pgmTopFreqs[i] = pgmBuilder.Apply(udfTopFreq_Serialize, params); - } + } } - + TRuntimeNode pgmMainTopFreq; { pgmvalue = pgmBuilder.NewDataLiteral<ui64>(Total + 2); @@ -413,15 +413,15 @@ namespace NYql { params = {pgmvalue, param}; pgmMainTopFreq = pgmBuilder.Apply(udfTopFreq_Create, params); } - + for (ui64 i = 0; i < BlockCount; i++) { params = {pgmTopFreqs[i]}; pgmTopFreqs[i] = pgmBuilder.Apply(udfTopFreq_Deserialize, params); - + params = {pgmMainTopFreq, pgmTopFreqs[i]}; pgmMainTopFreq = pgmBuilder.Apply(udfTopFreq_Merge, params); } - + TRuntimeNode pgmReturn; { param = pgmBuilder.NewDataLiteral<ui32>(AskFor); @@ -442,10 +442,10 @@ namespace NYql { ui64 current = item.GetElement(1).Get<ui64>(); if (current <= BigNum) found++; - } + } UNIT_ASSERT(!listIterator.Skip()); UNIT_ASSERT(found >= WorksIfAtLeast); - } - } -} + } + } +} diff --git a/ydb/library/yql/udfs/common/topfreq/ut/ya.make b/ydb/library/yql/udfs/common/topfreq/ut/ya.make index e848abb9cd0..50d941a9e2e 100644 --- a/ydb/library/yql/udfs/common/topfreq/ut/ya.make +++ b/ydb/library/yql/udfs/common/topfreq/ut/ya.make @@ -1,21 +1,21 @@ UNITTEST_FOR(ydb/library/yql/udfs/common/topfreq/static) - -OWNER( - vmakeev + +OWNER( + vmakeev g:yql -) - -SRCS( - ../topfreq_udf_ut.cpp -) - -PEERDIR( +) + +SRCS( + ../topfreq_udf_ut.cpp +) + +PEERDIR( ydb/library/yql/minikql ydb/library/yql/minikql/comp_nodes ydb/library/yql/minikql/computation ydb/library/yql/public/udf/service/exception_policy -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/yql/udfs/common/topfreq/ya.make b/ydb/library/yql/udfs/common/topfreq/ya.make index 63448b297d4..9b30b9ede8e 100644 --- a/ydb/library/yql/udfs/common/topfreq/ya.make +++ b/ydb/library/yql/udfs/common/topfreq/ya.make @@ -1,22 +1,22 @@ -YQL_UDF(topfreq_udf) - +YQL_UDF(topfreq_udf) + YQL_ABI_VERSION( 2 10 0 -) - +) + OWNER(g:yql g:yql_ydb_core) -SRCS( - topfreq_udf.cpp -) - -PEERDIR( +SRCS( + topfreq_udf.cpp +) + +PEERDIR( ydb/library/yql/udfs/common/topfreq/static -) - -END() +) + +END() RECURSE_FOR_TESTS( ut |