diff options
| author | vvvv <[email protected]> | 2025-06-18 11:37:13 +0300 |
|---|---|---|
| committer | vvvv <[email protected]> | 2025-06-18 13:38:30 +0300 |
| commit | fafaf82fef03d47443d3563fb5ea36af8fa71e64 (patch) | |
| tree | 517c43562240391278d797c4793eb9cee8b6986f | |
| parent | 06674e69d9005bafa2ac27df970398fd1c389fdf (diff) | |
YQL-20086 core
commit_hash:af5d81d51befa5cee331fbed69e7e5db2014a260
52 files changed, 2063 insertions, 2063 deletions
diff --git a/yql/essentials/core/cbo/cbo_hints.cpp b/yql/essentials/core/cbo/cbo_hints.cpp index 3a022b36d09..e7dc201c638 100644 --- a/yql/essentials/core/cbo/cbo_hints.cpp +++ b/yql/essentials/core/cbo/cbo_hints.cpp @@ -16,19 +16,19 @@ TString ToLower(TString s) { class TOptimizerHintsParser { public: TOptimizerHintsParser(const TString& text) - : Pos(-1) - , Size(static_cast<i32>(text.size()) - 1) - , Text(text) + : Pos_(-1) + , Size_(static_cast<i32>(text.size()) - 1) + , Text_(text) {} TOptimizerHints Parse() { Start(); - return Hints; + return Hints_; } private: void Start() { - while (Pos < Size) { + while (Pos_ < Size_) { auto hintType = Keyword({"JoinOrder", "Leading", "JoinType", "Rows"}); if (hintType == "JoinOrder" || hintType == "Leading") { JoinOrder(hintType == "Leading"); @@ -37,7 +37,7 @@ private: } else if (hintType == "Rows"){ Rows(); } else { - ParseError(Sprintf("Undefined hints type: %s", hintType.c_str()), Pos - hintType.size()); + ParseError(Sprintf("Undefined hints type: %s", hintType.c_str()), Pos_ - hintType.size()); } SkipWhiteSpaces(); @@ -53,11 +53,11 @@ private: } void JoinType() { - i32 beginPos = Pos + 1; + i32 beginPos = Pos_ + 1; Keyword({"("}); - i32 labelsBeginPos = Pos + 1; + i32 labelsBeginPos = Pos_ + 1; TVector<TString> labels = CollectLabels(); if (labels.size() <= 1) { ParseError(Sprintf("Bad labels for JoinType hint: %s, example of the format: JoinType(t1 t2 Shuffle)", JoinSeq(", ", labels).c_str()), labelsBeginPos); @@ -72,25 +72,25 @@ private: for (const auto& [JoinType, joinAlgoStr]: Zip(joinAlgos, joinAlgosStr)) { if (ToLower(reqJoinAlgoStr) == joinAlgoStr) { - Hints.JoinAlgoHints->PushBack(std::move(labels), JoinType, "JoinType" + Text.substr(beginPos, Pos - beginPos + 1)); + Hints_.JoinAlgoHints->PushBack(std::move(labels), JoinType, "JoinType" + Text_.substr(beginPos, Pos_ - beginPos + 1)); return; } } - ParseError(Sprintf("Unknown JoinType: '%s', supported algos: [%s]", reqJoinAlgoStr.c_str(), JoinSeq(", ", joinAlgosStr).c_str()), Pos - reqJoinAlgoStr.size()); + ParseError(Sprintf("Unknown JoinType: '%s', supported algos: [%s]", reqJoinAlgoStr.c_str(), JoinSeq(", ", joinAlgosStr).c_str()), Pos_ - reqJoinAlgoStr.size()); Y_UNREACHABLE(); } void JoinOrder(bool leading /* is keyword "Leading" or "JoinOrder" */) { - i32 beginPos = Pos + 1; + i32 beginPos = Pos_ + 1; Keyword({"("}); auto joinOrderHintTree = JoinOrderLabels(); Keyword({")"}); - Hints.JoinOrderHints->PushBack( + Hints_.JoinOrderHints->PushBack( std::move(joinOrderHintTree), - leading? "Leading" : "JoinOrder" + Text.substr(beginPos, Pos - beginPos + 1) + leading? "Leading" : "JoinOrder" + Text_.substr(beginPos, Pos_ - beginPos + 1) ); } @@ -109,12 +109,12 @@ private: return join; } - ParseError(Sprintf("JoinOrder args must be either a relation, either a join, example of the format: JoinOrder(t1 (t2 t3))"), Pos); + ParseError(Sprintf("JoinOrder args must be either a relation, either a join, example of the format: JoinOrder(t1 (t2 t3))"), Pos_); Y_UNREACHABLE(); } void Rows() { - i32 beginPos = Pos + 1; + i32 beginPos = Pos_ + 1; Keyword({"("}); @@ -131,10 +131,10 @@ private: case '/': { op = TCardinalityHints::ECardOperation::Divide; break; } case '*': { op = TCardinalityHints::ECardOperation::Multiply; break; } case '#': { op = TCardinalityHints::ECardOperation::Replace; break; } - default: {ParseError(Sprintf("Unknown operation: '%c'", sign), Pos - 1); Y_UNREACHABLE();} + default: {ParseError(Sprintf("Unknown operation: '%c'", sign), Pos_ - 1); Y_UNREACHABLE();} } - Hints.CardinalityHints->PushBack(std::move(labels), op, value, "Rows" + Text.substr(beginPos, Pos - beginPos + 1)); + Hints_.CardinalityHints->PushBack(std::move(labels), op, value, "Rows" + Text_.substr(beginPos, Pos_ - beginPos + 1)); } private: @@ -158,10 +158,10 @@ private: TString Term(const std::bitset<256>& allowedSym = {}) { SkipWhiteSpaces(); - Y_ENSURE(Pos < Size, "Expected <string>, but got end of the string."); + Y_ENSURE(Pos_ < Size_, "Expected <string>, but got end of the string."); TString term; - while (Pos < Size) { + while (Pos_ < Size_) { try { term.push_back(Char(allowedSym)); } catch (...) { @@ -170,7 +170,7 @@ private: } if (term.empty()) { - ParseError("Expected a term!", Pos); + ParseError("Expected a term!", Pos_); } return term; } @@ -190,22 +190,22 @@ private: } char Char(const std::bitset<256>& allowedSymbols = {}) { - Y_ENSURE(Pos < Size, Sprintf("Expected [%s], but got end of the string.", "")); + Y_ENSURE(Pos_ < Size_, Sprintf("Expected [%s], but got end of the string.", "")); - char nextSym = Text[Pos + 1]; + char nextSym = Text_[Pos_ + 1]; if (allowedSymbols.count() == 0) { - ++Pos; + ++Pos_; return nextSym; } for (size_t i = 0; i < allowedSymbols.size(); ++i) { if (allowedSymbols[i] && tolower(i) == tolower(nextSym)) { - ++Pos; + ++Pos_; return nextSym; } } - ParseError(Sprintf("Expected [%s], but got [%c]", "", nextSym), Pos); + ParseError(Sprintf("Expected [%s], but got [%c]", "", nextSym), Pos_); Y_UNREACHABLE(); } @@ -219,25 +219,25 @@ private: TString Keyword(const TVector<TString>& keywords) { SkipWhiteSpaces(); - Y_ENSURE(Pos < Size, Sprintf("Expected [%s], but got end of the string.", JoinSeq(", ", keywords).c_str())); + Y_ENSURE(Pos_ < Size_, Sprintf("Expected [%s], but got end of the string.", JoinSeq(", ", keywords).c_str())); for (const auto& keyword: keywords) { - size_t lowInclude = Pos + 1; + size_t lowInclude = Pos_ + 1; size_t highExclude = lowInclude + keyword.size(); - if (Text.substr(lowInclude, highExclude - lowInclude).equal(keyword)) { - Pos += keyword.size(); + if (Text_.substr(lowInclude, highExclude - lowInclude).equal(keyword)) { + Pos_ += keyword.size(); return keyword; } } - ParseError(Sprintf("Expected [%s], but got [%c]", JoinSeq(", ", keywords).c_str(), Text[Pos + 1]), Pos); + ParseError(Sprintf("Expected [%s], but got [%c]", JoinSeq(", ", keywords).c_str(), Text_[Pos_ + 1]), Pos_); Y_UNREACHABLE(); } double Number() { SkipWhiteSpaces(); - Y_ENSURE(Pos < Size, Sprintf("Expected number, but got end of the string.")); + Y_ENSURE(Pos_ < Size_, Sprintf("Expected number, but got end of the string.")); TString number; if (auto maybeSign = MaybeKeyword({"+", "-"})) { @@ -248,7 +248,7 @@ private: try { return std::stod(term); } catch (...) { - ParseError(Sprintf("Expected a number, got [%s]", term.c_str()), Pos - term.size()); + ParseError(Sprintf("Expected a number, got [%s]", term.c_str()), Pos_ - term.size()); } Y_UNREACHABLE(); } @@ -295,7 +295,7 @@ private: } void SkipWhiteSpaces() { - for (; Pos < Size && isspace(Text[Pos + 1]); ++Pos) { + for (; Pos_ < Size_ && isspace(Text_[Pos_ + 1]); ++Pos_) { } } @@ -303,8 +303,8 @@ private: i32 Line = 0; i32 LinePos = 0; - for (i32 i = 0; i <= pos && i < static_cast<i32>(Text.size()); ++i) { - if (Text[i] == '\n') { + for (i32 i = 0; i <= pos && i < static_cast<i32>(Text_.size()); ++i) { + if (Text_[i] == '\n') { LinePos = 0; ++Line; } else { @@ -316,12 +316,12 @@ private: } private: - i32 Pos; - const i32 Size; - const TString& Text; + i32 Pos_; + const i32 Size_; + const TString& Text_; private: - TOptimizerHints Hints; + TOptimizerHints Hints_; }; TOptimizerHints TOptimizerHints::Parse(const TString& text) { diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp index cd0f5dbe9d0..e2549c1bb5d 100644 --- a/yql/essentials/core/cbo/cbo_interesting_orderings.cpp +++ b/yql/essentials/core/cbo/cbo_interesting_orderings.cpp @@ -95,7 +95,7 @@ TString TFunctionalDependency::ToString() const { } void TTableAliasMap::AddMapping(const TString& table, const TString& alias) { - TableByAlias[alias] = table; + TableByAlias_[alias] = table; } void TTableAliasMap::AddRename(const TString& from, const TString& to) { @@ -104,27 +104,27 @@ void TTableAliasMap::AddRename(const TString& from, const TString& to) { TString baseTable = GetBaseTableByAlias(alias); TString columnName = from.substr(pointIdx + 1); - if (auto it = BaseColumnByRename.find(columnName); it != BaseColumnByRename.end()) { + if (auto it = BaseColumnByRename_.find(columnName); it != BaseColumnByRename_.end()) { auto baseColumn = it->second; - BaseColumnByRename[to] = BaseColumnByRename[from] = it->second; + BaseColumnByRename_[to] = BaseColumnByRename_[from] = it->second; return; } auto fromColumn = TBaseColumn(alias, columnName); auto baseColumn = TBaseColumn(baseTable, columnName); - BaseColumnByRename[to] = BaseColumnByRename[from] = baseColumn; + BaseColumnByRename_[to] = BaseColumnByRename_[from] = baseColumn; return; } - if (BaseColumnByRename.contains(from)) { - BaseColumnByRename[to] = BaseColumnByRename[from]; + if (BaseColumnByRename_.contains(from)) { + BaseColumnByRename_[to] = BaseColumnByRename_[from]; } } TTableAliasMap::TBaseColumn TTableAliasMap::GetBaseColumnByRename(const TString& renamedColumn) { - if (BaseColumnByRename.contains(renamedColumn)) { - return BaseColumnByRename[renamedColumn]; + if (BaseColumnByRename_.contains(renamedColumn)) { + return BaseColumnByRename_[renamedColumn]; } if (auto pointIdx = renamedColumn.find('.'); pointIdx != TString::npos) { @@ -134,8 +134,8 @@ TTableAliasMap::TBaseColumn TTableAliasMap::GetBaseColumnByRename(const TString& return TBaseColumn(std::move(baseTable), std::move(column)); } - if (alias.empty() && BaseColumnByRename.contains(column)) { - return BaseColumnByRename[column]; + if (alias.empty() && BaseColumnByRename_.contains(column)) { + return BaseColumnByRename_[column]; } return TBaseColumn(std::move(alias), std::move(column)); @@ -151,9 +151,9 @@ TTableAliasMap::TBaseColumn TTableAliasMap::GetBaseColumnByRename(const NDq::TJo TString TTableAliasMap::ToString() const { TString result; - if (!BaseColumnByRename.empty()) { + if (!BaseColumnByRename_.empty()) { result += "Renames: "; - for (const auto& [from, to] : BaseColumnByRename) { + for (const auto& [from, to] : BaseColumnByRename_) { result += from + " -> " + to.Relation + "." + to.Column + " "; } result.pop_back(); @@ -161,7 +161,7 @@ TString TTableAliasMap::ToString() const { } result += "TableAliases: "; - for (const auto& [alias, table] : TableByAlias) { + for (const auto& [alias, table] : TableByAlias_) { result += alias + " -> " + table + ", "; } result.pop_back(); @@ -170,19 +170,19 @@ TString TTableAliasMap::ToString() const { } void TTableAliasMap::Merge(const TTableAliasMap& other) { - for (const auto& [alias, table] : other.TableByAlias) { - TableByAlias[alias] = table; + for (const auto& [alias, table] : other.TableByAlias_) { + TableByAlias_[alias] = table; } - for (const auto& [from, to] : other.BaseColumnByRename) { - BaseColumnByRename[from] = TBaseColumn(to.Relation, to.Column); + for (const auto& [from, to] : other.BaseColumnByRename_) { + BaseColumnByRename_[from] = TBaseColumn(to.Relation, to.Column); } } TString TTableAliasMap::GetBaseTableByAlias(const TString& alias) { - if (!TableByAlias.contains(alias)) { + if (!TableByAlias_.contains(alias)) { return alias; } - return TableByAlias[alias]; + return TableByAlias_[alias]; } i64 TFDStorage::FindFDIdx( @@ -389,7 +389,7 @@ TVector<TJoinColumn> TFDStorage::GetInterestingOrderingsColumnNamesByIdx(std::si TVector<TJoinColumn> columns; columns.reserve(InterestingOrderings[interestingOrderingIdx].Items.size()); for (std::size_t columnIdx: InterestingOrderings[interestingOrderingIdx].Items) { - columns.push_back(ColumnByIdx[columnIdx]); + columns.push_back(ColumnByIdx_[columnIdx]); } return columns; @@ -408,8 +408,8 @@ TString TFDStorage::ToString() const { TStringBuilder ss; ss << "Columns mapping: "; - TVector<TString> columnsMapping(IdCounter); - for (const auto& [column, idx]: IdxByColumn) { + TVector<TString> columnsMapping(IdCounter_); + for (const auto& [column, idx]: IdxByColumn_) { TStringBuilder columnSs; columnSs << "{" << idx << ": " << column << "}"; columnsMapping[idx] = columnSs; @@ -488,8 +488,8 @@ std::size_t TFDStorage::GetIdxByColumn( const TString fullPath = baseColumn.RelName + "." + baseColumn.AttributeName; - if (IdxByColumn.contains(fullPath)) { - return IdxByColumn[fullPath]; + if (IdxByColumn_.contains(fullPath)) { + return IdxByColumn_[fullPath]; } Y_ENSURE(!baseColumn.AttributeName.empty()); @@ -497,43 +497,43 @@ std::size_t TFDStorage::GetIdxByColumn( return Max<size_t>(); } - ColumnByIdx.push_back(baseColumn); - IdxByColumn[fullPath] = IdCounter++; - return IdxByColumn[fullPath]; + ColumnByIdx_.push_back(baseColumn); + IdxByColumn_[fullPath] = IdCounter_++; + return IdxByColumn_[fullPath]; } bool TOrderingsStateMachine::TLogicalOrderings::ContainsShuffle(i64 orderingIdx) { - return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx]; + return IsInitialized() && HasState() && (orderingIdx >= 0) && Dfsm_->Nodes_[State_].InterestingOrderings[orderingIdx]; } bool TOrderingsStateMachine::TLogicalOrderings::ContainsSorting(i64 orderingIdx) { - return IsInitialized() && HasState() && (orderingIdx >= 0) && DFSM->Nodes[State].InterestingOrderings[orderingIdx]; + return IsInitialized() && HasState() && (orderingIdx >= 0) && Dfsm_->Nodes_[State_].InterestingOrderings[orderingIdx]; } void TOrderingsStateMachine::TLogicalOrderings::InduceNewOrderings(const TFDSet& fds) { - AppliedFDs |= fds; + AppliedFDs_ |= fds; if (!(HasState() && IsInitialized())) { return; } for (;;) { - auto availableTransitions = DFSM->Nodes[State].OutgoingFDs & AppliedFDs; + auto availableTransitions = Dfsm_->Nodes_[State_].OutgoingFDs & AppliedFDs_; if (availableTransitions.none()) { return; } std::size_t fdIdx = std::countr_zero(availableTransitions.to_ullong()); // take any edge - State = DFSM->TransitionMatrix[State][fdIdx]; + State_ = Dfsm_->TransitionMatrix_[State_][fdIdx]; } } void TOrderingsStateMachine::TLogicalOrderings::RemoveState() { - *this = TLogicalOrderings(DFSM); + *this = TLogicalOrderings(Dfsm_); } void TOrderingsStateMachine::TLogicalOrderings::SetOrdering(i64 orderingIdx) { - if (!IsInitialized() || orderingIdx < 0 || orderingIdx >= static_cast<i64>(DFSM->InitStateByOrderingIdx.size())) { + if (!IsInitialized() || orderingIdx < 0 || orderingIdx >= static_cast<i64>(Dfsm_->InitStateByOrderingIdx_.size())) { RemoveState(); return; } @@ -542,49 +542,49 @@ void TOrderingsStateMachine::TLogicalOrderings::SetOrdering(i64 orderingIdx) { return; } - auto state = DFSM->InitStateByOrderingIdx[orderingIdx]; - State = state.StateIdx; - ShuffleHashFuncArgsCount = state.ShuffleHashFuncArgsCount; + auto state = Dfsm_->InitStateByOrderingIdx_[orderingIdx]; + State_ = state.StateIdx; + ShuffleHashFuncArgsCount_ = state.ShuffleHashFuncArgsCount; } i64 TOrderingsStateMachine::TLogicalOrderings::GetShuffleHashFuncArgsCount() { - return ShuffleHashFuncArgsCount; + return ShuffleHashFuncArgsCount_; } void TOrderingsStateMachine::TLogicalOrderings::SetShuffleHashFuncArgsCount(std::size_t value) { - ShuffleHashFuncArgsCount = value; + ShuffleHashFuncArgsCount_ = value; } TOrderingsStateMachine::TFDSet TOrderingsStateMachine::TLogicalOrderings::GetFDs() { - return AppliedFDs; + return AppliedFDs_; } bool TOrderingsStateMachine::TLogicalOrderings::HasState() { - return State != -1; + return State_ != -1; } bool TOrderingsStateMachine::TLogicalOrderings::HasState() const { - return State != -1; + return State_ != -1; } bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() { - return DFSM != nullptr; + return Dfsm_ != nullptr; } bool TOrderingsStateMachine::TLogicalOrderings::IsInitialized() const { - return DFSM != nullptr; + return Dfsm_ != nullptr; } bool TOrderingsStateMachine::TLogicalOrderings::IsSubsetOf(const TLogicalOrderings& logicalOrderings) { return HasState() && logicalOrderings.HasState() && IsInitialized() && logicalOrderings.IsInitialized() && - DFSM == logicalOrderings.DFSM && - IsSubset(DFSM->Nodes[State].NFSMNodesBitset, logicalOrderings.DFSM->Nodes[logicalOrderings.State].NFSMNodesBitset); + Dfsm_ == logicalOrderings.Dfsm_ && + IsSubset(Dfsm_->Nodes_[State_].NFSMNodesBitset, logicalOrderings.Dfsm_->Nodes_[logicalOrderings.State_].NFSMNodesBitset); } i64 TOrderingsStateMachine::TLogicalOrderings::GetState() const { - return State; + return State_; } bool TOrderingsStateMachine::TLogicalOrderings::IsSubset(const std::bitset<EMaxNFSMStates>& lhs, const std::bitset<EMaxNFSMStates>& rhs) { @@ -592,17 +592,17 @@ bool TOrderingsStateMachine::TLogicalOrderings::IsSubset(const std::bitset<EMaxN } TOrderingsStateMachine::TLogicalOrderings TOrderingsStateMachine::CreateState() { - return TLogicalOrderings(DFSM.Get()); + return TLogicalOrderings(Dfsm_.Get()); } TOrderingsStateMachine::TLogicalOrderings TOrderingsStateMachine::CreateState(i64 orderingIdx) { - auto state = TLogicalOrderings(DFSM.Get()); + auto state = TLogicalOrderings(Dfsm_.Get()); state.SetOrdering(orderingIdx); return state; } bool TOrderingsStateMachine::IsBuilt() const { - return Built; + return Built_; } TOrderingsStateMachine::TFDSet TOrderingsStateMachine::GetFDSet(i64 fdIdx) { @@ -614,8 +614,8 @@ TOrderingsStateMachine::TFDSet TOrderingsStateMachine::GetFDSet(const std::vecto TFDSet fdSet; for (std::size_t fdIdx: fdIdxes) { - if (FdMapping[fdIdx] != -1) { - fdSet[FdMapping[fdIdx]] = 1; + if (FdMapping_[fdIdx] != -1) { + fdSet[FdMapping_[fdIdx]] = 1; } } @@ -625,11 +625,11 @@ TOrderingsStateMachine::TFDSet TOrderingsStateMachine::GetFDSet(const std::vecto TString TOrderingsStateMachine::ToString() const { TStringBuilder ss; ss << "TOrderingsStateMachine:\n"; - ss << "Built: " << (Built ? "true" : "false") << "\n"; - ss << "FdMapping: [" << JoinSeq(", ", FdMapping) << "]\n"; + ss << "Built: " << (Built_ ? "true" : "false") << "\n"; + ss << "FdMapping: [" << JoinSeq(", ", FdMapping_) << "]\n"; ss << "FDStorage:\n" << FDStorage.ToString() << "\n"; - ss << NFSM.ToString(); - ss << DFSM->ToString(NFSM); + ss << Nfsm_.ToString(); + ss << Dfsm_->ToString(Nfsm_); return ss; } @@ -657,7 +657,7 @@ void TOrderingsStateMachine::CollectItemInfo( maxItem = std::max({maxItem, fd.ConsequentItem, maxAntecedentItems}); } - ItemInfo.resize(maxItem + 1); + ItemInfo_.resize(maxItem + 1); for (const auto& ordering: interestingOrderings) { Y_ENSURE(ordering.Items.size() == ordering.Directions.size() || ordering.Directions.empty()); @@ -665,11 +665,11 @@ void TOrderingsStateMachine::CollectItemInfo( for (const auto& [item, direction]: Zip(ordering.Items, ordering.Directions)) { switch (direction) { case TOrdering::TItem::EDirection::EAscending: { - ItemInfo[item].UsedInAscOrdering = true; + ItemInfo_[item].UsedInAscOrdering = true; break; } case TOrdering::TItem::EDirection::EDescending: { - ItemInfo[item].UsedInDescOrdering = true; + ItemInfo_[item].UsedInDescOrdering = true; break; } default: {} @@ -685,10 +685,10 @@ void TOrderingsStateMachine::Build( ) { CollectItemInfo(fds, interestingOrderings); std::vector<TFunctionalDependency> processedFDs = PruneFDs(fds, interestingOrderings); - NFSM.Build(processedFDs, interestingOrderings, ItemInfo); - DFSM = MakeSimpleShared<TDFSM>(); - DFSM->Build(NFSM, processedFDs, interestingOrderings); - Built = true; + Nfsm_.Build(processedFDs, interestingOrderings, ItemInfo_); + Dfsm_ = MakeSimpleShared<TDFSM>(); + Dfsm_->Build(Nfsm_, processedFDs, interestingOrderings); + Built_ = true; } TString TOrderingsStateMachine::TNFSM::TNode::ToString() const { @@ -702,27 +702,27 @@ TString TOrderingsStateMachine::TNFSM::TNode::ToString() const { TString TOrderingsStateMachine::TNFSM::TEdge::ToString() const { TStringBuilder ss; - ss << "Edge{src=" << srcNodeIdx - << ", dst=" << dstNodeIdx - << ", fdIdx=" << (fdIdx == EPSILON ? "EPSILON" : std::to_string(fdIdx)) + ss << "Edge{src=" << SrcNodeIdx + << ", dst=" << DstNodeIdx + << ", fdIdx=" << (FdIdx == EPSILON ? "EPSILON" : std::to_string(FdIdx)) << "}"; return ss; } std::size_t TOrderingsStateMachine::TNFSM::Size() { - return Nodes.size(); + return Nodes_.size(); } TString TOrderingsStateMachine::TNFSM::ToString() const { TStringBuilder ss; ss << "NFSM:\n"; - ss << "Nodes (" << Nodes.size() << "):\n"; - for (std::size_t i = 0; i < Nodes.size(); ++i) { - ss << " " << i << ": " << Nodes[i].ToString() << "\n"; + ss << "Nodes (" << Nodes_.size() << "):\n"; + for (std::size_t i = 0; i < Nodes_.size(); ++i) { + ss << " " << i << ": " << Nodes_[i].ToString() << "\n"; } - ss << "Edges (" << Edges.size() << "):\n"; - for (std::size_t i = 0; i < Edges.size(); ++i) { - ss << " " << i << ": " << Edges[i].ToString() << "\n"; + ss << "Edges (" << Edges_.size() << "):\n"; + for (std::size_t i = 0; i < Edges_.size(); ++i) { + ss << " " << i << ": " << Edges_[i].ToString() << "\n"; } return ss; } @@ -739,42 +739,42 @@ void TOrderingsStateMachine::TNFSM::Build( ApplyFDs(fds, interesting, itemInfo); PrefixClosure(); - for (std::size_t idx = 0; idx < Edges.size(); ++idx) { - Nodes[Edges[idx].srcNodeIdx].OutgoingEdges.push_back(idx); + for (std::size_t idx = 0; idx < Edges_.size(); ++idx) { + Nodes_[Edges_[idx].SrcNodeIdx].OutgoingEdges.push_back(idx); } } std::size_t TOrderingsStateMachine::TNFSM::AddNode(const TOrdering& ordering, TNode::EType type, i64 interestingOrderingIdx) { - for (std::size_t i = 0; i < Nodes.size(); ++i) { - if (Nodes[i].Ordering == ordering) { + for (std::size_t i = 0; i < Nodes_.size(); ++i) { + if (Nodes_[i].Ordering == ordering) { return i; } } - Nodes.emplace_back(ordering, type, interestingOrderingIdx); - return Nodes.size() - 1; + Nodes_.emplace_back(ordering, type, interestingOrderingIdx); + return Nodes_.size() - 1; } bool TOrderingsStateMachine::TNFSM::TEdge::operator==(const TEdge& other) const { - return std::tie(srcNodeIdx, dstNodeIdx, fdIdx) == std::tie(other.srcNodeIdx, other.dstNodeIdx, other.fdIdx); + return std::tie(SrcNodeIdx, DstNodeIdx, FdIdx) == std::tie(other.SrcNodeIdx, other.DstNodeIdx, other.FdIdx); } void TOrderingsStateMachine::TNFSM::AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx) { auto newEdge = TNFSM::TEdge(srcNodeIdx, dstNodeIdx, fdIdx); - for (std::size_t i = 0; i < Edges.size(); ++i) { - if (Edges[i] == newEdge) { + for (std::size_t i = 0; i < Edges_.size(); ++i) { + if (Edges_[i] == newEdge) { return; } } - Edges.emplace_back(newEdge); + Edges_.emplace_back(newEdge); } void TOrderingsStateMachine::TNFSM::PrefixClosure() { - for (std::size_t i = 0; i < Nodes.size(); ++i) { - const auto& iItems = Nodes[i].Ordering.Items; + for (std::size_t i = 0; i < Nodes_.size(); ++i) { + const auto& iItems = Nodes_[i].Ordering.Items; - for (std::size_t j = 0; j < Nodes.size(); ++j) { - const auto& jItems = Nodes[j].Ordering.Items; + for (std::size_t j = 0; j < Nodes_.size(); ++j) { + const auto& jItems = Nodes_[j].Ordering.Items; if (i == j || iItems.size() >= jItems.size()) { continue; } @@ -784,11 +784,11 @@ void TOrderingsStateMachine::TNFSM::PrefixClosure() { } if (k == iItems.size()) { - if (Nodes[i].Ordering.Type == TOrdering::EShuffle) { + if (Nodes_[i].Ordering.Type == TOrdering::EShuffle) { AddEdge(i, j, TNFSM::TEdge::EPSILON); } - if (Nodes[i].Ordering.Type == TOrdering::ESorting) { + if (Nodes_[i].Ordering.Type == TOrdering::ESorting) { AddEdge(j, i, TNFSM::TEdge::EPSILON); } } @@ -811,40 +811,40 @@ void TOrderingsStateMachine::TNFSM::ApplyFDs( )->Items.size(); } - for (std::size_t nodeIdx = 0; nodeIdx < Nodes.size() && Nodes.size() < EMaxNFSMStates; ++nodeIdx) { - for (std::size_t fdIdx = 0; fdIdx < fds.size() && Nodes.size() < EMaxNFSMStates; ++fdIdx) { - if (Nodes[nodeIdx].Ordering.Items.empty()) { + for (std::size_t nodeIdx = 0; nodeIdx < Nodes_.size() && Nodes_.size() < EMaxNFSMStates; ++nodeIdx) { + for (std::size_t fdIdx = 0; fdIdx < fds.size() && Nodes_.size() < EMaxNFSMStates; ++fdIdx) { + if (Nodes_[nodeIdx].Ordering.Items.empty()) { continue; } TFunctionalDependency fd = fds[fdIdx]; auto applyFD = [this, &itemInfo, nodeIdx, maxInterestingOrderingSize](const TFunctionalDependency& fd, std::size_t fdIdx) { - if (Nodes.size() >= EMaxNFSMStates) { + if (Nodes_.size() >= EMaxNFSMStates) { return; } - if (fd.IsConstant() && Nodes[nodeIdx].Ordering.Items.size() > 1) { - std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + if (fd.IsConstant() && Nodes_[nodeIdx].Ordering.Items.size() > 1) { + std::vector<std::size_t> newOrdering = Nodes_[nodeIdx].Ordering.Items; auto it = std::find(newOrdering.begin(), newOrdering.end(), fd.ConsequentItem); if (it == newOrdering.end()) { return; } bool isNewOrderingPrefixOfOld = (it == (newOrdering.end() - 1)); - auto newDirections = Nodes[nodeIdx].Ordering.Directions; + auto newDirections = Nodes_[nodeIdx].Ordering.Directions; if (!newDirections.empty()) { newDirections.erase(newDirections.begin() + std::distance(newOrdering.begin(), it)); newOrdering.erase(it); } - std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), std::move(newDirections), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), std::move(newDirections), Nodes_[nodeIdx].Ordering.Type), TNode::EArtificial); - if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle) { + if (!isNewOrderingPrefixOfOld || Nodes_[nodeIdx].Ordering.Type == TOrdering::EShuffle) { AddEdge(nodeIdx, dstIdx, fdIdx); } - if (!isNewOrderingPrefixOfOld || Nodes[nodeIdx].Ordering.Type == TOrdering::ESorting) { + if (!isNewOrderingPrefixOfOld || Nodes_[nodeIdx].Ordering.Type == TOrdering::ESorting) { AddEdge(dstIdx, nodeIdx, fdIdx); } } @@ -853,21 +853,21 @@ void TOrderingsStateMachine::TNFSM::ApplyFDs( return; } - auto maybeAntecedentItemIdx = fd.MatchesAntecedentItems(Nodes[nodeIdx].Ordering); + auto maybeAntecedentItemIdx = fd.MatchesAntecedentItems(Nodes_[nodeIdx].Ordering); if (!maybeAntecedentItemIdx) { return; } std::size_t antecedentItemIdx = maybeAntecedentItemIdx.GetRef(); if ( - auto it = std::find(Nodes[nodeIdx].Ordering.Items.begin(), Nodes[nodeIdx].Ordering.Items.end(), fd.ConsequentItem); - it != Nodes[nodeIdx].Ordering.Items.end() + auto it = std::find(Nodes_[nodeIdx].Ordering.Items.begin(), Nodes_[nodeIdx].Ordering.Items.end(), fd.ConsequentItem); + it != Nodes_[nodeIdx].Ordering.Items.end() ) { if (fd.IsEquivalence()) { // swap (a, b) -> (b, a) - std::size_t consequentItemIdx = std::distance(Nodes[nodeIdx].Ordering.Items.begin(), it); - auto newOrdering = Nodes[nodeIdx].Ordering.Items; + std::size_t consequentItemIdx = std::distance(Nodes_[nodeIdx].Ordering.Items.begin(), it); + auto newOrdering = Nodes_[nodeIdx].Ordering.Items; std::swap(newOrdering[antecedentItemIdx], newOrdering[consequentItemIdx]); - std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Directions, Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes_[nodeIdx].Ordering.Directions, Nodes_[nodeIdx].Ordering.Type), TNode::EArtificial); AddEdge(nodeIdx, dstIdx, fdIdx); AddEdge(dstIdx, nodeIdx, fdIdx); } @@ -875,30 +875,30 @@ void TOrderingsStateMachine::TNFSM::ApplyFDs( return; } - Y_ENSURE(antecedentItemIdx < Nodes[nodeIdx].Ordering.Items.size()); + Y_ENSURE(antecedentItemIdx < Nodes_[nodeIdx].Ordering.Items.size()); if (fd.IsEquivalence()) { Y_ENSURE(fd.AntecedentItems.size() == 1); - std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + std::vector<std::size_t> newOrdering = Nodes_[nodeIdx].Ordering.Items; newOrdering[antecedentItemIdx] = fd.ConsequentItem; - std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes[nodeIdx].Ordering.Directions, Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), Nodes_[nodeIdx].Ordering.Directions, Nodes_[nodeIdx].Ordering.Type), TNode::EArtificial); AddEdge(nodeIdx, dstIdx, fdIdx); AddEdge(dstIdx, nodeIdx, fdIdx); } if ( - Nodes[nodeIdx].Ordering.Type == TOrdering::EShuffle || - Nodes[nodeIdx].Ordering.Items.size() == maxInterestingOrderingSize + Nodes_[nodeIdx].Ordering.Type == TOrdering::EShuffle || + Nodes_[nodeIdx].Ordering.Items.size() == maxInterestingOrderingSize ) { return; } if (fd.IsImplication() || fd.IsEquivalence()) { - for (std::size_t i = antecedentItemIdx + fd.AntecedentItems.size(); i <= Nodes[nodeIdx].Ordering.Items.size() && Nodes.size() < EMaxNFSMStates; ++i) { - std::vector<std::size_t> newOrdering = Nodes[nodeIdx].Ordering.Items; + for (std::size_t i = antecedentItemIdx + fd.AntecedentItems.size(); i <= Nodes_[nodeIdx].Ordering.Items.size() && Nodes_.size() < EMaxNFSMStates; ++i) { + std::vector<std::size_t> newOrdering = Nodes_[nodeIdx].Ordering.Items; newOrdering.insert(newOrdering.begin() + i, fd.ConsequentItem); - auto newDirections = Nodes[nodeIdx].Ordering.Directions; + auto newDirections = Nodes_[nodeIdx].Ordering.Directions; if (newDirections.empty()) { // smthing went wrong during ordering adding stage return; } @@ -908,13 +908,13 @@ void TOrderingsStateMachine::TNFSM::ApplyFDs( if (itemInfo[fd.ConsequentItem].UsedInAscOrdering) { newDirections[i] = TOrdering::TItem::EDirection::EAscending; - std::size_t dstIdx = AddNode(TOrdering(newOrdering, newDirections, Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + std::size_t dstIdx = AddNode(TOrdering(newOrdering, newDirections, Nodes_[nodeIdx].Ordering.Type), TNode::EArtificial); AddEdge(nodeIdx, dstIdx, fdIdx); // Epsilon edge will be added during PrefixClosure } if (itemInfo[fd.ConsequentItem].UsedInDescOrdering) { newDirections[i] = TOrdering::TItem::EDirection::EDescending; - std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), std::move(newDirections), Nodes[nodeIdx].Ordering.Type), TNode::EArtificial); + std::size_t dstIdx = AddNode(TOrdering(std::move(newOrdering), std::move(newDirections), Nodes_[nodeIdx].Ordering.Type), TNode::EArtificial); AddEdge(nodeIdx, dstIdx, fdIdx); // Epsilon edge will be added during PrefixClosure } } @@ -941,44 +941,44 @@ TString TOrderingsStateMachine::TDFSM::TNode::ToString() const { TString TOrderingsStateMachine::TDFSM::TEdge::ToString() const { TStringBuilder ss; - ss << "Edge{src=" << srcNodeIdx - << ", dst=" << dstNodeIdx - << ", fdIdx=" << fdIdx + ss << "Edge{src=" << SrcNodeIdx + << ", dst=" << DstNodeIdx + << ", fdIdx=" << FdIdx << "}"; return ss; } std::size_t TOrderingsStateMachine::TDFSM::Size() { - return Nodes.size(); + return Nodes_.size(); } TString TOrderingsStateMachine::TDFSM::ToString(const TNFSM& nfsm) const { TStringBuilder ss; ss << "DFSM:\n"; - ss << "Nodes (" << Nodes.size() << "):\n"; - for (std::size_t i = 0; i < Nodes.size(); ++i) { - ss << " " << i << ": " << Nodes[i].ToString() << "\n"; + ss << "Nodes (" << Nodes_.size() << "):\n"; + for (std::size_t i = 0; i < Nodes_.size(); ++i) { + ss << " " << i << ": " << Nodes_[i].ToString() << "\n"; ss << " NFSMNodes: ["; - for (std::size_t j = 0; j < Nodes[i].NFSMNodes.size(); ++j) { - std::size_t nfsmNodeIdx = Nodes[i].NFSMNodes[j]; + for (std::size_t j = 0; j < Nodes_[i].NFSMNodes.size(); ++j) { + std::size_t nfsmNodeIdx = Nodes_[i].NFSMNodes[j]; if (j > 0) { ss << ", "; } ss << nfsmNodeIdx; - if (nfsmNodeIdx < nfsm.Nodes.size()) { - ss << "(" << nfsm.Nodes[nfsmNodeIdx].Ordering.ToString() << ")"; + if (nfsmNodeIdx < nfsm.Nodes_.size()) { + ss << "(" << nfsm.Nodes_[nfsmNodeIdx].Ordering.ToString() << ")"; } } ss << "]\n"; } - ss << "Edges (" << Edges.size() << "):\n"; - for (std::size_t i = 0; i < Edges.size(); ++i) { - ss << " " << i << ": " << Edges[i].ToString() << "\n"; + ss << "Edges (" << Edges_.size() << "):\n"; + for (std::size_t i = 0; i < Edges_.size(); ++i) { + ss << " " << i << ": " << Edges_[i].ToString() << "\n"; } - ss << "InitStateByOrderingIdx (" << InitStateByOrderingIdx.size() << "):\n"; - for (std::size_t i = 0; i < InitStateByOrderingIdx.size(); ++i) { - ss << " " << i << ": StateIdx=" << InitStateByOrderingIdx[i].StateIdx - << ", ShuffleHashFuncArgsCount=" << InitStateByOrderingIdx[i].ShuffleHashFuncArgsCount << "\n"; + ss << "InitStateByOrderingIdx (" << InitStateByOrderingIdx_.size() << "):\n"; + for (std::size_t i = 0; i < InitStateByOrderingIdx_.size(); ++i) { + ss << " " << i << ": StateIdx=" << InitStateByOrderingIdx_[i].StateIdx + << ", ShuffleHashFuncArgsCount=" << InitStateByOrderingIdx_[i].ShuffleHashFuncArgsCount << "\n"; } return ss; } @@ -988,22 +988,22 @@ void TOrderingsStateMachine::TDFSM::Build( const std::vector<TFunctionalDependency>& fds, const std::vector<TOrdering>& interestingOrderings ) { - InitStateByOrderingIdx.resize(interestingOrderings.size()); + InitStateByOrderingIdx_.resize(interestingOrderings.size()); for (std::size_t i = 0; i < interestingOrderings.size(); ++i) { - for (std::size_t nfsmNodeIdx = 0; nfsmNodeIdx < nfsm.Nodes.size(); ++nfsmNodeIdx) { - if (nfsm.Nodes[nfsmNodeIdx].Ordering == interestingOrderings[i]) { + for (std::size_t nfsmNodeIdx = 0; nfsmNodeIdx < nfsm.Nodes_.size(); ++nfsmNodeIdx) { + if (nfsm.Nodes_[nfsmNodeIdx].Ordering == interestingOrderings[i]) { auto nfsmNodes = CollectNodesWithEpsOrFdEdge(nfsm, {i}, fds); - InitStateByOrderingIdx[i] = TInitState{AddNode(std::move(nfsmNodes)), interestingOrderings[i].Items.size()}; + InitStateByOrderingIdx_[i] = TInitState{AddNode(std::move(nfsmNodes)), interestingOrderings[i].Items.size()}; } } } - for (std::size_t nodeIdx = 0; nodeIdx < Nodes.size() && Nodes.size() < EMaxDFSMStates; ++nodeIdx) { + for (std::size_t nodeIdx = 0; nodeIdx < Nodes_.size() && Nodes_.size() < EMaxDFSMStates; ++nodeIdx) { std::unordered_set<i64> outgoingDFSMNodeFDs; - for (std::size_t nfsmNodeIdx: Nodes[nodeIdx].NFSMNodes) { - for (std::size_t nfsmEdgeIdx: nfsm.Nodes[nfsmNodeIdx].OutgoingEdges) { - outgoingDFSMNodeFDs.insert(nfsm.Edges[nfsmEdgeIdx].fdIdx); + for (std::size_t nfsmNodeIdx: Nodes_[nodeIdx].NFSMNodes) { + for (std::size_t nfsmEdgeIdx: nfsm.Nodes_[nfsmNodeIdx].OutgoingEdges) { + outgoingDFSMNodeFDs.insert(nfsm.Edges_[nfsmEdgeIdx].FdIdx); } } @@ -1012,13 +1012,13 @@ void TOrderingsStateMachine::TDFSM::Build( continue; } - std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes[nodeIdx].NFSMNodes, fds, fdIdx)); + std::size_t dstNodeIdx = AddNode(CollectNodesWithEpsOrFdEdge(nfsm, Nodes_[nodeIdx].NFSMNodes, fds, fdIdx)); if (nodeIdx == dstNodeIdx) { continue; } AddEdge(nodeIdx, dstNodeIdx, fdIdx); - Nodes[nodeIdx].OutgoingFDs[fdIdx] = 1; + Nodes_[nodeIdx].OutgoingFDs[fdIdx] = 1; } } @@ -1026,18 +1026,18 @@ void TOrderingsStateMachine::TDFSM::Build( } std::size_t TOrderingsStateMachine::TDFSM::AddNode(const std::vector<std::size_t>& nfsmNodes) { - for (std::size_t i = 0; i < Nodes.size(); ++i) { - if (Nodes[i].NFSMNodes == nfsmNodes) { + for (std::size_t i = 0; i < Nodes_.size(); ++i) { + if (Nodes_[i].NFSMNodes == nfsmNodes) { return i; } } - Nodes.emplace_back(nfsmNodes); - return Nodes.size() - 1; + Nodes_.emplace_back(nfsmNodes); + return Nodes_.size() - 1; } void TOrderingsStateMachine::TDFSM::AddEdge(std::size_t srcNodeIdx, std::size_t dstNodeIdx, i64 fdIdx) { - Edges.emplace_back(srcNodeIdx, dstNodeIdx, fdIdx); + Edges_.emplace_back(srcNodeIdx, dstNodeIdx, fdIdx); } std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdEdge( @@ -1056,10 +1056,10 @@ std::vector<std::size_t> TOrderingsStateMachine::TDFSM::CollectNodesWithEpsOrFdE visited.insert(nodeIdx); - for (std::size_t edgeIdx: nfsm.Nodes[nodeIdx].OutgoingEdges) { - const TNFSM::TEdge& edge = nfsm.Edges[edgeIdx]; - if (edge.fdIdx == fdIdx || edge.fdIdx == TNFSM::TEdge::EPSILON || fds[edge.fdIdx].AlwaysActive) { - DFS(edge.dstNodeIdx); + for (std::size_t edgeIdx: nfsm.Nodes_[nodeIdx].OutgoingEdges) { + const TNFSM::TEdge& edge = nfsm.Edges_[edgeIdx]; + if (edge.FdIdx == fdIdx || edge.FdIdx == TNFSM::TEdge::EPSILON || fds[edge.FdIdx].AlwaysActive) { + DFS(edge.DstNodeIdx); } } }; @@ -1075,21 +1075,21 @@ void TOrderingsStateMachine::TDFSM::Precompute( const TNFSM& nfsm, const std::vector<TFunctionalDependency>& fds ) { - TransitionMatrix = std::vector<std::vector<i64>>(Nodes.size(), std::vector<i64>(fds.size(), -1)); - for (const auto& edge: Edges) { - TransitionMatrix[edge.srcNodeIdx][edge.fdIdx] = edge.dstNodeIdx; + TransitionMatrix_ = std::vector<std::vector<i64>>(Nodes_.size(), std::vector<i64>(fds.size(), -1)); + for (const auto& edge: Edges_) { + TransitionMatrix_[edge.SrcNodeIdx][edge.FdIdx] = edge.DstNodeIdx; } - for (std::size_t dfsmNodeIdx = 0; dfsmNodeIdx < Nodes.size(); ++dfsmNodeIdx) { - for (std::size_t nfsmNodeIdx : Nodes[dfsmNodeIdx].NFSMNodes) { - auto interestingOrderIdx = nfsm.Nodes[nfsmNodeIdx].InterestingOrderingIdx; + for (std::size_t dfsmNodeIdx = 0; dfsmNodeIdx < Nodes_.size(); ++dfsmNodeIdx) { + for (std::size_t nfsmNodeIdx : Nodes_[dfsmNodeIdx].NFSMNodes) { + auto interestingOrderIdx = nfsm.Nodes_[nfsmNodeIdx].InterestingOrderingIdx; if (interestingOrderIdx == -1) { continue; } - Nodes[dfsmNodeIdx].InterestingOrderings[interestingOrderIdx] = 1; + Nodes_[dfsmNodeIdx].InterestingOrderings[interestingOrderIdx] = 1; } } - for (auto& node: Nodes) { + for (auto& node: Nodes_) { for (auto& nfsmNodeIdx: node.NFSMNodes) { node.NFSMNodesBitset[nfsmNodeIdx] = 1; } @@ -1102,7 +1102,7 @@ std::vector<TFunctionalDependency> TOrderingsStateMachine::PruneFDs( ) { std::vector<TFunctionalDependency> filteredFds; filteredFds.reserve(fds.size()); - FdMapping.resize(fds.size()); + FdMapping_.resize(fds.size()); for (std::size_t i = 0; i < fds.size(); ++i) { bool canLeadToInteresting = false; @@ -1123,9 +1123,9 @@ std::vector<TFunctionalDependency> TOrderingsStateMachine::PruneFDs( if (canLeadToInteresting && filteredFds.size() < EMaxFDCount) { filteredFds.push_back(std::move(fds[i])); - FdMapping[i] = filteredFds.size() - 1; + FdMapping_[i] = filteredFds.size() - 1; } else { - FdMapping[i] = -1; + FdMapping_[i] = -1; } } diff --git a/yql/essentials/core/cbo/cbo_interesting_orderings.h b/yql/essentials/core/cbo/cbo_interesting_orderings.h index e4e86bcb0d9..fd45db41a07 100644 --- a/yql/essentials/core/cbo/cbo_interesting_orderings.h +++ b/yql/essentials/core/cbo/cbo_interesting_orderings.h @@ -145,14 +145,14 @@ public: TBaseColumn GetBaseColumnByRename(const NDq::TJoinColumn& renamedColumn); TString ToString() const; void Merge(const TTableAliasMap& other); - bool Empty() const { return TableByAlias.empty() && BaseColumnByRename.empty(); } + bool Empty() const { return TableByAlias_.empty() && BaseColumnByRename_.empty(); } private: TString GetBaseTableByAlias(const TString& alias); private: - THashMap<TString, TString> TableByAlias; - THashMap<TString, TBaseColumn> BaseColumnByRename; + THashMap<TString, TString> TableByAlias_; + THashMap<TString, TBaseColumn> BaseColumnByRename_; }; /* @@ -273,9 +273,9 @@ private: ); private: - THashMap<TString, std::size_t> IdxByColumn; - std::vector<TJoinColumn> ColumnByIdx; - std::size_t IdCounter = 0; + THashMap<TString, std::size_t> IdxByColumn_; + std::vector<TJoinColumn> ColumnByIdx_; + std::size_t IdCounter_ = 0; }; /* @@ -314,7 +314,7 @@ public: TLogicalOrderings& operator= (const TLogicalOrderings&) = default; TLogicalOrderings(TDFSM* dfsm) - : DFSM(dfsm) + : Dfsm_(dfsm) {} public: // API @@ -339,11 +339,11 @@ public: bool IsSubset(const std::bitset<EMaxNFSMStates>& lhs, const std::bitset<EMaxNFSMStates>& rhs); private: - TDFSM* DFSM = nullptr; + TDFSM* Dfsm_ = nullptr; /* we can have different args in hash shuffle function, so shuffles can be incompitable in this case */ - i64 ShuffleHashFuncArgsCount = -1; - i64 State = -1; - TFDSet AppliedFDs{}; + i64 ShuffleHashFuncArgsCount_ = -1; + i64 State_ = -1; + TFDSet AppliedFDs_{}; }; TLogicalOrderings CreateState(); @@ -409,9 +409,9 @@ private: }; struct TEdge { - std::size_t srcNodeIdx; - std::size_t dstNodeIdx; - i64 fdIdx; + std::size_t SrcNodeIdx; + std::size_t DstNodeIdx; + i64 FdIdx; enum _ : i64 { EPSILON = -1 // eps edges with give us nodes without applying any FDs. @@ -441,8 +441,8 @@ private: ); private: - std::vector<TNode> Nodes; - std::vector<TEdge> Edges; + std::vector<TNode> Nodes_; + std::vector<TEdge> Edges_; }; class TDFSM { @@ -460,9 +460,9 @@ private: }; struct TEdge { - std::size_t srcNodeIdx; - std::size_t dstNodeIdx; - i64 fdIdx; + std::size_t SrcNodeIdx; + std::size_t DstNodeIdx; + i64 FdIdx; TString ToString() const; }; @@ -490,17 +490,17 @@ private: ); private: - std::vector<TNode> Nodes; - std::vector<TEdge> Edges; + std::vector<TNode> Nodes_; + std::vector<TEdge> Edges_; - std::vector<std::vector<i64>> TransitionMatrix; - std::vector<std::vector<bool>> ContainsMatrix; + std::vector<std::vector<i64>> TransitionMatrix_; + std::vector<std::vector<bool>> ContainsMatrix_; struct TInitState { std::size_t StateIdx; std::size_t ShuffleHashFuncArgsCount; }; - std::vector<TInitState> InitStateByOrderingIdx; + std::vector<TInitState> InitStateByOrderingIdx_; }; /* @@ -518,12 +518,12 @@ private: ); private: - TNFSM NFSM; - TSimpleSharedPtr<TDFSM> DFSM; // it is important to have sharedptr here, otherwise all logicalorderings will invalidate after copying of FSM + TNFSM Nfsm_; + TSimpleSharedPtr<TDFSM> Dfsm_; // it is important to have sharedptr here, otherwise all logicalorderings will invalidate after copying of FSM - std::vector<i64> FdMapping; // We to remap FD idxes after the pruning - std::vector<TItemInfo> ItemInfo; - bool Built = false; + std::vector<i64> FdMapping_; // We to remap FD idxes after the pruning + std::vector<TItemInfo> ItemInfo_; + bool Built_ = false; }; } diff --git a/yql/essentials/core/common_opt/yql_co_transformer.cpp b/yql/essentials/core/common_opt/yql_co_transformer.cpp index f249c80e0bd..1a807f76858 100644 --- a/yql/essentials/core/common_opt/yql_co_transformer.cpp +++ b/yql/essentials/core/common_opt/yql_co_transformer.cpp @@ -23,9 +23,9 @@ namespace { class TCommonOptTransformer final : public TSyncTransformerBase { public: TCommonOptTransformer(TTypeAnnotationContext* typeCtx, bool final, bool forPeephole) - : TypeCtx(typeCtx) - , Final(final) - , ForPeephole(forPeephole) + : TypeCtx_(typeCtx) + , Final_(final) + , ForPeephole_(forPeephole) {} IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final; @@ -41,15 +41,15 @@ private: bool ScanErrors(const TExprNode& node, TExprContext& ctx); private: - TProcessedNodesSet SimpleProcessedNodes[TCoCallableRules::SIMPLE_STEPS]; - TProcessedNodesSet FlowProcessedNodes[TCoCallableRules::FLOW_STEPS]; - TProcessedNodesSet FinalProcessedNodes; - TProcessedNodesSet ErrorProcessedNodes; - THashSet<TIssue> AddedErrors; - TTypeAnnotationContext* TypeCtx; - const bool Final; - const bool ForPeephole; - bool CheckMissingWorld = false; + TProcessedNodesSet SimpleProcessedNodes_[TCoCallableRules::SIMPLE_STEPS]; + TProcessedNodesSet FlowProcessedNodes_[TCoCallableRules::FLOW_STEPS]; + TProcessedNodesSet FinalProcessedNodes_; + TProcessedNodesSet ErrorProcessedNodes_; + THashSet<TIssue> AddedErrors_; + TTypeAnnotationContext* TypeCtx_; + const bool Final_; + const bool ForPeephole_; + bool CheckMissingWorld_ = false; }; } @@ -66,23 +66,23 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr in IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok; output = std::move(input); - if (IsOptimizerEnabled<CheckMissingWorldOptName>(*TypeCtx) && !IsOptimizerDisabled<CheckMissingWorldOptName>(*TypeCtx)) { - CheckMissingWorld = true; + if (IsOptimizerEnabled<CheckMissingWorldOptName>(*TypeCtx_) && !IsOptimizerDisabled<CheckMissingWorldOptName>(*TypeCtx_)) { + CheckMissingWorld_ = true; } - if (Final) { - return DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FinalCallables, FinalProcessedNodes, true); + if (Final_) { + return DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FinalCallables, FinalProcessedNodes_, true); } for (ui32 i = 0; i < TCoCallableRules::SIMPLE_STEPS; ++i) { - status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().SimpleCallables[i], SimpleProcessedNodes[i], true); + status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().SimpleCallables[i], SimpleProcessedNodes_[i], true); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } } for (ui32 i = 0; i < TCoCallableRules::FLOW_STEPS; ++i) { - status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FlowCallables[i], FlowProcessedNodes[i], true); + status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().FlowCallables[i], FlowProcessedNodes_[i], true); if (status.Level != IGraphTransformer::TStatus::Ok) { return status; } @@ -101,22 +101,22 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr in } void TCommonOptTransformer::Rewind() { - CheckMissingWorld = false; - AddedErrors.clear(); - ErrorProcessedNodes.clear(); - FinalProcessedNodes.clear(); + CheckMissingWorld_ = false; + AddedErrors_.clear(); + ErrorProcessedNodes_.clear(); + FinalProcessedNodes_.clear(); - for (auto& set : FlowProcessedNodes) { + for (auto& set : FlowProcessedNodes_) { set.clear(); } - for (auto& set : SimpleProcessedNodes) { + for (auto& set : SimpleProcessedNodes_) { set.clear(); } } bool TCommonOptTransformer::ScanErrors(const TExprNode& node, TExprContext& ctx) { - auto [it, inserted] = ErrorProcessedNodes.emplace(node.UniqueId()); + auto [it, inserted] = ErrorProcessedNodes_.emplace(node.UniqueId()); if (!inserted) { return true; } @@ -132,7 +132,7 @@ bool TCommonOptTransformer::ScanErrors(const TExprNode& node, TExprContext& ctx) } auto issue = node.GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TErrorExprType>()->GetError(); - if (AddedErrors.insert(issue).second) { + if (AddedErrors_.insert(issue).second) { ctx.AddError(issue); } @@ -144,13 +144,13 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform( const TCallableOptimizerMap& callables, TProcessedNodesSet& processedNodes, bool withParents) { - TOptimizeExprSettings settings(TypeCtx); + TOptimizeExprSettings settings(TypeCtx_); settings.ProcessedNodes = &processedNodes; - settings.CustomInstantTypeTransformer = TypeCtx->CustomInstantTypeTransformer.Get(); + settings.CustomInstantTypeTransformer = TypeCtx_->CustomInstantTypeTransformer.Get(); TParentsMap parentsMap; TOptimizeContext optCtx; - optCtx.Types = TypeCtx; - optCtx.ForPeephole = ForPeephole; + optCtx.Types = TypeCtx_; + optCtx.ForPeephole = ForPeephole_; if (withParents) { GatherParents(*input, parentsMap); optCtx.ParentsMap = &parentsMap; @@ -162,7 +162,7 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform( defaultOpt = defaultIt->second; } - return OptimizeExpr(input, output, [&callables, &optCtx, defaultOpt, checkMissingWorld = CheckMissingWorld](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + return OptimizeExpr(input, output, [&callables, &optCtx, defaultOpt, checkMissingWorld = CheckMissingWorld_](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { const auto rule = callables.find(node->Content()); TExprNode::TPtr result = node; if (rule != callables.cend()) { @@ -188,7 +188,7 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(const TExprNode::T { TParentsMap parentsMap; TOptimizeContext optCtx; - optCtx.Types = TypeCtx; + optCtx.Types = TypeCtx_; GatherParents(*input, parentsMap); optCtx.ParentsMap = &parentsMap; @@ -226,9 +226,9 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(const TExprNode::T } if (!toOptimize.empty()) { - TOptimizeExprSettings settings(TypeCtx); + TOptimizeExprSettings settings(TypeCtx_); settings.VisitTuples = true; - settings.CustomInstantTypeTransformer = TypeCtx->CustomInstantTypeTransformer.Get(); + settings.CustomInstantTypeTransformer = TypeCtx_->CustomInstantTypeTransformer.Get(); return RemapExpr(input, output, toOptimize, ctx, settings); } diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp index 1ebe47096e2..1026e8f1b66 100644 --- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp +++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp @@ -683,20 +683,20 @@ class TJoinTreeRebuilder { public: TJoinTreeRebuilder(TExprNode::TPtr joinTree, TStringBuf label1, TStringBuf column1, TStringBuf label2, TStringBuf column2, TExprContext& ctx, bool rotateJoinTree) - : JoinTree(joinTree) - , Labels{ label1, label2 } - , Columns{ column1, column2 } - , Ctx(ctx) - , RotateJoinTree(rotateJoinTree) + : JoinTree_(joinTree) + , Labels_{ label1, label2 } + , Columns_{ column1, column2 } + , Ctx_(ctx) + , RotateJoinTree_(rotateJoinTree) {} TExprNode::TPtr Run() { - auto joinTree = JoinTree; - if (RotateJoinTree) { - joinTree = RotateCrossJoin(JoinTree->Pos(), JoinTree); + auto joinTree = JoinTree_; + if (RotateJoinTree_) { + joinTree = RotateCrossJoin(JoinTree_->Pos(), JoinTree_); } auto newJoinTree = std::get<0>(AddLink(joinTree)); - YQL_ENSURE(Updated); + YQL_ENSURE(Updated_); return newJoinTree; } @@ -715,21 +715,21 @@ private: right = RotateCrossJoin(pos, right); } - return Ctx.ChangeChildren(*joinTree, std::move(children)); + return Ctx_.ChangeChildren(*joinTree, std::move(children)); } - CrossJoins.clear(); - RestJoins.clear(); + CrossJoins_.clear(); + RestJoins_.clear(); GatherCross(joinTree); - auto inCross1 = FindPtr(CrossJoins, Labels[0]); - auto inCross2 = FindPtr(CrossJoins, Labels[1]); + auto inCross1 = FindPtr(CrossJoins_, Labels_[0]); + auto inCross2 = FindPtr(CrossJoins_, Labels_[1]); if (inCross1 || inCross2) { if (inCross1 && inCross2) { // make them a leaf - joinTree = MakeCrossJoin(pos, Ctx.NewAtom(pos, Labels[0]), Ctx.NewAtom(pos, Labels[1]), Ctx); - for (auto label : CrossJoins) { - if (label != Labels[0] && label != Labels[1]) { - joinTree = MakeCrossJoin(pos, joinTree, Ctx.NewAtom(pos, label), Ctx); + joinTree = MakeCrossJoin(pos, Ctx_.NewAtom(pos, Labels_[0]), Ctx_.NewAtom(pos, Labels_[1]), Ctx_); + for (auto label : CrossJoins_) { + if (label != Labels_[0] && label != Labels_[1]) { + joinTree = MakeCrossJoin(pos, joinTree, Ctx_.NewAtom(pos, label), Ctx_); } } @@ -737,12 +737,12 @@ private: } else if (inCross1) { // leaf with table1 and subtree with table2 - auto rest = FindRestJoin(Labels[1]); + auto rest = FindRestJoin(Labels_[1]); YQL_ENSURE(rest); - joinTree = MakeCrossJoin(pos, Ctx.NewAtom(pos, Labels[0]), rest, Ctx); - for (auto label : CrossJoins) { - if (label != Labels[0]) { - joinTree = MakeCrossJoin(pos, joinTree, Ctx.NewAtom(pos, label), Ctx); + joinTree = MakeCrossJoin(pos, Ctx_.NewAtom(pos, Labels_[0]), rest, Ctx_); + for (auto label : CrossJoins_) { + if (label != Labels_[0]) { + joinTree = MakeCrossJoin(pos, joinTree, Ctx_.NewAtom(pos, label), Ctx_); } } @@ -750,12 +750,12 @@ private: } else { // leaf with table2 and subtree with table1 - auto rest = FindRestJoin(Labels[0]); + auto rest = FindRestJoin(Labels_[0]); YQL_ENSURE(rest); - joinTree = MakeCrossJoin(pos, Ctx.NewAtom(pos, Labels[1]), rest, Ctx); - for (auto label : CrossJoins) { - if (label != Labels[1]) { - joinTree = MakeCrossJoin(pos, joinTree, Ctx.NewAtom(pos, label), Ctx); + joinTree = MakeCrossJoin(pos, Ctx_.NewAtom(pos, Labels_[1]), rest, Ctx_); + for (auto label : CrossJoins_) { + if (label != Labels_[1]) { + joinTree = MakeCrossJoin(pos, joinTree, Ctx_.NewAtom(pos, label), Ctx_); } } @@ -767,19 +767,19 @@ private: } TExprNode::TPtr AddRestJoins(TPositionHandle pos, TExprNode::TPtr joinTree, TExprNode::TPtr exclude) { - for (auto join : RestJoins) { + for (auto join : RestJoins_) { if (join == exclude) { continue; } - joinTree = MakeCrossJoin(pos, joinTree, join, Ctx); + joinTree = MakeCrossJoin(pos, joinTree, join, Ctx_); } return joinTree; } TExprNode::TPtr FindRestJoin(TStringBuf label) { - for (auto join : RestJoins) { + for (auto join : RestJoins_) { if (HasTable(join, label)) { return join; } @@ -819,13 +819,13 @@ private: void GatherCross(TExprNode::TPtr joinTree) { auto type = joinTree->Child(0)->Content(); if (type != "Cross") { - RestJoins.push_back(joinTree); + RestJoins_.push_back(joinTree); return; } auto left = joinTree->ChildPtr(1); if (left->IsAtom()) { - CrossJoins.push_back(left->Content()); + CrossJoins_.push_back(left->Content()); } else { GatherCross(left); @@ -833,7 +833,7 @@ private: auto right = joinTree->ChildPtr(2); if (right->IsAtom()) { - CrossJoins.push_back(right->Content()); + CrossJoins_.push_back(right->Content()); } else { GatherCross(right); @@ -858,11 +858,11 @@ private: } } else { - if (left->Content() == Labels[0]) { + if (left->Content() == Labels_[0]) { found1 = 1u; } - if (left->Content() == Labels[1]) { + if (left->Content() == Labels_[1]) { found2 = 1u; } } @@ -880,19 +880,19 @@ private: } } else { - if (right->Content() == Labels[0]) { + if (right->Content() == Labels_[0]) { found1 = 2u; } - if (right->Content() == Labels[1]) { + if (right->Content() == Labels_[1]) { found2 = 2u; } } if (found1 && found2) { - if (!Updated) { + if (!Updated_) { if (joinTree->Child(0)->Content() == "Cross") { - children[0] = Ctx.NewAtom(joinTree->Pos(), "Inner"); + children[0] = Ctx_.NewAtom(joinTree->Pos(), "Inner"); } else { YQL_ENSURE(joinTree->Child(0)->Content() == "Inner"); @@ -902,33 +902,33 @@ private: ui32 index2 = 1 - index1; auto link1 = children[3]->ChildrenList(); - link1.push_back(Ctx.NewAtom(joinTree->Pos(), Labels[index1])); - link1.push_back(Ctx.NewAtom(joinTree->Pos(), Columns[index1])); - children[3] = Ctx.ChangeChildren(*children[3], std::move(link1)); + link1.push_back(Ctx_.NewAtom(joinTree->Pos(), Labels_[index1])); + link1.push_back(Ctx_.NewAtom(joinTree->Pos(), Columns_[index1])); + children[3] = Ctx_.ChangeChildren(*children[3], std::move(link1)); auto link2 = children[4]->ChildrenList(); - link2.push_back(Ctx.NewAtom(joinTree->Pos(), Labels[index2])); - link2.push_back(Ctx.NewAtom(joinTree->Pos(), Columns[index2])); - children[4] = Ctx.ChangeChildren(*children[4], std::move(link2)); + link2.push_back(Ctx_.NewAtom(joinTree->Pos(), Labels_[index2])); + link2.push_back(Ctx_.NewAtom(joinTree->Pos(), Columns_[index2])); + children[4] = Ctx_.ChangeChildren(*children[4], std::move(link2)); - Updated = true; + Updated_ = true; } } - return { Ctx.ChangeChildren(*joinTree, std::move(children)), found1, found2 }; + return { Ctx_.ChangeChildren(*joinTree, std::move(children)), found1, found2 }; } private: - TVector<TStringBuf> CrossJoins; - TVector<TExprNode::TPtr> RestJoins; + TVector<TStringBuf> CrossJoins_; + TVector<TExprNode::TPtr> RestJoins_; - bool Updated = false; + bool Updated_ = false; - TExprNode::TPtr JoinTree; - TStringBuf Labels[2]; - TStringBuf Columns[2]; - TExprContext& Ctx; - bool RotateJoinTree; + TExprNode::TPtr JoinTree_; + TStringBuf Labels_[2]; + TStringBuf Columns_[2]; + TExprContext& Ctx_; + bool RotateJoinTree_; }; TExprNode::TPtr DecayCrossJoinIntoInner(TExprNode::TPtr equiJoin, const TExprNode::TPtr& predicate, diff --git a/yql/essentials/core/expr_nodes/yql_expr_nodes.h b/yql/essentials/core/expr_nodes/yql_expr_nodes.h index b00185573bb..7705461917f 100644 --- a/yql/essentials/core/expr_nodes/yql_expr_nodes.h +++ b/yql/essentials/core/expr_nodes/yql_expr_nodes.h @@ -59,7 +59,7 @@ public: : NGenerated::TCoWorldBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} TCoWorld DoBuild() { - auto node = this->Ctx.NewWorld(this->Pos); + auto node = this->Ctx_.NewWorld(this->Pos_); return TCoWorld(node); } }; @@ -74,18 +74,18 @@ public: : NGenerated::TCoArgumentBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} TNodeBuilder<TParent, TCoArgument>& Name(const TStringBuf& value) { - this->NameHolder = this->Ctx.AppendString(value); + this->NameHolder_ = this->Ctx_.AppendString(value); return *this; } TCoArgument DoBuild() { - YQL_ENSURE(!NameHolder.empty()); - auto node = this->Ctx.NewArgument(this->Pos, NameHolder); + YQL_ENSURE(!NameHolder_.empty()); + auto node = this->Ctx_.NewArgument(this->Pos_, NameHolder_); return TCoArgument(node); } private: - TStringBuf NameHolder; + TStringBuf NameHolder_; }; template<typename TParent> @@ -98,19 +98,19 @@ public: : NGenerated::TCoAtomBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {} TNodeBuilder<TParent, TCoAtom>& Value(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) { - this->ValueHolder = this->Ctx.AppendString(value); - this->Flags = flags; + this->ValueHolder_ = this->Ctx_.AppendString(value); + this->Flags_ = flags; return *this; } TNodeBuilder<TParent, TCoAtom>& Value(i64 value) { return value >= 0 && value <= i64(std::numeric_limits<ui32>::max()) ? - Value(this->Ctx.GetIndexAsString(ui32(value)), TNodeFlags::Default) : + Value(this->Ctx_.GetIndexAsString(ui32(value)), TNodeFlags::Default) : Value(ToString(value), TNodeFlags::Default); } TCoAtom DoBuild() { - auto node = this->Ctx.NewAtom(this->Pos, ValueHolder, this->Flags); + auto node = this->Ctx_.NewAtom(this->Pos_, ValueHolder_, this->Flags_); return TCoAtom(node); } @@ -119,19 +119,19 @@ public: } TParent& Build(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) { - this->ValueHolder = this->Ctx.AppendString(value); - this->Flags = flags; + this->ValueHolder_ = this->Ctx_.AppendString(value); + this->Flags_ = flags; return this->Build(); } TParent& Build(i64 value) { return value >= 0 && value <= i64(std::numeric_limits<ui32>::max()) ? - Build(this->Ctx.GetIndexAsString(ui32(value)), TNodeFlags::Default) : + Build(this->Ctx_.GetIndexAsString(ui32(value)), TNodeFlags::Default) : Build(ToString(value), TNodeFlags::Default); } private: - TStringBuf ValueHolder; - ui32 Flags = TNodeFlags::ArbitraryContent; + TStringBuf ValueHolder_; + ui32 Flags_ = TNodeFlags::ArbitraryContent; }; template<typename TParent> @@ -149,13 +149,13 @@ public: return getArgFunc(argName); }) - , ArgsMap(argsStore) + , ArgsMap_(argsStore) {} TNodeBuilder<TParent, TCoLambda>& Args(const TCoArgument& node) { Y_DEBUG_ABORT_UNLESS(!this->ArgsHolder.IsValid()); - auto argsNode = this->Ctx.NewArguments(this->Pos, { node.Ptr() }); + auto argsNode = this->Ctx_.NewArguments(this->Pos_, { node.Ptr() }); this->ArgsHolder = TCoArguments(argsNode); return *this; @@ -174,14 +174,14 @@ public: TExprNode::TListType argNodes; for (auto name : list) { - auto argName = this->Ctx.AppendString(name); - auto argNode = this->Ctx.NewArgument(this->Pos, argName); + auto argName = this->Ctx_.AppendString(name); + auto argNode = this->Ctx_.NewArgument(this->Pos_, argName); argNodes.push_back(argNode); - ArgsMap->emplace(argName, TExprBase(argNode)); + ArgsMap_->emplace(argName, TExprBase(argNode)); } - auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes)); + auto argsNode = this->Ctx_.NewArguments(this->Pos_, std::move(argNodes)); this->ArgsHolder = TCoArguments(argsNode); return *this; } @@ -195,7 +195,7 @@ public: argNodes.push_back(arg.Ptr()); } - auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes)); + auto argsNode = this->Ctx_.NewArguments(this->Pos_, std::move(argNodes)); this->ArgsHolder = TCoArguments(argsNode); return *this; } @@ -204,19 +204,19 @@ public: { Y_DEBUG_ABORT_UNLESS(!this->ArgsHolder.IsValid()); - auto argsNode = this->Ctx.NewArguments(this->Pos, TExprNode::TListType(list)); + auto argsNode = this->Ctx_.NewArguments(this->Pos_, TExprNode::TListType(list)); this->ArgsHolder = TCoArguments(argsNode); return *this; } TCoLambda DoBuild() { - auto node = this->Ctx.NewLambda(this->Pos, this->ArgsHolder.Cast().Ptr(), this->BodyHolder.Cast().Ptr()); + auto node = this->Ctx_.NewLambda(this->Pos_, this->ArgsHolder.Cast().Ptr(), this->BodyHolder.Cast().Ptr()); return TCoLambda(node); } private: - std::shared_ptr<TMap<TStringBuf, TExprBase>> ArgsMap; + std::shared_ptr<TMap<TStringBuf, TExprBase>> ArgsMap_; }; class TExprApplier : public TExprBase { @@ -259,11 +259,11 @@ public: TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) - , BuildFunc(buildFunc) {} + , BuildFunc_(buildFunc) {} TParent& Build() { - YQL_ENSURE(Body); - return BuildFunc(TExprApplier(Body.Cast().Ptr())); + YQL_ENSURE(Body_); + return BuildFunc_(TExprApplier(Body_.Cast().Ptr())); } typename TParent::ResultType Done() { @@ -272,24 +272,24 @@ public: } TNodeBuilder<TParent, TExprApplier>& Apply(TCoLambda lambda) { - YQL_ENSURE(!Body); + YQL_ENSURE(!Body_); - Body = lambda.Body(); - Args = lambda.Args(); + Body_ = lambda.Body(); + Args_ = lambda.Args(); return *this; } TNodeBuilder<TParent, TExprApplier>& Apply(TExprBase node) { - YQL_ENSURE(!Body); + YQL_ENSURE(!Body_); - Body = node; + Body_ = node; return *this; } TNodeBuilder<TParent, TExprApplier>& With(TExprBase from, TExprBase to) { - YQL_ENSURE(Body); + YQL_ENSURE(Body_); DoApply(from, to); @@ -297,76 +297,76 @@ public: } TNodeBuilder<TParent, TExprApplier>& With(ui32 argIndex, TExprBase to) { - YQL_ENSURE(Body); - YQL_ENSURE(Args); + YQL_ENSURE(Body_); + YQL_ENSURE(Args_); - DoApply(Args.Cast().Arg(argIndex), to); + DoApply(Args_.Cast().Arg(argIndex), to); return *this; } TNodeBuilder<TParent, TExprApplier>& With(ui32 argIndex, const TStringBuf& argName) { - YQL_ENSURE(Body); - YQL_ENSURE(Args); + YQL_ENSURE(Body_); + YQL_ENSURE(Args_); - DoApply(Args.Cast().Arg(argIndex), GetArgFunc(argName)); + DoApply(Args_.Cast().Arg(argIndex), GetArgFunc_(argName)); return *this; } TNodeBuilder<TParent, TExprApplier>& With(TExprBase from, const TStringBuf& argName) { - YQL_ENSURE(Body); + YQL_ENSURE(Body_); - DoApply(from, GetArgFunc(argName)); + DoApply(from, GetArgFunc_(argName)); return *this; } template<typename TNode> TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode> With(ui32 argIndex) { - YQL_ENSURE(Body); - YQL_ENSURE(Args); + YQL_ENSURE(Body_); + YQL_ENSURE(Args_); auto buildFunc = [argIndex, this] (const TNode& node) mutable -> TNodeBuilder<TParent, TExprApplier>& { - DoApply(Args.Cast().Arg(argIndex), node); + DoApply(Args_.Cast().Arg(argIndex), node); return *this; }; - return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx, this->Pos, - buildFunc, GetArgFunc); + return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx_, this->Pos_, + buildFunc, GetArgFunc_); } template<typename TNode> TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode> With(TExprBase from) { - YQL_ENSURE(Body); + YQL_ENSURE(Body_); auto buildFunc = [from, this] (const TNode& node) mutable -> TNodeBuilder<TParent, TExprApplier>& { DoApply(from, node); return *this; }; - return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx, this->Pos, - buildFunc, GetArgFunc); + return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx_, this->Pos_, + buildFunc, GetArgFunc_); } private: void DoApply(TExprBase applyFrom, TExprBase applyTo) { - TExprNodeBuilder builder(this->Pos, this->Ctx, [this] (const TStringBuf& argName) { - return GetArgFunc(argName).Ptr(); + TExprNodeBuilder builder(this->Pos_, this->Ctx_, [this] (const TStringBuf& argName) { + return GetArgFunc_(argName).Ptr(); }); - auto args = Args.IsValid() ? Args.Cast().Ptr() : nullptr; + auto args = Args_.IsValid() ? Args_.Cast().Ptr() : nullptr; - Body = builder - .ApplyPartial(std::move(args), Body.Cast().Ptr()) + Body_ = builder + .ApplyPartial(std::move(args), Body_.Cast().Ptr()) .WithNode(applyFrom.Ref(), applyTo.Ptr()) .Seal().Build(); } private: - BuildFuncType BuildFunc; + BuildFuncType BuildFunc_; - TMaybeNode<TExprBase> Body; - TMaybeNode<TCoArguments> Args; + TMaybeNode<TExprBase> Body_; + TMaybeNode<TCoArguments> Args_; }; } // namespace NNodes diff --git a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h index 65c104cdfdc..c9a50b7b2c7 100644 --- a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h +++ b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h @@ -27,13 +27,13 @@ public: } explicit TExprBase(TExprNode::TPtr&& node) - : Raw_(node.Get()), Node(std::move(node)) + : Raw_(node.Get()), Node_(std::move(node)) { YQL_ENSURE(Raw_); } explicit TExprBase(const TExprNode::TPtr& node) - : Raw_(node.Get()), Node(node) + : Raw_(node.Get()), Node_(node) { YQL_ENSURE(node); } @@ -43,13 +43,13 @@ public: } TExprNode* MutableRaw() const { - YQL_ENSURE(Node); - return Node.Get(); + YQL_ENSURE(Node_); + return Node_.Get(); } TExprNode::TPtr Ptr() const { - YQL_ENSURE(Node); - return Node; + YQL_ENSURE(Node_); + return Node_; } const TExprNode& Ref() const { @@ -57,8 +57,8 @@ public: } TExprNode& MutableRef() const { - YQL_ENSURE(Node); - return *Node; + YQL_ENSURE(Node_); + return *Node_; } TExprBase NonOwning() const { @@ -76,12 +76,12 @@ public: template<typename TNode> TNode Cast() const { - return Node ? TNode(Node) : TNode(Raw_); + return Node_ ? TNode(Node_) : TNode(Raw_); } private: const TExprNode* Raw_; - TExprNode::TPtr Node; + TExprNode::TPtr Node_; }; template<> @@ -91,10 +91,10 @@ public: : Raw_(node) {} TMaybeNode(const TExprNode::TPtr& node) - : Raw_(node.Get()), Node(node) {} + : Raw_(node.Get()), Node_(node) {} TMaybeNode(const TExprBase& node) - : Raw_(node.Raw()), Node(node.Ptr()) {} + : Raw_(node.Raw()), Node_(node.Ptr()) {} const TExprNode& Ref() const { YQL_ENSURE(IsValid()); @@ -103,8 +103,8 @@ public: TExprNode& MutableRef() const { YQL_ENSURE(IsValid()); - YQL_ENSURE(Node); - return *Node; + YQL_ENSURE(Node_); + return *Node_; } const TExprNode* Raw() const { @@ -113,8 +113,8 @@ public: TExprNode* MutableRaw() const { YQL_ENSURE(IsValid()); - YQL_ENSURE(Node); - return Node.Get(); + YQL_ENSURE(Node_); + return Node_.Get(); } bool IsValid() const { @@ -127,23 +127,23 @@ public: template<typename TNode> TMaybeNode<TNode> Maybe() const { - return Node ? TMaybeNode<TNode>(Node) : TMaybeNode<TNode>(Raw_); + return Node_ ? TMaybeNode<TNode>(Node_) : TMaybeNode<TNode>(Raw_); } template<typename TNode> TNode Cast() const { YQL_ENSURE(IsValid()); - return Node ? TNode(Node) : TNode(Raw_); + return Node_ ? TNode(Node_) : TNode(Raw_); } TExprBase Cast() const { YQL_ENSURE(IsValid()); - return Node ? TExprBase(Node) : TExprBase(Raw_); + return Node_ ? TExprBase(Node_) : TExprBase(Raw_); } private: const TExprNode* Raw_; - TExprNode::TPtr Node; + TExprNode::TPtr Node_; }; template<typename TNode> @@ -151,28 +151,28 @@ class TChildIterator : public std::iterator<std::forward_iterator_tag, TNode> { public: TChildIterator() { - CurIt = EndIt = {}; + CurIt_ = EndIt_ = {}; } TChildIterator(const TExprBase &node, size_t startIndex = 0) - : CurIt(node.Ref().Children().begin() + startIndex) - , EndIt(node.Ref().Children().end()) + : CurIt_(node.Ref().Children().begin() + startIndex) + , EndIt_(node.Ref().Children().end()) { - if (CurIt != EndIt) { - CurNode = *CurIt; + if (CurIt_ != EndIt_) { + CurNode_ = *CurIt_; } } TChildIterator& operator++ () // Pre-increment { - YQL_ENSURE(CurNode); + YQL_ENSURE(CurNode_); Move(); return *this; } TChildIterator operator++ (int) // Post-increment { - YQL_ENSURE(CurNode); + YQL_ENSURE(CurNode_); TChildIterator<TNode> tmp(*this); Move(); return tmp; @@ -180,29 +180,29 @@ public: bool operator ==(const TChildIterator<TNode>& rhs) const { - return CurNode == rhs.CurNode; + return CurNode_ == rhs.CurNode_; } bool operator !=(const TChildIterator<TNode>& rhs) const { - return CurNode != rhs.CurNode; + return CurNode_ != rhs.CurNode_; } TNode operator*() const { - YQL_ENSURE(CurNode); - return TNode(CurNode); + YQL_ENSURE(CurNode_); + return TNode(CurNode_); } private: void Move() { - ++CurIt; - CurNode = CurIt == EndIt ? nullptr : *CurIt; + ++CurIt_; + CurNode_ = CurIt_ == EndIt_ ? nullptr : *CurIt_; } - TExprNode::TPtr CurNode; - TExprNode::TListType::const_iterator CurIt; - TExprNode::TListType::const_iterator EndIt; + TExprNode::TPtr CurNode_; + TExprNode::TListType::const_iterator CurIt_; + TExprNode::TListType::const_iterator EndIt_; }; template<typename TItem> @@ -366,17 +366,17 @@ class TArgs { public: TArgs(const TExprBase& node, size_t startIndex) - : Node(node) - , StartIndex(startIndex) {} + : Node_(node) + , StartIndex_(startIndex) {} - TExprBase Get(size_t index) const { return TExprBase(Node.Ref().ChildPtr(index)); } - size_t Count() const { return Node.Ref().ChildrenSize(); } - TChildIterator<TExprBase> begin() const { return TChildIterator<TExprBase>(Node, StartIndex); } + TExprBase Get(size_t index) const { return TExprBase(Node_.Ref().ChildPtr(index)); } + size_t Count() const { return Node_.Ref().ChildrenSize(); } + TChildIterator<TExprBase> begin() const { return TChildIterator<TExprBase>(Node_, StartIndex_); } TChildIterator<TExprBase> end() const { return TChildIterator<TExprBase>(); } private: - TExprBase Node; - size_t StartIndex; + TExprBase Node_; + size_t StartIndex_; }; template<const size_t FixedArgsCount> @@ -430,14 +430,14 @@ protected: typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType; TNodeBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) - : Ctx(ctx) - , Pos(pos) - , GetArgFunc(getArgFunc) {} + : Ctx_(ctx) + , Pos_(pos) + , GetArgFunc_(getArgFunc) {} protected: - TExprContext& Ctx; - TPositionHandle Pos; - GetArgFuncType GetArgFunc; + TExprContext& Ctx_; + TPositionHandle Pos_; + GetArgFuncType GetArgFunc_; }; template <typename TParent, typename TDerived, typename TItem> @@ -447,17 +447,17 @@ protected: TListBuilderBase(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) - , BuildFunc(buildFunc) {} + , BuildFunc_(buildFunc) {} - TVector<TExprBase> Items; - BuildFuncType BuildFunc; + TVector<TExprBase> Items_; + BuildFuncType BuildFunc_; public: typedef TDerived ResultType; TParent& Build() { TDerived node = static_cast<TNodeBuilder<TParent, TDerived>*>(this)->DoBuild(); - return BuildFunc(node); + return BuildFunc_(node); } typename TParent::ResultType Done() { @@ -466,58 +466,58 @@ public: } TNodeBuilder<TParent, TDerived>& InitFrom(const ResultType& item) { - Items.assign(item.begin(), item.end()); + Items_.assign(item.begin(), item.end()); return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } TNodeBuilder<TParent, TDerived>& Add(const TExprBase& node) { - Items.push_back(node); + Items_.push_back(node); return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } TNodeBuilder<TParent, TDerived>& Add(const TExprNode::TPtr& node) { - Items.push_back(TExprBase(node)); + Items_.push_back(TExprBase(node)); return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } TNodeBuilder<TNodeBuilder<TParent, TDerived>, TItem> Add() { - return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TItem>(this->Ctx, this->Pos, + return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TItem>(this->Ctx_, this->Pos_, [this] (const TExprBase& node) mutable -> TNodeBuilder<TParent, TDerived>& { return Add(node); - }, GetArgFunc); + }, GetArgFunc_); } template<typename TNode> TNodeBuilder<TNodeBuilder<TParent, TDerived>, TNode> Add() { - return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TNode>(this->Ctx, this->Pos, + return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TNode>(this->Ctx_, this->Pos_, [this] (const TNode& node) mutable -> TNodeBuilder<TParent, TDerived>&{ return Add(node); - }, GetArgFunc); + }, GetArgFunc_); } TNodeBuilder<TParent, TDerived>& Add(const TStringBuf& argName) { - return Add(TExprBase(this->GetArgFunc(argName))); + return Add(TExprBase(this->GetArgFunc_(argName))); } template <typename T, typename = std::enable_if_t<!std::is_same<T, TExprBase>::value && std::is_same<T, TItem>::value>> TNodeBuilder<TParent, TDerived>& Add(const TVector<T>& list) { for (auto item : list) { - Items.push_back(item); + Items_.push_back(item); } return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } TNodeBuilder<TParent, TDerived>& Add(const TVector<TExprBase>& list) { for (auto item : list) { - Items.push_back(item); + Items_.push_back(item); } return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } TNodeBuilder<TParent, TDerived>& Add(const TExprNode::TListType& list) { for (auto item : list) { - Items.push_back(TExprBase(item)); + Items_.push_back(TExprBase(item)); } return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); } @@ -537,7 +537,7 @@ public: : TListBuilderBase<TParent, TVector<TExprBase>, TExprBase>(ctx, pos, buildFunc, getArgFunc) {} TVector<TExprBase> DoBuild() { - return this->Items; + return this->Items_; } }; @@ -547,15 +547,15 @@ protected: TFreeArgCallableBuilderBase(TExprContext& ctx, TPositionHandle pos, GetArgFuncType getArgFunc) : TNodeBuilderBase(ctx, pos, getArgFunc) {} - TVector<TExprBase> FreeArgsHolder; + TVector<TExprBase> FreeArgsHolder_; public: TNodeBuilder<TNodeBuilder<TParent, TDerived>, TVector<TExprBase>> FreeArgs() { - return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TVector<TExprBase>>(this->Ctx, this->Pos, + return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TVector<TExprBase>>(this->Ctx_, this->Pos_, [this] (const TVector<TExprBase>& freeArgs) mutable -> TNodeBuilder<TParent, TDerived>& { - FreeArgsHolder = freeArgs; + FreeArgsHolder_ = freeArgs; return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); - }, GetArgFunc); + }, GetArgFunc_); } }; @@ -565,17 +565,17 @@ public: typedef TNode ResultType; void SetValue(const TNode& node) { - Node = node.template Maybe<TNode>(); + Node_ = node.template Maybe<TNode>(); } TNode Value() const { - YQL_ENSURE(Node); + YQL_ENSURE(Node_); - return Node.Cast(); + return Node_.Cast(); } private: - TMaybeNode<TNode> Node; + TMaybeNode<TNode> Node_; }; template<typename TNode> diff --git a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj index 7d074a070d6..fce7d1aceb1 100644 --- a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj +++ b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj @@ -203,7 +203,7 @@ public: {%- endfor %} {%- if node.Builder.Kind == "FreeArg" %} - TBase::FreeArgsHolder.assign(item.FreeArgs().begin(), item.FreeArgs().end()); + TBase::FreeArgsHolder_.assign(item.FreeArgs().begin(), item.FreeArgs().end()); {%- endif %} return *static_cast<TNodeBuilder<TParent, TDerived>*>(this); @@ -215,7 +215,7 @@ public: {% endif %} {%- for child in node.aux.allChildren %} - TBuilder<TParent, TDerived>& {{child.Name}}(const TStringBuf& argName) { {{child.aux.holderName}} = {{child.Type}}(this->GetArgFunc(argName)); return *static_cast<TBuilder<TParent, TDerived>*>(this); } + TBuilder<TParent, TDerived>& {{child.Name}}(const TStringBuf& argName) { {{child.aux.holderName}} = {{child.Type}}(this->GetArgFunc_(argName)); return *static_cast<TBuilder<TParent, TDerived>*>(this); } {% if child.Optional %} TBuilder<TParent, TDerived>& {{child.Name}}(const TMaybe<{{child.Type}}>& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); } {% endif %} @@ -224,8 +224,8 @@ public: TBuilder<TParent, TDerived>& {{child.Name}}(const TExprNode::TPtr& node) { {{child.aux.holderName}} = {{child.Type}}(node); return *static_cast<TBuilder<TParent, TDerived>*>(this); } TBuilder<TParent, TDerived>& {{child.Name}}(std::function<void(TExprNodeBuilder&)> fluentFunc) { - TExprNodeBuilder fluentBuilder(this->Pos, this->Ctx, [this] (const TStringBuf& argName) { - return this->GetArgFunc(argName).Ptr(); + TExprNodeBuilder fluentBuilder(this->Pos_, this->Ctx_, [this] (const TStringBuf& argName) { + return this->GetArgFunc_(argName).Ptr(); }); fluentFunc(fluentBuilder); @@ -234,20 +234,20 @@ public: } TBuilder<TBuilder<TParent, TDerived>, {{child.Type}}> {{child.Name}}() { - return TBuilder<TBuilder<TParent, TDerived>, {{child.Type}}>(this->Ctx, this->Pos, + return TBuilder<TBuilder<TParent, TDerived>, {{child.Type}}>(this->Ctx_, this->Pos_, [this] (const {{child.Type}}& node) mutable -> TBuilder<TParent, TDerived>& { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); - }, this->GetArgFunc); + }, this->GetArgFunc_); } template<typename TNode> TBuilder<TBuilder<TParent, TDerived>, TNode> {{child.Name}}() { - return TBuilder<TBuilder<TParent, TDerived>, TNode>(this->Ctx, this->Pos, + return TBuilder<TBuilder<TParent, TDerived>, TNode>(this->Ctx_, this->Pos_, [this] (const TNode& node) mutable -> TBuilder<TParent, TDerived>&{ {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); - }, this->GetArgFunc); + }, this->GetArgFunc_); } {%- endfor %} }; @@ -344,7 +344,7 @@ class UnknownBuilder { {%- if node.Match and (node.Match.Type == "Callable" or node.Match.Type == "CallableBase") %} TExprNode::TListType argsList; {%- if node.Builder.Kind == "List" %} - for (auto child : this->Items) { + for (auto child : this->Items_) { argsList.push_back(child.Ptr()); } {%- else %} @@ -357,15 +357,15 @@ class UnknownBuilder { {% if node.Builder.Kind == "FreeArg" %} size_t fixedChildCount = {{ node.aux.allChildren | length }}; YQL_ENSURE(argsList.size() == fixedChildCount); - for (size_t i = 0; i < this->FreeArgsHolder.size(); ++i) { - argsList.push_back(this->FreeArgsHolder[i].Ptr()); + for (size_t i = 0; i < this->FreeArgsHolder_.size(); ++i) { + argsList.push_back(this->FreeArgsHolder_[i].Ptr()); } {% endif %} {%- endif %} {%- if node.Match.Type == "CallableBase" %} - auto node = this->Ctx.NewCallable(this->Pos, this->CallableNameHolder, std::move(argsList)); + auto node = this->Ctx_.NewCallable(this->Pos_, this->CallableNameHolder, std::move(argsList)); {%- else %} - auto node = this->Ctx.NewCallable(this->Pos, "{{ node.Match.Name }}", std::move(argsList)); + auto node = this->Ctx_.NewCallable(this->Pos_, "{{ node.Match.Name }}", std::move(argsList)); {%- endif %} return {{ node.aux.qName }}(node); {%- elif node.Match and node.Match.Type == "Tuple"%} @@ -376,14 +376,14 @@ class UnknownBuilder { tupleItems.push_back(this->{{ child.aux.holderName }}.Cast().Ptr()); } {%- endfor %} - auto node = this->Ctx.NewList(this->Pos, std::move(tupleItems)); + auto node = this->Ctx_.NewList(this->Pos_, std::move(tupleItems)); return {{ node.aux.qName }}(node); {%- elif node.ListBase and node.Builder.Kind == "List"%} TExprNode::TListType nodeChildren; - for (auto child : this->Items) { + for (auto child : this->Items_) { nodeChildren.push_back(child.Ptr()); } - auto node = this->Ctx.NewList(this->Pos, std::move(nodeChildren)); + auto node = this->Ctx_.NewList(this->Pos_, std::move(nodeChildren)); return {{ node.aux.qName }}(node); {%- else %} static_assert(false, "Don't know how to build {{ node.Name }}."); diff --git a/yql/essentials/core/extract_predicate/extract_predicate_impl.cpp b/yql/essentials/core/extract_predicate/extract_predicate_impl.cpp index 2b9c12e87fa..bc8426dac5e 100644 --- a/yql/essentials/core/extract_predicate/extract_predicate_impl.cpp +++ b/yql/essentials/core/extract_predicate/extract_predicate_impl.cpp @@ -2176,10 +2176,10 @@ bool TPredicateRangeExtractor::Prepare(const TExprNode::TPtr& filterLambdaNode, THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) { possibleIndexKeys.clear(); - YQL_ENSURE(!FilterLambda, "Prepare() should be called only once"); - FilterLambda = filterLambdaNode; + YQL_ENSURE(!FilterLambda_, "Prepare() should be called only once"); + FilterLambda_ = filterLambdaNode; YQL_ENSURE(rowType.GetKind() == ETypeAnnotationKind::Struct); - RowType = rowType.Cast<TStructExprType>(); + RowType_ = rowType.Cast<TStructExprType>(); TCoLambda filterLambda = OptimizeLambdaForRangeExtraction(filterLambdaNode, ctx, typesCtx); @@ -2196,22 +2196,22 @@ bool TPredicateRangeExtractor::Prepare(const TExprNode::TPtr& filterLambdaNode, YQL_ENSURE(EnsureSpecificDataType(*pred, EDataSlot::Bool, ctx)); YQL_ENSURE(rowArgType->GetKind() == ETypeAnnotationKind::Struct); for (auto& item : rowArgType->Cast<TStructExprType>()->GetItems()) { - auto idx = RowType->FindItem(item->GetName()); + auto idx = RowType_->FindItem(item->GetName()); YQL_ENSURE(idx, "RowType/lambda arg mismatch: column " << item->GetName() << " is missing in original table"); - YQL_ENSURE(IsSameAnnotation(*RowType->GetItems()[*idx]->GetItemType(), *item->GetItemType()), + YQL_ENSURE(IsSameAnnotation(*RowType_->GetItems()[*idx]->GetItemType(), *item->GetItemType()), "RowType/lambda arg mismatch: column " << item->GetName() << " has type: " << *item->GetItemType() << - " expecting: " << *RowType->GetItems()[*idx]->GetItemType()); + " expecting: " << *RowType_->GetItems()[*idx]->GetItemType()); } TSet<TString> keysInScope; - DoBuildRanges(rowArg, pred, Settings, Range, keysInScope, ctx, false); + DoBuildRanges(rowArg, pred, Settings_, Range_, keysInScope, ctx, false); possibleIndexKeys.insert(keysInScope.begin(), keysInScope.end()); TOptimizeExprSettings settings(nullptr); settings.VisitChanges = true; // replace plain predicate in Range/RangeRest with lambda - auto status = OptimizeExpr(Range, Range, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + auto status = OptimizeExpr(Range_, Range_, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { if (node->IsCallable({"Range", "RangeRest"}) && node->ChildrenSize() == 1) { auto pos = node->Pos(); // use lambda in Range/RangeRest @@ -2237,10 +2237,10 @@ bool TPredicateRangeExtractor::Prepare(const TExprNode::TPtr& filterLambdaNode, TPredicateRangeExtractor::TBuildResult TPredicateRangeExtractor::BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) const { - YQL_ENSURE(FilterLambda && Range && RowType, "Prepare() is not called"); + YQL_ENSURE(FilterLambda_ && Range_ && RowType_, "Prepare() is not called"); TBuildResult result; - result.PrunedLambda = ctx.DeepCopyLambda(*FilterLambda); + result.PrunedLambda = ctx.DeepCopyLambda(*FilterLambda_); { THashSet<TString> uniqIndexKeys; @@ -2252,9 +2252,9 @@ TPredicateRangeExtractor::TBuildResult TPredicateRangeExtractor::BuildComputeNod THashMap<TString, size_t> indexKeysOrder; TVector<TString> effectiveIndexKeys = indexKeys; for (size_t i = 0; i < effectiveIndexKeys.size(); ++i) { - TMaybe<ui32> idx = RowType->FindItem(effectiveIndexKeys[i]); + TMaybe<ui32> idx = RowType_->FindItem(effectiveIndexKeys[i]); if (idx) { - auto keyBaseType = GetBasePgOrDataType(RowType->GetItems()[*idx]->GetItemType()); + auto keyBaseType = GetBasePgOrDataType(RowType_->GetItems()[*idx]->GetItemType()); if (!(keyBaseType && keyBaseType->IsComparable() && keyBaseType->IsEquatable())) { idx = {}; } @@ -2270,14 +2270,14 @@ TPredicateRangeExtractor::TBuildResult TPredicateRangeExtractor::BuildComputeNod return result; } - TExprNode::TPtr rebuiltRange = RebuildRangeForIndexKeys(*RowType, Range, indexKeysOrder, result.UsedPrefixLen, ctx); + TExprNode::TPtr rebuiltRange = RebuildRangeForIndexKeys(*RowType_, Range_, indexKeysOrder, result.UsedPrefixLen, ctx); TExprNode::TPtr prunedRange; - result.ComputeNode = BuildMultiColumnComputeNode(*RowType, rebuiltRange, effectiveIndexKeys, indexKeysOrder, - prunedRange, Settings, result.UsedPrefixLen, result.PointPrefixLen, ctx, typesCtx, result.LiteralRange); + result.ComputeNode = BuildMultiColumnComputeNode(*RowType_, rebuiltRange, effectiveIndexKeys, indexKeysOrder, + prunedRange, Settings_, result.UsedPrefixLen, result.PointPrefixLen, ctx, typesCtx, result.LiteralRange); if (result.ComputeNode) { result.ExpectedMaxRanges = CalcMaxRanges(rebuiltRange, indexKeysOrder); - if (!Settings.MaxRanges || (result.ExpectedMaxRanges && *result.ExpectedMaxRanges < *Settings.MaxRanges)) { + if (!Settings_.MaxRanges || (result.ExpectedMaxRanges && *result.ExpectedMaxRanges < *Settings_.MaxRanges)) { TCoLambda lambda(result.PrunedLambda); auto newPred = MakePredicateFromPrunedRange(prunedRange, lambda.Args().Arg(0).Ptr(), ctx); diff --git a/yql/essentials/core/extract_predicate/extract_predicate_impl.h b/yql/essentials/core/extract_predicate/extract_predicate_impl.h index 68524c25d62..6231a4c30f9 100644 --- a/yql/essentials/core/extract_predicate/extract_predicate_impl.h +++ b/yql/essentials/core/extract_predicate/extract_predicate_impl.h @@ -7,22 +7,22 @@ namespace NYql::NDetail { class TPredicateRangeExtractor : public IPredicateRangeExtractor { public: explicit TPredicateRangeExtractor(const TPredicateExtractorSettings& settings = {}) - : Settings(settings) + : Settings_(settings) {} bool Prepare(const TExprNode::TPtr& filterLambdaNode, const TTypeAnnotationNode& rowType, THashSet<TString>& possibleIndexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) override final; TExprNode::TPtr GetPreparedRange() const { - return Range; + return Range_; } TBuildResult BuildComputeNode(const TVector<TString>& indexKeys, TExprContext& ctx, TTypeAnnotationContext& typesCtx) const override final; private: - const TPredicateExtractorSettings Settings; - TExprNode::TPtr FilterLambda; - const TStructExprType* RowType = nullptr; - TExprNode::TPtr Range; + const TPredicateExtractorSettings Settings_; + TExprNode::TPtr FilterLambda_; + const TStructExprType* RowType_ = nullptr; + TExprNode::TPtr Range_; }; } diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index e06135256fe..9fb056953f5 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -163,7 +163,7 @@ TProgramFactory::TProgramFactory( } void TProgramFactory::UnrepeatableRandom() { - UseUnrepeatableRandom = true; + UseUnrepeatableRandom_ = true; } void TProgramFactory::EnableRangeComputeFor() { @@ -245,7 +245,7 @@ TProgramPtr TProgramFactory::Create( const TQContext& qContext, TMaybe<TString> gatewaysForMerge) { - auto randomProvider = UseRepeatableRandomAndTimeProviders_ && !UseUnrepeatableRandom && hiddenMode == EHiddenMode::Disable ? + auto randomProvider = UseRepeatableRandomAndTimeProviders_ && !UseUnrepeatableRandom_ && hiddenMode == EHiddenMode::Disable ? CreateDeterministicRandomProvider(1) : CreateDefaultRandomProvider(); auto timeProvider = UseRepeatableRandomAndTimeProviders_ ? CreateDeterministicTimeProvider(10000000) : CreateDefaultTimeProvider(); diff --git a/yql/essentials/core/facade/yql_facade.h b/yql/essentials/core/facade/yql_facade.h index f231a365011..caed085ec9a 100644 --- a/yql/essentials/core/facade/yql_facade.h +++ b/yql/essentials/core/facade/yql_facade.h @@ -83,7 +83,7 @@ public: void UnrepeatableRandom(); private: const bool UseRepeatableRandomAndTimeProviders_; - bool UseUnrepeatableRandom = false; + bool UseUnrepeatableRandom_ = false; const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; const ui64 NextUniqueId_; TVector<TDataProviderInitializer> DataProvidersInit_; diff --git a/yql/essentials/core/file_storage/file_storage.cpp b/yql/essentials/core/file_storage/file_storage.cpp index de24524ac30..95bbaebf930 100644 --- a/yql/essentials/core/file_storage/file_storage.cpp +++ b/yql/essentials/core/file_storage/file_storage.cpp @@ -46,12 +46,12 @@ namespace NYql { class TFileStorageImpl: public IFileStorage { public: explicit TFileStorageImpl(const TFileStorageConfig& params, const std::vector<NFS::IDownloaderPtr>& downloaders) - : Storage(params.GetMaxFiles(), ui64(params.GetMaxSizeMb()) << 20ull, params.GetPath()) - , Config(params) - , UseFakeChecksums(GetEnv("YQL_LOCAL") == "1") + : Storage_(params.GetMaxFiles(), ui64(params.GetMaxSizeMb()) << 20ull, params.GetPath()) + , Config_(params) + , UseFakeChecksums_(GetEnv("YQL_LOCAL") == "1") { - Downloaders.push_back(MakeHttpDownloader(params)); - Downloaders.insert(Downloaders.begin(), downloaders.begin(), downloaders.end()); + Downloaders_.push_back(MakeHttpDownloader(params)); + Downloaders_.insert(Downloaders_.begin(), downloaders.begin(), downloaders.end()); } ~TFileStorageImpl() { @@ -61,8 +61,8 @@ public: YQL_LOG(INFO) << "PutFile to cache: " << file; const auto md5 = FileChecksum(file); const TString storageFileName = md5 + ".file"; - auto lock = MultiResourceLock.Acquire(storageFileName); - return Storage.Put(storageFileName, outFileName, md5, [&file, &md5](const TFsPath& dstFile) { + auto lock = MultiResourceLock_.Acquire(storageFileName); + return Storage_.Put(storageFileName, outFileName, md5, [&file, &md5](const TFsPath& dstFile) { NFs::HardLinkOrCopy(file, dstFile); i64 length = GetFileLength(dstFile.c_str()); if (length == -1) { @@ -85,19 +85,19 @@ public: } const auto md5 = originalMd5.empty() ? MD5::File(file) : originalMd5; const auto strippedMetaFile = md5 + ".stripped_meta"; - auto lock = MultiResourceLock.Acquire(strippedMetaFile); + auto lock = MultiResourceLock_.Acquire(strippedMetaFile); TUrlMeta strippedMeta; strippedMeta.TryReadFrom(GetRoot() / strippedMetaFile); if (strippedMeta.ContentFile) { - if (auto result = Storage.HardlinkFromStorage(strippedMeta.ContentFile, strippedMeta.Md5, "")) { + if (auto result = Storage_.HardlinkFromStorage(strippedMeta.ContentFile, strippedMeta.Md5, "")) { return result; } } strippedMeta = TUrlMeta(); const TString storageFileName = md5 + ".file.stripped"; - TFileLinkPtr result = Storage.Put(storageFileName, "", "", [&file](const TFsPath& dstPath) { + TFileLinkPtr result = Storage_.Put(storageFileName, "", "", [&file](const TFsPath& dstPath) { ui64 size; TString md5; TShellCommand cmd("strip", {file, "-o", dstPath.GetPath()}); @@ -113,9 +113,9 @@ public: strippedMeta.ContentFile = result->GetStorageFileName(); strippedMeta.Md5 = result->GetMd5(); - auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); + auto metaTmpFile = Storage_.GetTemp() / Storage_.GetTempName(); strippedMeta.SaveTo(metaTmpFile); - Storage.MoveToStorage(metaTmpFile, strippedMetaFile); + Storage_.MoveToStorage(metaTmpFile, strippedMetaFile); return result; } @@ -123,8 +123,8 @@ public: const auto md5 = MD5::Calc(data); const TString storageFileName = md5 + ".file"; YQL_LOG(INFO) << "PutInline to cache. md5=" << md5; - auto lock = MultiResourceLock.Acquire(storageFileName); - return Storage.Put(storageFileName, TString(), md5, [&data, &md5](const TFsPath& dstFile) { + auto lock = MultiResourceLock_.Acquire(storageFileName); + return Storage_.Put(storageFileName, TString(), md5, [&data, &md5](const TFsPath& dstFile) { TStringInput in(data); TFile outFile(dstFile, CreateAlways | ARW | AX); TUnbufferedFileOutput out(outFile); @@ -145,7 +145,7 @@ public: try { YQL_LOG(INFO) << "PutUrl to cache: " << urlStr; THttpURL url = ParseURL(urlStr); - for (const auto& d: Downloaders) { + for (const auto& d: Downloaders_) { if (d->Accept(url)) { return PutUrl(url, token, d); } @@ -172,37 +172,37 @@ public: } TFsPath GetRoot() const final { - return Storage.GetRoot(); + return Storage_.GetRoot(); } TFsPath GetTemp() const final { - return Storage.GetTemp(); + return Storage_.GetTemp(); } const TFileStorageConfig& GetConfig() const final { - return Config; + return Config_; } private: TFileLinkPtr PutUrl(const THttpURL& url, const TString& token, const NFS::IDownloaderPtr& downloader) { - return WithRetry<TDownloadError>(Config.GetRetryCount(), [&, this]() { + return WithRetry<TDownloadError>(Config_.GetRetryCount(), [&, this]() { return this->DoPutUrl(url, token, downloader); }, [&](const auto& e, int attempt, int attemptCount) { YQL_LOG(WARN) << "Error while downloading url " << url.PrintS() << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what(); - Sleep(TDuration::MilliSeconds(Config.GetRetryDelayMs())); + Sleep(TDuration::MilliSeconds(Config_.GetRetryDelayMs())); }); } TFileLinkPtr DoPutUrl(const THttpURL& url, const TString& token, const NFS::IDownloaderPtr& downloader) { const auto urlMetaFile = BuildUrlMetaFileName(url, token); - auto lock = MultiResourceLock.Acquire(urlMetaFile); // let's use meta file as lock name + auto lock = MultiResourceLock_.Acquire(urlMetaFile); // let's use meta file as lock name TUrlMeta urlMeta; urlMeta.TryReadFrom(GetRoot() / urlMetaFile); TFileLinkPtr oldContentLink = nullptr; if (urlMeta.ContentFile) { - oldContentLink = Storage.HardlinkFromStorage(urlMeta.ContentFile, urlMeta.Md5, ""); + oldContentLink = Storage_.HardlinkFromStorage(urlMeta.ContentFile, urlMeta.Md5, ""); } if (!oldContentLink) { @@ -238,23 +238,23 @@ private: // todo: remove oldContentLink ? const auto urlChecksum = BuildUrlChecksum(url, token, etag, lastModified); const auto urlContentFile = BuildUrlContentFileName(urlChecksum); - TFileLinkPtr result = Storage.Put(urlContentFile, TString(), UseFakeChecksums ? urlChecksum : TString(), puller); + TFileLinkPtr result = Storage_.Put(urlContentFile, TString(), UseFakeChecksums_ ? urlChecksum : TString(), puller); // save meta using rename for atomicity urlMeta.ETag = etag; urlMeta.LastModified = lastModified; urlMeta.ContentFile = result->GetStorageFileName(); urlMeta.Md5 = result->GetMd5(); - auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); + auto metaTmpFile = Storage_.GetTemp() / Storage_.GetTempName(); urlMeta.SaveTo(metaTmpFile); - Storage.MoveToStorage(metaTmpFile, urlMetaFile); + Storage_.MoveToStorage(metaTmpFile, urlMetaFile); return result; } TString FileChecksum(const TString& file) const { - if (UseFakeChecksums) { + if (UseFakeChecksums_) { // Avoid heavy binaries recalculation in local mode (YQL-15353): // calculate MD5 sum of file name instead of file contents return MD5::Calc(file); @@ -277,31 +277,31 @@ private: } private: - TStorage Storage; - const TFileStorageConfig Config; - std::vector<NFS::IDownloaderPtr> Downloaders; - TMultiResourceLock MultiResourceLock; - const bool UseFakeChecksums; // YQL-15353 + TStorage Storage_; + const TFileStorageConfig Config_; + std::vector<NFS::IDownloaderPtr> Downloaders_; + TMultiResourceLock MultiResourceLock_; + const bool UseFakeChecksums_; // YQL-15353 }; class TFileStorageWithAsync: public TFileStorageDecorator { public: TFileStorageWithAsync(TFileStoragePtr fs) : TFileStorageDecorator(std::move(fs)) - , QueueStarted(0) + , QueueStarted_(0) { auto numThreads = Inner_->GetConfig().GetThreads(); if (1 == numThreads) { - MtpQueue.Reset(new TFakeThreadPool()); + MtpQueue_.Reset(new TFakeThreadPool()); } else { - MtpQueue.Reset(new TSimpleThreadPool(TThreadPoolParams{"FileStorage"})); + MtpQueue_.Reset(new TSimpleThreadPool(TThreadPoolParams{"FileStorage"})); } // do not call MtpQueue->Start here as we have to do it _after_ fork } ~TFileStorageWithAsync() { - MtpQueue->Stop(); + MtpQueue_->Stop(); } NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName) final { @@ -322,19 +322,19 @@ public: private: NThreading::TFuture<TFileLinkPtr> DoAsync(std::function<TFileLinkPtr(const TFileStoragePtr&)> action) { - if (AtomicTryLock(&QueueStarted)) { - MtpQueue->Start(Inner_->GetConfig().GetThreads()); + if (AtomicTryLock(&QueueStarted_)) { + MtpQueue_->Start(Inner_->GetConfig().GetThreads()); } auto logCtx = NLog::CurrentLogContextPath(); return NThreading::Async([logCtx, fs = Inner_, action]() { YQL_LOG_CTX_ROOT_SESSION_SCOPE(logCtx); return action(fs); - }, *MtpQueue); + }, *MtpQueue_); } private: - TAtomic QueueStarted; - THolder<IThreadPool> MtpQueue; + TAtomic QueueStarted_; + THolder<IThreadPool> MtpQueue_; }; diff --git a/yql/essentials/core/file_storage/sized_cache.cpp b/yql/essentials/core/file_storage/sized_cache.cpp index fe17d895d05..dc1699b011d 100644 --- a/yql/essentials/core/file_storage/sized_cache.cpp +++ b/yql/essentials/core/file_storage/sized_cache.cpp @@ -5,29 +5,29 @@ namespace NYql { TSizedCache::TSizedCache(size_t maxEntries, ui64 maxSize) - : Cache(Max<size_t>()) - , MaxEntries(maxEntries) - , MaxSize(maxSize) + : Cache_(Max<size_t>()) + , MaxEntries_(maxEntries) + , MaxSize_(maxSize) { - Y_ASSERT(MaxEntries > 0); - Y_ASSERT(MaxSize > 0); + Y_ASSERT(MaxEntries_ > 0); + Y_ASSERT(MaxSize_ > 0); } void TSizedCache::Put(const TIntrusivePtr<ICacheObj>& obj, bool lock) { const TString name = obj->GetName(); const ui64 size = obj->GetSize(); - auto guard = Guard(Lock); - auto it = Cache.Find(name); - if (it == Cache.End()) { - Cache.Insert(name, TEntry{obj, lock ? 1U : 0U}); - CurrentSize += size; + auto guard = Guard(Lock_); + auto it = Cache_.Find(name); + if (it == Cache_.End()) { + Cache_.Insert(name, TEntry{obj, lock ? 1U : 0U}); + CurrentSize_ += size; } else if (lock) { ++it->Locks; } - while (Cache.Size() > MaxEntries || CurrentSize > MaxSize) { - it = Cache.FindOldest(); - if (Cache.End() == it || 0 != it->Locks) { + while (Cache_.Size() > MaxEntries_ || CurrentSize_ > MaxSize_) { + it = Cache_.FindOldest(); + if (Cache_.End() == it || 0 != it->Locks) { break; } Remove(it); @@ -35,49 +35,49 @@ void TSizedCache::Put(const TIntrusivePtr<ICacheObj>& obj, bool lock) { } void TSizedCache::Release(const TString& name, bool remove) { - auto guard = Guard(Lock); - auto it = Cache.FindWithoutPromote(name); - if (it == Cache.End()) { + auto guard = Guard(Lock_); + auto it = Cache_.FindWithoutPromote(name); + if (it == Cache_.End()) { Y_ASSERT(false); } else { Y_ASSERT(it->Locks); if (it->Locks) { --it->Locks; } - if (!it->Locks && (remove || Cache.Size() > MaxEntries || CurrentSize > MaxSize)) { + if (!it->Locks && (remove || Cache_.Size() > MaxEntries_ || CurrentSize_ > MaxSize_)) { Remove(it); } } } TIntrusivePtr<TSizedCache::ICacheObj> TSizedCache::Get(const TString& name) { - auto guard = Guard(Lock); - auto it = Cache.Find(name); + auto guard = Guard(Lock_); + auto it = Cache_.Find(name); TIntrusivePtr<TSizedCache::ICacheObj> res; - if (it != Cache.End()) { + if (it != Cache_.End()) { res = it->Obj; } return res; } TIntrusivePtr<TSizedCache::ICacheObj> TSizedCache::Revoke(const TString& name) { - auto guard = Guard(Lock); - auto it = Cache.FindWithoutPromote(name); + auto guard = Guard(Lock_); + auto it = Cache_.FindWithoutPromote(name); TIntrusivePtr<TSizedCache::ICacheObj> res; - if (it != Cache.End()) { + if (it != Cache_.End()) { res = it->Obj; - Y_ASSERT(CurrentSize >= res->GetSize()); - CurrentSize -= res->GetSize(); - Cache.Erase(it); + Y_ASSERT(CurrentSize_ >= res->GetSize()); + CurrentSize_ -= res->GetSize(); + Cache_.Erase(it); } return res; } TMaybe<ui32> TSizedCache::GetLocks(const TString& name) const { - auto guard = Guard(Lock); - auto it = Cache.FindWithoutPromote(name); + auto guard = Guard(Lock_); + auto it = Cache_.FindWithoutPromote(name); TMaybe<ui32> res; - if (it != Cache.End()) { + if (it != Cache_.End()) { res = it->Locks; } return res; @@ -85,11 +85,11 @@ TMaybe<ui32> TSizedCache::GetLocks(const TString& name) const { void TSizedCache::Remove(TCache::TIterator it) { TIntrusivePtr<TSizedCache::ICacheObj> obj = it->Obj; - Y_ASSERT(CurrentSize >= obj->GetSize()); - CurrentSize -= obj->GetSize(); - Cache.Erase(it); + Y_ASSERT(CurrentSize_ >= obj->GetSize()); + CurrentSize_ -= obj->GetSize(); + Cache_.Erase(it); // Dismissing may take a long time. Unlock temporary - auto unguard = Unguard(Lock); + auto unguard = Unguard(Lock_); obj->Dismiss(); } diff --git a/yql/essentials/core/file_storage/sized_cache.h b/yql/essentials/core/file_storage/sized_cache.h index a08095cab7b..aa17f1796f1 100644 --- a/yql/essentials/core/file_storage/sized_cache.h +++ b/yql/essentials/core/file_storage/sized_cache.h @@ -47,11 +47,11 @@ public: TMaybe<ui32> GetLocks(const TString& name) const; // Total size of all objects in the cache ui64 GetOccupiedSize() const { - return CurrentSize; + return CurrentSize_; } // Total count of objects in the cache size_t GetCount() const { - return Cache.Size(); + return Cache_.Size(); } private: struct TEntry { @@ -63,11 +63,11 @@ private: void Remove(TCache::TIterator it); private: - TCache Cache; - size_t MaxEntries = 0; - ui64 MaxSize = 0; - ui64 CurrentSize = 0; - TAdaptiveLock Lock; + TCache Cache_; + size_t MaxEntries_ = 0; + ui64 MaxSize_ = 0; + ui64 CurrentSize_ = 0; + TAdaptiveLock Lock_; }; } // NYql diff --git a/yql/essentials/core/file_storage/storage.cpp b/yql/essentials/core/file_storage/storage.cpp index 1bbef336690..dece8d0b546 100644 --- a/yql/essentials/core/file_storage/storage.cpp +++ b/yql/essentials/core/file_storage/storage.cpp @@ -55,22 +55,22 @@ TFsPath ToFilePath(const TString& path) } // namespace TFileLink::TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy) - : Path(path) - , StorageFileName(storageFileName) - , Size(size) - , Md5(md5) - , DeleteOnDestroy(deleteOnDestroy) + : Path_(path) + , StorageFileName_(storageFileName) + , Size_(size) + , Md5_(md5) + , DeleteOnDestroy_(deleteOnDestroy) { } TFileLink::~TFileLink() { - if (!DeleteOnDestroy) { + if (!DeleteOnDestroy_) { return; } - YQL_LOG(INFO) << "Destroying TFileLink for " << Path.GetPath().Quote(); + YQL_LOG(INFO) << "Destroying TFileLink for " << Path_.GetPath().Quote(); try { - Path.ForceDelete(); + Path_.ForceDelete(); } catch (...) { YQL_LOG(ERROR) << CurrentExceptionMessage(); } @@ -121,12 +121,12 @@ public: } inline void Register(TImpl* obj) { - auto guard = Guard(Mutex); - Registered.PushBack(obj); + auto guard = Guard(Mutex_); + Registered_.PushBack(obj); } inline void Unregister(TImpl* obj) { - auto guard = Guard(Mutex); + auto guard = Guard(Mutex_); obj->Unlink(); } @@ -136,7 +136,7 @@ public: private: void Reinit() { - for (auto& v : Registered) { + for (auto& v : Registered_) { v.ResetAtFork(); } } @@ -145,48 +145,48 @@ public: Get().Reinit(); } - TIntrusiveList<TImpl> Registered; - TMutex Mutex; + TIntrusiveList<TImpl> Registered_; + TMutex Mutex_; }; TImpl(size_t maxFiles, ui64 maxSize, const TString& storagePath) - : StorageDir(ToFilePath(storagePath)) - , ProcessTempDir(StorageDir / ToString(GetPID())) // must be subfolder for fast hardlinking - , IsTemp(storagePath.empty()) - , MaxFiles(maxFiles) - , MaxSize(maxSize) - , CurrentFiles(0) - , CurrentSize(0) - , Dirty(false) + : StorageDir_(ToFilePath(storagePath)) + , ProcessTempDir_(StorageDir_ / ToString(GetPID())) // must be subfolder for fast hardlinking + , IsTemp_(storagePath.empty()) + , MaxFiles_(maxFiles) + , MaxSize_(maxSize) + , CurrentFiles_(0) + , CurrentSize_(0) + , Dirty_(false) { // TFsPath is not thread safe. It can initialize internal Split at any time. Force do it right now - StorageDir.PathSplit(); - ProcessTempDir.PathSplit(); + StorageDir_.PathSplit(); + ProcessTempDir_.PathSplit(); - StorageDir.MkDirs(MODE0711); - ProcessTempDir.MkDirs(MODE0711); + StorageDir_.MkDirs(MODE0711); + ProcessTempDir_.MkDirs(MODE0711); #ifdef _linux_ - ProcessTempDirLock.Reset(new TFileLock(ProcessTempDir / ".lockfile")); - ProcessTempDirLock->Acquire(); + ProcessTempDirLock_.Reset(new TFileLock(ProcessTempDir_ / ".lockfile")); + ProcessTempDirLock_->Acquire(); // We never explicitly release this lock. It will be released when all file handles (including those in child processes) will be closed #endif - if (!IsTemp) { + if (!IsTemp_) { LoadStats(); } TAtforkReinit::Get().Register(this); - YQL_LOG(INFO) << "FileStorage initialized in " << StorageDir.GetPath().Quote() - << ", temporary dir: " << ProcessTempDir.GetPath().Quote() - << ", files: " << CurrentFiles.load() - << ", total size: " << CurrentSize.load(); + YQL_LOG(INFO) << "FileStorage initialized in " << StorageDir_.GetPath().Quote() + << ", temporary dir: " << ProcessTempDir_.GetPath().Quote() + << ", files: " << CurrentFiles_.load() + << ", total size: " << CurrentSize_.load(); } ~TImpl() { TAtforkReinit::Get().Unregister(this); try { - ProcessTempDir.ForceDelete(); - if (IsTemp) { - StorageDir.ForceDelete(); + ProcessTempDir_.ForceDelete(); + if (IsTemp_) { + StorageDir_.ForceDelete(); } } catch (...) { YQL_LOG(ERROR) << CurrentExceptionMessage(); @@ -194,19 +194,19 @@ public: } const TFsPath& GetRoot() const { - return StorageDir; + return StorageDir_; } const TFsPath& GetTemp() const { - return ProcessTempDir; + return ProcessTempDir_; } TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const NYql::NFS::TDataProvider& puller) { bool newFileAdded = false; TFileLinkPtr result = HardlinkFromStorage(storageFileName, md5, outFileName); if (!result) { - TFsPath storageFile = StorageDir / storageFileName; - TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); + TFsPath storageFile = StorageDir_ / storageFileName; + TFsPath hardlinkFile = ProcessTempDir_ / (outFileName ? outFileName : GetTempName()); Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); ui64 fileSize = 0; @@ -224,8 +224,8 @@ public: SetCacheFilePermissionsNoThrow(hardlinkFile); if (NFs::HardLink(hardlinkFile, storageFile)) { - ++CurrentFiles; - CurrentSize += fileSize; + ++CurrentFiles_; + CurrentSize_ += fileSize; } // Ignore HardLink fail. Another process managed to download before us TouchFile(storageFile.c_str()); @@ -248,8 +248,8 @@ public: } TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) { - TFsPath storageFile = StorageDir / existingStorageFileName; - TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName()); + TFsPath storageFile = StorageDir_ / existingStorageFileName; + TFsPath hardlinkFile = ProcessTempDir_ / (outFileName ? outFileName : GetTempName()); Y_ENSURE(!hardlinkFile.Exists(), "FileStorage: temporary file " << hardlinkFile.GetPath().Quote() << " already exists"); if (!NFs::HardLink(storageFile, hardlinkFile)) { @@ -274,7 +274,7 @@ public: } void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) { - TFsPath dstStorageFile = StorageDir / dstStorageFileName; + TFsPath dstStorageFile = StorageDir_ / dstStorageFileName; const bool prevFileExisted = dstStorageFile.Exists(); const i64 prevFileSize = Max<i64>(0, GetFileLength(dstStorageFile.c_str())); @@ -286,14 +286,14 @@ public: const i64 newFileSize = Max<i64>(0, GetFileLength(dstStorageFile.c_str())); if (!prevFileExisted) { - ++CurrentFiles; + ++CurrentFiles_; } - CurrentSize += newFileSize - prevFileSize; + CurrentSize_ += newFileSize - prevFileSize; } bool RemoveFromStorage(const TString& existingStorageFileName) { - TFsPath storageFile = StorageDir / existingStorageFileName; + TFsPath storageFile = StorageDir_ / existingStorageFileName; if (!storageFile.Exists()) { // can't update statistics // not sure we had this file at all @@ -305,31 +305,31 @@ public: const bool result = NFs::Remove(storageFile); if (result || !storageFile.Exists()) { - ++CurrentFiles; - CurrentSize -= prevFileSize; + ++CurrentFiles_; + CurrentSize_ -= prevFileSize; } return result; } ui64 GetOccupiedSize() const { - return CurrentSize.load(); + return CurrentSize_.load(); } size_t GetCount() const { - return CurrentFiles.load(); + return CurrentFiles_.load(); } TString GetTempName() { - with_lock(RndLock) { - return Rnd.GenGuid(); + with_lock(RndLock_) { + return Rnd_.GenGuid(); } } private: void LoadStats() { TVector<TString> names; - StorageDir.ListNames(names); + StorageDir_.ListNames(names); ui64 actualFiles = 0; ui64 actualSize = 0; @@ -337,7 +337,7 @@ private: ui32 oldPid; for (const TString& name: names) { - TFsPath childPath(StorageDir / name); + TFsPath childPath(StorageDir_ / name); TFileStat stat(childPath, true); if (stat.IsFile()) { ++actualFiles; @@ -366,25 +366,25 @@ private: } } - CurrentFiles = actualFiles; - CurrentSize = actualSize; + CurrentFiles_ = actualFiles; + CurrentSize_ = actualSize; } bool NeedToCleanup() const { - return Dirty.load() - || static_cast<ui64>(CurrentFiles.load()) > MaxFiles - || static_cast<ui64>(CurrentSize.load()) > MaxSize; + return Dirty_.load() + || static_cast<ui64>(CurrentFiles_.load()) > MaxFiles_ + || static_cast<ui64>(CurrentSize_.load()) > MaxSize_; } void Cleanup() { if (!NeedToCleanup()) { return; } - Dirty.store(false); + Dirty_.store(false); - with_lock (CleanupLock) { + with_lock (CleanupLock_) { TVector<TString> names; - StorageDir.ListNames(names); + StorageDir_.ListNames(names); TVector<TFileObject> files; files.reserve(names.size()); @@ -393,7 +393,7 @@ private: ui64 actualSize = 0; for (const TString& name: names) { - TFsPath childPath(StorageDir / name); + TFsPath childPath(StorageDir_ / name); TFileStat stat(childPath, true); if (stat.IsFile()) { files.push_back(TFileObject{name, stat.MTime, stat.Size}); @@ -410,8 +410,8 @@ private: return f1.MTime < f2.MTime; }); - ui64 filesThreshold = MaxFiles / 2; - ui64 sizeThreshold = MaxSize / 2; + ui64 filesThreshold = MaxFiles_ / 2; + ui64 sizeThreshold = MaxSize_ / 2; for (const TFileObject& f: files) { if (actualFiles <= filesThreshold && actualSize <= sizeThreshold) { @@ -421,7 +421,7 @@ private: YQL_LOG(INFO) << "Removing file from cache (name: " << f.Name << ", size: " << f.Size << ", mtime: " << f.MTime << ")"; - if (!NFs::Remove(StorageDir / f.Name)) { + if (!NFs::Remove(StorageDir_ / f.Name)) { YQL_LOG(WARN) << "Failed to remove file " << f.Name.Quote() << ": " << LastSystemErrorText(); } else { --actualFiles; @@ -429,37 +429,37 @@ private: } } - CurrentFiles.store(actualFiles); - CurrentSize.store(actualSize); + CurrentFiles_.store(actualFiles); + CurrentSize_.store(actualSize); } } void ResetAtFork() { - RndLock.Release(); - with_lock(RndLock) { - Rnd.ResetSeed(); + RndLock_.Release(); + with_lock(RndLock_) { + Rnd_.ResetSeed(); } // Force cleanup on next file add, because other processes may change the state - Dirty.store(true); + Dirty_.store(true); } private: - TMutex CleanupLock; - const TFsPath StorageDir; - const TFsPath ProcessTempDir; - THolder<TFileLock> ProcessTempDirLock; - const bool IsTemp; - const ui64 MaxFiles; - const ui64 MaxSize; - std::atomic<i64> CurrentFiles = 0; - std::atomic<i64> CurrentSize = 0; - std::atomic_bool Dirty; - TSpinLock RndLock; - TRandGuid Rnd; + TMutex CleanupLock_; + const TFsPath StorageDir_; + const TFsPath ProcessTempDir_; + THolder<TFileLock> ProcessTempDirLock_; + const bool IsTemp_; + const ui64 MaxFiles_; + const ui64 MaxSize_; + std::atomic<i64> CurrentFiles_ = 0; + std::atomic<i64> CurrentSize_ = 0; + std::atomic_bool Dirty_; + TSpinLock RndLock_; + TRandGuid Rnd_; }; TStorage::TStorage(size_t maxFiles, ui64 maxSize, const TString& storagePath) - : Impl(new TImpl(maxFiles, maxSize, storagePath)) + : Impl_(new TImpl(maxFiles, maxSize, storagePath)) { } @@ -469,46 +469,46 @@ TStorage::~TStorage() TFsPath TStorage::GetRoot() const { - return Impl->GetRoot(); + return Impl_->GetRoot(); } TFsPath TStorage::GetTemp() const { - return Impl->GetTemp(); + return Impl_->GetTemp(); } TFileLinkPtr TStorage::Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const NFS::TDataProvider& puller) { - return Impl->Put(storageFileName, outFileName, md5, puller); + return Impl_->Put(storageFileName, outFileName, md5, puller); } TFileLinkPtr TStorage::HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) { - return Impl->HardlinkFromStorage(existingStorageFileName, storageFileMd5, outFileName); + return Impl_->HardlinkFromStorage(existingStorageFileName, storageFileMd5, outFileName); } void TStorage::MoveToStorage(const TFsPath& src, const TString& dstStorageFileName) { - return Impl->MoveToStorage(src, dstStorageFileName); + return Impl_->MoveToStorage(src, dstStorageFileName); } bool TStorage::RemoveFromStorage(const TString& existingStorageFileName) { - return Impl->RemoveFromStorage(existingStorageFileName); + return Impl_->RemoveFromStorage(existingStorageFileName); } ui64 TStorage::GetOccupiedSize() const { - return Impl->GetOccupiedSize(); + return Impl_->GetOccupiedSize(); } size_t TStorage::GetCount() const { - return Impl->GetCount(); + return Impl_->GetCount(); } TString TStorage::GetTempName() { - return Impl->GetTempName(); + return Impl_->GetTempName(); } } // NYql diff --git a/yql/essentials/core/file_storage/storage.h b/yql/essentials/core/file_storage/storage.h index a8898c42aea..1bb562320da 100644 --- a/yql/essentials/core/file_storage/storage.h +++ b/yql/essentials/core/file_storage/storage.h @@ -16,17 +16,17 @@ public: TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy = true); ~TFileLink(); - const TString& GetStorageFileName() const { return StorageFileName; } - const TFsPath& GetPath() const { return Path; } - ui64 GetSize() const { return Size; } - const TString& GetMd5() const { return Md5; } + const TString& GetStorageFileName() const { return StorageFileName_; } + const TFsPath& GetPath() const { return Path_; } + ui64 GetSize() const { return Size_; } + const TString& GetMd5() const { return Md5_; } private: - const TFsPath Path; - const TString StorageFileName; - const ui64 Size; - const TString Md5; - const bool DeleteOnDestroy; + const TFsPath Path_; + const TString StorageFileName_; + const ui64 Size_; + const TString Md5_; + const bool DeleteOnDestroy_; }; using TFileLinkPtr = TIntrusivePtr<TFileLink>; @@ -67,7 +67,7 @@ public: private: class TImpl; - THolder<TImpl> Impl; + THolder<TImpl> Impl_; }; constexpr int MODE0711 = S_IRWXU | S_IXGRP | S_IXOTH; diff --git a/yql/essentials/core/histogram/eq_width_histogram.cpp b/yql/essentials/core/histogram/eq_width_histogram.cpp index 3c5a452fdbd..b83fc1638c6 100644 --- a/yql/essentials/core/histogram/eq_width_histogram.cpp +++ b/yql/essentials/core/histogram/eq_width_histogram.cpp @@ -3,7 +3,7 @@ namespace NKikimr { TEqWidthHistogram::TEqWidthHistogram(ui32 numBuckets, EHistogramValueType valueType) - : ValueType(valueType), Buckets(numBuckets) { + : ValueType_(valueType), Buckets_(numBuckets) { // Exptected at least one bucket for histogram. Y_ASSERT(numBuckets >= 1); } @@ -13,11 +13,11 @@ TEqWidthHistogram::TEqWidthHistogram(const char *str, ui64 size) { const ui32 numBuckets = *reinterpret_cast<const ui32 *>(str); Y_ABORT_UNLESS(GetBinarySize(numBuckets) == size); ui32 offset = sizeof(ui32); - ValueType = *reinterpret_cast<const EHistogramValueType *>(str + offset); + ValueType_ = *reinterpret_cast<const EHistogramValueType *>(str + offset); offset += sizeof(EHistogramValueType); - Buckets = TVector<TBucket>(numBuckets); + Buckets_ = TVector<TBucket>(numBuckets); for (ui32 i = 0; i < numBuckets; ++i) { - std::memcpy(&Buckets[i], reinterpret_cast<const char *>(str + offset), sizeof(TBucket)); + std::memcpy(&Buckets_[i], reinterpret_cast<const char *>(str + offset), sizeof(TBucket)); offset += sizeof(TBucket); } } @@ -38,36 +38,36 @@ std::unique_ptr<char> TEqWidthHistogram::Serialize(ui64 &binarySize) const { std::memcpy(binaryData.get(), &numBuckets, sizeof(ui32)); offset += sizeof(ui32); // 1 byte - values type. - std::memcpy(binaryData.get() + offset, &ValueType, sizeof(EHistogramValueType)); + std::memcpy(binaryData.get() + offset, &ValueType_, sizeof(EHistogramValueType)); offset += sizeof(EHistogramValueType); // Buckets. for (ui32 i = 0; i < numBuckets; ++i) { - std::memcpy(binaryData.get() + offset, &Buckets[i], sizeof(TBucket)); + std::memcpy(binaryData.get() + offset, &Buckets_[i], sizeof(TBucket)); offset += sizeof(TBucket); } return binaryData; } TEqWidthHistogramEstimator::TEqWidthHistogramEstimator(std::shared_ptr<TEqWidthHistogram> histogram) - : Histogram(histogram) { - const auto numBuckets = Histogram->GetNumBuckets(); - PrefixSum = TVector<ui64>(numBuckets); - SuffixSum = TVector<ui64>(numBuckets); + : Histogram_(histogram) { + const auto numBuckets = Histogram_->GetNumBuckets(); + PrefixSum_ = TVector<ui64>(numBuckets); + SuffixSum_ = TVector<ui64>(numBuckets); CreatePrefixSum(numBuckets); CreateSuffixSum(numBuckets); } void TEqWidthHistogramEstimator::CreatePrefixSum(ui32 numBuckets) { - PrefixSum[0] = Histogram->GetNumElementsInBucket(0); + PrefixSum_[0] = Histogram_->GetNumElementsInBucket(0); for (ui32 i = 1; i < numBuckets; ++i) { - PrefixSum[i] = PrefixSum[i - 1] + Histogram->GetNumElementsInBucket(i); + PrefixSum_[i] = PrefixSum_[i - 1] + Histogram_->GetNumElementsInBucket(i); } } void TEqWidthHistogramEstimator::CreateSuffixSum(ui32 numBuckets) { - SuffixSum[numBuckets - 1] = Histogram->GetNumElementsInBucket(numBuckets - 1); + SuffixSum_[numBuckets - 1] = Histogram_->GetNumElementsInBucket(numBuckets - 1); for (i32 i = static_cast<i32>(numBuckets) - 2; i >= 0; --i) { - SuffixSum[i] = SuffixSum[i + 1] + Histogram->GetNumElementsInBucket(i); + SuffixSum_[i] = SuffixSum_[i + 1] + Histogram_->GetNumElementsInBucket(i); } } } // namespace NKikimr diff --git a/yql/essentials/core/histogram/eq_width_histogram.h b/yql/essentials/core/histogram/eq_width_histogram.h index 97c660af76b..261f2aaafc3 100644 --- a/yql/essentials/core/histogram/eq_width_histogram.h +++ b/yql/essentials/core/histogram/eq_width_histogram.h @@ -68,11 +68,11 @@ class TEqWidthHistogram { void AddElement(T val) { const auto index = FindBucketIndex(val); // The given `index` in range [0, numBuckets - 1]. - const T bucketValue = LoadFrom<T>(Buckets[index].Start); + const T bucketValue = LoadFrom<T>(Buckets_[index].Start); if (!index || ((CmpEqual<T>(bucketValue, val) || CmpLess<T>(bucketValue, val)))) { - Buckets[index].Count++; + Buckets_[index].Count++; } else { - Buckets[index - 1].Count++; + Buckets_[index - 1].Count++; } } @@ -85,7 +85,7 @@ class TEqWidthHistogram { ui32 end = GetNumBuckets() - 1; while (start < end) { auto it = start + (end - start) / 2; - if (CmpLess<T>(LoadFrom<T>(Buckets[it].Start), val)) { + if (CmpLess<T>(LoadFrom<T>(Buckets_[it].Start), val)) { start = it + 1; } else { end = it; @@ -95,15 +95,15 @@ class TEqWidthHistogram { } // Returns a number of buckets in a histogram. - ui32 GetNumBuckets() const { return Buckets.size(); } + ui32 GetNumBuckets() const { return Buckets_.size(); } template <typename T> ui32 GetBucketWidth() const { Y_ASSERT(GetNumBuckets()); if (GetNumBuckets() == 1) { - return std::max(static_cast<ui32>(LoadFrom<T>(Buckets.front().Start)), 1U); + return std::max(static_cast<ui32>(LoadFrom<T>(Buckets_.front().Start)), 1U); } else { - return std::max(static_cast<ui32>(LoadFrom<T>(Buckets[1].Start) - LoadFrom<T>(Buckets[0].Start)), 1U); + return std::max(static_cast<ui32>(LoadFrom<T>(Buckets_[1].Start) - LoadFrom<T>(Buckets_[0].Start)), 1U); } } @@ -113,46 +113,46 @@ class TEqWidthHistogram { } // Returns histogram type. - EHistogramValueType GetType() const { return ValueType; } + EHistogramValueType GetType() const { return ValueType_; } // Returns a number of elements in a bucket by the given `index`. - ui64 GetNumElementsInBucket(ui32 index) const { return Buckets[index].Count; } + ui64 GetNumElementsInBucket(ui32 index) const { return Buckets_[index].Count; } // Initializes buckets with a given `range`. template <typename T> void InitializeBuckets(const TBucketRange &range) { Y_ASSERT(CmpLess<T>(LoadFrom<T>(range.Start), LoadFrom<T>(range.End))); T rangeLen = LoadFrom<T>(range.End) - LoadFrom<T>(range.Start); - std::memcpy(Buckets[0].Start, range.Start, sizeof(range.Start)); + std::memcpy(Buckets_[0].Start, range.Start, sizeof(range.Start)); for (ui32 i = 1; i < GetNumBuckets(); ++i) { - const T prevStart = LoadFrom<T>(Buckets[i - 1].Start); - StoreTo<T>(Buckets[i].Start, prevStart + rangeLen); + const T prevStart = LoadFrom<T>(Buckets_[i - 1].Start); + StoreTo<T>(Buckets_[i].Start, prevStart + rangeLen); } } // Seriailizes to a binary representation std::unique_ptr<char> Serialize(ui64 &binSize) const; // Returns buckets. - const TVector<TBucket> &GetBuckets() const { return Buckets; } + const TVector<TBucket> &GetBuckets() const { return Buckets_; } template <typename T> void Aggregate(const TEqWidthHistogram &other) { - if ((this->ValueType != other.GetType()) || (!BucketsEqual<T>(other))) { + if ((this->ValueType_ != other.GetType()) || (!BucketsEqual<T>(other))) { // Should we fail? return; } - for (ui32 i = 0; i < Buckets.size(); ++i) { - Buckets[i].Count += other.GetBuckets()[i].Count; + for (ui32 i = 0; i < Buckets_.size(); ++i) { + Buckets_[i].Count += other.GetBuckets()[i].Count; } } private: template <typename T> bool BucketsEqual(const TEqWidthHistogram &other) { - if (Buckets.size() != other.GetNumBuckets()) { + if (Buckets_.size() != other.GetNumBuckets()) { return false; } - for (ui32 i = 0; i < Buckets.size(); ++i) { - if (!CmpEqual<T>(LoadFrom<T>(Buckets[i].Start), LoadFrom<T>(GetBuckets()[i].Start))) { + for (ui32 i = 0; i < Buckets_.size(); ++i) { + if (!CmpEqual<T>(LoadFrom<T>(Buckets_[i].Start), LoadFrom<T>(GetBuckets()[i].Start))) { return false; } } @@ -161,8 +161,8 @@ class TEqWidthHistogram { // Returns binary size of the histogram. ui64 GetBinarySize(ui32 nBuckets) const; - EHistogramValueType ValueType; - TVector<TBucket> Buckets; + EHistogramValueType ValueType_; + TVector<TBucket> Buckets_; }; // This class represents a machinery to estimate a value in a histogram. @@ -173,45 +173,45 @@ class TEqWidthHistogramEstimator { // Methods to estimate values. template <typename T> ui64 EstimateLessOrEqual(T val) const { - return EstimateOrEqual<T>(val, PrefixSum); + return EstimateOrEqual<T>(val, PrefixSum_); } template <typename T> ui64 EstimateGreaterOrEqual(T val) const { - return EstimateOrEqual<T>(val, SuffixSum); + return EstimateOrEqual<T>(val, SuffixSum_); } template <typename T> ui64 EstimateLess(T val) const { - return EstimateNotEqual<T>(val, PrefixSum); + return EstimateNotEqual<T>(val, PrefixSum_); } template <typename T> ui64 EstimateGreater(T val) const { - return EstimateNotEqual<T>(val, SuffixSum); + return EstimateNotEqual<T>(val, SuffixSum_); } template <typename T> ui64 EstimateEqual(T val) const { - const auto index = Histogram->FindBucketIndex(val); + const auto index = Histogram_->FindBucketIndex(val); // Assuming uniform distribution. - return std::max(1U, static_cast<ui32>(Histogram->GetNumElementsInBucket(index) / Histogram->template GetBucketWidth<T>())); + return std::max(1U, static_cast<ui32>(Histogram_->GetNumElementsInBucket(index) / Histogram_->template GetBucketWidth<T>())); } // Returns the total number elements in histogram. // Could be used to adjust scale. - ui64 GetNumElements() const { return PrefixSum.back(); } + ui64 GetNumElements() const { return PrefixSum_.back(); } private: template <typename T> ui64 EstimateOrEqual(T val, const TVector<ui64> &sumArray) const { - const auto index = Histogram->FindBucketIndex(val); + const auto index = Histogram_->FindBucketIndex(val); return sumArray[index]; } template <typename T> ui64 EstimateNotEqual(T val, const TVector<ui64> &sumArray) const { - const auto index = Histogram->FindBucketIndex(val); + const auto index = Histogram_->FindBucketIndex(val); // Take the previous backet if it's not the first one. if (!index) { return sumArray[index]; @@ -221,8 +221,8 @@ class TEqWidthHistogramEstimator { void CreatePrefixSum(ui32 numBuckets); void CreateSuffixSum(ui32 numBuckets); - std::shared_ptr<TEqWidthHistogram> Histogram; - TVector<ui64> PrefixSum; - TVector<ui64> SuffixSum; + std::shared_ptr<TEqWidthHistogram> Histogram_; + TVector<ui64> PrefixSum_; + TVector<ui64> SuffixSum_; }; } // namespace NKikimr diff --git a/yql/essentials/core/minsketch/count_min_sketch.cpp b/yql/essentials/core/minsketch/count_min_sketch.cpp index 90d36f51da0..2648154a73c 100644 --- a/yql/essentials/core/minsketch/count_min_sketch.cpp +++ b/yql/essentials/core/minsketch/count_min_sketch.cpp @@ -11,15 +11,15 @@ TCountMinSketch* TCountMinSketch::Create(ui64 width, ui64 depth) { auto* data = ::malloc(size); auto* sketch = reinterpret_cast<TCountMinSketch*>(data); std::memset(sketch, 0, size); - sketch->Width = width; - sketch->Depth = depth; - sketch->ElementCount = 0; + sketch->Width_ = width; + sketch->Depth_ = depth; + sketch->ElementCount_ = 0; return sketch; } TCountMinSketch* TCountMinSketch::FromString(const char* data, size_t size) { auto* from = reinterpret_cast<const TCountMinSketch*>(data); - Y_ABORT_UNLESS(StaticSize(from->Width, from->Depth) == size); + Y_ABORT_UNLESS(StaticSize(from->Width_, from->Depth_) == size); auto* dataDst = ::malloc(size); std::memcpy(dataDst, data, size); return reinterpret_cast<TCountMinSketch*>(dataDst); @@ -42,34 +42,34 @@ ui64 TCountMinSketch::Hash(const char* data, size_t size, size_t hashIndex) { void TCountMinSketch::Count(const char* data, size_t size) { ui32* start = Buckets(); - for (size_t d = 0; d < Depth; ++d, start += Width) { + for (size_t d = 0; d < Depth_; ++d, start += Width_) { ui64 hash = Hash(data, size, d); - ui32* bucket = start + hash % Width; + ui32* bucket = start + hash % Width_; if (Y_LIKELY(*bucket < std::numeric_limits<ui32>::max())) { ++*bucket; } } - ++ElementCount; + ++ElementCount_; } ui32 TCountMinSketch::Probe(const char* data, size_t size) const { ui32 minValue = std::numeric_limits<ui32>::max(); const ui32* start = Buckets(); - for (size_t d = 0; d < Depth; ++d, start += Width) { + for (size_t d = 0; d < Depth_; ++d, start += Width_) { ui64 hash = Hash(data, size, d); - const ui32* bucket = start + hash % Width; + const ui32* bucket = start + hash % Width_; minValue = std::min(minValue, *bucket); } return minValue; } TCountMinSketch& TCountMinSketch::operator+=(const TCountMinSketch& rhs) { - if (Width != rhs.Width || Depth != rhs.Depth) { + if (Width_ != rhs.Width_ || Depth_ != rhs.Depth_) { return *this; } ui32* dst = Buckets(); const ui32* src = rhs.Buckets(); - ui32* end = dst + Width * Depth; + ui32* end = dst + Width_ * Depth_; for (; dst != end; ++dst, ++src) { ui32 sum = *dst + *src; if (Y_UNLIKELY(sum < *dst)) { @@ -78,7 +78,7 @@ TCountMinSketch& TCountMinSketch::operator+=(const TCountMinSketch& rhs) { *dst = sum; } } - ElementCount += rhs.ElementCount; + ElementCount_ += rhs.ElementCount_; return *this; } diff --git a/yql/essentials/core/minsketch/count_min_sketch.h b/yql/essentials/core/minsketch/count_min_sketch.h index 4b8866d3d92..88e3962b330 100644 --- a/yql/essentials/core/minsketch/count_min_sketch.h +++ b/yql/essentials/core/minsketch/count_min_sketch.h @@ -7,9 +7,9 @@ namespace NKikimr { class TCountMinSketch { private: - ui64 Width; - ui64 Depth; - ui64 ElementCount; + ui64 Width_; + ui64 Depth_; + ui64 ElementCount_; private: static ui64 Hash(const char* data, size_t size, size_t hashIndex); @@ -36,19 +36,19 @@ public: TCountMinSketch(const TCountMinSketch&) = delete; size_t GetSize() const { - return StaticSize(Width, Depth); + return StaticSize(Width_, Depth_); } size_t GetWidth() const { - return Width; + return Width_; } size_t GetDepth() const { - return Depth; + return Depth_; } size_t GetElementCount() const { - return ElementCount; + return ElementCount_; } TStringBuf AsStringBuf() const { diff --git a/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.cpp b/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.cpp index e8d56d75f50..9fc2431638b 100644 --- a/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.cpp +++ b/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.cpp @@ -16,19 +16,19 @@ TVector<TQItem> DrainIterator(IQIterator& iterator) { return res; } -void QStorageTestEmpty_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestEmptyImpl(const NYql::IQStoragePtr& storage) { auto reader = storage->MakeReader("foo", {}); UNIT_ASSERT(!reader->Get({"comp", "label"}).GetValueSync().Defined()); auto iterator = storage->MakeIterator("foo", {}); UNIT_ASSERT(!iterator->Next().GetValueSync().Defined()); } -void QStorageTestNoCommit_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestNoCommitImpl(const NYql::IQStoragePtr& storage) { auto writer = storage->MakeWriter("foo", {}); writer->Put({"comp", "label"}, "value").GetValueSync(); } -void QStorageTestOne_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestOneImpl(const NYql::IQStoragePtr& storage) { auto writer = storage->MakeWriter("foo", {}); writer->Put({"comp", "label"}, "value").GetValueSync(); writer->Commit().GetValueSync(); @@ -48,7 +48,7 @@ void QStorageTestOne_Impl(const NYql::IQStoragePtr& storage) { UNIT_ASSERT(!value.Defined()); } -void QStorageTestManyKeys_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestManyKeysImpl(const NYql::IQStoragePtr& storage) { const size_t N = 10; auto writer = storage->MakeWriter("foo", {}); for (size_t i = 0; i < N; ++i) { @@ -76,7 +76,7 @@ void QStorageTestManyKeys_Impl(const NYql::IQStoragePtr& storage) { } } -void QStorageTestInterleaveReadWrite_Impl(const NYql::IQStoragePtr& storage, bool commit) { +void QStorageTestInterleaveReadWriteImpl(const NYql::IQStoragePtr& storage, bool commit) { auto reader = storage->MakeReader("foo", {}); auto value = reader->Get({"comp", "label"}).GetValueSync(); UNIT_ASSERT(!value.Defined()); @@ -108,7 +108,7 @@ void QStorageTestInterleaveReadWrite_Impl(const NYql::IQStoragePtr& storage, boo UNIT_ASSERT(!value.Defined()); } -void QStorageTestLimitWriterItems_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestLimitWriterItemsImpl(const NYql::IQStoragePtr& storage) { TQWriterSettings settings; settings.ItemsLimit = 1; auto writer = storage->MakeWriter("foo", settings); @@ -117,7 +117,7 @@ void QStorageTestLimitWriterItems_Impl(const NYql::IQStoragePtr& storage) { UNIT_ASSERT_EXCEPTION(writer->Commit().GetValueSync(), yexception); } -void QStorageTestLimitWriterBytes_Impl(const NYql::IQStoragePtr& storage) { +void QStorageTestLimitWriterBytesImpl(const NYql::IQStoragePtr& storage) { TQWriterSettings settings; settings.BytesLimit = 7; auto writer = storage->MakeWriter("foo", settings); diff --git a/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.h b/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.h index 729ac66bd82..85056cb8967 100644 --- a/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.h +++ b/yql/essentials/core/qplayer/storage/ut_common/yql_qstorage_ut_common.h @@ -3,19 +3,19 @@ #include <library/cpp/testing/unittest/registar.h> -void QStorageTestEmpty_Impl(const NYql::IQStoragePtr& storage); -void QStorageTestNoCommit_Impl(const NYql::IQStoragePtr& storage); -void QStorageTestOne_Impl(const NYql::IQStoragePtr& storage); -void QStorageTestManyKeys_Impl(const NYql::IQStoragePtr& storage); -void QStorageTestInterleaveReadWrite_Impl(const NYql::IQStoragePtr& storage, bool commit); -void QStorageTestLimitWriterItems_Impl(const NYql::IQStoragePtr& storage); -void QStorageTestLimitWriterBytes_Impl(const NYql::IQStoragePtr& storage); +void QStorageTestEmptyImpl(const NYql::IQStoragePtr& storage); +void QStorageTestNoCommitImpl(const NYql::IQStoragePtr& storage); +void QStorageTestOneImpl(const NYql::IQStoragePtr& storage); +void QStorageTestManyKeysImpl(const NYql::IQStoragePtr& storage); +void QStorageTestInterleaveReadWriteImpl(const NYql::IQStoragePtr& storage, bool commit); +void QStorageTestLimitWriterItemsImpl(const NYql::IQStoragePtr& storage); +void QStorageTestLimitWriterBytesImpl(const NYql::IQStoragePtr& storage); #define GENERATE_ONE_TEST(NAME, FACTORY) \ Y_UNIT_TEST(NAME) { \ auto storage = FACTORY(); \ if (storage) { \ - QStorageTest##NAME##_Impl(storage); \ + QStorageTest##NAME##Impl(storage); \ } \ } @@ -23,9 +23,9 @@ void QStorageTestLimitWriterBytes_Impl(const NYql::IQStoragePtr& storage); Y_UNIT_TEST(NAME) { \ auto storage = FACTORY(); \ if (storage) { \ - QStorageTest##NAME##_Impl(storage, OPT); \ + QStorageTest##NAME##Impl(storage, OPT); \ } \ - } + } #define GENERATE_TESTS(FACTORY, commit)\ GENERATE_ONE_TEST(Empty, FACTORY) \ diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp index 95dfbf3672e..70114fc11e8 100644 --- a/yql/essentials/core/services/mounts/yql_mounts.cpp +++ b/yql/essentials/core/services/mounts/yql_mounts.cpp @@ -26,13 +26,13 @@ namespace NYql { } TUserDataKey CreateKey(const NUserData::TUserData& item) { - TString name = (item.Disposition_ == EDisposition::RESOURCE) ? item.Content_ : item.Name_; + TString name = (item.Disposition == EDisposition::RESOURCE) ? item.Content : item.Name; if (!name.StartsWith('/')) { name = GetDefaultFilePrefix() + name; } - if (item.Type_ == EType::UDF) { + if (item.Type == EType::UDF) { return TUserDataKey::Udf(name); } else { return TUserDataKey::File(name); @@ -45,29 +45,29 @@ namespace NYql { auto& block = userDataTable[CreateKey(item)]; - switch (item.Disposition_) { + switch (item.Disposition) { case EDisposition::INLINE: - block.Data = item.Content_; + block.Data = item.Content; block.Type = EUserDataType::RAW_INLINE_DATA; break; case EDisposition::RESOURCE: case EDisposition::RESOURCE_FILE: - block.Data = NResource::Find(item.Content_); + block.Data = NResource::Find(item.Content); block.Type = EUserDataType::RAW_INLINE_DATA; break; case EDisposition::FILESYSTEM: - block.Data = item.Content_; + block.Data = item.Content; block.Type = EUserDataType::PATH; break; case EDisposition::URL: - block.Data = item.Content_; + block.Data = item.Content; block.Type = EUserDataType::URL; break; default: - ythrow yexception() << "Unknown disposition for user data \"" << item.Name_ << "\": " << item.Disposition_; + ythrow yexception() << "Unknown disposition for user data \"" << item.Name << "\": " << item.Disposition; } - switch (item.Type_) { + switch (item.Type) { case EType::LIBRARY: block.Usage.Set(EUserDataBlockUsage::Library, true); break; @@ -78,28 +78,28 @@ namespace NYql { block.Usage.Set(EUserDataBlockUsage::Udf, true); break; default: - ythrow yexception() << "Unknown type for user data \"" << item.Name_ << "\": " << item.Type_; + ythrow yexception() << "Unknown type for user data \"" << item.Name << "\": " << item.Type; } - if (item.Type_ == EType::LIBRARY) { - switch (item.Disposition_) { + if (item.Type == EType::LIBRARY) { + switch (item.Disposition) { case EDisposition::INLINE: case EDisposition::RESOURCE: case EDisposition::RESOURCE_FILE: case EDisposition::FILESYSTEM: { - if (item.Disposition_ == EDisposition::FILESYSTEM) { + if (item.Disposition == EDisposition::FILESYSTEM) { TFsPath path(block.Data); if (path.Exists() && path.IsFile()) { TFileInput input(path); block.Data = input.ReadAll(); } else { - ythrow yexception() << "File for user data \"" << item.Name_ << "\" does not exist: " << block.Data; + ythrow yexception() << "File for user data \"" << item.Name << "\" does not exist: " << block.Data; } block.Type = EUserDataType::RAW_INLINE_DATA; } } break; default: - ythrow yexception() << item.Disposition_ << " disposition is not yet supported for libraries (specified for \"" << item.Name_ << "\")"; + ythrow yexception() << item.Disposition << " disposition is not yet supported for libraries (specified for \"" << item.Name << "\")"; } } } diff --git a/yql/essentials/core/services/yql_eval_expr.cpp b/yql/essentials/core/services/yql_eval_expr.cpp index 4681a276646..ce6749c6418 100644 --- a/yql/essentials/core/services/yql_eval_expr.cpp +++ b/yql/essentials/core/services/yql_eval_expr.cpp @@ -108,9 +108,9 @@ public: if (pending) { const TStringBuf command = n.Child(2)->Content(); if (command == "AddFileByUrl") { - PendingFileAliases.insert(n.Child(3)->Content()); + PendingFileAliases_.insert(n.Child(3)->Content()); } else if (command == "AddFolderByUrl") { - PendingFolderPrefixes.insert(n.Child(3)->Content()); + PendingFolderPrefixes_.insert(n.Child(3)->Content()); } } } @@ -122,7 +122,7 @@ public: private: void ScanImpl(const TExprNode& node) { - if (!Visited.emplace(&node).second) { + if (!Visited_.emplace(&node).second) { return; } @@ -139,11 +139,11 @@ private: static THashSet<TStringBuf> FILE_CALLABLES = {"FilePath", "FileContent", "FolderPath"}; if (node.IsCallable(FILE_CALLABLES)) { const auto alias = node.Head().Content(); - if (PendingFileAliases.contains(alias) || AnyOf(PendingFolderPrefixes, [alias](const TStringBuf prefix) { + if (PendingFileAliases_.contains(alias) || AnyOf(PendingFolderPrefixes_, [alias](const TStringBuf prefix) { auto withSlash = TString(prefix) + "/"; return alias.StartsWith(withSlash); })) { - for (auto& curr: CurrentEvalNodes) { + for (auto& curr: CurrentEvalNodes_) { Reachable.erase(curr); } HasConfigPending = true; @@ -158,14 +158,14 @@ private: bool pop = false; if (node.IsCallable(EvaluationFuncs) || node.IsCallable(SubqueryExpandFuncs)) { Reachable.insert(&node); - CurrentEvalNodes.insert(&node); + CurrentEvalNodes_.insert(&node); pop = true; } if (node.IsCallable({ "EvaluateIf!", "EvaluateFor!", "EvaluateParallelFor!" })) { // scan predicate/list only if (node.ChildrenSize() > 1) { - CurrentEvalNodes.insert(&node); + CurrentEvalNodes_.insert(&node); pop = true; ScanImpl(*node.Child(1)); } @@ -177,7 +177,7 @@ private: } if (node.ChildrenSize() > index) { if (node.Child(index)->IsCallable(EvaluationFuncs)) { - CurrentEvalNodes.insert(&node); + CurrentEvalNodes_.insert(&node); pop = true; ScanImpl(*node.Child(index)); } else { @@ -192,15 +192,15 @@ private: } } if (pop) { - CurrentEvalNodes.erase(&node); + CurrentEvalNodes_.erase(&node); } } private: - TNodeSet Visited; - THashSet<TStringBuf> PendingFileAliases; - THashSet<TStringBuf> PendingFolderPrefixes; - TNodeSet CurrentEvalNodes; + TNodeSet Visited_; + THashSet<TStringBuf> PendingFileAliases_; + THashSet<TStringBuf> PendingFolderPrefixes_; + TNodeSet CurrentEvalNodes_; }; struct TEvalScope { diff --git a/yql/essentials/core/services/yql_out_transformers.cpp b/yql/essentials/core/services/yql_out_transformers.cpp index cc7b6bfd4e3..a8d7d63b47e 100644 --- a/yql/essentials/core/services/yql_out_transformers.cpp +++ b/yql/essentials/core/services/yql_out_transformers.cpp @@ -52,14 +52,14 @@ IGraphTransformer::TStatus TExprLogTransformer::operator()( { Y_UNUSED(ctx); output = input; - if (YQL_CVLOG_ACTIVE(Level, Component)) { + if (YQL_CVLOG_ACTIVE(Level_, Component_)) { TConvertToAstSettings settings; settings.AllowFreeArgs = true; settings.RefAtoms = true; auto ast = ConvertToAst(*input, ctx, settings); TStringStream out; ast.Root->PrettyPrintTo(out, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine); - YQL_CVLOG(Level, Component) << Description << ":\n" << out.Str(); + YQL_CVLOG(Level_, Component_) << Description_ << ":\n" << out.Str(); } return IGraphTransformer::TStatus::Ok; } diff --git a/yql/essentials/core/services/yql_out_transformers.h b/yql/essentials/core/services/yql_out_transformers.h index 893b0890c2d..6c83199fdec 100644 --- a/yql/essentials/core/services/yql_out_transformers.h +++ b/yql/essentials/core/services/yql_out_transformers.h @@ -70,9 +70,9 @@ private: class TExprLogTransformer { public: TExprLogTransformer(const TString& description, NYql::NLog::EComponent component, NYql::NLog::ELevel level) - : Description(description) - , Component(component) - , Level(level) {} + : Description_(description) + , Component_(component) + , Level_(level) {} NYql::IGraphTransformer::TStatus operator()(const NYql::TExprNode::TPtr& input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx); @@ -85,9 +85,9 @@ public: } private: - TString Description; - NYql::NLog::EComponent Component; - NYql::NLog::ELevel Level; + TString Description_; + NYql::NLog::EComponent Component_; + NYql::NLog::ELevel Level_; }; } // namespace NYql diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 75043a88ede..1cf269402d9 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -13283,7 +13283,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> class TIntentDeterminationTransformer : public TSyncTransformerBase { public: TIntentDeterminationTransformer(const TTypeAnnotationContext& types) - : Types(types) + : Types_(types) {} IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { @@ -13299,7 +13299,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> bool foundFunc = false; - for (auto& datasource : Types.DataSources) { + for (auto& datasource : Types_.DataSources) { if (!datasource->CanParse(*input)) { continue; } @@ -13309,7 +13309,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } if (!foundFunc) { - for (auto& datasink : Types.DataSinks) { + for (auto& datasink : Types_.DataSinks) { if (!datasink->CanParse(*input)) { continue; } @@ -13332,10 +13332,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } void Rewind() final { - for (auto& x : Types.DataSources) { + for (auto& x : Types_.DataSources) { x->GetIntentDeterminationTransformer().Rewind(); } - for (auto& x : Types.DataSinks) { + for (auto& x : Types_.DataSinks) { x->GetIntentDeterminationTransformer().Rewind(); } } @@ -13347,7 +13347,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } private: - const TTypeAnnotationContext& Types; + const TTypeAnnotationContext& Types_; }; class TExtCallableTypeAnnotationTransformer : public TCallableTransformerBase<TExtCallableTypeAnnotationTransformer> { @@ -13366,15 +13366,15 @@ template <NKikimr::NUdf::EDataSlot DataSlot> TContext funcCtx(ctx); status = (*func)(input, output, funcCtx); } else if (auto func = extFunctions.FindPtr(name)) { - TExtContext funcCtx(ctx, Types); + TExtContext funcCtx(ctx, Types_); status = (*func)(input, output, funcCtx); } else { return Nothing(); } - if (status == IGraphTransformer::TStatus::Ok && Types.DeriveColumnOrder && !Types.LookupColumnOrder(*input)) { + if (status == IGraphTransformer::TStatus::Ok && Types_.DeriveColumnOrder && !Types_.LookupColumnOrder(*input)) { if (auto func = columnOrderFunctions.FindPtr(name)) { - TExtContext funcCtx(ctx, Types); + TExtContext funcCtx(ctx, Types_); status = (*func)(input, output, funcCtx); } } @@ -13428,7 +13428,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> } IGraphTransformer& GetTransformer(IDataProvider& provider) const { - return provider.GetTypeAnnotationTransformer(InstantOnly); + return provider.GetTypeAnnotationTransformer(InstantOnly_); } }; } // namespace NTypeAnnInpl diff --git a/yql/essentials/core/type_ann/type_ann_expr.cpp b/yql/essentials/core/type_ann/type_ann_expr.cpp index 5f847223c72..88ca32b8329 100644 --- a/yql/essentials/core/type_ann/type_ann_expr.cpp +++ b/yql/essentials/core/type_ann/type_ann_expr.cpp @@ -21,17 +21,17 @@ class TTypeAnnotationTransformer : public TGraphTransformerBase { public: TTypeAnnotationTransformer(TAutoPtr<IGraphTransformer> callableTransformer, TTypeAnnotationContext& types, ETypeCheckMode mode) - : CallableTransformer(callableTransformer) - , Types(types) - , Mode(mode) + : CallableTransformer_(callableTransformer) + , Types_(types) + , Mode_(mode) { } ~TTypeAnnotationTransformer() { if (PrintCallableTimes) { std::vector<std::pair<TStringBuf, std::pair<ui64, ui64>>> pairs; - pairs.reserve(CallableTimes.size()); - for (auto& x : CallableTimes) { + pairs.reserve(CallableTimes_.size()); + for (auto& x : CallableTimes_) { pairs.emplace_back(x.first, x.second); } @@ -47,12 +47,12 @@ public: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { YQL_PROFILE_SCOPE(DEBUG, "TypeAnnotationTransformer::DoTransform"); output = input; - if (Mode == ETypeCheckMode::Initial && IsComplete) { + if (Mode_ == ETypeCheckMode::Initial && IsComplete_) { return TStatus::Ok; } - if (IsOptimizerEnabled<KeepWorldOptName>(Types) && !IsOptimizerDisabled<KeepWorldOptName>(Types)) { - KeepWorldEnabled = true; + if (IsOptimizerEnabled<KeepWorldOptName>(Types_) && !IsOptimizerDisabled<KeepWorldOptName>(Types_)) { + KeepWorldEnabled_ = true; } auto status = TransformNode(input, output, ctx); @@ -61,22 +61,22 @@ public: WriteRepeatCallableCount(); } - if (status.Level != TStatus::Error && HasRenames) { - output = ctx.ReplaceNodes(std::move(output), Processed); + if (status.Level != TStatus::Error && HasRenames_) { + output = ctx.ReplaceNodes(std::move(output), Processed_); } - Processed.clear(); + Processed_.clear(); if (status == TStatus::Ok) { - Types.ExpectedTypes.clear(); - Types.ExpectedColumnOrders.clear(); + Types_.ExpectedTypes.clear(); + Types_.ExpectedColumnOrders.clear(); } - HasRenames = false; - if (Mode == ETypeCheckMode::Initial && status == TStatus::Ok) { - IsComplete = true; + HasRenames_ = false; + if (Mode_ == ETypeCheckMode::Initial && status == TStatus::Ok) { + IsComplete_ = true; } - if (Mode == ETypeCheckMode::Repeat) { + if (Mode_ == ETypeCheckMode::Repeat) { CheckFatalTypeError(status); } @@ -87,8 +87,8 @@ public: YQL_PROFILE_SCOPE(DEBUG, "TypeAnnotationTransformer::DoGetAsyncFuture"); Y_UNUSED(input); TVector<NThreading::TFuture<void>> futures; - for (const auto& callable : CallableInputs) { - futures.push_back(CallableTransformer->GetAsyncFuture(*callable)); + for (const auto& callable : CallableInputs_) { + futures.push_back(CallableTransformer_->GetAsyncFuture(*callable)); } return WaitExceptionOrAll(futures); @@ -98,10 +98,10 @@ public: YQL_PROFILE_SCOPE(DEBUG, "TypeAnnotationTransformer::DoApplyAsyncChanges"); output = input; TStatus combinedStatus = TStatus::Ok; - for (const auto& callable : CallableInputs) { + for (const auto& callable : CallableInputs_) { callable->SetState(TExprNode::EState::TypePending); TExprNode::TPtr callableOutput; - auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx); + auto status = CallableTransformer_->ApplyAsyncChanges(callable, callableOutput, ctx); Y_ABORT_UNLESS(callableOutput); YQL_ENSURE(status != TStatus::Async); YQL_ENSURE(callableOutput == callable); @@ -111,12 +111,12 @@ public: } } - CallableInputs.clear(); + CallableInputs_.clear(); if (combinedStatus.Level == TStatus::Ok) { - Processed.clear(); + Processed_.clear(); } - if (Mode == ETypeCheckMode::Repeat) { + if (Mode_ == ETypeCheckMode::Repeat) { CheckFatalTypeError(combinedStatus); } @@ -124,25 +124,25 @@ public: } void Rewind() { - CallableTransformer->Rewind(); - CallableInputs.clear(); - Processed.clear(); - HasRenames = false; - RepeatCallableCount.clear(); - FunctionStack.Reset(); - CallableTimes.clear(); - IsComplete = false; + CallableTransformer_->Rewind(); + CallableInputs_.clear(); + Processed_.clear(); + HasRenames_ = false; + RepeatCallableCount_.clear(); + FunctionStack_.Reset(); + CallableTimes_.clear(); + IsComplete_ = false; } private: void WriteRepeatCallableCount() { - if (RepeatCallableCount.empty()) { + if (RepeatCallableCount_.empty()) { return; } TVector<std::pair<TString, ui64>> values; - for (const auto& x : RepeatCallableCount) { + for (const auto& x : RepeatCallableCount_) { values.push_back({ x.first, x.second }); } @@ -157,12 +157,12 @@ private: } YQL_CLOG(DEBUG, Core) << out.Str(); - RepeatCallableCount.clear(); + RepeatCallableCount_.clear(); } TStatus TransformNode(const TExprNode::TPtr& start, TExprNode::TPtr& output, TExprContext& ctx) { output = start; - auto processedPair = Processed.emplace(start.Get(), nullptr); // by default node is not changed + auto processedPair = Processed_.emplace(start.Get(), nullptr); // by default node is not changed if (!processedPair.second) { if (processedPair.first->second) { output = processedPair.first->second; @@ -206,9 +206,9 @@ private: auto input = start; for (size_t transformCount = 0; true; ++transformCount) { - FunctionStack.EnterFrame(*input, ctx); + FunctionStack_.EnterFrame(*input, ctx); Y_DEFER { - FunctionStack.LeaveFrame(*input, ctx); + FunctionStack_.LeaveFrame(*input, ctx); }; TStatus retStatus = TStatus::Error; @@ -420,12 +420,12 @@ private: } } - FunctionStack.MarkUsed(); + FunctionStack_.MarkUsed(); auto cyclesBefore = PrintCallableTimes ? GetCycleCount() : 0; - auto status = CallableTransformer->Transform(input, output, ctx); + auto status = CallableTransformer_->Transform(input, output, ctx); auto cyclesAfter = PrintCallableTimes ? GetCycleCount() : 0; if (PrintCallableTimes) { - auto& x = CallableTimes[input->Content()]; + auto& x = CallableTimes_[input->Content()]; x.first += (cyclesAfter - cyclesBefore); ++x.second; } @@ -447,13 +447,13 @@ private: CalculateWorld(*input); } else if (status == TStatus::Async) { - CallableInputs.push_back(input); + CallableInputs_.push_back(input); input->SetState(TExprNode::EState::TypeInProgress); } else { - RepeatCallableCount[input.Get()->Content()] += 1; + RepeatCallableCount_[input.Get()->Content()] += 1; if (output != input.Get()) { processedPair.first->second = output; - HasRenames = true; + HasRenames_ = true; } retStatus = status; @@ -522,11 +522,11 @@ private: } void CheckExpected(const TExprNode& input, TExprContext& ctx) { - CheckExpectedTypeAndColumnOrder(input, ctx, Types); + CheckExpectedTypeAndColumnOrder(input, ctx, Types_); } void CalculateWorld(TExprNode& input) { - if (!KeepWorldEnabled) { + if (!KeepWorldEnabled_) { return; } @@ -578,17 +578,17 @@ private: } private: - TAutoPtr<IGraphTransformer> CallableTransformer; - TTypeAnnotationContext& Types; - const ETypeCheckMode Mode; - bool IsComplete = false; - TDeque<TExprNode::TPtr> CallableInputs; - TNodeOnNodeOwnedMap Processed; - bool HasRenames = false; - THashMap<TString, ui64> RepeatCallableCount; - TFunctionStack FunctionStack; - THashMap<TStringBuf, std::pair<ui64, ui64>> CallableTimes; - bool KeepWorldEnabled = false; + TAutoPtr<IGraphTransformer> CallableTransformer_; + TTypeAnnotationContext& Types_; + const ETypeCheckMode Mode_; + bool IsComplete_ = false; + TDeque<TExprNode::TPtr> CallableInputs_; + TNodeOnNodeOwnedMap Processed_; + bool HasRenames_ = false; + THashMap<TString, ui64> RepeatCallableCount_; + TFunctionStack FunctionStack_; + THashMap<TStringBuf, std::pair<ui64, ui64>> CallableTimes_; + bool KeepWorldEnabled_ = false; }; } // namespace diff --git a/yql/essentials/core/type_ann/type_ann_pg.cpp b/yql/essentials/core/type_ann/type_ann_pg.cpp index 2b5f7625ca1..c46aeb53d07 100644 --- a/yql/essentials/core/type_ann/type_ann_pg.cpp +++ b/yql/essentials/core/type_ann/type_ann_pg.cpp @@ -5685,8 +5685,8 @@ IGraphTransformer::TStatus PgInWrapper(const TExprNode::TPtr& input, TExprNode:: }; struct TPgListCommonTypeConversion { - ui32 targetType = 0; - TExprNodeList items; + ui32 TargetType = 0; + TExprNodeList Items; }; bool castRequired = false; @@ -5698,7 +5698,7 @@ IGraphTransformer::TStatus PgInWrapper(const TExprNode::TPtr& input, TExprNode:: THashMap<ui32, TPgListCommonTypeConversion> elemsByType; for (size_t i = 1; i < input->ChildrenSize(); ++i) { auto& elemsOfType = elemsByType[pgTypes[i]]; - if (elemsOfType.items.empty()) { + if (elemsOfType.Items.empty()) { const NPg::TTypeDesc* elemCommonType; if (const auto issue = NPg::LookupCommonType({pgTypes[0], pgTypes[i]}, posGetter, elemCommonType)) @@ -5706,23 +5706,23 @@ IGraphTransformer::TStatus PgInWrapper(const TExprNode::TPtr& input, TExprNode:: ctx.Expr.AddError(*issue); return IGraphTransformer::TStatus::Error; } - elemsOfType.targetType = elemCommonType->TypeId; + elemsOfType.TargetType = elemCommonType->TypeId; - elemsOfType.items.push_back((lhsTypeId == elemsOfType.targetType) + elemsOfType.Items.push_back((lhsTypeId == elemsOfType.TargetType) ? input->HeadPtr() - : WrapWithPgCast(input->HeadPtr(), elemsOfType.targetType, ctx.Expr)); + : WrapWithPgCast(input->HeadPtr(), elemsOfType.TargetType, ctx.Expr)); } const auto rhsItemTypeId = input->Child(i)->GetTypeAnn()->Cast<TPgExprType>()->GetId(); - elemsOfType.items.push_back((rhsItemTypeId == elemsOfType.targetType) + elemsOfType.Items.push_back((rhsItemTypeId == elemsOfType.TargetType) ? input->Child(i) - : WrapWithPgCast(input->Child(i), elemsOfType.targetType, ctx.Expr)); + : WrapWithPgCast(input->Child(i), elemsOfType.TargetType, ctx.Expr)); } TExprNodeList orClausesOfIn; orClausesOfIn.reserve(elemsByType.size()); for (auto& elemsOfType: elemsByType) { auto& conversion = elemsOfType.second; - orClausesOfIn.push_back(BuildUniTypePgIn(std::move(conversion.items), ctx)); + orClausesOfIn.push_back(BuildUniTypePgIn(std::move(conversion.Items), ctx)); } output = ctx.Expr.Builder(input->Pos()) .Callable("Or") diff --git a/yql/essentials/core/url_lister/url_lister_manager.cpp b/yql/essentials/core/url_lister/url_lister_manager.cpp index ead04ad683e..b20d7c741a4 100644 --- a/yql/essentials/core/url_lister/url_lister_manager.cpp +++ b/yql/essentials/core/url_lister/url_lister_manager.cpp @@ -15,27 +15,27 @@ public: TUrlListerManager( TVector<IUrlListerPtr> urlListers ) - : UrlListers(std::move(urlListers)) + : UrlListers_(std::move(urlListers)) { } public: TVector<TUrlListEntry> ListUrl(const TString& url, const TString& tokenName) const override { - auto urlWithoutParameters = SubstParameters(url, Parameters, nullptr); + auto urlWithoutParameters = SubstParameters(url, Parameters_, nullptr); auto preprocessedUrl = urlWithoutParameters; TString alias; - if (UrlPreprocessing) { - std::tie(preprocessedUrl, alias) = UrlPreprocessing->Preprocess(urlWithoutParameters); + if (UrlPreprocessing_) { + std::tie(preprocessedUrl, alias) = UrlPreprocessing_->Preprocess(urlWithoutParameters); } TMaybe<TString> token; if (tokenName) { - if (!Credentials) { + if (!Credentials_) { ythrow yexception() << "Missing credentials"; } - auto credential = Credentials->FindCredential(tokenName); + auto credential = Credentials_->FindCredential(tokenName); if (!credential) { ythrow yexception() << "Unknown token name: " << tokenName; } @@ -43,15 +43,15 @@ public: token = credential->Content; } - if (!token && alias && Credentials) { - if (auto credential = Credentials->FindCredential("default_" + alias)) { + if (!token && alias && Credentials_) { + if (auto credential = Credentials_->FindCredential("default_" + alias)) { token = credential->Content; } } token = token.OrElse(""); - for (const auto& urlLister: UrlListers) { + for (const auto& urlLister: UrlListers_) { if (urlLister->Accept(preprocessedUrl)) { return urlLister->ListUrl(preprocessedUrl, *token); } @@ -62,36 +62,36 @@ public: public: IUrlListerManagerPtr Clone() const override { - auto clone = MakeUrlListerManager(UrlListers); + auto clone = MakeUrlListerManager(UrlListers_); - clone->SetCredentials(Credentials); - clone->SetUrlPreprocessing(UrlPreprocessing); + clone->SetCredentials(Credentials_); + clone->SetUrlPreprocessing(UrlPreprocessing_); - if (Parameters) { - clone->SetParameters(*Parameters); + if (Parameters_) { + clone->SetParameters(*Parameters_); } return clone; } void SetCredentials(TCredentials::TPtr credentials) override { - Credentials = std::move(credentials); + Credentials_ = std::move(credentials); } void SetUrlPreprocessing(IUrlPreprocessing::TPtr urlPreprocessing) override { - UrlPreprocessing = std::move(urlPreprocessing); + UrlPreprocessing_ = std::move(urlPreprocessing); } void SetParameters(const NYT::TNode& parameters) override { - Parameters = parameters; + Parameters_ = parameters; } private: - TVector<IUrlListerPtr> UrlListers; + TVector<IUrlListerPtr> UrlListers_; - TCredentials::TPtr Credentials; - IUrlPreprocessing::TPtr UrlPreprocessing; - TMaybe<NYT::TNode> Parameters; + TCredentials::TPtr Credentials_; + IUrlPreprocessing::TPtr UrlPreprocessing_; + TMaybe<NYT::TNode> Parameters_; }; } diff --git a/yql/essentials/core/url_preprocessing/pattern_group.cpp b/yql/essentials/core/url_preprocessing/pattern_group.cpp index 8185a84a7f1..ba848ddc721 100644 --- a/yql/essentials/core/url_preprocessing/pattern_group.cpp +++ b/yql/essentials/core/url_preprocessing/pattern_group.cpp @@ -3,20 +3,20 @@ namespace NYql { void TPatternGroup::Add(const TString& pattern, const TString& alias) { - auto it = CompiledPatterns.find(pattern); - if (it != CompiledPatterns.end()) { + auto it = CompiledPatterns_.find(pattern); + if (it != CompiledPatterns_.end()) { return; } - CompiledPatterns.emplace(pattern, std::make_pair(TRegExMatch(pattern), alias)); + CompiledPatterns_.emplace(pattern, std::make_pair(TRegExMatch(pattern), alias)); } bool TPatternGroup::IsEmpty() const { - return CompiledPatterns.empty(); + return CompiledPatterns_.empty(); } TMaybe<TString> TPatternGroup::Match(const TString& s) const { - for (auto& p : CompiledPatterns) { + for (auto& p : CompiledPatterns_) { if (p.second.first.Match(s.c_str())) { return p.second.second; } diff --git a/yql/essentials/core/url_preprocessing/pattern_group.h b/yql/essentials/core/url_preprocessing/pattern_group.h index 3137d4a28cb..fc30989e19a 100644 --- a/yql/essentials/core/url_preprocessing/pattern_group.h +++ b/yql/essentials/core/url_preprocessing/pattern_group.h @@ -17,7 +17,7 @@ public: TMaybe<TString> Match(const TString& s) const; private: - TMap<TString, std::pair<TRegExMatch, TString>> CompiledPatterns; + TMap<TString, std::pair<TRegExMatch, TString>> CompiledPatterns_; }; } diff --git a/yql/essentials/core/url_preprocessing/url_mapper.cpp b/yql/essentials/core/url_preprocessing/url_mapper.cpp index 0c646ed0d68..e56f0bf272a 100644 --- a/yql/essentials/core/url_preprocessing/url_mapper.cpp +++ b/yql/essentials/core/url_preprocessing/url_mapper.cpp @@ -3,11 +3,11 @@ namespace NYql { void TUrlMapper::AddMapping(const TString& pattern, const TString& targetUrl) { - CustomSchemes.push_back(TCustomScheme(pattern, targetUrl)); + CustomSchemes_.push_back(TCustomScheme(pattern, targetUrl)); } bool TUrlMapper::MapUrl(const TString& url, TString& mappedUrl) const { - for (const auto& sc : CustomSchemes) { + for (const auto& sc : CustomSchemes_) { if (sc.Pattern.Match(url.data())) { mappedUrl = TRegExSubst(sc.TargetUrlSubst).Replace(url.data()); return true; diff --git a/yql/essentials/core/url_preprocessing/url_mapper.h b/yql/essentials/core/url_preprocessing/url_mapper.h index f6438ecca3a..1a139460484 100644 --- a/yql/essentials/core/url_preprocessing/url_mapper.h +++ b/yql/essentials/core/url_preprocessing/url_mapper.h @@ -26,7 +26,7 @@ private: }; private: - TVector<TCustomScheme> CustomSchemes; + TVector<TCustomScheme> CustomSchemes_; }; } diff --git a/yql/essentials/core/user_data/yql_user_data.cpp b/yql/essentials/core/user_data/yql_user_data.cpp index 11db9c779e4..204f0df057c 100644 --- a/yql/essentials/core/user_data/yql_user_data.cpp +++ b/yql/essentials/core/user_data/yql_user_data.cpp @@ -9,11 +9,11 @@ void TUserData::UserDataToLibraries( THashMap<TString,TString>& modules ) { for (const TUserData& item : userData) { - if (item.Type_ == EType::LIBRARY) { - if (item.Disposition_ == EDisposition::RESOURCE) { // TODO: support other disposition options - modules[to_lower(item.Name_)] = item.Content_; - } else if (item.Disposition_ == EDisposition::RESOURCE_FILE) { - modules[to_lower(item.Name_)] = item.Name_; + if (item.Type == EType::LIBRARY) { + if (item.Disposition == EDisposition::RESOURCE) { // TODO: support other disposition options + modules[to_lower(item.Name)] = item.Content; + } else if (item.Disposition == EDisposition::RESOURCE_FILE) { + modules[to_lower(item.Name)] = item.Name; } } } diff --git a/yql/essentials/core/user_data/yql_user_data.h b/yql/essentials/core/user_data/yql_user_data.h index 8153f8253ed..60cf1441aa9 100644 --- a/yql/essentials/core/user_data/yql_user_data.h +++ b/yql/essentials/core/user_data/yql_user_data.h @@ -23,10 +23,10 @@ enum class EDisposition { }; struct TUserData { - EType Type_; - EDisposition Disposition_; - TString Name_; - TString Content_; + EType Type; + EDisposition Disposition; + TString Name; + TString Content; static void UserDataToLibraries( const TVector<TUserData>& userData, diff --git a/yql/essentials/core/yql_aggregate_expander.cpp b/yql/essentials/core/yql_aggregate_expander.cpp index 11df5c7f89e..c0406a2f912 100644 --- a/yql/essentials/core/yql_aggregate_expander.cpp +++ b/yql/essentials/core/yql_aggregate_expander.cpp @@ -11,18 +11,18 @@ namespace NYql { TExprNode::TPtr TAggregateExpander::ExpandAggregate() { - YQL_CLOG(DEBUG, Core) << "Expand " << Node->Content(); - if (Node->Head().GetTypeAnn()->HasErrors()) { - TErrorTypeVisitor errorVisitor(Ctx); - Node->Head().GetTypeAnn()->Accept(errorVisitor); + YQL_CLOG(DEBUG, Core) << "Expand " << Node_->Content(); + if (Node_->Head().GetTypeAnn()->HasErrors()) { + TErrorTypeVisitor errorVisitor(Ctx_); + Node_->Head().GetTypeAnn()->Accept(errorVisitor); return nullptr; } auto result = ExpandAggregateWithFullOutput(); if (result) { - auto outputColumns = GetSetting(*Node->Child(NNodes::TCoAggregate::idx_Settings), "output_columns"); + auto outputColumns = GetSetting(*Node_->Child(NNodes::TCoAggregate::idx_Settings), "output_columns"); if (outputColumns) { - result = Ctx.NewCallable(result->Pos(), "ExtractMembers", { result, outputColumns->ChildPtr(1) }); + result = Ctx_.NewCallable(result->Pos(), "ExtractMembers", { result, outputColumns->ChildPtr(1) }); } } return result; @@ -30,58 +30,58 @@ TExprNode::TPtr TAggregateExpander::ExpandAggregate() { TExprNode::TPtr TAggregateExpander::ExpandAggregateWithFullOutput() { - Suffix = Node->Content(); - YQL_ENSURE(Suffix.SkipPrefix("Aggregate")); - AggList = Node->HeadPtr(); - KeyColumns = Node->ChildPtr(1); - AggregatedColumns = Node->Child(2); - auto settings = Node->Child(3); + Suffix_ = Node_->Content(); + YQL_ENSURE(Suffix_.SkipPrefix("Aggregate")); + AggList_ = Node_->HeadPtr(); + KeyColumns_ = Node_->ChildPtr(1); + AggregatedColumns_ = Node_->Child(2); + auto settings = Node_->Child(3); bool allTraitsCollected = CollectTraits(); YQL_ENSURE(!HasSetting(*settings, "hopping"), "Aggregate with hopping unsupported here."); - HaveDistinct = AnyOf(AggregatedColumns->ChildrenList(), + HaveDistinct_ = AnyOf(AggregatedColumns_->ChildrenList(), [](const auto& child) { return child->ChildrenSize() == 3; }); - EffectiveCompact = (HaveDistinct && CompactForDistinct && !UseBlocks) || ForceCompact || HasSetting(*settings, "compact"); - for (const auto& trait : Traits) { + EffectiveCompact_ = (HaveDistinct_ && CompactForDistinct_ && !UseBlocks_) || ForceCompact_ || HasSetting(*settings, "compact"); + for (const auto& trait : Traits_) { auto mergeLambda = trait->Child(5); if (mergeLambda->Tail().IsCallable("Void")) { - EffectiveCompact = true; + EffectiveCompact_ = true; break; } } - if (Suffix == "Finalize") { - EffectiveCompact = true; - Suffix = ""; - } else if (Suffix != "") { - EffectiveCompact = false; + if (Suffix_ == "Finalize") { + EffectiveCompact_ = true; + Suffix_ = ""; + } else if (Suffix_ != "") { + EffectiveCompact_ = false; } - OriginalRowType = GetSeqItemType(*Node->Head().GetTypeAnn()).Cast<TStructExprType>(); - RowItems = OriginalRowType->GetItems(); + OriginalRowType_ = GetSeqItemType(*Node_->Head().GetTypeAnn()).Cast<TStructExprType>(); + RowItems_ = OriginalRowType_->GetItems(); ProcessSessionSetting(GetSetting(*settings, "session")); - RowType = Ctx.MakeType<TStructExprType>(RowItems); + RowType_ = Ctx_.MakeType<TStructExprType>(RowItems_); TVector<const TTypeAnnotationNode*> keyItemTypes = GetKeyItemTypes(); bool needPickle = IsNeedPickle(keyItemTypes); auto keyExtractor = GetKeyExtractor(needPickle); CollectColumnsSpecs(); - if (Suffix == "" && !HaveSessionSetting && !EffectiveCompact && UsePhases) { + if (Suffix_ == "" && !HaveSessionSetting_ && !EffectiveCompact_ && UsePhases_) { return GeneratePhases(); } - if (UseBlocks) { - if (Suffix == "Combine") { + if (UseBlocks_) { + if (Suffix_ == "Combine") { auto ret = TryGenerateBlockCombine(); if (ret) { return ret; } } - if (Suffix == "MergeFinalize" || Suffix == "MergeManyFinalize") { + if (Suffix_ == "MergeFinalize" || Suffix_ == "MergeManyFinalize") { auto ret = TryGenerateBlockMergeFinalize(); if (ret) { return ret; @@ -94,16 +94,16 @@ TExprNode::TPtr TAggregateExpander::ExpandAggregateWithFullOutput() } BuildNothingStates(); - if (Suffix == "MergeState" || Suffix == "MergeFinalize" || Suffix == "MergeManyFinalize") { - return GeneratePostAggregate(AggList, keyExtractor); + if (Suffix_ == "MergeState" || Suffix_ == "MergeFinalize" || Suffix_ == "MergeManyFinalize") { + return GeneratePostAggregate(AggList_, keyExtractor); } TExprNode::TPtr preAgg = GeneratePartialAggregate(keyExtractor, keyItemTypes, needPickle); - if (EffectiveCompact || !preAgg) { - preAgg = std::move(AggList); + if (EffectiveCompact_ || !preAgg) { + preAgg = std::move(AggList_); } - if (Suffix == "Combine" || Suffix == "CombineState") { + if (Suffix_ == "Combine" || Suffix_ == "CombineState") { return preAgg; } @@ -118,7 +118,7 @@ TExprNode::TPtr TAggregateExpander::ExpandAggApply(const TExprNode::TPtr& node) auto itemType = node->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); TVector<ui32> argTypes; bool needRetype = false; - auto status = ExtractPgTypesFromMultiLambda(node->ChildRef(2), argTypes, needRetype, Ctx); + auto status = ExtractPgTypesFromMultiLambda(node->ChildRef(2), argTypes, needRetype, Ctx_); YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); const NPg::TAggregateDesc* aggDescPtr; @@ -130,54 +130,54 @@ TExprNode::TPtr TAggregateExpander::ExpandAggApply(const TExprNode::TPtr& node) aggDescPtr = &NPg::LookupAggregation(TString(func), argTypes); } - return ExpandPgAggregationTraits(node->Pos(), *aggDescPtr, false, node->ChildPtr(2), argTypes, itemType, Ctx); + return ExpandPgAggregationTraits(node->Pos(), *aggDescPtr, false, node->ChildPtr(2), argTypes, itemType, Ctx_); } const TString modulePath = "/lib/yql/aggregate.yqls"; - auto exportsPtr = TypesCtx.Modules->GetModule(modulePath); + auto exportsPtr = TypesCtx_.Modules->GetModule(modulePath); YQL_ENSURE(exportsPtr, "Failed to get module " << modulePath); const auto& exports = exportsPtr->Symbols(); const auto ex = exports.find(TString(name) + "_traits_factory"); YQL_ENSURE(exports.cend() != ex); TNodeOnNodeOwnedMap deepClones; - auto lambda = Ctx.DeepCopy(*ex->second, exportsPtr->ExprCtx(), deepClones, true, false); + auto lambda = Ctx_.DeepCopy(*ex->second, exportsPtr->ExprCtx(), deepClones, true, false); - auto listTypeNode = Ctx.NewCallable(node->Pos(), "ListType", { node->ChildPtr(node->ChildrenSize() == 4 && !node->Child(3)->IsCallable("Void") ? 3 : 1) }); + auto listTypeNode = Ctx_.NewCallable(node->Pos(), "ListType", { node->ChildPtr(node->ChildrenSize() == 4 && !node->Child(3)->IsCallable("Void") ? 3 : 1) }); auto extractor = node->ChildPtr(2); - auto traits = Ctx.ReplaceNodes(lambda->TailPtr(), { + auto traits = Ctx_.ReplaceNodes(lambda->TailPtr(), { {lambda->Head().Child(0), listTypeNode}, {lambda->Head().Child(1), extractor} }); - Ctx.Step.Repeat(TExprStep::ExpandApplyForLambdas); - auto status = ExpandApplyNoRepeat(traits, traits, Ctx); + Ctx_.Step.Repeat(TExprStep::ExpandApplyForLambdas); + auto status = ExpandApplyNoRepeat(traits, traits, Ctx_); YQL_ENSURE(status != IGraphTransformer::TStatus::Error); return traits; } bool TAggregateExpander::CollectTraits() { bool allTraitsCollected = true; - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto trait = AggregatedColumns->Child(index)->ChildPtr(1); + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto trait = AggregatedColumns_->Child(index)->ChildPtr(1); if (trait->IsCallable({ "AggApply", "AggApplyState", "AggApplyManyState" })) { trait = ExpandAggApply(trait); allTraitsCollected = false; } - Traits.push_back(trait); + Traits_.push_back(trait); } return allTraitsCollected; } TExprNode::TPtr TAggregateExpander::RebuildAggregate() { - TExprNode::TListType newAggregatedColumnsItems = AggregatedColumns->ChildrenList(); - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto trait = AggregatedColumns->Child(index)->ChildPtr(1); + TExprNode::TListType newAggregatedColumnsItems = AggregatedColumns_->ChildrenList(); + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto trait = AggregatedColumns_->Child(index)->ChildPtr(1); if (trait->IsCallable("AggApply")) { - newAggregatedColumnsItems[index] = Ctx.ChangeChild(*(newAggregatedColumnsItems[index]), 1, std::move(Traits[index])); + newAggregatedColumnsItems[index] = Ctx_.ChangeChild(*(newAggregatedColumnsItems[index]), 1, std::move(Traits_[index])); } else if (trait->IsCallable("AggApplyState") || trait->IsCallable("AggApplyManyState")) { - auto newTrait = Ctx.Builder(Node->Pos()) + auto newTrait = Ctx_.Builder(Node_->Pos()) .Callable("AggregationTraits") .Add(0, trait->ChildPtr(1)) .Add(1, trait->ChildPtr(2)) // extractor for state, not initial value itself @@ -187,25 +187,25 @@ TExprNode::TPtr TAggregateExpander::RebuildAggregate() .Callable("Void") .Seal() .Seal() - .Add(3, Traits[index]->ChildPtr(3)) - .Add(4, Traits[index]->ChildPtr(4)) - .Add(5, Traits[index]->ChildPtr(5)) - .Add(6, Traits[index]->ChildPtr(6)) - .Add(7, Traits[index]->ChildPtr(7)) + .Add(3, Traits_[index]->ChildPtr(3)) + .Add(4, Traits_[index]->ChildPtr(4)) + .Add(5, Traits_[index]->ChildPtr(5)) + .Add(6, Traits_[index]->ChildPtr(6)) + .Add(7, Traits_[index]->ChildPtr(7)) .Seal() .Build(); - newAggregatedColumnsItems[index] = Ctx.ChangeChild(*(newAggregatedColumnsItems[index]), 1, std::move(newTrait)); + newAggregatedColumnsItems[index] = Ctx_.ChangeChild(*(newAggregatedColumnsItems[index]), 1, std::move(newTrait)); } } - return Ctx.ChangeChild(*Node, 2, Ctx.NewList(Node->Pos(), std::move(newAggregatedColumnsItems))); + return Ctx_.ChangeChild(*Node_, 2, Ctx_.NewList(Node_->Pos(), std::move(newAggregatedColumnsItems))); } TExprNode::TPtr TAggregateExpander::GetContextLambda() { - return HasContextFuncs(*AggregatedColumns) ? - Ctx.Builder(Node->Pos()) + return HasContextFuncs(*AggregatedColumns_) ? + Ctx_.Builder(Node_->Pos()) .Lambda() .Param("stream") .Callable("WithContext") @@ -214,7 +214,7 @@ TExprNode::TPtr TAggregateExpander::GetContextLambda() .Seal() .Seal() .Build() : - Ctx.Builder(Node->Pos()) + Ctx_.Builder(Node_->Pos()) .Lambda() .Param("stream") .Arg("stream") @@ -227,31 +227,31 @@ void TAggregateExpander::ProcessSessionSetting(TExprNode::TPtr sessionSetting) if (!sessionSetting) { return; } - HaveSessionSetting = true; + HaveSessionSetting_ = true; YQL_ENSURE(sessionSetting->Child(1)->Child(0)->IsAtom()); - SessionOutputColumn = sessionSetting->Child(1)->Child(0)->Content(); + SessionOutputColumn_ = sessionSetting->Child(1)->Child(0)->Content(); // remove session column from other keys - TExprNodeList keyColumnsList = KeyColumns->ChildrenList(); - EraseIf(keyColumnsList, [&](const auto& key) { return SessionOutputColumn == key->Content(); }); - KeyColumns = Ctx.NewList(KeyColumns->Pos(), std::move(keyColumnsList)); + TExprNodeList keyColumnsList = KeyColumns_->ChildrenList(); + EraseIf(keyColumnsList, [&](const auto& key) { return SessionOutputColumn_ == key->Content(); }); + KeyColumns_ = Ctx_.NewList(KeyColumns_->Pos(), std::move(keyColumnsList)); - SessionWindowParams.Traits = sessionSetting->Child(1)->ChildPtr(1); - ExtractSessionWindowParams(Node->Pos(), SessionWindowParams, Ctx); - ExtractSortKeyAndOrder(Node->Pos(), SessionWindowParams.SortTraits, SortParams, Ctx); + SessionWindowParams_.Traits = sessionSetting->Child(1)->ChildPtr(1); + ExtractSessionWindowParams(Node_->Pos(), SessionWindowParams_, Ctx_); + ExtractSortKeyAndOrder(Node_->Pos(), SessionWindowParams_.SortTraits, SortParams_, Ctx_); - if (HaveDistinct) { - auto keySelector = BuildKeySelector(Node->Pos(), *OriginalRowType, KeyColumns, Ctx); - const auto sessionStartMemberLambda = AddSessionParamsMemberLambda(Node->Pos(), SessionStartMemberName, keySelector, - SessionWindowParams, Ctx); + if (HaveDistinct_) { + auto keySelector = BuildKeySelector(Node_->Pos(), *OriginalRowType_, KeyColumns_, Ctx_); + const auto sessionStartMemberLambda = AddSessionParamsMemberLambda(Node_->Pos(), SessionStartMemberName, keySelector, + SessionWindowParams_, Ctx_); - AggList = Ctx.Builder(Node->Pos()) + AggList_ = Ctx_.Builder(Node_->Pos()) .Callable("PartitionsByKeys") - .Add(0, AggList) + .Add(0, AggList_) .Add(1, keySelector) - .Add(2, SortParams.Order) - .Add(3, SortParams.Key) + .Add(2, SortParams_.Order) + .Add(3, SortParams_.Key) .Lambda(4) .Param("partitionedStream") .Apply(sessionStartMemberLambda) @@ -261,26 +261,26 @@ void TAggregateExpander::ProcessSessionSetting(TExprNode::TPtr sessionSetting) .Seal() .Build(); - auto keyColumnsList = KeyColumns->ChildrenList(); - keyColumnsList.push_back(Ctx.NewAtom(Node->Pos(), SessionStartMemberName)); - KeyColumns = Ctx.NewList(Node->Pos(), std::move(keyColumnsList)); + auto keyColumnsList = KeyColumns_->ChildrenList(); + keyColumnsList.push_back(Ctx_.NewAtom(Node_->Pos(), SessionStartMemberName)); + KeyColumns_ = Ctx_.NewList(Node_->Pos(), std::move(keyColumnsList)); - RowItems.push_back(Ctx.MakeType<TItemExprType>(SessionStartMemberName, SessionWindowParams.KeyType)); + RowItems_.push_back(Ctx_.MakeType<TItemExprType>(SessionStartMemberName, SessionWindowParams_.KeyType)); - SessionWindowParams.Reset(); - SortParams.Key = SortParams.Order = VoidNode; + SessionWindowParams_.Reset(); + SortParams_.Key = SortParams_.Order = VoidNode_; } else { - EffectiveCompact = true; + EffectiveCompact_ = true; } } TVector<const TTypeAnnotationNode*> TAggregateExpander::GetKeyItemTypes() { TVector<const TTypeAnnotationNode*> keyItemTypes; - for (auto keyColumn : KeyColumns->Children()) { - auto index = RowType->FindItem(keyColumn->Content()); + for (auto keyColumn : KeyColumns_->Children()) { + auto index = RowType_->FindItem(keyColumn->Content()); YQL_ENSURE(index, "Unknown column: " << keyColumn->Content()); - auto type = RowType->GetItems()[*index]->GetItemType(); + auto type = RowType_->GetItems()[*index]->GetItemType(); keyItemTypes.push_back(type); } @@ -298,24 +298,24 @@ bool TAggregateExpander::IsNeedPickle(const TVector<const TTypeAnnotationNode*>& TExprNode::TPtr TAggregateExpander::GetKeyExtractor(bool needPickle) { - TExprNode::TPtr keyExtractor = Ctx.Builder(Node->Pos()) + TExprNode::TPtr keyExtractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (KeyColumns->ChildrenSize() == 0) { + if (KeyColumns_->ChildrenSize() == 0) { return parent.Callable("Uint32").Atom(0, "0", TNodeFlags::Default).Seal(); } - else if (KeyColumns->ChildrenSize() == 1) { - return parent.Callable("Member").Arg(0, "item").Add(1, KeyColumns->HeadPtr()).Seal(); + else if (KeyColumns_->ChildrenSize() == 1) { + return parent.Callable("Member").Arg(0, "item").Add(1, KeyColumns_->HeadPtr()).Seal(); } else { auto listBuilder = parent.List(); ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { listBuilder .Callable(pos++, "Member") .Arg(0, "item") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal(); } return listBuilder.Seal(); @@ -325,7 +325,7 @@ TExprNode::TPtr TAggregateExpander::GetKeyExtractor(bool needPickle) .Build(); if (needPickle) { - keyExtractor = Ctx.Builder(Node->Pos()) + keyExtractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Callable("StablePickle") @@ -341,36 +341,36 @@ TExprNode::TPtr TAggregateExpander::GetKeyExtractor(bool needPickle) void TAggregateExpander::CollectColumnsSpecs() { - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto child = AggregatedColumns->Child(index); + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto child = AggregatedColumns_->Child(index); if (const auto distinctField = (child->ChildrenSize() == 3) ? child->Child(2) : nullptr) { - const auto ins = Distinct2Columns.emplace(distinctField->Content(), TIdxSet()); + const auto ins = Distinct2Columns_.emplace(distinctField->Content(), TIdxSet()); if (ins.second) { - DistinctFields.push_back(distinctField); + DistinctFields_.push_back(distinctField); } - ins.first->second.insert(InitialColumnNames.size()); + ins.first->second.insert(InitialColumnNames_.size()); } else { - NonDistinctColumns.insert(InitialColumnNames.size()); + NonDistinctColumns_.insert(InitialColumnNames_.size()); } if (child->Head().IsAtom()) { - FinalColumnNames.push_back(child->HeadPtr()); + FinalColumnNames_.push_back(child->HeadPtr()); } else { - FinalColumnNames.push_back(child->Head().HeadPtr()); + FinalColumnNames_.push_back(child->Head().HeadPtr()); } - InitialColumnNames.push_back(Ctx.NewAtom(FinalColumnNames.back()->Pos(), "_yql_agg_" + ToString(InitialColumnNames.size()), TNodeFlags::Default)); + InitialColumnNames_.push_back(Ctx_.NewAtom(FinalColumnNames_.back()->Pos(), "_yql_agg_" + ToString(InitialColumnNames_.size()), TNodeFlags::Default)); } } void TAggregateExpander::BuildNothingStates() { - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto trait = Traits[index]; + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto trait = Traits_[index]; auto saveLambda = trait->Child(3); auto saveLambdaType = saveLambda->GetTypeAnn(); - auto typeNode = ExpandType(Node->Pos(), *saveLambdaType, Ctx); - NothingStates.push_back(Ctx.Builder(Node->Pos()) + auto typeNode = ExpandType(Node_->Pos(), *saveLambdaType, Ctx_); + NothingStates_.push_back(Ctx_.Builder(Node_->Pos()) .Callable("Nothing") .Callable(0, "OptionalType") .Add(0, std::move(typeNode)) @@ -387,24 +387,24 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregate(const TExprNode::TP TExprNode::TPtr pickleTypeNode = nullptr; if (needPickle) { const TTypeAnnotationNode* pickleType = nullptr; - pickleType = KeyColumns->ChildrenSize() > 1 ? Ctx.MakeType<TTupleExprType>(keyItemTypes) : keyItemTypes[0]; - pickleTypeNode = ExpandType(Node->Pos(), *pickleType, Ctx); + pickleType = KeyColumns_->ChildrenSize() > 1 ? Ctx_.MakeType<TTupleExprType>(keyItemTypes) : keyItemTypes[0]; + pickleTypeNode = ExpandType(Node_->Pos(), *pickleType, Ctx_); } TExprNode::TPtr partialAgg = nullptr; - if (!NonDistinctColumns.empty()) { + if (!NonDistinctColumns_.empty()) { partialAgg = GeneratePartialAggregateForNonDistinct(keyExtractor, pickleTypeNode); } - for (ui32 index = 0; index < DistinctFields.size(); ++index) { - auto distinctField = DistinctFields[index]; + for (ui32 index = 0; index < DistinctFields_.size(); ++index) { + auto distinctField = DistinctFields_[index]; - bool needDistinctPickle = EffectiveCompact ? false : needPickle; + bool needDistinctPickle = EffectiveCompact_ ? false : needPickle; auto distinctGrouper = GenerateDistinctGrouper(distinctField, keyItemTypes, needDistinctPickle); if (!partialAgg) { partialAgg = std::move(distinctGrouper); } else { - partialAgg = Ctx.Builder(Node->Pos()) + partialAgg = Ctx_.Builder(Node_->Pos()) .Callable("Extend") .Add(0, std::move(partialAgg)) .Add(1, std::move(distinctGrouper)) @@ -413,13 +413,13 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregate(const TExprNode::TP } } // If no aggregation functions then add additional combiner - if (AggregatedColumns->ChildrenSize() == 0 && KeyColumns->ChildrenSize() > 0 && !SessionWindowParams.Update) { + if (AggregatedColumns_->ChildrenSize() == 0 && KeyColumns_->ChildrenSize() > 0 && !SessionWindowParams_.Update) { if (!partialAgg) { - partialAgg = AggList; + partialAgg = AggList_; } auto uniqCombineInit = ReturnKeyAsIsForCombineInit(pickleTypeNode); - auto uniqCombineUpdate = Ctx.Builder(Node->Pos()) + auto uniqCombineUpdate = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") @@ -429,7 +429,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregate(const TExprNode::TP .Build(); // Return state as-is - auto uniqCombineSave = Ctx.Builder(Node->Pos()) + auto uniqCombineSave = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("state") @@ -439,10 +439,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregate(const TExprNode::TP .Seal() .Build(); - partialAgg = Ctx.Builder(Node->Pos()) + partialAgg = Ctx_.Builder(Node_->Pos()) .Callable("CombineByKey") .Add(0, std::move(partialAgg)) - .Add(1, PreMap) + .Add(1, PreMap_) .Add(2, keyExtractor) .Add(3, std::move(uniqCombineInit)) .Add(4, std::move(uniqCombineUpdate)) @@ -455,10 +455,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregate(const TExprNode::TP std::function<TExprNodeBuilder& (TExprNodeBuilder&)> TAggregateExpander::GetPartialAggArgExtractor(ui32 i, bool deserialize) { return [&, i, deserialize](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - auto trait = Traits[i]; + auto trait = Traits_[i]; auto extractorLambda = trait->Child(1); auto loadLambda = trait->Child(4); - if (Suffix == "CombineState") { + if (Suffix_ == "CombineState") { if (deserialize) { parent.Apply(*loadLambda) .With(0) @@ -466,7 +466,7 @@ std::function<TExprNodeBuilder& (TExprNodeBuilder&)> TAggregateExpander::GetPart .With(0) .Callable("CastStruct") .Arg(0, "item") - .Add(1, ExpandType(Node->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx)) + .Add(1, ExpandType(Node_->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx_)) .Seal() .Done() .Seal() @@ -477,7 +477,7 @@ std::function<TExprNodeBuilder& (TExprNodeBuilder&)> TAggregateExpander::GetPart .With(0) .Callable("CastStruct") .Arg(0, "item") - .Add(1, ExpandType(Node->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx)) + .Add(1, ExpandType(Node_->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx_)) .Seal() .Done() .Seal(); @@ -485,7 +485,7 @@ std::function<TExprNodeBuilder& (TExprNodeBuilder&)> TAggregateExpander::GetPart } else { parent.Callable("CastStruct") .Arg(0, "item") - .Add(1, ExpandType(Node->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx)) + .Add(1, ExpandType(Node_->Pos(), *extractorLambda->Head().Head().GetTypeAnn(), Ctx_)) .Seal(); } @@ -494,15 +494,15 @@ std::function<TExprNodeBuilder& (TExprNodeBuilder&)> TAggregateExpander::GetPart } TExprNode::TPtr TAggregateExpander::GetFinalAggStateExtractor(ui32 i) { - auto trait = Traits[i]; - if (Suffix.StartsWith("Merge")) { + auto trait = Traits_[i]; + if (Suffix_.StartsWith("Merge")) { auto lambda = trait->ChildPtr(1); - if (!Suffix.StartsWith("MergeMany")) { + if (!Suffix_.StartsWith("MergeMany")) { return lambda; } if (lambda->Tail().IsCallable("Unwrap")) { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .ApplyPartial(lambda->HeadPtr(), lambda->Tail().HeadPtr()) @@ -511,7 +511,7 @@ TExprNode::TPtr TAggregateExpander::GetFinalAggStateExtractor(ui32 i) { .Seal() .Build(); } else { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Callable("Just") @@ -524,9 +524,9 @@ TExprNode::TPtr TAggregateExpander::GetFinalAggStateExtractor(ui32 i) { } } - bool aggregateOnly = (Suffix != ""); - const auto& columnNames = aggregateOnly ? FinalColumnNames : InitialColumnNames; - return Ctx.Builder(Node->Pos()) + bool aggregateOnly = (Suffix_ != ""); + const auto& columnNames = aggregateOnly ? FinalColumnNames_ : InitialColumnNames_; + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Callable("Member") @@ -540,38 +540,38 @@ TExprNode::TPtr TAggregateExpander::GetFinalAggStateExtractor(ui32 i) { TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& stream, TExprNode::TListType& keyIdxs, TVector<TString>& outputColumns, TExprNode::TListType& aggs, bool overState, bool many, ui32* streamIdxColumn) { TVector<TString> inputColumns; - auto flow = Ctx.NewCallable(Node->Pos(), "ToFlow", { stream }); - for (ui32 i = 0; i < RowType->GetSize(); ++i) { - inputColumns.push_back(TString(RowType->GetItems()[i]->GetName())); + auto flow = Ctx_.NewCallable(Node_->Pos(), "ToFlow", { stream }); + for (ui32 i = 0; i < RowType_->GetSize(); ++i) { + inputColumns.push_back(TString(RowType_->GetItems()[i]->GetName())); } - auto wideFlow = MakeExpandMap(Node->Pos(), inputColumns, flow, Ctx); + auto wideFlow = MakeExpandMap(Node_->Pos(), inputColumns, flow, Ctx_); TExprNode::TListType extractorArgs; TExprNode::TListType newRowItems; - for (ui32 i = 0; i < RowType->GetSize(); ++i) { - extractorArgs.push_back(Ctx.NewArgument(Node->Pos(), "field" + ToString(i))); - newRowItems.push_back(Ctx.NewList(Node->Pos(), { Ctx.NewAtom(Node->Pos(), RowType->GetItems()[i]->GetName()), extractorArgs.back() })); + for (ui32 i = 0; i < RowType_->GetSize(); ++i) { + extractorArgs.push_back(Ctx_.NewArgument(Node_->Pos(), "field" + ToString(i))); + newRowItems.push_back(Ctx_.NewList(Node_->Pos(), { Ctx_.NewAtom(Node_->Pos(), RowType_->GetItems()[i]->GetName()), extractorArgs.back() })); } - const TExprNode::TPtr newRow = Ctx.NewCallable(Node->Pos(), "AsStruct", std::move(newRowItems)); + const TExprNode::TPtr newRow = Ctx_.NewCallable(Node_->Pos(), "AsStruct", std::move(newRowItems)); TExprNode::TListType extractorRoots; TVector<const TTypeAnnotationNode*> allKeyTypes; - for (ui32 index = 0; index < KeyColumns->ChildrenSize(); ++index) { - auto keyName = KeyColumns->Child(index)->Content(); - auto rowIndex = RowType->FindItem(keyName); + for (ui32 index = 0; index < KeyColumns_->ChildrenSize(); ++index) { + auto keyName = KeyColumns_->Child(index)->Content(); + auto rowIndex = RowType_->FindItem(keyName); YQL_ENSURE(rowIndex, "Unknown column: " << keyName); - auto type = RowType->GetItems()[*rowIndex]->GetItemType(); + auto type = RowType_->GetItems()[*rowIndex]->GetItemType(); extractorRoots.push_back(extractorArgs[*rowIndex]); allKeyTypes.push_back(type); - keyIdxs.push_back(Ctx.NewAtom(Node->Pos(), ToString(index))); + keyIdxs.push_back(Ctx_.NewAtom(Node_->Pos(), ToString(index))); outputColumns.push_back(TString(keyName)); } if (many) { - auto rowIndex = RowType->FindItem("_yql_group_stream_index"); + auto rowIndex = RowType_->FindItem("_yql_group_stream_index"); if (!rowIndex) { return nullptr; } @@ -582,22 +582,22 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea extractorRoots.push_back(extractorArgs[*rowIndex]); } - auto outputStructType = GetSeqItemType(*Node->GetTypeAnn()).Cast<TStructExprType>(); + auto outputStructType = GetSeqItemType(*Node_->GetTypeAnn()).Cast<TStructExprType>(); - auto resolveStatus = TypesCtx.ArrowResolver->AreTypesSupported(Ctx.GetPosition(Node->Pos()), allKeyTypes, Ctx); + auto resolveStatus = TypesCtx_.ArrowResolver->AreTypesSupported(Ctx_.GetPosition(Node_->Pos()), allKeyTypes, Ctx_); YQL_ENSURE(resolveStatus != IArrowResolver::ERROR); if (resolveStatus != IArrowResolver::OK) { return nullptr; } - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto trait = AggregatedColumns->Child(index)->ChildPtr(1); + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto trait = AggregatedColumns_->Child(index)->ChildPtr(1); TVector<const TTypeAnnotationNode*> allTypes; const TTypeAnnotationNode* originalType = nullptr; if (overState && !trait->Child(3)->IsCallable("Void")) { auto originalExtractorType = trait->Child(3)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); - originalType = GetOriginalResultType(trait->Pos(), many, originalExtractorType, Ctx); + originalType = GetOriginalResultType(trait->Pos(), many, originalExtractorType, Ctx_); YQL_ENSURE(originalType); } @@ -614,20 +614,20 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea auto root = trait->Child(2)->ChildPtr(i); allTypes.push_back(root->GetTypeAnn()); - auto status = RemapExpr(root, root, remaps, Ctx, TOptimizeExprSettings(&TypesCtx)); + auto status = RemapExpr(root, root, remaps, Ctx_, TOptimizeExprSettings(&TypesCtx_)); YQL_ENSURE(status.Level != IGraphTransformer::TStatus::Error); roots.push_back(root); } - aggs.push_back(Ctx.Builder(Node->Pos()) + aggs.push_back(Ctx_.Builder(Node_->Pos()) .List() .Callable(0, TString("AggBlockApply") + (overState ? "State" : "")) .Atom(0, trait->Child(0)->Content()) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { if (overState) { if (originalType) { - parent.Add(1, ExpandType(Node->Pos(), *originalType, Ctx)); + parent.Add(1, ExpandType(Node_->Pos(), *originalType, Ctx_)); } else { parent .Callable(1, "NullType") @@ -639,7 +639,7 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 1; i < argsCount + 1; ++i) { - parent.Add(i + (overState ? 1 : 0), ExpandType(Node->Pos(), *trait->Child(2)->Child(i)->GetTypeAnn(), Ctx)); + parent.Add(i + (overState ? 1 : 0), ExpandType(Node_->Pos(), *trait->Child(2)->Child(i)->GetTypeAnn(), Ctx_)); } return parent; @@ -660,7 +660,7 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea if (root->IsCallable("Unwrap")) { root = root->HeadPtr(); } else { - root = Ctx.Builder(Node->Pos()) + root = Ctx_.Builder(Node_->Pos()) .Callable("Just") .Add(0, root) .Seal() @@ -671,21 +671,21 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea extractorRoots.push_back(root); } - auto outPos = outputStructType->FindItem(FinalColumnNames[index]->Content()); + auto outPos = outputStructType->FindItem(FinalColumnNames_[index]->Content()); YQL_ENSURE(outPos); allTypes.push_back(outputStructType->GetItems()[*outPos]->GetItemType()); - auto resolveStatus = TypesCtx.ArrowResolver->AreTypesSupported(Ctx.GetPosition(Node->Pos()), allTypes, Ctx); + auto resolveStatus = TypesCtx_.ArrowResolver->AreTypesSupported(Ctx_.GetPosition(Node_->Pos()), allTypes, Ctx_); YQL_ENSURE(resolveStatus != IArrowResolver::ERROR); if (resolveStatus != IArrowResolver::OK) { return nullptr; } - outputColumns.push_back(TString(FinalColumnNames[index]->Content())); + outputColumns.push_back(TString(FinalColumnNames_[index]->Content())); } - auto extractorLambda = Ctx.NewLambda(Node->Pos(), Ctx.NewArguments(Node->Pos(), std::move(extractorArgs)), std::move(extractorRoots)); - auto mappedWideFlow = Ctx.NewCallable(Node->Pos(), "WideMap", { wideFlow, extractorLambda }); - return Ctx.Builder(Node->Pos()) + auto extractorLambda = Ctx_.NewLambda(Node_->Pos(), Ctx_.NewArguments(Node_->Pos(), std::move(extractorArgs)), std::move(extractorRoots)); + auto mappedWideFlow = Ctx_.NewCallable(Node_->Pos(), "WideMap", { wideFlow, extractorLambda }); + return Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "WideToBlocks") .Callable(0, "FromFlow") @@ -697,21 +697,21 @@ TExprNode::TPtr TAggregateExpander::MakeInputBlocks(const TExprNode::TPtr& strea } TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombineAllOrHashed() { - if (!TypesCtx.ArrowResolver) { + if (!TypesCtx_.ArrowResolver) { return nullptr; } - const bool hashed = (KeyColumns->ChildrenSize() > 0); - const bool isInputList = (AggList->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List); + const bool hashed = (KeyColumns_->ChildrenSize() > 0); + const bool isInputList = (AggList_->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List); TExprNode::TListType keyIdxs; TVector<TString> outputColumns; TExprNode::TListType aggs; TExprNode::TPtr stream = nullptr; if (isInputList) { - stream = Ctx.NewArgument(Node->Pos(), "stream"); + stream = Ctx_.NewArgument(Node_->Pos(), "stream"); } else { - stream = AggList; + stream = AggList_; } TExprNode::TPtr blocks = MakeInputBlocks(stream, keyIdxs, outputColumns, aggs, false, false); @@ -721,7 +721,7 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombineAllOrHashed() { TExprNode::TPtr aggWideFlow; if (hashed) { - aggWideFlow = Ctx.Builder(Node->Pos()) + aggWideFlow = Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "WideFromBlocks") .Callable(0, "BlockCombineHashed") @@ -730,14 +730,14 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombineAllOrHashed() { .Seal() .Callable(1, "Void") .Seal() - .Add(2, Ctx.NewList(Node->Pos(), std::move(keyIdxs))) - .Add(3, Ctx.NewList(Node->Pos(), std::move(aggs))) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(keyIdxs))) + .Add(3, Ctx_.NewList(Node_->Pos(), std::move(aggs))) .Seal() .Seal() .Seal() .Build(); } else { - aggWideFlow = Ctx.Builder(Node->Pos()) + aggWideFlow = Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "BlockCombineAll") .Callable(0, "FromFlow") @@ -745,20 +745,20 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombineAllOrHashed() { .Seal() .Callable(1, "Void") .Seal() - .Add(2, Ctx.NewList(Node->Pos(), std::move(aggs))) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(aggs))) .Seal() .Seal() .Build(); } - auto finalFlow = MakeNarrowMap(Node->Pos(), outputColumns, aggWideFlow, Ctx); + auto finalFlow = MakeNarrowMap(Node_->Pos(), outputColumns, aggWideFlow, Ctx_); if (isInputList) { - auto root = Ctx.NewCallable(Node->Pos(), "FromFlow", { finalFlow }); - auto lambdaStream = Ctx.NewLambda(Node->Pos(), Ctx.NewArguments(Node->Pos(), { stream }), std::move(root)); + auto root = Ctx_.NewCallable(Node_->Pos(), "FromFlow", { finalFlow }); + auto lambdaStream = Ctx_.NewLambda(Node_->Pos(), Ctx_.NewArguments(Node_->Pos(), { stream }), std::move(root)); - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Callable("LMap") - .Add(0, AggList) + .Add(0, AggList_) .Lambda(1) .Param("stream") .Apply(GetContextLambda()) @@ -778,20 +778,20 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombineAllOrHashed() { TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const TExprNode::TPtr& keyExtractor, const TExprNode::TPtr& pickleTypeNode) { - bool combineOnly = Suffix == "Combine" || Suffix == "CombineState"; - const auto& columnNames = combineOnly ? FinalColumnNames : InitialColumnNames; - auto initLambdaIndex = (Suffix == "CombineState") ? 4 : 1; - auto updateLambdaIndex = (Suffix == "CombineState") ? 5 : 2; + bool combineOnly = Suffix_ == "Combine" || Suffix_ == "CombineState"; + const auto& columnNames = combineOnly ? FinalColumnNames_ : InitialColumnNames_; + auto initLambdaIndex = (Suffix_ == "CombineState") ? 4 : 1; + auto updateLambdaIndex = (Suffix_ == "CombineState") ? 5 : 2; - auto combineInit = Ctx.Builder(Node->Pos()) + auto combineInit = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 ndx = 0; - for (ui32 i: NonDistinctColumns) { - auto trait = Traits[i]; + for (ui32 i: NonDistinctColumns_) { + auto trait = Traits_[i]; auto initLambda = trait->Child(initLambdaIndex); if (initLambda->Head().ChildrenSize() == 1) { parent.List(ndx++) @@ -824,7 +824,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const .Seal() .Build(); - auto combineUpdate = Ctx.Builder(Node->Pos()) + auto combineUpdate = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") @@ -832,8 +832,8 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 ndx = 0; - for (ui32 i: NonDistinctColumns) { - auto trait = Traits[i]; + for (ui32 i: NonDistinctColumns_) { + auto trait = Traits_[i]; auto updateLambda = trait->Child(updateLambdaIndex); if (updateLambda->Head().ChildrenSize() == 2) { parent.List(ndx++) @@ -878,7 +878,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const .Seal() .Build(); - auto combineSave = Ctx.Builder(Node->Pos()) + auto combineSave = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("state") @@ -886,15 +886,15 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const .Callable(0, "AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < columnNames.size(); ++i) { - if (NonDistinctColumns.find(i) == NonDistinctColumns.end()) { + if (NonDistinctColumns_.find(i) == NonDistinctColumns_.end()) { parent.List(i) .Add(0, columnNames[i]) - .Add(1, NothingStates[i]) + .Add(1, NothingStates_[i]) .Seal(); } else { - auto trait = Traits[i]; + auto trait = Traits_[i]; auto saveLambda = trait->Child(3); - if (!DistinctFields.empty()) { + if (!DistinctFields_.empty()) { parent.List(i) .Add(0, columnNames[i]) .Callable(1, "Just") @@ -927,10 +927,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { auto listBuilder = parent.List(columnNames.size() + i); - listBuilder.Add(0, KeyColumns->ChildPtr(i)); - if (KeyColumns->ChildrenSize() > 1) { + listBuilder.Add(0, KeyColumns_->ChildPtr(i)); + if (KeyColumns_->ChildrenSize() > 1) { if (pickleTypeNode) { listBuilder .Callable(1, "Nth") @@ -967,10 +967,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const .Seal() .Build(); - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Callable("CombineByKey") - .Add(0, AggList) - .Add(1, PreMap) + .Add(0, AggList_) + .Add(1, PreMap_) .Add(2, keyExtractor) .Add(3, std::move(combineInit)) .Add(4, std::move(combineUpdate)) @@ -981,11 +981,11 @@ TExprNode::TPtr TAggregateExpander::GeneratePartialAggregateForNonDistinct(const void TAggregateExpander::GenerateInitForDistinct(TExprNodeBuilder& parent, ui32& ndx, const TIdxSet& indicies, const TExprNode::TPtr& distinctField) { for (ui32 i: indicies) { - auto trait = Traits[i]; + auto trait = Traits_[i]; auto initLambda = trait->Child(1); if (initLambda->Head().ChildrenSize() == 1) { parent.List(ndx++) - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Apply(1, *initLambda) .With(0) .Callable("Member") @@ -997,7 +997,7 @@ void TAggregateExpander::GenerateInitForDistinct(TExprNodeBuilder& parent, ui32& .Seal(); } else { parent.List(ndx++) - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Apply(1, *initLambda) .With(0) .Callable("Member") @@ -1019,10 +1019,10 @@ void TAggregateExpander::GenerateInitForDistinct(TExprNodeBuilder& parent, ui32& TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPtr distinctField, const TVector<const TTypeAnnotationNode*>& keyItemTypes, bool needDistinctPickle) { - auto& indicies = Distinct2Columns[distinctField->Content()]; - auto distinctIndex = RowType->FindItem(distinctField->Content()); + auto& indicies = Distinct2Columns_[distinctField->Content()]; + auto distinctIndex = RowType_->FindItem(distinctField->Content()); YQL_ENSURE(distinctIndex, "Unknown field: " << distinctField->Content()); - auto distinctType = RowType->GetItems()[*distinctIndex]->GetItemType(); + auto distinctType = RowType_->GetItems()[*distinctIndex]->GetItemType(); TVector<const TTypeAnnotationNode*> distinctKeyItemTypes = keyItemTypes; distinctKeyItemTypes.push_back(distinctType); auto valueType = distinctType; @@ -1032,19 +1032,19 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt if (distinctType->GetKind() != ETypeAnnotationKind::Data) { needDistinctPickle = true; - valueType = Ctx.MakeType<TDataExprType>(EDataSlot::String); + valueType = Ctx_.MakeType<TDataExprType>(EDataSlot::String); } const auto expandedValueType = needDistinctPickle ? - Ctx.Builder(Node->Pos()) + Ctx_.Builder(Node_->Pos()) .Callable("DataType") .Atom(0, "String", TNodeFlags::Default) .Seal() .Build() - : ExpandType(Node->Pos(), *valueType, Ctx); + : ExpandType(Node_->Pos(), *valueType, Ctx_); - DistinctFieldNeedsPickle[distinctField->Content()] = needDistinctPickle; - auto udfSetCreateValue = Ctx.Builder(Node->Pos()) + DistinctFieldNeedsPickle_[distinctField->Content()] = needDistinctPickle; + auto udfSetCreateValue = Ctx_.Builder(Node_->Pos()) .Callable("Udf") .Atom(0, "Set.Create") .Callable(1, "Void").Seal() @@ -1061,8 +1061,8 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - UdfSetCreate[distinctField->Content()] = udfSetCreateValue; - auto resourceType = Ctx.Builder(Node->Pos()) + UdfSetCreate_[distinctField->Content()] = udfSetCreateValue; + auto resourceType = Ctx_.Builder(Node_->Pos()) .Callable("TypeOf") .Callable(0, "Apply") .Add(0, udfSetCreateValue) @@ -1076,7 +1076,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - UdfAddValue[distinctField->Content()] = Ctx.Builder(Node->Pos()) + UdfAddValue_[distinctField->Content()] = Ctx_.Builder(Node_->Pos()) .Callable("Udf") .Atom(0, "Set.AddValue") .Callable(1, "Void").Seal() @@ -1091,7 +1091,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - UdfWasChanged[distinctField->Content()] = Ctx.Builder(Node->Pos()) + UdfWasChanged_[distinctField->Content()] = Ctx_.Builder(Node_->Pos()) .Callable("Udf") .Atom(0, "Set.WasChanged") .Callable(1, "Void").Seal() @@ -1105,18 +1105,18 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - auto distinctKeyExtractor = Ctx.Builder(Node->Pos()) + auto distinctKeyExtractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (KeyColumns->ChildrenSize() != 0) { + if (KeyColumns_->ChildrenSize() != 0) { auto listBuilder = parent.List(); ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { listBuilder .Callable(pos++, "Member") .Arg(0, "item") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal(); } listBuilder @@ -1140,12 +1140,12 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt const TTypeAnnotationNode* distinctPickleType = nullptr; TExprNode::TPtr distinctPickleTypeNode; if (needDistinctPickle) { - distinctPickleType = KeyColumns->ChildrenSize() > 0 ? Ctx.MakeType<TTupleExprType>(distinctKeyItemTypes) : distinctKeyItemTypes.front(); - distinctPickleTypeNode = ExpandType(Node->Pos(), *distinctPickleType, Ctx); + distinctPickleType = KeyColumns_->ChildrenSize() > 0 ? Ctx_.MakeType<TTupleExprType>(distinctKeyItemTypes) : distinctKeyItemTypes.front(); + distinctPickleTypeNode = ExpandType(Node_->Pos(), *distinctPickleType, Ctx_); } if (needDistinctPickle) { - distinctKeyExtractor = Ctx.Builder(Node->Pos()) + distinctKeyExtractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Callable("StablePickle") @@ -1155,7 +1155,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Build(); } - auto distinctCombineInit = Ctx.Builder(Node->Pos()) + auto distinctCombineInit = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") @@ -1169,7 +1169,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - auto distinctCombineUpdate = Ctx.Builder(Node->Pos()) + auto distinctCombineUpdate = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") @@ -1179,7 +1179,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Build(); ui32 ndx = 0; - auto distinctCombineSave = Ctx.Builder(Node->Pos()) + auto distinctCombineSave = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("state") @@ -1187,15 +1187,15 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Callable(0, "AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i: indicies) { - auto trait = Traits[i]; + auto trait = Traits_[i]; auto saveLambda = trait->Child(3); parent.List(ndx++) - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Apply(1, *saveLambda) .With(0) .Callable("Member") .Arg(0, "state") - .Add(1, InitialColumnNames[i]) + .Add(1, InitialColumnNames_[i]) .Seal() .Done() .Seal() @@ -1204,12 +1204,12 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt return parent; }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (KeyColumns->ChildrenSize() > 0) { + if (KeyColumns_->ChildrenSize() > 0) { if (needDistinctPickle) { ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { parent.List(ndx++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Nth") .Callable(0, "Unpickle") .Add(0, distinctPickleTypeNode) @@ -1232,9 +1232,9 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt } else { ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { parent.List(ndx++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Nth") .Arg(0, "key") .Atom(1, ToString(pos++), TNodeFlags::Default) @@ -1272,10 +1272,10 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - auto distinctCombiner = Ctx.Builder(Node->Pos()) + auto distinctCombiner = Ctx_.Builder(Node_->Pos()) .Callable("CombineByKey") - .Add(0, AggList) - .Add(1, PreMap) + .Add(0, AggList_) + .Add(1, PreMap_) .Add(2, distinctKeyExtractor) .Add(3, std::move(distinctCombineInit)) .Add(4, std::move(distinctCombineUpdate)) @@ -1283,7 +1283,7 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Seal() .Build(); - auto distinctGrouper = Ctx.Builder(Node->Pos()) + auto distinctGrouper = Ctx_.Builder(Node_->Pos()) .Callable("PartitionsByKeys") .Add(0, std::move(distinctCombiner)) .Add(1, distinctKeyExtractor) @@ -1318,34 +1318,34 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt .Param("state") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < InitialColumnNames.size(); ++i) { + for (ui32 i = 0; i < InitialColumnNames_.size(); ++i) { if (indicies.find(i) != indicies.end()) { parent.List(i) - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Callable(1, "Just") .Callable(0, "Member") .Arg(0, "state") - .Add(1, InitialColumnNames[i]) + .Add(1, InitialColumnNames_[i]) .Seal() .Seal() .Seal(); } else { parent.List(i) - .Add(0, InitialColumnNames[i]) - .Add(1, NothingStates[i]) + .Add(0, InitialColumnNames_[i]) + .Add(1, NothingStates_[i]) .Seal(); } } return parent; }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (KeyColumns->ChildrenSize() > 0) { - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { - parent.List(InitialColumnNames.size() + i) - .Add(0, KeyColumns->ChildPtr(i)) + if (KeyColumns_->ChildrenSize() > 0) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { + parent.List(InitialColumnNames_.size() + i) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "state") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal().Seal(); } } @@ -1362,17 +1362,17 @@ TExprNode::TPtr TAggregateExpander::GenerateDistinctGrouper(const TExprNode::TPt TExprNode::TPtr TAggregateExpander::ReturnKeyAsIsForCombineInit(const TExprNode::TPtr& pickleTypeNode) { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("key") .Param("item") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { auto listBuilder = parent.List(i); - listBuilder.Add(0, KeyColumns->Child(i)); - if (KeyColumns->ChildrenSize() > 1) { + listBuilder.Add(0, KeyColumns_->Child(i)); + if (KeyColumns_->ChildrenSize() > 1) { if (pickleTypeNode) { listBuilder .Callable(1, "Nth") @@ -1409,7 +1409,7 @@ TExprNode::TPtr TAggregateExpander::ReturnKeyAsIsForCombineInit(const TExprNode: } TExprNode::TPtr TAggregateExpander::BuildFinalizeByKeyLambda(const TExprNode::TPtr& preprocessLambda, const TExprNode::TPtr& keyExtractor) { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("stream") .Callable("FinalizeByKey") @@ -1758,8 +1758,8 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregate(const TExprNode::TPtr& { auto preprocessLambda = GeneratePreprocessLambda(keyExtractor); TExprNode::TPtr postAgg; - if (!UsePartitionsByKeys && UseFinalizeByKeys && !HaveSessionSetting) { - postAgg = Ctx.Builder(Node->Pos()) + if (!UsePartitionsByKeys_ && UseFinalizeByKeys_ && !HaveSessionSetting_) { + postAgg = Ctx_.Builder(Node_->Pos()) .Callable("ShuffleByKeys") .Add(0, std::move(preAgg)) .Add(1, keyExtractor) @@ -1776,12 +1776,12 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregate(const TExprNode::TPtr& .Seal().Build(); } else { auto condenseSwitch = GenerateCondenseSwitch(keyExtractor); - postAgg = Ctx.Builder(Node->Pos()) + postAgg = Ctx_.Builder(Node_->Pos()) .Callable("PartitionsByKeys") .Add(0, std::move(preAgg)) .Add(1, keyExtractor) - .Add(2, SortParams.Order) - .Add(3, SortParams.Key) + .Add(2, SortParams_.Order) + .Add(3, SortParams_.Key) .Lambda(4) .Param("stream") .Apply(GetContextLambda()) @@ -1802,8 +1802,8 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregate(const TExprNode::TPtr& .Seal() .Seal().Build(); } - if (KeyColumns->ChildrenSize() == 0 && !HaveSessionSetting && (Suffix == "" || Suffix.EndsWith("Finalize"))) { - return MakeSingleGroupRow(*Node, postAgg, Ctx); + if (KeyColumns_->ChildrenSize() == 0 && !HaveSessionSetting_ && (Suffix_ == "" || Suffix_.EndsWith("Finalize"))) { + return MakeSingleGroupRow(*Node_, postAgg, Ctx_); } return postAgg; @@ -1812,17 +1812,17 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregate(const TExprNode::TPtr& TExprNode::TPtr TAggregateExpander::GeneratePreprocessLambda(const TExprNode::TPtr& keyExtractor) { TExprNode::TPtr preprocessLambda; - if (SessionWindowParams.Update) { - YQL_ENSURE(EffectiveCompact); - YQL_ENSURE(SessionWindowParams.Key); - YQL_ENSURE(SessionWindowParams.KeyType); - YQL_ENSURE(SessionWindowParams.Init); - - preprocessLambda = AddSessionParamsMemberLambda(Node->Pos(), SessionStartMemberName, "", keyExtractor, - SessionWindowParams.Key, SessionWindowParams.Init, SessionWindowParams.Update, Ctx); + if (SessionWindowParams_.Update) { + YQL_ENSURE(EffectiveCompact_); + YQL_ENSURE(SessionWindowParams_.Key); + YQL_ENSURE(SessionWindowParams_.KeyType); + YQL_ENSURE(SessionWindowParams_.Init); + + preprocessLambda = AddSessionParamsMemberLambda(Node_->Pos(), SessionStartMemberName, "", keyExtractor, + SessionWindowParams_.Key, SessionWindowParams_.Init, SessionWindowParams_.Update, Ctx_); } else { - YQL_ENSURE(!SessionWindowParams.Key); - preprocessLambda = MakeIdentityLambda(Node->Pos(), Ctx); + YQL_ENSURE(!SessionWindowParams_.Key); + preprocessLambda = MakeIdentityLambda(Node_->Pos(), Ctx_); } return preprocessLambda; } @@ -1830,13 +1830,13 @@ TExprNode::TPtr TAggregateExpander::GeneratePreprocessLambda(const TExprNode::TP TExprNode::TPtr TAggregateExpander::GenerateCondenseSwitch(const TExprNode::TPtr& keyExtractor) { TExprNode::TPtr condenseSwitch; - if (SessionWindowParams.Update) { - YQL_ENSURE(EffectiveCompact); - YQL_ENSURE(SessionWindowParams.Key); - YQL_ENSURE(SessionWindowParams.KeyType); - YQL_ENSURE(SessionWindowParams.Init); + if (SessionWindowParams_.Update) { + YQL_ENSURE(EffectiveCompact_); + YQL_ENSURE(SessionWindowParams_.Key); + YQL_ENSURE(SessionWindowParams_.KeyType); + YQL_ENSURE(SessionWindowParams_.Init); - condenseSwitch = Ctx.Builder(Node->Pos()) + condenseSwitch = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Param("state") @@ -1863,8 +1863,8 @@ TExprNode::TPtr TAggregateExpander::GenerateCondenseSwitch(const TExprNode::TPtr .Seal() .Build(); } else { - YQL_ENSURE(!SessionWindowParams.Key); - condenseSwitch = Ctx.Builder(Node->Pos()) + YQL_ENSURE(!SessionWindowParams_.Key); + condenseSwitch = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Param("state") @@ -1882,26 +1882,26 @@ TExprNode::TPtr TAggregateExpander::GenerateCondenseSwitch(const TExprNode::TPtr TExprNode::TPtr TAggregateExpander::GeneratePostAggregateInitPhase() { - bool aggregateOnly = (Suffix != ""); - const auto& columnNames = aggregateOnly ? FinalColumnNames : InitialColumnNames; + bool aggregateOnly = (Suffix_ != ""); + const auto& columnNames = aggregateOnly ? FinalColumnNames_ : InitialColumnNames_; ui32 index = 0U; - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { parent .List(index++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "item") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal() .Seal(); } - if (SessionWindowParams.Update) { + if (SessionWindowParams_.Update) { parent .List(index++) .Atom(0, SessionStartMemberName) @@ -1915,13 +1915,13 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateInitPhase() }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < columnNames.size(); ++i) { - auto child = AggregatedColumns->Child(i); - auto trait = Traits[i]; - if (!EffectiveCompact) { + auto child = AggregatedColumns_->Child(i); + auto trait = Traits_[i]; + if (!EffectiveCompact_) { auto loadLambda = trait->Child(4); auto extractorLambda = GetFinalAggStateExtractor(i); - if (!DistinctFields.empty() || Suffix == "MergeManyFinalize") { + if (!DistinctFields_.empty() || Suffix_ == "MergeManyFinalize") { parent.List(index++) .Add(0, columnNames[i]) .Callable(1, "Map") @@ -1959,7 +1959,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateInitPhase() parent .Callable("CastStruct") .Arg(0, "item") - .Add(1, ExpandType(Node->Pos(), *initLambda->Head().Head().GetTypeAnn(), Ctx)) + .Add(1, ExpandType(Node_->Pos(), *initLambda->Head().Head().GetTypeAnn(), Ctx_)) .Seal(); } @@ -1983,16 +1983,16 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateInitPhase() }; if (distinctField) { - const bool isFirst = *Distinct2Columns[distinctField->Content()].begin() == i; + const bool isFirst = *Distinct2Columns_[distinctField->Content()].begin() == i; if (isFirst) { parent.List(index++) .Add(0, columnNames[i]) .List(1) .Callable(0, "NamedApply") - .Add(0, UdfSetCreate[distinctField->Content()]) + .Add(0, UdfSetCreate_[distinctField->Content()]) .List(1) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (!DistinctFieldNeedsPickle[distinctField->Content()]) { + if (!DistinctFieldNeedsPickle_[distinctField->Content()]) { parent.Callable(0, "Member") .Arg(0, "item") .Add(1, distinctField) @@ -2045,33 +2045,33 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateInitPhase() TExprNode::TPtr TAggregateExpander::GeneratePostAggregateSavePhase() { - bool aggregateOnly = (Suffix != ""); - const auto& columnNames = aggregateOnly ? FinalColumnNames : InitialColumnNames; + bool aggregateOnly = (Suffix_ != ""); + const auto& columnNames = aggregateOnly ? FinalColumnNames_ : InitialColumnNames_; ui32 index = 0U; - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("state") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { - if (KeyColumns->Child(i)->Content() == SessionStartMemberName) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { + if (KeyColumns_->Child(i)->Content() == SessionStartMemberName) { continue; } parent .List(index++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "state") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal() .Seal(); } - if (SessionOutputColumn) { + if (SessionOutputColumn_) { parent .List(index++) - .Atom(0, *SessionOutputColumn) + .Atom(0, *SessionOutputColumn_) .Callable(1, "Member") .Arg(0, "state") .Atom(1, SessionStartMemberName) @@ -2082,14 +2082,14 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateSavePhase() }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < columnNames.size(); ++i) { - auto child = AggregatedColumns->Child(i); - auto trait = Traits[i]; - auto finishLambda = (Suffix == "MergeState") ? trait->Child(3) : trait->Child(6); + auto child = AggregatedColumns_->Child(i); + auto trait = Traits_[i]; + auto finishLambda = (Suffix_ == "MergeState") ? trait->Child(3) : trait->Child(6); - if (!EffectiveCompact && (!DistinctFields.empty() || Suffix == "MergeManyFinalize")) { + if (!EffectiveCompact_ && (!DistinctFields_.empty() || Suffix_ == "MergeManyFinalize")) { if (child->Head().IsAtom()) { parent.List(index++) - .Add(0, FinalColumnNames[i]) + .Add(0, FinalColumnNames_[i]) .Callable(1, "Unwrap") .Callable(0, "Map") .Callable(0, "Member") @@ -2123,7 +2123,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateSavePhase() } else { auto distinctField = (child->ChildrenSize() == 3) ? child->Child(2) : nullptr; auto stateExtractor = [&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - const bool isFirst = distinctField ? (*Distinct2Columns[distinctField->Content()].begin() == i) : false; + const bool isFirst = distinctField ? (*Distinct2Columns_[distinctField->Content()].begin() == i) : false; if (distinctField && isFirst) { parent.Callable("Nth") .Callable(0, "Member") @@ -2144,7 +2144,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateSavePhase() if (child->Head().IsAtom()) { parent.List(index++) - .Add(0, FinalColumnNames[i]) + .Add(0, FinalColumnNames_[i]) .Apply(1, *finishLambda) .With(0) .Do(stateExtractor) @@ -2178,27 +2178,27 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateSavePhase() TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() { - bool aggregateOnly = (Suffix != ""); - const auto& columnNames = aggregateOnly ? FinalColumnNames : InitialColumnNames; + bool aggregateOnly = (Suffix_ != ""); + const auto& columnNames = aggregateOnly ? FinalColumnNames_ : InitialColumnNames_; ui32 index = 0U; - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Lambda() .Param("item") .Param("state") .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { parent .List(index++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "state") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal() .Seal(); } - if (SessionWindowParams.Update) { + if (SessionWindowParams_.Update) { parent .List(index++) .Atom(0, SessionStartMemberName) @@ -2212,14 +2212,14 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() }) .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { for (ui32 i = 0; i < columnNames.size(); ++i) { - auto child = AggregatedColumns->Child(i); - auto trait = Traits[i]; - if (!EffectiveCompact) { + auto child = AggregatedColumns_->Child(i); + auto trait = Traits_[i]; + if (!EffectiveCompact_) { auto loadLambda = trait->Child(4); auto mergeLambda = trait->Child(5); auto extractorLambda = GetFinalAggStateExtractor(i); - if (!DistinctFields.empty() || Suffix == "MergeManyFinalize") { + if (!DistinctFields_.empty() || Suffix_ == "MergeManyFinalize") { parent.List(index++) .Add(0, columnNames[i]) .Callable(1, "OptionalReduce") @@ -2261,7 +2261,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() } else { auto updateLambda = trait->Child(2); auto distinctField = (child->ChildrenSize() == 3) ? child->Child(2) : nullptr; - const bool isFirst = distinctField ? (*Distinct2Columns[distinctField->Content()].begin() == i) : false; + const bool isFirst = distinctField ? (*Distinct2Columns_[distinctField->Content()].begin() == i) : false; auto updateApply = [&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { parent.Apply(1, *updateLambda) .With(0) @@ -2276,7 +2276,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() parent .Callable("CastStruct") .Arg(0, "item") - .Add(1, ExpandType(Node->Pos(), *updateLambda->Head().Head().GetTypeAnn(), Ctx)) + .Add(1, ExpandType(Node_->Pos(), *updateLambda->Head().Head().GetTypeAnn(), Ctx_)) .Seal(); } @@ -2321,11 +2321,11 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() }; if (distinctField) { - auto distinctIndex = *Distinct2Columns[distinctField->Content()].begin(); + auto distinctIndex = *Distinct2Columns_[distinctField->Content()].begin(); ui32 newValueIndex = 0; auto newValue = [&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { parent.Callable(newValueIndex, "NamedApply") - .Add(0, UdfAddValue[distinctField->Content()]) + .Add(0, UdfAddValue_[distinctField->Content()]) .List(1) .Callable(0, "Nth") .Callable(0, "Member") @@ -2335,7 +2335,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() .Atom(1, "0", TNodeFlags::Default) .Seal() .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (!DistinctFieldNeedsPickle[distinctField->Content()]) { + if (!DistinctFieldNeedsPickle_[distinctField->Content()]) { parent.Callable(1, "Member") .Arg(0, "item") .Add(1, distinctField) @@ -2362,10 +2362,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() .Add(0, columnNames[i]) .Callable(1, "If") .Callable(0, "NamedApply") - .Add(0, UdfWasChanged[distinctField->Content()]) + .Add(0, UdfWasChanged_[distinctField->Content()]) .List(1) .Callable(0, "NamedApply") - .Add(0, UdfAddValue[distinctField->Content()]) + .Add(0, UdfAddValue_[distinctField->Content()]) .List(1) .Callable(0, "Nth") .Callable(0, "Member") @@ -2375,7 +2375,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() .Atom(1, "0", TNodeFlags::Default) .Seal() .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - if (!DistinctFieldNeedsPickle[distinctField->Content()]) { + if (!DistinctFieldNeedsPickle_[distinctField->Content()]) { parent.Callable(1, "Member") .Arg(0, "item") .Add(1, distinctField) @@ -2431,7 +2431,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePostAggregateMergePhase() } TExprNode::TPtr TAggregateExpander::GenerateJustOverStates(const TExprNode::TPtr& input, const TIdxSet& indicies) { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .Callable("Map") .Add(0, input) .Lambda(1) @@ -2439,13 +2439,13 @@ TExprNode::TPtr TAggregateExpander::GenerateJustOverStates(const TExprNode::TPtr .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 pos = 0; - for (ui32 i = 0; i < KeyColumns->ChildrenSize(); ++i) { + for (ui32 i = 0; i < KeyColumns_->ChildrenSize(); ++i) { parent .List(pos++) - .Add(0, KeyColumns->ChildPtr(i)) + .Add(0, KeyColumns_->ChildPtr(i)) .Callable(1, "Member") .Arg(0, "row") - .Add(1, KeyColumns->ChildPtr(i)) + .Add(1, KeyColumns_->ChildPtr(i)) .Seal() .Seal(); } @@ -2453,11 +2453,11 @@ TExprNode::TPtr TAggregateExpander::GenerateJustOverStates(const TExprNode::TPtr for (ui32 i : indicies) { parent .List(pos++) - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Callable(1, "Just") .Callable(0, "Member") .Arg(0, "row") - .Add(1, InitialColumnNames[i]) + .Add(1, InitialColumnNames_[i]) .Seal() .Seal() .Seal(); @@ -2472,7 +2472,7 @@ TExprNode::TPtr TAggregateExpander::GenerateJustOverStates(const TExprNode::TPtr } TExprNode::TPtr TAggregateExpander::SerializeIdxSet(const TIdxSet& indicies) { - return Ctx.Builder(Node->Pos()) + return Ctx_.Builder(Node_->Pos()) .List() .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { ui32 pos = 0; @@ -2487,24 +2487,24 @@ TExprNode::TPtr TAggregateExpander::SerializeIdxSet(const TIdxSet& indicies) { } TExprNode::TPtr TAggregateExpander::GeneratePhases() { - const TExprNode::TPtr cleanOutputSettings = RemoveSetting(*Node->Child(3), "output_columns", Ctx); - const bool many = HaveDistinct; + const TExprNode::TPtr cleanOutputSettings = RemoveSetting(*Node_->Child(3), "output_columns", Ctx_); + const bool many = HaveDistinct_; YQL_CLOG(DEBUG, Core) << "Aggregate: generate " << (many ? "phases with distinct" : "simple phases"); TExprNode::TListType mergeTraits; - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - auto originalTrait = AggregatedColumns->Child(index)->ChildPtr(1); - auto extractor = Ctx.Builder(Node->Pos()) + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + auto originalTrait = AggregatedColumns_->Child(index)->ChildPtr(1); + auto extractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Callable("Member") .Arg(0, "row") - .Add(1, InitialColumnNames[index]) + .Add(1, InitialColumnNames_[index]) .Seal() .Seal() .Build(); if (many) { - extractor = Ctx.Builder(Node->Pos()) + extractor = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Callable("Unwrap") @@ -2517,16 +2517,16 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { } bool isAggApply = originalTrait->IsCallable("AggApply"); - auto serializedStateType = isAggApply ? AggApplySerializedStateType(originalTrait, Ctx) : originalTrait->Child(3)->GetTypeAnn(); + auto serializedStateType = isAggApply ? AggApplySerializedStateType(originalTrait, Ctx_) : originalTrait->Child(3)->GetTypeAnn(); if (many) { - serializedStateType = Ctx.MakeType<TOptionalExprType>(serializedStateType); + serializedStateType = Ctx_.MakeType<TOptionalExprType>(serializedStateType); } - auto extractorTypeNode = Ctx.Builder(Node->Pos()) + auto extractorTypeNode = Ctx_.Builder(Node_->Pos()) .Callable("StructType") .List(0) - .Add(0, InitialColumnNames[index]) - .Add(1, ExpandType(Node->Pos(), *serializedStateType, Ctx)) + .Add(0, InitialColumnNames_[index]) + .Add(1, ExpandType(Node_->Pos(), *serializedStateType, Ctx_)) .Seal() .Seal() .Build(); @@ -2534,14 +2534,14 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { if (isAggApply) { auto initialType = originalTrait->GetTypeAnn(); if (many) { - initialType = Ctx.MakeType<TOptionalExprType>(initialType); + initialType = Ctx_.MakeType<TOptionalExprType>(initialType); } - auto originalExtractorTypeNode = Ctx.Builder(Node->Pos()) + auto originalExtractorTypeNode = Ctx_.Builder(Node_->Pos()) .Callable("StructType") .List(0) - .Add(0, InitialColumnNames[index]) - .Add(1, ExpandType(Node->Pos(), *initialType, Ctx)) + .Add(0, InitialColumnNames_[index]) + .Add(1, ExpandType(Node_->Pos(), *initialType, Ctx_)) .Seal() .Seal() .Build(); @@ -2551,13 +2551,13 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { auto func = name.substr(3); TVector<ui32> argTypes; bool needRetype = false; - auto status = ExtractPgTypesFromMultiLambda(originalTrait->ChildRef(2), argTypes, needRetype, Ctx); + auto status = ExtractPgTypesFromMultiLambda(originalTrait->ChildRef(2), argTypes, needRetype, Ctx_); YQL_ENSURE(status == IGraphTransformer::TStatus::Ok); const NPg::TAggregateDesc& aggDesc = NPg::LookupAggregation(TString(func), argTypes); name = "pg_" + aggDesc.Name + "#" + ToString(aggDesc.AggId); } - mergeTraits.push_back(Ctx.Builder(Node->Pos()) + mergeTraits.push_back(Ctx_.Builder(Node_->Pos()) .Callable(many ? "AggApplyManyState" : "AggApplyState") .Atom(0, name) .Add(1, extractorTypeNode) @@ -2567,7 +2567,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { .Build()); } else { YQL_ENSURE(originalTrait->IsCallable("AggregationTraits")); - mergeTraits.push_back(Ctx.Builder(Node->Pos()) + mergeTraits.push_back(Ctx_.Builder(Node_->Pos()) .Callable("AggregationTraits") .Add(0, extractorTypeNode) .Add(1, extractor) @@ -2588,10 +2588,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { } TExprNode::TListType finalizeColumns; - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - finalizeColumns.push_back(Ctx.Builder(Node->Pos()) + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + finalizeColumns.push_back(Ctx_.Builder(Node_->Pos()) .List() - .Add(0, AggregatedColumns->Child(index)->ChildPtr(0)) + .Add(0, AggregatedColumns_->Child(index)->ChildPtr(0)) .Add(1, mergeTraits[index]) .Seal() .Build()); @@ -2600,29 +2600,29 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { if (!many) { // simple Combine + MergeFinalize TExprNode::TListType combineColumns; - for (ui32 index = 0; index < AggregatedColumns->ChildrenSize(); ++index) { - combineColumns.push_back(Ctx.Builder(Node->Pos()) + for (ui32 index = 0; index < AggregatedColumns_->ChildrenSize(); ++index) { + combineColumns.push_back(Ctx_.Builder(Node_->Pos()) .List() - .Add(0, InitialColumnNames[index]) - .Add(1, AggregatedColumns->Child(index)->ChildPtr(1)) + .Add(0, InitialColumnNames_[index]) + .Add(1, AggregatedColumns_->Child(index)->ChildPtr(1)) .Seal() .Build()); } - auto combine = Ctx.Builder(Node->Pos()) + auto combine = Ctx_.Builder(Node_->Pos()) .Callable("AggregateCombine") - .Add(0, AggList) - .Add(1, KeyColumns) - .Add(2, Ctx.NewList(Node->Pos(), std::move(combineColumns))) + .Add(0, AggList_) + .Add(1, KeyColumns_) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(combineColumns))) .Add(3, cleanOutputSettings) .Seal() .Build(); - auto mergeFinalize = Ctx.Builder(Node->Pos()) + auto mergeFinalize = Ctx_.Builder(Node_->Pos()) .Callable("AggregateMergeFinalize") .Add(0, combine) - .Add(1, KeyColumns) - .Add(2, Ctx.NewList(Node->Pos(), std::move(finalizeColumns))) + .Add(1, KeyColumns_) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(finalizeColumns))) .Add(3, cleanOutputSettings) .Seal() .Build(); @@ -2641,40 +2641,40 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { TExprNode::TListType unionAllInputs; TExprNode::TListType streams; - if (!NonDistinctColumns.empty()) { + if (!NonDistinctColumns_.empty()) { TExprNode::TListType combineColumns; - for (ui32 i : NonDistinctColumns) { - combineColumns.push_back(Ctx.Builder(Node->Pos()) + for (ui32 i : NonDistinctColumns_) { + combineColumns.push_back(Ctx_.Builder(Node_->Pos()) .List() - .Add(0, InitialColumnNames[i]) - .Add(1, AggregatedColumns->Child(i)->ChildPtr(1)) + .Add(0, InitialColumnNames_[i]) + .Add(1, AggregatedColumns_->Child(i)->ChildPtr(1)) .Seal() .Build()); } - auto combine = Ctx.Builder(Node->Pos()) + auto combine = Ctx_.Builder(Node_->Pos()) .Callable("AggregateCombine") - .Add(0, AggList) - .Add(1, KeyColumns) - .Add(2, Ctx.NewList(Node->Pos(), std::move(combineColumns))) + .Add(0, AggList_) + .Add(1, KeyColumns_) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(combineColumns))) .Add(3, cleanOutputSettings) .Seal() .Build(); - unionAllInputs.push_back(GenerateJustOverStates(combine, NonDistinctColumns)); - streams.push_back(SerializeIdxSet(NonDistinctColumns)); + unionAllInputs.push_back(GenerateJustOverStates(combine, NonDistinctColumns_)); + streams.push_back(SerializeIdxSet(NonDistinctColumns_)); } - for (ui32 index = 0; index < DistinctFields.size(); ++index) { - auto distinctField = DistinctFields[index]; - auto& indicies = Distinct2Columns[distinctField->Content()]; - TExprNode::TListType allKeyColumns = KeyColumns->ChildrenList(); + for (ui32 index = 0; index < DistinctFields_.size(); ++index) { + auto distinctField = DistinctFields_[index]; + auto& indicies = Distinct2Columns_[distinctField->Content()]; + TExprNode::TListType allKeyColumns = KeyColumns_->ChildrenList(); allKeyColumns.push_back(distinctField); - auto distinct = Ctx.Builder(Node->Pos()) + auto distinct = Ctx_.Builder(Node_->Pos()) .Callable("Aggregate") - .Add(0, AggList) - .Add(1, Ctx.NewList(Node->Pos(), std::move(allKeyColumns))) + .Add(0, AggList_) + .Add(1, Ctx_.NewList(Node_->Pos(), std::move(allKeyColumns))) .List(2) .Seal() .Add(3, cleanOutputSettings) @@ -2683,10 +2683,10 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { TExprNode::TListType combineColumns; for (ui32 i : indicies) { - auto trait = AggregatedColumns->Child(i)->ChildPtr(1); + auto trait = AggregatedColumns_->Child(i)->ChildPtr(1); bool isAggApply = trait->IsCallable("AggApply"); if (isAggApply) { - trait = Ctx.Builder(Node->Pos()) + trait = Ctx_.Builder(Node_->Pos()) .Callable("AggApply") .Add(0, trait->ChildPtr(0)) .Callable(1, "StructType") @@ -2711,7 +2711,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { } else { TExprNode::TPtr newInit; if (trait->ChildPtr(1)->Head().ChildrenSize() == 1) { - newInit = Ctx.Builder(Node->Pos()) + newInit = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Apply(trait->ChildPtr(1)) @@ -2725,7 +2725,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { .Seal() .Build(); } else { - newInit = Ctx.Builder(Node->Pos()) + newInit = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Param("parent") @@ -2744,7 +2744,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { TExprNode::TPtr newUpdate; if (trait->ChildPtr(2)->Head().ChildrenSize() == 2) { - newUpdate = Ctx.Builder(Node->Pos()) + newUpdate = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Param("state") @@ -2760,7 +2760,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { .Seal() .Build(); } else { - newUpdate = Ctx.Builder(Node->Pos()) + newUpdate = Ctx_.Builder(Node_->Pos()) .Lambda() .Param("row") .Param("state") @@ -2779,7 +2779,7 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { .Build(); } - trait = Ctx.Builder(Node->Pos()) + trait = Ctx_.Builder(Node_->Pos()) .Callable("AggregationTraits") .Callable(0, "StructType") .List(0) @@ -2798,19 +2798,19 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { .Build(); } - combineColumns.push_back(Ctx.Builder(Node->Pos()) + combineColumns.push_back(Ctx_.Builder(Node_->Pos()) .List() - .Add(0, InitialColumnNames[i]) + .Add(0, InitialColumnNames_[i]) .Add(1, trait) .Seal() .Build()); } - auto combine = Ctx.Builder(Node->Pos()) + auto combine = Ctx_.Builder(Node_->Pos()) .Callable("AggregateCombine") .Add(0, distinct) - .Add(1, KeyColumns) - .Add(2, Ctx.NewList(Node->Pos(), std::move(combineColumns))) + .Add(1, KeyColumns_) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(combineColumns))) .Add(3, cleanOutputSettings) .Seal() .Build(); @@ -2819,9 +2819,9 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { streams.push_back(SerializeIdxSet(indicies)); } - if (UseBlocks) { + if (UseBlocks_) { for (ui32 i = 0; i < unionAllInputs.size(); ++i) { - unionAllInputs[i] = Ctx.Builder(Node->Pos()) + unionAllInputs[i] = Ctx_.Builder(Node_->Pos()) .Callable("Map") .Add(0, unionAllInputs[i]) .Lambda(1) @@ -2840,16 +2840,16 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { } auto settings = cleanOutputSettings; - if (UseBlocks) { - settings = AddSetting(*settings, Node->Pos(), "many_streams", Ctx.NewList(Node->Pos(), std::move(streams)), Ctx); + if (UseBlocks_) { + settings = AddSetting(*settings, Node_->Pos(), "many_streams", Ctx_.NewList(Node_->Pos(), std::move(streams)), Ctx_); } - auto unionAll = Ctx.NewCallable(Node->Pos(), "UnionAll", std::move(unionAllInputs)); - auto mergeManyFinalize = Ctx.Builder(Node->Pos()) + auto unionAll = Ctx_.NewCallable(Node_->Pos(), "UnionAll", std::move(unionAllInputs)); + auto mergeManyFinalize = Ctx_.Builder(Node_->Pos()) .Callable("AggregateMergeManyFinalize") .Add(0, unionAll) - .Add(1, KeyColumns) - .Add(2, Ctx.NewList(Node->Pos(), std::move(finalizeColumns))) + .Add(1, KeyColumns_) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(finalizeColumns))) .Add(3, settings) .Seal() .Build(); @@ -2858,11 +2858,11 @@ TExprNode::TPtr TAggregateExpander::GeneratePhases() { } TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombine() { - if (HaveSessionSetting || HaveDistinct) { + if (HaveSessionSetting_ || HaveDistinct_) { return nullptr; } - for (const auto& x : AggregatedColumns->Children()) { + for (const auto& x : AggregatedColumns_->Children()) { auto trait = x->ChildPtr(1); if (!trait->IsCallable("AggApply")) { return nullptr; @@ -2873,15 +2873,15 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockCombine() { } TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalize() { - if (UsePartitionsByKeys || !UseBlocks) { + if (UsePartitionsByKeys_ || !UseBlocks_) { return nullptr; } - if (HaveSessionSetting || HaveDistinct) { + if (HaveSessionSetting_ || HaveDistinct_) { return nullptr; } - for (const auto& x : AggregatedColumns->Children()) { + for (const auto& x : AggregatedColumns_->Children()) { auto trait = x->ChildPtr(1); if (!trait->IsCallable({ "AggApplyState", "AggApplyManyState" })) { return nullptr; @@ -2892,16 +2892,16 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalize() { } TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalizeHashed() { - if (!TypesCtx.ArrowResolver) { + if (!TypesCtx_.ArrowResolver) { return nullptr; } - if (KeyColumns->ChildrenSize() == 0) { + if (KeyColumns_->ChildrenSize() == 0) { return nullptr; } - bool isMany = Suffix == "MergeManyFinalize"; - auto streamArg = Ctx.NewArgument(Node->Pos(), "stream"); + bool isMany = Suffix_ == "MergeManyFinalize"; + auto streamArg = Ctx_.NewArgument(Node_->Pos(), "stream"); TExprNode::TListType keyIdxs; TVector<TString> outputColumns; TExprNode::TListType aggs; @@ -2913,29 +2913,29 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalizeHashed() { TExprNode::TPtr aggBlocks; if (!isMany) { - aggBlocks = Ctx.Builder(Node->Pos()) + aggBlocks = Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "BlockMergeFinalizeHashed") .Callable(0, "FromFlow") .Add(0, blocks) .Seal() - .Add(1, Ctx.NewList(Node->Pos(), std::move(keyIdxs))) - .Add(2, Ctx.NewList(Node->Pos(), std::move(aggs))) + .Add(1, Ctx_.NewList(Node_->Pos(), std::move(keyIdxs))) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(aggs))) .Seal() .Seal() .Build(); } else { - auto manyStreamsSetting = GetSetting(*Node->Child(3), "many_streams"); + auto manyStreamsSetting = GetSetting(*Node_->Child(3), "many_streams"); YQL_ENSURE(manyStreamsSetting, "Missing many_streams setting"); - aggBlocks = Ctx.Builder(Node->Pos()) + aggBlocks = Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "BlockMergeManyFinalizeHashed") .Callable(0, "FromFlow") .Add(0, blocks) .Seal() - .Add(1, Ctx.NewList(Node->Pos(), std::move(keyIdxs))) - .Add(2, Ctx.NewList(Node->Pos(), std::move(aggs))) + .Add(1, Ctx_.NewList(Node_->Pos(), std::move(keyIdxs))) + .Add(2, Ctx_.NewList(Node_->Pos(), std::move(aggs))) .Atom(3, ToString(streamIdxColumn)) .Add(4, manyStreamsSetting->TailPtr()) .Seal() @@ -2943,7 +2943,7 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalizeHashed() { .Build(); } - auto aggWideFlow = Ctx.Builder(Node->Pos()) + auto aggWideFlow = Ctx_.Builder(Node_->Pos()) .Callable("ToFlow") .Callable(0, "WideFromBlocks") .Callable(0, "FromFlow") @@ -2952,14 +2952,14 @@ TExprNode::TPtr TAggregateExpander::TryGenerateBlockMergeFinalizeHashed() { .Seal() .Seal() .Build(); - auto finalFlow = MakeNarrowMap(Node->Pos(), outputColumns, aggWideFlow, Ctx); - auto root = Ctx.NewCallable(Node->Pos(), "FromFlow", { finalFlow }); - auto lambdaStream = Ctx.NewLambda(Node->Pos(), Ctx.NewArguments(Node->Pos(), { streamArg }), std::move(root)); + auto finalFlow = MakeNarrowMap(Node_->Pos(), outputColumns, aggWideFlow, Ctx_); + auto root = Ctx_.NewCallable(Node_->Pos(), "FromFlow", { finalFlow }); + auto lambdaStream = Ctx_.NewLambda(Node_->Pos(), Ctx_.NewArguments(Node_->Pos(), { streamArg }), std::move(root)); - auto keySelector = BuildKeySelector(Node->Pos(), *OriginalRowType, KeyColumns, Ctx); - return Ctx.Builder(Node->Pos()) + auto keySelector = BuildKeySelector(Node_->Pos(), *OriginalRowType_, KeyColumns_, Ctx_); + return Ctx_.Builder(Node_->Pos()) .Callable("ShuffleByKeys") - .Add(0, AggList) + .Add(0, AggList_) .Add(1, keySelector) .Lambda(2) .Param("stream") diff --git a/yql/essentials/core/yql_aggregate_expander.h b/yql/essentials/core/yql_aggregate_expander.h index eed24b61f31..e3c9eae5626 100644 --- a/yql/essentials/core/yql_aggregate_expander.h +++ b/yql/essentials/core/yql_aggregate_expander.h @@ -10,31 +10,31 @@ class TAggregateExpander { public: TAggregateExpander(bool usePartitionsByKeys, const bool useFinalizeByKeys, const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& typesCtx, bool forceCompact = false, bool compactForDistinct = false, bool usePhases = false, bool useBlocks = false) - : Node(node) - , Ctx(ctx) - , TypesCtx(typesCtx) - , UsePartitionsByKeys(usePartitionsByKeys) - , UseFinalizeByKeys(useFinalizeByKeys) - , ForceCompact(forceCompact) - , CompactForDistinct(compactForDistinct) - , UsePhases(usePhases) - , AggregatedColumns(nullptr) - , VoidNode(ctx.NewCallable(node->Pos(), "Void", {})) - , HaveDistinct(false) - , EffectiveCompact(false) - , HaveSessionSetting(false) - , OriginalRowType(nullptr) - , RowType(nullptr) - , UseBlocks(useBlocks) + : Node_(node) + , Ctx_(ctx) + , TypesCtx_(typesCtx) + , UsePartitionsByKeys_(usePartitionsByKeys) + , UseFinalizeByKeys_(useFinalizeByKeys) + , ForceCompact_(forceCompact) + , CompactForDistinct_(compactForDistinct) + , UsePhases_(usePhases) + , AggregatedColumns_(nullptr) + , VoidNode_(ctx.NewCallable(node->Pos(), "Void", {})) + , HaveDistinct_(false) + , EffectiveCompact_(false) + , HaveSessionSetting_(false) + , OriginalRowType_(nullptr) + , RowType_(nullptr) + , UseBlocks_(useBlocks) { - PreMap = Ctx.Builder(node->Pos()) + PreMap_ = Ctx_.Builder(node->Pos()) .Lambda() .Param("premap") .Callable("Just").Arg(0, "premap").Seal() .Seal().Build(); - SortParams = { - .Key = VoidNode, - .Order = VoidNode + SortParams_ = { + .Key = VoidNode_, + .Order = VoidNode_ }; } @@ -91,45 +91,45 @@ private: private: static constexpr TStringBuf SessionStartMemberName = "_yql_group_session_start"; - const TExprNode::TPtr Node; - TExprContext& Ctx; - TTypeAnnotationContext& TypesCtx; - bool UsePartitionsByKeys; - bool UseFinalizeByKeys = false; - bool ForceCompact; - bool CompactForDistinct; - bool UsePhases; - TStringBuf Suffix; - - TSessionWindowParams SessionWindowParams; - TExprNode::TPtr AggList; - TExprNode::TListType Traits; - TExprNode::TPtr KeyColumns; - TExprNode::TPtr AggregatedColumns; - const TExprNode::TPtr VoidNode; - TMaybe<TStringBuf> SessionOutputColumn; - TSortParams SortParams; - bool HaveDistinct; - bool EffectiveCompact; - bool HaveSessionSetting; - const TStructExprType* OriginalRowType; - const TStructExprType* RowType; - TVector<const TItemExprType*> RowItems; - TExprNode::TPtr PreMap; - bool UseBlocks; - - TExprNode::TListType InitialColumnNames; - TExprNode::TListType FinalColumnNames; - TExprNode::TListType DistinctFields; - TExprNode::TListType NothingStates; - - std::unordered_map<std::string_view, TIdxSet> Distinct2Columns; - TIdxSet NonDistinctColumns; - - std::unordered_map<std::string_view, bool> DistinctFieldNeedsPickle; - std::unordered_map<std::string_view, TExprNode::TPtr> UdfSetCreate; - std::unordered_map<std::string_view, TExprNode::TPtr> UdfAddValue; - std::unordered_map<std::string_view, TExprNode::TPtr> UdfWasChanged; + const TExprNode::TPtr Node_; + TExprContext& Ctx_; + TTypeAnnotationContext& TypesCtx_; + bool UsePartitionsByKeys_; + bool UseFinalizeByKeys_ = false; + bool ForceCompact_; + bool CompactForDistinct_; + bool UsePhases_; + TStringBuf Suffix_; + + TSessionWindowParams SessionWindowParams_; + TExprNode::TPtr AggList_; + TExprNode::TListType Traits_; + TExprNode::TPtr KeyColumns_; + TExprNode::TPtr AggregatedColumns_; + const TExprNode::TPtr VoidNode_; + TMaybe<TStringBuf> SessionOutputColumn_; + TSortParams SortParams_; + bool HaveDistinct_; + bool EffectiveCompact_; + bool HaveSessionSetting_; + const TStructExprType* OriginalRowType_; + const TStructExprType* RowType_; + TVector<const TItemExprType*> RowItems_; + TExprNode::TPtr PreMap_; + bool UseBlocks_; + + TExprNode::TListType InitialColumnNames_; + TExprNode::TListType FinalColumnNames_; + TExprNode::TListType DistinctFields_; + TExprNode::TListType NothingStates_; + + std::unordered_map<std::string_view, TIdxSet> Distinct2Columns_; + TIdxSet NonDistinctColumns_; + + std::unordered_map<std::string_view, bool> DistinctFieldNeedsPickle_; + std::unordered_map<std::string_view, TExprNode::TPtr> UdfSetCreate_; + std::unordered_map<std::string_view, TExprNode::TPtr> UdfAddValue_; + std::unordered_map<std::string_view, TExprNode::TPtr> UdfWasChanged_; }; inline TExprNode::TPtr ExpandAggregatePeepholeImpl(const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& typesCtx, diff --git a/yql/essentials/core/yql_callable_transform.h b/yql/essentials/core/yql_callable_transform.h index 1c683774a05..c8b5de61b42 100644 --- a/yql/essentials/core/yql_callable_transform.h +++ b/yql/essentials/core/yql_callable_transform.h @@ -23,8 +23,8 @@ template <class TDerived> class TCallableTransformerBase : public TGraphTransformerBase { public: TCallableTransformerBase(TTypeAnnotationContext& types, bool instantOnly) - : Types(types) - , InstantOnly(instantOnly) + : Types_(types) + , InstantOnly_(instantOnly) {} IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { @@ -38,7 +38,7 @@ public: auto name = input->Content(); TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), + return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << NormalizeCallableName(name)); }); @@ -88,7 +88,7 @@ public: } } else { bool foundFunc = false; - for (auto& datasource : Types.DataSources) { + for (auto& datasource : Types_.DataSources) { if (!datasource->CanParse(*input)) { continue; } @@ -99,7 +99,7 @@ public: } if (!foundFunc) { - for (auto& datasink : Types.DataSinks) { + for (auto& datasink : Types_.DataSinks) { if (!datasink->CanParse(*input)) { continue; } @@ -120,23 +120,23 @@ public: } NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { - const auto it = PendingNodes.find(&input); - YQL_ENSURE(it != PendingNodes.cend()); + const auto it = PendingNodes_.find(&input); + YQL_ENSURE(it != PendingNodes_.cend()); return static_cast<TDerived*>(this)->GetTransformer(*it->second.second).GetAsyncFuture(input); } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - const auto it = PendingNodes.find(input.Get()); - YQL_ENSURE(it != PendingNodes.cend()); + const auto it = PendingNodes_.find(input.Get()); + YQL_ENSURE(it != PendingNodes_.cend()); const auto provider = it->second.second; IGraphTransformer& transformer = static_cast<TDerived*>(this)->GetTransformer(*provider); const auto status = transformer.ApplyAsyncChanges(it->second.first, output, ctx); - PendingNodes.erase(it); + PendingNodes_.erase(it); return status; } void Rewind() override { - PendingNodes.clear(); + PendingNodes_.clear(); } protected: @@ -171,7 +171,7 @@ protected: } auto datasinkName = input.Child(1)->Child(0)->Content(); - auto datasink = Types.DataSinkMap.FindPtr(datasinkName); + auto datasink = Types_.DataSinkMap.FindPtr(datasinkName); if (!datasink) { ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; @@ -194,7 +194,7 @@ protected: } auto datasourceName = input.Child(1)->Child(0)->Content(); - auto datasource = Types.DataSourceMap.FindPtr(datasourceName); + auto datasource = Types_.DataSourceMap.FindPtr(datasourceName); if (!datasource) { ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); return nullptr; @@ -217,7 +217,7 @@ protected: } auto datasinkName = input.Child(1)->Child(0)->Content(); - auto datasink = Types.DataSinkMap.FindPtr(datasinkName); + auto datasink = Types_.DataSinkMap.FindPtr(datasinkName); if (!datasink) { ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; @@ -241,7 +241,7 @@ protected: if (input.Child(1)->IsCallable("DataSource")) { auto datasourceName = input.Child(1)->Child(0)->Content(); - auto datasource = Types.DataSourceMap.FindPtr(datasourceName); + auto datasource = Types_.DataSourceMap.FindPtr(datasourceName); if (!datasource) { ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName)); return nullptr; @@ -252,7 +252,7 @@ protected: if (input.Child(1)->IsCallable("DataSink")) { auto datasinkName = input.Child(1)->Child(0)->Content(); - auto datasink = Types.DataSinkMap.FindPtr(datasinkName); + auto datasink = Types_.DataSinkMap.FindPtr(datasinkName); if (!datasink) { ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName)); return nullptr; @@ -268,22 +268,22 @@ protected: const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { auto status = static_cast<TDerived*>(this)->GetTransformer(dataProvider).Transform(input, output, ctx); if (status.Level == IGraphTransformer::TStatus::Async) { - if (InstantOnly) { + if (InstantOnly_) { ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Async status is not allowed for instant transform, provider name: " << dataProvider.GetName())); return IGraphTransformer::TStatus::Error; } - PendingNodes[input.Get()] = std::make_pair(input, &dataProvider); + PendingNodes_[input.Get()] = std::make_pair(input, &dataProvider); } return status; } protected: - TTypeAnnotationContext& Types; - const bool InstantOnly; - TNodeMap<std::pair<TExprNode::TPtr, IDataProvider*>> PendingNodes; + TTypeAnnotationContext& Types_; + const bool InstantOnly_; + TNodeMap<std::pair<TExprNode::TPtr, IDataProvider*>> PendingNodes_; }; } // NYql diff --git a/yql/essentials/core/yql_execution.cpp b/yql/essentials/core/yql_execution.cpp index 57892229b36..0199ce9f596 100644 --- a/yql/essentials/core/yql_execution.cpp +++ b/yql/essentials/core/yql_execution.cpp @@ -40,18 +40,18 @@ public: TExecutionTransformer(TTypeAnnotationContext& types, TOperationProgressWriter writer, bool withFinalize) - : Types(types) - , Writer(writer) - , WithFinalize(withFinalize) - , DeterministicMode(GetEnv("YQL_DETERMINISTIC_MODE")) + : Types_(types) + , Writer_(writer) + , WithFinalize_(withFinalize) + , DeterministicMode_(GetEnv("YQL_DETERMINISTIC_MODE")) { Rewind(); } TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - if (FinalizingTransformer) { + if (FinalizingTransformer_) { YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer, root #" << input->UniqueId(); - auto status = FinalizingTransformer->Transform(input, output, ctx); + auto status = FinalizingTransformer_->Transform(input, output, ctx); YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer done, output #" << output->UniqueId() << ", status: " << status; return status; } @@ -60,7 +60,7 @@ public: output = input; if (RewriteSanityCheck) { VisitExpr(input, [&](const TExprNode::TPtr& localInput) { - if (NewNodes.cend() != NewNodes.find(localInput.Get())) { + if (NewNodes_.cend() != NewNodes_.find(localInput.Get())) { Cerr << "found old node: #" << localInput->UniqueId() << "\n" << input->Dump(); YQL_ENSURE(false); } @@ -75,35 +75,35 @@ public: } status = status.Combine(ExecuteNode(input, output, ctx, 0)); - for (auto node: FreshPendingNodes) { + for (auto node: FreshPendingNodes_) { if (TExprNode::EState::ExecutionPending == node->GetState()) { node->SetState(TExprNode::EState::ConstrComplete); } } - FreshPendingNodes.clear(); + FreshPendingNodes_.clear(); if (!ReplaceNewNodes(output, ctx)) { return TStatus::Error; } YQL_CLOG(INFO, CoreExecution) << "Finish, output #" << output->UniqueId() << ", status: " << status; - if (status != TStatus::Ok || !WithFinalize) { + if (status != TStatus::Ok || !WithFinalize_) { return status; } YQL_CLOG(INFO, CoreExecution) << "Creating finalizing transformer, output #" << output->UniqueId(); - FinalizingTransformer = CreateCompositeFinalizingTransformer(Types); - return FinalizingTransformer->Transform(input, output, ctx); + FinalizingTransformer_ = CreateCompositeFinalizingTransformer(Types_); + return FinalizingTransformer_->Transform(input, output, ctx); } NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { - return FinalizingTransformer ? - FinalizingTransformer->GetAsyncFuture(input) : - State->Promise.GetFuture(); + return FinalizingTransformer_ ? + FinalizingTransformer_->GetAsyncFuture(input) : + State_->Promise.GetFuture(); } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - if (FinalizingTransformer) { - return FinalizingTransformer->ApplyAsyncChanges(input, output, ctx); + if (FinalizingTransformer_) { + return FinalizingTransformer_->ApplyAsyncChanges(input, output, ctx); } output = input; @@ -113,26 +113,26 @@ public: TState::TQueueType completed; auto newPromise = NThreading::NewPromise(); { - TGuard<TAdaptiveLock> guard(State->Lock); - completed.Swap(State->Completed); - State->Promise.Swap(newPromise); - State->HasResult = false; + TGuard<TAdaptiveLock> guard(State_->Lock); + completed.Swap(State_->Completed); + State_->Promise.Swap(newPromise); + State_->HasResult = false; } for (auto& item : completed) { - auto collectedIt = CollectingNodes.find(item.Node); - if (collectedIt != CollectingNodes.end()) { + auto collectedIt = CollectingNodes_.find(item.Node); + if (collectedIt != CollectingNodes_.end()) { YQL_CLOG(INFO, CoreExecution) << "Completed async cleanup for node #" << item.Node->UniqueId(); TExprNode::TPtr callableOutput; auto status = item.DataProvider->GetTrackableNodeProcessor().GetCleanupTransformer().ApplyAsyncChanges(item.Node, callableOutput, ctx); combinedStatus = combinedStatus.Combine(status); - CollectingNodes.erase(collectedIt); + CollectingNodes_.erase(collectedIt); continue; } YQL_CLOG(INFO, CoreExecution) << "Completed async execution for node #" << item.Node->UniqueId(); - auto asyncIt = AsyncNodes.find(item.Node); - YQL_ENSURE(asyncIt != AsyncNodes.end()); + auto asyncIt = AsyncNodes_.find(item.Node); + YQL_ENSURE(asyncIt != AsyncNodes_.end()); TExprNode::TPtr callableOutput; auto status = item.DataProvider->GetCallableExecutionTransformer().ApplyAsyncChanges(item.Node, callableOutput, ctx); Y_ABORT_UNLESS(callableOutput); @@ -144,7 +144,7 @@ public: if (callableOutput != item.Node) { YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << item.Node->UniqueId() << " to #" << callableOutput->UniqueId() << " in ApplyAsyncChanges()"; - NewNodes[item.Node] = callableOutput; + NewNodes_[item.Node] = callableOutput; combinedStatus = combinedStatus.Combine(TStatus(TStatus::Repeat, true)); FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput); } @@ -160,7 +160,7 @@ public: FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput); } - AsyncNodes.erase(asyncIt); + AsyncNodes_.erase(asyncIt); } if (!ReplaceNewNodes(output, ctx)) { @@ -175,14 +175,14 @@ public: } bool ReplaceNewNodes(TExprNode::TPtr& output, TExprContext& ctx) { - if (!NewNodes.empty()) { - TOptimizeExprSettings settings(&Types); + if (!NewNodes_.empty()) { + TOptimizeExprSettings settings(&Types_); settings.VisitChanges = true; settings.VisitStarted = true; auto replaceStatus = OptimizeExpr(output, output, [&](const TExprNode::TPtr& input, TExprContext& ctx) -> TExprNode::TPtr { Y_UNUSED(ctx); - const auto replace = NewNodes.find(input.Get()); - if (NewNodes.cend() != replace) { + const auto replace = NewNodes_.find(input.Get()); + if (NewNodes_.cend() != replace) { return replace->second; } @@ -190,7 +190,7 @@ public: }, ctx, settings); if (!RewriteSanityCheck) { - NewNodes.clear(); + NewNodes_.clear(); } if (replaceStatus.Level == TStatus::Error) { @@ -200,7 +200,7 @@ public: if (RewriteSanityCheck) { VisitExpr(output, [&](const TExprNode::TPtr& localInput) { - if (NewNodes.cend() != NewNodes.find(localInput.Get())) { + if (NewNodes_.cend() != NewNodes_.find(localInput.Get())) { Cerr << "found old node: #" << localInput->UniqueId() << "\n" << output->Dump(); YQL_ENSURE(false); } @@ -211,23 +211,23 @@ public: } void Rewind() override { - State = MakeIntrusive<TState>(); - State->Promise = NThreading::NewPromise(); - State->HasResult = false; - NewNodes.clear(); - FinalizingTransformer.Reset(); + State_ = MakeIntrusive<TState>(); + State_->Promise = NThreading::NewPromise(); + State_->HasResult = false; + NewNodes_.clear(); + FinalizingTransformer_.Reset(); - TrackableNodes.clear(); - CollectingNodes.clear(); - ProvidersCache.clear(); - AsyncNodes.clear(); + TrackableNodes_.clear(); + CollectingNodes_.clear(); + ProvidersCache_.clear(); + AsyncNodes_.clear(); } TStatus ExecuteNode(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExprContext& ctx, ui32 depth) { output = node; bool changed = false; - const auto knownNode = NewNodes.find(node.Get()); - if (NewNodes.cend() != knownNode) { + const auto knownNode = NewNodes_.find(node.Get()); + if (NewNodes_.cend() != knownNode) { output = knownNode->second; changed = true; } @@ -284,7 +284,7 @@ public: output->SetState(TExprNode::EState::ExecutionPending); status = ExecuteChildren(output, output, ctx, depth + 1); if (TExprNode::EState::ExecutionPending == output->GetState()) { - FreshPendingNodes.push_back(output.Get()); + FreshPendingNodes_.push_back(output.Get()); } if (status.Level != TStatus::Repeat) { return status; @@ -292,7 +292,7 @@ public: } if (output != prevOutput) { YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << node->UniqueId() << " to #" << output->UniqueId(); - NewNodes[node.Get()] = output; + NewNodes_[node.Get()] = output; } return TStatus(TStatus::Repeat, output != prevOutput); } else if (status.Level == TStatus::Async) { @@ -366,7 +366,7 @@ public: IDataProvider* GetDataProvider(const TExprNode& node) const { IDataProvider* dataProvider = nullptr; - for (const auto& x : Types.DataSources) { + for (const auto& x : Types_.DataSources) { if (x->CanExecute(node)) { dataProvider = x.Get(); break; @@ -374,7 +374,7 @@ public: } if (!dataProvider) { - for (const auto& x : Types.DataSinks) { + for (const auto& x : Types_.DataSinks) { if (x->CanExecute(node)) { dataProvider = x.Get(); } @@ -393,7 +393,7 @@ public: } auto category = node->Child(1)->Child(0)->Content(); - auto datasink = Types.DataSinkMap.FindPtr(category); + auto datasink = Types_.DataSinkMap.FindPtr(category); YQL_ENSURE(datasink); output = node; auto status = (*datasink)->GetCallableExecutionTransformer().Transform(node, output, ctx); @@ -465,7 +465,7 @@ public: Y_UNUSED(ctx); YQL_CLOG(INFO, CoreExecution) << "Register async execution for node #" << node->UniqueId(); auto future = dataProvider->GetCallableExecutionTransformer().GetAsyncFuture(*node); - AsyncNodes[node.Get()] = node; + AsyncNodes_[node.Get()] = node; SubscribeAsyncFuture(node, dataProvider, future); } @@ -514,8 +514,8 @@ public: void SubscribeAsyncFuture(const TExprNode::TPtr& node, IDataProvider* dataProvider, const NThreading::TFuture<void>& future) { - auto state = State; - if (DeterministicMode) { + auto state = State_; + if (DeterministicMode_) { TAutoPtr<TState::TItem> item = new TState::TItem; item->Node = node.Get(); item->DataProvider = dataProvider; item->Future = future; @@ -523,7 +523,7 @@ public: state->Inflight.PushBack(item.Release()); } - if (DeterministicMode) { + if (DeterministicMode_) { future.Subscribe([state](const NThreading::TFuture<void>& future) { HandleFutureException(future); ProcessFutureResultQueue(state); @@ -540,25 +540,25 @@ public: } void StartNode(TStringBuf category, const TExprNode& node) { - auto publicId = Types.TranslateOperationId(node.UniqueId()); + auto publicId = Types_.TranslateOperationId(node.UniqueId()); if (publicId) { - auto x = Progresses.insert({ *publicId, + auto x = Progresses_.insert({ *publicId, TOperationProgress(TString(category), *publicId, TOperationProgress::EState::Started) }); if (x.second) { - Writer(x.first->second); + Writer_(x.first->second); } } } void FinishNode(TStringBuf category, const TExprNode& node, const TExprNode& newNode) { - auto publicId = Types.TranslateOperationId(node.UniqueId()); + auto publicId = Types_.TranslateOperationId(node.UniqueId()); if (publicId) { if (newNode.UniqueId() != node.UniqueId()) { - Types.NodeToOperationId[newNode.UniqueId()] = *publicId; + Types_.NodeToOperationId[newNode.UniqueId()] = *publicId; } - auto progIt = Progresses.find(*publicId); - YQL_ENSURE(progIt != Progresses.end()); + auto progIt = Progresses_.find(*publicId); + YQL_ENSURE(progIt != Progresses_.end()); auto newState = (node.GetState() == TExprNode::EState::ExecutionComplete) ? TOperationProgress::EState::Finished @@ -567,7 +567,7 @@ public: if (progIt->second.State != newState) { TString stage = progIt->second.Stage.first; progIt->second = TOperationProgress(TString(category), *publicId, newState, stage); - Writer(progIt->second); + Writer_(progIt->second); } } } @@ -586,7 +586,7 @@ public: TVector<TString> ids; for (const auto& c : createdNodes) { - auto& info = TrackableNodes[c.Id]; + auto& info = TrackableNodes_[c.Id]; info.Provider = dataProvider; info.Node = c.Node; ids.push_back(c.Id); @@ -596,7 +596,7 @@ public: } TStatus CollectUnusedNodes(const TExprNode& root, TExprContext& ctx) { - if (TrackableNodes.empty()) { + if (TrackableNodes_.empty()) { return TStatus::Ok; } @@ -613,14 +613,14 @@ public: visited.insert(nodeId); TIntrusivePtr<IDataProvider> dataProvider; - auto providerIt = ProvidersCache.find(nodeId); - if (providerIt != ProvidersCache.end()) { + auto providerIt = ProvidersCache_.find(nodeId); + if (providerIt != ProvidersCache_.end()) { YQL_ENSURE(providerIt->second); dataProvider = providerIt->second; } else { dataProvider = GetDataProvider(node); if (dataProvider) { - ProvidersCache[nodeId] = dataProvider; + ProvidersCache_[nodeId] = dataProvider; } } @@ -633,16 +633,16 @@ public: return true; }); - for (auto i = ProvidersCache.begin(); i != ProvidersCache.end();) { + for (auto i = ProvidersCache_.begin(); i != ProvidersCache_.end();) { if (visited.count(i->first) == 0) { - ProvidersCache.erase(i++); + ProvidersCache_.erase(i++); } else { ++i; } } THashMap<TIntrusivePtr<IDataProvider>, TExprNode::TListType> toCollect; - for (auto i = TrackableNodes.begin(); i != TrackableNodes.end();) { + for (auto i = TrackableNodes_.begin(); i != TrackableNodes_.end();) { TString id = i->first; TTrackableNodeInfo info = i->second; if (!usedIds.contains(id)) { @@ -650,7 +650,7 @@ public: YQL_ENSURE(info.Provider); YQL_CLOG(INFO, CoreExecution) << "Marking node " << id << " for collection"; toCollect[info.Provider].push_back(info.Node); - TrackableNodes.erase(i++); + TrackableNodes_.erase(i++); } else { ++i; } @@ -672,7 +672,7 @@ public: } if (status == TStatus::Async) { - CollectingNodes[collectNode.Get()] = collectNode; + CollectingNodes_[collectNode.Get()] = collectNode; auto future = provider->GetTrackableNodeProcessor().GetCleanupTransformer().GetAsyncFuture(*collectNode); SubscribeAsyncFuture(collectNode, provider.Get(), future); } @@ -682,13 +682,13 @@ public: } private: - TTypeAnnotationContext& Types; - TOperationProgressWriter Writer; - const bool WithFinalize; - TStatePtr State; - TNodeOnNodeOwnedMap NewNodes; - TAutoPtr<IGraphTransformer> FinalizingTransformer; - THashMap<ui32, TOperationProgress> Progresses; + TTypeAnnotationContext& Types_; + TOperationProgressWriter Writer_; + const bool WithFinalize_; + TStatePtr State_; + TNodeOnNodeOwnedMap NewNodes_; + TAutoPtr<IGraphTransformer> FinalizingTransformer_; + THashMap<ui32, TOperationProgress> Progresses_; struct TTrackableNodeInfo { @@ -696,13 +696,13 @@ private: TExprNode::TPtr Node; }; - THashMap<TString, TTrackableNodeInfo> TrackableNodes; - TNodeOnNodeOwnedMap CollectingNodes; - THashMap<ui64, TIntrusivePtr<IDataProvider>> ProvidersCache; - TExprNode::TListType FreshPendingNodes; + THashMap<TString, TTrackableNodeInfo> TrackableNodes_; + TNodeOnNodeOwnedMap CollectingNodes_; + THashMap<ui64, TIntrusivePtr<IDataProvider>> ProvidersCache_; + TExprNode::TListType FreshPendingNodes_; - bool DeterministicMode; - TNodeOnNodeOwnedMap AsyncNodes; + bool DeterministicMode_; + TNodeOnNodeOwnedMap AsyncNodes_; }; IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited); diff --git a/yql/essentials/core/yql_expr_constraint.cpp b/yql/essentials/core/yql_expr_constraint.cpp index dede320a721..cef1fdbdf85 100644 --- a/yql/essentials/core/yql_expr_constraint.cpp +++ b/yql/essentials/core/yql_expr_constraint.cpp @@ -77,187 +77,187 @@ class TCallableConstraintTransformer : public TCallableTransformerBase<TCallable public: TCallableConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly, bool subGraph) : TCallableTransformerBase<TCallableConstraintTransformer>(types, instantOnly) - , SubGraph(subGraph) + , SubGraph_(subGraph) { - Functions["FailMe"] = &TCallableConstraintTransformer::FailMe; - Functions["Unordered"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["UnorderedSubquery"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Sort"] = &TCallableConstraintTransformer::SortWrap; - Functions["AssumeSorted"] = &TCallableConstraintTransformer::SortWrap; - Functions["AssumeUnique"] = &TCallableConstraintTransformer::AssumeUniqueWrap<false, true>; - Functions["AssumeDistinct"] = &TCallableConstraintTransformer::AssumeUniqueWrap<true, true>; - Functions["AssumeUniqueHint"] = &TCallableConstraintTransformer::AssumeUniqueWrap<false, false>; - Functions["AssumeDistinctHint"] = &TCallableConstraintTransformer::AssumeUniqueWrap<true, false>; - Functions["AssumeConstraints"] = &TCallableConstraintTransformer::AssumeConstraintsWrap; - Functions["AssumeChopped"] = &TCallableConstraintTransformer::AssumeChoppedWrap; - Functions["AssumeColumnOrder"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["AssumeAllMembersNullableAtOnce"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["Top"] = &TCallableConstraintTransformer::TopWrap<false>; - Functions["TopSort"] = &TCallableConstraintTransformer::TopWrap<true>; - Functions["TakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["SkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["TakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["SkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["WideTakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["WideSkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["WideTakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["WideSkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["Iterator"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ForwardList"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["LazyList"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["TableSource"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["WideTableSource"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ToFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["FromFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ToStream"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ToSequence"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["Collect"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["PruneAdjacentKeys"] = &TCallableConstraintTransformer::PruneKeysWrap<true>; - Functions["PruneKeys"] = &TCallableConstraintTransformer::PruneKeysWrap<false>; - Functions["FilterNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; - Functions["SkipNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; - Functions["FilterNullElements"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; - Functions["SkipNullElements"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; - Functions["Right!"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["Cons!"] = &TCallableConstraintTransformer::CopyAllFrom<1>; - Functions["ExtractMembers"] = &TCallableConstraintTransformer::ExtractMembersWrap; - Functions["RemoveSystemMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap; - Functions["RemovePrefixMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap; - Functions["FlattenMembers"] = &TCallableConstraintTransformer::FlattenMembersWrap; - Functions["SelectMembers"] = &TCallableConstraintTransformer::SelectMembersWrap; - Functions["FilterMembers"] = &TCallableConstraintTransformer::SelectMembersWrap; - Functions["CastStruct"] = &TCallableConstraintTransformer::SelectMembersWrap; - Functions["SafeCast"] = &TCallableConstraintTransformer::CastWrap<false>; - Functions["StrictCast"] = &TCallableConstraintTransformer::CastWrap<true>; - Functions["ToString"] = &TCallableConstraintTransformer::CastWrap<true>; - Functions["ToBytes"] = &TCallableConstraintTransformer::CastWrap<true>; - Functions["DivePrefixMembers"] = &TCallableConstraintTransformer::DivePrefixMembersWrap; - Functions["OrderedFilter"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["Filter"] = &TCallableConstraintTransformer::FilterWrap<false>; - Functions["WideFilter"] = &TCallableConstraintTransformer::FilterWrap<true>; - Functions["OrderedMap"] = &TCallableConstraintTransformer::MapWrap<true, false>; - Functions["Map"] = &TCallableConstraintTransformer::MapWrap<false, false>; - Functions["MapNext"] = &TCallableConstraintTransformer::MapWrap<true, false>; - Functions["OrderedFlatMap"] = &TCallableConstraintTransformer::MapWrap<true, true>; - Functions["FlatMap"] = &TCallableConstraintTransformer::MapWrap<false, true>; - Functions["OrderedMultiMap"] = &TCallableConstraintTransformer::MapWrap<true, false>; - Functions["MultiMap"] = &TCallableConstraintTransformer::MapWrap<false, false>; - Functions["ExpandMap"] = &TCallableConstraintTransformer::MapWrap<true, false, false, true>; - Functions["WideMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, true>; - Functions["NarrowMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, false>; - Functions["NarrowFlatMap"] = &TCallableConstraintTransformer::MapWrap<true, true, true, false>; - Functions["NarrowMultiMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, false>; - Functions["OrderedFlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<true, true>; - Functions["FlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<false, true>; - Functions["OrderedLMap"] = &TCallableConstraintTransformer::LMapWrap<true>; - Functions["LMap"] = &TCallableConstraintTransformer::LMapWrap<false>; - Functions["Extract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; - Functions["OrderedExtract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; - Functions["OrderedExtend"] = &TCallableConstraintTransformer::ExtendWrap<true>; - Functions["Extend"] = &TCallableConstraintTransformer::ExtendWrap<false>; - Functions["UnionAll"] = &TCallableConstraintTransformer::ExtendWrap<false>; - Functions["Merge"] = &TCallableConstraintTransformer::MergeWrap<false>; - Functions["UnionMerge"] = &TCallableConstraintTransformer::MergeWrap<true>; - Functions["Skip"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["Take"] = &TCallableConstraintTransformer::TakeWrap; - Functions["Limit"] = &TCallableConstraintTransformer::TakeWrap; - Functions["Member"] = &TCallableConstraintTransformer::MemberWrap; - Functions["AsStruct"] = &TCallableConstraintTransformer::AsStructWrap; - Functions["BlockAsStruct"] = &TCallableConstraintTransformer::AsStructWrap; - Functions["Just"] = &TCallableConstraintTransformer::FromFirst<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Unwrap"] = &TCallableConstraintTransformer::FromFirst<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Ensure"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ToList"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["ToOptional"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Head"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Last"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["Reverse"] = &TCallableConstraintTransformer::ReverseWrap; - Functions["Replicate"] = &TCallableConstraintTransformer::FromFirst<TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["AddMember"] = &TCallableConstraintTransformer::AddMemberWrap; - Functions["RemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap; - Functions["ForceRemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap; - Functions["ReplaceMember"] = &TCallableConstraintTransformer::ReplaceMemberWrap; - Functions["AsList"] = &TCallableConstraintTransformer::AsListWrap; - Functions["OptionalIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<false, false>; - Functions["FlatOptionalIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<false, true>; - Functions["ListIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<true, false>; - Functions["FlatListIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<true, true>; - Functions["EmptyIterator"] = &TCallableConstraintTransformer::FromEmpty; - Functions["EmptyFrom"] = &TCallableConstraintTransformer::EmptyFromWrap; - Functions["List"] = &TCallableConstraintTransformer::ListWrap; - Functions["Dict"] = &TCallableConstraintTransformer::DictWrap; - Functions["EmptyList"] = &TCallableConstraintTransformer::FromEmpty; - Functions["EmptyDict"] = &TCallableConstraintTransformer::FromEmpty; - Functions["DictFromKeys"] = &TCallableConstraintTransformer::DictFromKeysWrap; - Functions["If"] = &TCallableConstraintTransformer::IfWrap; - Functions["Nothing"] = &TCallableConstraintTransformer::FromEmpty; - Functions["IfPresent"] = &TCallableConstraintTransformer::IfPresentWrap; - Functions["Coalesce"] = &TCallableConstraintTransformer::CommonFromChildren<0, TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; - Functions["CombineByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineByKey::idx_FinishHandlerLambda>; - Functions["FinalizeByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoFinalizeByKey::idx_FinishHandlerLambda>; - Functions["CombineCore"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineCore::idx_FinishHandler>; - Functions["PartitionByKey"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<true>; - Functions["PartitionsByKeys"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<true>; - Functions["ShuffleByKeys"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<false>; - Functions["Switch"] = &TCallableConstraintTransformer::SwitchWrap; - Functions["Visit"] = &TCallableConstraintTransformer::VisitWrap; - Functions["VariantItem"] = &TCallableConstraintTransformer::VariantItemWrap; - Functions["Variant"] = &TCallableConstraintTransformer::VariantWrap; - Functions["DynamicVariant"] = &TCallableConstraintTransformer::DynamicVariantWrap; - Functions["Guess"] = &TCallableConstraintTransformer::GuessWrap; - Functions["Mux"] = &TCallableConstraintTransformer::MuxWrap; - Functions["Nth"] = &TCallableConstraintTransformer::NthWrap; - Functions["EquiJoin"] = &TCallableConstraintTransformer::EquiJoinWrap; - Functions["JoinDict"] = &TCallableConstraintTransformer::JoinDictWrap; - Functions["MapJoinCore"] = &TCallableConstraintTransformer::MapJoinCoreWrap; - Functions["GraceJoinCore"] = &TCallableConstraintTransformer::GraceJoinCoreWrap; - Functions["GraceSelfJoinCore"] = &TCallableConstraintTransformer::GraceSelfJoinCoreWrap; - Functions["CommonJoinCore"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; - Functions["ToDict"] = &TCallableConstraintTransformer::ToDictWrap; - Functions["DictItems"] = &TCallableConstraintTransformer::DictItemsWrap; - Functions["DictKeys"] = &TCallableConstraintTransformer::DictHalfWrap<true>; - Functions["DictPayloads"] = &TCallableConstraintTransformer::DictHalfWrap<false>; - Functions["Chain1Map"] = &TCallableConstraintTransformer::Chain1MapWrap<false>; - Functions["WideChain1Map"] = &TCallableConstraintTransformer::Chain1MapWrap<true>; - Functions["IsKeySwitch"] = &TCallableConstraintTransformer::IsKeySwitchWrap; - Functions["Condense"] = &TCallableConstraintTransformer::CondenseWrap; - Functions["Condense1"] = &TCallableConstraintTransformer::Condense1Wrap<false>; - Functions["GroupingCore"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; - Functions["Chopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; - Functions["WideChopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; - Functions["WideCombiner"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; - Functions["WideCondense1"] = &TCallableConstraintTransformer::Condense1Wrap<true>; - Functions["Aggregate"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["AggregateMergeState"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["AggregateMergeFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["AggregateMergeManyFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["AggregateFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["AggregateCombine"] = &TCallableConstraintTransformer::AggregateWrap<false>; - Functions["AggregateCombineState"] = &TCallableConstraintTransformer::AggregateWrap<false>; - Functions["Fold"] = &TCallableConstraintTransformer::FoldWrap; - Functions["Fold1"] = &TCallableConstraintTransformer::FoldWrap; - Functions["WithContext"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["WithWorld"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["WideTop"] = &TCallableConstraintTransformer::WideTopWrap<false>; - Functions["WideTopSort"] = &TCallableConstraintTransformer::WideTopWrap<true>; - Functions["WideSort"] = &TCallableConstraintTransformer::WideTopWrap<true>; - Functions["WideTopBlocks"] = &TCallableConstraintTransformer::WideTopWrap<false>; - Functions["WideTopSortBlocks"] = &TCallableConstraintTransformer::WideTopWrap<true>; - Functions["WideSortBlocks"] = &TCallableConstraintTransformer::WideTopWrap<true>; - Functions["WideToBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ListToBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["WideFromBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ListFromBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["ReplicateScalars"] = &TCallableConstraintTransformer::CopyAllFrom<0>; - Functions["BlockMergeFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["BlockMergeManyFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>; - Functions["MultiHoppingCore"] = &TCallableConstraintTransformer::MultiHoppingCoreWrap; - Functions["StablePickle"] = &TCallableConstraintTransformer::PickleWrap; - Functions["Unpickle"] = &TCallableConstraintTransformer::FromSecond<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode>; + Functions_["FailMe"] = &TCallableConstraintTransformer::FailMe; + Functions_["Unordered"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["UnorderedSubquery"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Sort"] = &TCallableConstraintTransformer::SortWrap; + Functions_["AssumeSorted"] = &TCallableConstraintTransformer::SortWrap; + Functions_["AssumeUnique"] = &TCallableConstraintTransformer::AssumeUniqueWrap<false, true>; + Functions_["AssumeDistinct"] = &TCallableConstraintTransformer::AssumeUniqueWrap<true, true>; + Functions_["AssumeUniqueHint"] = &TCallableConstraintTransformer::AssumeUniqueWrap<false, false>; + Functions_["AssumeDistinctHint"] = &TCallableConstraintTransformer::AssumeUniqueWrap<true, false>; + Functions_["AssumeConstraints"] = &TCallableConstraintTransformer::AssumeConstraintsWrap; + Functions_["AssumeChopped"] = &TCallableConstraintTransformer::AssumeChoppedWrap; + Functions_["AssumeColumnOrder"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["AssumeAllMembersNullableAtOnce"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["Top"] = &TCallableConstraintTransformer::TopWrap<false>; + Functions_["TopSort"] = &TCallableConstraintTransformer::TopWrap<true>; + Functions_["TakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["SkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["TakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["SkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["WideTakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["WideSkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["WideTakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["WideSkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["Iterator"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ForwardList"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["LazyList"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["TableSource"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["WideTableSource"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ToFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["FromFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ToStream"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ToSequence"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["Collect"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["PruneAdjacentKeys"] = &TCallableConstraintTransformer::PruneKeysWrap<true>; + Functions_["PruneKeys"] = &TCallableConstraintTransformer::PruneKeysWrap<false>; + Functions_["FilterNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; + Functions_["SkipNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; + Functions_["FilterNullElements"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; + Functions_["SkipNullElements"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode>; + Functions_["Right!"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["Cons!"] = &TCallableConstraintTransformer::CopyAllFrom<1>; + Functions_["ExtractMembers"] = &TCallableConstraintTransformer::ExtractMembersWrap; + Functions_["RemoveSystemMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap; + Functions_["RemovePrefixMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap; + Functions_["FlattenMembers"] = &TCallableConstraintTransformer::FlattenMembersWrap; + Functions_["SelectMembers"] = &TCallableConstraintTransformer::SelectMembersWrap; + Functions_["FilterMembers"] = &TCallableConstraintTransformer::SelectMembersWrap; + Functions_["CastStruct"] = &TCallableConstraintTransformer::SelectMembersWrap; + Functions_["SafeCast"] = &TCallableConstraintTransformer::CastWrap<false>; + Functions_["StrictCast"] = &TCallableConstraintTransformer::CastWrap<true>; + Functions_["ToString"] = &TCallableConstraintTransformer::CastWrap<true>; + Functions_["ToBytes"] = &TCallableConstraintTransformer::CastWrap<true>; + Functions_["DivePrefixMembers"] = &TCallableConstraintTransformer::DivePrefixMembersWrap; + Functions_["OrderedFilter"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["Filter"] = &TCallableConstraintTransformer::FilterWrap<false>; + Functions_["WideFilter"] = &TCallableConstraintTransformer::FilterWrap<true>; + Functions_["OrderedMap"] = &TCallableConstraintTransformer::MapWrap<true, false>; + Functions_["Map"] = &TCallableConstraintTransformer::MapWrap<false, false>; + Functions_["MapNext"] = &TCallableConstraintTransformer::MapWrap<true, false>; + Functions_["OrderedFlatMap"] = &TCallableConstraintTransformer::MapWrap<true, true>; + Functions_["FlatMap"] = &TCallableConstraintTransformer::MapWrap<false, true>; + Functions_["OrderedMultiMap"] = &TCallableConstraintTransformer::MapWrap<true, false>; + Functions_["MultiMap"] = &TCallableConstraintTransformer::MapWrap<false, false>; + Functions_["ExpandMap"] = &TCallableConstraintTransformer::MapWrap<true, false, false, true>; + Functions_["WideMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, true>; + Functions_["NarrowMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, false>; + Functions_["NarrowFlatMap"] = &TCallableConstraintTransformer::MapWrap<true, true, true, false>; + Functions_["NarrowMultiMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, false>; + Functions_["OrderedFlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<true, true>; + Functions_["FlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<false, true>; + Functions_["OrderedLMap"] = &TCallableConstraintTransformer::LMapWrap<true>; + Functions_["LMap"] = &TCallableConstraintTransformer::LMapWrap<false>; + Functions_["Extract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; + Functions_["OrderedExtract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; + Functions_["OrderedExtend"] = &TCallableConstraintTransformer::ExtendWrap<true>; + Functions_["Extend"] = &TCallableConstraintTransformer::ExtendWrap<false>; + Functions_["UnionAll"] = &TCallableConstraintTransformer::ExtendWrap<false>; + Functions_["Merge"] = &TCallableConstraintTransformer::MergeWrap<false>; + Functions_["UnionMerge"] = &TCallableConstraintTransformer::MergeWrap<true>; + Functions_["Skip"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["Take"] = &TCallableConstraintTransformer::TakeWrap; + Functions_["Limit"] = &TCallableConstraintTransformer::TakeWrap; + Functions_["Member"] = &TCallableConstraintTransformer::MemberWrap; + Functions_["AsStruct"] = &TCallableConstraintTransformer::AsStructWrap; + Functions_["BlockAsStruct"] = &TCallableConstraintTransformer::AsStructWrap; + Functions_["Just"] = &TCallableConstraintTransformer::FromFirst<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Unwrap"] = &TCallableConstraintTransformer::FromFirst<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Ensure"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ToList"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfSortedConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["ToOptional"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Head"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Last"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["Reverse"] = &TCallableConstraintTransformer::ReverseWrap; + Functions_["Replicate"] = &TCallableConstraintTransformer::FromFirst<TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["AddMember"] = &TCallableConstraintTransformer::AddMemberWrap; + Functions_["RemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap; + Functions_["ForceRemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap; + Functions_["ReplaceMember"] = &TCallableConstraintTransformer::ReplaceMemberWrap; + Functions_["AsList"] = &TCallableConstraintTransformer::AsListWrap; + Functions_["OptionalIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<false, false>; + Functions_["FlatOptionalIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<false, true>; + Functions_["ListIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<true, false>; + Functions_["FlatListIf"] = &TCallableConstraintTransformer::PassOrEmptyWrap<true, true>; + Functions_["EmptyIterator"] = &TCallableConstraintTransformer::FromEmpty; + Functions_["EmptyFrom"] = &TCallableConstraintTransformer::EmptyFromWrap; + Functions_["List"] = &TCallableConstraintTransformer::ListWrap; + Functions_["Dict"] = &TCallableConstraintTransformer::DictWrap; + Functions_["EmptyList"] = &TCallableConstraintTransformer::FromEmpty; + Functions_["EmptyDict"] = &TCallableConstraintTransformer::FromEmpty; + Functions_["DictFromKeys"] = &TCallableConstraintTransformer::DictFromKeysWrap; + Functions_["If"] = &TCallableConstraintTransformer::IfWrap; + Functions_["Nothing"] = &TCallableConstraintTransformer::FromEmpty; + Functions_["IfPresent"] = &TCallableConstraintTransformer::IfPresentWrap; + Functions_["Coalesce"] = &TCallableConstraintTransformer::CommonFromChildren<0, TSortedConstraintNode, TPartOfSortedConstraintNode, TChoppedConstraintNode, TPartOfChoppedConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>; + Functions_["CombineByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineByKey::idx_FinishHandlerLambda>; + Functions_["FinalizeByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoFinalizeByKey::idx_FinishHandlerLambda>; + Functions_["CombineCore"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineCore::idx_FinishHandler>; + Functions_["PartitionByKey"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<true>; + Functions_["PartitionsByKeys"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<true>; + Functions_["ShuffleByKeys"] = &TCallableConstraintTransformer::ShuffleByKeysWrap<false>; + Functions_["Switch"] = &TCallableConstraintTransformer::SwitchWrap; + Functions_["Visit"] = &TCallableConstraintTransformer::VisitWrap; + Functions_["VariantItem"] = &TCallableConstraintTransformer::VariantItemWrap; + Functions_["Variant"] = &TCallableConstraintTransformer::VariantWrap; + Functions_["DynamicVariant"] = &TCallableConstraintTransformer::DynamicVariantWrap; + Functions_["Guess"] = &TCallableConstraintTransformer::GuessWrap; + Functions_["Mux"] = &TCallableConstraintTransformer::MuxWrap; + Functions_["Nth"] = &TCallableConstraintTransformer::NthWrap; + Functions_["EquiJoin"] = &TCallableConstraintTransformer::EquiJoinWrap; + Functions_["JoinDict"] = &TCallableConstraintTransformer::JoinDictWrap; + Functions_["MapJoinCore"] = &TCallableConstraintTransformer::MapJoinCoreWrap; + Functions_["GraceJoinCore"] = &TCallableConstraintTransformer::GraceJoinCoreWrap; + Functions_["GraceSelfJoinCore"] = &TCallableConstraintTransformer::GraceSelfJoinCoreWrap; + Functions_["CommonJoinCore"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>; + Functions_["ToDict"] = &TCallableConstraintTransformer::ToDictWrap; + Functions_["DictItems"] = &TCallableConstraintTransformer::DictItemsWrap; + Functions_["DictKeys"] = &TCallableConstraintTransformer::DictHalfWrap<true>; + Functions_["DictPayloads"] = &TCallableConstraintTransformer::DictHalfWrap<false>; + Functions_["Chain1Map"] = &TCallableConstraintTransformer::Chain1MapWrap<false>; + Functions_["WideChain1Map"] = &TCallableConstraintTransformer::Chain1MapWrap<true>; + Functions_["IsKeySwitch"] = &TCallableConstraintTransformer::IsKeySwitchWrap; + Functions_["Condense"] = &TCallableConstraintTransformer::CondenseWrap; + Functions_["Condense1"] = &TCallableConstraintTransformer::Condense1Wrap<false>; + Functions_["GroupingCore"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; + Functions_["Chopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; + Functions_["WideChopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; + Functions_["WideCombiner"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; + Functions_["WideCondense1"] = &TCallableConstraintTransformer::Condense1Wrap<true>; + Functions_["Aggregate"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["AggregateMergeState"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["AggregateMergeFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["AggregateMergeManyFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["AggregateFinalize"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["AggregateCombine"] = &TCallableConstraintTransformer::AggregateWrap<false>; + Functions_["AggregateCombineState"] = &TCallableConstraintTransformer::AggregateWrap<false>; + Functions_["Fold"] = &TCallableConstraintTransformer::FoldWrap; + Functions_["Fold1"] = &TCallableConstraintTransformer::FoldWrap; + Functions_["WithContext"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["WithWorld"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["WideTop"] = &TCallableConstraintTransformer::WideTopWrap<false>; + Functions_["WideTopSort"] = &TCallableConstraintTransformer::WideTopWrap<true>; + Functions_["WideSort"] = &TCallableConstraintTransformer::WideTopWrap<true>; + Functions_["WideTopBlocks"] = &TCallableConstraintTransformer::WideTopWrap<false>; + Functions_["WideTopSortBlocks"] = &TCallableConstraintTransformer::WideTopWrap<true>; + Functions_["WideSortBlocks"] = &TCallableConstraintTransformer::WideTopWrap<true>; + Functions_["WideToBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ListToBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["WideFromBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ListFromBlocks"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["ReplicateScalars"] = &TCallableConstraintTransformer::CopyAllFrom<0>; + Functions_["BlockMergeFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["BlockMergeManyFinalizeHashed"] = &TCallableConstraintTransformer::AggregateWrap<true>; + Functions_["MultiHoppingCore"] = &TCallableConstraintTransformer::MultiHoppingCoreWrap; + Functions_["StablePickle"] = &TCallableConstraintTransformer::PickleWrap; + Functions_["Unpickle"] = &TCallableConstraintTransformer::FromSecond<TUniqueConstraintNode, TPartOfUniqueConstraintNode, TDistinctConstraintNode, TPartOfDistinctConstraintNode, TPartOfChoppedConstraintNode, TVarIndexConstraintNode>; } std::optional<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - if (const auto func = Functions.find(input->Content()); Functions.cend() != func) { + if (const auto func = Functions_.find(input->Content()); Functions_.cend() != func) { return (this->*func->second)(input, output, ctx); } return std::nullopt; @@ -290,7 +290,7 @@ public: } IGraphTransformer& GetTransformer(IDataProvider& provider) const { - return provider.GetConstraintTransformer(InstantOnly, SubGraph); + return provider.GetConstraintTransformer(InstantOnly_, SubGraph_); } private: @@ -3171,8 +3171,8 @@ private: return valueNode; } private: - const bool SubGraph; - std::unordered_map<std::string_view, THandler> Functions; + const bool SubGraph_; + std::unordered_map<std::string_view, THandler> Functions_; }; template<> const TPartOfSortedConstraintNode* @@ -3268,8 +3268,8 @@ template<bool DisableCheck> class TConstraintTransformer : public TGraphTransformerBase { public: TConstraintTransformer(TAutoPtr<IGraphTransformer> callableTransformer, TTypeAnnotationContext& types) - : CallableTransformer(callableTransformer) - , Types(types) + : CallableTransformer_(callableTransformer) + , Types_(types) { } @@ -3281,16 +3281,16 @@ public: auto status = TransformNode(input, output, ctx); UpdateStatusIfChanged(status, input, output); - if (status.Level != TStatus::Error && HasRenames) { - output = ctx.ReplaceNodes(std::move(output), Processed); + if (status.Level != TStatus::Error && HasRenames_) { + output = ctx.ReplaceNodes(std::move(output), Processed_); } - Processed.clear(); + Processed_.clear(); if (status == TStatus::Ok) { - Types.ExpectedConstraints.clear(); + Types_.ExpectedConstraints.clear(); } - HasRenames = false; + HasRenames_ = false; return status; } @@ -3298,8 +3298,8 @@ public: YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoGetAsyncFuture"); Y_UNUSED(input); TVector<NThreading::TFuture<void>> futures; - for (const auto& callable : CallableInputs) { - futures.push_back(CallableTransformer->GetAsyncFuture(*callable)); + for (const auto& callable : CallableInputs_) { + futures.push_back(CallableTransformer_->GetAsyncFuture(*callable)); } return WaitExceptionOrAll(futures); @@ -3309,10 +3309,10 @@ public: YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoApplyAsyncChanges"); output = input; TStatus combinedStatus = TStatus::Ok; - for (const auto& callable : CallableInputs) { + for (const auto& callable : CallableInputs_) { callable->SetState(TExprNode::EState::ConstrPending); TExprNode::TPtr callableOutput; - auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx); + auto status = CallableTransformer_->ApplyAsyncChanges(callable, callableOutput, ctx); Y_ABORT_UNLESS(callableOutput); YQL_ENSURE(status != TStatus::Async); YQL_ENSURE(callableOutput == callable); @@ -3322,26 +3322,26 @@ public: } } - CallableInputs.clear(); + CallableInputs_.clear(); if (combinedStatus.Level == TStatus::Ok) { - Processed.clear(); + Processed_.clear(); } return combinedStatus; } void Rewind() final { - CallableTransformer->Rewind(); - CallableInputs.clear(); - Processed.clear(); - HasRenames = false; - CurrentFunctions = {}; + CallableTransformer_->Rewind(); + CallableInputs_.clear(); + Processed_.clear(); + HasRenames_ = false; + CurrentFunctions_ = {}; } private: TStatus TransformNode(const TExprNode::TPtr& start, TExprNode::TPtr& output, TExprContext& ctx) { output = start; - auto processedPair = Processed.emplace(start.Get(), nullptr); // by default node is not changed + auto processedPair = Processed_.emplace(start.Get(), nullptr); // by default node is not changed if (!processedPair.second) { if (processedPair.first->second) { output = processedPair.first->second; @@ -3391,32 +3391,32 @@ private: str << "At "; switch (input->Type()) { case TExprNode::Callable: - if (!CurrentFunctions.empty() && CurrentFunctions.top().second) { + if (!CurrentFunctions_.empty() && CurrentFunctions_.top().second) { return nullptr; } - if (!CurrentFunctions.empty()) { - CurrentFunctions.top().second = true; + if (!CurrentFunctions_.empty()) { + CurrentFunctions_.top().second = true; } str << "function: " << input->Content(); break; case TExprNode::List: - if (CurrentFunctions.empty()) { + if (CurrentFunctions_.empty()) { str << "tuple"; - } else if (!CurrentFunctions.top().second) { - CurrentFunctions.top().second = true; - str << "function: " << CurrentFunctions.top().first; + } else if (!CurrentFunctions_.top().second) { + CurrentFunctions_.top().second = true; + str << "function: " << CurrentFunctions_.top().first; } else { return nullptr; } break; case TExprNode::Lambda: - if (CurrentFunctions.empty()) { + if (CurrentFunctions_.empty()) { str << "lambda"; - } else if (!CurrentFunctions.top().second) { - CurrentFunctions.top().second = true; - str << "function: " << CurrentFunctions.top().first; + } else if (!CurrentFunctions_.top().second) { + CurrentFunctions_.top().second = true; + str << "function: " << CurrentFunctions_.top().first; } else { return nullptr; } @@ -3429,13 +3429,13 @@ private: }); if (input->IsCallable()) { - CurrentFunctions.emplace(input->Content(), false); + CurrentFunctions_.emplace(input->Content(), false); } Y_SCOPE_EXIT(this, input) { if (input->IsCallable()) { - CurrentFunctions.pop(); - if (!CurrentFunctions.empty() && CurrentFunctions.top().first.ends_with('!')) { - CurrentFunctions.top().second = true; + CurrentFunctions_.pop(); + if (!CurrentFunctions_.empty() && CurrentFunctions_.top().first.ends_with('!')) { + CurrentFunctions_.top().second = true; } } }; @@ -3475,7 +3475,7 @@ private: { retStatus = TransformChildren(input, output, ctx); if (retStatus == TStatus::Ok) { - retStatus = CallableTransformer->Transform(input, output, ctx); + retStatus = CallableTransformer_->Transform(input, output, ctx); if (retStatus == TStatus::Ok) { input->SetState(TExprNode::EState::ConstrComplete); CheckExpected(*input); @@ -3525,7 +3525,7 @@ private: else if (updatedChildren) { output = ctx.DeepCopyLambda(*input, std::move(newBody)); processedPair.first->second = output; - HasRenames = true; + HasRenames_ = true; } } else { if (input->ChildrenSize() != 2U) @@ -3582,8 +3582,8 @@ private: break; } - CurrentFunctions.top().second = true; - retStatus = CallableTransformer->Transform(input, output, ctx); + CurrentFunctions_.top().second = true; + retStatus = CallableTransformer_->Transform(input, output, ctx); if (retStatus == TStatus::Error) { input->SetState(TExprNode::EState::Error); } else if (retStatus == TStatus::Ok) { @@ -3595,12 +3595,12 @@ private: input->SetState(TExprNode::EState::ConstrComplete); CheckExpected(*input); } else if (retStatus == TStatus::Async) { - CallableInputs.push_back(input); + CallableInputs_.push_back(input); input->SetState(TExprNode::EState::ConstrInProgress); } else { if (output != input.Get()) { processedPair.first->second = output; - HasRenames = true; + HasRenames_ = true; } } break; @@ -3637,7 +3637,7 @@ private: } else if (updatedChildren) { output = ctx.ChangeChildren(*input, std::move(newChildren)); - HasRenames = true; + HasRenames_ = true; } } return combinedStatus; @@ -3653,12 +3653,12 @@ private: if constexpr (DisableCheck) return; - if (const auto it = Types.ExpectedConstraints.find(input.UniqueId()); it != Types.ExpectedConstraints.cend()) { + if (const auto it = Types_.ExpectedConstraints.find(input.UniqueId()); it != Types_.ExpectedConstraints.cend()) { for (const auto expectedConstr: it->second) { - if (!Types.DisableConstraintCheck.contains(expectedConstr->GetName())) { + if (!Types_.DisableConstraintCheck.contains(expectedConstr->GetName())) { if (auto newConstr = input.GetConstraint(expectedConstr->GetName())) { if (expectedConstr->GetName() == TMultiConstraintNode::Name()) { - YQL_ENSURE(static_cast<const TMultiConstraintNode*>(newConstr)->FilteredIncludes(*expectedConstr, Types.DisableConstraintCheck), "Rewrite error, unequal " << *newConstr + YQL_ENSURE(static_cast<const TMultiConstraintNode*>(newConstr)->FilteredIncludes(*expectedConstr, Types_.DisableConstraintCheck), "Rewrite error, unequal " << *newConstr << " constraint in node " << input.Content() << ", previous was " << *expectedConstr); } else { YQL_ENSURE(newConstr->Includes(*expectedConstr), "Rewrite error, unequal " << *newConstr @@ -3676,12 +3676,12 @@ private: } } private: - TAutoPtr<IGraphTransformer> CallableTransformer; - std::deque<TExprNode::TPtr> CallableInputs; - TNodeOnNodeOwnedMap Processed; - bool HasRenames = false; - std::stack<std::pair<std::string_view, bool>> CurrentFunctions; - TTypeAnnotationContext& Types; + TAutoPtr<IGraphTransformer> CallableTransformer_; + std::deque<TExprNode::TPtr> CallableInputs_; + TNodeOnNodeOwnedMap Processed_; + bool HasRenames_ = false; + std::stack<std::pair<std::string_view, bool>> CurrentFunctions_; + TTypeAnnotationContext& Types_; }; } // namespace diff --git a/yql/essentials/core/yql_gc_transformer.cpp b/yql/essentials/core/yql_gc_transformer.cpp index 94365d8de88..3e950943b72 100644 --- a/yql/essentials/core/yql_gc_transformer.cpp +++ b/yql/essentials/core/yql_gc_transformer.cpp @@ -13,10 +13,10 @@ public: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override { output = input; - if (!CurrentThreshold) - CurrentThreshold = ctx.GcConfig.Settings.NodeCountThreshold; + if (!CurrentThreshold_) + CurrentThreshold_ = ctx.GcConfig.Settings.NodeCountThreshold; - if (ctx.NodeAllocationCounter < LastGcCount + CurrentThreshold) { + if (ctx.NodeAllocationCounter < LastGcCount_ + CurrentThreshold_) { return TStatus::Ok; } @@ -46,9 +46,9 @@ public: ++ctx.GcConfig.Statistics.CollectCount; ctx.GcConfig.Statistics.TotalCollectedNodes += oldSize - liveSize; - LastGcCount = ctx.NodeAllocationCounter; + LastGcCount_ = ctx.NodeAllocationCounter; // adjust next treshold - CurrentThreshold = Max(ctx.GcConfig.Settings.NodeCountThreshold, liveSize); + CurrentThreshold_ = Max(ctx.GcConfig.Settings.NodeCountThreshold, liveSize); if (liveSize > ctx.NodesAllocationLimit) { ctx.AddError(YqlIssue(TPosition(), TIssuesIds::CORE_GC_NODES_LIMIT_EXCEEDED, TStringBuilder() @@ -68,13 +68,13 @@ public: } void Rewind() final { - LastGcCount = 0; - CurrentThreshold = 0; + LastGcCount_ = 0; + CurrentThreshold_ = 0; } private: - ui64 LastGcCount = 0; - ui64 CurrentThreshold = 0; + ui64 LastGcCount_ = 0; + ui64 CurrentThreshold_ = 0; }; } diff --git a/yql/essentials/core/yql_opt_proposed_by_data.cpp b/yql/essentials/core/yql_opt_proposed_by_data.cpp index d9653a2ea25..8639d2fa46b 100644 --- a/yql/essentials/core/yql_opt_proposed_by_data.cpp +++ b/yql/essentials/core/yql_opt_proposed_by_data.cpp @@ -15,48 +15,48 @@ class TDataProposalsInspector : public TGraphTransformerBase { public: TDataProposalsInspector(const TTypeAnnotationContext& types, TGetTransformer getTransformer, TFinish finish) - : Types(types) - , GetTransformer(getTransformer) - , Finish(finish) + : Types_(types) + , GetTransformer_(getTransformer) + , Finish_(finish) {} private: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - HasRepeats = false; + HasRepeats_ = false; if (Source == ESource::DataSource || Source == ESource::All) { - for (auto& x : Types.DataSources) { + for (auto& x : Types_.DataSources) { auto s = HandleProvider(x.Get(), input, output, ctx); if (s.Level == TStatus::Error) return s; - HasRepeats = HasRepeats || s == TStatus::Repeat; + HasRepeats_ = HasRepeats_ || s == TStatus::Repeat; - if (!NewRoots.empty()) { + if (!NewRoots_.empty()) { break; } } } if (Source == ESource::DataSink || Source == ESource::All) { - for (auto& x : Types.DataSinks) { + for (auto& x : Types_.DataSinks) { auto s = HandleProvider(x.Get(), input, output, ctx); if (s.Level == TStatus::Error) return s; - HasRepeats = HasRepeats || s == TStatus::Repeat; + HasRepeats_ = HasRepeats_ || s == TStatus::Repeat; - if (!NewRoots.empty()) { + if (!NewRoots_.empty()) { break; } } } - if (!PendingProviders.empty()) { + if (!PendingProviders_.empty()) { return TStatus::Async; } - if (NewRoots.empty()) { - if (HasRepeats) { + if (NewRoots_.empty()) { + if (HasRepeats_) { return TStatus::Repeat; } - Finish(ctx); + Finish_(ctx); return TStatus::Ok; } @@ -66,32 +66,32 @@ private: void Rewind() final { if (Source == ESource::DataSource || Source == ESource::All) { - for (auto& x : Types.DataSources) { - GetTransformer(x.Get()).Rewind(); + for (auto& x : Types_.DataSources) { + GetTransformer_(x.Get()).Rewind(); } } if (Source == ESource::DataSink || Source == ESource::All) { - for (auto& x : Types.DataSinks) { - GetTransformer(x.Get()).Rewind(); + for (auto& x : Types_.DataSinks) { + GetTransformer_(x.Get()).Rewind(); } } - PendingProviders.clear(); - HasRepeats = false; - NewRoots.clear(); + PendingProviders_.clear(); + HasRepeats_ = false; + NewRoots_.clear(); } TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { TExprNode::TPtr newRoot; - TStatus status = GetTransformer(provider).Transform(input, newRoot, ctx); + TStatus status = GetTransformer_(provider).Transform(input, newRoot, ctx); if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) { if (newRoot && newRoot != input) { - NewRoots.push_back(newRoot); + NewRoots_.push_back(newRoot); } } else if (status.Level == TStatus::Async) { - PendingProviders.push_back(provider); + PendingProviders_.push_back(provider); if (newRoot && newRoot != input) { output = newRoot; } @@ -102,36 +102,36 @@ private: NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { TVector<NThreading::TFuture<void>> futures; - for (auto& x : PendingProviders) { - futures.push_back(GetTransformer(x).GetAsyncFuture(input)); + for (auto& x : PendingProviders_) { + futures.push_back(GetTransformer_(x).GetAsyncFuture(input)); } return WaitExceptionOrAll(futures); } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - for (auto& x : PendingProviders) { + for (auto& x : PendingProviders_) { TExprNode::TPtr newRoot; - TStatus status = GetTransformer(x).ApplyAsyncChanges(input, newRoot, ctx); + TStatus status = GetTransformer_(x).ApplyAsyncChanges(input, newRoot, ctx); if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) { if (newRoot && newRoot != input) { - NewRoots.push_back(newRoot); + NewRoots_.push_back(newRoot); } - HasRepeats = HasRepeats || status == TStatus::Repeat; + HasRepeats_ = HasRepeats_ || status == TStatus::Repeat; } else { return status; } } - PendingProviders.clear(); - bool hasRepeats = HasRepeats; - HasRepeats = false; - if (NewRoots.empty()) { + PendingProviders_.clear(); + bool hasRepeats = HasRepeats_; + HasRepeats_ = false; + if (NewRoots_.empty()) { if (hasRepeats) { return TStatus::Repeat; } - Finish(ctx); + Finish_(ctx); return TStatus::Ok; } @@ -141,16 +141,16 @@ private: void ChooseRoot(TExprNode::TPtr&& input, TExprNode::TPtr& output) { // just get first - output = std::move(NewRoots.empty() ? input : NewRoots.front()); - NewRoots.clear(); + output = std::move(NewRoots_.empty() ? input : NewRoots_.front()); + NewRoots_.clear(); } - const TTypeAnnotationContext& Types; - TGetTransformer GetTransformer; - TFinish Finish; - TVector<IDataProvider*> PendingProviders; - bool HasRepeats = false; - TExprNode::TListType NewRoots; + const TTypeAnnotationContext& Types_; + TGetTransformer GetTransformer_; + TFinish Finish_; + TVector<IDataProvider*> PendingProviders_; + bool HasRepeats_ = false; + TExprNode::TListType NewRoots_; }; template <ESource Source, typename TGetTransformer, typename TFinish> @@ -158,17 +158,17 @@ class TSpecificDataProposalsInspector : public TGraphTransformerBase { public: TSpecificDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider, TGetTransformer getTransformer, TFinish finish) - : Types(types) - , Provider(provider) - , GetTransformer(getTransformer) - , Finish(finish) + : Types_(types) + , Provider_(provider) + , GetTransformer_(getTransformer) + , Finish_(finish) {} private: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { bool hasRepeats = false; if (Source == ESource::DataSource || Source == ESource::All) { - if (auto p = Types.DataSourceMap.FindPtr(Provider)) { + if (auto p = Types_.DataSourceMap.FindPtr(Provider_)) { auto s = HandleProvider(p->Get(), input, output, ctx); if (s.Level == TStatus::Error) return s; hasRepeats = hasRepeats || s == TStatus::Repeat; @@ -176,23 +176,23 @@ private: } if (Source == ESource::DataSink || Source == ESource::All) { - if (auto p = Types.DataSinkMap.FindPtr(Provider)) { + if (auto p = Types_.DataSinkMap.FindPtr(Provider_)) { auto s = HandleProvider(p->Get(), input, output, ctx); if (s.Level == TStatus::Error) return s; hasRepeats = hasRepeats || s == TStatus::Repeat; } } - if (!PendingProviders.empty()) { + if (!PendingProviders_.empty()) { return TStatus::Async; } - if (NewRoots.empty()) { + if (NewRoots_.empty()) { if (hasRepeats) { return TStatus::Repeat; } - Finish(ctx); + Finish_(ctx); return TStatus::Ok; } @@ -202,31 +202,31 @@ private: void Rewind() final { if (Source == ESource::DataSource || Source == ESource::All) { - if (auto p = Types.DataSourceMap.FindPtr(Provider)) { - GetTransformer(p->Get()).Rewind(); + if (auto p = Types_.DataSourceMap.FindPtr(Provider_)) { + GetTransformer_(p->Get()).Rewind(); } } if (Source == ESource::DataSink || Source == ESource::All) { - if (auto p = Types.DataSinkMap.FindPtr(Provider)) { - GetTransformer(p->Get()).Rewind(); + if (auto p = Types_.DataSinkMap.FindPtr(Provider_)) { + GetTransformer_(p->Get()).Rewind(); } } - PendingProviders.clear(); - NewRoots.clear(); + PendingProviders_.clear(); + NewRoots_.clear(); } TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { TExprNode::TPtr newRoot; - TStatus status = GetTransformer(provider).Transform(input, newRoot, ctx); + TStatus status = GetTransformer_(provider).Transform(input, newRoot, ctx); if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) { if (newRoot && newRoot != input) { - NewRoots.push_back(newRoot); + NewRoots_.push_back(newRoot); } } else if (status.Level == TStatus::Async) { - PendingProviders.push_back(provider); + PendingProviders_.push_back(provider); if (newRoot && newRoot != input) { output = newRoot; } @@ -237,8 +237,8 @@ private: NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final { TVector<NThreading::TFuture<void>> futures; - for (auto& x : PendingProviders) { - futures.push_back(GetTransformer(x).GetAsyncFuture(input)); + for (auto& x : PendingProviders_) { + futures.push_back(GetTransformer_(x).GetAsyncFuture(input)); } return WaitExceptionOrAll(futures); @@ -246,12 +246,12 @@ private: TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { bool hasRepeats = false; - for (auto& x : PendingProviders) { + for (auto& x : PendingProviders_) { TExprNode::TPtr newRoot; - TStatus status = GetTransformer(x).ApplyAsyncChanges(input, newRoot, ctx); + TStatus status = GetTransformer_(x).ApplyAsyncChanges(input, newRoot, ctx); if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) { if (newRoot && newRoot != input) { - NewRoots.push_back(newRoot); + NewRoots_.push_back(newRoot); } hasRepeats = hasRepeats || status == TStatus::Repeat; } else { @@ -259,8 +259,8 @@ private: } } - PendingProviders.clear(); - if (NewRoots.empty()) { + PendingProviders_.clear(); + if (NewRoots_.empty()) { return hasRepeats ? TStatus::Repeat : TStatus::Ok; @@ -272,16 +272,16 @@ private: void ChooseRoot(TExprNode::TPtr&& input, TExprNode::TPtr& output) { // just get first - output = std::move(NewRoots.empty() ? input : NewRoots.front()); - NewRoots.clear(); + output = std::move(NewRoots_.empty() ? input : NewRoots_.front()); + NewRoots_.clear(); } - const TTypeAnnotationContext& Types; - TString Provider; - TGetTransformer GetTransformer; - TFinish Finish; - TVector<IDataProvider*> PendingProviders; - TExprNode::TListType NewRoots; + const TTypeAnnotationContext& Types_; + TString Provider_; + TGetTransformer GetTransformer_; + TFinish Finish_; + TVector<IDataProvider*> PendingProviders_; + TExprNode::TListType NewRoots_; }; auto DefaultFinish = [](TExprContext&){}; diff --git a/yql/essentials/core/yql_opt_window.cpp b/yql/essentials/core/yql_opt_window.cpp index 0329ee61040..af0b1e992d3 100644 --- a/yql/essentials/core/yql_opt_window.cpp +++ b/yql/essentials/core/yql_opt_window.cpp @@ -758,17 +758,17 @@ public: using TPtr = TIntrusivePtr<TChain1MapTraits>; TChain1MapTraits(TStringBuf name, TPositionHandle pos) - : Name(name) - , Pos(pos) + : Name_(name) + , Pos_(pos) { } TStringBuf GetName() const { - return Name; + return Name_; } TPositionHandle GetPos() const { - return Pos; + return Pos_; } // Lambda(row) -> AsTuple(output, state) @@ -788,16 +788,16 @@ public: virtual ~TChain1MapTraits() = default; private: - const TStringBuf Name; - const TPositionHandle Pos; + const TStringBuf Name_; + const TPositionHandle Pos_; }; class TChain1MapTraitsLagLead : public TChain1MapTraits { public: TChain1MapTraitsLagLead(TStringBuf name, const TRawTrait& raw, TMaybe<ui64> queueOffset) : TChain1MapTraits(name, raw.Pos) - , QueueOffset(queueOffset) - , LeadLagLambda(raw.CalculateLambda) + , QueueOffset_(queueOffset) + , LeadLagLambda_(raw.CalculateLambda) { } @@ -835,8 +835,8 @@ public: private: TExprNode::TPtr CalculateOutputLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const { - if (!QueueOffset.Defined()) { - return AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx); + if (!QueueOffset_.Defined()) { + return AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda_, ctx); } YQL_ENSURE(dataQueue); @@ -845,8 +845,8 @@ private: auto body = ctx.Builder(GetPos()) .Callable("IfPresent") - .Add(0, BuildQueuePeek(GetPos(), dataQueue, *QueueOffset, rowArg, ctx)) - .Add(1, AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda, ctx)) + .Add(0, BuildQueuePeek(GetPos(), dataQueue, *QueueOffset_, rowArg, ctx)) + .Add(1, AddOptionalIfNotAlreadyOptionalOrNull(LeadLagLambda_, ctx)) .Callable(2, "Null") .Seal() .Seal() @@ -855,8 +855,8 @@ private: return ctx.NewLambda(GetPos(), ctx.NewArguments(GetPos(), {rowArg}), std::move(body)); } - const TMaybe<ui64> QueueOffset; - const TExprNode::TPtr LeadLagLambda; + const TMaybe<ui64> QueueOffset_; + const TExprNode::TPtr LeadLagLambda_; }; class TChain1MapTraitsRowNumber : public TChain1MapTraits { @@ -904,7 +904,7 @@ class TChain1MapTraitsCumeDist : public TChain1MapTraits { public: TChain1MapTraitsCumeDist(TStringBuf name, const TRawTrait& raw, const TString& partitionRowsColumn) : TChain1MapTraits(name, raw.Pos) - , PartitionRowsColumn(partitionRowsColumn) + , PartitionRowsColumn_(partitionRowsColumn) { } @@ -919,7 +919,7 @@ public: .Add(0, BuildDouble(GetPos(), 1.0, ctx)) .Callable(1, "Member") .Arg(0, "row") - .Atom(1, PartitionRowsColumn) + .Atom(1, PartitionRowsColumn_) .Seal() .Seal() .Add(1, BuildUint64(GetPos(), 1, ctx)) @@ -945,7 +945,7 @@ public: .Seal() .Callable(1, "Member") .Arg(0, "row") - .Atom(1, PartitionRowsColumn) + .Atom(1, PartitionRowsColumn_) .Seal() .Seal() .Callable(1, "Inc") @@ -957,17 +957,17 @@ public: } private: - const TString PartitionRowsColumn; + const TString PartitionRowsColumn_; }; class TChain1MapTraitsNTile : public TChain1MapTraits { public: TChain1MapTraitsNTile(TStringBuf name, const TRawTrait& raw, const TString& partitionRowsColumn) : TChain1MapTraits(name, raw.Pos) - , PartitionRowsColumn(partitionRowsColumn) + , PartitionRowsColumn_(partitionRowsColumn) { YQL_ENSURE(raw.Params.size() == 1); - Param = raw.Params[0]; + Param_ = raw.Params[0]; } // Lambda(row) -> AsTuple(output, state) @@ -997,14 +997,14 @@ public: .Callable(0, "/") .Callable(0, "*") .Callable(0, "SafeCast") - .Add(0, Param) + .Add(0, Param_) .Atom(1, "Uint64") .Seal() .Arg(1, "state") .Seal() .Callable(1, "Member") .Arg(0, "row") - .Atom(1, PartitionRowsColumn) + .Atom(1, PartitionRowsColumn_) .Seal() .Seal() .Seal() @@ -1018,17 +1018,17 @@ public: } private: - const TString PartitionRowsColumn; - TExprNode::TPtr Param; + const TString PartitionRowsColumn_; + TExprNode::TPtr Param_; }; class TChain1MapTraitsRankBase : public TChain1MapTraits { public: TChain1MapTraitsRankBase(TStringBuf name, const TRawTrait& raw) : TChain1MapTraits(name, raw.Pos) - , ExtractForCompareLambda(raw.CalculateLambda->ChildPtr(1)) - , Ansi(HasSetting(*raw.CalculateLambda->Child(2), "ansi")) - , KeyType(raw.OutputType) + , ExtractForCompareLambda_(raw.CalculateLambda->ChildPtr(1)) + , Ansi_(HasSetting(*raw.CalculateLambda->Child(2), "ansi")) + , KeyType_(raw.OutputType) { } @@ -1050,8 +1050,8 @@ public: auto initKeyLambda = BuildRawInitLambda(ctx); - if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { - auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); + if (!Ansi_ && KeyType_->GetKind() == ETypeAnnotationKind::Optional) { + auto stateType = GetStateType(KeyType_->Cast<TOptionalExprType>()->GetItemType(), ctx); initKeyLambda = BuildOptKeyInitLambda(initKeyLambda, stateType, ctx); } @@ -1060,7 +1060,7 @@ public: .Param("row") .Apply(initKeyLambda) .With(0) - .Apply(ExtractForCompareLambda) + .Apply(ExtractForCompareLambda_) .With(0, "row") .Seal() .Done() @@ -1075,11 +1075,11 @@ public: TExprNode::TPtr BuildUpdateLambda(const TExprNode::TPtr& dataQueue, TExprContext& ctx) const final { Y_UNUSED(dataQueue); - bool useAggrEquals = Ansi; + bool useAggrEquals = Ansi_; auto updateKeyLambda = BuildRawUpdateLambda(useAggrEquals, ctx); - if (!Ansi && KeyType->GetKind() == ETypeAnnotationKind::Optional) { - auto stateType = GetStateType(KeyType->Cast<TOptionalExprType>()->GetItemType(), ctx); + if (!Ansi_ && KeyType_->GetKind() == ETypeAnnotationKind::Optional) { + auto stateType = GetStateType(KeyType_->Cast<TOptionalExprType>()->GetItemType(), ctx); updateKeyLambda = ctx.Builder(GetPos()) .Lambda() .Param("key") @@ -1118,7 +1118,7 @@ public: .Param("state") .Apply(updateKeyLambda) .With(0) - .Apply(ExtractForCompareLambda) + .Apply(ExtractForCompareLambda_) .With(0, "row") .Seal() .Done() @@ -1160,9 +1160,9 @@ private: .Build(); } - const TExprNode::TPtr ExtractForCompareLambda; - const bool Ansi; - const TTypeAnnotationNode* const KeyType; + const TExprNode::TPtr ExtractForCompareLambda_; + const bool Ansi_; + const TTypeAnnotationNode* const KeyType_; }; class TChain1MapTraitsRank : public TChain1MapTraitsRankBase { @@ -1236,7 +1236,7 @@ class TChain1MapTraitsPercentRank : public TChain1MapTraitsRank { public: TChain1MapTraitsPercentRank(TStringBuf name, const TRawTrait& raw, const TString& partitionRowsColumn) : TChain1MapTraitsRank(name, raw) - , PartitionRowsColumn(partitionRowsColumn) + , PartitionRowsColumn_(partitionRowsColumn) { } @@ -1258,7 +1258,7 @@ public: .Callable(1, "Dec") .Callable(0, "Member") .Arg(0, "row") - .Atom(1, PartitionRowsColumn) + .Atom(1, PartitionRowsColumn_) .Seal() .Seal() .Seal() @@ -1267,7 +1267,7 @@ public: } private: - const TString PartitionRowsColumn; + const TString PartitionRowsColumn_; }; class TChain1MapTraitsDenseRank : public TChain1MapTraitsRankBase { @@ -1332,46 +1332,46 @@ class TChain1MapTraitsStateBase : public TChain1MapTraits { public: TChain1MapTraitsStateBase(TStringBuf name, const TRawTrait& raw) : TChain1MapTraits(name, raw.Pos) - , FrameNeverEmpty(raw.FrameSettings.IsNonEmpty()) - , InitLambda(raw.InitLambda) - , UpdateLambda(raw.UpdateLambda) - , CalculateLambda(raw.CalculateLambda) - , DefaultValue(raw.DefaultValue) + , FrameNeverEmpty_(raw.FrameSettings.IsNonEmpty()) + , InitLambda_(raw.InitLambda) + , UpdateLambda_(raw.UpdateLambda) + , CalculateLambda_(raw.CalculateLambda) + , DefaultValue_(raw.DefaultValue) { } protected: TExprNode::TPtr GetInitLambda() const { - return InitLambda; + return InitLambda_; } TExprNode::TPtr GetUpdateLambda() const { - return UpdateLambda; + return UpdateLambda_; } TExprNode::TPtr GetCalculateLambda() const { - return CalculateLambda; + return CalculateLambda_; } TExprNode::TPtr GetDefaultValue() const { - return DefaultValue; + return DefaultValue_; } - const bool FrameNeverEmpty; + const bool FrameNeverEmpty_; private: - const TExprNode::TPtr InitLambda; - const TExprNode::TPtr UpdateLambda; - const TExprNode::TPtr CalculateLambda; - const TExprNode::TPtr DefaultValue; + const TExprNode::TPtr InitLambda_; + const TExprNode::TPtr UpdateLambda_; + const TExprNode::TPtr CalculateLambda_; + const TExprNode::TPtr DefaultValue_; }; class TChain1MapTraitsCurrentOrLagging : public TChain1MapTraitsStateBase { public: TChain1MapTraitsCurrentOrLagging(TStringBuf name, const TRawTrait& raw, TMaybe<ui64> lagQueueIndex) : TChain1MapTraitsStateBase(name, raw) - , LaggingQueueIndex(lagQueueIndex) - , OutputIsOptional(raw.OutputType->IsOptionalOrNull()) + , LaggingQueueIndex_(lagQueueIndex) + , OutputIsOptional_(raw.OutputType->IsOptionalOrNull()) { } @@ -1390,14 +1390,14 @@ public: TExprNode::TPtr ExtractLaggingOutput(const TExprNode::TPtr& lagQueue, const TExprNode::TPtr& dependsOn, TExprContext& ctx) const override { - if (!LaggingQueueIndex.Defined()) { + if (!LaggingQueueIndex_.Defined()) { return {}; } - YQL_ENSURE(!FrameNeverEmpty); + YQL_ENSURE(!FrameNeverEmpty_); auto output = ctx.Builder(GetPos()) .Callable("Map") - .Add(0, BuildQueuePeek(GetPos(), lagQueue, *LaggingQueueIndex, dependsOn, ctx)) + .Add(0, BuildQueuePeek(GetPos(), lagQueue, *LaggingQueueIndex_, dependsOn, ctx)) .Lambda(1) .Param("struct") .Callable("Member") @@ -1407,20 +1407,20 @@ public: .Seal() .Seal() .Build(); - return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); + return CoalesceQueueOutput(GetPos(), output, OutputIsOptional_, GetDefaultValue(), ctx); } private: - const TMaybe<ui64> LaggingQueueIndex; - const bool OutputIsOptional; + const TMaybe<ui64> LaggingQueueIndex_; + const bool OutputIsOptional_; }; class TChain1MapTraitsLeading : public TChain1MapTraitsStateBase { public: TChain1MapTraitsLeading(TStringBuf name, const TRawTrait& raw, ui64 currentRowIndex, ui64 lastRowIndex) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(currentRowIndex + 1) - , QueueEnd(lastRowIndex + 1) + , QueueBegin_(currentRowIndex + 1) + , QueueEnd_(lastRowIndex + 1) { } @@ -1434,7 +1434,7 @@ public: auto rowArg = ctx.NewArgument(GetPos(), "row"); auto state = ctx.Builder(GetPos()) .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin_, QueueEnd_, rowArg, ctx)) .Apply(1, originalInit) .With(0, rowArg) .Seal() @@ -1470,7 +1470,7 @@ public: auto state = ctx.Builder(GetPos()) .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin_, QueueEnd_, rowArg, ctx)) .Apply(1, originalUpdate) .With(0, rowArg) .With(1, stateArg) @@ -1497,15 +1497,15 @@ public: } private: - const ui64 QueueBegin; - const ui64 QueueEnd; + const ui64 QueueBegin_; + const ui64 QueueEnd_; }; class TChain1MapTraitsFull : public TChain1MapTraitsStateBase { public: TChain1MapTraitsFull(TStringBuf name, const TRawTrait& raw, ui64 currentRowIndex) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(currentRowIndex + 1) + , QueueBegin_(currentRowIndex + 1) { } @@ -1519,7 +1519,7 @@ public: auto rowArg = ctx.NewArgument(GetPos(), "row"); auto state = ctx.Builder(GetPos()) .Callable("Fold") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, Max<ui64>(), rowArg, ctx)) + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin_, Max<ui64>(), rowArg, ctx)) .Apply(1, originalInit) .With(0, rowArg) .Seal() @@ -1559,16 +1559,16 @@ public: } private: - const ui64 QueueBegin; + const ui64 QueueBegin_; }; class TChain1MapTraitsGeneric : public TChain1MapTraitsStateBase { public: TChain1MapTraitsGeneric(TStringBuf name, const TRawTrait& raw, ui64 queueBegin, ui64 queueEnd) : TChain1MapTraitsStateBase(name, raw) - , QueueBegin(queueBegin) - , QueueEnd(queueEnd) - , OutputIsOptional(raw.OutputType->IsOptionalOrNull()) + , QueueBegin_(queueBegin) + , QueueEnd_(queueEnd) + , OutputIsOptional_(raw.OutputType->IsOptionalOrNull()) { } @@ -1607,7 +1607,7 @@ private: auto fold1 = ctx.Builder(GetPos()) .Callable("Fold1") - .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin, QueueEnd, rowArg, ctx)) + .Add(0, BuildQueueRange(GetPos(), dataQueue, QueueBegin_, QueueEnd_, rowArg, ctx)) .Add(1, ctx.DeepCopyLambda(*originalInit)) .Add(2, ctx.DeepCopyLambda(*originalUpdate)) .Seal() @@ -1622,7 +1622,7 @@ private: .Seal() .Build(); - if (FrameNeverEmpty) { + if (FrameNeverEmpty_) { // output is always non-empty optional in this case // we do IfPresent with some fake output value to remove optional // this will have exactly the same result as Unwrap(output) @@ -1644,19 +1644,19 @@ private: .Build(); } - return CoalesceQueueOutput(GetPos(), output, OutputIsOptional, GetDefaultValue(), ctx); + return CoalesceQueueOutput(GetPos(), output, OutputIsOptional_, GetDefaultValue(), ctx); } - const ui64 QueueBegin; - const ui64 QueueEnd; - const bool OutputIsOptional; + const ui64 QueueBegin_; + const ui64 QueueEnd_; + const bool OutputIsOptional_; }; class TChain1MapTraitsEmpty : public TChain1MapTraitsStateBase { public: TChain1MapTraitsEmpty(TStringBuf name, const TRawTrait& raw) : TChain1MapTraitsStateBase(name, raw) - , RawOutputType(raw.OutputType) + , RawOutputType_(raw.OutputType) { } @@ -1693,10 +1693,10 @@ public: private: TExprNode::TPtr BuildFinalOutput(TExprContext& ctx) const { const auto defaultValue = GetDefaultValue(); - YQL_ENSURE(!FrameNeverEmpty); + YQL_ENSURE(!FrameNeverEmpty_); if (defaultValue->IsCallable("Null")) { - auto resultingType = RawOutputType; + auto resultingType = RawOutputType_; if (!resultingType->IsOptionalOrNull()) { resultingType = ctx.MakeType<TOptionalExprType>(resultingType); } @@ -1710,7 +1710,7 @@ private: return defaultValue; } - const TTypeAnnotationNode* const RawOutputType; + const TTypeAnnotationNode* const RawOutputType_; }; struct TQueueParams { @@ -3474,12 +3474,12 @@ TMaybe<TWindowFrameSettings> TWindowFrameSettings::TryParse(const TExprNode& nod TWindowFrameSettings settings; if (node.IsCallable("WinOnRows")) { - settings.Type = EFrameType::FrameByRows; + settings.Type_ = EFrameType::FrameByRows; } else if (node.IsCallable("WinOnRange")) { - settings.Type = EFrameType::FrameByRange; + settings.Type_ = EFrameType::FrameByRange; } else { YQL_ENSURE(node.IsCallable("WinOnGroups")); - settings.Type = EFrameType::FrameByGroups; + settings.Type_ = EFrameType::FrameByGroups; } auto frameSpec = node.Child(0); if (frameSpec->Type() == TExprNode::List) { @@ -3503,7 +3503,7 @@ TMaybe<TWindowFrameSettings> TWindowFrameSettings::TryParse(const TExprNode& nod } if (settingName == "compact") { - settings.Compact = true; + settings.Compact_ = true; continue; } @@ -3519,8 +3519,8 @@ TMaybe<TWindowFrameSettings> TWindowFrameSettings::TryParse(const TExprNode& nod } hasBound = true; - TMaybe<i32>& boundOffset = (settingName == "begin") ? settings.FirstOffset : settings.LastOffset; - TExprNode::TPtr& frameBound = (settingName == "begin") ? settings.First : settings.Last; + TMaybe<i32>& boundOffset = (settingName == "begin") ? settings.FirstOffset_ : settings.LastOffset_; + TExprNode::TPtr& frameBound = (settingName == "begin") ? settings.First_ : settings.Last_; if (setting->Tail().IsList()) { TExprNode::TPtr fb = setting->TailPtr(); @@ -3638,8 +3638,8 @@ TMaybe<TWindowFrameSettings> TWindowFrameSettings::TryParse(const TExprNode& nod return {}; } } else if (frameSpec->IsCallable("Void")) { - settings.FirstOffset = {}; - settings.LastOffset = 0; + settings.FirstOffset_ = {}; + settings.LastOffset_ = 0; } else { const TTypeAnnotationNode* type = frameSpec->GetTypeAnn(); ctx.AddError(TIssue(ctx.GetPosition(frameSpec->Pos()), @@ -3648,35 +3648,35 @@ TMaybe<TWindowFrameSettings> TWindowFrameSettings::TryParse(const TExprNode& nod } // frame will always contain rows if it includes current row - if (!settings.FirstOffset) { - settings.NeverEmpty = !settings.LastOffset.Defined() || *settings.LastOffset >= 0; - } else if (!settings.LastOffset.Defined()) { - settings.NeverEmpty = !settings.FirstOffset.Defined() || *settings.FirstOffset <= 0; + if (!settings.FirstOffset_) { + settings.NeverEmpty_ = !settings.LastOffset_.Defined() || *settings.LastOffset_ >= 0; + } else if (!settings.LastOffset_.Defined()) { + settings.NeverEmpty_ = !settings.FirstOffset_.Defined() || *settings.FirstOffset_ <= 0; } else { - settings.NeverEmpty = *settings.FirstOffset <= *settings.LastOffset && *settings.FirstOffset <= 0 && *settings.LastOffset >= 0; + settings.NeverEmpty_ = *settings.FirstOffset_ <= *settings.LastOffset_ && *settings.FirstOffset_ <= 0 && *settings.LastOffset_ >= 0; } return settings; } TMaybe<i32> TWindowFrameSettings::GetFirstOffset() const { - YQL_ENSURE(Type == FrameByRows); - return FirstOffset; + YQL_ENSURE(Type_ == FrameByRows); + return FirstOffset_; } TMaybe<i32> TWindowFrameSettings::GetLastOffset() const { - YQL_ENSURE(Type == FrameByRows); - return LastOffset; + YQL_ENSURE(Type_ == FrameByRows); + return LastOffset_; } TCoFrameBound TWindowFrameSettings::GetFirst() const { - YQL_ENSURE(First); - return TCoFrameBound(First); + YQL_ENSURE(First_); + return TCoFrameBound(First_); } TCoFrameBound TWindowFrameSettings::GetLast() const { - YQL_ENSURE(Last); - return TCoFrameBound(Last); + YQL_ENSURE(Last_); + return TCoFrameBound(Last_); } TExprNode::TPtr ZipWithSessionParamsLambda(TPositionHandle pos, const TExprNode::TPtr& partitionKeySelector, diff --git a/yql/essentials/core/yql_opt_window.h b/yql/essentials/core/yql_opt_window.h index 679b71c8b7d..979f1535474 100644 --- a/yql/essentials/core/yql_opt_window.h +++ b/yql/essentials/core/yql_opt_window.h @@ -33,17 +33,17 @@ public: TCoFrameBound GetFirst() const; TCoFrameBound GetLast() const; - bool IsNonEmpty() const { return NeverEmpty; } - bool IsCompact() const { return Compact; } - EFrameType GetFrameType() const { return Type; } + bool IsNonEmpty() const { return NeverEmpty_; } + bool IsCompact() const { return Compact_; } + EFrameType GetFrameType() const { return Type_; } private: - EFrameType Type = FrameByRows; - TExprNode::TPtr First; - TMaybe<i32> FirstOffset; - TExprNode::TPtr Last; - TMaybe<i32> LastOffset; - bool NeverEmpty = false; - bool Compact = false; + EFrameType Type_ = FrameByRows; + TExprNode::TPtr First_; + TMaybe<i32> FirstOffset_; + TExprNode::TPtr Last_; + TMaybe<i32> LastOffset_; + bool NeverEmpty_ = false; + bool Compact_ = false; }; struct TSessionWindowParams { @@ -56,7 +56,7 @@ struct TSessionWindowParams { , Update(nullptr) , SortTraits(nullptr) {} - + void Reset(); TExprNode::TPtr Traits; diff --git a/yql/essentials/core/yql_type_annotation.cpp b/yql/essentials/core/yql_type_annotation.cpp index eb403cdbede..3c4c2238f47 100644 --- a/yql/essentials/core/yql_type_annotation.cpp +++ b/yql/essentials/core/yql_type_annotation.cpp @@ -347,36 +347,36 @@ TString TModuleResolver::NormalizeModuleName(const TString& path) { } void TModuleResolver::RegisterPackage(const TString& package) { - KnownPackages.insert(package); + KnownPackages_.insert(package); } bool TModuleResolver::SetPackageDefaultVersion(const TString& package, ui32 version) { - if (!KnownPackages.contains(package)) { + if (!KnownPackages_.contains(package)) { return false; } - PackageVersions[package] = version; + PackageVersions_[package] = version; return true; } const TExportTable* TModuleResolver::GetModule(const TString& module) const { // ParentModules and Modules should not have common keys const TString normalizedModuleName = NormalizeModuleName(module); - if (ParentModules) { - if (auto table = ParentModules->FindPtr(normalizedModuleName)) { + if (ParentModules_) { + if (auto table = ParentModules_->FindPtr(normalizedModuleName)) { return table; } } - return Modules.FindPtr(normalizedModuleName); + return Modules_.FindPtr(normalizedModuleName); } void TModuleResolver::WriteStatistics(NYson::TYsonWriter& writer) { writer.OnKeyedItem("UsedSuffixes"); - writer.OnStringScalar(JoinRange(",", UsedSuffixes.begin(), UsedSuffixes.end())); + writer.OnStringScalar(JoinRange(",", UsedSuffixes_.begin(), UsedSuffixes_.end())); } bool TModuleResolver::AddFromUrl(const std::string_view& file, const std::string_view& url, const std::string_view& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) { - if (!UserData) { + if (!UserData_) { ctx.AddError(TIssue(pos, "Loading libraries is prohibited")); return false; } @@ -386,11 +386,11 @@ bool TModuleResolver::AddFromUrl(const std::string_view& file, const std::string block.Data = url; block.Data = SubstParameters(block.Data); if (!tokenName.empty()) { - if (!Credentials) { + if (!Credentials_) { ctx.AddError(TIssue(pos, "Missing credentials")); return false; } - auto cred = Credentials->FindCredential(tokenName); + auto cred = Credentials_->FindCredential(tokenName); if (!cred) { ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown token name: " << tokenName)); return false; @@ -398,13 +398,13 @@ bool TModuleResolver::AddFromUrl(const std::string_view& file, const std::string block.UrlToken = cred->Content; } - UserData->AddUserDataBlock(file, block); + UserData_->AddUserDataBlock(file, block); return AddFromFile(file, ctx, syntaxVersion, packageVersion, pos); } bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) { - if (!UserData) { + if (!UserData_) { ctx.AddError(TIssue(pos, "Loading libraries is prohibited")); return false; } @@ -418,8 +418,8 @@ bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ct return false; } - UsedSuffixes.insert(TString(file.substr(1 + file.rfind('.')))); - const TUserDataBlock* block = UserData->FindUserDataBlock(fullName); + UsedSuffixes_.insert(TString(file.substr(1 + file.rfind('.')))); + const TUserDataBlock* block = UserData_->FindUserDataBlock(fullName); if (!block) { ctx.AddError(TIssue(pos, TStringBuilder() << "File not found: " << file)); @@ -427,9 +427,9 @@ bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ct } auto moduleName = TModuleResolver::NormalizeModuleName(TString(file)); - if (GetModule(moduleName) || Libs.contains(moduleName)) { - auto it = Libs.find(moduleName); - if (it != Libs.end() && it->second.contains(packageVersion)) { + if (GetModule(moduleName) || Libs_.contains(moduleName)) { + auto it = Libs_.find(moduleName); + if (it != Libs_.end() && it->second.contains(packageVersion)) { // TODO (YQL-7170): find better fix // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); return true; // false @@ -437,7 +437,7 @@ bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ct } TString body; - if (!QContext.CanRead()) { + if (!QContext_.CanRead()) { switch (block->Type) { case EUserDataType::RAW_INLINE_DATA: body = block->Data; @@ -446,13 +446,13 @@ bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ct body = TFileInput(block->Data).ReadAll(); break; case EUserDataType::URL: - if (!UrlLoader) { + if (!UrlLoader_) { ctx.AddError(TIssue(pos, TStringBuilder() << "Unable to load file \"" << file << "\" from url, because url loader is not available")); return false; } - body = UrlLoader->Load(block->Data, block->UrlToken); + body = UrlLoader_->Load(block->Data, block->UrlToken); break; default: throw yexception() << "Unknown block type " << block->Type; @@ -477,11 +477,11 @@ bool TModuleResolver::AddFromMemory(const std::string_view& file, const TString& return false; } - UsedSuffixes.insert(TString(file.substr(1 + file.rfind('.')))); + UsedSuffixes_.insert(TString(file.substr(1 + file.rfind('.')))); moduleName = TModuleResolver::NormalizeModuleName(TString(file)); - if (GetModule(moduleName) || Libs.contains(moduleName)) { - auto it = Libs.find(moduleName); - if (it != Libs.end() && it->second.contains(packageVersion)) { + if (GetModule(moduleName) || Libs_.contains(moduleName)) { + auto it = Libs_.find(moduleName); + if (it != Libs_.end() && it->second.contains(packageVersion)) { // TODO (YQL-7170): find better fix // ctx.AddError(TIssue({0,0,TString(fullName)}, TStringBuilder() << "File is already loaded as library")); return true; // false @@ -493,15 +493,15 @@ bool TModuleResolver::AddFromMemory(const std::string_view& file, const TString& bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& moduleName, bool sExpr, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, std::vector<TString>* exports, std::vector<TString>* imports) { auto query = body; - if (QContext.CanRead()) { - auto item = QContext.GetReader()->Get({ModuleResolverComponent, fullName}).GetValueSync(); + if (QContext_.CanRead()) { + auto item = QContext_.GetReader()->Get({ModuleResolverComponent, fullName}).GetValueSync(); if (!item) { throw yexception() << "Missing replay data"; } query = item->Value; - } else if (QContext.CanWrite()) { - QContext.GetWriter()->Put({ModuleResolverComponent, fullName}, query).GetValueSync(); + } else if (QContext_.CanWrite()) { + QContext_.GetWriter()->Put({ModuleResolverComponent, fullName}, query).GetValueSync(); } const auto addSubIssues = [](TIssue&& issue, const TIssues& issues) { @@ -518,12 +518,12 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu NSQLTranslation::TTranslationSettings settings; settings.Mode = NSQLTranslation::ESqlMode::LIBRARY; settings.File = fullName; - settings.ClusterMapping = ClusterMapping; - settings.Flags = SqlFlags; + settings.ClusterMapping = ClusterMapping_; + settings.Flags = SqlFlags_; settings.SyntaxVersion = syntaxVersion; settings.V0Behavior = NSQLTranslation::EV0Behavior::Silent; - settings.FileAliasPrefix = FileAliasPrefix; - astRes = SqlToYql(Translators, query, settings); + settings.FileAliasPrefix = FileAliasPrefix_; + astRes = SqlToYql(Translators_, query, settings); } if (!astRes.IsOk()) { @@ -537,21 +537,21 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu ctx.IssueManager.RaiseIssue(addSubIssues(std::move(issue), astRes.Issues)); } - if (!sExpr && ModuleChecker) { - if (!ModuleChecker(query, fullName, ctx)) { + if (!sExpr && ModuleChecker_) { + if (!ModuleChecker_(query, fullName, ctx)) { return false; } } TLibraryCohesion cohesion; - if (!CompileExpr(*astRes.Root, cohesion, LibsContext)) { - ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to compile: " << fullName), LibsContext.IssueManager.GetIssues())); + if (!CompileExpr(*astRes.Root, cohesion, LibsContext_)) { + ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to compile: " << fullName), LibsContext_.IssueManager.GetIssues())); return false; } - if (OptimizeLibraries) { - if (!OptimizeLibrary(cohesion, LibsContext)) { - ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to optimize: " << fullName), LibsContext.IssueManager.GetIssues())); + if (OptimizeLibraries_) { + if (!OptimizeLibrary(cohesion, LibsContext_)) { + ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to optimize: " << fullName), LibsContext_.IssueManager.GetIssues())); return false; } } @@ -570,7 +570,7 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu } } - Libs[moduleName][packageVersion] = std::move(cohesion); + Libs_[moduleName][packageVersion] = std::move(cohesion); return true; } @@ -580,22 +580,22 @@ bool TModuleResolver::Link(TExprContext& ctx) { }; THashMap<TString, TLibraryCohesion> libs = FilterLibsByVersion(); - if (!LinkLibraries(libs, ctx, LibsContext, f)) { + if (!LinkLibraries(libs, ctx, LibsContext_, f)) { return false; } for (auto& x : libs) { - Modules.emplace(x.first, std::move(x.second.Exports)); + Modules_.emplace(x.first, std::move(x.second.Exports)); } - Libs.clear(); - PackageVersions.clear(); + Libs_.clear(); + PackageVersions_.clear(); return true; } THashMap<TString, TLibraryCohesion> TModuleResolver::FilterLibsByVersion() const { THashMap<TString, TLibraryCohesion> result; - for (auto p : Libs) { + for (auto p : Libs_) { YQL_ENSURE(!p.second.empty()); auto packageName = ExtractPackageNameFromModule(p.first); @@ -605,12 +605,12 @@ THashMap<TString, TLibraryCohesion> TModuleResolver::FilterLibsByVersion() const continue; } - if (!KnownPackages.contains(packageName)) { + if (!KnownPackages_.contains(packageName)) { ythrow yexception() << "Unknown package " << packageName << " is used in module " << p.first; } - auto it = PackageVersions.find(packageName); - const ui32 version = (it != PackageVersions.end()) ? it->second : 0; + auto it = PackageVersions_.find(packageName); + const ui32 version = (it != PackageVersions_.end()) ? it->second : 0; auto cohesionIt = p.second.find(version); if (cohesionIt == p.second.end()) { ythrow yexception() << "Unable to find library version " << version << " for package " << packageName << " and module " << p.first; @@ -639,37 +639,37 @@ TString TModuleResolver::ExtractPackageNameFromModule(TStringBuf moduleName) { } void TModuleResolver::UpdateNextUniqueId(TExprContext& ctx) const { - if (UserData && ctx.NextUniqueId < LibsContext.NextUniqueId) { - ctx.NextUniqueId = LibsContext.NextUniqueId; + if (UserData_ && ctx.NextUniqueId < LibsContext_.NextUniqueId) { + ctx.NextUniqueId = LibsContext_.NextUniqueId; } } ui64 TModuleResolver::GetNextUniqueId() const { - return LibsContext.NextUniqueId; + return LibsContext_.NextUniqueId; } IModuleResolver::TPtr TModuleResolver::CreateMutableChild() const { - if (UserData || UrlLoader) { + if (UserData_ || UrlLoader_) { throw yexception() << "Module resolver should not contain user data and URL loader"; } - return std::make_shared<TModuleResolver>(Translators, &Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs, FileAliasPrefix, ModuleChecker); + return std::make_shared<TModuleResolver>(Translators_, &Modules_, LibsContext_.NextUniqueId, ClusterMapping_, SqlFlags_, OptimizeLibraries_, KnownPackages_, Libs_, FileAliasPrefix_, ModuleChecker_); } void TModuleResolver::SetFileAliasPrefix(TString&& prefix) { - FileAliasPrefix = std::move(prefix); + FileAliasPrefix_ = std::move(prefix); } TString TModuleResolver::GetFileAliasPrefix() const { - return FileAliasPrefix; + return FileAliasPrefix_; } TString TModuleResolver::SubstParameters(const TString& str) { - if (!Parameters) { + if (!Parameters_) { return str; } - return ::NYql::SubstParameters(str, Parameters, nullptr); + return ::NYql::SubstParameters(str, Parameters_, nullptr); } } // namespace NYql diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index 64ad1207cad..79766beee7e 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -53,17 +53,17 @@ public: ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping, const THashSet<TString>& sqlFlags, bool optimizeLibraries = true, THolder<TExprContext> ownedCtx = {}, TModuleChecker moduleChecker = {}) - : Translators(translators) - , OwnedCtx(std::move(ownedCtx)) - , LibsContext(nextUniqueId) - , Modules(std::move(modules)) - , ClusterMapping(clusterMapping) - , SqlFlags(sqlFlags) - , ModuleChecker(moduleChecker) - , OptimizeLibraries(optimizeLibraries) + : Translators_(translators) + , OwnedCtx_(std::move(ownedCtx)) + , LibsContext_(nextUniqueId) + , Modules_(std::move(modules)) + , ClusterMapping_(clusterMapping) + , SqlFlags_(sqlFlags) + , ModuleChecker_(moduleChecker) + , OptimizeLibraries_(optimizeLibraries) { - if (OwnedCtx) { - FreezeGuard = MakeHolder<TExprContext::TFreezeGuard>(*OwnedCtx); + if (OwnedCtx_) { + FreezeGuard_ = MakeHolder<TExprContext::TFreezeGuard>(*OwnedCtx_); } } @@ -71,50 +71,50 @@ public: ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping, const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString, THashMap<int, TLibraryCohesion>>& libs, const TString& fileAliasPrefix, TModuleChecker moduleChecker) - : Translators(translators) - , ParentModules(parentModules) - , LibsContext(nextUniqueId) - , KnownPackages(knownPackages) - , Libs(libs) - , ClusterMapping(clusterMapping) - , SqlFlags(sqlFlags) - , ModuleChecker(moduleChecker) - , OptimizeLibraries(optimizeLibraries) - , FileAliasPrefix(fileAliasPrefix) + : Translators_(translators) + , ParentModules_(parentModules) + , LibsContext_(nextUniqueId) + , KnownPackages_(knownPackages) + , Libs_(libs) + , ClusterMapping_(clusterMapping) + , SqlFlags_(sqlFlags) + , ModuleChecker_(moduleChecker) + , OptimizeLibraries_(optimizeLibraries) + , FileAliasPrefix_(fileAliasPrefix) { } static TString NormalizeModuleName(const TString& path); void AttachUserData(TUserDataStorage::TPtr userData) { - UserData = userData; + UserData_ = userData; } void SetUrlLoader(IUrlLoader::TPtr loader) { - UrlLoader = loader; + UrlLoader_ = loader; } void SetParameters(const NYT::TNode& node) { - Parameters = node; + Parameters_ = node; } void SetCredentials(TCredentials::TPtr credentials) { - Credentials = std::move(credentials); + Credentials_ = std::move(credentials); } void SetQContext(const TQContext& qContext) { - QContext = qContext; + QContext_ = qContext; } void SetClusterMapping(const THashMap<TString, TString>& clusterMapping) { - ClusterMapping = clusterMapping; + ClusterMapping_ = clusterMapping; } void SetSqlFlags(const THashSet<TString>& flags) { - SqlFlags = flags; + SqlFlags_ = flags; } void SetModuleChecker(TModuleChecker moduleChecker) { - ModuleChecker = moduleChecker; + ModuleChecker_ = moduleChecker; } void RegisterPackage(const TString& package) override; @@ -139,26 +139,26 @@ private: TString SubstParameters(const TString& str); private: - const NSQLTranslation::TTranslators Translators; - THolder<TExprContext> OwnedCtx; - const TModulesTable* ParentModules = nullptr; - TUserDataStorage::TPtr UserData; - IUrlLoader::TPtr UrlLoader; - TMaybe<NYT::TNode> Parameters; - TCredentials::TPtr Credentials; - TQContext QContext; - TExprContext LibsContext; - TSet<TString> KnownPackages; - THashMap<TString, ui32> PackageVersions; - THashMap<TString, THashMap<int, TLibraryCohesion>> Libs; - TModulesTable Modules; - THashMap<TString, TString> ClusterMapping; - THashSet<TString> SqlFlags; - TModuleChecker ModuleChecker; - const bool OptimizeLibraries; - THolder<TExprContext::TFreezeGuard> FreezeGuard; - TString FileAliasPrefix; - TSet<TString> UsedSuffixes; + const NSQLTranslation::TTranslators Translators_; + THolder<TExprContext> OwnedCtx_; + const TModulesTable* ParentModules_ = nullptr; + TUserDataStorage::TPtr UserData_; + IUrlLoader::TPtr UrlLoader_; + TMaybe<NYT::TNode> Parameters_; + TCredentials::TPtr Credentials_; + TQContext QContext_; + TExprContext LibsContext_; + TSet<TString> KnownPackages_; + THashMap<TString, ui32> PackageVersions_; + THashMap<TString, THashMap<int, TLibraryCohesion>> Libs_; + TModulesTable Modules_; + THashMap<TString, TString> ClusterMapping_; + THashSet<TString> SqlFlags_; + TModuleChecker ModuleChecker_; + const bool OptimizeLibraries_; + THolder<TExprContext::TFreezeGuard> FreezeGuard_; + TString FileAliasPrefix_; + TSet<TString> UsedSuffixes_; }; bool SplitUdfName(TStringBuf name, TStringBuf& moduleName, TStringBuf& funcName); @@ -286,18 +286,18 @@ public: TColumnOrderStorage() = default; TMaybe<TColumnOrder> Lookup(ui64 uniqueId) const { - auto it = Storage.find(uniqueId); - if (it == Storage.end()) { + auto it = Storage_.find(uniqueId); + if (it == Storage_.end()) { return {}; } return it->second; } void Set(ui64 uniqueId, const TColumnOrder& order) { - Storage[uniqueId] = order; + Storage_[uniqueId] = order; } private: - THashMap<ui64, TColumnOrder> Storage; + THashMap<ui64, TColumnOrder> Storage_; }; enum class EHiddenMode { diff --git a/yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h b/yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h index ec0868d9415..4ea8ed93d0c 100644 --- a/yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h +++ b/yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h @@ -50,8 +50,8 @@ public: , BuildFunc(buildFunc) {} TParent& Build() { - auto atom = this->Ctx.NewAtom(this->Pos, ResultProviderName); - auto node = this->Ctx.NewCallable(this->Pos, "DataSink", { atom }); + auto atom = this->Ctx_.NewAtom(this->Pos_, ResultProviderName); + auto node = this->Ctx_.NewCallable(this->Pos_, "DataSink", { atom }); return BuildFunc(TResultDataSink(node)); } diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp index 8b97278887a..3a01f5f6da3 100644 --- a/yql/essentials/public/fastcheck/fastcheck.cpp +++ b/yql/essentials/public/fastcheck/fastcheck.cpp @@ -91,10 +91,10 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro TVector<NUserData::TUserData> userData; for (const auto& x : options.SqlLibs) { NUserData::TUserData data; - data.Type_ = NUserData::EType::LIBRARY; - data.Disposition_ = NUserData::EDisposition::INLINE; - data.Name_ = x.first; - data.Content_ = x.second; + data.Type = NUserData::EType::LIBRARY; + data.Disposition = NUserData::EDisposition::INLINE; + data.Name = x.first; + data.Content = x.second; userData.push_back(data); } diff --git a/yql/essentials/public/purecalc/common/interface.cpp b/yql/essentials/public/purecalc/common/interface.cpp index 9c95c0454ad..d9dd9fa634f 100644 --- a/yql/essentials/public/purecalc/common/interface.cpp +++ b/yql/essentials/public/purecalc/common/interface.cpp @@ -51,10 +51,10 @@ TProgramFactoryOptions& TProgramFactoryOptions::SetUDFsDir(TStringBuf dir) { TProgramFactoryOptions& TProgramFactoryOptions::AddLibrary(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content) { auto& ref = UserData_.emplace_back(); - ref.Type_ = NUserData::EType::LIBRARY; - ref.Disposition_ = disposition; - ref.Name_ = name; - ref.Content_ = content; + ref.Type = NUserData::EType::LIBRARY; + ref.Disposition = disposition; + ref.Name = name; + ref.Content = content; return *this; } @@ -62,10 +62,10 @@ TProgramFactoryOptions& TProgramFactoryOptions::AddLibrary(NUserData::EDispositi TProgramFactoryOptions& TProgramFactoryOptions::AddFile(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content) { auto& ref = UserData_.emplace_back(); - ref.Type_ = NUserData::EType::FILE; - ref.Disposition_ = disposition; - ref.Name_ = name; - ref.Content_ = content; + ref.Type = NUserData::EType::FILE; + ref.Disposition = disposition; + ref.Name = name; + ref.Content = content; return *this; } @@ -73,10 +73,10 @@ TProgramFactoryOptions& TProgramFactoryOptions::AddFile(NUserData::EDisposition TProgramFactoryOptions& TProgramFactoryOptions::AddUDF(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content) { auto& ref = UserData_.emplace_back(); - ref.Type_ = NUserData::EType::UDF; - ref.Disposition_ = disposition; - ref.Name_ = name; - ref.Content_ = content; + ref.Type = NUserData::EType::UDF; + ref.Disposition = disposition; + ref.Name = name; + ref.Content = content; return *this; } diff --git a/yql/essentials/public/purecalc/common/program_factory.cpp b/yql/essentials/public/purecalc/common/program_factory.cpp index dae0b52be95..051c4cfa465 100644 --- a/yql/essentials/public/purecalc/common/program_factory.cpp +++ b/yql/essentials/public/purecalc/common/program_factory.cpp @@ -35,10 +35,10 @@ TProgramFactory::TProgramFactory(const TProgramFactoryOptions& options) TVector<TString> UDFsPaths; for (const auto& item: Options_.UserData_) { if ( - item.Type_ == NUserData::EType::UDF && - item.Disposition_ == NUserData::EDisposition::FILESYSTEM + item.Type == NUserData::EType::UDF && + item.Disposition == NUserData::EDisposition::FILESYSTEM ) { - UDFsPaths.push_back(item.Content_); + UDFsPaths.push_back(item.Content); } } |