diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-10-31 17:11:37 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-10-31 17:11:37 +0000 |
commit | a4a4e847216dbe1e32056717eb466537d2128bce (patch) | |
tree | 27070fcd1102d57042f56b8eb602789b020a1d04 /yql | |
parent | e13dea6e57e441f5dc2fe09409a2932cdc4f821c (diff) | |
parent | 6c9f2f9532a9812c29ae9f3ed08ad266b93993c0 (diff) | |
download | ydb-a4a4e847216dbe1e32056717eb466537d2128bce.tar.gz |
Merge branch 'rightlib' into mergelibs-241031-1710
Diffstat (limited to 'yql')
22 files changed, 2410 insertions, 0 deletions
diff --git a/yql/essentials/public/issue/protos/issue_message.proto b/yql/essentials/public/issue/protos/issue_message.proto new file mode 100644 index 0000000000..d581a5d8f0 --- /dev/null +++ b/yql/essentials/public/issue/protos/issue_message.proto @@ -0,0 +1,17 @@ +package NYql.NIssue.NProto; +option java_package = "com.yandex.yql.issue.proto"; + +message IssueMessage { + message Position { + optional uint32 row = 1; + optional uint32 column = 2; + optional string file = 3; + } + + optional Position position = 1; + optional string message = 2; + optional Position end_position = 3; + optional uint32 issue_code = 4; + optional uint32 severity = 5; + repeated IssueMessage issues = 6; +} diff --git a/yql/essentials/public/issue/protos/issue_severity.proto b/yql/essentials/public/issue/protos/issue_severity.proto new file mode 100644 index 0000000000..70fd61f5d9 --- /dev/null +++ b/yql/essentials/public/issue/protos/issue_severity.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package NYql; + +option java_package = "com.yandex.yql.proto"; + +message TSeverityIds { + enum ESeverityId { + S_FATAL = 0; + S_ERROR = 1; + S_WARNING = 2; + S_INFO = 3; + } +} diff --git a/yql/essentials/public/issue/protos/ya.make b/yql/essentials/public/issue/protos/ya.make new file mode 100644 index 0000000000..2451631ea1 --- /dev/null +++ b/yql/essentials/public/issue/protos/ya.make @@ -0,0 +1,10 @@ +PROTO_LIBRARY() + +SRCS( + issue_message.proto + issue_severity.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/yql/essentials/public/issue/ut/ya.make b/yql/essentials/public/issue/ut/ya.make new file mode 100644 index 0000000000..46f35941bc --- /dev/null +++ b/yql/essentials/public/issue/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(yql/essentials/public/issue) + +FORK_SUBTESTS() + +SRCS( + yql_issue_ut.cpp + yql_issue_manager_ut.cpp + yql_issue_utils_ut.cpp + yql_warning_ut.cpp +) + +PEERDIR( + library/cpp/unicode/normalization +) + +END() diff --git a/yql/essentials/public/issue/ya.make b/yql/essentials/public/issue/ya.make new file mode 100644 index 0000000000..0179b23bbb --- /dev/null +++ b/yql/essentials/public/issue/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +SRCS( + yql_issue.cpp + yql_issue_message.cpp + yql_issue_manager.cpp + yql_issue_utils.cpp + yql_warning.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/colorizer + library/cpp/resource + contrib/ydb/public/api/protos + yql/essentials/public/issue/protos + contrib/ydb/library/yql/utils +) + +GENERATE_ENUM_SERIALIZATION(yql_warning.h) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/public/issue/yql_issue.cpp b/yql/essentials/public/issue/yql_issue.cpp new file mode 100644 index 0000000000..3fc8e3767c --- /dev/null +++ b/yql/essentials/public/issue/yql_issue.cpp @@ -0,0 +1,310 @@ +#include "yql_issue.h" +#include "yql_issue_id.h" + +#include <contrib/ydb/library/yql/utils/utf8.h> + +#include <library/cpp/colorizer/output.h> + +#include <util/charset/utf8.h> +#include <util/string/ascii.h> +#include <util/string/split.h> +#include <util/string/strip.h> +#include <util/string/subst.h> +#include <util/system/compiler.h> +#include <util/generic/map.h> +#include <util/generic/stack.h> +#include <cstdlib> + + +namespace NYql { + +void SanitizeNonAscii(TString& s) { + if (!NYql::IsUtf8(s)) { + TString escaped; + escaped.reserve(s.size()); + const unsigned char* i = reinterpret_cast<const unsigned char*>(s.data()); + const unsigned char* end = i + s.size(); + while (i < end) { + wchar32 rune; + size_t runeLen; + const RECODE_RESULT result = SafeReadUTF8Char(rune, runeLen, i, end); + if (result == RECODE_OK) { + escaped.insert(escaped.end(), reinterpret_cast<const char*>(i), reinterpret_cast<const char*>(i + runeLen)); + i += runeLen; + } else { + escaped.push_back('?'); + ++i; + } + } + s = escaped; + } +} + +TTextWalker& TTextWalker::Advance(char c) { + if (c == '\n') { + HaveCr = false; + ++LfCount; + return *this; + } + + + if (c == '\r' && !HaveCr) { + HaveCr = true; + return *this; + } + + ui32 charDistance = 1; + if (Utf8Aware && IsUtf8Intermediate(c)) { + charDistance = 0; + } + + // either not '\r' or second '\r' + if (LfCount) { + Position.Row += LfCount; + Position.Column = charDistance; + LfCount = 0; + } else { + Position.Column += charDistance + (HaveCr && c != '\r'); + } + HaveCr = (c == '\r'); + return *this; +} + +void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { + out << Range() << ": " << SeverityToString(GetSeverity()) << ": "; + if (oneLine) { + TString message = StripString(Message); + SubstGlobal(message, '\n', ' '); + out << message; + } else { + out << Message; + } + if (GetCode()) { + out << ", code: " << GetCode(); + } +} + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn) { + enum class EFnType { + Main, + AfterChildren, + }; + + const bool hasAfterChildrenFn = bool(afterChildrenFn); + TStack<std::tuple<ui16, const TIssue*, EFnType>> issuesStack; + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(0, &topIssue, EFnType::Main)); + while (!issuesStack.empty()) { + auto level = std::get<0>(issuesStack.top()); + const auto& curIssue = *std::get<1>(issuesStack.top()); + const EFnType fnType = std::get<2>(issuesStack.top()); + issuesStack.pop(); + if (!leafOnly || curIssue.GetSubIssues().empty()) { + if (fnType == EFnType::Main) { + fn(curIssue, level); + } else { + afterChildrenFn(curIssue, level); + } + } + if (fnType == EFnType::Main) { + level++; + const auto& subIssues = curIssue.GetSubIssues(); + for (int i = subIssues.size() - 1; i >= 0; i--) { + if (hasAfterChildrenFn) { + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::AfterChildren)); + } + issuesStack.push(std::make_tuple(level, subIssues[i].Get(), EFnType::Main)); + } + } + } +} + +namespace { + +Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { + char* whitespaces = reinterpret_cast<char*>(alloca(indentation)); + memset(whitespaces, ' ', indentation); + out.Write(whitespaces, indentation); +} + +void ProgramLinesWithErrors( + const TString& programText, + const TVector<TIssue>& errors, + TMap<ui32, TStringBuf>& lines) +{ + TVector<ui32> rows; + for (const auto& topIssue: errors) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { + for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { + rows.push_back(row); + } + }); + } + std::sort(rows.begin(), rows.end()); + + auto prog = StringSplitter(programText).Split('\n'); + auto progIt = prog.begin(), progEnd = prog.end(); + ui32 progRow = 1; + + for (ui32 row: rows) { + while (progRow < row && progIt != progEnd) { + ++progRow; + ++progIt; + } + if (progIt != progEnd) { + lines[row] = progIt->Token(); + } + } +} + +} // namspace + +void TIssues::PrintTo(IOutputStream& out, bool oneLine) const +{ + if (oneLine) { + bool printWithSpace = false; + if (Issues_.size() > 1) { + printWithSpace = true; + out << "["; + } + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + if (level > 0) { + out << " subissue: { "; + } else { + out << (printWithSpace ? " { " : "{ "); + } + issue.PrintTo(out, true); + }, + [&](const TIssue&, ui16) { + out << " }"; + }); + } + if (Issues_.size() > 1) { + out << " ]"; + } + } else { + for (const auto& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + auto shift = level * 4; + Indent(out, shift); + out << issue << Endl; + }); + } + } +} + +void TIssues::PrintWithProgramTo( + IOutputStream& out, + const TString& programFilename, + const TString& programText) const +{ + using namespace NColorizer; + + TMap<ui32, TStringBuf> lines; + ProgramLinesWithErrors(programText, Issues_, lines); + + for (const TIssue& topIssue: Issues_) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + auto shift = level * 4; + Indent(out, shift); + out << DarkGray() << programFilename << Old() << ':'; + out << Purple() << issue.Range() << Old(); + auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed(); + auto severityName = SeverityToString(issue.GetSeverity()); + out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n'; + Indent(out, shift); + if (issue.Position.HasValue()) { + out << '\t' << lines[issue.Position.Row] << '\n'; + out << '\t'; + if (issue.Position.Column > 0) { + Indent(out, issue.Position.Column - 1); + } + out << '^'; + } + out << Endl; + }); + } +} + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { + TStringBuf messageBuf = e.what(); + auto parsedPos = TryParseTerminationMessage(messageBuf); + auto issue = TIssue(parsedPos.GetOrElse(pos), messageBuf); + const TErrorException* errorException = dynamic_cast<const TErrorException*>(&e); + if (errorException) { + issue.SetCode(errorException->GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + } else { + issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL); + } + return issue; +} + +static constexpr TStringBuf TerminationMessageMarker = "Terminate was called, reason("; + +TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) { + size_t len = 0; + size_t startPos = message.find(TerminationMessageMarker); + size_t endPos = 0; + if (startPos != TString::npos) { + endPos = message.find(')', startPos + TerminationMessageMarker.size()); + if (endPos != TString::npos) { + TStringBuf lenText = message.Tail(startPos + TerminationMessageMarker.size()) + .Trunc(endPos - startPos - TerminationMessageMarker.size()); + try { + len = FromString<size_t>(lenText); + } catch (const TFromStringException&) { + len = 0; + } + } + } + + if (len) { + message = message.Tail(endPos + 3).Trunc(len); + auto s = message; + TMaybe<TStringBuf> file; + TMaybe<TStringBuf> row; + TMaybe<TStringBuf> column; + GetNext(s, ':', file); + GetNext(s, ':', row); + GetNext(s, ':', column); + ui32 rowValue, columnValue; + if (file && row && column && TryFromString(*row, rowValue) && TryFromString(*column, columnValue)) { + message = StripStringLeft(s); + return TPosition(columnValue, rowValue, TString(*file)); + } + } + + return Nothing(); +} + +} // namspace NYql + +template <> +void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) { + out << (pos.File ? pos.File : "<main>"); + if (pos) { + out << ":" << pos.Row << ':' << pos.Column; + } +} + +template<> +void Out<NYql::TRange>(IOutputStream & out, const NYql::TRange & range) { + if (range.IsRange()) { + out << '[' << range.Position << '-' << range.EndPosition << ']'; + } else { + out << range.Position; + } +} + +template <> +void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error) { + error.PrintTo(out); +} + +template <> +void Out<NYql::TIssues>(IOutputStream& out, const NYql::TIssues& error) { + error.PrintTo(out); +}
\ No newline at end of file diff --git a/yql/essentials/public/issue/yql_issue.h b/yql/essentials/public/issue/yql_issue.h new file mode 100644 index 0000000000..00c49fb1fc --- /dev/null +++ b/yql/essentials/public/issue/yql_issue.h @@ -0,0 +1,372 @@ +#pragma once + +#include <util/system/types.h> +#include <util/generic/hash.h> +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/generic/string.h> +#include <util/generic/strbuf.h> +#include <util/generic/ptr.h> +#include <util/stream/output.h> +#include <util/stream/str.h> +#include <util/digest/numeric.h> +#include <google/protobuf/message.h> + +#include "yql_issue_id.h" + +namespace NYql { + +void SanitizeNonAscii(TString& s); + +/////////////////////////////////////////////////////////////////////////////// +// TPosition +/////////////////////////////////////////////////////////////////////////////// +struct TPosition { + ui32 Column = 0U; + ui32 Row = 0U; + TString File; + + TPosition() = default; + + TPosition(ui32 column, ui32 row, const TString& file = {}) + : Column(column) + , Row(row) + , File(file) + { + SanitizeNonAscii(File); + } + + explicit operator bool() const { + return HasValue(); + } + + inline bool HasValue() const { + return Row | Column; + } + + inline bool operator==(const TPosition& other) const { + return Column == other.Column && Row == other.Row && File == other.File; + } + + inline bool operator<(const TPosition& other) const { + return std::tie(Row, Column, File) < std::tie(other.Row, other.Column, other.File); + } +}; + +class TTextWalker { +public: + TTextWalker(TPosition& position, bool utf8Aware) + : Position(position) + , Utf8Aware(utf8Aware) + , HaveCr(false) + , LfCount(0) + { + } + + static inline bool IsUtf8Intermediate(char c) { + return (c & 0xC0) == 0x80; + } + + template<typename T> + TTextWalker& Advance(const T& buf) { + for (char c : buf) { + Advance(c); + } + return *this; + } + + TTextWalker& Advance(char c); + +private: + TPosition& Position; + const bool Utf8Aware; + bool HaveCr; + ui32 LfCount; +}; + +struct TRange { + TPosition Position; + TPosition EndPosition; + + TRange() = default; + + TRange(TPosition position) + : Position(position) + , EndPosition(position) + { + } + + TRange(TPosition position, TPosition endPosition) + : Position(position) + , EndPosition(endPosition) + { + } + + inline bool IsRange() const { + return !(Position == EndPosition); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// TIssue +/////////////////////////////////////////////////////////////////////////////// + +class TIssue; +using TIssuePtr = TIntrusivePtr<TIssue>; +class TIssue: public TThrRefBase { + TVector<TIntrusivePtr<TIssue>> Children_; + TString Message; +public: + TPosition Position; + TPosition EndPosition; + TIssueCode IssueCode = 0U; + ESeverity Severity = TSeverityIds::S_ERROR; + + TIssue() = default; + + template <typename T> + explicit TIssue(const T& message) + : Message(message) + , Position(TPosition()) + , EndPosition(TPosition()) + { + SanitizeNonAscii(Message); + } + + template <typename T> + TIssue(TPosition position, const T& message) + : Message(message) + , Position(position) + , EndPosition(position) + { + SanitizeNonAscii(Message); + } + + inline TRange Range() const { + return{ Position, EndPosition }; + } + + template <typename T> + TIssue(TPosition position, TPosition endPosition, const T& message) + : Message(message) + , Position(position) + , EndPosition(endPosition) + { + SanitizeNonAscii(Message); + } + + inline bool operator==(const TIssue& other) const { + return Position == other.Position && Message == other.Message + && IssueCode == other.IssueCode; + } + + ui64 Hash() const noexcept { + return CombineHashes( + CombineHashes( + (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), + ComputeHash(Position.File) + ), + (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message))); + } + + TIssue& SetCode(TIssueCode id, ESeverity severity) { + IssueCode = id; + Severity = severity; + return *this; + } + + TIssue& SetMessage(const TString& msg) { + Message = msg; + SanitizeNonAscii(Message); + return *this; + } + + ESeverity GetSeverity() const { + return Severity; + } + + TIssueCode GetCode() const { + return IssueCode; + } + + const TString& GetMessage() const { + return Message; + } + + TIssue& AddSubIssue(TIntrusivePtr<TIssue> issue) { + Severity = (ESeverity)Min((ui32)issue->GetSeverity(), (ui32)Severity); + Children_.push_back(issue); + return *this; + } + + const TVector<TIntrusivePtr<TIssue>>& GetSubIssues() const { + return Children_; + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + + TString ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + // Unsafe method. Doesn't call SanitizeNonAscii(Message) + TString* MutableMessage() { + return &Message; + } + + TIssue& CopyWithoutSubIssues(const TIssue& src) { + Message = src.Message; + IssueCode = src.IssueCode; + Severity = src.Severity; + Position = src.Position; + EndPosition = src.EndPosition; + return *this; + } +}; + +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function<void(const TIssue&, ui16 level)> fn, std::function<void(const TIssue&, ui16 level)> afterChildrenFn = {}); + +/////////////////////////////////////////////////////////////////////////////// +// TIssues +/////////////////////////////////////////////////////////////////////////////// +class TIssues { +public: + TIssues() = default; + + inline TIssues(const TVector<TIssue>& issues) + : Issues_(issues) + { + } + + inline TIssues(const std::initializer_list<TIssue>& issues) + : TIssues(TVector<TIssue>(issues)) + { + } + + inline TIssues(const TIssues& rhs) + : Issues_(rhs.Issues_) + { + } + + inline TIssues& operator=(const TIssues& rhs) { + Issues_ = rhs.Issues_; + return *this; + } + + inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) + { + } + + inline TIssues& operator=(TIssues&& rhs) { + Issues_ = std::move(rhs.Issues_); + return *this; + } + + template <typename ... Args> void AddIssue(Args&& ... args) { + Issues_.emplace_back(std::forward<Args>(args)...); + } + + inline void AddIssues(const TIssues& errors) { + Issues_.insert(Issues_.end(), + errors.Issues_.begin(), errors.Issues_.end()); + } + + inline void AddIssues(const TPosition& pos, const TIssues& errors) { + Issues_.reserve(Issues_.size() + errors.Size()); + for (const auto& e: errors) { + TIssue& issue = Issues_.emplace_back(); + *issue.MutableMessage() = e.GetMessage(); // No need to sanitize message, it has already been sanitized. + issue.Position = pos; + issue.SetCode(e.IssueCode, e.Severity); + } + } + + inline const TIssue* begin() const { + return Issues_.begin(); + } + + inline const TIssue* end() const { + return Issues_.end(); + } + + inline TIssue& back() { + return Issues_.back(); + } + + inline const TIssue& back() const { + return Issues_.back(); + } + + inline bool Empty() const { + return Issues_.empty(); + } + + explicit operator bool() const noexcept { + return !Issues_.empty(); + } + + inline size_t Size() const { + return Issues_.size(); + } + + void PrintTo(IOutputStream& out, bool oneLine = false) const; + void PrintWithProgramTo( + IOutputStream& out, + const TString& programFilename, + const TString& programText) const; + + inline TString ToString(bool oneLine = false) const { + TStringStream out; + PrintTo(out, oneLine); + return out.Str(); + } + + TString ToOneLineString() const { + return ToString(true); + } + + inline void Clear() { + Issues_.clear(); + } + + void Reserve(size_t capacity) { + Issues_.reserve(capacity); + } + +private: + TVector<TIssue> Issues_; +}; + +class TErrorException : public yexception { + const TIssueCode Code_; +public: + explicit TErrorException(TIssueCode code) + : Code_(code) + {} + TIssueCode GetCode() const { + return Code_; + } +}; + +TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos = TPosition()); +TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message); + +} // namespace NYql + +template <> +void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos); + +template <> +void Out<NYql::TRange>(IOutputStream& out, const NYql::TRange& pos); + +template <> +void Out<NYql::TIssue>(IOutputStream& out, const NYql::TIssue& error); + +template <> +struct THash<NYql::TIssue> { + inline size_t operator()(const NYql::TIssue& err) const { + return err.Hash(); + } +}; diff --git a/yql/essentials/public/issue/yql_issue_id.h b/yql/essentials/public/issue/yql_issue_id.h new file mode 100644 index 0000000000..b4889319a7 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_id.h @@ -0,0 +1,128 @@ +#pragma once + +#include <yql/essentials/public/issue/protos/issue_severity.pb.h> + +#include <library/cpp/resource/resource.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/text_format.h> +#include <google/protobuf/message.h> + +#include <util/generic/hash.h> +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> +#include <util/string/subst.h> + +#ifdef _win_ +#ifdef GetMessage +#undef GetMessage +#endif +#endif + +namespace NYql { + +using TIssueCode = ui32; +using ESeverity = NYql::TSeverityIds::ESeverityId; +const TIssueCode DEFAULT_ERROR = 0; +const TIssueCode UNEXPECTED_ERROR = 1; + +inline TString SeverityToString(ESeverity severity) +{ + auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); + return ret.empty() ? "Unknown" : to_title(ret.substr(2)); //remove prefix "S_" +} + +template <typename T> +inline TString IssueCodeToString(TIssueCode id) { + auto ret = T::EIssueCode_Name(static_cast<typename T::EIssueCode>(id)); + if (!ret.empty()) { + SubstGlobal(ret, '_', ' '); + return to_title(ret); + } else { + return "Unknown"; + } +} + +template<typename TProto, const char* ResourceName> +class TIssueId { + TProto ProtoIssues_; + THashMap<TIssueCode, NYql::TSeverityIds::ESeverityId> IssuesMap_; + THashMap<TIssueCode, TString> IssuesFormatMap_; + + const google::protobuf::Descriptor* GetProtoDescriptor() const { + auto ret = ProtoIssues_.GetDescriptor(); + Y_ENSURE(ret != nullptr, "Bad proto file"); + return ret; + } + + bool CheckSeverityNameFormat(const TString& name) const { + if (name.size() > 2 && name.substr(0,2) == "S_") { + return true; + } + return false; + } + +public: + ESeverity GetSeverity(TIssueCode id) const { + auto it = IssuesMap_.find(id); + Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: " + << id << "(" << IssueCodeToString<TProto>(id) << ")"); + return it->second; + } + + TString GetMessage(TIssueCode id) const { + auto it = IssuesFormatMap_.find(id); + Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: " + << id << "(" << IssueCodeToString<TProto>(id) << ")"); + return it->second; + } + + TIssueId() { + auto configData = NResource::Find(TStringBuf(ResourceName)); + if (!::google::protobuf::TextFormat::ParseFromString(configData, &ProtoIssues_)) { + Y_ENSURE(false, "Bad format of protobuf data file, resource: " << ResourceName); + } + + auto sDesc = TSeverityIds::ESeverityId_descriptor(); + for (int i = 0; i < sDesc->value_count(); i++) { + const auto& name = sDesc->value(i)->name(); + Y_ENSURE(CheckSeverityNameFormat(name), + "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); + } + + for (const auto& x : ProtoIssues_.ids()) { + auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity())); + Y_ENSURE(rv.second, "Duplicate issue code found, code: " + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + } + + // Check all IssueCodes have mapping to severity + auto eDesc = TProto::EIssueCode_descriptor(); + for (int i = 0; i < eDesc->value_count(); i++) { + auto it = IssuesMap_.find(eDesc->value(i)->number()); + Y_ENSURE(it != IssuesMap_.end(), "IssueCode: " + << eDesc->value(i)->name() + << " is not found in protobuf data file"); + } + + for (const auto& x : ProtoIssues_.ids()) { + auto rv = IssuesFormatMap_.insert(std::make_pair(x.code(), x.format())); + Y_ENSURE(rv.second, "Duplicate issue code found, code: " + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + } + } +}; + +template<typename TProto, const char* ResourceName> +inline ESeverity GetSeverity(TIssueCode id) { + return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id); +} + +template<typename TProto, const char* ResourceName> +inline TString GetMessage(TIssueCode id) { + return Singleton<TIssueId<TProto, ResourceName>>()->GetMessage(id); +} + +} diff --git a/yql/essentials/public/issue/yql_issue_manager.cpp b/yql/essentials/public/issue/yql_issue_manager.cpp new file mode 100644 index 0000000000..8ffeed9fb0 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager.cpp @@ -0,0 +1,234 @@ +#include "yql_issue_manager.h" + +#include <util/string/cast.h> +#include <util/string/builder.h> + +using namespace NYql; + +void TIssueManager::AddScope(std::function<TIssuePtr()> fn) { + RawIssues_.emplace(std::make_pair(TMaybe<TIssuePtr>(), fn)); +} + +void TIssueManager::LeaveScope() { + Y_ASSERT(HasOpenScopes()); + if (RawIssues_.size() == 1) { + // Last scope, just dump it + auto maybeIssue = RawIssues_.top().first; + if (maybeIssue) { + if ((*maybeIssue)->GetCode() == Max<ui32>()) { + for (const auto& nestedIssue : (*maybeIssue.Get())->GetSubIssues()) { + CompletedIssues_.AddIssue(*nestedIssue); + } + } else { + CompletedIssues_.AddIssue(**maybeIssue); + } + } + RawIssues_.pop(); + return; + } + + if (RawIssues_.top().first.Empty()) { + // No issues in this scope + RawIssues_.pop(); + return; + } + + auto subIssue = *RawIssues_.top().first; + if (subIssue->GetSubIssues().size() == 1) { + auto nestedIssue = subIssue->GetSubIssues().front(); + if (!nestedIssue->GetSubIssues().empty() && nestedIssue->Position == subIssue->Position && nestedIssue->EndPosition == subIssue->EndPosition) { + auto msg = subIssue->GetMessage(); + if (nestedIssue->GetMessage()) { + if (msg) { + msg.append(", "); + } + msg.append(nestedIssue->GetMessage()); + } + subIssue = nestedIssue; + subIssue->SetMessage(msg); + } + } + RawIssues_.pop(); + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); + if (!*RawIssues_.top().first) { + RawIssues_.top().first = new TIssue(); + (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); + } else { + (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; + } + } + + if (subIssue->GetCode() == Max<ui32>()) { + for (const auto& nestedIssue : subIssue->GetSubIssues()) { + RawIssues_.top().first->Get()->AddSubIssue(nestedIssue); + } + } else { + RawIssues_.top().first->Get()->AddSubIssue(subIssue); + } +} + +void TIssueManager::RaiseIssueForEmptyScope() { + if (RawIssues_.top().first.Empty()) { + TIssuePtr materialized = RawIssues_.top().second(); + if (auto p = CheckUniqAndLimit(materialized)) { + RawIssues_.top().first = p; + } + } +} + +void TIssueManager::LeaveAllScopes() { + while (!RawIssues_.empty()) { + LeaveScope(); + } +} + +TIssuePtr TIssueManager::CheckUniqAndLimit(TIssuePtr issue) { + const auto severity = issue->GetSeverity(); + if (OverflowIssues_[severity]) { + return {}; + } + if (UniqueIssues_[severity].contains(issue)) { + return {}; + } + if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) { + OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder() + << "Too many " << SeverityToString(issue->GetSeverity()) << " issues"); + OverflowIssues_[severity]->Severity = severity; + return {}; + } + UniqueIssues_[severity].insert(issue); + return issue; +} + +TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) { + const auto severity = issue.GetSeverity(); + if (OverflowIssues_[severity]) { + return {}; + } + return CheckUniqAndLimit(MakeIntrusive<TIssue>(issue)); +} + +void TIssueManager::RaiseIssue(const TIssue& issue) { + TIssuePtr p = CheckUniqAndLimit(issue); + if (!p) { + return; + } + if (RawIssues_.empty()) { + CompletedIssues_.AddIssue(issue); + return; + } + if (RawIssues_.top().first.Empty()) { + RawIssues_.top().first = RawIssues_.top().second(); + if (!*RawIssues_.top().first) { + RawIssues_.top().first = new TIssue(); + (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); + } else { + (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; + } + } + RawIssues_.top().first->Get()->AddSubIssue(p); +} + +void TIssueManager::RaiseIssues(const TIssues& issues) { + for (const auto& x : issues) { + RaiseIssue(x); + } +} + +bool TIssueManager::RaiseWarning(TIssue issue) { + bool isWarning = true; + if (issue.GetSeverity() == ESeverity::TSeverityIds_ESeverityId_S_WARNING) { + const auto action = WarningPolicy_.GetAction(issue.GetCode()); + switch (action) { + case EWarningAction::DISABLE: + return isWarning; + case EWarningAction::ERROR: + issue.Severity = ESeverity::TSeverityIds_ESeverityId_S_ERROR; + if (WarningToErrorTreatMessage_) { + TIssue newIssue; + newIssue.SetCode(issue.GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + newIssue.SetMessage(WarningToErrorTreatMessage_.GetRef()); + newIssue.AddSubIssue(new TIssue(issue)); + issue = newIssue; + } + isWarning = false; + break; + case EWarningAction::DEFAULT: + break; + default: + Y_ENSURE(false, "Unknown action"); + } + } + + RaiseIssue(issue); + return isWarning; +} + +bool TIssueManager::HasOpenScopes() const { + return !RawIssues_.empty(); +} + +TIssues TIssueManager::GetIssues() { + auto tmp = RawIssues_; + LeaveAllScopes(); + auto result = GetCompletedIssues(); + RawIssues_ = tmp; + return result; +} + +TIssues TIssueManager::GetCompletedIssues() const { + TIssues res; + for (auto& p: OverflowIssues_) { + if (p) { + res.AddIssue(*p); + } + } + res.AddIssues(CompletedIssues_); + return res; +} + +void TIssueManager::AddIssues(const TIssues& issues) { + for (auto& issue: issues) { + if (auto p = CheckUniqAndLimit(issue)) { + CompletedIssues_.AddIssue(*p); + } + } +} + +void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { + for (auto& issue: issues) { + if (auto p = CheckUniqAndLimit(TIssue(pos, issue.GetMessage()))) { + CompletedIssues_.AddIssue(*p); + } + } +} + +void TIssueManager::Reset(const TIssues& issues) { + for (auto& p: OverflowIssues_) { + p.Drop(); + } + + for (auto& s: UniqueIssues_) { + s.clear(); + } + CompletedIssues_.Clear(); + + while (!RawIssues_.empty()) { + RawIssues_.pop(); + } + AddIssues(issues); +} + +void TIssueManager::Reset() { + Reset(TIssues()); +} + +void TIssueManager::AddWarningRule(const TWarningRule &rule) +{ + WarningPolicy_.AddRule(rule); +} + +void TIssueManager::SetWarningToErrorTreatMessage(const TString& msg) { + WarningToErrorTreatMessage_ = msg; +} diff --git a/yql/essentials/public/issue/yql_issue_manager.h b/yql/essentials/public/issue/yql_issue_manager.h new file mode 100644 index 0000000000..6b6ec2c719 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager.h @@ -0,0 +1,83 @@ +#pragma once + +#include "yql_issue.h" +#include "yql_warning.h" + +#include <util/generic/maybe.h> +#include <util/generic/stack.h> +#include <util/generic/hash_set.h> + +#include <array> + +namespace NYql { + +class TIssueManager: private TNonCopyable { +public: + void AddScope(std::function<TIssuePtr()> fn); + void LeaveScope(); + void RaiseIssue(const TIssue& issue); + void RaiseIssues(const TIssues& issues); + bool RaiseWarning(TIssue issue); + void AddIssues(const TIssues& errors); + void AddIssues(const TPosition& pos, const TIssues& issues); + + TIssues GetIssues(); + TIssues GetCompletedIssues() const; + + void Reset(const TIssues& issues); + void Reset(); + + void AddWarningRule(const TWarningRule &rule); + void SetWarningToErrorTreatMessage(const TString& msg); + + void SetIssueCountLimit(size_t limit) { + IssueLimit_ = limit; + } + + void RaiseIssueForEmptyScope(); + +private: + TIssuePtr CheckUniqAndLimit(const TIssue& issue); + TIssuePtr CheckUniqAndLimit(TIssuePtr issue); + void LeaveAllScopes(); + bool HasOpenScopes() const; + + struct TIssueHash { + ui64 operator()(const TIssuePtr& p) { + return p->Hash(); + } + }; + struct TIssueEqual { + bool operator()(const TIssuePtr& l, const TIssuePtr& r) { + return *l == *r; + } + }; + TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_; + TIssues CompletedIssues_; + TMaybe<TString> WarningToErrorTreatMessage_; + TWarningPolicy WarningPolicy_; + std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_; + std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_; + size_t IssueLimit_ = 0; +}; + +class TIssueScopeGuard: private TNonCopyable { + TIssueManager& Manager_; +public: + TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn) + : Manager_(manager) + { + Manager_.AddScope(fn); + } + + void RaiseIssueForEmptyScope() { + Manager_.RaiseIssueForEmptyScope(); + } + + ~TIssueScopeGuard() + { + Manager_.LeaveScope(); + } +}; + +} diff --git a/yql/essentials/public/issue/yql_issue_manager_ut.cpp b/yql/essentials/public/issue/yql_issue_manager_ut.cpp new file mode 100644 index 0000000000..e3308163c3 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_manager_ut.cpp @@ -0,0 +1,206 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include "yql_issue_manager.h" + +using namespace NYql; + +static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) { + return [name, column, row]() { + return new TIssue(TPosition(column, row), name); + }; +} + +Y_UNIT_TEST_SUITE(TIssueManagerTest) { + Y_UNIT_TEST(NoErrorNoLevelTest) { + TIssueManager issueManager; + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(NoErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(NoErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.LeaveScope(); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); + } + + Y_UNIT_TEST(OneErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues1 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + auto completedIssues2 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); + issueManager.LeaveScope(); + auto completedIssues3 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); + + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "IssueOne"); + } + + Y_UNIT_TEST(OneErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "B"); + + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->GetMessage(), "IssueOne"); + } + + Y_UNIT_TEST(MultiErrorsMultiLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2)); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1 + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "WarningScope1"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->GetMessage(), "ErrorScope3"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->GetMessage(), "ErrorScope2"); + auto ref = R"___(<main>: Error: A + <main>: Error: WarningScope1 + <main>:1:1: Error: B + <main>:2:2: Error: C + <main>: Error: ErrorScope3 + <main>: Error: ErrorScope2 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(TIssueScopeGuardSimpleTest) { + TIssueManager issueManager; + { + TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1")); + } + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "ErrorScope1"); + } + + Y_UNIT_TEST(FuseScopesTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + auto ref = R"___(<main>: Error: A + <main>: Error: B, C + <main>: Error: D + <main>:2:1: Error: IssueOne + <main>:2:1: Error: IssueTwo +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(UniqIssues) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); + issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne")); + issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto ref = R"___(<main>: Error: A + <main>:1:1: Error: B + <main>:2:1: Error: IssueOne + <main>:2:1: Error: IssueTwo + <main>:3:2: Error: IssueOne + <main>:3:2: Warning: IssueOne, code: 1 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } + + Y_UNIT_TEST(Limits) { + TIssueManager issueManager; + issueManager.SetIssueCountLimit(2); + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3")); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3); + auto ref = R"___(<main>: Error: Too many Error issues +<main>: Warning: Too many Warning issues +<main>: Error: A + <main>:1:1: Error: B + <main>:1:1: Error: Issue1 + <main>:1:1: Error: Issue2 + <main>:1:1: Warning: Issue4, code: 1 + <main>:1:1: Warning: Issue5, code: 1 + <main>:1:1: Info: Issue7, code: 2 + <main>:1:1: Info: Issue8, code: 2 +)___"; + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); + } +} diff --git a/yql/essentials/public/issue/yql_issue_message.cpp b/yql/essentials/public/issue/yql_issue_message.cpp new file mode 100644 index 0000000000..757bba3946 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_message.cpp @@ -0,0 +1,149 @@ +#include "yql_issue_message.h" + +#include <yql/essentials/public/issue/protos/issue_message.pb.h> +#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h> + +#include <util/generic/deque.h> +#include <util/generic/yexception.h> +#include <util/stream/output.h> +#include <util/string/join.h> + +#include <tuple> + +namespace NYql { + +using namespace NIssue::NProto; + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage) { + TIssue topIssue; + TDeque<std::pair<TIssue*, const TIssueMessage*>> queue; + queue.push_front(std::make_pair(&topIssue, &issueMessage)); + while (!queue.empty()) { + TIssue& issue = *queue.back().first; + const auto& message = *queue.back().second; + queue.pop_back(); + TPosition position(message.position().column(), message.position().row(), message.position().file()); + TPosition endPosition(message.end_position().column(), message.end_position().row()); + if (position.HasValue()) { + if (endPosition.HasValue()) { + issue = TIssue(position, endPosition, message.message()); + } else { + issue = TIssue(position, message.message()); + } + } else { + issue = TIssue(message.message()); + } + + for (const auto& subMessage : message.issues()) { + auto subIssue = new TIssue(); + issue.AddSubIssue(subIssue); + queue.push_front(std::make_pair(subIssue, &subMessage)); + } + + issue.SetCode(message.issue_code(), static_cast<ESeverity>(message.severity())); + } + return topIssue; +} + +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) { + issues.Clear(); + if (message.size()) { + issues.Reserve(message.size()); + for (auto &x : message) + issues.AddIssue(IssueFromMessage(x)); + } +} + +template<typename TIssueMessage> +void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { + TDeque<std::pair<const TIssue*, TIssueMessage*>> queue; + queue.push_front(std::make_pair(&topIssue, issueMessage)); + while (!queue.empty()) { + const TIssue& issue = *queue.back().first; + auto& message = *queue.back().second; + queue.pop_back(); + if (issue.Position) { + auto& position = *message.mutable_position(); + position.set_row(issue.Position.Row); + position.set_column(issue.Position.Column); + position.set_file(issue.Position.File); + } + if (issue.EndPosition) { + auto& endPosition = *message.mutable_end_position(); + endPosition.set_row(issue.EndPosition.Row); + endPosition.set_column(issue.EndPosition.Column); + } + message.set_message(issue.GetMessage()); + message.set_issue_code(issue.GetCode()); + message.set_severity(issue.GetSeverity()); + + for (auto subIssue : issue.GetSubIssues()) { + TIssueMessage* subMessage = message.add_issues(); + queue.push_front(std::make_pair(subIssue.Get(), subMessage)); + } + } +} + +template<typename TIssueMessage> +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) { + message->Clear(); + if (!issues) + return; + message->Reserve(issues.Size()); + for (const auto &issue : issues) { + IssueToMessage(issue, message->Add()); + } +} + +template +TIssue IssueFromMessage<Ydb::Issue::IssueMessage>(const Ydb::Issue::IssueMessage& issueMessage); +template +TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); + +template +void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues); +template +void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues); + +template +void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage); +template +void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); + +template +void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message); +template +void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message); + +NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { + NIssue::NProto::IssueMessage issueMessage; + IssueToMessage(topIssue, &issueMessage); + return issueMessage; +} + +TString IssueToBinaryMessage(const TIssue& issue) { + TString result; + Ydb::Issue::IssueMessage protobuf; + IssueToMessage(issue, &protobuf); + Y_PROTOBUF_SUPPRESS_NODISCARD protobuf.SerializeToString(&result); + return result; +} + +TIssue IssueFromBinaryMessage(const TString& binaryMessage) { + Ydb::Issue::IssueMessage protobuf; + if (!protobuf.ParseFromString(binaryMessage)) { + ythrow yexception() << "unable to parse binary string as issue protobuf"; + } + return IssueFromMessage(protobuf); +} + +} + +Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>, stream, issues) { + stream << JoinSeq("", issues); +} +Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>, stream, issues) { + stream << JoinSeq("", issues); +}
\ No newline at end of file diff --git a/yql/essentials/public/issue/yql_issue_message.h b/yql/essentials/public/issue/yql_issue_message.h new file mode 100644 index 0000000000..a3cb63e4f4 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_message.h @@ -0,0 +1,37 @@ +#pragma once + +#include "yql_issue.h" + +#include <util/generic/ylimits.h> + +namespace NYql { + +namespace NIssue { +namespace NProto { +class IssueMessage; +} +} + +template<typename TIssueMessage> +TIssue IssueFromMessage(const TIssueMessage& issueMessage); +template<typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues); + +template<typename TIssueMessage> +TString IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message) { + TIssues issues; + IssuesFromMessage(message, issues); + return issues.ToOneLineString(); +} + +NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue); + +template<typename TIssueMessage> +void IssueToMessage(const TIssue& topIssue, TIssueMessage* message); +template<typename TIssueMessage> +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message); + +TString IssueToBinaryMessage(const TIssue& issue); +TIssue IssueFromBinaryMessage(const TString& binaryMessage); + +} diff --git a/yql/essentials/public/issue/yql_issue_ut.cpp b/yql/essentials/public/issue/yql_issue_ut.cpp new file mode 100644 index 0000000000..02f917fa54 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_ut.cpp @@ -0,0 +1,249 @@ +#include "yql_issue.h" +#include "yql_issue_message.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/public/issue/protos/issue_message.pb.h> +#include <yql/essentials/public/issue/yql_issue_message.h> +#include <contrib/ydb/public/api/protos/ydb_issue_message.pb.h> + +#include <library/cpp/unicode/normalization/normalization.h> + +#include <util/charset/utf8.h> +#include <util/charset/wide.h> +#include <util/string/builder.h> + +#include <google/protobuf/message.h> +#include <google/protobuf/descriptor.h> + +using namespace google::protobuf; +using namespace NYql; + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes); +void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + + UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type())); + UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label()); + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number()); + UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated()); + UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed()); + UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index()); + if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) { + ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes); + } +} + +void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<TString>* visitedTypes) { + UNIT_ASSERT(a); + UNIT_ASSERT(b); + if (!visitedTypes->insert(a->name()).second) { + return; + } + + UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); + UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count()); + + for (int i = 0; i < a->field_count(); i++) { + ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes); + } +} + +Y_UNIT_TEST_SUITE(IssueTest) { + Y_UNIT_TEST(Ascii) { + TIssue issue1("тест abc"); + UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); + TIssue issue2("\xFF abc"); + UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); + TIssue issue3(""); + UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); + TIssue issue4("abc"); + UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); + } +} + +Y_UNIT_TEST_SUITE(IssueProtoTest) { + Y_UNIT_TEST(KikimrYqlSameLayout) { + Ydb::Issue::IssueMessage yqlIssue; + NYql::NIssue::NProto::IssueMessage kikimrIssue; + THashSet<TString> visitedTypes; + ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes); + } + + Y_UNIT_TEST(BinarySerialization) { + TIssue issueTo("root_issue"); + TString bin = IssueToBinaryMessage(issueTo); + TIssue issueFrom = IssueFromBinaryMessage(bin); + UNIT_ASSERT_EQUAL(issueTo, issueFrom); + } + + Y_UNIT_TEST(WrongBinStringException) { + UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); + } +} + + +Y_UNIT_TEST_SUITE(TextWalkerTest) { + Y_UNIT_TEST(BasicTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\r\taa")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); + walker.Advance(TStringBuf("\na")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); + } + + Y_UNIT_TEST(CrLfTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\raa\r")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); + walker.Advance(TStringBuf("\r\r\ra")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\r'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('a'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); + } + + Y_UNIT_TEST(UnicodeTest) { + { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("привет")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1)); + } + + { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, true); + walker.Advance(TStringBuf("привет")); + + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1)); + } + } +} + +Y_UNIT_TEST_SUITE(ToOneLineStringTest) { + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); + issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); + + TIssues issues; + issues.AddIssue(issue); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: { <main>: Error: suberror subissue: { <main>: Error: subsuberror } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); + issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: { <main>: Error: suberror } } { abc.file:2:100: Error: my message } ]"); + } +} + +Y_UNIT_TEST_SUITE(ToStreamTest) { + template <typename TIssueMessage> + void CheckSerializationToStream(const TIssues& issues, const TString& expected) { + google::protobuf::RepeatedPtrField<TIssueMessage> protoIssues; + IssuesToMessage(issues, &protoIssues); + TStringBuilder stream; + stream << protoIssues; + UNIT_ASSERT_STRINGS_EQUAL(stream, expected); + }; + + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue("error"); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ message: \"error\" severity: 1 }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ message: \"error\" issue_code: 0 severity: 1 }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); + issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); + TIssues issues; + issues.AddIssue(issue); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 issues { message: \"subsuberror\" severity: 1 } } }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 issues { message: \"subsuberror\" issue_code: 0 severity: 1 } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my message"); + CheckSerializationToStream<Ydb::Issue::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } severity: 1 }"); + CheckSerializationToStream<NYql::NIssue::NProto::IssueMessage>(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } issue_code: 0 severity: 1 }"); + } +} + +Y_UNIT_TEST_SUITE(ToMessage) { + Y_UNIT_TEST(NonUtf8) { + const TString nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + TIssue issue; + issue.SetMessage(nonUtf8String); + + Ydb::Issue::IssueMessage msg; + IssueToMessage(issue, &msg); + TString serialized; + UNIT_ASSERT(msg.SerializeToString(&serialized)); + Ydb::Issue::IssueMessage msg2; + UNIT_ASSERT(msg2.ParseFromString(serialized)); + } +} + +Y_UNIT_TEST_SUITE(EscapeNonUtf8) { + Y_UNIT_TEST(Escape) { + const TString nonUtf8String = "\xfe\xfa\xf5\xc2"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + + // Check that our escaping correctly processes unicode pairs + const TString toNormalize = "Ёлка"; + const TString nfd = WideToUTF8(Normalize<NUnicode::ENormalization::NFD>(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol + const TString nfc = WideToUTF8(Normalize<NUnicode::ENormalization::NFC>(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter + UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); + std::pair<TString, TString> nonUtf8Messages[] = { + { nonUtf8String, "????" }, + { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, + { nfd, nfd }, + { nfc, nfc }, + { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, + { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, + }; + + for (const auto& [src, dst] : nonUtf8Messages) { + TIssue issue(src); + UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); + } + } +} diff --git a/yql/essentials/public/issue/yql_issue_utils.cpp b/yql/essentials/public/issue/yql_issue_utils.cpp new file mode 100644 index 0000000000..2be0b8e57b --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils.cpp @@ -0,0 +1,105 @@ +#include "yql_issue_utils.h" + +#include <util/system/yassert.h> + +#include <tuple> +#include <list> +#include <algorithm> +#include <deque> + +namespace NYql { + +TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) { + // [issue, level, parent, visibleParent] + std::list<std::tuple<const TIssue*, ui32, size_t, size_t>> issueQueue; + // [issue, targetIssue, level, parent, visible, visibleParent, targetSkipIssue] + std::deque<std::tuple<const TIssue*, TIssue*, ui32, size_t, bool, size_t, TIssue*>> issues; + // [depth from bottom, position] + std::list<size_t> leafs; + + const auto depthBeforeLeaf = std::max(opts.KeepTailLevels, ui32(1)) - 1; + const auto maxLevels = std::max(opts.MaxLevels - std::min(opts.MaxLevels, depthBeforeLeaf + 1), ui32(1)); + + issueQueue.emplace_front(&topIssue, 0, 0, 0); + while (!issueQueue.empty()) { + const auto issue = std::get<0>(issueQueue.back()); + const auto level = std::get<1>(issueQueue.back()); + const auto parent = std::get<2>(issueQueue.back()); + const auto visibleParent = std::get<3>(issueQueue.back()); + issueQueue.pop_back(); + + const bool visible = issue->GetSubIssues().empty() || level < maxLevels; + const auto pos = issues.size(); + issues.emplace_back(issue, nullptr, level, parent, visible, visibleParent, nullptr); + if (issue->GetSubIssues().empty()) { + if (level != 0) { + leafs.push_back(pos); + } + } else { + for (auto subIssue : issue->GetSubIssues()) { + issueQueue.emplace_front(subIssue.Get(), level + 1, pos, visible ? pos : visibleParent); + } + } + } + + if (depthBeforeLeaf && !leafs.empty()) { + for (size_t pos: leafs) { + ui32 depth = depthBeforeLeaf; + auto parent = std::get<3>(issues.at(pos)); + while (depth && parent) { + auto& visible = std::get<4>(issues.at(parent)); + auto& visibleParent = std::get<5>(issues.at(pos)); + if (!visible || visibleParent != parent) { + visible = true; + visibleParent = parent; // Update visible parent + --depth; + pos = parent; + parent = std::get<3>(issues.at(parent)); + } else { + break; + } + } + } + } + leafs.clear(); + + TIssue result; + for (auto& i: issues) { + const auto srcIssue = std::get<0>(i); + auto& targetIssue = std::get<1>(i); + const auto level = std::get<2>(i); + const auto parent = std::get<3>(i); + const auto visible = std::get<4>(i); + const auto visibleParent = std::get<5>(i); + + if (0 == level) { + targetIssue = &result; + targetIssue->CopyWithoutSubIssues(*srcIssue); + } else if (visible) { + auto& parentRec = issues.at(visibleParent); + auto& parentTargetIssue = std::get<1>(parentRec); + if (parent != visibleParent) { + auto& parentSkipIssue = std::get<6>(parentRec); + if (!parentSkipIssue) { + const auto parentIssue = std::get<0>(parentRec); + auto newIssue = MakeIntrusive<TIssue>("(skipped levels)"); + newIssue->SetCode(parentIssue->GetCode(), parentIssue->GetSeverity()); + parentTargetIssue->AddSubIssue(newIssue); + parentSkipIssue = newIssue.Get(); + } + auto newIssue = MakeIntrusive<TIssue>(TString{}); + newIssue->CopyWithoutSubIssues(*srcIssue); + parentSkipIssue->AddSubIssue(newIssue); + targetIssue = newIssue.Get(); + } else { + auto newIssue = MakeIntrusive<TIssue>(TString{}); + newIssue->CopyWithoutSubIssues(*srcIssue); + parentTargetIssue->AddSubIssue(newIssue); + targetIssue = newIssue.Get(); + } + } + } + return result; +} + +} // namspace NYql diff --git a/yql/essentials/public/issue/yql_issue_utils.h b/yql/essentials/public/issue/yql_issue_utils.h new file mode 100644 index 0000000000..023ea0edbf --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils.h @@ -0,0 +1,31 @@ +#pragma once + +#include "yql_issue.h" + +#include <util/generic/ylimits.h> +#include <util/system/types.h> + +namespace NYql { + +struct TTruncateIssueOpts { + +#define YQL_TRUNC_DECL_FIELD(type, name, def) \ + TTruncateIssueOpts& Set##name(type arg##name)& { \ + name = arg##name; \ + return *this; \ + } \ + TTruncateIssueOpts&& Set##name(type arg##name)&& { \ + name = arg##name; \ + return std::move(*this); \ + } \ + type name = def; + + YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>()) + YQL_TRUNC_DECL_FIELD(ui32, KeepTailLevels, 1) + +#undef YQL_TRUNC_DECL_FIELD +}; + +TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts = {}); + +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_utils_ut.cpp b/yql/essentials/public/issue/yql_issue_utils_ut.cpp new file mode 100644 index 0000000000..657643a613 --- /dev/null +++ b/yql/essentials/public/issue/yql_issue_utils_ut.cpp @@ -0,0 +1,200 @@ +#include "yql_issue_utils.h" +#include "yql_issue.h" +#include "yql_issue_id.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/subst.h> + +using namespace NYql; + +Y_UNIT_TEST_SUITE(TIssueUtilsTest) { + Y_UNIT_TEST(TruncLevels1) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level30 = MakeIntrusive<TIssue>("level30"); + auto level31 = MakeIntrusive<TIssue>("level31"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level51 = MakeIntrusive<TIssue>("level51"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 -- + * | | + * * 30 * 31 + * | | + * * 40 * 41 + * | + * * 51 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level30); + level2->AddSubIssue(level31); + level30->AddSubIssue(level40); + level31->AddSubIssue(level41); + level41->AddSubIssue(level51); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level30 + <main>: Error: level40 + <main>: Error: level41 + <main>: Error: level51 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level40 + <main>: Error: level51 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(TruncLevels2) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level3 + <main>: Error: level40 + <main>: Error: level41 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(TruncLevels3) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level50 = MakeIntrusive<TIssue>("level50"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 | + * | | + * * 50 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + level40->AddSubIssue(level50); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: level2 + <main>: Error: (skipped levels) + <main>: Error: level41 + <main>: Error: level50 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: level1 + <main>: Error: (skipped levels) + <main>: Error: level3 + <main>: Error: level41 + <main>: Error: level40 + <main>: Error: level50 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString(); + const auto expected = +R"___(<main>: Error: level0 + <main>: Error: (skipped levels) + <main>: Error: level2 + <main>: Error: level3 + <main>: Error: level40 + <main>: Error: level50 + <main>: Error: level41 +)___"; + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } + + Y_UNIT_TEST(KeepSeverity) { + const auto templ = +R"___(<main>: {severity}: level0, code: 1 + <main>: {severity}: level1, code: 1 +)___"; + for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) { + + auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity)); + auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity)); + + level0->AddSubIssue(level1); + + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString(); + const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity)); + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } + } +} diff --git a/yql/essentials/public/issue/yql_warning.cpp b/yql/essentials/public/issue/yql_warning.cpp new file mode 100644 index 0000000000..881e63e95f --- /dev/null +++ b/yql/essentials/public/issue/yql_warning.cpp @@ -0,0 +1,73 @@ +#include "yql_warning.h" + +#include <util/string/cast.h> +#include <util/string/join.h> + + +namespace NYql { + +TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage) +{ + errorMessage.clear(); + result.IssueCodePattern.clear(); + result.Action = EWarningAction::DEFAULT; + + if (!TryFromString<EWarningAction>(to_upper(action), result.Action)) { + errorMessage = "unknown warning action '" + action + "', expecting one of " + + Join(", ", EWarningAction::DEFAULT, EWarningAction::ERROR, EWarningAction::DISABLE); + return EParseResult::PARSE_ACTION_FAIL; + } + + if (codePattern != "*") { + ui32 code; + if (!TryFromString(codePattern, code)) { + errorMessage = "unknown warning code '" + codePattern + "', expecting integer or '*'"; + return EParseResult::PARSE_PATTERN_FAIL; + } + } + + result.IssueCodePattern = codePattern; + return EParseResult::PARSE_OK; +} + +void TWarningPolicy::AddRule(const TWarningRule& rule) +{ + TString pattern = rule.GetPattern(); + if (pattern.empty()) { + return; + } + + Rules.push_back(rule); + + EWarningAction action = rule.GetAction(); + if (pattern == "*") { + BaseAction = action; + Overrides.clear(); + return; + } + + TIssueCode code; + Y_ENSURE(TryFromString(pattern, code)); + + if (action == BaseAction) { + Overrides.erase(Overrides.find(code)); + } else { + Overrides[code] = action; + } +} + +EWarningAction TWarningPolicy::GetAction(TIssueCode code) const +{ + auto it = Overrides.find(code); + return (it == Overrides.end()) ? BaseAction : it->second; +} + +void TWarningPolicy::Clear() +{ + BaseAction = EWarningAction::DEFAULT; + Overrides.clear(); + Rules.clear(); +} + +} diff --git a/yql/essentials/public/issue/yql_warning.h b/yql/essentials/public/issue/yql_warning.h new file mode 100644 index 0000000000..d1d6a90922 --- /dev/null +++ b/yql/essentials/public/issue/yql_warning.h @@ -0,0 +1,51 @@ +#pragma once + +#include "yql_issue_id.h" + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +#if defined(_win_) +#undef ERROR +#endif + +namespace NYql { + +enum class EWarningAction { + DISABLE, + ERROR, + DEFAULT, +}; + +class TWarningRule { +public: + const TString& GetPattern() const { return IssueCodePattern; } + EWarningAction GetAction() const { return Action; } + + enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL }; + static EParseResult ParseFrom(const TString& codePattern, const TString& action, + TWarningRule& result, TString& errorMessage); +private: + TString IssueCodePattern; + EWarningAction Action = EWarningAction::DEFAULT; +}; + +using TWarningRules = TVector<TWarningRule>; + +class TWarningPolicy { +public: + void AddRule(const TWarningRule& rule); + + EWarningAction GetAction(TIssueCode code) const; + + const TWarningRules& GetRules() const { return Rules; } + + void Clear(); + +private: + TWarningRules Rules; + EWarningAction BaseAction = EWarningAction::DEFAULT; + THashMap<TIssueCode, EWarningAction> Overrides; +}; + +} diff --git a/yql/essentials/public/issue/yql_warning_ut.cpp b/yql/essentials/public/issue/yql_warning_ut.cpp new file mode 100644 index 0000000000..5edf7ed363 --- /dev/null +++ b/yql/essentials/public/issue/yql_warning_ut.cpp @@ -0,0 +1,94 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include "yql_warning.h" + +using namespace NYql; + +Y_UNIT_TEST_SUITE(TWarningRuleTest) { + + Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + } + + Y_UNIT_TEST(InvalidActionIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + } + + Y_UNIT_TEST(InvalidPatternIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + } +} + + +void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action) +{ + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom(codePattern, action, rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + policy.AddRule(rule); +} + + +Y_UNIT_TEST_SUITE(TWarningPolicyTest) { + + Y_UNIT_TEST(BasicIntegerRules) { + TWarningPolicy policy; + AddRule(policy, "123", "error"); + AddRule(policy, "456", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); + } + + + Y_UNIT_TEST(AsteriskRules) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + + AddRule(policy, "*", "default"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + + } + + Y_UNIT_TEST(ReactionOnPull) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "default"); + AddRule(policy, "999", "disable"); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + policy.Clear(); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); + + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + } +} diff --git a/yql/essentials/public/ya.make b/yql/essentials/public/ya.make new file mode 100644 index 0000000000..14a283e5a9 --- /dev/null +++ b/yql/essentials/public/ya.make @@ -0,0 +1,4 @@ +RECURSE( + issue +) + diff --git a/yql/essentials/ya.make b/yql/essentials/ya.make index 48f46c7cd8..57841ee7e2 100644 --- a/yql/essentials/ya.make +++ b/yql/essentials/ya.make @@ -1,5 +1,6 @@ SUBSCRIBER(g:yql) RECURSE( + public ) |