additional checkers for canonization

4 files changed, 171 insertions, 58 deletions

diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp
index b57c0b54c9f..94f47188a35 100644
--- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp
+++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp
@@ -78,10 +78,10 @@ private:
     TString ErrorInfo;
 public:
     virtual ~IQueryResultScanner() = default;
-    virtual void OnStart() = 0;
+    virtual void OnStart(const TVector<NYdb::TColumn>& columns) = 0;
     virtual void OnBeforeRow() = 0;
     virtual void OnAfterRow() = 0;
-    virtual void OnRowItem(const NYdb::TValue& value) = 0;
+    virtual void OnRowItem(const NYdb::TColumn& c, const NYdb::TValue& value) = 0;
     virtual void OnFinish() = 0;
     void OnError(const TString& info) {
         ErrorInfo = info;
@@ -91,7 +91,6 @@ public:
     }
 
     bool Scan(NTable::TScanQueryPartIterator& it) {
-        OnStart();
         for (;;) {
             auto streamPart = it.ReadNext().GetValueSync();
             if (!streamPart.IsSuccess()) {
@@ -106,17 +105,18 @@ public:
                 auto result = streamPart.ExtractResultSet();
                 auto columns = result.GetColumnsMeta();
 
+                OnStart(columns);
                 NYdb::TResultSetParser parser(result);
                 while (parser.TryNextRow()) {
                     OnBeforeRow();
                     for (ui32 i = 0; i < columns.size(); ++i) {
-                        OnRowItem(parser.GetValue(i));
+                        OnRowItem(columns[i], parser.GetValue(i));
                     }
                     OnAfterRow();
                 }
+                OnFinish();
             }
         }
-        OnFinish();
         return true;
     }
 };
@@ -129,9 +129,9 @@ public:
         Scanners.emplace_back(scanner);
     }
 
-    virtual void OnStart() override {
+    virtual void OnStart(const TVector<NYdb::TColumn>& columns) override {
         for (auto&& i : Scanners) {
-            i->OnStart();
+            i->OnStart(columns);
         }
     }
     virtual void OnBeforeRow() override {
@@ -144,9 +144,9 @@ public:
             i->OnAfterRow();
         }
     }
-    virtual void OnRowItem(const NYdb::TValue& value) override {
+    virtual void OnRowItem(const NYdb::TColumn& c, const NYdb::TValue& value) override {
         for (auto&& i : Scanners) {
-            i->OnRowItem(value);
+            i->OnRowItem(c, value);
         }
     }
     virtual void OnFinish() override {
@@ -167,7 +167,7 @@ public:
         Writer.reset();
         return ResultString.Str();
     }
-    virtual void OnStart() override {
+    virtual void OnStart(const TVector<NYdb::TColumn>& /*columns*/) override {
         Writer = std::make_unique<NYson::TYsonWriter>(&ResultString, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true);
         Writer->OnBeginList();
     }
@@ -178,7 +178,7 @@ public:
     virtual void OnAfterRow() override {
         Writer->OnEndList();
     }
-    virtual void OnRowItem(const NYdb::TValue& value) override {
+    virtual void OnRowItem(const NYdb::TColumn& /*c*/, const NYdb::TValue& value) override {
         Writer->OnListItem();
         FormatValueYson(value, *Writer);
     }
@@ -187,34 +187,20 @@ public:
     }
 };
 
-class TCSVResultScanner: public IQueryResultScanner {
-private:
-    TStringStream ResultString;
-    bool IsFirstInRow = false;
-    bool IsFirstRow = true;
+class TCSVResultScanner: public IQueryResultScanner, public TQueryResultInfo {
 public:
     TCSVResultScanner() {
     }
-    TString GetResult() const {
-        return ResultString.Str();
-    }
-    virtual void OnStart() override {
+    virtual void OnStart(const TVector<NYdb::TColumn>& columns) override {
+        Columns = columns;
     }
     virtual void OnBeforeRow() override {
-        if (!IsFirstRow) {
-            ResultString << "\n";
-            IsFirstRow = false;
-        }
-        IsFirstInRow = true;
+        Result.emplace_back(std::vector<NYdb::TValue>());
     }
     virtual void OnAfterRow() override {
     }
-    virtual void OnRowItem(const NYdb::TValue& value) override {
-        if (!IsFirstInRow) {
-            ResultString << ",";
-            IsFirstInRow = false;
-        }
-        ResultString << FormatValueYson(value);
+    virtual void OnRowItem(const NYdb::TColumn& /*c*/, const NYdb::TValue& value) override {
+        Result.back().emplace_back(value);
     }
     virtual void OnFinish() override {
     }
@@ -234,7 +220,7 @@ TQueryBenchmarkResult Execute(const TString& query, NTable::TTableClient& client
     if (!composite.Scan(it)) {
         return TQueryBenchmarkResult::Error(composite.GetErrorInfo());
     } else {
-        return TQueryBenchmarkResult::Result(scannerYson->GetResult(), scannerCSV->GetResult());
+        return TQueryBenchmarkResult::Result(scannerYson->GetResult(), *scannerCSV);
     }
 }
 
diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h
index 8154589ea47..afd1d28b6d6 100644
--- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h
+++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h
@@ -18,17 +18,39 @@ struct TTestInfo {
     explicit TTestInfo(std::vector<TDuration>&& timings);
 };
 
+class TQueryResultInfo {
+protected:
+    std::vector<std::vector<NYdb::TValue>> Result;
+    TVector<NYdb::TColumn> Columns;
+public:
+    std::map<TString, ui32> GetColumnsRemap() const {
+        std::map<TString, ui32> result;
+        ui32 idx = 0;
+        for (auto&& i : Columns) {
+            result.emplace(i.Name, idx++);
+        }
+        return result;
+    }
+
+    const std::vector<std::vector<NYdb::TValue>>& GetResult() const {
+        return Result;
+    }
+    const TVector<NYdb::TColumn>& GetColumns() const {
+        return Columns;
+    }
+};
+
 class TQueryBenchmarkResult {
 private:
     TString ErrorInfo;
     TString YSONResult;
-    TString CSVResult;
+    TQueryResultInfo QueryResult;
     TQueryBenchmarkResult() = default;
 public:
-    static TQueryBenchmarkResult Result(const TString& yson, const TString& csv) {
+    static TQueryBenchmarkResult Result(const TString& yson, const TQueryResultInfo& queryResult) {
         TQueryBenchmarkResult result;
         result.YSONResult = yson;
-        result.CSVResult = csv;
+        result.QueryResult = queryResult;
         return result;
     }
     static TQueryBenchmarkResult Error(const TString& error) {
@@ -45,8 +67,8 @@ public:
     const TString& GetYSONResult() const {
         return YSONResult;
     }
-    const TString& GetCSVResult() const {
-        return CSVResult;
+    const TQueryResultInfo& GetQueryResult() const {
+        return QueryResult;
     }
 };
 
diff --git a/ydb/public/lib/ydb_cli/commands/click_bench.cpp b/ydb/public/lib/ydb_cli/commands/click_bench.cpp
index 7bfe5fc43d1..2bac0071992 100644
--- a/ydb/public/lib/ydb_cli/commands/click_bench.cpp
+++ b/ydb/public/lib/ydb_cli/commands/click_bench.cpp
@@ -63,6 +63,103 @@ public:
 
 }
 
+bool TClickBenchCommandRun::TQueryFullInfo::CompareValue(const NYdb::TValue& v, const TStringBuf vExpected) const {
+    const auto& vp = v.GetProto();
+    if (vp.has_bool_value()) {
+        return CompareValueImpl<bool>(vp.bool_value(), vExpected);
+    }
+    if (vp.has_int32_value()) {
+        return CompareValueImpl<i32>(vp.int32_value(), vExpected);
+    }
+    if (vp.has_uint32_value()) {
+        return CompareValueImpl<ui32>(vp.uint32_value(), vExpected);
+    }
+    if (vp.has_int64_value()) {
+        return CompareValueImpl<i64>(vp.int64_value(), vExpected);
+    }
+    if (vp.has_uint64_value()) {
+        return CompareValueImpl<ui64>(vp.uint64_value(), vExpected);
+    }
+    if (vp.has_float_value()) {
+        return CompareValueImpl<float>(vp.float_value(), vExpected);
+    }
+    if (vp.has_double_value()) {
+        return CompareValueImpl<double>(vp.double_value(), vExpected);
+    }
+    if (vp.has_text_value()) {
+        return CompareValueImpl<TString>(TString(vp.text_value().data(), vp.text_value().size()), vExpected);
+    }
+    if (vp.has_null_flag_value()) {
+        return vExpected == "";
+    }
+    Cerr << "unexpected type for comparision: " << vp.DebugString() << Endl;
+    return false;
+}
+
+bool TClickBenchCommandRun::TQueryFullInfo::IsCorrectResult(const BenchmarkUtils::TQueryResultInfo& resultFull) const {
+    if (!ExpectedResult) {
+        return true;
+    }
+    const auto expectedLines = StringSplitter(ExpectedResult).Split('\n').SkipEmpty().ToList<TString>();
+    auto& result = resultFull.GetResult();
+    if (result.size() + 1 != expectedLines.size()) {
+        Cerr << "has diff: incorrect lines count (" << result.size() << " in result, but " << expectedLines.size() << " expected with header)" << Endl;
+        return false;
+    }
+
+    std::vector<ui32> columnIndexes;
+    {
+        const std::map<TString, ui32> columns = resultFull.GetColumnsRemap();
+        auto copy = expectedLines.front();
+        NCsvFormat::CsvSplitter splitter(copy);
+        while (true) {
+            auto cName = splitter.Consume();
+            auto it = columns.find(TString(cName.data(), cName.size()));
+            if (it == columns.end()) {
+                columnIndexes.clear();
+                for (ui32 i = 0; i < columns.size(); ++i) {
+                    columnIndexes.emplace_back(i);
+                }
+                break;
+            } else {
+                columnIndexes.emplace_back(it->second);
+            }
+
+            if (!splitter.Step()) {
+                break;
+            }
+        }
+        if (columnIndexes.size() != columns.size()) {
+            Cerr << "there are unexpected columns in result" << Endl;
+            return false;
+        }
+    }
+
+    for (ui32 i = 0; i < result.size(); ++i) {
+        TString copy = expectedLines[i + 1];
+        NCsvFormat::CsvSplitter splitter(copy);
+        bool isCorrectCurrent = true;
+        for (ui32 cIdx = 0; cIdx < columnIndexes.size(); ++cIdx) {
+            const NYdb::TValue& resultValue = result[i][columnIndexes[cIdx]];
+            if (!isCorrectCurrent) {
+                Cerr << "has diff: no element in expectation" << Endl;
+                return false;
+            }
+            TStringBuf cItem = splitter.Consume();
+            if (!CompareValue(resultValue, cItem)) {
+                Cerr << "has diff: " << resultValue.GetProto().DebugString() << ";EXPECTED:" << cItem << Endl;
+                return false;
+            }
+            isCorrectCurrent = splitter.Step();
+        }
+        if (isCorrectCurrent) {
+            Cerr << "expected more items than have in result" << Endl;
+            return false;
+        }
+    }
+    return true;
+}
+
 TVector<TClickBenchCommandRun::TQueryFullInfo> TClickBenchCommandRun::GetQueries(const TString& fullTablePath) const {
     TVector<TString> queries;
     const TMap<ui32, TString> qResults = LoadExternalResults();
@@ -152,6 +249,8 @@ bool TClickBenchCommandRun::RunBench(TConfig& config)
         Cerr << query << Endl << Endl;
 
         ui32 successIteration = 0;
+        ui32 failsCount = 0;
+        ui32 diffsCount = 0;
         std::optional<TString> prevResult;
         for (ui32 i = 0; i < IterationsCount * 10 && successIteration < IterationsCount; ++i) {
             auto t1 = TInstant::Now();
@@ -172,13 +271,18 @@ bool TClickBenchCommandRun::RunBench(TConfig& config)
                     outFStream << queryN << ": " << Endl
                         << res.GetYSONResult() << Endl << Endl;
                 }
-                if ((!prevResult || *prevResult != res.GetCSVResult()) && !qInfo.IsCorrectResult(res.GetCSVResult())) {
-                    outFStream << queryN << ": UNEXPECTED DIFF: " << Endl
-                        << "RESULT: " << Endl << res.GetCSVResult() << Endl
-                        << "EXPECTATION: " << Endl << qInfo.GetExpectedResult() << Endl;
-                    prevResult = res.GetCSVResult();
+                if ((!prevResult || *prevResult != res.GetYSONResult()) && !qInfo.IsCorrectResult(res.GetQueryResult())) {
+                    outFStream << queryN << ":" << Endl <<
+                        "Query text:" << Endl <<
+                        query << Endl << Endl <<
+                        "UNEXPECTED DIFF: " << Endl
+                            << "RESULT: " << Endl << res.GetYSONResult() << Endl
+                            << "EXPECTATION: " << Endl << qInfo.GetExpectedResult() << Endl;
+                    prevResult = res.GetYSONResult();
+                    ++diffsCount;
                 }
             } else {
+                ++failsCount;
                 Cout << "failed\t" << duration << " seconds" << Endl;
                 Cerr << queryN << ": " << query << Endl
                     << res.GetErrorInfo() << Endl;
@@ -203,6 +307,8 @@ bool TClickBenchCommandRun::RunBench(TConfig& config)
             jsonReport.AppendValue(GetSensorValue("Max", testInfo.Max, queryN));
             jsonReport.AppendValue(GetSensorValue("Mean", testInfo.Mean, queryN));
             jsonReport.AppendValue(GetSensorValue("Std", testInfo.Std, queryN));
+            jsonReport.AppendValue(GetSensorValue("DiffsCount", diffsCount, queryN));
+            jsonReport.AppendValue(GetSensorValue("FailsCount", failsCount, queryN));
         }
     }
 
diff --git a/ydb/public/lib/ydb_cli/commands/click_bench.h b/ydb/public/lib/ydb_cli/commands/click_bench.h
index 8a4ef19ffd9..ee2939d708a 100644
--- a/ydb/public/lib/ydb_cli/commands/click_bench.h
+++ b/ydb/public/lib/ydb_cli/commands/click_bench.h
@@ -1,8 +1,10 @@
 #pragma once
 
+#include <library/cpp/string_utils/csv/csv.h>
 #include <util/generic/set.h>
 
 #include "ydb_command.h"
+#include "benchmark_utils.h"
 
 namespace NYdb::NConsoleClient {
 
@@ -51,6 +53,18 @@ public:
     private:
         TString Query;
         TString ExpectedResult;
+
+        template <class T>
+        bool CompareValueImpl(const T valResult, const TStringBuf vExpected) const {
+            T valExpected;
+            if (!TryFromString<T>(vExpected, valExpected)) {
+                Cerr << "cannot parse expected as " << typeid(valResult).name() << "(" << vExpected << ")" << Endl;
+                return false;
+            }
+            return valResult == valExpected;
+        }
+
+        bool CompareValue(const NYdb::TValue& v, const TStringBuf vExpected) const;
     public:
         TQueryFullInfo(const TString& query, const TString& expectedResult)
             : Query(query)
@@ -59,22 +73,7 @@ public:
 
         }
 
-        bool IsCorrectResult(const TString& result) const {
-            if (!ExpectedResult) {
-                return true;
-            }
-            const auto resultLines = StringSplitter(result).Split('\n').SkipEmpty().ToList<TString>();
-            const auto expectedLines = StringSplitter(ExpectedResult).Split('\n').SkipEmpty().ToList<TString>();
-            if (resultLines.size() + 1 != expectedLines.size()) {
-                return false;
-            }
-            for (ui32 i = 0; i < resultLines.size(); ++i) {
-                if (expectedLines[i + 1] != resultLines[i]) {
-                    return false;
-                }
-            }
-            return true;
-        }
+        bool IsCorrectResult(const BenchmarkUtils::TQueryResultInfo& result) const;
 
         const TString& GetQuery() const {
             return Query;