limits and debug information has been improved

10 files changed, 241 insertions, 9 deletions

diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt
index ed5ff00d81..7195d142d5 100644
--- a/CMakeLists.darwin.txt
+++ b/CMakeLists.darwin.txt
@@ -759,6 +759,7 @@ add_subdirectory(ydb/public/sdk/cpp/client/ydb_coordination)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_common_client/impl)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_rate_limiter)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_scheme)
+add_subdirectory(ydb/library/protobuf_printer)
 add_subdirectory(ydb/core/yq/libs/db_schema)
 add_subdirectory(ydb/library/yql/providers/s3/path_generator)
 add_subdirectory(ydb/core/yq/libs/grpc)
@@ -823,7 +824,6 @@ add_subdirectory(ydb/core/ymq/base)
 add_subdirectory(ydb/core/ymq/proto)
 add_subdirectory(ydb/library/http_proxy/authorization)
 add_subdirectory(ydb/library/http_proxy/error)
-add_subdirectory(ydb/library/protobuf_printer)
 add_subdirectory(ydb/core/ymq/queues/common)
 add_subdirectory(ydb/core/ymq/queues/fifo)
 add_subdirectory(ydb/core/ymq/queues/std)
diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt
index 7709d6ef8b..d8532e17e8 100644
--- a/CMakeLists.linux.txt
+++ b/CMakeLists.linux.txt
@@ -763,6 +763,7 @@ add_subdirectory(ydb/public/sdk/cpp/client/ydb_coordination)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_common_client/impl)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_rate_limiter)
 add_subdirectory(ydb/public/sdk/cpp/client/ydb_scheme)
+add_subdirectory(ydb/library/protobuf_printer)
 add_subdirectory(ydb/core/yq/libs/db_schema)
 add_subdirectory(ydb/library/yql/providers/s3/path_generator)
 add_subdirectory(ydb/core/yq/libs/grpc)
@@ -827,7 +828,6 @@ add_subdirectory(ydb/core/ymq/base)
 add_subdirectory(ydb/core/ymq/proto)
 add_subdirectory(ydb/library/http_proxy/authorization)
 add_subdirectory(ydb/library/http_proxy/error)
-add_subdirectory(ydb/library/protobuf_printer)
 add_subdirectory(ydb/core/ymq/queues/common)
 add_subdirectory(ydb/core/ymq/queues/fifo)
 add_subdirectory(ydb/core/ymq/queues/std)
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.txt b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.txt
index 97b1f6c32d..0302cfa6aa 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.txt
+++ b/ydb/core/yq/libs/control_plane_storage/internal/CMakeLists.txt
@@ -29,6 +29,7 @@ target_link_libraries(libs-control_plane_storage-internal PUBLIC
   ydb-library-security
   cpp-client-ydb_scheme
   cpp-client-ydb_value
+  ydb-library-protobuf_printer
   yql-public-issue
 )
 target_sources(libs-control_plane_storage-internal PRIVATE
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
index 417d8ec78c..41473835ed 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
@@ -3,6 +3,7 @@
 #include <util/datetime/base.h>
 
 #include <ydb/core/yq/libs/db_schema/db_schema.h>
+#include <ydb/library/protobuf_printer/size_printer.h>
 
 #include <google/protobuf/util/time_util.h>
 
@@ -21,8 +22,8 @@ bool IsFinishedStatus(YandexQuery::QueryMeta::ComputeStatus status) {
 
 std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructHardPingTask(
     const Fq::Private::PingTaskRequest& request, std::shared_ptr<Fq::Private::PingTaskResult> response,
-    const TString& tablePathPrefix, const TDuration& automaticQueriesTtl, const TDuration& taskLeaseTtl, const THashMap<ui64, TRetryPolicyItem>& retryPolicies,
-    ::NMonitoring::TDynamicCounterPtr rootCounters) {
+    const TString& tablePathPrefix, const TDuration& automaticQueriesTtl, const TDuration& taskLeaseTtl,
+    const THashMap<ui64, TRetryPolicyItem>& retryPolicies, ::NMonitoring::TDynamicCounterPtr rootCounters, uint64_t maxRequestSize) {
 
     auto scope = request.scope();
     auto query_id = request.query_id().value();
@@ -269,6 +270,18 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
             internal.set_dq_graph_index(request.dq_graph_index());
         }
 
+        if (job.ByteSizeLong() > maxRequestSize) {
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Job proto exceeded the size limit: " << job.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(job).ToString();
+        }
+
+        if (query.ByteSizeLong() > maxRequestSize) {
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query proto exceeded the size limit: " << query.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(query).ToString();
+        }
+
+        if (internal.ByteSizeLong() > maxRequestSize) {
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "QueryInternal proto exceeded the size limit: " << internal.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(internal).ToString();
+        }
+
         TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)");
         writeQueryBuilder.AddString("tenant", request.tenant());
         writeQueryBuilder.AddString("scope", request.scope());
@@ -447,7 +460,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskReq
     if (request.status())
         Counters.GetFinalStatusCounters(cloudId, scope)->IncByStatus(request.status());
     auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ?
-        ConstructHardPingTask(request, response, YdbConnection->TablePathPrefix, Config.AutomaticQueriesTtl, Config.TaskLeaseTtl, Config.RetryPolicies, Counters.Counters) :
+        ConstructHardPingTask(request, response, YdbConnection->TablePathPrefix, Config.AutomaticQueriesTtl, Config.TaskLeaseTtl, Config.RetryPolicies, Counters.Counters, Config.Proto.GetMaxRequestSize()) :
         ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix, Config.TaskLeaseTtl);
     auto readQuery = std::get<0>(pingTaskParams); // Use std::get for win compiler
     auto readParams = std::get<1>(pingTaskParams);
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
index 28ad0d70e6..a8be5b0cb4 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
@@ -275,11 +275,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery
         }
 
         if (query.ByteSizeLong() > Config.Proto.GetMaxRequestSize()) {
-            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query data is not placed in the table. Please shorten your request";
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Incoming request exceeded the size limit: " << query.ByteSizeLong() << " of " << Config.Proto.GetMaxRequestSize() <<  ". Please shorten your request";
         }
 
         if (queryInternal.ByteSizeLong() > Config.Proto.GetMaxRequestSize()) {
-            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query internal data is not placed in the table. Please reduce the number of connections and bindings";
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "The size of all connections and bindings in the project exceeded the limit: " << queryInternal.ByteSizeLong() << " of " << Config.Proto.GetMaxRequestSize() << ". Please reduce the number of connections and bindings";
         }
 
         response->second.After.ConstructInPlace().CopyFrom(query);
@@ -604,6 +604,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue
         if (internal.plan_compressed().data()) { // todo: remove this if after migration
             TCompressor compressor(internal.plan_compressed().method());
             result.mutable_query()->mutable_plan()->set_json(compressor.Decompress(internal.plan_compressed().data()));
+            if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) {
+                if (result.query().plan().json().size() > 1000) {
+                    // modifing plan this way should definitely reduce query msg size
+                    result.mutable_query()->mutable_plan()->set_json(TStringBuilder() << "Message is too big: " << result.query().ByteSizeLong() << " bytes, dropping plan of size " << result.query().plan().json().size() << " bytes");
+                }
+            }
         }
         if (!permissions.Check(TPermissions::VIEW_AST)) {
             result.mutable_query()->clear_ast();
@@ -923,11 +929,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
         }
 
         if (query.ByteSizeLong() > Config.Proto.GetMaxRequestSize()) {
-            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query data is not placed in the table. Please shorten your request";
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Incoming request exceeded the size limit: " << query.ByteSizeLong() << " of " << Config.Proto.GetMaxRequestSize() <<  ". Please shorten your request";
         }
 
         if (internal.ByteSizeLong() > Config.Proto.GetMaxRequestSize()) {
-            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Internal data is not placed in the table. Please reduce the number of connections and bindings";
+            ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "The size of all connections and bindings in the project exceeded the limit: " << internal.ByteSizeLong() << " of " << Config.Proto.GetMaxRequestSize() << ". Please reduce the number of connections and bindings";
         }
 
         YandexQuery::Job job;
diff --git a/ydb/library/protobuf_printer/CMakeLists.txt b/ydb/library/protobuf_printer/CMakeLists.txt
index 081c1bc36f..4fcb0bc301 100644
--- a/ydb/library/protobuf_printer/CMakeLists.txt
+++ b/ydb/library/protobuf_printer/CMakeLists.txt
@@ -17,6 +17,7 @@ target_link_libraries(ydb-library-protobuf_printer PUBLIC
 )
 target_sources(ydb-library-protobuf_printer PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/protobuf_printer/hide_field_printer.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/protobuf_printer/size_printer.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/protobuf_printer/stream_helper.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/protobuf_printer/token_field_printer.cpp
 )
diff --git a/ydb/library/protobuf_printer/protobuf_printer_ut.cpp b/ydb/library/protobuf_printer/protobuf_printer_ut.cpp
index f71c330d46..38d098b817 100644
--- a/ydb/library/protobuf_printer/protobuf_printer_ut.cpp
+++ b/ydb/library/protobuf_printer/protobuf_printer_ut.cpp
@@ -1,6 +1,7 @@
 #include "hide_field_printer.h"
 #include "protobuf_printer.h"
 #include "security_printer.h"
+#include "size_printer.h"
 #include "stream_helper.h"
 #include "token_field_printer.h"
 #include <ydb/library/protobuf_printer/ut/test_proto.pb.h>
@@ -91,3 +92,29 @@ Y_UNIT_TEST_SUITE(SecurityPrinterTest) {
         }
     }
 }
+
+Y_UNIT_TEST_SUITE(FieldSizePrinterTest) {
+    Y_UNIT_TEST(PrintSuccess) {
+        NTestProto::TBigObject bo;
+        (*bo.mutable_map())["a1"] = "a";
+        (*bo.mutable_map())["a2"] = "a2";
+        (*bo.mutable_map())["a3"] = "a3";
+        bo.add_list("b");
+        bo.add_list("bb");
+        bo.add_list("bbb");
+        (*bo.mutable_object()->mutable_inner_map())["t"] = "1";
+        bo.mutable_object()->set_value(2);
+        bo.mutable_object()->mutable_inner()->add_a("aba");
+        bo.mutable_object()->mutable_inner()->add_a("caba");
+        UNIT_ASSERT_STRINGS_EQUAL(TSizeFormatPrinter(bo).ToString(), "map: 23 bytes list: 9 bytes object { inner_map: 6 bytes value: 8 bytes inner { a: 9 bytes } } ");
+    }
+
+    Y_UNIT_TEST(PrintRecursiveType) {
+        NTestProto::TRecursiveType response;
+        response.set_name("name1");
+        response.set_login("login1");
+        response.add_types()->set_login("login2");
+        response.add_types()->set_name("name3");
+        UNIT_ASSERT_STRINGS_EQUAL(TSizeFormatPrinter(response).ToString(), "name: 6 bytes login: 7 bytes types: 15 bytes ");
+    }
+}
diff --git a/ydb/library/protobuf_printer/size_printer.cpp b/ydb/library/protobuf_printer/size_printer.cpp
new file mode 100644
index 0000000000..e68adc18b7
--- /dev/null
+++ b/ydb/library/protobuf_printer/size_printer.cpp
@@ -0,0 +1,141 @@
+#include "size_printer.h"
+
+#include <contrib/libs/protobuf/src/google/protobuf/descriptor.pb.h>
+
+namespace NKikimr {
+
+namespace {
+
+struct FieldIndexSorter {
+    bool operator()(const google::protobuf::FieldDescriptor* left, const google::protobuf::FieldDescriptor* right) const {
+        if (left->is_extension() && right->is_extension()) {
+            return left->number() < right->number();
+        } else if (left->is_extension()) {
+            return false;
+        } else if (right->is_extension()) {
+            return true;
+        } else {
+            return left->index() < right->index();
+        }
+    }
+};
+
+}
+
+TSizeFormatPrinter::TSizeFormatPrinter(google::protobuf::Message& message)
+    : Message(message)
+{}
+
+TString TSizeFormatPrinter::ToString() const {
+    TStringBuilder builder;
+    PrintMessage(Message, builder);
+    return builder;
+}
+
+void TSizeFormatPrinter::PrintMessage(const google::protobuf::Message& message, TStringBuilder& builder) const {
+    const google::protobuf::Reflection* reflection = message.GetReflection();
+    if (!reflection) {
+        return;
+    }
+    const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
+    std::vector<const google::protobuf::FieldDescriptor*> fields;
+    if (!descriptor->options().map_entry()) {
+        reflection->ListFields(message, &fields);
+    }
+    std::sort(fields.begin(), fields.end(), FieldIndexSorter());
+    for (const google::protobuf::FieldDescriptor* field : fields) {
+        PrintField(message, reflection, field, builder);
+    }
+}
+
+void TSizeFormatPrinter::PrintFieldName(const google::protobuf::FieldDescriptor* field, TStringBuilder& builder) const {
+    if (field->is_extension()) {
+        builder << "[" << field->PrintableNameForExtension() << "]";
+    } else if (field->type() == google::protobuf::FieldDescriptor::TYPE_GROUP) {
+        // Groups must be serialized with their original capitalization.
+        builder << field->message_type()->name();
+    } else {
+        builder << field->name();
+    }
+}
+
+size_t TSizeFormatPrinter::RepeatedByteSizeLong(const google::protobuf::Message& message, const google::protobuf::Reflection* reflection, const google::protobuf::FieldDescriptor* field) const {
+    const int fieldSize = reflection->FieldSize(message, field);
+    switch (field->cpp_type()) {
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_FLOAT:
+            return fieldSize * sizeof(float);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_UINT32:
+            return fieldSize * sizeof(uint32_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_INT32:
+            return fieldSize * sizeof(int32_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_DOUBLE:
+            return fieldSize * sizeof(double);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_UINT64:
+            return fieldSize * sizeof(uint64_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_INT64:
+            return fieldSize * sizeof(int64_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_STRING: {
+            size_t size = 0;
+            for (int i = 0; i < fieldSize; i++) {
+                size += reflection->GetRepeatedString(message, field, i).Size() + 1 /* null terminated */;
+            }
+            return size;
+        }
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_BOOL:
+            return fieldSize * sizeof(bool);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_ENUM:
+            return fieldSize * sizeof(int);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_MESSAGE: {
+            size_t size = 0;
+            for (int i = 0; i < fieldSize; i++) {
+                size += reflection->GetRepeatedMessage(message, field, i).ByteSizeLong();
+            }
+            return size;
+        }
+    }
+}
+
+size_t TSizeFormatPrinter::ItemByteSizeLong(const google::protobuf::Message& message, const google::protobuf::Reflection* reflection, const google::protobuf::FieldDescriptor* field) const {
+    switch (field->cpp_type()) {
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_FLOAT:
+            return sizeof(float);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_UINT32:
+            return sizeof(uint32_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_INT32:
+            return sizeof(int32_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_DOUBLE:
+            return sizeof(double);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_UINT64:
+            return sizeof(uint64_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_INT64:
+            return sizeof(int64_t);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_STRING:
+            return reflection->GetString(message, field).Size() + 1 /* null terminated */;
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_BOOL:
+            return sizeof(bool);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_ENUM:
+            return sizeof(int);
+        case google::protobuf::FieldDescriptor::CppType::CPPTYPE_MESSAGE:
+            return message.ByteSizeLong();
+    }
+}
+
+void TSizeFormatPrinter::PrintField(const google::protobuf::Message& message,
+                const google::protobuf::Reflection* reflection,
+                const google::protobuf::FieldDescriptor* field,
+                TStringBuilder& builder) const {
+    PrintFieldName(field, builder);
+    if (field->is_map()) {
+        builder << ": " << RepeatedByteSizeLong(message, reflection, field) << " bytes ";
+    } else if (field->is_repeated()) {
+        builder << ": " << RepeatedByteSizeLong(message, reflection, field) << " bytes ";
+    } else if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
+        const google::protobuf::Message& subMessage = reflection->GetMessage(message, field);
+        builder << " { ";
+        PrintMessage(subMessage, builder);
+        builder << "} ";
+    } else {
+        builder << ": " << ItemByteSizeLong(message, reflection, field) << " bytes ";
+    }
+}
+}
diff --git a/ydb/library/protobuf_printer/size_printer.h b/ydb/library/protobuf_printer/size_printer.h
new file mode 100644
index 0000000000..3026db2867
--- /dev/null
+++ b/ydb/library/protobuf_printer/size_printer.h
@@ -0,0 +1,27 @@
+#pragma once
+
+#include <google/protobuf/message.h>
+
+#include <util/string/builder.h>
+
+namespace NKikimr {
+
+class TSizeFormatPrinter {
+    const google::protobuf::Message& Message;
+
+public:
+    TSizeFormatPrinter(google::protobuf::Message& message);
+    TString ToString() const;
+
+private:
+    void PrintMessage(const google::protobuf::Message& message, TStringBuilder& builder) const;
+    void PrintFieldName(const google::protobuf::FieldDescriptor* field, TStringBuilder& builder) const;
+    size_t RepeatedByteSizeLong(const google::protobuf::Message& message, const google::protobuf::Reflection* reflection, const google::protobuf::FieldDescriptor* field) const;
+    size_t ItemByteSizeLong(const google::protobuf::Message& message, const google::protobuf::Reflection* reflection, const google::protobuf::FieldDescriptor* field) const;
+    void PrintField(const google::protobuf::Message& message,
+                    const google::protobuf::Reflection* reflection,
+                    const google::protobuf::FieldDescriptor* field,
+                    TStringBuilder& builder) const;
+};
+
+} // namespace NKikimr
diff --git a/ydb/library/protobuf_printer/ut/test_proto.proto b/ydb/library/protobuf_printer/ut/test_proto.proto
index 80bf23c804..678fc22f1c 100644
--- a/ydb/library/protobuf_printer/ut/test_proto.proto
+++ b/ydb/library/protobuf_printer/ut/test_proto.proto
@@ -44,3 +44,19 @@ message TRecursiveType {
     string login = 2 [(Ydb.sensitive) = true];
     repeated TRecursiveType types = 3;
 }
+
+message TInnerObject2 {
+    repeated string a = 1;
+}
+
+message TInnerObject {
+    map<string, string> inner_map = 1;
+    uint64 value = 2;
+    TInnerObject2 inner = 3;
+}
+
+message TBigObject {
+    map<string, string> map = 1;
+    repeated string list = 2;
+    TInnerObject object = 3;
+}