feat(data_integrity_trails): add configurable log modes for query_text and pk (#12732)

8 files changed, 64 insertions, 13 deletions

diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp
index 469ecc921ed..4bee1034dfe 100644
--- a/ydb/core/base/appdata.cpp
+++ b/ydb/core/base/appdata.cpp
@@ -19,6 +19,7 @@
 #include <ydb/core/protos/blobstorage.pb.h>
 #include <ydb/core/protos/cms.pb.h>
 #include <ydb/core/protos/config.pb.h>
+#include <ydb/core/protos/data_integrity_trails.pb.h>
 #include <ydb/core/protos/key.pb.h>
 #include <ydb/core/protos/memory_controller_config.pb.h>
 #include <ydb/core/protos/pqconfig.pb.h>
@@ -68,6 +69,7 @@ struct TAppData::TImpl {
     NKikimrConfig::TMetadataCacheConfig MetadataCacheConfig;
     NKikimrConfig::TMemoryControllerConfig MemoryControllerConfig;
     NKikimrReplication::TReplicationDefaults ReplicationConfig;
+    NKikimrProto::TDataIntegrityTrailsConfig DataIntegrityTrailsConfig;
 };
 
 TAppData::TAppData(
@@ -123,6 +125,7 @@ TAppData::TAppData(
     , MetadataCacheConfig(Impl->MetadataCacheConfig)
     , MemoryControllerConfig(Impl->MemoryControllerConfig)
     , ReplicationConfig(Impl->ReplicationConfig)
+    , DataIntegrityTrailsConfig(Impl->DataIntegrityTrailsConfig)
     , KikimrShouldContinue(kikimrShouldContinue)
     , TracingConfigurator(MakeIntrusive<NJaegerTracing::TSamplingThrottlingConfigurator>(TimeProvider, RandomProvider))
 {}
diff --git a/ydb/core/base/appdata_fwd.h b/ydb/core/base/appdata_fwd.h
index c88f82f2af7..ae2bc9ab9fa 100644
--- a/ydb/core/base/appdata_fwd.h
+++ b/ydb/core/base/appdata_fwd.h
@@ -38,6 +38,7 @@ namespace NKikimrSharedCache {
 namespace NKikimrProto {
     class TKeyConfig;
     class TAuthConfig;
+    class TDataIntegrityTrailsConfig;
 
     namespace NFolderService {
         class TFolderServiceConfig;
@@ -228,6 +229,7 @@ struct TAppData {
     NKikimrConfig::TMetadataCacheConfig& MetadataCacheConfig;
     NKikimrConfig::TMemoryControllerConfig& MemoryControllerConfig;
     NKikimrReplication::TReplicationDefaults& ReplicationConfig;
+    NKikimrProto::TDataIntegrityTrailsConfig& DataIntegrityTrailsConfig;
     bool EnforceUserTokenRequirement = false;
     bool EnforceUserTokenCheckRequirement = false; // check token if it was specified
     bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request
diff --git a/ydb/core/data_integrity_trails/data_integrity_trails.h b/ydb/core/data_integrity_trails/data_integrity_trails.h
index 7c4f41f726d..003505d5dc1 100644
--- a/ydb/core/data_integrity_trails/data_integrity_trails.h
+++ b/ydb/core/data_integrity_trails/data_integrity_trails.h
@@ -2,6 +2,8 @@
 
 #include <util/stream/str.h>
 
+#include <ydb/core/protos/data_integrity_trails.pb.h>
+
 namespace NKikimr {
 namespace NDataIntegrity {
 
diff --git a/ydb/core/kqp/common/kqp_data_integrity_trails.h b/ydb/core/kqp/common/kqp_data_integrity_trails.h
index d7b514c7786..37c73bc4eff 100644
--- a/ydb/core/kqp/common/kqp_data_integrity_trails.h
+++ b/ydb/core/kqp/common/kqp_data_integrity_trails.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <openssl/sha.h>
+#include <ydb/core/base/appdata.h>
 #include <library/cpp/string_utils/base64/base64.h>
 
 #include <ydb/core/data_integrity_trails/data_integrity_trails.h>
@@ -45,6 +46,22 @@ inline void LogIntegrityTrails(const NKqp::TEvKqp::TEvQueryRequest::TPtr& reques
         LogKeyValue("QueryAction", ToString(request->Get()->GetAction()), ss);
         LogKeyValue("QueryType", ToString(request->Get()->GetType()), ss);
 
+        const auto queryTextLogMode = AppData()->DataIntegrityTrailsConfig.HasQueryTextLogMode()
+            ? AppData()->DataIntegrityTrailsConfig.GetQueryTextLogMode()
+            : NKikimrProto::TDataIntegrityTrailsConfig_ELogMode_HASHED;
+        if (queryTextLogMode == NKikimrProto::TDataIntegrityTrailsConfig_ELogMode_ORIGINAL) {
+            LogKeyValue("QueryText", request->Get()->GetQuery(), ss);
+        } else {
+            std::string hashedQueryText;
+            hashedQueryText.resize(SHA256_DIGEST_LENGTH);
+
+            SHA256_CTX sha256;
+            SHA256_Init(&sha256);
+            SHA256_Update(&sha256, request->Get()->GetQuery().data(), request->Get()->GetQuery().size());
+            SHA256_Final(reinterpret_cast<unsigned char*>(&hashedQueryText[0]), &sha256);
+            LogKeyValue("QueryText", Base64Encode(hashedQueryText), ss);
+        }
+
         if (request->Get()->HasTxControl()) {
             LogTxControl(request->Get()->GetTxControl(), ss);
         }
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index b2a2ca09d08..35ecd803d64 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -12,6 +12,7 @@ import "ydb/core/protos/bootstrap.proto";
 import "ydb/core/protos/cms.proto";
 import "ydb/core/protos/config_units.proto";
 import "ydb/core/protos/counters_schemeshard.proto";
+import "ydb/core/protos/data_integrity_trails.proto";
 import "ydb/core/protos/datashard_config.proto";
 import "ydb/core/protos/drivemodel.proto";
 import "ydb/core/protos/feature_flags.proto";
@@ -2132,6 +2133,7 @@ message TAppConfig {
     optional TShutdownConfig ShutdownConfig = 84;
     optional TPrioritiesQueueConfig CompPrioritiesConfig = 85;
     optional TSelfManagementConfig SelfManagementConfig = 86;
+    optional NKikimrProto.TDataIntegrityTrailsConfig DataIntegrityTrailsConfig = 87;
 
     repeated TNamedConfig NamedConfigs = 100;
     optional string ClusterYamlConfig = 101;
diff --git a/ydb/core/protos/data_integrity_trails.proto b/ydb/core/protos/data_integrity_trails.proto
new file mode 100644
index 00000000000..314157bd950
--- /dev/null
+++ b/ydb/core/protos/data_integrity_trails.proto
@@ -0,0 +1,11 @@
+package NKikimrProto;
+
+message TDataIntegrityTrailsConfig {
+    enum ELogMode {
+        ORIGINAL = 1;
+        HASHED = 2;
+    }
+
+    optional ELogMode QueryTextLogMode = 1 [default = HASHED];
+    optional ELogMode KeysLogMode = 2 [default = HASHED];
+};
\ No newline at end of file
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
index 17daa84b88c..d2f4bec42af 100644
--- a/ydb/core/protos/ya.make
+++ b/ydb/core/protos/ya.make
@@ -63,6 +63,7 @@ SRCS(
     counters_tx_allocator.proto
     counters_tx_proxy.proto
     data_events.proto
+    data_integrity_trails.proto
     database_basic_sausage_metainfo.proto
     datashard_config.proto
     datashard_load.proto
diff --git a/ydb/core/tx/datashard/datashard_integrity_trails.h b/ydb/core/tx/datashard/datashard_integrity_trails.h
index 1dd74f2f784..de0a65569af 100644
--- a/ydb/core/tx/datashard/datashard_integrity_trails.h
+++ b/ydb/core/tx/datashard/datashard_integrity_trails.h
@@ -8,6 +8,7 @@
 #include <ydb/core/engine/mkql_engine_flat.h>
 #include <ydb/core/protos/tx_datashard.pb.h>
 #include <ydb/core/scheme/scheme_tabledefs.h>
+#include <ydb/core/tx/datashard/range_ops.h>
 #include <ydb/core/tx/locks/sys_tables.h>
 #include <ydb/library/actors/core/log.h>
 #include <ydb/library/actors/core/actor.h>
@@ -19,36 +20,48 @@ namespace NDataIntegrity {
 
 inline void WriteTablePoint(const TConstArrayRef<NKikimr::TCell>& point, TStringStream& output) {
     std::string result;
-    result.resize(SHA_DIGEST_LENGTH);
+    result.resize(SHA256_DIGEST_LENGTH);
 
-    SHA_CTX sha1;
-    if (!SHA1_Init(&sha1)) {
+    SHA256_CTX sha256;
+    if (!SHA256_Init(&sha256)) {
         return;
     }
 
     for (size_t i = 0; i < point.size(); ++i) {
         const NKikimr::TCell& cell = point[i];
-        if (!SHA1_Update(&sha1, cell.Data(), cell.Size())) {
+        if (!SHA256_Update(&sha256, cell.Data(), cell.Size())) {
             return;
         }
     }
 
-    if (!SHA1_Final(reinterpret_cast<unsigned char*>(&result[0]), &sha1)) {
+    if (!SHA256_Final(reinterpret_cast<unsigned char*>(&result[0]), &sha256)) {
         return;
     }
 
     output << Base64Encode(result);
 }
 
-inline void WriteTableRange(const NKikimr::TTableRange &range, TStringStream& output) {
+inline void WriteTableRange(const NKikimr::TTableRange& range, const TVector<NScheme::TTypeInfo>& types, TStringStream& output) {
+    const auto keysLogMode = AppData()->DataIntegrityTrailsConfig.HasKeysLogMode()
+        ? AppData()->DataIntegrityTrailsConfig.GetKeysLogMode()
+        : NKikimrProto::TDataIntegrityTrailsConfig_ELogMode_HASHED;
+
     if (range.Point) {
-        WriteTablePoint(range.From, output);
+        if (keysLogMode == NKikimrProto::TDataIntegrityTrailsConfig_ELogMode_ORIGINAL) {
+            output << DebugPrintPoint(types, range.From, *AppData()->TypeRegistry);
+        } else {
+            WriteTablePoint(range.From, output);
+        }
     } else {
-        output << (range.InclusiveFrom ? "[" : "(");
-        WriteTablePoint(range.From, output);
-        output << " ; ";
-        WriteTablePoint(range.To, output);
-        output << (range.InclusiveTo ? "]" : ")");
+        if (keysLogMode == NKikimrProto::TDataIntegrityTrailsConfig_ELogMode_ORIGINAL) {
+            output << DebugPrintRange(types, range, *AppData()->TypeRegistry);
+        } else {
+            output << (range.InclusiveFrom ? "[" : "(");
+            WriteTablePoint(range.From, output);
+            output << " ; ";
+            WriteTablePoint(range.To, output);
+            output << (range.InclusiveTo ? "]" : ")");
+        }
     }
 }
 
@@ -100,7 +113,7 @@ inline void LogIntegrityTrailsKeys(const NActors::TActorContext& ctx, const ui64
                     LogKeyValue("Op", rowOp, ss);
 
                     ss << "Key: ";
-                    WriteTableRange(range, ss);
+                    WriteTableRange(range, keyDef->KeyColumnTypes, ss);
 
                     if (i + 1 < keys.Keys.size() && j + 1 < batchSize) {
                         ss << ",";