diff options
82 files changed, 3825 insertions, 686 deletions
diff --git a/.github/actions/test_cpp_sdk_with_coverage/action.yaml b/.github/actions/test_cpp_sdk_with_coverage/action.yaml index 6a3822f64b8..73aaf7f67a9 100644 --- a/.github/actions/test_cpp_sdk_with_coverage/action.yaml +++ b/.github/actions/test_cpp_sdk_with_coverage/action.yaml @@ -41,6 +41,7 @@ runs: --clang-coverage --coverage-report -DCOVERAGE_TARGET_REGEXP=ydb/public/sdk/cpp --coverage-prefix-filter=ydb/public/sdk/cpp + --coverage-exclude-regexp='/(ut|tests)/|_(ut|it)\.(c|cc|cpp|cxx|h|hh|hpp|hxx)' --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" --output "$out_dir" diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index c61c33c0d38..3495a401501 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -27,7 +27,6 @@ ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.ExternalTableMatc ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedEdsPathMatches ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedTopicRemoteNameMatches ydb/core/kqp/ut/federated_query/datastreams unittest.[*/*] chunk -ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.IndexesActualization-QueryService ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectDisabled ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectEnabled ydb/core/kqp/ut/query KqpAnalyzeOperations.AnalyzeOperationsLifecycle @@ -74,6 +73,7 @@ ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[kick_tablets-std] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-fifo] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-std] +ydb/tests/functional/tenants test_remove_storage_groups.py.test_remove_storage_group ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test diff --git a/.github/config/muted_ya_asan.txt b/.github/config/muted_ya_asan.txt index 7e8d0f3ecae..d23a7816d30 100644 --- a/.github/config/muted_ya_asan.txt +++ b/.github/config/muted_ya_asan.txt @@ -183,6 +183,7 @@ ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageFifoQueue ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageFifoQueue_TableImplementation ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageWithAttributes ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageWithAttributes_TableImplementation +ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageWithIAMWithoutFolderId ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessageWithIAMWithoutFolderId_TableImplementation ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_Deduplication_Compatibility ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_Deduplication_Compatibility_WithUserSettings @@ -501,6 +502,7 @@ ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap-UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs-IsOlap+UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclInterruptInheritance ydb/core/kqp/ut/scheme KqpAcl.AclTemporaryInterruptInheritance-IsOlap +ydb/core/kqp/ut/scheme KqpConstraints.CompileTimeDefaults_WithGlobalAsyncIndex-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.AlterTableCompactAccessDenied+UseQueryService ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableViaApiMultipleTime ydb/core/kqp/ut/scheme KqpScheme.ParallelDropStreamingQuery @@ -541,6 +543,7 @@ ydb/core/kqp/ut/view TCreateAndDropViewTest.DropViewIfExists ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidColumnRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidIndexRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidTableRef +ydb/core/kqp/ut/view TCreateAndDropViewTest.ParsingSecurityInvoker ydb/core/kqp/ut/view TSelectFromViewTest.LeftJoinViewsWithNoMatch ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher @@ -828,6 +831,7 @@ ydb/tests/fq/s3 test_streaming_join.py.TestStreamingJoin.test_grace_join[v1-clie ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[False] ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[True] ydb/tests/fq/streaming test_udfs.py.TestUdfsUsage.test_precompute_recovery[False] +ydb/tests/fq/streaming/streaming_large test_cluster_restarts.py.TestStreamingLarge.test_rolling_restart[kikimr0] ydb/tests/fq/streaming_optimize py3test.[test_sql_negative.py */*] chunk ydb/tests/fq/streaming_optimize py3test.[test_sql_streaming.py */*] chunk ydb/tests/fq/streaming_optimize py3test.sole chunk @@ -1102,8 +1106,11 @@ ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_doc ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_list_with_path_filter ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_tool_schema_in_request ydb/tests/olap data_read_correctness.py.TestDataReadPerformanceNoIntersections.test +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_20_compression-algorithm=zstd,level=20] +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_8_compression-algorithm=zstd,level=8] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_to_compression[zstd_20_compression-algorithm=zstd,level=20] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_zstd_level_from_0_to_19_compression +ydb/tests/olap/column_compression/alter py3test.[*/*] chunk ydb/tests/olap/s3_import py3test.[*/*] chunk ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_drop_table_during_export ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_rename_table_during_export diff --git a/.github/workflows/cpp_sdk_peerdirs_check.yml b/.github/workflows/cpp_sdk_peerdirs_check.yml index ad124c0c8fc..d4000fb789a 100644 --- a/.github/workflows/cpp_sdk_peerdirs_check.yml +++ b/.github/workflows/cpp_sdk_peerdirs_check.yml @@ -3,18 +3,27 @@ name: Check CPP SDK PEERDIRs on: pull_request_target: branches: [main, stable-*] + types: [opened, synchronize, reopened, labeled] paths: ['ydb/public/sdk/cpp/**', '.github/workflows/cpp_sdk_peerdirs_check.yml'] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Irrelevant label events must not cancel a running check for the same PR. + group: >- + ${{ github.workflow }}-${{ + github.event_name == 'pull_request_target' + && format('pr-{0}{1}', github.event.pull_request.number, + github.event.action == 'labeled' && github.event.label.name != 'ok-to-test' && format('-ignore-{0}', github.run_id) || '') + || github.ref + }} cancel-in-progress: true jobs: check: if: >- github.event_name == 'workflow_dispatch' || - (vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).cpp_sdk_peerdirs_check == true) + (github.event.pull_request.state == 'open' && + (github.event.action != 'labeled' || github.event.label.name == 'ok-to-test')) runs-on: [self-hosted, tiny-worker] steps: - uses: actions/github-script@v8 diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index cdf6adbc47d..f478f7c836b 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -59,6 +59,7 @@ jobs: 'stable-26-1-1', 'stable-26-2', 'stable-26-2-1', + 'stable-26-1-1-enterprise', ]; let baseRef; if (context.eventName === 'workflow_dispatch') { diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index a2d4fc87259..0235b0a2b3a 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -248,6 +248,9 @@ jobs: with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 + # Fork code is checked out only after the check-running-allowed gate + # (collaborator/owner or a maintainer-applied `ok-to-test` label) + allow-unsafe-pr-checkout: true - name: Setup ydb access uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials with: diff --git a/ydb/core/blobstorage/ddisk/ddisk.h b/ydb/core/blobstorage/ddisk/ddisk.h index f55dcc5c91d..25e9c7642fa 100644 --- a/ydb/core/blobstorage/ddisk/ddisk.h +++ b/ydb/core/blobstorage/ddisk/ddisk.h @@ -639,7 +639,7 @@ struct TPersistentBufferFormat { TEvReadPersistentBufferResult(NKikimrBlobStorage::NDDisk::TReplyStatus::E status, const std::optional<TString>& errorReason = std::nullopt, ui64 vChunkIndex = 0, ui32 offsetInBytes = 0, ui32 sizeInBytes = 0, - TRope data = {}) { + TRope data = {}, const std::vector<ui64>& checksums = {}) { Record.SetStatus(status); if (errorReason) { Record.SetErrorReason(*errorReason); @@ -649,6 +649,12 @@ struct TPersistentBufferFormat { Record.SetOffsetInBytes(offsetInBytes); Record.SetSizeInBytes(sizeInBytes); TReadResult(AddPayload(std::move(data))).Serialize(Record.MutableReadResult()); + // Opt-in, mirrors TEvWritePersistentBuffer.Checksums: only attached when the persisted + // record actually carries sender-supplied payload checksums (see + // TPersistentBuffer::TRecord::PayloadChecksums). + for (ui64 checksum : checksums) { + Record.AddChecksums(checksum); + } } } }; diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor.h b/ydb/core/blobstorage/ddisk/ddisk_actor.h index e780c0f362b..f3d0606c8b6 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor.h +++ b/ydb/core/blobstorage/ddisk/ddisk_actor.h @@ -753,6 +753,9 @@ namespace NKikimr::NDDisk { std::map<ui64, TRope> DataParts; ui32 PartsCount; std::vector<TPersistentBufferSectorInfo> Sectors; + // Sender-supplied per-MinSectorSize-block payload checksums for this record, in order. + // Empty when the write carried no checksums. See TPersistentBuffer::TRecord::PayloadChecksums. + std::vector<ui64> PayloadChecksums; TRope JoinData(ui32 sectorSize); }; @@ -813,7 +816,7 @@ namespace NKikimr::NDDisk { void IssuePersistentBufferChunkAllocation(); void ProcessDeallocatePersistentBufferChunk(bool forceToNextChunk = false); void ProcessPersistentBufferQueue(); - std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors); + std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, const std::vector<ui64>& payloadChecksums); std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBufferData(TRope& data, std::vector<TPersistentBufferSectorInfo>& sectors); void StartRestorePersistentBuffer(); void RestorePersistentBufferChunk(TEvPrivate::TEvReadPersistentBufferPart::TPtr ev); @@ -822,7 +825,12 @@ namespace NKikimr::NDDisk { bool PreprocessPersistentBufferWrite(NActors::TEventHandle<TEvWritePersistentBuffer>& ev); void ProcessPersistentBufferWrite(TEvWritePersistentBuffer::TPtr ev); - bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev); + // ev is taken by reference (not TPtr by value, unlike its sibling above): TPtr is a TAutoPtr + // with ownership-transferring copy semantics, so a by-value parameter here would null out the + // caller's ev as soon as this is invoked -- including on the "doesn't fit, fall back" (false) + // return path, where Handle(TEvWritePersistentBuffer) still needs a valid ev afterwards to retry + // via ProcessPersistentBufferWrite. + bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev); void ProcessPersistentBufferBatchWrite(); double GetPersistentBufferFreeSpace(); void ErasePersistentBuffer(IEventHandle& queryEv, const TQueryCredentials& creds, const std::vector<TEraseLsnId>& erases); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp index 611a01228a1..3660a33cf16 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp @@ -15,6 +15,20 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxLsnsPerPack = (DataAlignment - sizeof(TPersistentBufferHeader)) / (sizeof(TPersistentBufferLsnRecordHeader) + TPersistentBufferLsnRecordHeader::MaxSectorsPerPackBufferRecord * sizeof(TPersistentBufferSectorInfo)); + // Size, in bytes, of one LSN record's metadata block within the shared 4 KiB batch header sector: + // TPersistentBufferLsnRecordHeader + its TPersistentBufferSectorInfo[] location array, plus an + // optional trailing ui64 PayloadChecksums[] (one entry per data sector) when the record carries + // sender-supplied checksums. Used both to lay out the header sector (SlicePersistentBuffer / + // ProcessPersistentBufferBatchWrite) and to decide whether one more record still fits in an + // already-open batch (ProcessPersistentBufferBatchWriteData) - MaxLsnsPerPack above only bounds + // the pre-allocated data-sector budget and is deliberately not tightened for checksums, since + // checksums add header-sector bytes, not data sectors. + static ui32 LsnRecordMetadataSize(ui32 dataSectorsCnt, bool hasPayloadChecksums) { + return sizeof(TPersistentBufferLsnRecordHeader) + + dataSectorsCnt * sizeof(TPersistentBufferSectorInfo) + + (hasPayloadChecksums ? dataSectorsCnt * sizeof(ui64) : 0); + } + void TDDiskActor::IssuePersistentBufferChunkAllocation() { Y_ABORT_UNLESS(IsPersistentBufferActor); if (!IssuePersistentBufferChunkAllocationInflight) { @@ -229,7 +243,8 @@ namespace NKikimr::NDDisk { std::vector<std::tuple<ui32, ui32, TRope>> TDDiskActor::SlicePersistentBuffer( ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 sizeInBytes, - TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors) + TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, + const std::vector<ui64>& payloadChecksums) { TRope fullData(std::move(payloadWithHeader)); @@ -263,7 +278,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -281,6 +295,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = offsetInBytes; lsnRecordHeader->Size = sizeInBytes; lsnRecordHeader->Lsn = lsn; + const bool hasPayloadChecksums = !payloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + header->Version = hasPayloadChecksums ? 1 : 0; auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(fullData.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader) @@ -295,6 +312,16 @@ namespace NKikimr::NDDisk { sectors[i].Checksum = loc.Checksum; } + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(payloadChecksums.size() == sectors.size() - 1); + auto* checksums = reinterpret_cast<ui64*>(fullData.Begin().UnsafeContiguousDataMut() + + sizeof(TPersistentBufferHeader) + + sizeof(TPersistentBufferLsnRecordHeader) + + (sectors.size() - 1) * sizeof(TPersistentBufferSectorInfo) + ); + memcpy(checksums, payloadChecksums.data(), payloadChecksums.size() * sizeof(ui64)); + } + header->Checksum = CalculateChecksum(fullData.Begin(), SectorSize); // Slice fullData ([header | payload], a single contiguous, SectorSize-aligned buffer) into one part @@ -587,6 +614,12 @@ namespace NKikimr::NDDisk { pr.Sectors.push_back(locations[i]); } pos += sectorsCnt * sizeof(TPersistentBufferSectorInfo); + + if (recordHeader->Flags & TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + const ui64* checksums = reinterpret_cast<const ui64*>(pos); + pr.PayloadChecksums.assign(checksums, checksums + sectorsCnt); + pos += sectorsCnt * sizeof(ui64); + } } } else { PersistentBufferSectorsChecksum[chunkIdx][sectorIdx] = CalculateChecksum(dataPos); @@ -721,7 +754,8 @@ namespace NKikimr::NDDisk { .Size = (ui32)record.Size, .Sectors = std::move(record.Sectors), .VChunkIndex = record.VChunkIndex, - .Timestamp = TInstant::Now() + .Timestamp = TInstant::Now(), + .PayloadChecksums = std::move(record.PayloadChecksums), }; auto& pbh = PersistentBufferHeaders[{pr.Sectors[0].ChunkIdx, pr.Sectors[0].SectorIdx}]; @@ -898,6 +932,7 @@ namespace NKikimr::NDDisk { if (instr.PayloadId) { payload = ev->Get()->GetPayload(*instr.PayloadId); } + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferWrite", {"marker", "BSPB"}, {"PBufferId", SelfId()}, @@ -959,7 +994,8 @@ namespace NKikimr::NDDisk { TRcBuf payloadWithHeader = ev->Get()->GetPayloadWithHeader(*instr.PayloadId); auto parts = SlicePersistentBuffer(creds.TabletId, creds.Generation, - selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors); + selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors, + payloadChecksums); auto opCookie = NextCookie++; auto& inflightRecord = PersistentBufferDiskOperationInflight[opCookie]; @@ -982,6 +1018,9 @@ namespace NKikimr::NDDisk { .DataParts = {{0, std::move(payload)}}, .PartsCount = 1, .Sectors = std::move(sectors), + // payloadChecksums was only read (never mutated) above by SlicePersistentBuffer; safe to + // move into the inflight record now that this is its last use. + .PayloadChecksums = std::move(payloadChecksums), }); PersistentBufferWriteInflightsByRecord[TPersistentBufferRecordId{creds.TabletId, creds.Generation, lsn}].emplace_back( opCookie, @@ -1008,7 +1047,7 @@ namespace NKikimr::NDDisk { } } - bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev) { + bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev) { const auto& record = ev->Get()->Record; const TQueryCredentials creds(record.GetCredentials()); const TBlockSelector selector(record.GetSelector()); @@ -1020,6 +1059,9 @@ namespace NKikimr::NDDisk { const TWriteInstruction instr(record.GetInstruction()); Y_ABORT_UNLESS(instr.PayloadId, "WritePersistentBuffer without a payload"); TRope payload = ev->Get()->GetPayload(*instr.PayloadId); + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); + const ui32 sectorsCnt = selector.Size / SectorSize; + Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferBatchWriteData", {"marker", "BSPB"}, @@ -1043,6 +1085,22 @@ namespace NKikimr::NDDisk { }); Y_ABORT_UNLESS(inserted); Schedule(TDuration::MicroSeconds(PersistentBufferFormat.WritesBatchingPeriodMicroseconds), new TEvents::TEvWakeup(EWakeupTag::WakeupProcessPersistentBufferBatchWrite)); + } else { + // Reusing an already-open batch: this record's metadata (TPersistentBufferLsnRecordHeader + + // sector locations + optional payload checksums) must still fit in the shared 4 KiB header + // sector alongside every record already queued. A brand-new batch (handled above) always + // fits a single record trivially, so this check only matters here. Bail out (fall back to + // the direct, non-batched write path in the caller) rather than overflow the header sector. + const auto it = PersistentBufferDiskOperationInflight.find(PersistentBufferBatchWriteCookie); + Y_ABORT_UNLESS(it != PersistentBufferDiskOperationInflight.end()); + const auto& existingRecords = it->second.Records; + ui32 usedBytes = sizeof(TPersistentBufferHeader); + for (auto& r : existingRecords) { + usedBytes += LsnRecordMetadataSize(r.Sectors.size() - 1, !r.PayloadChecksums.empty()); + } + if (usedBytes + LsnRecordMetadataSize(sectorsCnt, !payloadChecksums.empty()) > SectorSize) { + return false; + } } auto& inflight = PersistentBufferDiskOperationInflight[PersistentBufferBatchWriteCookie]; @@ -1071,11 +1129,10 @@ namespace NKikimr::NDDisk { .Size = selector.Size, .DataParts = {{0, payload}}, .PartsCount = 1, + .PayloadChecksums = std::move(payloadChecksums), }); auto& r = inflight.Records.back(); - ui32 sectorsCnt = selector.Size / SectorSize; - Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); r.Sectors.push_back(inflight.OccupiedSectors[0]); auto sectorsIt = inflight.OccupiedSectors.begin() + inflight.DataToWrite.size() / SectorSize; @@ -1119,7 +1176,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -1129,6 +1185,7 @@ namespace NKikimr::NDDisk { header->Flags = 0; header->BatchSize = inflight.Records.size(); + bool anyPayloadChecksums = false; auto* pos = inflight.DataToWrite.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader); for (auto& record : inflight.Records) { auto* lsnRecordHeader = reinterpret_cast<TPersistentBufferLsnRecordHeader*>(pos); @@ -1139,6 +1196,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = record.OffsetInBytes; lsnRecordHeader->Size = record.Size; lsnRecordHeader->Lsn = record.Lsn; + const bool hasPayloadChecksums = !record.PayloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + anyPayloadChecksums = anyPayloadChecksums || hasPayloadChecksums; pos += sizeof(TPersistentBufferLsnRecordHeader); auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(pos); @@ -1148,7 +1208,16 @@ namespace NKikimr::NDDisk { loc = record.Sectors[i]; } pos += sizeof(TPersistentBufferSectorInfo) * (record.Sectors.size() - 1); + + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(record.PayloadChecksums.size() == record.Sectors.size() - 1); + memcpy(pos, record.PayloadChecksums.data(), record.PayloadChecksums.size() * sizeof(ui64)); + pos += record.PayloadChecksums.size() * sizeof(ui64); + } } + Y_ABORT_UNLESS(static_cast<size_t>(pos - inflight.DataToWrite.Begin().UnsafeContiguousDataMut()) <= SectorSize, + "persistent buffer batch header overflow"); + header->Version = anyPayloadChecksums ? 1 : 0; header->Checksum = CalculateChecksum(inflight.DataToWrite.Begin(), SectorSize); auto parts = SlicePersistentBufferData(inflight.DataToWrite, inflight.OccupiedSectors); @@ -1407,11 +1476,23 @@ namespace NKikimr::NDDisk { Y_ABORT_UNLESS(inflight.Records.size() == 1); auto& inflightRecord = inflight.Records[0]; TRope data; + std::vector<ui64> checksums; if (pr.Data) { data = std::move(TrimData(pr.Data, pr.OffsetInBytes, pr.Size, inflightRecord.OffsetInBytes, inflightRecord.Size)); + if (!pr.PayloadChecksums.empty()) { + // Persisted checksums cover [pr.OffsetInBytes, pr.OffsetInBytes + pr.Size) one entry + // per MinSectorSize block, same order as the trimmed data above - slice out the + // sub-range the selector actually asked for. Returned as-is (never recomputed from + // on-disk bytes), matching the write path's opt-in, pre-signature-correction values. + Y_ABORT_UNLESS(pr.PayloadChecksums.size() == pr.Size / SectorSize); + const ui32 firstBlock = (inflightRecord.OffsetInBytes - pr.OffsetInBytes) / SectorSize; + const ui32 blockCount = inflightRecord.Size / SectorSize; + Y_ABORT_UNLESS(firstBlock + blockCount <= pr.PayloadChecksums.size()); + checksums.assign(pr.PayloadChecksums.begin() + firstBlock, pr.PayloadChecksums.begin() + firstBlock + blockCount); + } } auto replyEv = std::make_unique<TEvReadPersistentBufferResult>(status, errorMessage, - pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data)); + pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data), checksums); auto h = std::make_unique<IEventHandle>(inflightRecord.Sender, SelfId(), replyEv.release(), 0, inflightRecord.Cookie); if (inflightRecord.Session) { h->Rewrite(TEvInterconnect::EvForward, inflightRecord.Session); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp index a1db18c0ae6..0c4c1071914 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp @@ -89,13 +89,14 @@ namespace NKikimr::NDDisk { } } + // Unlike TEvWritePersistentBuffer, TEvWrite's checksums (if any) are validated here but never + // persisted: TEvWrite goes straight to the final DDisk chunk, which has no per-record metadata + // slot to carry them (see RFC 006). Note this validation intentionally runs again for events + // that get parked in PendingEventsForChunk below and re-dispatched to this same handler once + // their chunk allocation completes -- harmless (checksums don't change), just redundant. if (record.ChecksumsSize() > 0) { - // Checksum validation is opt-in (see TEvWritePersistentBuffer for details); checked here, - // before any chunk allocation queueing or disk I/O, and the checksums are not persisted. - // Note: an event parked in PendingEventsForChunk below re-enters this handler in full once - // the chunk is allocated, so its checksums get validated twice. This is harmless (validation - // stays right next to the actual write) and intentional, not an oversight. Y_ABORT_UNLESS(instr.PayloadId, "TEvWrite without a payload, but with checksums"); + const TRope& payload = ev->Get()->GetPayload(*instr.PayloadId); if (const auto result = ValidatePayloadChecksums(record, payload)) { const bool isCorrupted = result->Status == NKikimrBlobStorage::NDDisk::TReplyStatus::CORRUPTED; diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer.h b/ydb/core/blobstorage/ddisk/persistent_buffer.h index 9c94108d446..1710bb0554c 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer.h @@ -43,6 +43,10 @@ namespace NKikimr::NDDisk { ui64 VChunkIndex; TInstant Timestamp; std::unordered_set<ui64> ReadInflight; + // Sender-supplied per-MinSectorSize-block payload checksums, in order, covering + // [OffsetInBytes, OffsetInBytes + Size). Empty when the record was written without + // checksums (legacy / internal writes) - opt-in, mirrors the wire-level semantics. + std::vector<ui64> PayloadChecksums; }; std::map<ui64, TRecord> Records; diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h index 8f83c4d1142..7949fdfbbfd 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h @@ -73,9 +73,21 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxSectorsPerBufferRecord = 128; static constexpr ui32 MaxSectorsPerPackBufferRecord = 8; + enum EFlags : ui32 { + NONE = 0, + // When set, a ui64 payload checksum (sender-computed, unsalted XXH3-64 per MinSectorSize + // block - see CalculatePayloadChecksums) immediately follows this record's + // TPersistentBufferSectorInfo[] location array, one entry per data sector, same order. + // Opt-in per record: only present when the write that produced it carried sender-supplied + // checksums (TEvWritePersistentBuffer.Checksums). Legacy/internal records without checksums + // have this flag clear (the field was always zero-initialized via the full-sector memset + // that precedes header assembly, so old on-disk records are unaffected by this rename). + HAS_PAYLOAD_CHECKSUMS = 1, + }; + ui64 TabletId; ui32 Generation; - ui32 Reserved1; + ui32 Flags; ui64 VChunkIndex; ui32 OffsetInBytes; ui32 Size; @@ -83,8 +95,10 @@ namespace NKikimr::NDDisk { }; static_assert(sizeof(TPersistentBufferLsnRecordHeader) <= DataAlignment); + // Worst case: a single record spanning MaxSectorsPerBufferRecord sectors, all carrying a + // persisted payload checksum. static_assert(DataAlignment >= sizeof(TPersistentBufferHeader) + sizeof(TPersistentBufferLsnRecordHeader) - + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * sizeof(TPersistentBufferSectorInfo) + + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * (sizeof(TPersistentBufferSectorInfo) + sizeof(ui64)) ); #pragma pack(pop) } diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp index bea5e6eb01d..d7c92eb19ae 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp @@ -1129,12 +1129,16 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { // TPersistentBufferHeader // [record 0] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // [record 1] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // ... // Each record here has exactly 1 data sector, so Sectors.size() == 2 // (header sector + 1 data sector), meaning (Sectors.size()-1) == 1 - // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. + // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. Both A and B call + // ChecksumPayload(), so recB also carries 1 persisted payload checksum (ui64) after its + // TPersistentBufferSectorInfo entry. const ui32 headerChunkIdx = rawBC_combined->Get()->ChunkIdx; const ui32 headerOffset = rawBC_combined->Get()->Offset; const ui64 actualUniqueId = [&]() -> ui64 { @@ -1152,10 +1156,14 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { UNIT_ASSERT_VALUES_EQUAL_C(recB->Generation, credsB.Generation, "First batched record generation must match tablet B"); - // Advance past recB's LsnRecordHeader and its 1 TPersistentBufferSectorInfo - // (Sectors.size()-1 == 1 for a single-sector data write). + // Advance past recB's LsnRecordHeader, its 1 TPersistentBufferSectorInfo + // (Sectors.size()-1 == 1 for a single-sector data write), and (since B carries a + // sender checksum) its 1 persisted payload checksum. pos += sizeof(NDDisk::TPersistentBufferLsnRecordHeader); pos += sizeof(NDDisk::TPersistentBufferSectorInfo); // (Sectors.size()-1) entries + if (recB->Flags & NDDisk::TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + pos += sizeof(ui64); // (Sectors.size()-1) checksums + } // Record C: immediately follows. const auto* recC = reinterpret_cast<const NDDisk::TPersistentBufferLsnRecordHeader*>(pos); diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp index b36d7ebc065..19758b86f74 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp @@ -11,6 +11,7 @@ #include <ydb/core/blobstorage/ddisk/ddisk.h> #include <ydb/core/blobstorage/ddisk/ddisk_actor.h> +#include <ydb/core/blobstorage/ddisk/persistent_buffer_header.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h> @@ -22,6 +23,8 @@ #include <util/string/printf.h> +#include <cstring> + namespace NKikimr { namespace { @@ -184,6 +187,113 @@ public: auto res = new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0); SendPDiskResponse(disk, *checkSpace, res); } + + // Create a DDisk instance that simulates a restart where the PB chunks from a + // previous instance are passed via StartingPoints. Mirrors the helper of the same + // name in ddisk_actor_batch_write_ut.cpp. + TDiskHandle CreateDDiskWithRestoredChunkData(ui32 pdiskId, ui32 slotId, + const std::vector<ui32>& preExistingChunkIds, ui64 oldUniqueId, + const std::unordered_map<ui32, TString>& chunkData) { + const TActorId pdiskEdge = Runtime.AllocateEdgeActor(NodeId, __FILE__, __LINE__); + const TActorId pdiskServiceId = MakeBlobStoragePDiskID(NodeId, pdiskId); + Runtime.RegisterService(pdiskServiceId, pdiskEdge); + + TVector<TActorId> actorIds = { + MakeBlobStorageDDiskId(NodeId, pdiskId, slotId), + }; + auto groupInfo = MakeIntrusive<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureNone, ui32(1), ui32(1), + ui32(1), &actorIds); + + TVDiskConfig::TBaseInfo baseInfo( + TVDiskIdShort(groupInfo->GetVDiskId(0)), + pdiskServiceId, + 0x100000 + pdiskId, + pdiskId, + NPDisk::DEVICE_TYPE_NVME, + slotId, + NKikimrBlobStorage::TVDiskKind::Default, + 1, + "ddisk_pool"); + NDDisk::TPersistentBufferFormat pbFormat{256, 4, BlockSize * 128, 8, 5000, 512 * 1024}; + const TActorId ddiskActor = Runtime.Register(NDDisk::CreateDDiskActor(std::move(baseInfo), groupInfo, + std::move(pbFormat), NDDisk::TDDiskConfig{}, Counters), + NodeId); + const TActorId ddiskServiceId = MakeBlobStorageDDiskId(NodeId, pdiskId, slotId); + const TActorId pbServiceId = MakeBlobStoragePersistentBufferId(NodeId, pdiskId, slotId); + Runtime.RegisterService(ddiskServiceId, ddiskActor); + + TDiskHandle disk{ddiskServiceId, pbServiceId, pdiskEdge, pdiskId, slotId, 100000 + pdiskId * 1000}; + + const NPDisk::TOwner Owner = 1; + const NPDisk::TOwnerRound OwnerRound = 1; + + auto init = WaitPDiskRequest<NPDisk::TEvYardInit>(disk); + TVector<ui32> ownedChunks; + auto initReply = std::make_unique<NPDisk::TEvYardInitResult>( + NKikimrProto::OK, + 0, 0, 0, + BlockSize, BlockSize, BlockSize, + ChunkSize, + BlockSize, + Owner, + OwnerRound, + 1, + 0, + std::move(ownedChunks), + NPDisk::DEVICE_TYPE_NVME, + false, + BlockSize, + ""); + + NPDisk::TDiskFormat format = {}; + format.Clear(false); + initReply->DiskFormat = NPDisk::TDiskFormatPtr(new NPDisk::TDiskFormat(format), +[](NPDisk::TDiskFormat* ptr) { + delete ptr; + }); + + { + NKikimrBlobStorage::NDDisk::NInternal::TPersistentBufferChunkMapLogRecord pbChunkMap; + for (ui32 chunkIdx : preExistingChunkIds) { + pbChunkMap.AddChunkIdxs(chunkIdx); + } + pbChunkMap.SetUniqueId(oldUniqueId + 1); + + TString pbChunkMapData; + const bool serializeOk = pbChunkMap.SerializeToString(&pbChunkMapData); + Y_ABORT_UNLESS(serializeOk); + initReply->StartingPoints[TLogSignature::SignaturePersistentBufferChunkMap] = + NPDisk::TLogRecord(TLogSignature::SignaturePersistentBufferChunkMap, + TRcBuf(pbChunkMapData), 1 /*lsn*/); + } + + SendPDiskResponse(disk, *init, initReply.release()); + + auto readLog = WaitPDiskRequest<NPDisk::TEvReadLog>(disk); + auto readLogReply = std::make_unique<NPDisk::TEvReadLogResult>( + NKikimrProto::OK, + readLog->Get()->Position, + readLog->Get()->Position, + true, + 0, + "", + Owner); + SendPDiskResponse(disk, *readLog, readLogReply.release()); + + for (ui32 i = 0; i < preExistingChunkIds.size(); ++i) { + auto readRaw = WaitPDiskRequest<NPDisk::TEvChunkReadRaw>(disk); + const ui32 chunkIdx = readRaw->Get()->ChunkIdx; + auto it = chunkData.find(chunkIdx); + TString data; + if (it != chunkData.end()) { + data = it->second; + } else { + data = TString(TTestContext::ChunkSize, '\0'); + } + SendPDiskResponse(disk, *readRaw, new NPDisk::TEvChunkReadRawResult(TRope(data))); + } + + return disk; + } }; void SendToDDisk(TTestContext& ctx, const TActorId& serviceId, IEventBase* event, ui64 cookie = 0) { @@ -560,6 +670,765 @@ Y_UNIT_TEST_SUITE(TDDiskChecksumTests) { UNIT_ASSERT_VALUES_EQUAL(GetChecksumCounter(ctx, disk, "WritesWithoutChecksums")->Val(), 1); } + + // 9. Persistence: a PB write carrying sender checksums stores them, and an in-memory read + // (record still cached, no restart) returns exactly those checksums, in order, alongside the + // original payload. + Y_UNIT_TEST(SinglePBWriteWithChecksumsReadBackInMemory) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(60, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 110, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('Z', 2 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->GetPayload(0).ConvertToString(), payload); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i]); + } + } + + // 10. Persistence across restart, both the direct (non-batch) write path and the batched write + // path: record A is written directly, B and C are accumulated into a single batch header + // sector while A's disk write is in-flight. All three carry sender checksums. After a + // simulated restart (StartingPoints replay from a fresh DDisk instance), all three records + // must be listed and readable with the same checksums that were originally sent. + Y_UNIT_TEST(WritesWithChecksumsAndBatchSurviveRestart) { + TTestContext ctx; + + const TDiskHandle disk1 = ctx.CreateDDisk(61, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 111, 1); + + const ui64 lsnA = 1; + const ui64 lsnB = 77; + const ui64 lsnC = 88; + const TString payloadA = MakeData('A', BlockSize); + const TString payloadB = MakeData('B', BlockSize); + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + + // Accumulate all raw PDisk writes into per-chunk buffers so we can feed them back to + // disk2 during restore. + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + std::vector<ui64> checksumsA, checksumsB, checksumsC; + + // Step 1: send A, keep its PDisk write pending so B and C enter the batch path. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorA, lsnA, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadA)); + checksumsA.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1ActorId = rawA->Sender; + captureWrite(rawA); + + // Step 2/3: B and C accumulate into the same batch while A is in-flight. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorB, lsnB, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadB)); + checksumsB.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorC, lsnC, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Step 4: complete A. + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + // Step 5: wakeup fires -> combined write for B and C (header + B data + C data). + auto rawBC = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBC->Get()->Data.size(), 3 * BlockSize, + "Combined write must contain header + B data + C data (3 sectors)"); + captureWrite(rawBC); + + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawBC->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBC->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Header must record BatchSize=2 for B and C"); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 1u, + "Header Version must be 1 when any record in the batch carries payload checksums"); + return hdr->PersistentBufferUniqueId; + }(); + UNIT_ASSERT_C(actualUniqueId != 0, "PersistentBufferUniqueId must not be zero"); + + // Step 6: complete the combined write -> B and C both reply. + ctx.SendPDiskResponse(disk1, *rawBC, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + // Phase 2: restart with instance 2 (same PDiskId, same UniqueId). + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1ActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 61, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 111, 1); + + auto listResult = SendToDDiskAndWait<NDDisk::TEvListPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvListPersistentBuffer(creds2)); + AssertStatus(listResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(listResult->Get()->Record.RecordsSize(), 3, + "Records A, B and C must all be restored after restart"); + + auto checkRead = [&](const NDDisk::TBlockSelector& selector, ui64 lsn, const TString& payload, + const std::vector<ui64>& expectedChecksums, const TString& label) { + auto read = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(read, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->GetPayload(0).ConvertToString(), payload, + TStringBuilder() << label << ": restored payload must match"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(read->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + TStringBuilder() << label << ": checksum count mismatch after restore"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->Record.GetChecksums(i), expectedChecksums[i], + TStringBuilder() << label << ": checksum " << i << " mismatch after restore"); + } + }; + + checkRead(selectorA, lsnA, payloadA, checksumsA, "record A (non-batch)"); + checkRead(selectorB, lsnB, payloadB, checksumsB, "record B (batched)"); + checkRead(selectorC, lsnC, payloadC, checksumsC, "record C (batched)"); + } + + // 11. Signature-correction sector: the sender's checksum is validated over, and must be + // persisted as, the ORIGINAL payload bytes -- even though PB zeroes the colliding byte on disk + // and restores it on read. If the checksum were ever recomputed from the on-disk (corrected) + // bytes it would not match what the sender sent. + Y_UNIT_TEST(SignatureCorrectionChecksumPersistedAsOriginal) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(62, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 112, 1); + + const ui64 lsn = 1; + // The second data sector's first byte deliberately equals the PB header signature byte, + // forcing TDDiskActor to zero it on disk (HasSignatureCorrection) and restore it on read. + TString payload = MakeData('S', 2 * BlockSize); + payload.Detach()[BlockSize] = static_cast<char>(NDDisk::TPersistentBufferHeader::PersistentBufferHeaderSignature[0]); + const NDDisk::TBlockSelector selector{7, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + // Sanity: the on-disk sector's colliding byte was actually zeroed (header sector + 2 data + // sectors are written contiguously, so the second data sector starts at offset 2*BlockSize). + const TString writtenData = pbWriteRaw->Get()->Data.ConvertToString(); + UNIT_ASSERT_VALUES_EQUAL_C(writtenData[2 * BlockSize], '\0', + "Signature-correction must zero the colliding byte on disk"); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Original signature byte must be restored on read"); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Persisted checksum must be the pre-correction (original) value, never recomputed from on-disk bytes"); + } + } + + // 12. Legacy / no-checksum compatibility across restart: a write without sender checksums + // (opt-in, same as SinglePBWriteWithoutChecksums) must restore successfully and its read must + // return an empty Checksums list, both immediately and after a restart. + Y_UNIT_TEST(LegacyWriteWithoutChecksumsSurvivesRestartWithEmptyChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(63, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 113, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('L', BlockSize); + const NDDisk::TBlockSelector selector{4, 0, BlockSize}; + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payload)); + // Deliberately not calling AddPayloadThenChecksum(): checksum persistence is opt-in. + UNIT_ASSERT_VALUES_EQUAL(write->Record.ChecksumsSize(), 0u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + + auto rawWrite = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1SenderActorId = rawWrite->Sender; + captureWrite(rawWrite); + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawWrite->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawWrite->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 0u, "Header Version must stay 0 for a checksum-less write"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawWrite, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1SenderActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 63, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 113, 1); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Legacy record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 0u, + "A checksum-less record must return an empty Checksums list after restore"); + } + + // 13. Re-replication (TEvReadThenWritePersistentBuffers) must forward the checksums that were + // persisted on the source record, not silently drop them: the coordinator PB reads its own + // record (which carries sender-supplied checksums) and re-writes it to the target PBs via + // TEvWritePersistentBuffers -- the checksums must reach and be validated/persisted on every + // target, exactly as if the original sender had written them directly. Modeled on + // PersistentBufferReadThenWriteTunnel (ddisk_actor_ut.cpp). + Y_UNIT_TEST(ReadThenWriteForwardsPersistedChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(64, 1); + const TDiskHandle disk2 = ctx.CreateDDisk(65, 1); + const TDiskHandle disk3 = ctx.CreateDDisk(66, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk1.PBServiceId, 120, 1); + + const ui64 lsn = 10; + const TString payload = MakeData('P', 2 * BlockSize); + const NDDisk::TBlockSelector selector{3, 0, 2 * BlockSize}; + + auto pbs = std::vector<std::tuple<ui32, ui32, ui32>>{ + {NodeId, disk2.PDiskId, disk2.SlotId}, + {NodeId, disk3.PDiskId, disk3.SlotId}}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + ctx.SendPDiskResponse(disk1, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readThenWrite = std::make_unique<NDDisk::TEvReadThenWritePersistentBuffers>(creds, lsn, 1, pbs, 1000); + SendToDDisk(ctx, disk1.PBServiceId, readThenWrite.release()); + for (int i = 0; i < 2; ++i) { + auto peerWriteRaw = ctx.WaitPDiskRequests<NPDisk::TEvChunkWriteRaw>({disk2.PDiskEdge, disk3.PDiskEdge}); + const TDiskHandle& respondingDisk = (peerWriteRaw->Recipient == disk2.PDiskEdge) ? disk2 : disk3; + ctx.SendPDiskResponse(respondingDisk, *peerWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + } + + auto tunnelResult = ctx.Runtime.WaitForEdgeActorEvent<NDDisk::TEvWritePersistentBuffersResult>(ctx.Edge, false); + UNIT_ASSERT_VALUES_EQUAL(tunnelResult->Get()->Record.ResultSize(), 2u); + for (ui32 i = 0; i < tunnelResult->Get()->Record.ResultSize(); ++i) { + const auto& wr = tunnelResult->Get()->Record.GetResult(i); + UNIT_ASSERT_C(wr.GetResult().GetStatus() == TReplyStatus::OK, TStringBuilder() + << "Re-replicated write " << i << " expected OK, got " + << TReplyStatus::E_Name(static_cast<TReplyStatus::E>(wr.GetResult().GetStatus()))); + } + + for (auto disk : {disk2, disk3}) { + NDDisk::TQueryCredentials peerCreds = Connect(ctx, disk.PBServiceId, 120, 1); + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(peerCreds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Re-replicated payload must match the original"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + "Re-replicated record must carry the same checksums as the original write, not an empty list"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Checksum " << i << " must survive TEvReadThenWritePersistentBuffers re-replication"); + } + } + } + + // 14. Batch header overflow: checksummed 8-sector records (the largest size still eligible for + // batching) accumulate in an open batch until one more record's metadata (TPersistentBufferLsnRecordHeader + // + its sector-location array + one checksum per data sector, see LsnRecordMetadataSize) no longer + // fits the shared 4 KiB header sector -- even though the batch's outer record-count limit + // (MaxLsnsPerPack, sized for checksum-less records) is not yet reached. That overflowing record + // must fall back to a direct (own-header) write instead of corrupting the header sector, and every + // record -- batched and direct alike -- must restore with correct payload and checksums after a + // restart. + Y_UNIT_TEST(BatchHeaderOverflowFallsBackToDirectWrite) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(67, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 121, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path instead of a direct write. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Compute, the same way LsnRecordMetadataSize does, exactly how many checksummed 8-sector + // records fit in the shared 4 KiB header sector alongside TPersistentBufferHeader: that many + // are sent first (all must batch, i.e. produce no PDisk write of their own), then one more is + // sent (must overflow and fall back to a direct write). This is deterministic -- no need to + // race for "did a write appear immediately", since A's own raw write was already dequeued above + // and batched records never produce a raw write, so the next TEvChunkWriteRaw on disk1 can only + // be the overflowing record's direct write. + constexpr ui32 kSectorsPerRecord = 8; + const ui32 recordMetadataSize = sizeof(NDDisk::TPersistentBufferLsnRecordHeader) + + kSectorsPerRecord * sizeof(NDDisk::TPersistentBufferSectorInfo) + + kSectorsPerRecord * sizeof(ui64); + const ui32 headerBudget = BlockSize - sizeof(NDDisk::TPersistentBufferHeader); + const ui32 batchedCount = headerBudget / recordMetadataSize; + UNIT_ASSERT_C(batchedCount >= 1, "Sanity: at least one checksummed 8-sector record must fit the header sector"); + + struct TRecordInfo { + TString Payload; + std::vector<ui64> Checksums; + NDDisk::TBlockSelector Selector; + ui64 Lsn; + }; + auto makeRecord = [&](ui32 i) { + TString payload = MakeData(static_cast<char>('a' + (i % 26)), kSectorsPerRecord * BlockSize); + const NDDisk::TBlockSelector selector{ui64(10 + i), 0, kSectorsPerRecord * BlockSize}; + const ui64 lsn = 100 + i; + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + TRecordInfo info{payload, + std::vector<ui64>(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()), + selector, lsn}; + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + return info; + }; + + std::vector<TRecordInfo> batched; + for (ui32 i = 0; i < batchedCount; ++i) { + batched.push_back(makeRecord(i)); + } + const TRecordInfo overflow = makeRecord(batchedCount); + + auto rawOverflow = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawOverflow->Get()->Data.size(), (kSectorsPerRecord + 1) * BlockSize, + "Overflowing record must fall back to its own direct (header + data) write"); + captureWrite(rawOverflow); + ctx.SendPDiskResponse(disk1, *rawOverflow, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto overflowResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(overflowResult, TReplyStatus::OK); + + // Complete A's write, then the batch's combined write (header + all batched records' data). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), (1 + batched.size() * kSectorsPerRecord) * BlockSize, + "Batch combined write must contain the header sector plus every batched record's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, static_cast<ui32>(batched.size()), + "BatchSize must equal the number of successfully batched records"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + for (ui32 i = 0; i < batched.size(); ++i) { + auto wr = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wr, TReplyStatus::OK); + } + + // Restart: restore from the captured chunk data and verify every record -- the overflowing + // direct write and every batched record -- comes back with the correct payload and checksums. + // Reading a restored record fetches its data sectors from PDisk on demand (restore only + // reconstructs in-memory metadata, not payload bytes), so intercept TEvChunkReadRaw to disk2 + // and serve it from the captured chunkBufs; also auto-answer TEvCheckSpace, which the actor + // may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 67, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 121, 1); + + auto checkRecord = [&](const TRecordInfo& info, const TString& label) { + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, info.Selector, info.Lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), info.Payload, + label << " payload mismatch"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), info.Checksums.size(), + label << " checksum count mismatch"); + for (ui32 i = 0; i < info.Checksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), info.Checksums[i], + label << " checksum " << i << " mismatch"); + } + }; + + checkRecord(overflow, "overflow (direct) record"); + for (ui32 i = 0; i < batched.size(); ++i) { + checkRecord(batched[i], TStringBuilder() << "batched record " << i); + } + } + + // 15. Partial-range read: a read selector narrower than the written record (and starting at a + // nonzero offset within it) must return both the correct payload sub-range and the matching + // checksum sub-range -- exercising the firstBlock/blockCount slicing in ReplyReadPersistentBuffer. + Y_UNIT_TEST(PartialRangeReadSlicesChecksums) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(68, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 122, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('R', 4 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 4 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> allChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(allChecksums.size(), 4u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + // Read blocks [1, 3) out of the 4-block record, i.e. a 2-block selector starting one block in. + const NDDisk::TBlockSelector partialSelector{5, BlockSize, 2 * BlockSize}; + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, partialSelector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload.substr(BlockSize, 2 * BlockSize), + "Partial read must return only the requested payload sub-range"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 2u, + "Partial read must return only the checksums for the requested sub-range, not all 4"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(0), allChecksums[1], + "First returned checksum must be the record's block 1 (not block 0)"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(1), allChecksums[2], + "Second returned checksum must be the record's block 2"); + } + + // 16. Mixed batch: a checksummed record and a checksum-less record share the same batch header. + // After a restart, the checksummed record must restore with its checksums and the checksum-less + // one must restore with an empty Checksums list -- exercising the per-record HAS_PAYLOAD_CHECKSUMS + // flag check during batch restore parsing, as opposed to a single Version-wide switch. + Y_UNIT_TEST(MixedChecksummedAndLegacyBatchRestoresCorrectly) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(69, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 123, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Write B: batched, WITHOUT checksums (legacy/opt-out). + const TString payloadB = MakeData('B', BlockSize); + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorB, 2, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadB)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Write C: batched, WITH checksums -- lands in the same batch header as B. + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + std::vector<ui64> checksumsC; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorC, 3, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + UNIT_ASSERT_VALUES_EQUAL(checksumsC.size(), 1u); + + // Complete A's write, then the batch's combined write (header + B's + C's data sectors). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), 3 * BlockSize, + "Batch combined write must contain the header sector plus B's and C's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Both B and C must land in the same batch"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + // Restart. Reading a restored record fetches its data sectors from PDisk on demand, so + // intercept TEvChunkReadRaw to disk2 and serve it from the captured chunkBufs; also + // auto-answer TEvCheckSpace, which the actor may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 69, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 123, 1); + + auto readB = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorB, 2, 1, {true})); + AssertStatus(readB, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readB->Get()->GetPayload(0).ConvertToString(), payloadB, + "Checksum-less batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readB->Get()->Record.ChecksumsSize()), 0u, + "Checksum-less batched record must restore with an empty Checksums list"); + + auto readC = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorC, 3, 1, {true})); + AssertStatus(readC, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->GetPayload(0).ConvertToString(), payloadC, + "Checksummed batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readC->Get()->Record.ChecksumsSize()), checksumsC.size(), + "Checksummed batched record must restore with its checksums, not an empty list"); + for (ui32 i = 0; i < checksumsC.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->Record.GetChecksums(i), checksumsC[i], + "Checksum " << i << " must survive restart for the checksummed batched record"); + } + } } } // anonymous namespace diff --git a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp index cec89be4f79..6b1e3b13457 100644 --- a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp +++ b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp @@ -147,6 +147,11 @@ namespace NKikimr::NDDisk { auto msg = std::make_unique<TEvWritePersistentBuffers>(creds, selector, inflight.Lsn, NDDisk::TWriteInstruction(0), inflight.PersistentBufferIds, inflight.Timeout); msg->AddPayload(TRope(payload)); + // Forward the checksums persisted on the source record, if any (TEvReadPersistentBufferResult + // carries them opt-in, see TPersistentBuffer::TRecord::PayloadChecksums). Without this, records + // re-replicated via TEvReadThenWritePersistentBuffers would silently lose their checksums on + // every target PB, exactly the recovery scenario where corruption detection matters most. + msg->Record.MutableChecksums()->CopyFrom(record.GetChecksums()); auto h = std::make_unique<IEventHandle>(SelfId(), inflight.Sender, msg.release(), 0, inflight.Cookie); TActivationContext::Send(h.release()); diff --git a/ydb/core/blobstorage/nodewarden/distconf.h b/ydb/core/blobstorage/nodewarden/distconf.h index f22c277079f..a194af1a701 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.h +++ b/ydb/core/blobstorage/nodewarden/distconf.h @@ -476,17 +476,25 @@ namespace NKikimr::NStorage { std::optional<TString> GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID); - void AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, ui32 groupGeneration, - TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, - i64 requiredSpace, NKikimrBlobStorage::TBaseConfig *baseConfig, - bool convertToDonor, bool ignoreVSlotQuotaCheck, bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, - std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes = {}, - bool applyNodeAllowList = false); + struct TAllocateStaticGroupParams { + NKikimrBlobStorage::TStorageConfig *Config = nullptr; + TGroupId GroupId; + ui32 GroupGeneration = 0; + TBlobStorageGroupType GroupType; + THashMap<TVDiskIdShort, NBsController::TPDiskId> ReplacedDisks; + NBsController::TGroupMapper::TForbiddenPDisks ForbiddenPDisks; + i64 RequiredSpace = 0; + const NKikimrBlobStorage::TBaseConfig *BaseConfig = nullptr; + bool ConvertToDonor = false; + bool IgnoreVSlotQuotaCheck = false; + bool AllowUnusableDisks = false; + bool IsSelfHealReasonDecommit = false; + TBridgePileId BridgePileId; + std::optional<TGroupId> BridgeProxyGroupId; + bool ApplySelfHealNodeAllowList = false; + }; + + void AllocateStaticGroup(TAllocateStaticGroupParams params); bool UpdateConfig(NKikimrBlobStorage::TStorageConfig *config); @@ -518,7 +526,9 @@ namespace NKikimr::NStorage { bool GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig , std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse = {} , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {} + , bool automaticManagement = true , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 200 diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp index 4dec85b0917..96e83a946b7 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp @@ -37,10 +37,15 @@ namespace NKikimr::NStorage { TGroupId groupId = TGroupId::Zero(); auto allocateGroup = [&](TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId) { - AllocateStaticGroup(config, groupId, /*groupGeneration=*/ 1, TBlobStorageGroupType(species), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, {}, {}, 0, - nullptr, false, true, false, bridgePileId, bridgeProxyGroupId); + AllocateStaticGroup({ + .Config = config, + .GroupId = groupId, + .GroupGeneration = 1, + .GroupType = TBlobStorageGroupType(species), + .IgnoreVSlotQuotaCheck = true, + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + }); const auto& groups = config->GetBlobStorageConfig().GetServiceSet().GetGroups(); const auto& allocatedGroup = groups.at(groups.size() - 1); @@ -127,7 +132,12 @@ namespace NKikimr::NStorage { config->Mutable##NAME##Config()->CopyFrom(Cfg->DomainsConfig->GetExplicit##NAME##Config()); \ } \ if (!config->Has##NAME##Config()) { \ - GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes); \ + std::unordered_set<ui32> nodesToUse; \ + if (Cfg->SelfManagementConfig && Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes, nodesToUse); \ } UPDATE_EXPLICIT_CONFIG(StateStorage) @@ -139,32 +149,28 @@ namespace NKikimr::NStorage { return std::nullopt; } - void TDistributedConfigKeeper::AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, - ui32 groupGeneration, TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, i64 requiredSpace, - NKikimrBlobStorage::TBaseConfig *baseConfig, bool convertToDonor, bool ignoreVSlotQuotaCheck, - bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes, bool applyNodeAllowList) { + void TDistributedConfigKeeper::AllocateStaticGroup(TAllocateStaticGroupParams params) { using TPDiskId = NBsController::TPDiskId; - NKikimrConfig::TBlobStorageConfig *bsConfig = config->MutableBlobStorageConfig(); + auto& config = *params.Config; + auto& replacedDisks = params.ReplacedDisks; + const auto& selfManagementConfig = config.GetSelfManagementConfig(); + const auto& selfHealAllowedNodes = selfManagementConfig.GetStaticGroupSelfHealAllowedNodes(); + NKikimrConfig::TBlobStorageConfig *bsConfig = config.MutableBlobStorageConfig(); // build node location map THashMap<ui32, TNodeLocation> nodeLocations; THashSet<ui32> allowedNodeIds; - for (const auto& node : config->GetAllNodes()) { + for (const auto& node : config.GetAllNodes()) { TNodeLocation location(node.GetLocation()); nodeLocations.try_emplace(node.GetNodeId(), location); - if (bridgePileId == ResolveNodePileId(location)) { + if (params.BridgePileId == ResolveNodePileId(location)) { allowedNodeIds.insert(node.GetNodeId()); } } // when restricting self-heal targets, only these node ids may host relocated vdisks - const bool restrictSelfHealNodes = applyNodeAllowList && !selfHealAllowedNodes.empty(); + const bool restrictSelfHealNodes = params.ApplySelfHealNodeAllowList && !selfHealAllowedNodes.empty(); const THashSet<ui32> selfHealAllowedNodeSet = restrictSelfHealNodes ? THashSet<ui32>(selfHealAllowedNodes.begin(), selfHealAllowedNodes.end()) : THashSet<ui32>{}; @@ -181,10 +187,10 @@ namespace NKikimr::NStorage { THashMap<TPDiskId, TPDiskInfo> pdisks; auto checkMatch = [&](NKikimrBlobStorage::EPDiskType type, bool sharedWithOs, bool readCentric, ui64 kind) { - if (type == pdiskType) { + if (selfManagementConfig.HasPDiskType() && type == selfManagementConfig.GetPDiskType()) { return true; } - for (const auto& pdiskFilter : pdiskFilters) { + for (const auto& pdiskFilter : selfManagementConfig.GetPDiskFilter()) { bool m = true; for (const auto& p : pdiskFilter.GetProperty()) { bool pMatch = false; @@ -218,12 +224,13 @@ namespace NKikimr::NStorage { ui32 defaultMaxSlots = 16; - if (baseConfig) { + if (params.BaseConfig) { + const auto& baseConfig = *params.BaseConfig; std::optional<NKikimrBlobStorage::TPDiskSpaceColor::E> pdiskSpaceColorBorder; ui32 pdiskSpaceMarginPromille = 150; - if (baseConfig->HasSettings()) { - const auto& settings = baseConfig->GetSettings(); + if (baseConfig.HasSettings()) { + const auto& settings = baseConfig.GetSettings(); if (settings.DefaultMaxSlotsSize()) { defaultMaxSlots = settings.GetDefaultMaxSlots(0); } @@ -235,7 +242,7 @@ namespace NKikimr::NStorage { } } - for (const auto& pdisk : baseConfig->GetPDisk()) { + for (const auto& pdisk : baseConfig.GetPDisk()) { if (!checkMatch(pdisk.GetType(), pdisk.GetSharedWithOs(), pdisk.GetReadCentric(), pdisk.GetKind())) { continue; } @@ -261,13 +268,13 @@ namespace NKikimr::NStorage { } const bool usableInTermsOfDecommission = pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE || - pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !isSelfHealReasonDecommit; + pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !params.IsSelfHealReasonDecommit; if (!usableInTermsOfDecommission) { pdiskInfo.Usable = false; pdiskInfo.WhyUnusable += 'D'; } - if (!ignoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig->HasSettings()) { + if (!params.IgnoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig.HasSettings()) { const auto& m = pdisk.GetPDiskMetrics(); if (m.HasEnforcedDynamicSlotSize() && pdiskSpaceColorBorder >= NKikimrBlobStorage::TPDiskSpaceColor::YELLOW) { pdiskInfo.SpaceAvailable = m.GetEnforcedDynamicSlotSize() * (1000 - pdiskSpaceMarginPromille) / 1000; @@ -282,7 +289,7 @@ namespace NKikimr::NStorage { } THashMap<ui32, ui64> maxGroupSlotSize; - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { if (vslot.HasVDiskMetrics()) { if (const auto& m = vslot.GetVDiskMetrics(); m.HasAllocatedSize()) { ui64& size = maxGroupSlotSize[vslot.GetGroupId()]; @@ -291,7 +298,7 @@ namespace NKikimr::NStorage { } } - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { const auto& vslotId = vslot.GetVSlotId(); const TPDiskId pdiskId(vslotId.GetNodeId(), vslotId.GetPDiskId()); if (const auto it = pdisks.find(pdiskId); it != pdisks.end()) { @@ -321,7 +328,7 @@ namespace NKikimr::NStorage { std::optional<ui32> generation; for (const auto& group : ss.GetGroups()) { - if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { generation.emplace(group.GetGroupGeneration()); ui32 failRealmIdx = 0; @@ -362,7 +369,7 @@ namespace NKikimr::NStorage { for (const auto& vdisk : ss.GetVDisks()) { const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID == groupId) { + if (vdiskId.GroupID == params.GroupId) { if (!generation) { throw TExConfigError() << "missing record for group being reconfigured"; } else if (vdiskId.GroupGeneration == *generation && !replacedDisks.contains(vdiskId)) { @@ -375,10 +382,12 @@ namespace NKikimr::NStorage { for (const auto& pdisk : ss.GetPDisks()) { const TPDiskId pdiskId(pdisk.GetNodeID(), pdisk.GetPDiskID()); - if (requiredPDiskIds.contains(pdiskId)) { + if (params.AllowUnusableDisks && requiredPDiskIds.contains(pdiskId)) { if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) { - auto& r = it->second.Record; - r.CopyFrom(pdisk); + TPDiskInfo& pdiskInfo = it->second; + pdiskInfo.Record.CopyFrom(pdisk); + pdiskInfo.Usable = false; + pdiskInfo.WhyUnusable += 'X'; } } @@ -438,10 +447,10 @@ namespace NKikimr::NStorage { } } }; - EnumerateConfigDrives(*config, 0, processDrive, nullptr, true); + EnumerateConfigDrives(config, 0, processDrive, nullptr, true); // group mapper - NBsController::TGroupGeometryInfo geom(gtype.GetErasure(), geometry); + NBsController::TGroupGeometryInfo geom(params.GroupType.GetErasure(), selfManagementConfig.GetGeometry()); NBsController::TGroupMapper mapper(geom); for (const auto& [pdiskId, item] : pdisks) { @@ -460,7 +469,7 @@ namespace NKikimr::NStorage { slotSizeInUnits = pdiskConfig.GetSlotSizeInUnits(); } - const bool pileFilter = !bridgePileId || allowedNodeIds.contains(pdiskId.NodeId); + const bool pileFilter = !params.BridgePileId || allowedNodeIds.contains(pdiskId.NodeId); const bool nodeAllowFilter = !restrictSelfHealNodes || selfHealAllowedNodeSet.contains(pdiskId.NodeId); TString whyUnusable = item.WhyUnusable; @@ -502,8 +511,8 @@ namespace NKikimr::NStorage { NBsController::TGroupMapperError error; const ui32 groupSizeInUnits = 1; // static groups are always single-unit - if (!mapper.AllocateGroup(groupId.GetRawId(), groupDefinition, replacedDisks, forbid, - groupSizeInUnits, requiredSpace, false, {}, error)) { + if (!mapper.AllocateGroup(params.GroupId.GetRawId(), groupDefinition, replacedDisks, + params.ForbiddenPDisks, groupSizeInUnits, params.RequiredSpace, false, {}, error)) { throw TExConfigError() << "group allocation failed Error# " << error.ErrorMessage << " groupDefinition# " << dumpGroupDefinition(); } @@ -513,38 +522,38 @@ namespace NKikimr::NStorage { NKikimrBlobStorage::TGroupInfo *sGroup = nullptr; for (size_t i = 0; i < sSet->GroupsSize(); ++i) { if (const auto& group = sSet->GetGroups(i); TGroupId::FromProto(&group, - &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { sGroup = sSet->MutableGroups(i); break; } } if (!sGroup) { sGroup = sSet->AddGroups(); - groupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); - sGroup->SetErasureSpecies(gtype.GetErasure()); + params.GroupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); + sGroup->SetErasureSpecies(params.GroupType.GetErasure()); } else { sGroup->ClearRings(); } - sGroup->SetGroupGeneration(groupGeneration); + sGroup->SetGroupGeneration(params.GroupGeneration); - if (bridgeProxyGroupId) { - bridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); + if (params.BridgeProxyGroupId) { + params.BridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); } - bridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); + params.BridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); THashMap<TVDiskIdShort, NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk::TDonor>> donors; for (size_t i = 0; i < sSet->VDisksSize(); ++i) { const auto& vdisk = sSet->GetVDisks(i); const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID != groupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { + if (vdiskId.GroupID != params.GroupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { continue; } auto *m = sSet->MutableVDisks(i); if (replacedDisks.contains(vdiskId)) { if (m->HasDonorMode()) { // this disk is already a donor, nothing to do about it - } else if (convertToDonor) { + } else if (params.ConvertToDonor) { // make this disk a donor auto *donorMode = m->MutableDonorMode(); donorMode->SetNumFailRealms(groupDefinition.size()); @@ -560,7 +569,7 @@ namespace NKikimr::NStorage { donor->MutableVDiskId()->CopyFrom(m->GetVDiskID()); donor->MutableVDiskLocation()->CopyFrom(m->GetVDiskLocation()); } else { - m->MutableVDiskID()->SetGroupGeneration(groupGeneration); + m->MutableVDiskID()->SetGroupGeneration(params.GroupGeneration); } } @@ -596,7 +605,7 @@ namespace NKikimr::NStorage { sLoc->SetPDiskGuid(pdisk.GetPDiskGuid()); auto *sDisk = sSet->AddVDisks(); - VDiskIDFromVDiskID(TVDiskID(groupId, groupGeneration, vdiskId), sDisk->MutableVDiskID()); + VDiskIDFromVDiskID(TVDiskID(params.GroupId, params.GroupGeneration, vdiskId), sDisk->MutableVDiskID()); sDisk->SetVDiskKind(NKikimrBlobStorage::TVDiskKind::Default); sDisk->MutableVDiskLocation()->CopyFrom(*sLoc); if (const auto it = donors.find(vdiskId); it != donors.end()) { @@ -608,14 +617,23 @@ namespace NKikimr::NStorage { bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig + , bool automaticManagement , ui32 overrideReplicasInRingCount , ui32 overrideRingsCount , ui32 replicasSpecificVolume ) { + if (!automaticManagement) { + ss->CopyFrom(oldConfig); + return true; + } std::map<TBridgePileId, THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>> nodes; bool goodConfig = true; for (const auto& node : baseConfig.GetAllNodes()) { + if (!nodesToUse.empty() && !nodesToUse.contains(node.GetNodeId())) { + continue; + } TNodeLocation location(node.GetLocation()); TBridgePileId pileId = ResolveNodePileId(location); nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location); diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp index 2dc6f9d839d..9b3d68d6c2f 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp @@ -15,15 +15,27 @@ namespace NKikimr::NStorage { const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig; bool result = true; std::unordered_set<ui32> usedNodes; - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + +#define GENERATE_CONFIG(NAME) \ + { \ + std::unordered_set<ui32> nodesToUse; \ + if (Self->Cfg->SelfManagementConfig && Self->Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Self->Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + const bool automaticManagement = !Self->Cfg->SelfManagementConfig || Self->Cfg->SelfManagementConfig->GetAutomatic##NAME##Management(); \ + result &= Self->GenerateStateStorageConfig(currentConfig->Mutable##NAME##Config(), config, usedNodes, nodesToUse, config.Get##NAME##Config(), automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); \ + if (pileupReplicas) { \ + usedNodes.clear(); \ + } \ + } + + GENERATE_CONFIG(StateStorage) + GENERATE_CONFIG(StateStorageBoard) + GENERATE_CONFIG(SchemeBoard) + +#undef GENERATE_CONFIG + return result; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp index 30884b3f0cf..fc3e718fe03 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp @@ -176,8 +176,6 @@ namespace NKikimr::NStorage { } const auto& ss = bsConfig.GetServiceSet(); - const auto& smConfig = config.GetSelfManagementConfig(); - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; NBsController::TGroupMapper::TForbiddenPDisks forbid; for (const auto& vdisk : ss.GetVDisks()) { @@ -207,14 +205,23 @@ namespace NKikimr::NStorage { std::optional<TGroupId> bridgeProxyGroupId = group.HasBridgeProxyGroupId() ? std::make_optional(TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetBridgeProxyGroupId)) : std::nullopt; - Self->AllocateStaticGroup(&config, vdiskId.GroupID, vdiskId.GroupGeneration + 1, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, - replacedDisks, forbid, maxSlotSize, - &BaseConfig.value(), cmd.GetConvertToDonor(), cmd.GetIgnoreVSlotQuotaCheck(), - cmd.GetIsSelfHealReasonDecommit(), bridgePileId, bridgeProxyGroupId, - smConfig.GetStaticGroupSelfHealAllowedNodes(), cmd.GetFromSelfHeal()); + Self->AllocateStaticGroup({ + .Config = &config, + .GroupId = vdiskId.GroupID, + .GroupGeneration = vdiskId.GroupGeneration + 1, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .RequiredSpace = static_cast<i64>(maxSlotSize), + .BaseConfig = &BaseConfig.value(), + .ConvertToDonor = cmd.GetConvertToDonor(), + .IgnoreVSlotQuotaCheck = cmd.GetIgnoreVSlotQuotaCheck(), + .AllowUnusableDisks = cmd.GetAllowUnusableDisks(), + .IsSelfHealReasonDecommit = cmd.GetIsSelfHealReasonDecommit(), + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + .ApplySelfHealNodeAllowList = cmd.GetFromSelfHeal(), + }); } catch (const TExConfigError& ex) { YDB_LOG_NOTICE("ReassignGroupDisk failed to allocate group", {"marker", "NWDC76"}, diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp index aa8b5bc41fb..052c77e5e6d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp @@ -70,13 +70,23 @@ namespace NKikimr::NStorage { RingsInGroupCount = minNodesInGroup < 8 ? minNodesInGroup : 8; if (RingsInGroupCount > minNodesInGroup) { RingsInGroupCount = minNodesInGroup; + if (OverrideRingsCount > 0) { + EnoughNodesForOverride = false; + } } NToSelect = RingsInGroupCount < 3 ? 1 : (RingsInGroupCount < 5 ? 3 : 5); ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / ReplicasSpecificVolume); } else { if (OverrideRingsCount == 3 || OverrideRingsCount == 9) { RingsInGroupCount = OverrideRingsCount / 3; + if (RingsInGroupCount > minNodesInGroup) { + RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; + EnoughNodesForOverride = false; + } } else { + // any other value (including 0, meaning "no override") is not applicable to the + // multi-group (3-DC) topology and is silently ignored - this is not an insufficient + // nodes condition, just a non-applicable override RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; } NToSelect = RingsInGroupCount < 3 ? 3 : 9; @@ -87,11 +97,17 @@ namespace NKikimr::NStorage { ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / ReplicasSpecificVolume); } if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) { + if (OverrideReplicasInRingCount > 1) { + EnoughNodesForOverride = false; + } ReplicasInRingCount = 1; } } bool TStateStoragePerPileGenerator::IsGoodConfig() const { + if (!EnoughNodesForOverride) { + return false; + } for (auto &nodes : Rings) { for (auto nodeId : nodes) { if (CalcNodeState(nodeId, false) > 1) { @@ -102,6 +118,10 @@ namespace NKikimr::NStorage { return true; } + bool TStateStoragePerPileGenerator::IsEnoughNodesForOverride() const { + return EnoughNodesForOverride; + } + void TStateStoragePerPileGenerator::AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss) { auto *rg = ss->AddRingGroups(); PileId.CopyToProto(rg, &NKikimrConfig::TDomainsConfig::TStateStorage::TRing::SetBridgePileId); diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h index 1a55b2c7bd0..4fb5702b0f9 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h @@ -20,6 +20,8 @@ namespace NKikimr::NStorage { bool IsGoodConfig() const; void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss); + bool IsEnoughNodesForOverride() const; + private: static constexpr ui32 NodeStatesSize = (ui32)ENodeState::NODE_STATE_MAX; @@ -48,5 +50,6 @@ namespace NKikimr::NStorage { ui32 OverrideReplicasInRingCount = 0; ui32 OverrideRingsCount = 0; ui32 ReplicasSpecificVolume = 200; + bool EnoughNodesForOverride = true; }; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp index 28d707927f0..cbb9d93aa45 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp @@ -23,7 +23,7 @@ namespace NBlobStorageNodeWardenTest{ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { - NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200) { + NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200, std::unordered_set<ui32> nodesToUse = {}, bool *goodConfigOut = nullptr, const NKikimrConfig::TDomainsConfig::TStateStorage& oldSS = {}, bool automaticManagement = true) { NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); NKikimrConfig::TDomainsConfig::TStateStorage ss; NKikimrBlobStorage::TStorageConfig config; @@ -31,7 +31,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { auto *node = config.AddAllNodes(); node->SetNodeId(i + 1); } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -46,6 +49,9 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 1000 + , std::unordered_set<ui32> nodesToUse = {} + , bool *goodConfigOut = nullptr + , bool automaticManagement = true ) { NKikimrBlobStorage::TStorageConfig config; ui32 nodeId = 1; @@ -74,7 +80,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { for (auto [nodeId, state] : nodesState) { keeper.SelfHealNodesState[nodeId] = state; } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -209,6 +218,158 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { " Ring { Node: 4 Node: 5 Node: 6 } Ring { Node: 7 Node: 8 Node: 9 } } }"); CheckStateStorage2(GenerateSimpleStateStorage(16, {}, 0, 2, 5), "{ RingGroups { NToSelect: 1 Ring { Node: 1 Node: 2 Node: 3 Node: 4 } Ring { Node: 5 Node: 6 Node: 7 Node: 8 } } }"); } + + Y_UNIT_TEST(NodesToUseEmptyMeansNoRestriction) { + // Explicit regression check: passing an empty nodesToUse set behaves exactly + // like not passing the parameter at all (default argument in other tests). + CheckStateStorage(GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}), 5, {1, 2, 3, 4, 5, 6, 7, 8}); + CheckStateStorage(GenerateDCStateStorage(3, 1, 1, {}, {}, {}, 9, 0, 0, 1000, {}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseRestrictsSimplePool) { + // Out of 10 available nodes, only {5, 6, 7} are allowed to be used. + CheckStateStorage(GenerateSimpleStateStorage(10, {}, 0, 0, 200, {5, 6, 7}), 3, {5, 6, 7}); + } + + Y_UNIT_TEST(NodesToUseKeepsOriginalNodeIds) { + // Restrict a big pool of 100 nodes down to 8 specific nodes; the algorithm + // should behave exactly as if only those 8 nodes existed, while preserving + // their original (non-contiguous) node ids. + CheckStateStorage( + GenerateSimpleStateStorage(100, {}, 0, 0, 200, {10, 20, 30, 40, 50, 60, 70, 80}), + 5, {10, 20, 30, 40, 50, 60, 70, 80}); + } + + Y_UNIT_TEST(NodesToUseIgnoresUnknownNodeIds) { + // Node ids in nodesToUse that are not present in baseConfig's AllNodes + // must simply be ignored, not cause a crash or a bad config. + CheckStateStorage(GenerateSimpleStateStorage(3, {}, 0, 0, 200, {1, 2, 3, 9999}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseForcesAlternateNodeSelectionInDC) { + // Without restriction, the default pick within each 3-node rack is its + // first node: {1, 4, 7, 10, 13, 16, 19, 22, 25}. + CheckStateStorage(GenerateDCStateStorage(3, 3, 3), 9, {1, 4, 7, 10, 13, 16, 19, 22, 25}); + + // Excluding those default nodes from nodesToUse forces the generator to + // fall back to the next available node in each rack. + std::unordered_set<ui32> nodesToUse = { + 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, 24, 26, 27 + }; + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, 0, 1000, nodesToUse), + 9, {2, 5, 8, 11, 14, 17, 20, 23, 26}); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Cases where the requested configuration (via overrideRingsCount/overrideReplicasInRingCount) + // can not be honored because nodesToUse restricts the available nodes too much. + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountSingleGroup) { + // Only 3 nodes are usable (all with a known GOOD self-heal state, via GenerateDCStateStorage), + // but the caller explicitly asks for 8 rings: impossible to satisfy. + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig); + UNIT_ASSERT(!goodConfig); + + // Sanity check: with exactly as many nodes as requested rings, the override is satisfiable. + bool goodConfig2 = false; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, 0, /*overrideRingsCount=*/3, 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig2); + UNIT_ASSERT(goodConfig2); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideReplicasInRingCountSingleGroup) { + // Only 2 nodes are usable, but the caller explicitly asks for 3 replicas per ring: + // impossible to satisfy (would need at least 3 nodes for a single ring). + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/3, /*overrideRingsCount=*/1, + 1000, /*nodesToUse=*/{1, 2}, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountMultiGroup) { + // 3-DC topology, but nodesToUse leaves only 2 nodes in one of the DCs while the + // caller explicitly requests 9 rings (3 per group) - the smallest group can't supply 3 rings. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, // dc-0: full rack representation (3 nodes) + 10, 13, // dc-1: only 2 nodes available + 19, 22, 25, // dc-2: full rack representation (3 nodes) + }; + bool goodConfig = true; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseSufficientForOverrideKeepsGoodConfig) { + // Same restriction style as above, but this time nodesToUse leaves exactly enough + // nodes in every group to satisfy the override - the config must be reported as good. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, + 10, 13, 16, + 19, 22, 25, + }; + bool goodConfig = false; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(goodConfig); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // AutomaticManagement (AutomaticStateStorageManagement / AutomaticStateStorageBoardManagement / + // AutomaticSchemeBoardManagement) semantics at the GenerateStateStorageConfig level: when the + // subsystem's automatic management is disabled, self-heal must keep the current (old) config + // untouched and report the result as "good", regardless of node states or overrides. + + Y_UNIT_TEST(AutomaticManagementDisabledKeepsOldConfigUnchanged) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/false); + + // The generator must not have touched node selection at all: result equals oldSS verbatim. + UNIT_ASSERT_EQUAL(TStringBuilder() << ss, TStringBuilder() << oldSS); + UNIT_ASSERT(goodConfig); + } + + Y_UNIT_TEST(AutomaticManagementEnabledGeneratesNewConfig) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/true); + + // With automatic management enabled (default), a fresh configuration must be generated + // from the current node pool, ignoring the stale oldSS content. + UNIT_ASSERT(TStringBuilder() << ss != TStringBuilder() << oldSS); + CheckStateStorage(ss, 5, {1, 2, 3, 4, 5, 6, 7, 8}); + } + + Y_UNIT_TEST(AutomaticManagementDisabledIgnoresBadNodeStates) { + // Even if nodesToUse/overrides would normally produce a bad config, disabling automatic + // management must short-circuit generation entirely and still report "good", since the + // old config is kept as-is and self-heal is not supposed to touch it. + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(7); + } + bool goodConfig = false; + GenerateSimpleStateStorage(100, {}, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 200, /*nodesToUse=*/{1, 2, 3}, &goodConfig, oldSS, /*automaticManagement=*/false); + UNIT_ASSERT(goodConfig); + } } Y_UNIT_TEST_SUITE(TDeriveStorageConfigCleanupTest) { @@ -438,7 +599,7 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { NKikimrBlobStorage::TStorageConfig Config; NKikimrBlobStorage::TBaseConfig BaseConfig; - void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, + void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, const TString& dataCenter = "", const TString& rack = "") { auto *node = Config.AddAllNodes(); node->SetNodeId(nodeId); @@ -552,6 +713,16 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } return result; } + + void SetPDiskType(ui32 nodeId, NKikimrBlobStorage::EPDiskType type) { + for (auto& pdisk : *BaseConfig.MutablePDisk()) { + if (pdisk.GetNodeId() == nodeId) { + pdisk.SetType(type); + return; + } + } + UNIT_FAIL("PDisk not found"); + } }; NKikimrBlobStorage::TGroupGeometry Geometry(ui32 numFailDomains) { @@ -563,16 +734,28 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } void Reallocate(TSetup& s, const NProtoBuf::RepeatedField<ui32>& allowedNodeIds, bool applyNodeAllowList, - i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1) { + i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1, bool allowUnusableDisks = false) { + auto *selfManagementConfig = s.Config.MutableSelfManagementConfig(); + selfManagementConfig->MutableGeometry()->CopyFrom(Geometry(numFailDomains)); + selfManagementConfig->SetPDiskType(NKikimrBlobStorage::EPDiskType::ROT); + selfManagementConfig->MutableStaticGroupSelfHealAllowedNodes()->CopyFrom(allowedNodeIds); + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; replacedDisks.emplace(TVDiskIdShort(0, 0, 0), NBsController::TPDiskId()); NBsController::TGroupMapper::TForbiddenPDisks forbid; - keeper.AllocateStaticGroup(&s.Config, TGroupId::FromValue(0), /*groupGeneration=*/ 2, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), Geometry(numFailDomains), - /*pdiskFilters=*/ {}, NKikimrBlobStorage::EPDiskType::ROT, replacedDisks, forbid, - /*requiredSpace=*/ 0, &s.BaseConfig, /*convertToDonor=*/ false, /*ignoreVSlotQuotaCheck=*/ true, - /*isSelfHealReasonDecommit=*/ false, TBridgePileId(), std::nullopt, allowedNodeIds, applyNodeAllowList); + keeper.AllocateStaticGroup({ + .Config = &s.Config, + .GroupId = TGroupId::FromValue(0), + .GroupGeneration = 2, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .BaseConfig = &s.BaseConfig, + .IgnoreVSlotQuotaCheck = true, + .AllowUnusableDisks = allowUnusableDisks, + .ApplySelfHealNodeAllowList = applyNodeAllowList, + }); } NProtoBuf::RepeatedField<ui32> NodeIds(const std::vector<ui32>& nodeIds) { @@ -651,6 +834,23 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { Reallocate(s, NodeIds({2}), /*applyNodeAllowList=*/ true, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); } + + Y_UNIT_TEST(Block42RejectsUnusablePreservedPDiskByDefault) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), + NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); + } + + Y_UNIT_TEST(Block42AllowsUnusablePreservedPDiskWhenRequested) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_NO_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8, + /*allowUnusableDisks=*/ true)); + UNIT_ASSERT_VALUES_EQUAL(s.GetGroupDomainNodes()[1], 2u); + } } } diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp index 317c3594504..8e20e6b7ca0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp @@ -9,7 +9,9 @@ #include <yql/essentials/types/binary_json/write.h> #include <util/generic/ylimits.h> +#include <util/stream/str.h> #include <util/string/cast.h> +#include <util/string/escape.h> #include <cmath> #include <limits> @@ -19,8 +21,11 @@ namespace NKikimr::NArrow::NAccessor { namespace { NBinaryJson::TBinaryJson JsonValueToBinaryJsonVerified(const NJson::TJsonValue& json) { - auto result = NBinaryJson::SerializeToBinaryJson(NJson::WriteJson(&json, false)); - AFL_VERIFY(std::holds_alternative<NBinaryJson::TBinaryJson>(result)); + const auto dumpedJson = WriteJsonRoundTripSafe(json); + auto result = NBinaryJson::SerializeToBinaryJson(dumpedJson); + if (std::holds_alternative<TString>(result)) { + AFL_VERIFY(false)("error", std::get<TString>(result))("type", json.GetType())("input_dump", EscapeC(dumpedJson)); + } return std::get<NBinaryJson::TBinaryJson>(std::move(result)); } @@ -159,4 +164,13 @@ std::optional<TStringBuf> TJsonValueView::ScalarFromBinaryJson() const { } } +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json) { + NJson::TJsonWriterConfig config; + // avoid precision loss on double conversion + config.FloatToStringMode = PREC_AUTO; + TStringStream dumpedJson; + NJson::WriteJson(&dumpedJson, &json, config); + return dumpedJson.Str(); +} + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.h b/ydb/core/formats/arrow/accessor/common/json_value_view.h index a2aa5c21b16..3d6bbc39fc0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.h +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.h @@ -57,4 +57,7 @@ private: mutable TString ScalarHolder; }; +// Serialize a JSON value to text with round-trip-safe number formatting. +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json); + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp index 58894ffd10f..2e69ef7d567 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp @@ -1,4 +1,5 @@ #include "accessor.h" +#include "types.h" #include "direct_builder.h" #include "signals.h" @@ -137,7 +138,7 @@ TConclusion<NBinaryJson::TBinaryJson> ToBinaryJson(const TJsonRestorer& restorer [](NBinaryJson::TBinaryJson&& val) -> TConclusion<NBinaryJson::TBinaryJson> { return std::move(val); }}, - NBinaryJson::SerializeToBinaryJson(restorer.GetResult().GetStringRobust())); + NBinaryJson::SerializeToBinaryJson(WriteJsonRoundTripSafe(restorer.GetResult()))); } std::shared_ptr<arrow::Array> TSubColumnsArray::BuildBJsonArray(const TColumnConstructionContext& context) const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp index 7c01560f1fc..f6f8fd975e0 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp @@ -121,7 +121,7 @@ public: TDictStats::TBuilder statBuilder; for (auto&& i : UsedKeys) { statBuilder.Add(i.second.GetKeyName(), i.second.GetRecordsCount(), i.second.GetDataSize(), - i.second.GetAccessorType(settings, recordsCount), i.second.GetValueType()); + i.second.GetAccessorType(settings, recordsCount), OthersExplicitBinaryJson); } return statBuilder.Finish(); } @@ -250,7 +250,7 @@ TOthersData TOthersData::BuildEmpty() { TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(const std::string_view path, const ui32 recordsCount) const { auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < Stats.GetColumnsCount(); ++i) { - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("error", insertResult.GetErrorMessage()); } auto accessorResult = jsonPathAccessorTrie->GetAccessor(path); @@ -266,7 +266,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con auto idx = accessor->GetCookie(); if (!idx) { return std::make_shared<TJsonPathAccessor>( - std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, EValueType::BinaryJson); + std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, OthersExplicitBinaryJson); } TColumnFilter filter = TColumnFilter::BuildAllowFilter(); for (TIterator it(Records); it.IsValid(); it.Next()) { @@ -278,7 +278,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con TSparsedArray::TBuilder builder(nullptr, arrow::binary()); auto batch = ToBatch(table); builder.AddChunk(recordsCount, batch->GetColumnByName("record_idx"), batch->GetColumnByName("value")); - return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), EValueType::BinaryJson); + return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), OthersExplicitBinaryJson); } NArrow::NAccessor::TJsonValueView TOthersData::TIterator::GetValue() const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp index b08040273ae..02e05754011 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp @@ -12,7 +12,7 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 recordsCount) const { +TDictStats TDictStats::SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const { std::map<ui64, std::vector<TRTStats>> bySize; ui64 sumSize = 0; for (ui32 i = 0; i < GetColumnsCount(); ++i) { @@ -20,31 +20,21 @@ TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 sumSize += GetColumnSize(i); } std::vector<TRTStats> columnStats; - std::vector<TRTStats> otherStats; TSettings::TColumnsDistributor distributor = settings.BuildDistributor(sumSize, recordsCount); for (auto it = bySize.rbegin(); it != bySize.rend(); ++it) { for (auto&& i : it->second) { - switch (distributor.TakeAndDetect(it->first, i.GetRecordsCount())) { - case TSettings::TColumnsDistributor::EColumnType::Separated: - columnStats.emplace_back(std::move(i)); - break; - case TSettings::TColumnsDistributor::EColumnType::Other: - otherStats.emplace_back(std::move(i)); - break; + // Keys not taken as separated fall into the Others store, whose stats are built elsewhere. + if (distributor.TakeAndDetect(it->first, i.GetRecordsCount()) == TSettings::TColumnsDistributor::EColumnType::Separated) { + columnStats.emplace_back(std::move(i)); } } } std::sort(columnStats.begin(), columnStats.end()); - std::sort(otherStats.begin(), otherStats.end()); auto columnsBuilder = MakeBuilder(); - auto othersBuilder = MakeBuilder(); for (auto&& i : columnStats) { columnsBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); } - for (auto&& i : otherStats) { - othersBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); - } - return TSplittedColumns(columnsBuilder.Finish(), othersBuilder.Finish()); + return columnsBuilder.Finish(); } TDictStats TDictStats::Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount) { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.h b/ydb/core/formats/arrow/accessor/sub_columns/stats.h index e93b4bce0c6..b065ce7290f 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.h @@ -15,8 +15,6 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -class TSplittedColumns; - class TDictStats { private: using TJsonPathAccessorTriePtr = std::shared_ptr<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>; @@ -33,7 +31,7 @@ private: auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < DataNames->length(); ++i) { const auto arrView = DataNames->GetView(i); - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("key_name", TStringBuf(arrView.data(), arrView.size())) ("json_path", ToJsonPath(TStringBuf(arrView.data(), arrView.size()))) ("error", insertResult.GetErrorMessage()); @@ -156,7 +154,9 @@ public: static TDictStats Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount); - TSplittedColumns SplitByVolume(const TSettings& settings, const ui32 recordsCount) const; + // Selects which keys become separated columns; + // the rest fall into the Others store, whose stats are built by the caller. + TDictStats SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const; class TBuilder: TNonCopyable { private: @@ -242,22 +242,4 @@ public: TDictStats(const std::shared_ptr<arrow::RecordBatch>& original); }; - -class TSplittedColumns { -private: - TDictStats Columns; - TDictStats Others; - -public: - TDictStats ExtractColumns() { - return std::move(Columns); - } - TDictStats ExtractOthers() { - return std::move(Others); - } - TSplittedColumns(TDictStats&& columns, TDictStats&& others) - : Columns(std::move(columns)) - , Others(std::move(others)) { - } -}; } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp index fb0819375b7..6a2e18197b8 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp @@ -60,6 +60,7 @@ public: std::shared_ptr<arrow::DataType> GetArrowType() const override { return arrow::binary(); } + ui32 GetElementSize(const arrow::Array& array, const i64 index) const override { return BinaryView(array, index).size(); } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp index 8335760024a..5f83768df24 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp @@ -11,6 +11,7 @@ #include <yql/essentials/types/binary_json/write.h> #include <algorithm> +#include <limits> using NKikimr::NArrow::NAccessor::NSubColumns::NTesting::PrintBinaryJsons; @@ -220,4 +221,25 @@ Y_UNIT_TEST_SUITE(SubColumnsNativeScalars) { UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.n"), "1;2.5;<null>;-3;"); UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.b"), "true;false;<null>;true;"); } + + Y_UNIT_TEST(ReencodeNearMaxDouble) { + const std::vector<std::pair<TString, double>> cases = { + {"1.7976931348623157e308", 1.7976931348623157e308}, + {"-1.7976931348623157e308", -1.7976931348623157e308}, + }; + for (const auto& [lit, expected] : cases) { + const TString doc = TString(TStringBuilder() << "{\"n\":" << lit << "}"); + auto native = BuildSubColumns({doc}, NativeSettings(0)); + UNIT_ASSERT_VALUES_EQUAL_C(CountValueType(native, EValueType::Double), 1, native->DebugJson().GetStringRobust()); + UNIT_ASSERT(native->GetChunkedArray()); // read-back document reconstruction must not abort + const auto assertExact = [&](const std::shared_ptr<TSubColumnsArray>& arr) { + auto it = arr->GetColumnsData().BuildIterator(0); + auto bj = GetCodecForValueType(EValueType::Double)->ReadValueView(it.GetArray(), it.GetLocalIndex()).ToBinaryJson(); + auto reader = NBinaryJson::TBinaryJsonReader::Make(bj); + UNIT_ASSERT_VALUES_EQUAL(reader->GetRootCursor().GetElement(0).GetNumber(), expected); + }; + assertExact(native); + assertExact(SerializeRoundTrip(native, NativeSettings(0))); + } + } }; diff --git a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h index a6174a04bf2..a42012d4807 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h @@ -13,4 +13,7 @@ enum class EValueType : ui8 { String = 3, }; +// The Others store holds and reads every value as BinaryJson, whatever its logical type. +inline constexpr EValueType OthersExplicitBinaryJson = EValueType::BinaryJson; + } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp index ec839ca3fd9..3e440fd148b 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp +++ b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp @@ -323,6 +323,29 @@ private: return; } + if (!featureFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("has_full_scan") || configMap.count("has_path") || + configMap.count("has_stream") || configMap.count("has_app_name")) { + FailAndPassAway("Advanced resource pool classifier properties are disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + + if (!featureFlags.GetEnableRejectActionInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("action")) { + FailAndPassAway("Resource pool classifier action property is disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + FeatureFlagChecked = true; ValidateExistence(); ValidatePools(); diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp index cd7e2a8f400..df938397094 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp @@ -165,6 +165,26 @@ TVector<TExprNode::TPtr> TPhysicalQueryBuilder::BuildPhysicalStageGraph() { TExprNode::TPtr stage; if (Graph.IsSourceStageRowType(id)) { stage = Stages.at(id); + // Want to build materialize for ranges. + // TODO: Actually old optimizer has some machinery to compute ranges during compilation for some cases, for + // example when `LiteralRange` is defined, but currenlty we put any case in separate tx. Performance improvement is possible here. + auto rowSettingsPtr = FindNode(stage, [](const TExprNode::TPtr& node) { return !!TMaybeNode<TKqpReadRangesSourceSettings>(node); }); + if (rowSettingsPtr) { + auto rowSettings = TExprBase(rowSettingsPtr).Cast<TKqpReadRangesSourceSettings>(); + if (!rowSettings.RangesExpr().Maybe<TCoVoid>()) { + const auto materializeResult = BuildMaterialize(rowSettings.RangesExpr().Ptr()); + // clang-fomrat off + const auto newRowSettings = Build<TKqpReadRangesSourceSettings>(ctx, rowSettingsPtr->Pos()) + .Table(rowSettings.Table()) + .Columns(rowSettings.Columns()) + .Settings(rowSettings.Settings()) + .RangesExpr(materializeResult) + .ExplainPrompt(rowSettings.ExplainPrompt()) + .Done().Ptr(); + // clang-format on + stage = ctx.ReplaceNode(std::move(stage), rowSettings.Ref(), newRowSettings); + } + } } else { TVector<TExprNode::TPtr> stageInputConnections; TVector<TExprNode::TPtr> stageInputArgs; diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp index 53efaa30b8f..bcb8695fa2c 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp @@ -34,6 +34,8 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { for (const auto& column : Read->Columns) { columns.push_back(Ctx.NewAtom(Pos, column)); } + // Extract ranges. + TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); switch (Read->GetTableStorageType()) { case NYql::EStorageType::RowStorage: { @@ -48,7 +50,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { .Add(columns) .Build() .Settings<TCoNameValueTupleList>().Build() - .RangesExpr<TCoVoid>().Build() + .RangesExpr(ranges) .ExplainPrompt<TCoNameValueTupleList>().Build() .Build() .Done().Ptr(); @@ -108,7 +110,6 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { settings.SequentialInFlight = 1; } - TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); // clang-format off auto olapRead = Build<TKqpBlockReadOlapTableRanges>(Ctx, Pos) .Table(Read->TableCallable) @@ -139,7 +140,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { break; } default: - Y_ENSURE(false, "Unsupported table source type"); + Y_ENSURE(false, "Unsupported table source type."); } YQL_CLOG(TRACE, CoreDq) << "[NEW RBO Physical source] " << KqpExprToPrettyString(TExprBase(source), Ctx); diff --git a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp index 80a8978fc6b..3614c0aa3b0 100644 --- a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp +++ b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp @@ -123,8 +123,8 @@ bool IsSuitableToExtractAndPushRanges(const TIntrusivePtr<IOperator>& input) { } const auto read = CastOperator<TOpRead>(maybeRead); - // Currently supported only for cs. - return !read->GetRanges() && read->GetTableStorageType() == NYql::EStorageType::ColumnStorage; + const auto tableType = read->GetTableStorageType(); + return !read->GetRanges() && (tableType == NYql::EStorageType::ColumnStorage || tableType == NYql::EStorageType::RowStorage); } TPredicateExtractorSettings PrepareExtractorSettings(TKqpOptimizeContext& kqpCtx) { diff --git a/ydb/core/kqp/ut/federated_query/common/common.cpp b/ydb/core/kqp/ut/federated_query/common/common.cpp index de1411b5a50..0903c656efb 100644 --- a/ydb/core/kqp/ut/federated_query/common/common.cpp +++ b/ydb/core/kqp/ut/federated_query/common/common.cpp @@ -99,6 +99,10 @@ std::shared_ptr<TKikimrRunner> MakeKikimrRunner( if (appFlags.GetEnableColumnStore()) { featureFlags.SetEnableColumnStore(true); } + + if (appFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); + } } if (!appConfig) { diff --git a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp index 7d5ae2fdc07..3d3b992c865 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp @@ -73,7 +73,9 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // The topic path lands in tx.Tables (B walk). Y_UNIT_TEST_F(DirectTopicMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); constexpr TStringBuf topicName = "test_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -96,7 +98,9 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // direct topic; path is `<table>/<stream>`. Y_UNIT_TEST_F(CdcStreamMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); ExecSchemeQuery(RejectClassifierDdl( "hp_cdc", "/Root/t_cdc/cf")); @@ -129,6 +133,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // Uses a loopback Ydb-typed EDS pointing at our own runner + a local topic // as the target object (SetupMockPqGateway + real local PQ serves reads). Y_UNIT_TEST_F(FederatedEdsPathMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "eds_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -162,6 +167,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // appears in TDqPqTopicSource.TopicPath — the (D) walk's unique-coverage // case verified on real cluster). Y_UNIT_TEST_F(FederatedTopicRemoteNameMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "remote_by_name_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -196,6 +202,8 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // KindExternalTable — External Table on top of an ObjectStorage EDS. // Both the ET path and the underlying EDS path land in tx.Tables (B walk). Y_UNIT_TEST_F(ExternalTableMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); + ExecSchemeQuery(RejectClassifierDdl( "hp_et", "/Root/et_s3")); diff --git a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp index 383e457683b..7d367abc7d4 100644 --- a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp @@ -161,9 +161,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( @@ -219,8 +219,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); Variator::ToExecutor(Variator::SingleScript(scriptChunkDetailsMinMaxWithLocalDBStorage)).Execute(settings); - } - + } + TString scriptChunkDetailsMinMaxWithBSStorage = R"( STOP_COMPACTION ------ @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { READ: SELECT COALESCE(sum(CAST(ChunkDetails = "{\"min\":\"x\",\"max\":\"x\"}" as Uint32) ), 0) = count(ChunkDetails) FROM `/Root/ColumnTable/.sys/primary_index_stats` WHERE EntityName="field_mm"; EXPECTED: [[%true]] )"; - + Y_UNIT_TEST(ChunkDetailsMinMaxWithBSStorage, ELocalIndexAsSchemeObject) { const bool LocalIndexAsSchemeObject = (Arg<0>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); @@ -309,14 +309,14 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_mm2` LOCAL USING min_max ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom` LOCAL USING bloom_filter ON(`value`); )"); assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom2` LOCAL USING bloom_filter ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); @@ -326,7 +326,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram2` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); )"); - + assertDDLQueryNotOk(R"( CREATE TABLE `/Root/two_minmax_in_create` ( `key` Int32 NOT NULL, @@ -360,7 +360,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); } - + TString scriptMinMaxIndexOnInsertEnabled = R"( STOP_COMPACTION @@ -695,9 +695,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); NYdb::TStatus status = runDDLQuery(R"( - ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); + ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); )"); - + UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToString()); } @@ -917,7 +917,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item+1 AS String) AS `value_str`, CAST(item+1 AS Utf8) AS `value_utf`, @@ -928,7 +928,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item AS String) AS `value_str`, CAST(item AS Utf8) AS `value_utf`, @@ -939,7 +939,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { csController->WaitCompactions(TDuration::Seconds(5)); - using TQueryResult = TVector<TString>; + using TQueryResult = TVector<TString>; auto runDMLQueryTyped = [&](TString text) -> TQueryResult { auto columns = runDMLQuery(text); @@ -950,7 +950,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } return res; }; - + { TQueryResult tierNamesStr = runDMLQueryTyped(R"( SELECT TierName FROM `/Root/minmax_test_appropriate_storage_location/.sys/primary_index_stats` WHERE EntityName == "value_str_mm"; @@ -958,7 +958,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { UNIT_ASSERT_VALUES_UNEQUAL_C(tierNamesStr.size(), 0, "portions are not min_max indexed with String column type"); for (const auto& tierName: tierNamesStr) { UNIT_ASSERT_VALUES_EQUAL_C(tierName, "__DEFAULT", "min_max index must store its data is BS when building over String column"); - } + } } { TQueryResult tierNamesUtf8 = runDMLQueryTyped(R"( @@ -2038,9 +2038,9 @@ Y_UNIT_TEST(RenameLocalBloomIndex, EUseQueryService) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( diff --git a/ydb/core/kqp/ut/olap/types/json_ut.cpp b/ydb/core/kqp/ut/olap/types/json_ut.cpp index 50649a8d967..15210705b18 100644 --- a/ydb/core/kqp/ut/olap/types/json_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/json_ut.cpp @@ -1699,6 +1699,29 @@ Y_UNIT_TEST_SUITE(KqpOlapJsonNativeScalars) { Variator::ToExecutor(Variator::SingleScript(script)).Execute(); } + // A full cycle test for doubles near max magnitude. The stored value stays exact; reading back renders + // it with fewer digits (17 in, 16 out) on BinaryJson level. + Y_UNIT_TEST(CompactionNearMaxDouble) { + const TString script = TStringBuilder() << NativeTableSetup() << R"( + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (1u, JsonDocument('{"n" : 1.7976931348623157e308}')), + (2u, JsonDocument('{"n" : -1.7976931348623157e308}')) + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]]] + ------ + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (3u, JsonDocument('{"n" : 1.5}')) + ------ + ONE_COMPACTION + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]];[3u;["{\"n\":1.5}"]]] + ------ + )" << NativeValueTypeCheck(EValueType::Double); + Variator::ToExecutor(Variator::SingleScript(script)).Execute(); + } + Y_UNIT_TEST(CompactionDivergentTypesFallback) { const TString script = TStringBuilder() << NativeTableSetup() << R"( DATA: diff --git a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp index 84a7ba07175..8fc3e67375b 100644 --- a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp +++ b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp @@ -2021,9 +2021,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Execute)) .ExtractValueSync(); @@ -2072,10 +2070,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); auto params = paramsVector[i]; // clang-format off diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp index b0cb6c030d7..db35abb6bb3 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp +++ b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp @@ -302,6 +302,8 @@ private: featureFlags.SetEnableResourcePoolsCounters(true); featureFlags.SetEnableStreamingQueries(Settings_.EnableStreamingQueries_); featureFlags.SetEnableResourcePoolsScheduler(Settings_.EnableResourcePoolsScheduler_); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(Settings_.EnableHasPredicatesInResourcePoolClassifiers_); + featureFlags.SetEnableRejectActionInResourcePoolClassifiers(Settings_.EnableRejectActionInResourcePoolClassifiers_); auto& queryServiceConfig = *appConfig.MutableQueryServiceConfig(); queryServiceConfig.SetAllExternalDataSourcesAreAvailable(true); diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h index 47b84ed1ef7..1b6b2d020bd 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h +++ b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h @@ -81,6 +81,8 @@ struct TYdbSetupSettings { FLUENT_SETTING_DEFAULT(bool, EnableMetadataObjectsOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableExternalDataSourcesOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableStreamingQueries, true); + FLUENT_SETTING_DEFAULT(bool, EnableHasPredicatesInResourcePoolClassifiers, true); + FLUENT_SETTING_DEFAULT(bool, EnableRejectActionInResourcePoolClassifiers, true); FLUENT_SETTING(NKikimrConfig::TWorkloadManagerConfig, WorkloadManagerConfig); FLUENT_SETTING_DEFAULT(i32, DedicatedDiskQuota, -1); diff --git a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp index 5dd825f9adb..45ccc9a364f 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp @@ -14,7 +14,9 @@ using namespace NYdb; namespace { TIntrusivePtr<NWorkload::IYdbSetup> MakeStreamingYdb() { - return NWorkload::TYdbSetupSettings().Create([](auto) {}); + return NWorkload::TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(true) + .Create([](auto) {}); } void CreateTopic(TIntrusivePtr<NWorkload::IYdbSetup> ydb, TString name) { diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp index c011a23cf45..85490c90c74 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp @@ -11,6 +11,18 @@ using namespace NWorkload; using namespace NYdb; +TIntrusivePtr<IYdbSetup> MakeYdbWithAdvancedDisabled() { + return TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(false) + .Create(); +} + +TIntrusivePtr<IYdbSetup> MakeYdbWithRejectActionDisabled() { + return TYdbSetupSettings() + .EnableRejectActionInResourcePoolClassifiers(false) + .Create(); +} + void StartTestConcurrentQueryLimit(const ui64 activeCountLimit, const ui64 queueSize, const ui64 nodeCount = 1) { auto ydb = TYdbSetupSettings() .NodeCount(nodeCount) @@ -2065,4 +2077,154 @@ Y_UNIT_TEST_SUITE(DefaultPoolSettings) { +Y_UNIT_TEST_SUITE(HasPredicatesClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithHasFullScanIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasPathIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_path WITH ( + RESOURCE_POOL="default", + HAS_PATH='/Root/mydb/.*', + RANK=200 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasStreamIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_stream WITH ( + RESOURCE_POOL="default", + HAS_STREAM='true', + RANK=300 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasAppNameIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_app_name WITH ( + RESOURCE_POOL="default", + HAS_APP_NAME='myapp', + RANK=350 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(AlterWithHasFullScanIsRejected) { + // Create a basic classifier first (no advanced props) — should succeed. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_basic WITH ( + RESOURCE_POOL="default", + RANK=500 + ); + )"); + + // ALTER to add HAS_FULL_SCAN — should fail. + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + ALTER RESOURCE POOL CLASSIFIER cl_basic SET ( + HAS_FULL_SCAN='/Root/.*' + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithoutAdvancedPropsSucceeds) { + // Plain classifier must still work even when the flag is off. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_plain WITH ( + RESOURCE_POOL="default", + RANK=600 + ); + )"); + // If we get here without an exception, the creation succeeded. + } +} + +Y_UNIT_TEST_SUITE(HasPredicatesFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithHasFullScanSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan_ok WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithActionRejectIsRejected) { + auto ydb = MakeYdbWithRejectActionDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_action_reject WITH ( + ACTION="reject", + RANK=400 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Resource pool classifier action property is disabled"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithActionRejectSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_reject_ok WITH ( + ACTION="reject", + RANK=200 + ); + )"); + } +} + } // namespace NKikimr::NKqp diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index 13bd7c56fb4..4dfee75874b 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -181,6 +181,7 @@ namespace NKikimr::NBsController { auto *cmd = record.MutableReassignGroupDisk(); VDiskIDFromVDiskID(*VDiskToReplace, cmd->MutableVDiskId()); cmd->SetConvertToDonor(DonorMode); + cmd->SetAllowUnusableDisks(true); cmd->SetIsSelfHealReasonDecommit(IsSelfHealReasonDecommit); cmd->SetFromSelfHeal(true); Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), ev.release()); diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index e71178384a4..ad1d1c15efe 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -9736,7 +9736,6 @@ Y_UNIT_TEST_SUITE(THiveTest) { } Y_UNIT_TEST(TestLockedTabletVolatileStateDependsOnMetricsFlag) { - return; // https://github.com/ydb-platform/ydb/issues/46172 for (bool lockedTabletsSendMetrics : {false, true}) { const ui64 hiveTablet = MakeDefaultHiveID(); const ui64 testerTablet = MakeTabletID(false, 1); @@ -9783,7 +9782,12 @@ Y_UNIT_TEST_SUITE(THiveTest) { : NKikimrHive::ETabletVolatileState::TABLET_VOLATILE_STATE_STOPPED, tabletState(tabletIdExt)); - RebootTablet(runtime, hiveTablet, runtime.AllocateEdgeActor(0)); + runtime.Register(CreateTabletKiller(hiveTablet)); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvStatus, runtime.GetNodeCount()); + runtime.DispatchEvents(options); + } SendLockTabletExecution(runtime, hiveTablet, tabletIdExt, 0, NKikimrProto::OK, owner, 60000, true); diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index d4c06eb6444..7d268747719 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -111,38 +111,10 @@ public: return !NeedToReleaseFromParent && State == ETabletState::ReadyToWork && !IsBootingSuppressed(); } - bool IsReadyToBoot() const { - return IsReadyToWork() && TTabletInfo::IsReadyToBoot(); - } - - bool IsReadyToStart(TInstant now) const { - return IsReadyToWork() && TTabletInfo::IsReadyToStart(now); - } - bool IsReadyToBlockStorage() const { return State == ETabletState::BlockStorage; } - bool IsStarting() const { - return IsReadyToWork() && TTabletInfo::IsStarting(); - } - - bool IsStartingOnNode(TNodeId nodeId) const { - return IsReadyToWork() && TTabletInfo::IsStartingOnNode(nodeId); - } - - bool IsRunning() const { - return IsReadyToWork() && TTabletInfo::IsRunning(); - } - - bool IsAlive() const { - return IsReadyToWork() && TTabletInfo::IsAlive(); - } - - bool IsAliveOnLocal(const TActorId& local) const { - return IsReadyToWork() && TTabletInfo::IsAliveOnLocal(local); - } - bool IsDeleting() const { return State == ETabletState::Deleting; } diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 55ec6e8442a..9fa41bcd477 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -150,17 +150,28 @@ void TTabletInfo::ChangeVolatileState(EVolatileState state) { VolatileStateChangeTime = TActivationContext::Now(); } +bool TTabletInfo::IsReadyToWork() const { + if (IsLeader()) { + return AsLeader().IsReadyToWork(); + } + return true; +} + +bool TTabletInfo::IsReadyToBoot() const { + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; +} + bool TTabletInfo::IsReadyToStart(TInstant now) const { if (IsFollower()) { if (!GetLeader().IsRunning()) { return false; } } - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; } bool TTabletInfo::IsStarting() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; } bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { @@ -168,28 +179,31 @@ bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { } bool TTabletInfo::IsRunning() const { - return Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; + return IsReadyToWork() && Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; } bool TTabletInfo::IsBooting() const { - return VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; + return IsReadyToWork() && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; } bool TTabletInfo::IsAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING); } bool TTabletInfo::CanBeAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_UNKNOWN); // KIKIMR-12558 } bool TTabletInfo::IsAliveOnLocal(const TActorId& local) const { - return Node != nullptr + return IsReadyToWork() + && Node != nullptr && Node->Local == local && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index c88c635e08a..2edbab742dd 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -204,10 +204,8 @@ public: TString FamilyString() const; void ChangeVolatileState(EVolatileState state); - bool IsReadyToBoot() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; - } - + bool IsReadyToWork() const; + bool IsReadyToBoot() const; bool IsReadyToStart(TInstant now) const; bool IsStarting() const; bool IsStartingOnNode(TNodeId nodeId) const; diff --git a/ydb/core/persqueue/pqrb/mirror_describer.cpp b/ydb/core/persqueue/pqrb/mirror_describer.cpp index 47185c8846d..6cd1e3e3687 100644 --- a/ydb/core/persqueue/pqrb/mirror_describer.cpp +++ b/ydb/core/persqueue/pqrb/mirror_describer.cpp @@ -4,6 +4,8 @@ #include <google/protobuf/util/message_differencer.h> +#define YDB_LOG_THIS_FILE_COMPONENT Service + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("topic", TopicName) using namespace NPersQueue; @@ -34,7 +36,8 @@ void TMirrorDescriber::StartInit(const TActorContext& ctx) { } void TMirrorDescriber::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_N("killed"); + YDB_LOG_NOTICE("Killed", + {"logPrefix", NPQ_LOG_PREFIX}); CredentialsProvider = nullptr; Die(ctx); } @@ -44,10 +47,13 @@ void TMirrorDescriber::HandleChangeConfig(TEvPQ::TEvChangePartitionConfig::TPtr& Config, ev->Get()->Config.GetPartitionConfig().GetMirrorFrom() ); - LOG_D("got new config, equal with previous: " << equalConfigs); + YDB_LOG_DEBUG("Got new config, equal with", + {"logPrefix", NPQ_LOG_PREFIX}, + {"previous", equalConfigs}); if (!equalConfigs) { Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom(); - LOG_I("changing config"); + YDB_LOG_INFO("Changing config", + {"logPrefix", NPQ_LOG_PREFIX}); StartInit(ctx); } } @@ -56,13 +62,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: DescribeTopicRequestInFlight = false; const auto& description = ev->Get()->Description; if (!description.has_value()) { - LOG_E("cannot describe topic " << description.error()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"error", description.error()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } const NYdb::NTopic::TDescribeTopicResult& result = description.value(); if (!result.IsSuccess()) { - LOG_E("cannot describe topic " << result.GetIssues().ToString()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"issues", result.GetIssues()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } @@ -71,14 +81,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: descr.SerializeTo(req); return req.ShortUtf8DebugString(); }; - LOG_T("topic description: " << debugTopicDescriptionString(description.value().GetTopicDescription())); + YDB_LOG_TRACE("Topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"description", debugTopicDescriptionString(description.value().GetTopicDescription())}); ctx.Send(TabletActorId, ev->Release()); ctx.Schedule(DESCRIBE_RETRY_TIMEOUT_MAX, new TEvents::TEvWakeup()); } void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { if (DescribeTopicRequestInFlight) { - LOG_I("description request already inflight."); + YDB_LOG_INFO("Description request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } @@ -111,7 +124,8 @@ void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) { if (CredentialsRequestInFlight) { - LOG_W("credentials request already inflight."); + YDB_LOG_WARN("Credentials request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } CredentialsProvider = nullptr; @@ -145,13 +159,17 @@ void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /* void TMirrorDescriber::HandleCredentialsCreated(TEvPQ::TEvCredentialsCreated::TPtr& ev, const TActorContext& ctx) { CredentialsRequestInFlight = false; if (ev->Get()->Error) { - LOG_W("cannot initialize credentials provider: " << ev->Get()->Error.value()); + YDB_LOG_WARN("Cannot initialize credentials", + {"logPrefix", NPQ_LOG_PREFIX}, + {"provider", ev->Get()->Error.value()}); ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), CredentialsInitInterval, INIT_INTERVAL_MAX, ctx); return; } CredentialsProvider = ev->Get()->Credentials; - LOG_N("credentials provider created " << bool(CredentialsProvider)); + YDB_LOG_NOTICE("Credentials provider created", + {"logPrefix", NPQ_LOG_PREFIX}, + {"hasCredentialsProvider", bool(CredentialsProvider)}); CredentialsInitInterval = INIT_INTERVAL_START; ScheduleDescription(ctx); } diff --git a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp index 12b49163dbc..517fe6591ba 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp @@ -6,6 +6,8 @@ #include <algorithm> #include <ranges> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -35,9 +37,13 @@ void TPartitionScaleManager::HandleScaleStatusChange(const ui32 partitionId, NKi TMaybe<NKikimrPQ::TPartitionScaleParticipants> participants, TMaybe<TString> splitBoundary, const TActorContext& ctx) { - PQ_LOG_D("Handle HandleScaleStatusChange. Scale status: " << NKikimrPQ::EScaleStatus_Name(scaleStatus)); + YDB_LOG_DEBUG("Handle HandleScaleStatusChange. Scale", + {"logPrefix", LogPrefix()}, + {"status", NKikimrPQ::EScaleStatus_Name(scaleStatus)}); if (scaleStatus == NKikimrPQ::EScaleStatus::NEED_SPLIT) { - PQ_LOG_D("::HandleScaleStatusChange need to split partition " << partitionId); + YDB_LOG_DEBUG("::HandleScaleStatusChange need to split partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); TPartitionScaleOperationInfo op{ .PartitionId = partitionId, .PartitionScaleParticipants = std::move(participants), @@ -58,13 +64,15 @@ void TPartitionScaleManager::TrySendScaleRequest(const TActorContext& ctx) { auto splitMergeRequest = BuildScaleRequest(ctx); if (splitMergeRequest.Empty()) { - PQ_LOG_D("splitMergeRequest empty"); + YDB_LOG_DEBUG("SplitMergeRequest empty", + {"logPrefix", LogPrefix()}); return; } RequestInflight = true; RootPartitionsResetRequestInflight = !splitMergeRequest.SetBoundary.empty(); - PQ_LOG_D("send split request"); + YDB_LOG_DEBUG("Send split request", + {"logPrefix", LogPrefix()}); CurrentScaleRequest = ctx.Register(new TPartitionScaleRequest( TopicName, TopicPath, @@ -129,12 +137,12 @@ TPartitionScaleManager::TScaleRequest TPartitionScaleManager::BuildScaleRequest( auto mergesToApply = BuildMergeRequest(allowedSplitsCount); auto splitsToApply = BuildSplitRequest(allowedSplitsCount); - PQ_LOG_D(fmt::format("Scale request: #splits={}, #unprocessed={}, splitsLimit={}, #merges={}", - splitsToApply.Requests.size(), - splitsToApply.Unprocessed, - allowedSplitsCountLimit, - mergesToApply.Requests.size() - )); + YDB_LOG_DEBUG("Scale request", + {"logPrefix", LogPrefix()}, + {"splits", splitsToApply.Requests.size()}, + {"unprocessed", splitsToApply.Unprocessed}, + {"splitsLimit", allowedSplitsCountLimit}, + {"merges", mergesToApply.Requests.size()}); return { .Split = std::move(splitsToApply.Requests), @@ -166,19 +174,20 @@ TPartitionScaleManager::TRequests<TPartitionScaleManager::TPartitionBoundary> TP allowedSplitsCount -= cost; boundsToApply.push_back(std::move(part)); } else { - PQ_LOG_W(fmt::format("MaxActivePartitions ({}) is too low to recreate {} root partitions from the mirror source topic", - BalancerConfig.MaxActivePartitions, - RootPartitionsToCreate->size())); + YDB_LOG_WARN("MaxActivePartitions is too low to recreate root partitions from the mirror source topic", + {"logPrefix", LogPrefix()}, + {"maxActivePartitions", BalancerConfig.MaxActivePartitions}, + {"createPartitions", RootPartitionsToCreate->size()}); // don't send request at all, if there is not enough quota return { .Unprocessed = RootPartitionsToCreate->size(), }; } } - PQ_LOG_D(fmt::format("Set partition boundaries requsts: #modify={}, #create{}", - modifyPartitions, - createPartitions - )); + YDB_LOG_DEBUG("Set partition boundaries requsts", + {"logPrefix", LogPrefix()}, + {"modify", modifyPartitions}, + {"create", createPartitions}); } return { .Requests = std::move(boundsToApply), @@ -221,21 +230,29 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const ui32 partitionId = splitParameters.PartitionId; if (MirroredFromSomewhere) { if (!AppData()->FeatureFlags.GetEnableMirroredTopicSplitMerge()) { - PQ_LOG_D("split request for mirrored topic is disabled. Partition# " << partitionId); + YDB_LOG_DEBUG("Split request for mirrored topic is disabled", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } if (!splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("split request for mirrored topic doesn't have prescribed partition ids. Partition# " << partitionId); + YDB_LOG_NOTICE("Split request for mirrored topic doesn't have prescribed partition ids", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } const auto* node = PartitionGraph.GetPartition(partitionId); if (node == nullptr) { if (splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("attempt to split partition that was not created yet. Partition# " << partitionId); + YDB_LOG_NOTICE("Attempt to split partition that was not created yet", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } else { - PQ_LOG_ERROR("partition not found. Partition# " << partitionId); + YDB_LOG_ERROR("Partition not found", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } @@ -244,18 +261,25 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu auto to = node->To; auto mid = splitParameters.SplitBoundary.GetOrElse(MiddleOf(from, to)); if (mid.empty()) { - PQ_LOG_ERROR("wrong partition key range. Can't get mid. Partition# " << partitionId); + YDB_LOG_ERROR("Wrong partition key range. Can't get mid", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } if (splitParameters.PartitionScaleParticipants.Defined() && splitParameters.PartitionScaleParticipants->AdjacentPartitionIdsSize() != 0) { - PQ_LOG_ERROR("split request cannot have adjacent partitions. Partition# " << partitionId); + YDB_LOG_ERROR("Split request cannot have adjacent partitions", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } - PQ_LOG_D("partition split ranges. From# '" << ToHex(from) - << "'. To# '" << ToHex(to) << "'. Mid# '" << ToHex(mid) - << "'. Partition# " << partitionId); + YDB_LOG_DEBUG("Partition split ranges", + {"logPrefix", LogPrefix()}, + {"fromHex", ToHex(from)}, + {"toHex", ToHex(to)}, + {"midHex", ToHex(mid)}, + {"partition", partitionId}); TPartitionSplit split; split.set_partition(partitionId); @@ -264,7 +288,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu for (const auto& childPartitionId : splitParameters.PartitionScaleParticipants->GetChildPartitionIds()) { split.add_childpartitionids(childPartitionId); if (const auto* childNode = PartitionGraph.GetPartition(childPartitionId); childNode != nullptr) { - PQ_LOG_NOTICE(fmt::format("Child partition# {} already exists. Performing unordered split. Partition# {}", childPartitionId, partitionId)); + YDB_LOG_NOTICE("Child partition already exists. Performing unordered split", + {"logPrefix", LogPrefix()}, + {"childPartition", childPartitionId}, + {"partition", partitionId}); split.set_createrootlevelsibling(true); } } @@ -276,7 +303,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const auto& prescribedChildrenIds = splitParameters.PartitionScaleParticipants->GetChildPartitionIds(); if (!std::ranges::is_permutation(nodeChildrenIds, prescribedChildrenIds)) { const std::string mappingStr = fmt::format("([{}]->[{}])", fmt::join(nodeChildrenIds, ","), fmt::join(prescribedChildrenIds, ",")); - PQ_LOG_ERROR("trying to split partition into different set of children partitions " << mappingStr << ". Partition# " << partitionId); + YDB_LOG_ERROR("Trying to split partition into different set of children partitions", + {"logPrefix", LogPrefix()}, + {"mappingStr", mappingStr}, + {"partition", partitionId}); } } return {.Split = Nothing(), .Remove = true}; @@ -287,7 +317,9 @@ void TPartitionScaleManager::HandleScaleRequestResult(TPartitionScaleRequest::TE RequestInflight = false; LastResponseTime = ctx.Now(); auto result = ev->Get(); - PQ_LOG_D("HandleScaleRequestResult scale request result: " << result->Status); + YDB_LOG_DEBUG("HandleScaleRequestResult scale request", + {"logPrefix", LogPrefix()}, + {"result", result->Status}); if (result->Status == TEvTxUserProxy::TResultStatus::ExecComplete) { RequestTimeout = TDuration::Zero(); Backoff.Reset(); @@ -312,24 +344,27 @@ void TPartitionScaleManager::UpdateMirrorRootPartitionsSet() { NMirror::TMirrorGraphComparisonResult cmp = NMirror::ComparePartitionGraphs(PartitionGraph, MirrorTopicDescription->GetPartitions()); if (!cmp.RootPartitionsMismatch.has_value()) { - PQ_LOG_D("Topic has all root partitions from the source topic"); + YDB_LOG_DEBUG("Topic has all root partitions from the source topic", + {"logPrefix", LogPrefix()}); RootPartitionsToCreate.reset(); MirrorTopicError.reset(); return; } auto& rootPartitionsMismatch = cmp.RootPartitionsMismatch.value(); if (rootPartitionsMismatch.Error.has_value()) { - std::string msg = TStringBuilder() << "Incompatable configuration of root partitions between source and target topics:" << rootPartitionsMismatch.Error.value(); - PQ_LOG_ERROR(msg); + YDB_LOG_ERROR("Incompatable configuration of root partitions between source and target topics", + {"logPrefix", LogPrefix()}, + {"topics", rootPartitionsMismatch.Error.value()}); RootPartitionsToCreate.reset(); MirrorTopicError = std::move(*rootPartitionsMismatch.Error); return; } const size_t existingPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Modify, &NMirror::TPartitionWithBounds::Action); const size_t newPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Create, &NMirror::TPartitionWithBounds::Action); - PQ_LOG_I(fmt::format("Topic has less root partitions than the mirror source. New configuration has {}+{} partitions.", - existingPartitions, - newPartitions)); + YDB_LOG_INFO("Topic has less root partitions than the mirror source", + {"logPrefix", LogPrefix()}, + {"existing", existingPartitions}, + {"new", newPartitions}); RootPartitionsToCreate = std::move(rootPartitionsMismatch.AlterRootPartitions); MirrorTopicError.reset(); @@ -341,7 +376,8 @@ std::expected<void, std::string> TPartitionScaleManager::HandleMirrorTopicDescri } else { auto& description = ev->Get()->Description; if (!description.has_value() || !description.value().IsSuccess()) { - PQ_LOG_W("Ignoring invalid mirror source description"); + YDB_LOG_WARN("Ignoring invalid mirror source description", + {"logPrefix", LogPrefix()}); return {}; } MirrorTopicDescription.emplace(std::move(description->GetTopicDescription())); diff --git a/ydb/core/persqueue/pqrb/partition_scale_request.cpp b/ydb/core/persqueue/pqrb/partition_scale_request.cpp index e09adfe7b0e..30d1298c230 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_request.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_request.cpp @@ -3,6 +3,8 @@ #include <ydb/core/protos/schemeshard/operations.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -76,7 +78,8 @@ void TPartitionScaleRequest::FillProposeRequest(TEvTxUserProxy::TEvProposeTransa } logMessage << "."; } - PQ_LOG_D( logMessage); + YDB_LOG_DEBUG(logMessage, + {"logPrefix", LogPrefix()}); for(const auto& merge: Merges) { auto* newMerge = groupDescription.AddMerge(); @@ -124,7 +127,9 @@ void TPartitionScaleRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatus: for (auto& issue : ev->Get()->Record.GetIssues()) { issues << issue.ShortDebugString() + ", "; } - PQ_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split request: " << issues); + YDB_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split", + {"logPrefix", LogPrefix()}, + {"request", issues}); Send(ParentActorId, scaleRequestResult.release()); Die(ctx); } else { diff --git a/ydb/core/persqueue/pqrb/read_balancer.cpp b/ydb/core/persqueue/pqrb/read_balancer.cpp index 08ad274f8de..5eafeb10d21 100644 --- a/ydb/core/persqueue/pqrb/read_balancer.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer.cpp @@ -16,6 +16,8 @@ #include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/random_provider/random_provider.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("tablet_id", TabletID())("path", Path)("topic", Topic) namespace NKikimr { @@ -142,7 +144,9 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } return s; }; - PQ_LOG_D(getInitLog()); + YDB_LOG_DEBUG("BALANCER INIT DONE dump logPrefix, getInitLog", + {"logPrefix", LogPrefix()}, + {"getInitLog", getInitLog()}); for (auto &ev : UpdateEvents) { ctx.Send(ctx.SelfID, ev.Release()); @@ -161,7 +165,8 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } void TPersQueueReadBalancer::HandleWakeup(TEvents::TEvWakeup::TPtr& ev, const TActorContext &ctx) { - PQ_LOG_D("TPersQueueReadBalancer::HandleWakeup"); + YDB_LOG_DEBUG("TPersQueueReadBalancer::HandleWakeup", + {"logPrefix", LogPrefix()}); switch (ev->Get()->Tag) { case TPartitionScaleManager::TRY_SCALE_REQUEST_WAKE_UP_TAG: { @@ -209,9 +214,13 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvUpdateBalancerConfig::TPtr if (!WaitingResponse.empty()) { //got transaction infly WaitingResponse.push_back(ev->Sender); } else { //version already applied - PQ_LOG_D("BALANCER Topic " << Topic << "Tablet " << TabletID() - << " Config already applied version " << record.GetVersion() << " actor " << ev->Sender - << " txId " << record.GetTxId()); + YDB_LOG_DEBUG("BALANCER Topic Tablet Config already applied version actor txId", + {"logPrefix", LogPrefix()}, + {"topic", Topic}, + {"tabletID", TabletID()}, + {"version", record.GetVersion()}, + {"sender", ev->Sender}, + {"txId", record.GetTxId()}); THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; res->Record.SetStatus(NKikimrPQ::OK); res->Record.SetTxId(record.GetTxId()); @@ -374,7 +383,9 @@ TStringBuilder TPersQueueReadBalancer::LogPrefix() const { void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { auto tabletId = ev->Get()->TabletId; - PQ_LOG_D("TEvClientDestroyed " << tabletId); + YDB_LOG_DEBUG("TEvClientDestroyed", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); ClosePipe(tabletId, ctx); RequestTabletIfNeeded(tabletId, ctx, true); @@ -391,7 +402,10 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, ClosePipe(ev->Get()->TabletId, ctx); RequestTabletIfNeeded(ev->Get()->TabletId, ctx, true); - PQ_LOG_ERROR("TEvClientConnected Status " << ev->Get()->Status << ", TabletId " << tabletId); + YDB_LOG_ERROR("TEvClientConnected Status TabletId", + {"logPrefix", LogPrefix()}, + {"status", ev->Get()->Status}, + {"tabletId", tabletId}); return; } @@ -402,10 +416,16 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, it->second.Generation = ev->Get()->Generation; it->second.NodeId = ev->Get()->ServerId.NodeId(); - PQ_LOG_D("TEvClientConnected TabletId " << tabletId << ", NodeId " << ev->Get()->ServerId.NodeId() << ", Generation " << ev->Get()->Generation); + YDB_LOG_DEBUG("TEvClientConnected TabletId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"nodeId", ev->Get()->ServerId.NodeId()}, + {"generation", ev->Get()->Generation}); } else - PQ_LOG_I("TEvClientConnected Pipe is not found, TabletId " << tabletId); + YDB_LOG_INFO("TEvClientConnected Pipe is not found, TabletId", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); } void TPersQueueReadBalancer::ClosePipe(const ui64 tabletId, const TActorContext& ctx) @@ -453,7 +473,10 @@ void TPersQueueReadBalancer::RequestTabletIfNeeded(const ui64 tabletId, const TA StatsRequestTracker.Cookies[tabletId] = cookie; } - PQ_LOG_D("Send TEvPersQueue::TEvStatus TabletId: " << tabletId << " Cookie: " << cookie); + YDB_LOG_DEBUG("Send TEvPersQueue::TEvStatus", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"cookie", cookie}); NTabletPipe::SendData(ctx, pipeClient, new TEvPersQueue::TEvStatus("", true), cookie); } } @@ -531,11 +554,13 @@ void TPersQueueReadBalancer::CheckStat(const TActorContext& ctx) { UpdateCounters(ctx); TEvPersQueue::TEvPeriodicTopicStats* ev = GetStatsEvent(); - PQ_LOG_D("Send TEvPeriodicTopicStats PathId: " << PathId - << " Generation: " << Generation - << " StatsReportRound: " << StatsReportRound - << " DataSize: " << TopicMetricsHandler->GetTopicMetrics().TotalDataSize - << " UsedReserveSize: " << TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize); + YDB_LOG_DEBUG("Send TEvPeriodicTopicStats", + {"logPrefix", LogPrefix()}, + {"pathId", PathId}, + {"generation", Generation}, + {"statsReportRound", StatsReportRound}, + {"dataSize", TopicMetricsHandler->GetTopicMetrics().TotalDataSize}, + {"usedReserveSize", TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize}); NTabletPipe::SendData(ctx, GetPipeClient(SchemeShardId, ctx), ev); @@ -622,8 +647,12 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetPartitionsLocation::TPtr pResponse->SetNodeId(iter->second.NodeId.GetRef()); pResponse->SetGeneration(iter->second.Generation.GetRef()); - PQ_LOG_D("The partition location was added to response: TabletId " << tabletId << ", PartitionId " << partitionId - << ", NodeId " << pResponse->GetNodeId() << ", Generation " << pResponse->GetGeneration()); + YDB_LOG_DEBUG("The partition location was added to response: TabletId PartitionId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"partitionId", partitionId}, + {"pResponseNodeId", pResponse->GetNodeId()}, + {"pResponseGeneration", pResponse->GetGeneration()}); return true; }; @@ -713,7 +742,10 @@ void TPersQueueReadBalancer::Handle(NSchemeShard::TEvSchemeShard::TEvSubDomainPa if (SchemeShardId == msg->SchemeShardId && (!SubDomainPathId || SubDomainPathId->OwnerId != msg->SchemeShardId)) { - PQ_LOG_D("Discovered subdomain " << msg->LocalPathId << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain at RB", + {"logPrefix", LogPrefix()}, + {"localPathId", msg->LocalPathId}, + {"tabletID", TabletID()}); SubDomainPathId.emplace(msg->SchemeShardId, msg->LocalPathId); Execute(new TTxWriteSubDomainPathId(this), ctx); @@ -767,8 +799,11 @@ void TPersQueueReadBalancer::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated .GetDomainState() .GetDiskQuotaExceeded(); - PQ_LOG_D("Discovered subdomain " << msg->PathId << " state, outOfSpace = " << outOfSpace - << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain state, at RB", + {"logPrefix", LogPrefix()}, + {"pathId", msg->PathId}, + {"outOfSpace", outOfSpace}, + {"tabletID", TabletID()}); SubDomainOutOfSpace = outOfSpace; @@ -847,7 +882,9 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetReadSessionsInfo::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ctx); - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); MLPBalancer->Handle(ev); } @@ -874,13 +911,16 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ev, const TActorContext& ctx) { if (!SplitMergeEnabled(TabletConfig)) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled", + {"logPrefix", LogPrefix()}); return; } auto& record = ev->Get()->Record; auto* node = PartitionGraph.GetPartition(record.GetPartitionId()); if (!node) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: partition " << record.GetPartitionId() << " not found."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: partition not found", + {"logPrefix", LogPrefix()}, + {"partitionId", record.GetPartitionId()}); return; } @@ -893,7 +933,8 @@ void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ctx ); } else { - PQ_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized."); + YDB_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized", + {"logPrefix", LogPrefix()}); } } @@ -907,7 +948,8 @@ void TPersQueueReadBalancer::Handle(TPartitionScaleRequest::TEvPartitionScaleReq } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMirrorTopicDescription::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("Received TEvMirrorTopicDescription"); + YDB_LOG_DEBUG("Received TEvMirrorTopicDescription", + {"logPrefix", LogPrefix()}); if (!MirroringEnabled(TabletConfig)) { return; } @@ -928,13 +970,17 @@ void TPersQueueReadBalancer::BroadcastPartitionError(const TString& message, con } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetPartitionRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetPartitionRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPGetPartitionRequest", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); PendingMLPGetPartitionRequests.push_back(std::move(ev)); ProcessMLPGetPartitionRequests(ActorContext()); } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetRuntimeAttributesRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle", + {"logPrefix", LogPrefix()}, + {"mlpGetRuntimeAttributesRequest", ev->Get()->Record.ShortDebugString()}); if (StatsRequestTracker.StatsReceived) { return MLPBalancer->Handle(ev); } diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp index 98ba0856039..b91d5afe224 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp @@ -3,8 +3,7 @@ #include <ydb/core/persqueue/public/utils.h> -#define DEBUG(message) - +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER namespace NKikimr::NPQ::NBalancing { @@ -164,17 +163,23 @@ TString TPartitionFamily::LogPrefix() const { void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetStatus) { if (Status != EStatus::Active) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that isn't active"); + YDB_LOG_CRIT("Releasing the family that isn't active", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } if (!Session) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that does not have a session"); + YDB_LOG_CRIT("Releasing the family that does not have a session", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } - PQ_LOG_I(" release partitions [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) - << "]. Target status " << targetStatus); + YDB_LOG_INFO("Release partitions. Target status", + {"logPrefix", LogPrefix()}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}, + {"targetStatus", targetStatus}); Status = EStatus::Releasing; TargetStatus = targetStatus; @@ -193,25 +198,35 @@ void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetSta bool TPartitionFamily::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { if (!Session || Session->Pipe != sender) { - PQ_LOG_D("try unlock the partition " << partitionId << " from other sender"); + YDB_LOG_DEBUG("Try unlock the partition from other sender", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } if (Status != EStatus::Releasing) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but family status is " << Status); + YDB_LOG_CRIT("Try unlock partition but family status is", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"status", Status}); return false; } if (!LockedPartitions.erase(partitionId)) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but partition isn't locked." - << " Locked partitions are [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_CRIT("Try unlock partition but partition isn't locked. Locked partitions are", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } --Session->ReleasingPartitionCount; if (!LockedPartitions.empty()) { - PQ_LOG_D("partition " << partitionId << " was unlocked but wait else [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Partition was unlocked, but wait", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } @@ -238,7 +253,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct return false; case ETargetStatus::Free: - PQ_LOG_T(" is free."); + YDB_LOG_TRACE("Is free", + {"logPrefix", LogPrefix()}); Status = EStatus::Free; AfterRelease(); @@ -251,7 +267,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct auto it = Consumer.Families.find(MergeTo); if (it == Consumer.Families.end()) { - PQ_LOG_D(" has been released for merge but target family is not exists."); + YDB_LOG_DEBUG("Has been released for merge but target family is not exists", + {"logPrefix", LogPrefix()}); return true; } auto* targetFamily = it->second.get(); @@ -266,7 +283,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct } void TPartitionFamily::Destroy(const TActorContext&) { - PQ_LOG_D(" destroyed."); + YDB_LOG_DEBUG("Destroyed", + {"logPrefix", LogPrefix()}); if (Session) { Session->Families.erase(Id); @@ -301,11 +319,14 @@ void TPartitionFamily::AfterRelease() { void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) { if (Status != EStatus::Free) { - PQ_LOG_CRIT("try start reading but the family status is " << Status); + YDB_LOG_CRIT("Try start reading but the family status is", + {"logPrefix", LogPrefix()}, + {"status", Status}); return; } - PQ_LOG_T("start reading"); + YDB_LOG_TRACE("Start reading", + {"logPrefix", LogPrefix()}); Status = EStatus::Active; @@ -327,7 +348,9 @@ void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) } void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, const TActorContext& ctx) { - PQ_LOG_D("attaching partitions [" << JoinRange(", ", partitions.begin(), partitions.end()) << "]"); + YDB_LOG_DEBUG("Attaching partitions", + {"logPrefix", LogPrefix()}, + {"partitions", JoinRange(", ", partitions.begin(), partitions.end())}); std::unordered_set<ui32> existedPartitions; existedPartitions.insert(Partitions.begin(), Partitions.end()); @@ -376,13 +399,17 @@ void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, co } void TPartitionFamily::ActivatePartition(ui32 partitionId) { - PQ_LOG_D("activating partition " << partitionId); + YDB_LOG_DEBUG("Activating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(1, -1); } void TPartitionFamily::InactivatePartition(ui32 partitionId) { - PQ_LOG_D("inactivating partition " << partitionId); + YDB_LOG_DEBUG("Inactivating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(-1, 1); } @@ -401,7 +428,9 @@ void TPartitionFamily::InactivatePartition(ui32 partitionId) { } void TPartitionFamily::Merge(TPartitionFamily* other) { - PQ_LOG_D("merge family with " << other->DebugStr()); + YDB_LOG_DEBUG("Merge family with", + {"logPrefix", LogPrefix()}, + {"debug", other->DebugStr()}); Y_VERIFY(this != other); @@ -545,8 +574,12 @@ void TPartitionFamily::UpdateSpecialSessions() { void TPartitionFamily::LockPartition(ui32 partitionId, const TActorContext& ctx) { auto step = NextStep(); - PQ_LOG_I("lock partition " << partitionId << " for " << Session->DebugStr() - << " generation " << TabletGeneration() << " step " << step); + YDB_LOG_INFO("Lock partition for generation step", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"debug", Session->DebugStr()}, + {"tabletGeneration", TabletGeneration()}, + {"step", step}); ctx.Send(Session->Sender, MakeEvLockPartition(partitionId, step).release()); } @@ -636,7 +669,9 @@ ui32 TConsumer::NextStep() { void TConsumer::RegisterPartition(ui32 partitionId, const TActorContext& ctx) { auto [_, inserted] = Partitions.try_emplace(partitionId, TPartition()); if (inserted && IsReadable(partitionId)) { - PQ_LOG_D("register readable partition " << partitionId); + YDB_LOG_DEBUG("Register readable partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); CreateFamily({partitionId}, ctx); } @@ -666,7 +701,9 @@ TPartitionFamily* TConsumer::CreateFamily(std::vector<ui32>&& partitions, TParti UnreadableFamilies[id] = family; } - PQ_LOG_D("family created " << family->DebugStr()); + YDB_LOG_DEBUG("Family created", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); return family; } @@ -706,7 +743,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d std::vector<TPartitionFamily*> newFamilies; if (!family->IsLonely()) { - PQ_LOG_D("break up " << family->DebugStr() << " partition=" << partitionId); + YDB_LOG_DEBUG("Break up", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); std::unordered_set<ui32> partitions; partitions.insert(family->Partitions.begin(), family->Partitions.end()); @@ -786,7 +826,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d } } } else { - PQ_LOG_D("can't break up " << family->DebugStr() << " because partition=" << partitionId << " is not root of family"); + YDB_LOG_DEBUG("Can't break up because is not root of family", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); } } @@ -868,7 +911,9 @@ TPartitionFamily* TConsumer::FindFamily(ui32 partitionId) { } void TConsumer::RegisterReadingSession(TSession* session, const TActorContext& ctx) { - PQ_LOG_I("register reading session " << session->DebugStr()); + YDB_LOG_INFO("Register reading session", + {"logPrefix", LogPrefix()}, + {"debug", session->DebugStr()}); Sessions[session->Pipe] = session; @@ -952,7 +997,9 @@ void TConsumer::UnregisterReadingSession(TSession* session, const TActorContext& bool TConsumer::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_CRIT("unlocking the partition " << partitionId << " from unknown family."); + YDB_LOG_CRIT("Unlocking the partition from unknown family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } @@ -1034,7 +1081,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons }); if (partition.NeedReleaseChildren()) { - PQ_LOG_D("Attache partitions [" << JoinRange(", ", newPartitions.begin(), newPartitions.end()) << "] to " << family->DebugStr()); + YDB_LOG_DEBUG("Attache partitions", + {"logPrefix", LogPrefix()}, + {"newPartitions", JoinRange(", ", newPartitions.begin(), newPartitions.end())}, + {"family", family->DebugStr()}); for (auto id : newPartitions) { if (family->CanAttach(std::vector{id})) { auto* node = GetPartitionGraph().GetPartition(id); @@ -1066,7 +1116,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons } } } else { - PQ_LOG_D("Can't attache partition " << id << " to " << family->DebugStr()); + YDB_LOG_DEBUG("Can't attache partition", + {"logPrefix", LogPrefix()}, + {"id", id}, + {"family", family->DebugStr()}); } } } else { @@ -1083,19 +1136,28 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons void TConsumer::StartReading(ui32 partitionId, const TActorContext& ctx) { if (!GetPartitionInfo(partitionId)) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition has been deleted."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition has been deleted", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* partition = GetPartition(partitionId); if (!partition) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition does not exist."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition does not exist", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto wasInactive = partition->IsInactive(); if (partition->StartReading()) { - PQ_LOG_D("Reading of the partition " << partitionId << " was started by " << ConsumerName << ". We stop reading from child partitions."); + YDB_LOG_DEBUG("Reading of the partition was started by We stop reading from child partitions", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); auto* family = FindFamily(partitionId); if (!family) { @@ -1137,21 +1199,27 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: auto partitionId = r.GetPartitionId(); if (!IsReadable(partitionId)) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't family"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } if (!family->Session) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't reading session"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't reading session", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } @@ -1159,8 +1227,12 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: const bool wasInactive = partition.IsInactive(); if (partition.SetFinishedState(r.GetScaleAwareSDK(), r.GetStartedReadingFromEndOffset()) || wasInactive) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ", firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); if (ProccessReadingFinished(partitionId, wasInactive, ctx)) { ScheduleBalance(ctx); @@ -1168,9 +1240,13 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: } else if (!partition.IsInactive()) { auto delay = std::min<size_t>(1ul << partition.Iteration, Balancer.GetLifetimeSeconds()); // TODO use split/merge time - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ". Scheduled release of the partition for re-reading. Delay=" << delay << " seconds," - << " firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by Scheduled release of the partition for re-reading. seconds", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"delay", delay}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); ctx.Schedule(TDuration::Seconds(delay), new TEvPQ::TEvWakeupReleasePartition(ConsumerName, partitionId, partition.Cookie)); } @@ -1178,13 +1254,15 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: void TConsumer::ScheduleBalance(const TActorContext& ctx) { if (BalanceScheduled) { - PQ_LOG_T("rebalancing already was scheduled"); + YDB_LOG_TRACE("Rebalancing already was scheduled", + {"logPrefix", LogPrefix()}); return; } BalanceScheduled = true; - PQ_LOG_D("rebalancing was scheduled"); + YDB_LOG_DEBUG("Rebalancing was scheduled", + {"logPrefix", LogPrefix()}); ctx.Send(Balancer.TopicActor.SelfId(), new TEvPQ::TEvBalanceConsumer(ConsumerName)); } @@ -1254,9 +1332,14 @@ size_t GetMaxFamilySize(const std::unordered_map<size_t, const std::unique_ptr<T } void TConsumer::Balance(const TActorContext& ctx) { - PQ_LOG_D("balancing. Sessions=" << Sessions.size() << ", Families=" << Families.size() - << ", UnreadableFamilies=" << UnreadableFamilies.size() << " [" << DebugStr(UnreadableFamilies) - << "], RequireBalancing=" << FamiliesRequireBalancing.size() << " [" << DebugStr(FamiliesRequireBalancing) << "]"); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"sessions", Sessions.size()}, + {"families", Families.size()}, + {"unreadableFamilies", UnreadableFamilies.size()}, + {"unreadableFamiliesDebug", DebugStr(UnreadableFamilies)}, + {"requireBalancing", FamiliesRequireBalancing.size()}, + {"familiesRequireBalancingDebug", DebugStr(FamiliesRequireBalancing)}); if (Sessions.empty()) { return; @@ -1270,7 +1353,9 @@ void TConsumer::Balance(const TActorContext& ctx) { continue; } if (!family->SpecialSessions.contains(family->Session->Pipe)) { - PQ_LOG_D("rebalance " << family->DebugStr() << " because exists the special session for it"); + YDB_LOG_DEBUG("Rebalance because exists the special session for it", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); family->Release(ctx); } } @@ -1293,7 +1378,9 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (sit == sessions.end()) { - PQ_LOG_D("balancing of the " << family->DebugStr() << " failed because there are no suitable reading sessions."); + YDB_LOG_DEBUG("Balancing of the failed because there are no suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); continue; } @@ -1303,7 +1390,10 @@ void TConsumer::Balance(const TActorContext& ctx) { // Reorder sessions sessions.erase(sit); - PQ_LOG_D("balancing " << family->DebugStr() << " for " << session->DebugStr()); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"debug", session->DebugStr()}); family->StartReading(*session, ctx); // Reorder sessions @@ -1322,8 +1412,12 @@ void TConsumer::Balance(const TActorContext& ctx) { auto desiredFamilyCount = familyCount / commonSessions.size(); auto allowPlusOne = familyCount % commonSessions.size(); - PQ_LOG_D("start rebalancing. familyCount=" << familyCount << ", sessionCount=" << commonSessions.size() - << ", desiredFamilyCount=" << desiredFamilyCount << ", allowPlusOne=" << allowPlusOne); + YDB_LOG_DEBUG("Start rebalancing", + {"logPrefix", LogPrefix()}, + {"familyCount", familyCount}, + {"sessionCount", commonSessions.size()}, + {"desiredFamilyCount", desiredFamilyCount}, + {"allowPlusOne", allowPlusOne}); if (!OrderedSessions) { OrderedSessions.emplace(); @@ -1352,7 +1446,9 @@ void TConsumer::Balance(const TActorContext& ctx) { auto* family = it->second; if (!family->IsActive()) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is not active."); + YDB_LOG_DEBUG("Skip balancing because it is not active", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1365,14 +1461,18 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (family->Session->ActiveFamilyCount == 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is considered a session that does not read anything else."); + YDB_LOG_DEBUG("Skip balancing because it is considered a session that does not read anything else", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; } if (family->SpecialSessions.size() <= 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because there are no other suitable reading sessions."); + YDB_LOG_DEBUG("Skip balancing because there are no other suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1394,14 +1494,18 @@ void TConsumer::Balance(const TActorContext& ctx) { family->Release(ctx); it = FamiliesRequireBalancing.erase(it); } else { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is already being read by the best session."); + YDB_LOG_DEBUG("Skip balancing because it is already being read by the best session", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); ++it; } } } auto duration = TInstant::Now() - startTime; - PQ_LOG_D("balancing duration: " << duration); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"duration", duration}); } void TConsumer::Release(ui32 partitionId, const TActorContext& ctx) { @@ -1517,8 +1621,10 @@ const std::unordered_map<TActorId, std::unique_ptr<TSession>>& TBalancer::GetSes void TBalancer::UpdateConfig(const std::vector<ui32>& addedPartitions, const std::vector<ui32>& deletedPartitions, const TActorContext& ctx) { - PQ_LOG_D("updating configuration. Deleted partitions [" << JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end()) - << "]. Added partitions [" << JoinRange(", ", addedPartitions.begin(), addedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Updating configuration. Deleted partitions Added partitions", + {"logPrefix", LogPrefix()}, + {"deletedPartitions", JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end())}, + {"addedPartitions", JoinRange(", ", addedPartitions.begin(), addedPartitions.end())}); for (auto partitionId : deletedPartitions) { for (auto& [_, consumer] : Consumers) { @@ -1544,14 +1650,19 @@ bool TBalancer::SetCommittedState(const TString& consumerName, ui32 partitionId, } if (!consumer->IsReadable(partitionId)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("The offset of the partition was commited by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); return false; } auto wasInactive = consumer->IsInactive(partitionId); if (consumer->SetCommittedState(partitionId, generation, cookie)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName); + YDB_LOG_DEBUG("The offset of the partition was commited by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); if (consumer->ProccessReadingFinished(partitionId, wasInactive, ctx)) { consumer->ScheduleBalance(ctx); @@ -1575,13 +1686,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionStartedRequest::TPtr& ev auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1593,13 +1708,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionFinishedRequest::TPtr& e auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1614,17 +1733,28 @@ void TBalancer::Handle(TEvPersQueue::TEvPartitionReleased::TPtr& ev, const TActo auto* partitionInfo = GetPartitionInfo(partitionId); if (!partitionInfo) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender << " got deleted partition " << r); + YDB_LOG_CRIT("Client pipe got deleted partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"r", r}); return; } - PQ_LOG_I("client " << r.GetClientId() << " released partition from pipe " << sender - << " session " << r.GetSession() << " partition " << partitionId); + YDB_LOG_INFO("Client released partition from pipe session partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}, + {"partitionId", partitionId}); auto* consumer = GetConsumer(consumerName); if (!consumer) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender - << " is not connected and got release partitions request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got release partitions request for session", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}); return; } @@ -1646,11 +1776,17 @@ void TBalancer::Handle(TEvPQ::TEvWakeupReleasePartition::TPtr &ev, const TActorC } if (partition->Commited) { - PQ_LOG_D("skip releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout because offset is commited"); + YDB_LOG_DEBUG("Skip releasing partition of consumer by reading finished timeout because offset is commited", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); return; } - PQ_LOG_I("releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout"); + YDB_LOG_INFO("Releasing partition of consumer by reading finished timeout", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); consumer->Release(msg->PartitionId, ctx); } @@ -1666,22 +1802,31 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerConnected::TPtr& ev, const TActor auto& session = it->second; ++session->ServerActors; - PQ_LOG_I("pipe " << sender << " connected; active server actors: " << session->ServerActors); + YDB_LOG_INFO("Pipe connected; active server", + {"logPrefix", LogPrefix()}, + {"sender", sender}, + {"actors", session->ServerActors}); } void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected."); + YDB_LOG_DEBUG("Pipe disconnected", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Subscriptions.erase(ev->Get()->ClientId); auto it = Sessions.find(ev->Get()->ClientId); if (it == Sessions.end()) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected but there aren't sessions exists."); + YDB_LOG_DEBUG("Pipe disconnected but there aren't sessions exists", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); return; } - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected; active server actors: " - << (it != Sessions.end() ? it->second->ServerActors : -1)); + YDB_LOG_INFO("Pipe disconnected; active server", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}, + {"actors", (it != Sessions.end() ? it->second->ServerActors : -1)}); auto& session = it->second; if (--(session->ServerActors) > 0) { @@ -1689,8 +1834,11 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc } if (!session->SessionName.empty()) { - PQ_LOG_NOTICE("pipe " << ev->Get()->ClientId << " client " - << session->ClientId << " disconnected session " << session->SessionName); + YDB_LOG_NOTICE("Pipe client disconnected session", + {"logPrefix", LogPrefix()}, + {"eventClientId", ev->Get()->ClientId}, + {"sessionClientId", session->ClientId}, + {"sessionName", session->SessionName}); auto* consumer = GetConsumer(session->ClientId); if (consumer) { @@ -1706,7 +1854,9 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc Sessions.erase(it); } else { - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected no session"); + YDB_LOG_INFO("Pipe disconnected no session", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Sessions.erase(it); } @@ -1717,27 +1867,37 @@ void TBalancer::Handle(TEvPersQueue::TEvRegisterReadSession::TPtr& ev, const TAc auto& consumerName = r.GetClientId(); TActorId pipe = ActorIdFromProto(r.GetPipeClient()); - PQ_LOG_NOTICE("consumer \"" << consumerName << "\" register session for pipe " << pipe << " session " << r.GetSession()); + YDB_LOG_NOTICE("Consumer register session for pipe session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); if (consumerName.empty()) { - PQ_LOG_CRIT("ignored the session registration with empty consumer name."); + YDB_LOG_CRIT("Ignored the session registration with empty consumer name", + {"logPrefix", LogPrefix()}); return; } if (r.GetSession().empty()) { - PQ_LOG_CRIT("ignored the session registration with empty session name."); + YDB_LOG_CRIT("Ignored the session registration with empty session name", + {"logPrefix", LogPrefix()}); return; } if (!pipe) { - PQ_LOG_CRIT("ignored the session registration with empty Pipe."); + YDB_LOG_CRIT("Ignored the session registration with empty Pipe", + {"logPrefix", LogPrefix()}); return; } auto jt = Sessions.find(pipe); if (jt == Sessions.end()) { - PQ_LOG_CRIT("client \"" << consumerName << "\" pipe " << pipe - << " is not connected and got register session request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got register session request for session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); return; } @@ -1852,7 +2012,9 @@ void TBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorCo } void TBalancer::ProcessPendingStats(const TActorContext& ctx) { - PQ_LOG_D("ProcessPendingStats. PendingUpdates size " << PendingUpdates.size()); + YDB_LOG_DEBUG("ProcessPendingStats. PendingUpdates size", + {"logPrefix", LogPrefix()}, + {"pendingUpdatesSize", PendingUpdates.size()}); GetPartitionGraph().Travers([&](ui32 id) { for (auto& d : PendingUpdates[id]) { @@ -1868,7 +2030,9 @@ void TBalancer::ProcessPendingStats(const TActorContext& ctx) { void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TActorContext& ctx) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingSubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingSubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto status = Consumers.contains(record.GetConsumer()) ? @@ -1880,7 +2044,9 @@ void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TAct void TBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingUnsubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingUnsubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto& consumer = record.GetConsumer(); diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp index d62bd55c870..370de6c38d0 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp @@ -3,9 +3,6 @@ #include <library/cpp/monlib/service/pages/templates.h> #include <ydb/core/persqueue/common/common_app.h> -#define DEBUG(message) - - namespace NKikimr::NPQ::NBalancing { void TBalancer::RenderApp(NApp::TNavigationBar& __navigationBar) const { diff --git a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp index 61030f78738..2fdf743951d 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp @@ -3,6 +3,8 @@ #include <util/generic/ymath.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr::NPQ::NBalancing { namespace { @@ -192,7 +194,9 @@ void TMLPConsumer::Rebuild() { } } - PQ_LOG_D("Rebuild " << JoinSeq(",", PartitionsForBalancing) << " partitions for balancing"); + YDB_LOG_DEBUG("Rebuild partitions for balancing", + {"logPrefix", LogPrefix()}, + {"partitionsForBalancing", JoinSeq(",", PartitionsForBalancing)}); } const TMLPConsumer::TMetrics& TMLPConsumer::GetMetrics() const { @@ -212,7 +216,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' does not exist"; @@ -220,7 +226,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"; @@ -270,14 +278,18 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { const auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' does not exist"), 0, ev->Cookie); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"), 0, ev->Cookie); return; @@ -285,7 +297,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { auto it = Consumers.find(consumerName); if (it == Consumers.end()) { - PQ_LOG_D("Consumer '" << consumerName << "' is not initialized"); + YDB_LOG_DEBUG("Consumer is not initialized", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPGetRuntimeAttributesResponse(0, 0, 0), 0, ev->Cookie); return; } @@ -302,7 +316,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { } void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorContext&) { - PQ_LOG_D("Handle TEvPersQueue::TEvStatusResponse " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvStatusResponse", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); absl::flat_hash_map<TString, bool> mlpConsumers; for (const auto& consumer : GetConfig().GetConsumers()) { @@ -347,7 +363,9 @@ void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActo void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvReadingPartitionStatusRequest " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvReadingPartitionStatusRequest", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), true, // reading is finished @@ -363,7 +381,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, con void TMLPBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), std::nullopt, // reading is finished @@ -418,12 +438,16 @@ void TMLPBalancer::SetUseForReading(const TString& consumerName, ui64 cookie) { auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } diff --git a/ydb/core/persqueue/pqrb/read_balancer_log.h b/ydb/core/persqueue/pqrb/read_balancer_log.h index 96890d1d810..934eb09106e 100644 --- a/ydb/core/persqueue/pqrb/read_balancer_log.h +++ b/ydb/core/persqueue/pqrb/read_balancer_log.h @@ -7,12 +7,5 @@ namespace NKikimr::NPQ { inline TString LogPrefix() { return {}; } -#define PQ_LOG_T(stream) LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_NOTICE(stream) LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/public/schema/schema_operation.cpp b/ydb/core/persqueue/public/schema/schema_operation.cpp index 0f98bb13e42..99ab505f3cf 100644 --- a/ydb/core/persqueue/public/schema/schema_operation.cpp +++ b/ydb/core/persqueue/public/schema/schema_operation.cpp @@ -177,10 +177,9 @@ private: } void ReplyErrorAndDie(Ydb::StatusIds::StatusCode errorCode, TString&& errorMessage) { - YDB_LOG_DEBUG("Dump NPQLOGPREFIX, replyErrorAndDie, errorMessage", + YDB_LOG_DEBUG(errorMessage, {"logPrefix", NPQ_LOG_PREFIX}, - {"replyErrorAndDie", errorCode}, - {"errorMessage", errorMessage}); + {"replyErrorAndDie", errorCode}); Send(ParentId, new TEvSchemaOperationResponse(errorCode, std::move(errorMessage)), 0, Cookie); PassAway(); } diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index f1ca11529d9..9a68f23d15a 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -318,7 +318,7 @@ class TPartitionWriter : public TActorBootstrapped<TPartitionWriter>, public TPa WriteId = NPQ::GetWriteId(record.GetResponse().GetTopicOperations()); - YDB_LOG_DEBUG("Dump LOGPREFIX, sessionId, txId, writeId", + YDB_LOG_DEBUG("", {"logPrefix", LOG_PREFIX}, {"sessionId", Opts.SessionId}, {"txId", Opts.TxId}, diff --git a/ydb/core/protos/blobstorage_ddisk.proto b/ydb/core/protos/blobstorage_ddisk.proto index d5db486bc4c..b453c39f312 100644 --- a/ydb/core/protos/blobstorage_ddisk.proto +++ b/ydb/core/protos/blobstorage_ddisk.proto @@ -267,8 +267,11 @@ message TEvReadPersistentBufferResult { uint64 VChunkIndex = 4; uint32 OffsetInBytes = 5; uint32 SizeInBytes = 6; - // Reserved for checksum piggybacking: same raw XXH3_64(data) per MinSectorSize block of ReadResult - // payload as TEvWritePersistentBuffer.Checksums, no identity/salt mixed in; not filled yet. + // Same raw XXH3_64(data) per MinSectorSize block of ReadResult payload as + // TEvWritePersistentBuffer.Checksums, no identity/salt mixed in. Opt-in: populated only when the + // persisted record itself carries sender-supplied checksums (empty for legacy/checksum-less + // records). Forwarded verbatim by TWritePersistentBuffersRequestActor when re-replicating a + // record via TEvReadThenWritePersistentBuffers. repeated fixed64 Checksums = 7; } diff --git a/ydb/core/protos/blobstorage_distributed_config.proto b/ydb/core/protos/blobstorage_distributed_config.proto index e051fcb28b5..f0411a11f4d 100644 --- a/ydb/core/protos/blobstorage_distributed_config.proto +++ b/ydb/core/protos/blobstorage_distributed_config.proto @@ -198,6 +198,7 @@ message TEvNodeConfigInvokeOnRoot { bool IgnoreVSlotQuotaCheck = 6; bool IsSelfHealReasonDecommit = 7; bool FromSelfHeal = 8; + bool AllowUnusableDisks = 9; } // Regenerate configuration so the slain VDisk is no more reported as DESTROY one in the list. diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index b26510e50ef..5c4c5453267 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -393,13 +393,25 @@ message TSelfManagementConfig { optional bool AutomaticBoxManagement = 21 [default = true]; // invoke BSC DefineHostConfig/DefineBox automatically optional bool AutomaticBootstrap = 22; // whether bootstrap should be performed automatically; PROHIBITED for production optional bool AutomaticStaticGroupManagement = 23; // whether distconf/SelfHeal can change static group on its behalf - optional bool AutomaticStateStorageManagement = 24; // the same for state storage - optional bool AutomaticStateStorageBoardManagement = 25; // the same for state storage board - optional bool AutomaticSchemeBoardManagement = 26; // the same for scheme board + optional bool AutomaticStateStorageManagement = 24 [default = true]; // the same for state storage; if disabled, SelfHeal must not change current state storage config + optional bool AutomaticStateStorageBoardManagement = 25 [default = true]; // the same for state storage board; if disabled, SelfHeal must not change current state storage board config + optional bool AutomaticSchemeBoardManagement = 26 [default = true]; // the same for scheme board; if disabled, SelfHeal must not change current scheme board config // node ids onto which vdisks of static groups may be relocated during self-heal; // empty list means no restriction repeated uint32 StaticGroupSelfHealAllowedNodes = 27; + + // node ids onto which state storage may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageSelfHealAllowedNodes = 28; + + // node ids onto which state storage board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageBoardSelfHealAllowedNodes = 29; + + // node ids onto which state storage scheme board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 SchemeBoardSelfHealAllowedNodes = 30; } message TBlobStorageFormatConfig { diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index c05170d7d96..4eeb18b8486 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -353,4 +353,6 @@ message TFeatureFlags { // RequireRestart: the in-memory counter is rebuilt from all FileStores at SchemeShard start, // the counter can be re-enabled, which requires restarting the SchemeShard tablet. optional bool DisableFileStoreSSDSystemSpaceAccounting = 302 [default = false, (RequireRestart) = true]; + optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = false]; + optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = false]; } diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp index 60aabe3784b..372845d292a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp @@ -27,12 +27,8 @@ void TSubColumnsMerger::DoStart(const std::vector<std::shared_ptr<NArrow::NAcces AFL_VERIFY(stats.size()); AFL_VERIFY(statRecordsCount); auto commonStats = TDictStats::Merge(stats, GetSettings(), statRecordsCount); - auto splitted = commonStats.SplitByVolume(GetSettings(), statRecordsCount); - ResultColumnStats = splitted.ExtractColumns(); + ResultColumnStats = commonStats.SelectSeparatedColumns(GetSettings(), statRecordsCount); ResultColumnStats->CreateJsonPathAccessorTrieCache(); - // YDB_LOG_ERROR("", - // {"columns", ResultColumnStats->DebugJson()}, - // {"others", splitted.ExtractOthers().DebugJson()}); RemapKeyIndex.RegisterColumnStats(*ResultColumnStats); for (auto&& i : OrderedIterators) { i.Start(); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp index c1206d05bf0..07480a5a5fa 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp @@ -20,8 +20,7 @@ TRemapColumns::TOthersData::TFinishContext TRemapColumns::BuildRemapInfo( builder.Add(i.first, statsByKeyIndex[i.second].GetRecordsCount(), statsByKeyIndex[i.second].GetDataSize(), settings.IsSparsed(statsByKeyIndex[i.second].GetRecordsCount(), recordsCount) ? NArrow::NAccessor::IChunkedArray::EType::SparsedArray : NArrow::NAccessor::IChunkedArray::EType::Array, - // For now others always encode in BinaryJson - NArrow::NAccessor::NSubColumns::EValueType::BinaryJson); + NArrow::NAccessor::NSubColumns::OthersExplicitBinaryJson); remap[i.second] = idx++; } return TOthersData::TFinishContext(builder.Finish(), remap); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 8fff773c8be..240c0750a78 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -8010,7 +8010,7 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, // Control op completed; finalize the tracked record. Execute(CreateTxFullBackupProgress(ui64(txId)), ctx); } - if (TxIdToSetColumnConstraintOperations.contains(txId)) { + if (TxIdToSetColumnConstraintOperations.contains(txId) || TxIdToDependentSetColumnConstraint.contains(txId)) { Execute(CreateTxReplyCompletedSetColumnConstraint(txId), ctx); executed = true; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 1fa19209ba2..90ae47b49ad 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1993,6 +1993,10 @@ public: THashMap<TString, std::shared_ptr<TSetColumnConstraintOperationInfo>> SetColumnConstraintOperationsByUid; TSet<std::pair<TInstant, TIndexBuildId>> SetColumnConstraintOperationsByTime; THashMap<TTxId, TIndexBuildId> TxIdToSetColumnConstraintOperations; + // txIds of concurrent operations (e.g. a backup CopyTable) that a + // SetColumnConstraint operation is currently waiting on before it can retry + // one of its own internal sub-transactions. + THashMap<TTxId, THashSet<TIndexBuildId>> TxIdToDependentSetColumnConstraint; void PersistCreateSetColumnConstraint(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); void PersistSetColumnConstraintState(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp index 288caedd7c8..8e11a524455 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp @@ -205,45 +205,109 @@ public: << ", status# " << NKikimrScheme::EStatus_Name(record.GetStatus())); NIceDb::TNiceDb db(txc.DB); - - auto replyOnCreation = [&] { + bool shouldForget = false; + auto replyOnCreation = [&]() -> void { auto statusCode = TranslateStatusCode(record.GetStatus()); if (statusCode != Ydb::StatusIds::SUCCESS) { - // TODO(flown4qqqq): persist issue - // TODO(flown4qqqq): forget operation on error - // TODO(flown4qqqq): EraseBuildInfo(operationInfo); + shouldForget = true; } ReplyOnCreation(operationInfo, statusCode); - return true; }; - if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Locking) { - Y_ENSURE(txId == operationInfo.LockTxId); - operationInfo.LockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + // Most modifications of the main table are forbidden while SetColumnConstraint + // holds its lock, however copying the table (e.g. for a backup) is allowed to + // proceed concurrently (see TCopyTable::Propose: CheckLocks is skipped for + // IsBackup copies). While such a copy is in flight the table has + // PathStateCopying, and our own internal sub-transactions (which are gated by + // NotUnderOperation()) may transiently fail with StatusMultipleModifications. + // Just like build_index does, retry the very same proposal once the copy is + // done instead of failing the whole operation. + // Waiting for the conflicting operation's completion notification, not for + // an immediate self-retry - see the comment on shouldRetry() below for why. + bool waitingForDependency = false; - if (!replyOnCreation()) { + auto shouldRetry = [&]() { + if (record.GetStatus() != NKikimrScheme::StatusMultipleModifications) { return false; } - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites) { - Y_ENSURE(txId == operationInfo.LockNullWritesTxId); - operationInfo.LockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Finishing) { - Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); - operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Unlocking) { - Y_ENSURE(txId == operationInfo.UnlockTxId); - operationInfo.UnlockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); - } else { - Y_UNREACHABLE(); + + auto it = Self->PathsById.find(operationInfo.TablePathId); + if (it == Self->PathsById.end() || it->second->PathState != NKikimrSchemeOp::EPathStateCopying) { + return false; + } + + auto copyTxId = it->second->LastTxId; + LOG_I("TTxReplyModify : Waiting for txId " << copyTxId << " to retry SetColumnConstraint id# " << BuildId); + operationInfo.DependencyTxIds.insert(copyTxId); + Self->TxIdToDependentSetColumnConstraint[copyTxId].insert(BuildId); + Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(copyTxId))); + return true; + }; + + switch (operationInfo.OperationState) { + case TSetColumnConstraintOperationInfo::EOperationState::Locking: { + Y_ENSURE(txId == operationInfo.LockTxId); + operationInfo.LockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + replyOnCreation(); + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites: { + Y_ENSURE(txId == operationInfo.LockNullWritesTxId); + if (shouldRetry()) { + operationInfo.LockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.LockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Finishing: { + Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); + if (shouldRetry()) { + operationInfo.UnlockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Unlocking: { + Y_ENSURE(txId == operationInfo.UnlockTxId); + if (shouldRetry()) { + operationInfo.UnlockTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Invalid: + case TSetColumnConstraintOperationInfo::EOperationState::Validating: + case TSetColumnConstraintOperationInfo::EOperationState::Done: { + Y_UNREACHABLE(); + } + } + + if (!waitingForDependency) { + Progress(BuildId); + } + + if (shouldForget) { + Self->ForgetSetColumnConstraint(db, operationInfo); } - Progress(BuildId); return true; } @@ -271,6 +335,23 @@ public: const auto txId = CompletedTxId; auto* operationIdPtr = Self->TxIdToSetColumnConstraintOperations.FindPtr(txId); + + if (Self->TxIdToDependentSetColumnConstraint.contains(txId)) { + THashSet<TIndexBuildId> deps = std::move(Self->TxIdToDependentSetColumnConstraint.at(txId)); + Self->TxIdToDependentSetColumnConstraint.erase(txId); + for (auto& dependentBuildId : deps) { + LOG_I("TTxReplyCompleted txId: " << txId << " : trying to resume dependent SetColumnConstraint " << dependentBuildId); + if (auto* dependentOperationInfoPtr = Self->SetColumnConstraintOperations.FindPtr(dependentBuildId)) { + auto& dependentOperationInfo = **dependentOperationInfoPtr; + dependentOperationInfo.DependencyTxIds.erase(txId); + Progress(dependentBuildId); + } + } + if (!operationIdPtr) { + return true; + } + } + if (!operationIdPtr) { LOG_I("TTxReplyCompleted: operation not found, txId# " << txId); return true; @@ -638,6 +719,11 @@ public: Y_ENSURE(operationInfoPtr); auto& operationInfo = *operationInfoPtr->get(); + if (!operationInfo.DependencyTxIds.empty()) { + LOG_N("TTxProgressSetColumnConstraint: " << BuildId << ": waiting for dependencies"); + return true; + } + LOG_D("TTxProgressSetColumnConstraint::DoExecute, id# " << BuildId << "; OperationState = " << ToString(operationInfo.OperationState) << "; IsCancelled = " << operationInfo.IsCancelled); diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp index a444ad78332..c73f6e485b6 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp @@ -2383,5 +2383,459 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { Cerr << "=== Successfully cancelled at stage: " << stage.StageName << " ===" << Endl; } } + + Y_UNIT_TEST(MoveTableFailsWhileValidating) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Block validation requests to datashards so that the operation is stuck + // in the Validating stage (the table lock is already held at this point). + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + // While SetNotNull is stuck validating shards (table is locked), attempt to move + // the table. MoveTable must be rejected immediately at Propose (no TxState is even + // created for it), because CheckLocks() sees the path is locked by the SetNotNull + // operation and MoveTable's transaction carries no matching LockGuard. + ui64 moveTxId = ++txId; + TestMoveTable(runtime, moveTxId, tablePath, movedTablePath, + {NKikimrScheme::StatusMultipleModifications}); + + // The table must still be exactly where it was - move never actually started. + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathNotExist}); + + // Let SetNotNull finish normally. + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + // Now that the lock is released, the very same move must succeed. + TestMoveTable(runtime, ++txId, tablePath, movedTablePath); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + } + + Y_UNIT_TEST(SetNotNullFailsWhileMoveTableInProgress) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep MoveTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, MoveTable's local Propose has already committed - the source + // path is already marked EPathStateMoving. + TBlockEvents<TEvDataShard::TEvProposeTransaction> moveProposeBlocker(runtime); + + ui64 moveTxId = ++txId; + AsyncMoveTable(runtime, moveTxId, tablePath, movedTablePath); + + runtime.WaitFor("block move propose to datashards", [&]{ return moveProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (moving). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while moving): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_UNEQUAL_C( + response.GetStatus(), + Ydb::StatusIds::SUCCESS, + response.ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let MoveTable finish normally. + moveProposeBlocker.Stop(); + moveProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, moveTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + + // Now that the move is done, SetNotNull on the (now moved) table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + movedTablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, movedTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableNoBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the first scenario. + + For a standalone CopyTable, it is fine to fail the request and tell the user that the table is currently locked + by another long-running operation. In that scenario, CopyTable is a single explicit operation that the user can simply retry. + This is also the case where CopyTable is expected to check whether the table is locked by other schema operations. + */ + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx1 = ++txId; + TestConsistentCopyTables(runtime, copyTx1, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )", {NKikimrScheme::StatusMultipleModifications}); + env.TestWaitNotification(runtime, copyTx1); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathNotExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + ui64 copyTx2 = ++txId; + TestConsistentCopyTables(runtime, copyTx2, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )"); + env.TestWaitNotification(runtime, copyTx2); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableWithBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the second scenario. + + When CopyTable is invoked as part of Backup, it should not fail just because the table is locked by another + long-running operation. If ConsistentCopyTable gets blocked this way during a backup, it may lead to a large + number of retries. Each retry effectively means trying to copy all tables in the database again, which is a very + expensive and long-running operation. + + Therefore, the practical way to avoid such retries is to make the new long-running operation pause and wait + until the backup's CopyTable finishes. + */ + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx = ++txId; + TestConsistentCopyTables(runtime, copyTx, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: true + } + )"); + env.TestWaitNotification(runtime, copyTx); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", false}}); + + { + const auto describeResult = DescribePath(runtime, tablePath); + const auto& columns = describeResult.GetPathDescription().GetTable().GetColumns(); + for (const auto& column : columns) { + if (column.GetName() == "value") { + UNIT_ASSERT_VALUES_EQUAL(column.GetSetNotNullInProgress(), false); + } + } + } + } + + Y_UNIT_TEST_TWIN(SetNotNullFailsWhileCopyTableInProgress, IsBackup) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableCopy"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep CopyTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, CopyTable's local Propose has already committed - the source + // path is already marked EPathStateCopying. + TBlockEvents<TEvDataShard::TEvProposeTransaction> copyProposeBlocker(runtime); + + ui64 copyTxId = ++txId; + if (IsBackup) { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: true + } + )"); + } else { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: false + } + )"); + } + + runtime.WaitFor("block copy propose to datashards", [&]{ return copyProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (copying). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while copying): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let CopyTable finish normally. + copyProposeBlocker.Stop(); + copyProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, copyTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + // Now that the copy is done, SetNotNull on the source table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + } + + Y_UNIT_TEST(SetNotNullWithUsualSchemeOperation) { + // As an example of a 'usual' scheme operation, this test uses TRUNCATE TABLE. + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + + ui64 txId = 100; + + TString root = "/MyRoot"; + TString tablePath = root + "/Table"; + + TestCreateTable(runtime, ++txId, root, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvProposeTransaction> truncateTableBlocker(runtime); + const ui64 truncateTxId = ++txId; + AsyncTruncateTable(runtime, truncateTxId, root, "Table", TTestTxConfig::SchemeShard); + runtime.WaitFor("block truncate propose to datashards", [&]{ return truncateTableBlocker.size() > 0; }); + + ui64 setConstraintTxId = ++txId; + auto response = TestSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + truncateTableBlocker.Stop(); + truncateTableBlocker.Unblock(); + env.TestWaitNotification(runtime, truncateTxId); + } } // Y_UNIT_TEST_SUITE(SetNotNullTest) diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make index 13dfda64b9e..9df7e2698c9 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make @@ -1,6 +1,8 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) FORK_SUBTESTS() +SPLIT_FACTOR(60) +SIZE(MEDIUM) PEERDIR( library/cpp/getopt diff --git a/ydb/library/actors/core/servicemap.h b/ydb/library/actors/core/servicemap.h index d72e50cae5f..209d8cf13e3 100644 --- a/ydb/library/actors/core/servicemap.h +++ b/ydb/library/actors/core/servicemap.h @@ -2,23 +2,60 @@ #include "defs.h" +#include <util/generic/algorithm.h> + +#include <array> +#include <atomic> +#include <cstring> +#include <type_traits> + namespace NActors { // wait-free one writer multi reader hash-tree for service mapping purposes // on fast updates on same key - could lead to false-negatives, we don't care as such cases are broken from service-map app logic template <typename TKey, typename TValue, typename THash, ui64 BaseSize = 256 * 1024, ui64 ExtCount = 4, ui64 ExtBranching = 4> class TServiceMap : TNonCopyable { + // CounterIn/CounterOut validate the whole snapshot, but every part still + // has to be accessed atomically to make a discarded torn read well-defined. + template <typename T> + class TAtomicStorage : TNonCopyable { + static_assert(std::is_trivially_copyable_v<T>); + static_assert(sizeof(T) % sizeof(ui64) == 0); + + static constexpr size_t PartCount = sizeof(T) / sizeof(ui64); + std::array<std::atomic<ui64>, PartCount> Parts{}; + + public: + T Load() const noexcept { + std::array<ui64, PartCount> parts; + for (size_t i = 0; i != PartCount; ++i) { + parts[i] = Parts[i].load(std::memory_order_relaxed); + } + + T value; + std::memcpy(&value, parts.data(), sizeof(value)); + return value; + } + + void Store(const T& value) noexcept { + std::array<ui64, PartCount> parts; + std::memcpy(parts.data(), &value, sizeof(value)); + + for (size_t i = 0; i != PartCount; ++i) { + Parts[i].store(parts[i], std::memory_order_relaxed); + } + } + }; + struct TEntry : TNonCopyable { ui32 CounterIn; ui32 CounterOut; - TKey Key; - TValue Value; + TAtomicStorage<TKey> Key; + TAtomicStorage<TValue> Value; TEntry() : CounterIn(0) , CounterOut(0) - , Key() - , Value() { } }; @@ -39,8 +76,8 @@ namespace NActors { for (ui32 i = 0; i != ExtCount; ++i) { const TEntry& entry = branch->Entries[i]; const ui32 counterIn = AtomicLoad(&entry.CounterIn); - if (counterIn != 0 && entry.Key == key) { - ret = entry.Value; + if (counterIn != 0 && entry.Key.Load() == key) { + ret = entry.Value.Load(); const ui32 counterOut = AtomicLoad(&entry.CounterOut); if (counterOut == counterIn) return true; @@ -64,7 +101,7 @@ namespace NActors { return; } } else { - if (entry.Key == key) { + if (entry.Key.Load() == key) { oldEntry = &entry; if (!zeroEntry || *zeroEntry) return; @@ -131,12 +168,12 @@ namespace NActors { // now we got both entries, first - push new one const ui32 counter = AtomicUi32Increment(&Counter); AtomicStore(&zeroEntry->CounterOut, counter); - zeroEntry->Key = key; - zeroEntry->Value = value; + zeroEntry->Key.Store(key); + zeroEntry->Value.Store(value); AtomicStore(&zeroEntry->CounterIn, counter); if (oldEntry != nullptr) { - const TValue ret = oldEntry->Value; + const TValue ret = oldEntry->Value.Load(); AtomicStore<ui32>(&oldEntry->CounterOut, 0); AtomicStore<ui32>(&oldEntry->CounterIn, 0); return ret; diff --git a/ydb/library/actors/core/ut/servicemap_ut.cpp b/ydb/library/actors/core/ut/servicemap_ut.cpp new file mode 100644 index 00000000000..5f32b68ffef --- /dev/null +++ b/ydb/library/actors/core/ut/servicemap_ut.cpp @@ -0,0 +1,57 @@ +#include "../actorid.h" +#include "../servicemap.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <atomic> +#include <latch> +#include <thread> +#include <vector> + +namespace NActors { + +Y_UNIT_TEST_SUITE(ServiceMap) { + + Y_UNIT_TEST(ConcurrentUpdateAndFind) { + constexpr size_t readerCount = 4; + constexpr size_t updateCount = 1'000'000; + + TServiceMap<TActorId, TActorId, TActorId::THash, 1, 2, 2> map; + const TActorId serviceId(1, TStringBuf("service")); + const TActorId actorId1(1, 0, 1, 0); + const TActorId actorId2(1, 0, 2, 0); + map.Update(serviceId, actorId1); + + std::atomic<bool> stop = false; + std::atomic<size_t> invalidReads = 0; + std::latch start(readerCount + 1); + std::vector<std::thread> readers; + readers.reserve(readerCount); + + for (size_t i = 0; i != readerCount; ++i) { + readers.emplace_back([&] { + start.arrive_and_wait(); + while (!stop.load(std::memory_order_relaxed)) { + const TActorId actorId = map.Find(serviceId); + if (actorId && actorId != actorId1 && actorId != actorId2) { + invalidReads.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + + start.arrive_and_wait(); + for (size_t i = 0; i != updateCount; ++i) { + map.Update(serviceId, i % 2 ? actorId1 : actorId2); + } + stop.store(true, std::memory_order_relaxed); + + for (std::thread& reader : readers) { + reader.join(); + } + + UNIT_ASSERT_VALUES_EQUAL(invalidReads.load(), 0); + } +} + +} // namespace NActors diff --git a/ydb/library/actors/core/ut/ya.make b/ydb/library/actors/core/ut/ya.make index 6c18ed3f37a..c68796ac696 100644 --- a/ydb/library/actors/core/ut/ya.make +++ b/ydb/library/actors/core/ut/ya.make @@ -38,6 +38,7 @@ SRCS( mon_ut.cpp scheduler_actor_ut.cpp mailbox_lockfree_ut.cpp + servicemap_ut.cpp struct_log_ut.cpp ) diff --git a/ydb/library/actors/interconnect/load.cpp b/ydb/library/actors/interconnect/load.cpp index 1f2e2db20c1..727ae35f1f3 100644 --- a/ydb/library/actors/interconnect/load.cpp +++ b/ydb/library/actors/interconnect/load.cpp @@ -364,12 +364,13 @@ namespace NInterconnect { msg << " Throughput# "; const TDuration duration = now - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); const ui64 traffic = *Traffic; msg << "{window# " << duration << " bytes# " << ThroughputBytes << " samples# " << ThroughputSamples - << " b/s# " << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()) - << " common# " << ui64((traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()) + << " b/s# " << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)) + << " common# " << (durationUs ? ui64((traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)) << "}"; if (RunFirstSample == TMonotonic::Zero()) { @@ -478,6 +479,7 @@ namespace NInterconnect { TString RenderHTML(bool finished, const TActorContext& ctx) { const auto duration = ctx.Monotonic() - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); TStringStream msg; @@ -498,8 +500,8 @@ namespace NInterconnect { TABLED() { str << duration; } TABLED() { str << ThroughputBytes; } TABLED() { str << ThroughputSamples; } - TABLED() { str << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()); } - TABLED() { str << ui64((*Traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()); } + TABLED() { str << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)); } + TABLED() { str << (durationUs ? ui64((*Traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)); } } } } diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp index a3555725f02..ba828b2907d 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + namespace NKikimr { namespace NGRpcProxy { @@ -78,7 +80,8 @@ void TClustersUpdater::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TA } ctx.Schedule(TDuration::Seconds(AppData(ctx)->PQConfig.GetClustersUpdateTimeoutSec()), new TEvPQClustersUpdater::TEvUpdateClusters()); } else { - LOG_ERROR_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "can't update clusters " << record); + YDB_LOG_ERROR_CTX(ctx, "Can't update clusters", + {"record", record}); ctx.Schedule(TDuration::Seconds(CLUSTERS_UPDATER_TIMEOUT_ON_ERROR), new TEvPQClustersUpdater::TEvUpdateClusters()); } } diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index 54e10ae2006..0c20b05f00f 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -24,6 +24,8 @@ #include <algorithm> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_READ_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -39,8 +41,8 @@ using namespace NSchemeCache; #ifdef PQ_LOG_PREFIX #undef PQ_LOG_PREFIX #endif -#define PQ_LOG_PREFIX "session cookie " << Cookie << " client " << InternalClientId << " session " << Session +#define PQ_LOG_PREFIX (TStringBuilder() << "session cookie " << Cookie << " client " << InternalClientId << " session " << Session) //11 tries = 10,23 seconds, then each try for 5 seconds , so 21 retries will take near 1 min static const NTabletPipe::TClientRetryPolicy RetryPolicyForPipes = { @@ -352,7 +354,8 @@ void TReadSessionActor::Die(const TActorContext& ctx) { if (t.second->PipeClient) NTabletPipe::CloseClient(ctx, t.second->PipeClient); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Is DEAD", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); if (SessionsActive) { --(*SessionsActive); @@ -408,7 +411,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorCont CloseSession(TStringBuilder() << "double commit of cookie " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx); return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from client for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request from client", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readId", readId}); MakeCommit(ctx); } @@ -427,7 +432,10 @@ void TReadSessionActor::MakeCommit(const TActorContext& ctx) { if (readId == ReadIdCommitted) return; NextCommits.erase(NextCommits.begin(), it); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from " << ReadIdCommitted + 1 << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdCommittedPlus1", ReadIdCommitted + 1}, + {"readId", readId}); auto& commit = Commits[readId]; commit.StartReadId = ReadIdCommitted + 1; @@ -486,8 +494,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const TActorC if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL get status request from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL get status request from client for generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"generation", ev->Get()->Generation}); return; } @@ -514,13 +525,22 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvLocked::TPtr& ev, const TActorCont if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL lock from client for " << topic - << ":" << ev->Get()->Partition << " at offset " << ev->Get()->ReadOffset << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL lock from client for at offset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"generation", ev->Get()->Generation}); return; } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got lock from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " at readOffset " << ev->Get()->ReadOffset << " commitOffset " << ev->Get()->CommitOffset << " generation " << ev->Get()->Generation); + YDB_LOG_INFO_CTX(ctx, "Got lock from client for at readOffset commitOffset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"commitOffset", ev->Get()->CommitOffset}, + {"generation", ev->Get()->Generation}); //proxy request to partition - allow initing //TODO: add here VerifyReadOffset too and check it against Committed position @@ -582,8 +602,10 @@ void TReadSessionActor::AnswerForCommitsIfCan(const TActorContext& ctx) { for (ui64 i = it->second.StartReadId; i <= readId; ++i){ result.MutableCommit()->AddCookie(i); } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits from " << it->second.StartReadId - << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Replying for commits", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"startReadId", it->second.StartReadId}, + {"readId", readId}); ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; @@ -706,7 +728,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo topicsToResolve.insert(t); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "From", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"init", event->Request}, + {"peerName", PeerName}); ClientsideLocksAllowed = init.GetClientsideLocksAllowed(); BalanceRightNow = init.GetBalancePartitionRightNow() || CommitsDisabled; @@ -716,7 +741,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo } if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, "session without AuthInfo : " << ExternalClientId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo", + {"externalClientId", ExternalClientId}, + {"peerName", PeerName}); if (SessionsWithoutAuth) { ++(*SessionsWithoutAuth); } @@ -811,8 +838,10 @@ void TReadSessionActor::CreateInitAndAuthActor(const TActorContext& ctx) { void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) { TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_DEBUG_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -827,7 +856,9 @@ void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& void TReadSessionActor::RegisterSession(const TActorId& pipe, const TString& topic, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " register session to " << topic); + YDB_LOG_INFO_CTX(ctx, "Register session", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}); THolder<TEvPersQueue::TEvRegisterReadSession> request; request.Reset(new TEvPersQueue::TEvRegisterReadSession); auto& req = request->Record; @@ -940,7 +971,10 @@ void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const LastACLCheckTimestamp = ctx.Now(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone); + YDB_LOG_DEBUG_CTX(ctx, "Auth ok, got topics, init done", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topicAndTablets", ev->Get()->TopicAndTablets.size()}, + {"initDone", InitDone}); AuthInitActor = TActorId(); AuthInflight = false; @@ -1046,10 +1080,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path)); if (converterIter.IsEnd()) { - LOG_ALERT_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for event = " << record.ShortDebugString() << " path not recognized" - ); + YDB_LOG_ALERT_CTX(ctx, "Ignored ev lock for event path not recognized", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"record", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse lock event: " << record.ShortDebugString() << ", reason: topic not found", NPersQueue::NErrorCode::ERROR, ctx @@ -1062,21 +1095,18 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto jt = Topics.find(intName); if (jt == Topics.end() || pipe != jt->second->PipeClient) { //this is message from old version of pipe - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for topic = " << converterIter->second->GetPrintableString() - << " path recognized, but topic is unknown, this is unexpected" - ); + YDB_LOG_DEBUG_CTX(ctx, "Ignored ev lock for topic path recognized, but topic is unknown, this is unexpected", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", converterIter->second->GetPrintableString()}); return; } auto partitionGraph = jt->second->GetPartitionGraph(); auto* partitionNode = partitionGraph->GetPartition(record.GetPartition()); if (!partitionNode) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " lock for unknown partition = " << record.GetPartition() - ); + YDB_LOG_DEBUG_CTX(ctx, "Lock for unknown partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.GetPartition()}); Locks.push_back(ev->Release()); if (!AuthInflight) { SendAuthRequest(ctx); @@ -1124,7 +1154,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T it->second.PartitionsLocked.Inc(); it->second.PartitionsInfly.Inc(); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " lock: " << record); + YDB_LOG_INFO_CTX(ctx, "Lock partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"ev", record}); ctx.Send(actorId, new TEvPQProxy::TEvLockPartition(0, 0, false, !ClientsideLocksAllowed)); } @@ -1137,8 +1169,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T auto it = Partitions.find(std::make_pair(evTopic->GetClientsideName(), ev->Get()->Partition)); Y_ABORT_UNLESS(it != Partitions.end()); if (!it->second.LockGeneration) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " the unlocked partition " << ev->Get()->Partition << " status has been requested"); + YDB_LOG_ALERT_CTX(ctx, "The unlocked partition status has been requested", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", ev->Get()->Partition}); CloseSession( TStringBuilder() << "Internal server error, the unlocked partition " << ev->Get()->Partition << " status has been requested", NPersQueue::NErrorCode::ERROR, ctx @@ -1212,7 +1245,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(topic)); if (converterIter.IsEnd()) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Failed to parse balancer response: " << record.ShortDebugString()); + YDB_LOG_ALERT_CTX(ctx, "Failed to parse balancer", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"response", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse release event: " << record.ShortDebugString() << ", path not recognized", NPersQueue::NErrorCode::ERROR, ctx @@ -1232,7 +1267,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons } auto onUnknownPartition = [&](auto& marker) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Releasing unknown partition: " << record.ShortDebugString() << " " << marker); + YDB_LOG_ALERT_CTX(ctx, "Releasing unknown", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.ShortDebugString()}, + {"marker", marker}); CloseSession( TStringBuilder() << "Internal server error, releasing unknown partition: " << record.ShortDebugString() << " " << marker, NPersQueue::NErrorCode::ERROR, ctx @@ -1265,7 +1303,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons it->second.PartitionsToBeReleased.Inc(); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " releasing " << jt->first.first << ":" << jt->first.second); + YDB_LOG_INFO_CTX(ctx, "Releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", jt->first.first}, + {"partition", jt->first.second}); jt->second.Releasing = true; ctx.Send(actorId, new TEvPQProxy::TEvReleasePartition()); @@ -1319,7 +1360,10 @@ void TReadSessionActor::InformBalancerAboutRelease(const THashMap<std::pair<TStr req.SetTopic(it->first.first); req.SetPartition(it->first.second); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->first.first << ":" << it->first.second); + YDB_LOG_INFO_CTX(ctx, "Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", it->first.first}, + {"partition", it->first.second}); NTabletPipe::SendData(ctx, jt->second->PipeClient, request.Release()); } @@ -1347,17 +1391,21 @@ void TReadSessionActor::CloseSession(const TString& errorReason, const NPersQueu error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed with error reason: " << errorReason); + YDB_LOG_INFO_CTX(ctx, "Closed with error", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"reason", errorReason}); if (!Handler->IsShuttingDown()) { ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; Handler->Reply(std::move(result)); } else { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " GRps is shutting dows, skip reply"); + YDB_LOG_WARN_CTX(ctx, "GRps is shutting down, skip reply", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed"); + YDB_LOG_INFO_CTX(ctx, "Closed", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } Die(ctx); @@ -1457,7 +1505,9 @@ bool TReadSessionActor::ProcessReleasePartition(const THashMap<std::pair<TString bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorContext& ctx) { for (auto& t : Topics) { if (t.second->TabletID == tablet) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " balancer for topic " << t.first << " is dead, restarting all from this topic"); + YDB_LOG_INFO_CTX(ctx, "Balancer for topic is dead, restarting all from this topic", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", t.first}); //Drop all partitions from this topic for (auto it = Partitions.begin(); it != Partitions.end();) { @@ -1520,7 +1570,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContex ProcessAuth(event->Request.GetCredentials()); event->Request.ClearCredentials(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got read request: " << event->Request.GetRead() << " with guid: " << event->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Got read with", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", event->Request.GetRead()}, + {"guid", event->Guid}); Reads.emplace_back(event.Release()); @@ -1562,9 +1615,12 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TAct TFormedReadResponse::TPtr formedResponse = it->second; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " read done guid " << formedResponse->Guid - << " " << key.first << ":" << key.second - << " size " << event->Response.ByteSize()); + YDB_LOG_DEBUG_CTX(ctx, "Read done guid size", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}, + {"keyFirst", key.first}, + {"keySecond", key.second}, + {"byteSize", event->Response.ByteSize()}); const i64 diff = formedResponse->ApplyResponse(std::move(event->Response)); if (event->FromDisk) { @@ -1609,7 +1665,10 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo const bool hasMessages = HasMessages(formedResponse->Response.GetBatchedData()); if (hasMessages) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " assign read id " << ReadIdToResponse << " to read request " << formedResponse->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Assign read id to read request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdToResponse", ReadIdToResponse}, + {"guid", formedResponse->Guid}); formedResponse->Response.MutableBatchedData()->SetCookie(ReadIdToResponse); // reply to client if (ProtocolVersion < NPersQueue::TReadRequest::Batching) { @@ -1618,7 +1677,9 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo diff -= formedResponse->Response.ByteSize(); // Bytes will be tracked inside handler Handler->Reply(std::move(formedResponse->Response)); } else { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " empty read result " << formedResponse->Guid << ", start new reading"); + YDB_LOG_DEBUG_CTX(ctx, "Empty read result start new reading", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}); } BytesInflight_ -= diff; @@ -1764,11 +1825,16 @@ bool TReadSessionActor::ProcessReads(const TActorContext& ctx) { readR->SetMaxTimeLagMs(MaxTimeLagMs); readR->SetReadTimestampMs(ReadTimestampMs); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX - << " performing read request: " << (*readR) << " with guid " << read->Guid - << " from " << part.Topic->GetPrintableString() << ", partition:" << part.Partition - << " count " << ccount << " size " << csize - << " partitionsAsked " << partitionsAsked << " maxTimeLag " << MaxTimeLagMs << "ms"); + YDB_LOG_DEBUG_CTX(ctx, "Performing read with guid from count size partitionsAsked maxTimeLag ms", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", (*readR)}, + {"guid", read->Guid}, + {"topic", part.Topic->GetPrintableString()}, + {"partition", part.Partition}, + {"ccount", ccount}, + {"csize", csize}, + {"partitionsAsked", partitionsAsked}, + {"maxTimeLagMs", MaxTimeLagMs}); Y_ABORT_UNLESS(!it->second.Reading); @@ -1807,10 +1873,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionReady::TPtr& ev, const TA if (!ActualPartitionActor(ev->Sender)) return; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << ev->Get()->Topic->GetPrintableString() - << " partition:" << ev->Get()->Partition << " ready for read with readOffset " - << ev->Get()->ReadOffset << " endOffset " << ev->Get()->EndOffset << " WTime " - << ev->Get()->WTime << " sizeLag " << ev->Get()->SizeLag); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset WTime sizeLag", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic->GetPrintableString()}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"endOffset", ev->Get()->EndOffset}, + {"WTime", ev->Get()->WTime}, + {"sizeLag", ev->Get()->SizeLag}); const auto it = PartitionToReadResponse.find(ev->Sender); // check whether this partition is taking part in read response auto& container = it != PartitionToReadResponse.end() ? it->second->PartitionsBecameAvailable : AvailablePartitions; @@ -1834,7 +1904,8 @@ void TReadSessionActor::HandleWakeup(const TActorContext& ctx) { ForceACLCheck = false; RequestNotChecked = false; Y_ABORT_UNLESS(!AuthInitActor); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout"); + YDB_LOG_DEBUG_CTX(ctx, "Checking auth because of timeout", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); SendAuthRequest(ctx); } @@ -1935,17 +2006,29 @@ void TPartitionActor::Bootstrap(const TActorContext&) { void TPartitionActor::CheckRelease(const TActorContext& ctx) { const bool hasUncommittedData = ReadOffset > ClientCommitOffset && ReadOffset > ClientReadOffset; if (NeedRelease) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " checking release readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_DEBUG_CTX(ctx, "Checking release readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } if (NeedRelease && ReadGuid.empty() && CommitsInfly.empty() && !hasUncommittedData && !Released) { Released = true; ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReleased(Topic, Partition)); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " check release done - releasing; readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_INFO_CTX(ctx, "Check release done - releasing; readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } } @@ -1991,9 +2074,14 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc Y_ABORT_UNLESS(!Session.empty()); commit->SetSessionId(Session); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" - << Partition << " committing to position " << offset << " prev " << CommittedOffset - << " end " << EndOffset << " by cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Committing to position prev end by cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); @@ -2004,17 +2092,22 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { if (CommitsInfly.empty()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for nothing"); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting for nothing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}); return; } ui64 readId = CommitsInfly.front().first; if (cookie != readId) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}, + {"readId", readId}); return; } @@ -2025,17 +2118,23 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { CommitsInfly.pop_front(); if (readId != Max<ui64>()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit done to position endOffset with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); while (!CommitsInfly.empty() && CommitsInfly.front().second == Max<ui64>()) { //this is cookies that have no effect on this partition readId = CommitsInfly.front().first; CommitsInfly.pop_front(); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " commit done with no effect with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Partition commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); } CheckRelease(ctx); @@ -2043,8 +2142,12 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { } void TPartitionActor::SendPartitionReady(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << Topic->GetPrintableString() << " partition:" << Partition - << " ready for read with readOffset " << ReadOffset << " endOffset " << EndOffset); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}); if (FirstRead) { ctx.Send(ParentId, new TEvPQProxy::TEvReadingStarted(Topic->GetInternalName(), Partition)); FirstRead = false; @@ -2072,8 +2175,12 @@ void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reaso ctx.Schedule(TDuration::MilliSeconds(RESTART_PIPE_DELAY_MS), new TEvPQProxy::TEvRestartPipe()); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " schedule pipe restart attempt " << PipeGeneration << " reason: " << reason); + YDB_LOG_INFO_CTX(ctx, "Schedule pipe restart attempt", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"reason", reason}); } @@ -2092,13 +2199,23 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); Y_ABORT_UNLESS(TabletID); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " RequestInfly " << RequestInfly << " ReadOffset " << ReadOffset << " EndOffset " << EndOffset - << " InitDone " << InitDone << " WaitForData " << WaitForData); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt RequestInfly ReadOffset EndOffset InitDone WaitForData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"requestInfly", RequestInfly}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"initDone", InitDone}, + {"waitForData", WaitForData}); if (RequestInfly) { //got read infly - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " resend " << CurrentRequest); + YDB_LOG_INFO_CTX(ctx, "Resend", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"currentRequest", CurrentRequest}); TAutoPtr<TEvPersQueue::TEvRequest> event(new TEvPersQueue::TEvRequest); event->Record = CurrentRequest; @@ -2152,16 +2269,21 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " initDone " << InitDone << " event " << PartitionResponseToLog(result)); + YDB_LOG_DEBUG_CTX(ctx, "InitDone event", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"initDone", InitDone}, + {"partitionResponseToLog", PartitionResponseToLog(result)}); if (!InitDone) { if (result.GetCookie() != INIT_COOKIE) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited response in init with cookie " << result.GetCookie()); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response in init with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}); return; } Y_ABORT_UNLESS(RequestInfly); @@ -2183,9 +2305,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo ClientHasAnyCommits = resp.GetClientHasAnyCommits(); InitDone = true; PipeGeneration = 0; //reset tries counter - all ok - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " INIT DONE " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " << CommittedOffset); + YDB_LOG_INFO_CTX(ctx, "INIT DONE EndOffset readOffset committedOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}); if (!StartReading) { @@ -2207,9 +2333,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo const auto& res = result.GetCmdReadResult(); if (result.GetCookie() != (ui64)ReadOffset) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " unwaited read-response with cookie " << result.GetCookie() << "; waiting for " << ReadOffset << "; current read guid is " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "Partition unwaited read-response with cookie waiting for current read guid is", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); return; } @@ -2250,11 +2380,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo TString sourceId = ""; if (!r.GetSourceId().empty()) { if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "read bad sourceId from topic " << Topic->GetPrintableString() - << " partition:" << Partition - << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "' ReadGuid " << ReadGuid); + YDB_LOG_ERROR_CTX(ctx, "Read bad sourceId from topic offset seqNo sourceId ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"readGuid", ReadGuid}); } sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); } @@ -2311,13 +2444,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo if (proto.HasCodec()) { const auto codec = proto.GetCodec(); if (codec < Min<int>() || codec > Max<int>() || !NPersQueueCommon::ECodec_IsValid(codec)) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "data chunk (topic " << Topic->GetInternalName() << ", partition " << Partition - << ", offset " << r.GetOffset() << ", seqNo " << r.GetSeqNo() << ", sourceId " - << r.GetSourceId() << ") codec (id " << codec - << ") is not valid NPersQueueCommon::ECodec, loss of data compression codec information" - ); + YDB_LOG_ERROR_CTX(ctx, "Data chunk codec is not a valid NPersQueueCommon::ECodec; compression codec info may be lost", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetInternalName()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"codec", codec}); } message->SetCodec((NPersQueueCommon::ECodec)proto.GetCodec()); } @@ -2341,11 +2475,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo WaitDataInPartition(ctx); } - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " after read state " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "After read state EndOffset ReadOffset ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); ReadGuid = TString(); @@ -2364,10 +2500,12 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo void TPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { TEvTabletPipe::TEvClientConnected *msg = ev->Get(); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " pipe creation result: " << msg->Status); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt pipe creation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"result", msg->Status}); if (msg->Status != NKikimrProto::OK) { RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << msg->TabletId, NPersQueue::NErrorCode::ERROR); @@ -2381,12 +2519,13 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const void TPartitionActor::Handle(TEvPQProxy::TEvReleasePartition::TPtr&, const TActorContext& ctx) { - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " (partition)releasing " << Topic->GetPrintableString() << " partition:" << Partition - << " ReadOffset " << ReadOffset << " ClientCommitOffset " << ClientCommitOffset - << " CommittedOffst " << CommittedOffset - ); + YDB_LOG_INFO_CTX(ctx, "(partition)releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}); NeedRelease = true; CheckRelease(ctx); } @@ -2456,13 +2595,15 @@ void TPartitionActor::InitStartReading(const TActorContext& ctx) { Y_ABORT_UNLESS(AllPrepareInited); Y_ABORT_UNLESS(!WaitForData); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " Start reading " << Topic->GetPrintableString() << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " - << CommittedOffset << " clientCommittedOffset " << ClientCommitOffset - << " clientReadOffset " << ClientReadOffset - ); + YDB_LOG_INFO_CTX(ctx, "Start reading EndOffset readOffset committedOffset clientCommittedOffset clientReadOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"clientReadOffset", ClientReadOffset}); Counters.PartitionsToBeLocked.Dec(); LockCounted = false; @@ -2547,9 +2688,10 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) { cmd->SetGeneration(Generation); cmd->SetStep(Step); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " INITING " << Topic->GetPrintableString() << " partition:" << Partition); + YDB_LOG_INFO_CTX(ctx, "INITING", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); Y_ABORT_UNLESS(!RequestInfly); @@ -2588,11 +2730,11 @@ void TPartitionActor::WaitDataInPartition(const TActorContext& ctx) { event->Record.SetDeadline(deadline); event->Record.SetClientId(InternalClientId); - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait data in partition inited, cookie " << WaitDataCookie - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait data in partition inited", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"waitDataCookie", WaitDataCookie}); NTabletPipe::SendData(ctx, PipeClient, event.Release()); @@ -2609,10 +2751,11 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con auto it = WaitDataInfly.find(ev->Get()->Record.GetCookie()); if (it == WaitDataInfly.end()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " unwaited response for WaitData " << ev->Get()->Record); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response for WaitData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"record", ev->Get()->Record}); return; } WaitDataInfly.erase(it); @@ -2626,13 +2769,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con Y_ABORT_UNLESS(EndOffset <= record.GetEndOffset()); //end offset could not be changed if no data arrived, but signal will be sended anyway after timeout Y_ABORT_UNLESS(ReadOffset >= EndOffset); //otherwise no WaitData were needed - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait for data done: " << " readOffset " << ReadOffset << " EndOffset " << EndOffset - << " newEndOffset " << record.GetEndOffset() << " commitOffset " << CommittedOffset - << " clientCommitOffset " << ClientCommitOffset << " cookie " << ev->Get()->Record.GetCookie() - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait for data done: readOffset EndOffset newEndOffset commitOffset clientCommitOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"recordEndOffset", record.GetEndOffset()}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"cookie", ev->Get()->Record.GetCookie()}); EndOffset = record.GetEndOffset(); SizeLag = record.GetSizeLag(); @@ -2664,13 +2810,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " READ FROM " << Topic->GetPrintableString() << " partition:" << Partition - << " event " << ev->Get()->Request << " readOffset " << ReadOffset - << " EndOffset " << EndOffset << " ClientCommitOffset " << ClientCommitOffset - << " committedOffset " << CommittedOffset << " Guid " << ev->Get()->Guid - ); + YDB_LOG_DEBUG_CTX(ctx, "READ FROM event readOffset EndOffset ClientCommitOffset committedOffset Guid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"request", ev->Get()->Request}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}, + {"guid", ev->Get()->Guid}); Y_ABORT_UNLESS(!NeedRelease); Y_ABORT_UNLESS(!Released); @@ -2727,12 +2876,13 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex const ui64 offset = ev->Get()->Offset; Y_ABORT_UNLESS(offset != Max<ui64>()); // has concreete offset if (offset < ClientCommitOffset) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done to too small position " << offset - << " committedOffset " << ClientCommitOffset << " cookie " << readId - ); + YDB_LOG_ERROR_CTX(ctx, "Commit done to too small position committedOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"clientCommitOffset", ClientCommitOffset}, + {"readId", readId}); } Y_ABORT_UNLESS(offset >= ClientCommitOffset); @@ -2740,11 +2890,11 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex if (!hasProgress) {//nothing to commit for this partition if (CommitsInfly.empty()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done with no effect with cookie " << readId - ); + YDB_LOG_DEBUG_CTX(ctx, "Commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); CheckRelease(ctx); } else { diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp index d9a9abc1dcb..3a409920805 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp @@ -17,6 +17,8 @@ #include <util/string/vector.h> #include <util/string/escape.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -133,7 +135,9 @@ void TWriteSessionActor::Die(const TActorContext& ctx) { BytesInflightTotal.Dec(BytesInflightTotal_); } - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Session is DEAD", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); if (PartitionChooser) { ctx.Send(PartitionChooser, new TEvents::TEvPoison()); @@ -161,7 +165,9 @@ void TWriteSessionActor::CheckFinish(const TActorContext& ctx) { } void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " got TEvDone"); + YDB_LOG_INFO_CTX(ctx, "Session got TEvDone", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); WritesDone = true; CheckFinish(ctx); } @@ -231,7 +237,11 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor event->Request.ClearCredentials(); Y_PROTOBUF_SUPPRESS_NODISCARD Auth.SerializeToString(&AuthStr); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << ", Database: '" << Database << "' from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "Session request Database", + {"cookie", Cookie}, + {"init", init}, + {"database", Database}, + {"peerName", PeerName}); UserAgent = init.GetVersion(); LogSession(ctx); @@ -402,8 +412,10 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { //ACLCheckInProgress is still true - no recheck will be done - LOG_WARN_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session without AuthInfo : " << DiscoveryConverter->GetPrintableString() - << " sourceId " << SourceId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo sourceId", + {"discoveryConverter", DiscoveryConverter->GetPrintableString()}, + {"sourceId", SourceId}, + {"peerName", PeerName}); SessionsWithoutAuth.Inc(); if (FirstACLCheck) { FirstACLCheck = false; @@ -445,9 +457,10 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " - << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_INFO_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -459,7 +472,9 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check"); + YDB_LOG_INFO_CTX(ctx, "Session describe result for acl check", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); CheckACL(ctx); } @@ -548,13 +563,17 @@ void TWriteSessionActor::CloseSession(const TString& errorReason, const NPersQue error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, - "session error cookie: " << Cookie << " reason: \"" << errorReason << "\" code: " - << EErrorCode_Name(errorCode) << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session error reason", + {"cookie", Cookie}, + {"errorReason", errorReason}, + {"code", EErrorCode_Name(errorCode)}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(result)); } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session closed cookie: " << Cookie << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session closed", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); } Die(ctx); @@ -586,8 +605,11 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev Y_ABORT_UNLESS(FullConverter); init->SetTopic(FullConverter->GetClientsideName()); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition - << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session inited", + {"cookie", Cookie}, + {"partition", Partition}, + {"maxSeqNo", maxSeqNo}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(response)); @@ -886,9 +908,14 @@ void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const void TWriteSessionActor::LogSession(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie - << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v0 " - << " topic=" << DiscoveryConverter->GetPrintableString() << " durationSec=" << (ctx.Now() - StartTime).Seconds()); + YDB_LOG_INFO_CTX(ctx, "Write session", + {"cookie", Cookie}, + {"proto", "v0"}, + {"sessionId", OwnerCookie}, + {"userAgent", UserAgent}, + {"ip", PeerName}, + {"topic", DiscoveryConverter->GetPrintableString()}, + {"durationSec", (ctx.Now() - StartTime).Seconds()}); LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60); } diff --git a/ydb/tests/functional/config/test_distconf_self_heal.py b/ydb/tests/functional/config/test_distconf_self_heal.py index 91c9846795e..bd14fa0c77e 100644 --- a/ydb/tests/functional/config/test_distconf_self_heal.py +++ b/ydb/tests/functional/config/test_distconf_self_heal.py @@ -26,6 +26,28 @@ def get_ring_group(request_config, config_name): return config["Ring"] +def extract_node_ids(ring): + """Recursively collect all node ids referenced by a (possibly nested) TRing structure.""" + node_ids = set() + if "Node" in ring: + node_ids.update(ring["Node"]) + for nested in ring.get("Ring", []): + node_ids.update(extract_node_ids(nested)) + return node_ids + + +def extract_all_node_ids(config, config_name): + """Collect all node ids present in either the RingGroups or single Ring representation.""" + config = config[f"{config_name}Config"] + node_ids = set() + if "RingGroups" in config: + for rg in config["RingGroups"]: + node_ids.update(extract_node_ids(rg)) + elif "Ring" in config: + node_ids.update(extract_node_ids(config["Ring"])) + return node_ids + + class KiKiMRDistConfSelfHealTest(object): nodes_count = 9 erasure = Erasure.MIRROR_3_DC @@ -33,6 +55,11 @@ class KiKiMRDistConfSelfHealTest(object): separate_node_configs = True state_storage_rings = None n_to_select = None + # extra fields merged into the static self_management_config section of the initial + # (bootstrap) YAML config; NB: Cfg->SelfManagementConfig used by distconf self-heal is read + # once at node startup and is NOT affected by dynamic ReplaceConfig calls, so options like + # automatic_*_management / *_self_heal_allowed_nodes must be set here, before cluster start. + self_management_extra_options = {} metadata_section = { "kind": "MainConfig", "version": 0, @@ -59,6 +86,9 @@ class KiKiMRDistConfSelfHealTest(object): n_to_select=cls.n_to_select, state_storage_rings=cls.state_storage_rings) + if cls.self_management_extra_options: + cls.configurator.yaml_config["self_management_config"].update(cls.self_management_extra_options) + cls.cluster = KiKiMR(configurator=cls.configurator) cls.cluster.start() @@ -234,3 +264,73 @@ class TestKiKiMRDistConfSelfHealParallelCall2(KiKiMRDistConfSelfHealTest): rg = self.do_request_config()[f"{configName}Config"]["Ring"] assert_eq(rg["NToSelect"], 9) assert_eq(len(rg["Ring"]), 9) + + +class TestKiKiMRDistConfSelfHealAllowedNodes(KiKiMRDistConfSelfHealTest): + # 3 extra spare nodes (10, 11, 12) relative to the base 9-node MIRROR_3_DC topology so that + # forbidding one node still leaves enough nodes for a valid, fully-healthy configuration. + erasure = Erasure.MIRROR_3_DC + nodes_count = 12 + rgOffset = 1 + + forbidden_node_id = nodes_count + allowed_node_ids = sorted(set(range(1, nodes_count + 1)) - {forbidden_node_id}) + + self_management_extra_options = { + "automatic_state_storage_management": True, + "state_storage_self_heal_allowed_nodes": allowed_node_ids, + "automatic_state_storage_board_management": True, + "state_storage_board_self_heal_allowed_nodes": allowed_node_ids, + "automatic_scheme_board_management": True, + "scheme_board_self_heal_allowed_nodes": allowed_node_ids, + } + + def do_test(self, configName): + self.do_bad_config(configName) + + logger.info("Start SelfHeal with allowed nodes restriction (forbidding node %s) for %s", + self.forbidden_node_id, configName) + logger.info(self.do_request({"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}})) + time.sleep(10) + + healed_config = self.do_request_config() + node_ids = extract_all_node_ids(healed_config, configName) + + assert_that(len(node_ids) > 0, f"Healed {configName} config has no nodes: {healed_config}") + assert_that(self.forbidden_node_id not in node_ids, + f"Forbidden node {self.forbidden_node_id} present in healed {configName} config: {node_ids}") + assert_that(node_ids <= set(self.allowed_node_ids), + f"Healed {configName} config uses nodes outside the allowed list: {node_ids - set(self.allowed_node_ids)}") + + +class TestKiKiMRDistConfSelfHealAutomaticManagementDisabled(KiKiMRDistConfSelfHealTest): + erasure = Erasure.MIRROR_3_DC + nodes_count = 9 + rgOffset = 1 + + self_management_extra_options = { + "automatic_state_storage_management": False, + "automatic_state_storage_board_management": False, + "automatic_scheme_board_management": False, + } + + def check_failed(self, req, message): + resp = self.do_request(req) + assert_that(resp.get("ErrorReason", "").startswith(message), {"Response": resp, "Expected": message}) + + def do_test(self, configName): + # Manually push the config into a "bad"/suboptimal state (ReconfigStateStorage is a + # direct manual API and is unaffected by AutomaticManagement flags). + self.do_bad_config(configName) + bad_config = get_ring_group(self.do_request_config(), configName) + + logger.info("Attempting SelfHeal with automatic management disabled for %s", configName) + self.check_failed( + {"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}}, + "Current configuration is recommended. Nothing to self-heal.") + time.sleep(5) + + # The config must remain completely untouched: self-heal must not have modified it since + # automatic management for this subsystem is disabled. + unchanged_config = get_ring_group(self.do_request_config(), configName) + assert_eq(unchanged_config, bad_config) diff --git a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py index 66817523528..95985bb74e8 100644 --- a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py +++ b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py @@ -773,7 +773,7 @@ class ScenarioTestHelper: def get_volumes_columns(self, table_name: str, name_column: str) -> tuple[int, int]: path = table_name if table_name.startswith('/') else self.get_full_path(table_name) - query = f'''SELECT * FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' + query = f'''SELECT SUM(RawBytes) AS RawBytes, SUM(BlobRangeSize) AS BlobRangeSize FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' if (len(name_column)): query += f' AND EntityName = \"{name_column}\"' result_sets = self.execute_query(query) diff --git a/ydb/tools/stress_tool/README.md b/ydb/tools/stress_tool/README.md index e0608e7afab..d636240ee2b 100644 --- a/ydb/tools/stress_tool/README.md +++ b/ydb/tools/stress_tool/README.md @@ -84,10 +84,42 @@ Each `DDiskTestList` entry contains one or more `DDiskLoad` sources. These describe network load generated through the actors library interconnect subsystem (`ydb/library/actors/interconnect/load.h`), reusing the same `TInterconnectLoad` load actor used by the `InterconnectLoad` load type of the -regular YDB load actor service. Since the stress tool runs a single node, the -load actor and its counterpart (the load responder) both live in the same -process; traffic is routed through loopback using `NodeHops`, so no real -network connection is required to generate and measure interconnect traffic. +regular YDB load actor service. + +By default (no `--server`/`--client` options), the stress tool runs a single +node; the load actor and its counterpart (the load responder) both live in the +same process, and traffic is routed through loopback using `NodeHops`, so no +real network connection is required to generate and measure interconnect +traffic. + +To measure interconnect performance over a **real network** between two +hosts, run two copies of `ydb_stress_tool`: + +- On the **second** (remote) host, start a responder-only instance. Its + `--server NODE_ID` must match the NodeId that the client will assign this + server via `--endpoint` (see below) -- for a single server this is `1`: + ``` + ydb_stress_tool --cfg cfg.txt --server 1 --client 2 --ic-port 19001 + ``` + This registers the interconnect load responder for node 1 and listens on + port 19001 for an incoming connection from the client (node 2). No device + `--path` is required. + +- On the **first** host, run the actual `InterconnectTestList` from the config + as a client that connects out to the responder: + ``` + ydb_stress_tool --cfg cfg.txt --client 2 --endpoint remote-host:19001 + ``` + The client's own NodeId is `2` (from `--client`, must differ from all + server NodeIds); the server passed via `--endpoint` is assigned NodeId `1` + (the i-th `--endpoint` gets NodeId `i + 1`, matching the `--server NODE_ID` + used on the remote host). The config's `InterconnectLoad.NodeHops` must + reference this NodeId (e.g. `NodeHops: [1]`) so that load messages are + routed to the remote responder over the network instead of looping back + locally. + + Optional: `--use-uring` selects the io_uring transport instead of epoll for + the interconnect connection itself (Linux 5.19+). Each `InterconnectTestList` entry contains one or more `InterconnectLoad` sources, run sequentially. Detailed periodic throughput/RTT statistics are diff --git a/ydb/tools/stress_tool/device_test_tool.cpp b/ydb/tools/stress_tool/device_test_tool.cpp index 6362368ef29..53311967ed1 100644 --- a/ydb/tools/stress_tool/device_test_tool.cpp +++ b/ydb/tools/stress_tool/device_test_tool.cpp @@ -88,6 +88,15 @@ static void InstallServerSignalHandler() { signal(SIGTERM, ServerSignalHandler); } +static void InterconnectServerSignalHandler(int) { + NKikimr::InterconnectServerStopEvent.Signal(); +} + +static void InstallInterconnectServerSignalHandler() { + signal(SIGINT, InterconnectServerSignalHandler); + signal(SIGTERM, InterconnectServerSignalHandler); +} + static const char* ydb_logo = R"__( @@ -158,7 +167,9 @@ int main(int argc, char **argv) { // DDisk client/server options are hidden from the auto-generated --help list and // re-rendered below as a dedicated "DDisk client/server options" section. Keep the // section text in sync with the option definitions here. - opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has no DDiskTestList but has InterconnectTestList, runs as an interconnect load responder instead " + "(for testing interconnect over a real network between two ydb_stress_tool instances)") .RequiredArgument("NODE_ID").StoreResult(&serverNodeId).Hidden(); opts.AddLongOption("client", "client node ID; in server mode this is the expected client's ID (used in the nameserver), in client mode this is the client's own ID") .RequiredArgument("NODE_ID").StoreResult(&clientNodeId).Hidden(); @@ -166,7 +177,7 @@ int main(int argc, char **argv) { .RequiredArgument("HOST:PORT").AppendTo(&clientEndpoints).Hidden(); opts.AddLongOption("ic-port", "interconnect port for server to listen on (server only)") .RequiredArgument("PORT").StoreResult(&icPort).Hidden(); - opts.AddLongOption("num-server-devices", "number of devices per server (default 1)") + opts.AddLongOption("num-server-devices", "number of devices per server (default 1); ignored in interconnect server/client mode") .RequiredArgument("N").DefaultValue("1").Hidden(); bool useUring = false; opts.AddLongOption("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)") @@ -185,7 +196,9 @@ int main(int argc, char **argv) { TStringBuilder ddiskHelp; ddiskHelp << row("server NODE_ID", - "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has InterconnectTestList instead of DDiskTestList, runs as an interconnect load " + "responder for the network interconnect test (see InterconnectTestList in README)") << row("client NODE_ID", "client node ID; in server mode this is the expected client's ID (used in the nameserver), " "in client mode this is the client's own ID") @@ -194,11 +207,11 @@ int main(int argc, char **argv) { << row("ic-port PORT", "interconnect port for server to listen on (server only)") << row("num-server-devices N", - "number of devices per server (default 1)") + "number of devices per server (default 1); ignored for InterconnectTestList") << row("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)"); - opts.AddSection("DDisk client/server options", ddiskHelp); + opts.AddSection("DDisk / Interconnect client/server options", ddiskHelp); } TOptsParseResult res(&opts, argc, argv); @@ -206,6 +219,9 @@ int main(int argc, char **argv) { // Server mode is selected by --server. Client mode by --client without --server. // In server mode, --client is also required and provides the expected client's NodeID // so the interconnect handshake can resolve the peer in the static nameserver table. + // Both modes are shared between DDisk (real device paths, PDisk/DDisk actors) and + // Interconnect (no device paths, just a load responder / load actor) tests; which + // one is used is decided later based on the config file contents. const bool serverMode = res.Has("server"); const bool clientMode = !serverMode && res.Has("client"); if (serverMode) { @@ -221,10 +237,6 @@ int main(int argc, char **argv) { Cerr << "Error: --server and --client NODE_IDs must differ" << Endl; return 1; } - if (paths.empty()) { - Cerr << "Error: --server requires at least one --path" << Endl; - return 1; - } if (!icPort) { Cerr << "Error: --server requires --ic-port" << Endl; return 1; @@ -241,7 +253,7 @@ int main(int argc, char **argv) { } } - if (!clientMode && paths.empty()) { + if (!clientMode && !serverMode && paths.empty()) { Cerr << "Error: at least one --path must be specified" << Endl; return 1; } @@ -250,8 +262,11 @@ int main(int argc, char **argv) { Cout << ydb_logo << Flush; } - // For client mode, paths can be empty; use a dummy for TPerfTestConfig - if (clientMode && paths.empty()) { + const bool pathsProvided = !paths.empty(); + + // For client mode, and for server mode when running as an interconnect + // responder (no real devices), paths can be empty; use a dummy for TPerfTestConfig. + if ((clientMode || serverMode) && paths.empty()) { paths.push_back("unused"); } @@ -288,10 +303,77 @@ int main(int argc, char **argv) { } } - // Client-server mode dispatch + // Client-server mode dispatch. Two independent flavors share --server/--client/ + // --endpoint/--ic-port: DDisk (real device paths, PDisk/DDisk actors) is used when + // the config has DDiskTestList; Interconnect (no device paths, just a load + // responder / load actor talking over a real network connection) is used when the + // config has InterconnectTestList instead. This lets a second ydb_stress_tool + // instance on another host act as the interconnect load responder. if (serverMode || clientMode) { - if (protoTests.DDiskTestListSize() == 0) { - Cerr << "Error: --server/--client mode requires DDiskTestList in config" << Endl; + const bool hasDDiskTests = protoTests.DDiskTestListSize() > 0; + const bool hasInterconnectTests = protoTests.InterconnectTestListSize() > 0; + + if (!hasDDiskTests && !hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + if (hasDDiskTests && hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires exactly one of DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + + if (hasInterconnectTests) { + // Interconnect network test: no real device paths are used. + NDevicePerfTest::TInterconnectTest testProto = protoTests.GetInterconnectTestList(0); + + if (serverMode) { + InstallInterconnectServerSignalHandler(); + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + THolder<NKikimr::TPerfTest> test(new NKikimr::TInterconnectServer(config, serverNodeId, clientNodeId, icPort, useUring)); + test->SetPrinter(printer); + test->RunTest(); + return 0; + } + + if (clientMode) { + // Build server peer list. Server with index i (0-based) gets NodeId = i + 1. + // Config's InterconnectLoad.NodeHops entries should reference these NodeIds + // to route traffic to the corresponding remote server. + TVector<NKikimr::TInterconnectPeer> serverPeers; + serverPeers.reserve(clientEndpoints.size()); + for (size_t i = 0; i < clientEndpoints.size(); ++i) { + try { + auto [host, port] = ParseHostPort(clientEndpoints[i]); + serverPeers.push_back({static_cast<ui32>(i + 1), host, port}); + } catch (const yexception& ex) { + Cerr << "Error: --endpoint #" << (i + 1) << ": " << ex.what() << Endl; + return 1; + } + } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } + + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + for (ui32 run = 0; run < config.RunCount; ++run) { + THolder<NKikimr::TPerfTest> test( + new NKikimr::TInterconnectClient(config, testProto, clientNodeId, serverPeers, useUring)); + test->SetPrinter(printer); + test->RunTest(); + } + printer->EndTest(); + return 0; + } + } + + // DDisk test: real device paths (--path) are required. + if (serverMode && !pathsProvided) { + Cerr << "Error: --server requires at least one --path (DDisk mode)" << Endl; return 1; } NDevicePerfTest::TDDiskTest testProto = protoTests.GetDDiskTestList(0); @@ -320,6 +402,14 @@ int main(int argc, char **argv) { return 1; } } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } auto overrideDDiskInFlight = [](NDevicePerfTest::TDDiskTest& tp, ui32 inFlight) { for (size_t j = 0; j < tp.DDiskTestListSize(); ++j) { diff --git a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h index cdf4356b59e..57920621f43 100644 --- a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h +++ b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h @@ -30,11 +30,13 @@ #include <util/system/event.h> #include "device_test_tool.h" +#include "device_test_tool_ddisk_client_server.h" namespace NKikimr { static TSystemEvent InterconnectDoneEvent(TSystemEvent::rAuto); static TSystemEvent InterconnectResultsPrintedEvent(TSystemEvent::rAuto); +static TSystemEvent InterconnectServerStopEvent(TSystemEvent::rAuto); // Runs a sequence of interconnect load tests (NKikimr::TEvLoadTestRequest::TInterconnectLoad) // inside a single actor system, one after another, printing a short summary for each. @@ -258,7 +260,7 @@ struct TInterconnectTest : public TPerfTest { Printer->PrintResults(); InterconnectResultsPrintedEvent.Signal(); } - } catch (yexception ex) { + } catch (yexception& ex) { LastException = ex; IsLastExceptionSet = true; VERBOSE_COUT(ex.what()); @@ -282,4 +284,265 @@ struct TInterconnectTest : public TPerfTest { } }; +//////////////////////////////////////////////////////////////////////////////// +// Network (multi-host) interconnect test: a real TCP interconnect connection is +// established between two separate ydb_stress_tool processes running on +// different hosts. One side runs as a responder-only "server" (registers just +// the load responder actor and listens for incoming connections); the other +// side runs as a "client" that drives the InterconnectTestList load and +// connects out to the server via NodeHops pointing at the server's NodeId. +//////////////////////////////////////////////////////////////////////////////// + +// Server: registers the interconnect load responder actor for this node and +// listens for an incoming connection from the client. Does not run any load +// itself; it only replies to messages forwarded through TLoadResponderActor. +struct TInterconnectServer : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + volatile bool IsLastExceptionSet = false; + ui32 ServerNodeId; + ui32 ClientNodeId; + ui16 Port; + bool UseUring; + + TInterconnectServer(const TPerfTestConfig& cfg, ui32 serverNodeId, ui32 clientNodeId, ui16 port, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(serverNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , ServerNodeId(serverNodeId) + , ClientNodeId(clientNodeId) + , Port(port) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ServerNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 2, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 2, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Register the interconnect load responder actor for this node; it acts as + // the remote endpoint that the client's load actor routes messages to via + // NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ServerNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Server only accepts incoming connections, but the interconnect handshake + // still resolves the peer NodeId via the local nameserver, so register the + // client with a placeholder address (port 0; server never initiates a + // connection to the client). + auto common = MakeInterconnectCommon(Counters, ServerNodeId, UseUring); + TVector<TInterconnectPeer> peers = {{ClientNodeId, "::", 0}}; + SetupInterconnectServices(Setup.Get(), common, ServerNodeId, "::", Port, peers, /*listen=*/true); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ServerNodeId, "logger"), std::move(loggerActorCmd)); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect server init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + Cerr << "Interconnect load responder (node " << ServerNodeId << ", expecting client node " << ClientNodeId << ")" + << " is listening on port " << Port << ". Press Ctrl+C to stop." << Endl; + + InterconnectServerStopEvent.Wait(); + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + } + + void Finish() override { + } + + ~TInterconnectServer() override { + } +}; + +// Client: connects out to a set of remote interconnect servers and drives the +// InterconnectTestList load. The load actor's NodeHops must reference the +// server's NodeId (see ServerPeers) so that traffic is routed over the real +// network connection rather than looped back locally. +struct TInterconnectClient : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + yexception LastException; + volatile bool IsLastExceptionSet = false; + TActorId TestId; + NDevicePerfTest::TInterconnectTest TestProto; + ui32 ClientNodeId; + TVector<TInterconnectPeer> ServerPeers; + bool UseUring; + + TInterconnectClient(const TPerfTestConfig& cfg, const NDevicePerfTest::TInterconnectTest& testProto, + ui32 clientNodeId, const TVector<TInterconnectPeer>& serverPeers, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(clientNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , TestProto(testProto) + , ClientNodeId(clientNodeId) + , ServerPeers(serverPeers) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ClientNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 4, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 4, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Set up interconnect with all server peers + auto common = MakeInterconnectCommon(Counters, ClientNodeId, UseUring); + SetupInterconnectServices(Setup.Get(), common, ClientNodeId, "::", 0, ServerPeers, /*listen=*/false); + + // The load actor queries a local load responder for a shared traffic + // counter on Bootstrap (see NInterconnect::TLoadActor::Bootstrap), so a + // load responder must be registered on this node too, even though the + // actual load messages are routed to the remote server via NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ClientNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + // INTERCONNECT is too chatty at DEBUG/TRACE; floor it at INFO regardless of --log-level. + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ClientNodeId, "logger"), std::move(loggerActorCmd)); + + TestId = MakeBlobStorageProxyID(1); + TActorSetupCmd testSetup(new TInterconnectPerfTestActor(Cfg, TestProto, Printer, Counters), + TMailboxType::ReadAsFilled, 2); + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(TestId, std::move(testSetup))); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + + // Give interconnect some time to establish connections to the server(s) + Sleep(TDuration::Seconds(3)); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect client init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + try { + ActorSystem->Send(TestId, new TEvTablet::TEvBoot(MakeTabletID(0, 0, 1), 0, nullptr, TActorId(), nullptr)); + + for (ui32 i = 0; i < TestProto.InterconnectTestListSize(); ++i) { + InterconnectDoneEvent.Wait(); + Printer->PrintResults(); + InterconnectResultsPrintedEvent.Signal(); + } + } catch (yexception& ex) { + LastException = ex; + IsLastExceptionSet = true; + VERBOSE_COUT(ex.what()); + } + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + InterconnectDoneEvent.Reset(); + if (IsLastExceptionSet) { + IsLastExceptionSet = false; + ythrow LastException; + } + } + + void Finish() override { + } + + ~TInterconnectClient() override { + } +}; + } // namespace NKikimr |