diff options
707 files changed, 26813 insertions, 11846 deletions
diff --git a/.github/actions/test_cpp_sdk_with_coverage/action.yaml b/.github/actions/test_cpp_sdk_with_coverage/action.yaml index 6a3822f64b8..73aaf7f67a9 100644 --- a/.github/actions/test_cpp_sdk_with_coverage/action.yaml +++ b/.github/actions/test_cpp_sdk_with_coverage/action.yaml @@ -41,6 +41,7 @@ runs: --clang-coverage --coverage-report -DCOVERAGE_TARGET_REGEXP=ydb/public/sdk/cpp --coverage-prefix-filter=ydb/public/sdk/cpp + --coverage-exclude-regexp='/(ut|tests)/|_(ut|it)\.(c|cc|cpp|cxx|h|hh|hpp|hxx)' --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" --output "$out_dir" diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index f17c0a68923..cea861c52ea 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -1,4 +1,3 @@ -ydb/core/blobstorage/ddisk/ut TDDiskActorPDiskTest.PDiskRestartWithReservedChunks_DDiskZombie_Uring ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk ydb/core/blobstorage/pdisk/ut RDMA.TestChunkReadWithRdmaAllocatorEncryptedChunks ydb/core/blobstorage/pdisk/ut RDMA.TestChunkReadWithRdmaAllocatorPlainChunks @@ -25,21 +24,19 @@ ydb/core/kafka_proxy/ut KafkaProtocol.TopicsCompactionGapFromRetention ydb/core/kafka_proxy/ut unittest.sole chunk ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.CdcStreamMatches ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.ExternalTableMatches +ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedEdsPathMatches ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedTopicRemoteNameMatches ydb/core/kqp/ut/federated_query/datastreams unittest.[*/*] chunk ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.IndexesActualization-QueryService ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectDisabled ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectEnabled -ydb/core/kqp/ut/olap/storage KqpOlapDictionary.GroupBySomeDictionaryWithCompaction -ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable-ColumnStore ydb/core/kqp/ut/query KqpAnalyzeOperations.AnalyzeOperationsLifecycle ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableMultipleTime ydb/core/load_test/ut GroupWriteTest.SimpleRdma +ydb/core/mind/ut TDynamicNameserverTest.OnlyAliveDynamicNodesStaticNodeChangeSubscribers ydb/core/tx/columnshard/backup/iscan/ut unittest.sole chunk ydb/core/tx/schemeshard/ut_compaction_reboots SchemeshardForcedCompactionTestReboots.ForceCompactWithIndexCreation[TabletRebootsBucket0] -ydb/core/tx/schemeshard/ut_compaction_reboots SchemeshardForcedCompactionTestReboots.ForceCompactWithIndexCreation[TabletRebootsBucket1] ydb/core/tx/schemeshard/ut_compaction_reboots unittest.[*/*] chunk -ydb/core/tx/schemeshard/ut_subdomain TSchemeShardSubDomainTest.SimultaneousCreateTenantTable ydb/library/actors/core/ut TestThreadContextQueueTimestamps.CurrentQueueTimestamps ydb/library/actors/http/ut Http2Integration.Http2UpgradeFrom101 ydb/library/actors/interconnect/ut InterconnectDirectSession.UnregisterRestoresActorSystemDelivery @@ -47,17 +44,8 @@ ydb/public/sdk/cpp/tests/integration/topic gtest.[*/*] chunk ydb/public/sdk/cpp/tests/integration/topic/with_direct_read gtest.[*/*] chunk ydb/services/persqueue_v1/ut TPersQueueTest.TopicServiceCommitOffset ydb/services/persqueue_v1/ut unittest.[*/*] chunk -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] @@ -67,10 +55,8 @@ ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-3-1_to_prestable-26-4-False] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] -ydb/tests/fq/http_api test_http_api.py.TestHttpApi.test_restart_idempotency ydb/tests/fq/streaming py3test.[*/*] chunk ydb/tests/fq/streaming test_streaming.py.TestStreamingInYdb.test_read_topic_shared_reading_limit[False] ydb/tests/functional/blobstorage test_node_warden_cache.py.TestNodeWardenCacheFile.test_cache_file_lifecycle_and_restart @@ -79,6 +65,7 @@ ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_disk_creation ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_disk_creation_name_with_symbols ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_multiple_disks_creation ydb/tests/functional/query_cache/warmup test_warmup.py.TestCompileCacheViewPeerWarnings.test_peer_scan_warnings_increment_on_dead_peer +ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_healthcheck_authentication-with_schema_grants] ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[fifo] ydb/tests/functional/sqs/migration/compatibility/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters ydb/tests/functional/sqs/migration/finished/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters @@ -86,6 +73,7 @@ ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[kick_tablets-std] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-fifo] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-std] +ydb/tests/functional/tenants test_remove_storage_groups.py.test_remove_storage_group ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test @@ -96,7 +84,6 @@ ydb/tests/stress/topic_sqs/tests test_workload.py.TestYdbTopicSqsWorkload.test ydb/tools/mnc/viewer/ut py3test.sole chunk ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_operations_tab_shows_running_steps_and_final_output ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_operation_tab_navigation_matches_state_machine -ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_tab_navigation_matches_state_machine_for_core_command_sequences ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite ydb/tools/stress_tool/ut TDeviceTestTool.PersistentBufferTestWrite ydb/tools/stress_tool/ut unittest.sole chunk diff --git a/.github/config/muted_ya_asan.txt b/.github/config/muted_ya_asan.txt index ce0e5acad28..0caa100d347 100644 --- a/.github/config/muted_ya_asan.txt +++ b/.github/config/muted_ya_asan.txt @@ -49,6 +49,7 @@ ydb/core/blobstorage/ut_blobstorage/ut_replication unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOneWithReconnects ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk unittest.[*/*] chunk ydb/core/blobstorage/ut_vdisk unittest.[*/*] chunk +ydb/core/formats/arrow/accessor/sub_columns/ut SubColumnsSparsedBuilder.BinaryJsonMatchesPlain ydb/core/graph/shard/ut unittest.sole chunk ydb/core/http_proxy/ut TestMalformedRequest.CompressedDeflateContentLengthCorrect ydb/core/http_proxy/ut TestMalformedRequest.CompressedDeflateContentLengthHigher @@ -124,7 +125,6 @@ ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestChangeMessageVisibilit ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestChangeMessageVisibilityBatch ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestChangeMessageVisibilityBatch_TableImplementation ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestChangeMessageVisibility_TableImplementation -ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestCreateQueue ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestCreateQueueSetsContentBasedDeduplicationMessageRateLimit ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestCreateQueueSetsDefaultTopicMessageRateLimit ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestCreateQueueWithAllAttributes @@ -474,7 +474,6 @@ ydb/core/kqp/ut/olap KqpOlapOptimizer.TilingPlusPlusTtlDeletion ydb/core/kqp/ut/olap/types KqpOlapJson.BloomMixIndexesOldSyntaxVariants[false,true,1024,0,0,0.5,BLOOM_FILTER] ydb/core/kqp/ut/olap/types KqpOlapJson.QuotedFilterVariants[10,false,1,0,100,0.5] ydb/core/kqp/ut/olap/types KqpOlapJson.SimpleExistsVariants[10,false,0,10,100,0.5] -ydb/core/kqp/ut/query KqpLimits.ComputeNodeMemoryLimit ydb/core/kqp/ut/query KqpLimits.OutOfSpaceYQLUpsertFail ydb/core/kqp/ut/query unittest.[*/*] chunk ydb/core/kqp/ut/rbo KqpRboOlap.BlockChannelAuto @@ -486,7 +485,6 @@ ydb/core/kqp/ut/rbo KqpRboOlap.DisableBlocksOnColumnsLimit ydb/core/kqp/ut/rbo KqpRboOlap.DoubleOutOfRangeInJson ydb/core/kqp/ut/rbo KqpRboOlap.OlapFilterPeephole ydb/core/kqp/ut/rbo KqpRboOlap.PredicatePushdownCastErrors -ydb/core/kqp/ut/rbo KqpRboYql.ANALYZE_TPCН_11 ydb/core/kqp/ut/rbo KqpRboYql.AggregateKeyColumnsNotLeakedToInputMap ydb/core/kqp/ut/rbo KqpRboYql.Aggregation-ColumnStore ydb/core/kqp/ut/rbo KqpRboYql.AliasesRenames @@ -502,7 +500,7 @@ ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap-UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs-IsOlap+UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclInterruptInheritance ydb/core/kqp/ut/scheme KqpAcl.AclTemporaryInterruptInheritance-IsOlap -ydb/core/kqp/ut/scheme KqpConstraints.DropDefaultNotNullColumn_InsertOrReplace-IsInsert +ydb/core/kqp/ut/scheme KqpConstraints.CompileTimeDefaults_WithGlobalAsyncIndex-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.AlterTableCompactAccessDenied+UseQueryService ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableViaApiMultipleTime ydb/core/kqp/ut/scheme KqpScheme.ParallelDropStreamingQuery @@ -532,7 +530,6 @@ ydb/core/kqp/ut/tli KqpTli.SnapshotThenReadWrite ydb/core/kqp/ut/tli KqpTli.TwoVictimsOneBreaker ydb/core/kqp/ut/tli KqpTli.VictimReadThenWriteSameTable ydb/core/kqp/ut/tli KqpTli.VictimReadThenWriteSameTableMultiTable -ydb/core/kqp/ut/tx KqpSinkMvcc.OlapMultiSinks ydb/core/kqp/ut/tx KqpSnapshotIsolation.ConflictWrite+IsOlap+FillTables ydb/core/kqp/ut/tx unittest.[*/*] chunk ydb/core/kqp/ut/view TCreateAndDropViewTest.CallDropViewOnTable @@ -543,6 +540,7 @@ ydb/core/kqp/ut/view TCreateAndDropViewTest.DropViewIfExists ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidColumnRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidIndexRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidTableRef +ydb/core/kqp/ut/view TCreateAndDropViewTest.ParsingSecurityInvoker ydb/core/kqp/ut/view TSelectFromViewTest.LeftJoinViewsWithNoMatch ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher @@ -550,6 +548,7 @@ ydb/core/load_test/ut unittest.[*/*] chunk ydb/core/load_test/ut_ycsb ReadLoad.ShouldReadIterate ydb/core/mind/hive/ut THiveTestWithTenants.TestDrain ydb/core/mind/hive/ut unittest.[*/*] chunk +ydb/core/mind/ut TDynamicNameserverTest.OnlyAliveDynamicNodesStaticNodeChangeSubscribers ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TWriteRequestWithPBufferReplicationTest.ShouldWorkWithMultipleResponsesAndHedge ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut unittest.sole chunk @@ -572,7 +571,6 @@ ydb/core/persqueue/ut/ut_with_sdk CommitOffset.Commit_WithSession_ToPastParentPa ydb/core/quoter/ut QuoterWithKesusTest.AllocationStatistics ydb/core/statistics/service/ut ColumnStatistics.ManyColumns ydb/core/statistics/service/ut unittest.[*/*] chunk -ydb/core/tablet_flat/ut TIterator.GetKeyWithVersionSkips ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Local ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Remote ydb/core/transfer/ut/column_table unittest.sole chunk @@ -582,13 +580,11 @@ ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_LocalRead_TopicAutoPa ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_RowTable_TopicAutoPartitioning ydb/core/tx/columnshard/backup/async_jobs/ut unittest.sole chunk ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption -ydb/core/tx/columnshard/backup/iscan/ut unittest.sole chunk ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranuleStrKey_PKUtf8 ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranule_PKDatetime ydb/core/tx/columnshard/ut_rw unittest.[*/*] chunk ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.BuildToBuild_Ranges ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.EmptyEmbeddingSkipped -ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.MainToPostingWithKeyRange ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.BadRequest ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.BuildToBuild @@ -600,7 +596,7 @@ ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.DimensionMisma ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.EmptyEmbeddingSkipped ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.BuildTable+WithForeign -ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.MainTable +ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.DimensionMismatchError ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardValidateUniqueIndexScan.BadRequest @@ -613,6 +609,7 @@ ydb/core/tx/datashard/ut_stats DataShardStats.BackupTableStatsReportInterval ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-0-0 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-0-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-1-0 +ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-1-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-0-0 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-0-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-1-0 @@ -628,6 +625,7 @@ ydb/core/tx/schemeshard/ut_background_cleaning TSchemeshardBackgroundCleaningTes ydb/core/tx/schemeshard/ut_full_backup unittest.sole chunk ydb/core/tx/schemeshard/ut_login TSchemeShardLoginTest.CheckThatLockedOutParametersIsRestoredFromLocalDb ydb/core/tx/schemeshard/ut_metrics unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_olap TOlap.StoreStatsSmallBlobsVolumeQuota ydb/core/tx/schemeshard/ut_olap unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_restore TRestoreTests.ShouldHandleOverloadedShard ydb/core/tx/schemeshard/ut_serverless TSchemeShardServerLess.StorageBilling @@ -648,18 +646,15 @@ ydb/library/actors/interconnect/ut InterconnectDirectSession.ReceivesExpectedRep ydb/library/actors/interconnect/ut InterconnectDirectSession.UnregisterRestoresActorSystemDelivery ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster unittest.sole chunk -ydb/library/yql/providers/generic/actors/ut unittest.sole chunk ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.Producer_CloseTimeout ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.ReadWithRestartsAndLargeData ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.ReadWithRestartsAndLargeDataAndShuffle -ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.ReadWithoutConsumerWithRestarts ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.TypedProducer_BasicWrite ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut Describe.DescribePartitionPermissions ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut LocalPartition.DirectWriteWithoutDescribeResourcesPermission ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut LocalPartition.WithoutPartitionWithRestart ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopic_3_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopic_3_Table -ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopics_2_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopics_2_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.WriteToTopic_Demo_27_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut unittest.[*/*] chunk @@ -685,26 +680,15 @@ ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_UseUserPasswordWi ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_WithConnectionStringFakeTopicPath ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp+UseSchemeSecret ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp-UseSchemeSecret -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_prestable-26-3-row] ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-3-1_to_stable-26-3-1-row] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/common test_user_management.py.TestUserManagementRollingUpgrade.test_user_management[rolling_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] @@ -712,233 +696,14 @@ ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex. ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_prestable-26-4-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingMixedCluster.test_mixed_cluster[mixed_prestable-26-3_and_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-2-1_to_prestable-26-3-False] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-4_and_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-2-1_and_stable-26-1-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-3-1_and_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-1-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/datashard/truncate/concurrency py3test.[test_truncate_table_concurrency.py */*] chunk @@ -958,24 +723,14 @@ ydb/tests/fq/s3 test_streaming_join.py.TestStreamingJoin.test_grace_join[v1-clie ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[False] ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[True] ydb/tests/fq/streaming test_udfs.py.TestUdfsUsage.test_precompute_recovery[False] +ydb/tests/fq/streaming/streaming_large test_cluster_restarts.py.TestStreamingLarge.test_rolling_restart[kikimr0] ydb/tests/fq/streaming_optimize py3test.[test_sql_negative.py */*] chunk ydb/tests/fq/streaming_optimize py3test.[test_sql_streaming.py */*] chunk ydb/tests/fq/streaming_optimize py3test.sole chunk -ydb/tests/fq/streaming_optimize test_sql_negative.py.test[hopping_window-GroupByHoppingWindowPolicyBad-default.txt] ydb/tests/fq/streaming_optimize test_sql_negative.py.test[hopping_window-GroupByHoppingWindowTimeLimitBad-default.txt] ydb/tests/fq/streaming_optimize test_sql_negative.py.test[watermarks-bad_column-default.txt] -ydb/tests/fq/streaming_optimize test_sql_negative.py.test[watermarks-bad_pushdown-default.txt] -ydb/tests/fq/streaming_optimize test_sql_negative.py.test[watermarks-bad_type-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hop-GroupByHop-default.txt] ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hop-GroupByHopByStringKey-default.txt] ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hop-GroupByHopExprKey-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hop-GroupByHopPercentile-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hopping_window-GroupByHoppingWindowExprKey-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[hopping_window-GroupByHoppingWindowTimeExtractorUnusedColumns-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[pq-ReadTopic-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[pq-ReadTopicWithMetadataWithFilter-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[pq-ReadWriteTopic-default.txt] -ydb/tests/fq/streaming_optimize test_sql_streaming.py.test[watermarks-watermark_over_unused_column-default.txt] ydb/tests/fq/yds test_disposition.py.TestDisposition.test_disposition_fresh[v1-mvp_external_ydb_endpoint0] ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_3_sessions ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_start_new_query @@ -990,12 +745,15 @@ ydb/tests/fq/yt/kqp_yt_file/part0 test.py.test[aggregate-group_by_gs_and_having- ydb/tests/fq/yt/kqp_yt_file/part1 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_expr_mul_col-] ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_expr_with_join-] +ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_rollup_grouping_hum-] ydb/tests/fq/yt/kqp_yt_file/part10 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[action-eval_skip_take-] ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[aggregate-avg_and_sum_float-] +ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[aggregate-avg_with_having-default.txt] ydb/tests/fq/yt/kqp_yt_file/part11 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_expr_alias_on_subexp-] ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_expr_order_by_expr-] +ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_gs_grouping-] ydb/tests/fq/yt/kqp_yt_file/part12 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part12 test.py.test[action-eval_for_over_subquery-default.txt] ydb/tests/fq/yt/kqp_yt_file/part13 py3test.sole chunk @@ -1018,12 +776,14 @@ ydb/tests/fq/yt/kqp_yt_file/part19 test.py.test[aggregate-aggregate_with_const_y ydb/tests/fq/yt/kqp_yt_file/part2 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-compare_tuple-] ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-group_by_expr_semi_join-] +ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-group_by_gs_alt_duo-] ydb/tests/fq/yt/kqp_yt_file/part3 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part3 test.py.test[action-eval_column-] ydb/tests/fq/yt/kqp_yt_file/part3 test.py.test[aggregate-aggregate_distinct_expr_with_groupby_expr-default.txt] ydb/tests/fq/yt/kqp_yt_file/part4 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggregate_with_deep_aggregated_column-] ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggregate_with_lambda_inside_avg-] +ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggrs_no_grouping_via_map-default.txt] ydb/tests/fq/yt/kqp_yt_file/part5 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part5 test.py.test[action-action_nested_query-default.txt] ydb/tests/fq/yt/kqp_yt_file/part6 py3test.sole chunk @@ -1033,13 +793,14 @@ ydb/tests/fq/yt/kqp_yt_file/part7 test.py.test[action-eval_if-default.txt] ydb/tests/fq/yt/kqp_yt_file/part8 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-agg_filter_pushdown-] ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-aggrs_no_grouping-] +ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-group_by_cube_expr_trio-] ydb/tests/fq/yt/kqp_yt_file/part9 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-aggregate_distinct_expr_with_udf-] ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_expr-] +ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_rollup_aggr_expr-] ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc ydb/tests/functional/blobstorage test_node_warden_cache.py.TestNodeWardenCacheFile.test_cache_file_lifecycle_and_restart ydb/tests/functional/config test_distconf_sentinel_node_status.py.TestKiKiMRDistConfSelfHealOverrides.test_state_storage -ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_add ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx ydb/tests/functional/kqp/kqp_indexes unittest.sole chunk ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach @@ -1049,6 +810,7 @@ ydb/tests/functional/nbs test_nbs_load_actor.py.TestNbsLoadActor.test_nbs_load_a ydb/tests/functional/nbs test_nbs_load_actor.py.TestNbsLoadActor.test_nbs_load_actor_write ydb/tests/functional/query_cache/warmup test_warmup.py.TestCompileCacheViewPeerWarnings.test_peer_scan_warnings_increment_on_dead_peer ydb/tests/functional/query_cache/warmup test_warmup.py.TestWarmupRestart.test_warmup_basic +ydb/tests/functional/restarts test_restarts.py.TestRestartClusterMirror3DC.test_when_create_many_tablets_and_restart_cluster_then_every_thing_is_ok ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[enforce_user_token_disabled-no_schema_grants] ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[enforce_user_token_disabled-with_schema_grants] ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[enforce_user_token_enabled-no_schema_grants] @@ -1057,8 +819,6 @@ ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[requ ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_counters_authentication-with_schema_grants] ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_healthcheck_authentication-no_schema_grants] ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_healthcheck_authentication-with_schema_grants] -ydb/tests/functional/serverless test_serverless.py.test_create_table_with_alter_quotas[enable_alter_database_create_hive_first--false-enable_pool_encryption--false] -ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[fifo] ydb/tests/functional/sqs/migration/finished/cloud ydb.tests.functional.sqs.cloud.test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_yc_events_processor ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-fifo] ydb/tests/functional/sqs_topic test_send_message_batch.py.TestSqsTopicSendMessageBatch.test_send_message_batch_with_delay_seconds @@ -1068,8 +828,6 @@ ydb/tests/functional/statistics test_restarts.py.test_basic ydb/tests/functional/tpc/large py3test.[test_tpcds.py] chunk ydb/tests/functional/tpc/large py3test.[test_tpch_spilling.py] chunk ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[14] -ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[47] -ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[50] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[64] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[14] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[64] @@ -1227,15 +985,17 @@ ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_doc ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_list_with_path_filter ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_tool_schema_in_request ydb/tests/olap data_read_correctness.py.TestDataReadPerformanceNoIntersections.test +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_20_compression-algorithm=zstd,level=20] +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_8_compression-algorithm=zstd,level=8] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_to_compression[zstd_20_compression-algorithm=zstd,level=20] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_zstd_level_from_0_to_19_compression +ydb/tests/olap/column_compression/alter py3test.[*/*] chunk ydb/tests/olap/s3_import py3test.[*/*] chunk ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_drop_table_during_export ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_rename_table_during_export ydb/tests/olap/s3_import test_tpch_import.py.TestS3TpchImport.test_import_and_export ydb/tests/stress/kafka/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test -ydb/tests/stress/kv_volume_tool/tests py3test.sole chunk ydb/tests/stress/olap_workload/tests test_workload.py.TestYdbWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test @@ -1251,9 +1011,8 @@ ydb/tests/stress/topic_kafka/tests test_workload_topic.py.TestYdbTopicWorkload.t ydb/tests/stress/vector_workload/tests py3test.sole chunk ydb/tests/stress/vector_workload/tests test_workload.py.TestYdbVectorWorkload.test ydb/tools/mnc/viewer/ut py3test.sole chunk -ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_copy_cluster_config_creates_new_file_without_selecting_it ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_general_keeps_single_highlighted_card +ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_settings_save_deploy_flags_from_checkboxes ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_agents_tab_navigation_matches_state_machine ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_operation_tab_navigation_matches_state_machine ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_tab_navigation_matches_state_machine_for_core_command_sequences -ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite diff --git a/.github/config/muted_ya_msan.txt b/.github/config/muted_ya_msan.txt index 1bd95b8fc85..322615d4946 100644 --- a/.github/config/muted_ya_msan.txt +++ b/.github/config/muted_ya_msan.txt @@ -1,11 +1,10 @@ -ydb/core/blobstorage/backpressure/ut_client Backpressure.MonteCarlo -ydb/core/blobstorage/backpressure/ut_client unittest.sole chunk ydb/core/blobstorage/ddisk/ut TDDiskActorTest.LateOutdatedSyncReadResultDoesNotLogUnknownSync ydb/core/blobstorage/ut_blobstorage/ut_balancing VDiskBalancing.TestRandom_Block42 ydb/core/blobstorage/ut_blobstorage/ut_balancing VDiskBalancing.TestRandom_Mirror3dc ydb/core/blobstorage/ut_blobstorage/ut_balancing unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_bridge BridgeGet.PartRestorationAcrossBridgeOnRange ydb/core/blobstorage/ut_blobstorage/ut_bridge unittest.[*/*] chunk +ydb/core/blobstorage/ut_blobstorage/ut_ddisk DDisk.PersistentBufferFillAndRead ydb/core/blobstorage/ut_blobstorage/ut_ddisk unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_phantom_blobs PhantomBlobs.TestBlobSizeLimitAboveLimit ydb/core/blobstorage/ut_blobstorage/ut_phantom_blobs PhantomBlobs.TestBlobSizeLimitBelowLimit @@ -34,6 +33,7 @@ ydb/core/kqp/ut/service KqpQueryService.FlowControllOnHugeRealTable-LongRow ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher ydb/core/load_test/ut unittest.[*/*] chunk +ydb/core/mind/ut TDynamicNameserverTest.OnlyAliveDynamicNodesStaticNodeChangeSubscribers ydb/core/transfer/ut/functional Transfer.LocalTopic_BigMessage ydb/library/actors/interconnect/ut Interconnect.KernelLivenessReconnectLocalFallbackNotAppliedRdma ydb/library/actors/interconnect/ut Interconnect.RdmaSimultaneousReconnectDoesNotLoop @@ -41,66 +41,31 @@ ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialPa ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialRdmaRead ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialRdmaReadOnPeerSocketClose ydb/library/actors/interconnect/ut Interconnect.SetupRdmaSession -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/common test_user_management.py.TestUserManagementRollingUpgrade.test_user_management[rolling_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -113,30 +78,6 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -161,54 +102,24 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -221,30 +132,6 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -269,18 +156,13 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingMixedCluster.test_mixed_cluster[mixed_prestable-26-3_and_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc ydb/tests/functional/nbs py3test.sole chunk ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_500gb_disk_read_write -ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_multiple_disks_creation ydb/tests/functional/sqs/cloud test_common.py.TestCommonYandexWithPath.test_private_create_queue[tables_format_v0-fifo] ydb/tests/functional/sqs/cloud test_common.py.TestCommonYandexWithPath.test_private_create_queue[tables_format_v0-std] ydb/tests/functional/sqs/cloud test_common.py.TestCommonYandexWithPath.test_private_create_queue[tables_format_v1-fifo] diff --git a/.github/config/muted_ya_tsan.txt b/.github/config/muted_ya_tsan.txt index a6cc808299d..e2de00f2b7d 100644 --- a/.github/config/muted_ya_tsan.txt +++ b/.github/config/muted_ya_tsan.txt @@ -44,10 +44,7 @@ ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOne ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOneWithReconnects ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk unittest.[*/*] chunk -ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers VDiskStartupBrokers.LocalRecoveryBrokerReleasesTokenOnStartupFailurePerPDisk -ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers VDiskStartupBrokers.LocalRecoveryBrokerSerializesStartupPerNode ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers VDiskStartupBrokers.LocalRecoveryBrokerSerializesStartupPerPDisk -ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers VDiskStartupBrokers.StartupDataSyncBrokerDoesNotHangWhenQueuedOwnerDiesBeforeActivation ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers VDiskStartupBrokers.StartupDataSyncBrokerReleasesTokenWhenPeerIsOfflinePerNode ydb/core/blobstorage/ut_blobstorage/ut_startup_brokers unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_vdisk_internals ChunkKeeper.Random @@ -85,9 +82,7 @@ ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery22 ydb/core/kqp/executer_actor/ut unittest.sole chunk ydb/core/kqp/rm_service/ut KqpRm.ConcurrentChannels ydb/core/kqp/rm_service/ut KqpRm.ConcurrentTasks -ydb/core/kqp/ut/arrow unittest.[*/*] chunk ydb/core/kqp/ut/indexes/fulltext unittest.[*/*] chunk -ydb/core/kqp/ut/opt KqpNamedExpressions.NamedExpressionRandomUpsertRevert+UseDataQuery-EnableIndexStreamWrite ydb/core/kqp/ut/opt unittest.[*/*] chunk ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable-ColumnStore ydb/core/kqp/ut/query KqpAnalyzeOperations.AnalyzeOperationsLifecycle @@ -104,7 +99,6 @@ ydb/core/kqp/ut/runtime KqpScanLogs.WideCombine-EnabledLogs ydb/core/kqp/ut/runtime unittest.[*/*] chunk ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi-EnableIndexStreamWrite -ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexDeniesTableModificationsSql-EnableIndexStreamWrite ydb/core/kqp/ut/scheme unittest.[*/*] chunk @@ -128,10 +122,8 @@ ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.Shoul ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToTemporaryOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/vhost/ut TServerTest.ShouldCancelRequestsInFlightWhenStopEndpointOrStopServer -ydb/core/persqueue/pqtablet/partition/mlp/ut TMLPConsumerFIFOWithSplitDedup.Dedup_DoubleSplit_3min ydb/core/persqueue/pqtablet/partition/mlp/ut TMLPConsumerFIFOWithSplitDedup.Dedup_WithGroup_SameIds_6min ydb/core/persqueue/public/describer/ut TDescriberTests.TopicNotCanonizedPath -ydb/core/persqueue/public/describer/ut TDescriberTests.TopicWithoutDatabase ydb/core/persqueue/public/describer/ut unittest.sole chunk ydb/core/statistics/service/ut ColumnStatistics.ManyColumns ydb/core/statistics/service/ut unittest.[*/*] chunk @@ -172,81 +164,41 @@ ydb/core/transfer/ut/functional Transfer.WriteToNotExists ydb/core/transfer/ut/functional Transfer.WriteToNotTable ydb/core/transfer/ut/functional unittest.sole chunk ydb/core/transfer/ut/row_table Transfer_RowTable.ComplexKey -ydb/core/transfer/ut/row_table Transfer_RowTable.TableWithAsyncIndex ydb/core/transfer/ut/row_table unittest.sole chunk ydb/core/tx/columnshard/backup/iscan/ut IScan.MultiExport ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption ydb/core/tx/columnshard/backup/iscan/ut IScan.UploaderExport +ydb/core/tx/datashard/ut_truncate unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_set_column_constraint SetNotNullTest.CancelAtLockingStage ydb/core/tx/schemeshard/ut_set_column_constraint unittest.[*/*] chunk ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDomainAndTable-AlterDatabaseCreateHiveFirst-true ydb/core/tx/tx_proxy/ut_storage_tenant TStorageTenantTest.CreateDummyTabletsInDifferentDomains -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_prestable-26-3_to_stable-26-2-1-row] -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_prestable-26-3-row] -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_stable-26-1-1-row] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes py3test.[test_vector_index.py */*] chunk -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_vector_index.py.TestVectorIndex.test_vector_index[rolling_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/indexes test_vector_index.py.TestVectorIndex.test_vector_index[rolling_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -259,30 +211,6 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -307,54 +235,24 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -367,30 +265,6 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -415,24 +289,14 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-2-1_to_prestable-26-3-False] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-1-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/fq/http_api py3test.sole chunk @@ -457,6 +321,7 @@ ydb/tests/functional/dstool test_canonical_requests.py.Test.test_cluster_get_set ydb/tests/functional/dstool test_canonical_requests.py.Test.test_essential ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_add ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_take_snapshot +ydb/tests/functional/dstool test_canonical_requests.py.Test.test_infer_pdisk_slot_count ydb/tests/functional/dstool test_canonical_requests.py.Test.test_pdisk_set_status_inactive ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach @@ -465,6 +330,7 @@ ydb/tests/functional/nbs py3test.sole chunk ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_500gb_disk_read_write ydb/tests/functional/nbs test_nbs_load_actor.py.TestNbsLoadActor.test_nbs_load_actor_mixed ydb/tests/functional/nbs test_nbs_load_actor.py.TestNbsLoadActor.test_nbs_load_actor_write +ydb/tests/functional/nbs test_nbs_load_actor.py.TestNbsLoadActor.test_nbs_load_actor_write_then_read ydb/tests/functional/replication Replication.TopicAutopartitioning ydb/tests/functional/replication unittest.sole chunk ydb/tests/functional/serverless test_serverless.py.test_discovery_exclusive_nodes[enable_alter_database_create_hive_first--false-enable_pool_encryption--false] @@ -915,7 +781,6 @@ ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing. ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.NO] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.ERROR] -ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsv.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW10T4SdkCompaction.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW5T4LegacyCompaction.test @@ -931,7 +796,6 @@ ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[21] ydb/tests/functional/tpc/medium test_workload_manager.py.TestClickbenchWM.test ydb/tests/functional/tpc/medium test_workload_manager.py.TestClickbenchWMComputeSchedulerP1T1.test ydb/tests/functional/tpc/medium test_workload_manager.py.TestTpchWMS0_1.test -ydb/tests/functional/tpc/medium test_workload_manager.py.TestWMOltpAdHoc.test ydb/tests/functional/tpc/medium test_workload_manager.py.TestWMOltpTpchS0_1.test ydb/tests/functional/tpc/medium/tpch py3test.[test_decimal.py */*] chunk ydb/tests/functional/tpc/medium/tpch py3test.[test_duplicates.py */*] chunk @@ -1097,7 +961,6 @@ ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnComp ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_zstd_level_from_0_to_19_compression ydb/tests/olap/scenario py3test.[*/*] chunk ydb/tests/olap/scenario test_insert.py.TestInsert.test_multi[read_data_during_bulk_upsert] -ydb/tests/stress/backup/tests test_workload.py.TestYdbBackupWorkload.test ydb/tests/stress/compare_index_performance/tests test_compare.py.TestCompareIndexPerformance.test_vector ydb/tests/stress/kafka/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test @@ -1105,11 +968,11 @@ ydb/tests/stress/kv_volume_tool/tests py3test.sole chunk ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[128MB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[1GB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[256MB] +ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[512MB] +ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[column] ydb/tests/stress/mixedpy/tests py3test.sole chunk ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[column] ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[row] -ydb/tests/stress/nfs_backups/tests py3test.sole chunk -ydb/tests/stress/nfs_backups/tests test_workload.py.TestYdbNfsWorkload.test ydb/tests/stress/node_broker/tests py3test.sole chunk ydb/tests/stress/node_broker/tests test_workload.py.TestDeltaProtocol.test ydb/tests/stress/olap_workload/tests py3test.sole chunk @@ -1120,7 +983,6 @@ ydb/tests/stress/reconfig_state_storage_workload/tests test_scheme_board_workloa ydb/tests/stress/result_set_format/tests py3test.sole chunk ydb/tests/stress/result_set_format/tests test_arrow_workload.py.TestResultSetArrowWorkload.test ydb/tests/stress/result_set_format/tests test_value_workload.py.TestResultSetValueWorkload.test -ydb/tests/stress/scheme_board/pile_promotion/tests test_scheme_board_workload.py.TestSchemeBoard.test_scheme_board ydb/tests/stress/show_create/table/tests test_workload.py.TestYdbWorkload.test_show_create_table_workload[30-None] ydb/tests/stress/simple_queue/tests py3test.sole chunk ydb/tests/stress/simple_queue/tests test_workload.py.TestYdbWorkload.test[heavy-column] diff --git a/.github/workflows/cpp_sdk_peerdirs_check.yml b/.github/workflows/cpp_sdk_peerdirs_check.yml index ad124c0c8fc..d4000fb789a 100644 --- a/.github/workflows/cpp_sdk_peerdirs_check.yml +++ b/.github/workflows/cpp_sdk_peerdirs_check.yml @@ -3,18 +3,27 @@ name: Check CPP SDK PEERDIRs on: pull_request_target: branches: [main, stable-*] + types: [opened, synchronize, reopened, labeled] paths: ['ydb/public/sdk/cpp/**', '.github/workflows/cpp_sdk_peerdirs_check.yml'] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Irrelevant label events must not cancel a running check for the same PR. + group: >- + ${{ github.workflow }}-${{ + github.event_name == 'pull_request_target' + && format('pr-{0}{1}', github.event.pull_request.number, + github.event.action == 'labeled' && github.event.label.name != 'ok-to-test' && format('-ignore-{0}', github.run_id) || '') + || github.ref + }} cancel-in-progress: true jobs: check: if: >- github.event_name == 'workflow_dispatch' || - (vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).cpp_sdk_peerdirs_check == true) + (github.event.pull_request.state == 'open' && + (github.event.action != 'labeled' || github.event.label.name == 'ok-to-test')) runs-on: [self-hosted, tiny-worker] steps: - uses: actions/github-script@v8 diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index cdf6adbc47d..f478f7c836b 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -59,6 +59,7 @@ jobs: 'stable-26-1-1', 'stable-26-2', 'stable-26-2-1', + 'stable-26-1-1-enterprise', ]; let baseRef; if (context.eventName === 'workflow_dispatch') { diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index d5e5ca549af..a910f8bab92 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -1,7 +1,7 @@ name: Nightly-Build # workflow used to upload built binaries to s3 on: schedule: - - cron: "0 3 * * *" # At 03:00 every day + - cron: "45 22 * * *" # At 22:45 every day workflow_dispatch: inputs: use_default_branches: diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index a2d4fc87259..0235b0a2b3a 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -248,6 +248,9 @@ jobs: with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 + # Fork code is checked out only after the check-running-allowed gate + # (collaborator/owner or a maintainer-applied `ok-to-test` label) + allow-unsafe-pr-checkout: true - name: Setup ydb access uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials with: diff --git a/.github/workflows/regression_run_compatibility.yml b/.github/workflows/regression_run_compatibility.yml index 393f5226376..0f702ee423e 100644 --- a/.github/workflows/regression_run_compatibility.yml +++ b/.github/workflows/regression_run_compatibility.yml @@ -7,7 +7,7 @@ run-name: >- on: schedule: - - cron: "0 23 * * *" # At 23:00 every day + - cron: "0 2 * * *" # At 02:00 every day workflow_dispatch: inputs: pull_request_input: diff --git a/build/conf/ts/ts_check.conf b/build/conf/ts/ts_check.conf index 366665f9bf3..430c9b57572 100644 --- a/build/conf/ts/ts_check.conf +++ b/build/conf/ts/ts_check.conf @@ -5,8 +5,9 @@ macro _TS_CHECK(SCRIPT_NAME, IS_MEDIUM, CHECK_TYPE) { SET_APPEND(_TS_CHECK_LIST ${_TS_CHECK_SEPARATOR} ${SCRIPT_NAME} ${IS_MEDIUM} ${CHECK_TYPE}) } +# todo: FBP-3291: когда-нибудь прибраться и убрать тернарник, оставить медиум для всех проверок macro TS_LINT(SCRIPT_NAME, TIMEOUT_MEDIUM?"yes":"no") { - _TS_CHECK($SCRIPT_NAME $TIMEOUT_MEDIUM lint) + _TS_CHECK($SCRIPT_NAME yes lint) } macro TS_TEST(SCRIPT_NAME) { diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 308f8343545..75dcc48d3b9 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:12759441213" + "uri": "sbr:12836461311" }, "darwin-arm64": { - "uri": "sbr:12759440320" + "uri": "sbr:12836459950" }, "linux": { - "uri": "sbr:12759442670" + "uri": "sbr:12836463794" }, "linux-aarch64": { - "uri": "sbr:12759439073" + "uri": "sbr:12836458633" }, "win32": { - "uri": "sbr:12759441947" + "uri": "sbr:12836462531" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 3366243d6dc..cde2455e67d 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:12759447218" + "uri": "sbr:12836456989" }, "darwin-arm64": { - "uri": "sbr:12759446141" + "uri": "sbr:12836455885" }, "linux": { - "uri": "sbr:12759448668" + "uri": "sbr:12836459916" }, "linux-aarch64": { - "uri": "sbr:12759445467" + "uri": "sbr:12836454677" }, "win32": { - "uri": "sbr:12759447921" + "uri": "sbr:12836458452" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index aa1d8378f29..685cb5b6154 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -623,6 +623,7 @@ "12765030473": "{registry_endpoint}/12765030473", "12778660629": "{registry_endpoint}/12778660629", "12807488115": "{registry_endpoint}/12807488115", + "12845364131": "{registry_endpoint}/12845364131", "5486713852": "{registry_endpoint}/5486713852", "5514352253": "{registry_endpoint}/5514352253", "5523579199": "{registry_endpoint}/5523579199", @@ -1021,6 +1022,7 @@ "12592467517": "{registry_endpoint}/12592467517", "12675295547": "{registry_endpoint}/12675295547", "12759441213": "{registry_endpoint}/12759441213", + "12836461311": "{registry_endpoint}/12836461311", "5766172292": "{registry_endpoint}/5766172292", "5805431504": "{registry_endpoint}/5805431504", "5829027626": "{registry_endpoint}/5829027626", @@ -1165,6 +1167,7 @@ "12592466452": "{registry_endpoint}/12592466452", "12675293631": "{registry_endpoint}/12675293631", "12759440320": "{registry_endpoint}/12759440320", + "12836459950": "{registry_endpoint}/12836459950", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -1309,6 +1312,7 @@ "12592469929": "{registry_endpoint}/12592469929", "12675298739": "{registry_endpoint}/12675298739", "12759442670": "{registry_endpoint}/12759442670", + "12836463794": "{registry_endpoint}/12836463794", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -1453,6 +1457,7 @@ "12592464955": "{registry_endpoint}/12592464955", "12675292272": "{registry_endpoint}/12675292272", "12759439073": "{registry_endpoint}/12759439073", + "12836458633": "{registry_endpoint}/12836458633", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -1597,6 +1602,7 @@ "12592468857": "{registry_endpoint}/12592468857", "12675297027": "{registry_endpoint}/12675297027", "12759441947": "{registry_endpoint}/12759441947", + "12836462531": "{registry_endpoint}/12836462531", "8270821739": "{registry_endpoint}/8270821739", "8295446553": "{registry_endpoint}/8295446553", "8326170338": "{registry_endpoint}/8326170338", @@ -3051,6 +3057,7 @@ "12765030473": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "12778660629": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "12807488115": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "12845364131": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486713852": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5514352253": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5523579199": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", @@ -3449,6 +3456,7 @@ "12592467517": "devtools/ymake/bin/ymake for darwin", "12675295547": "devtools/ymake/bin/ymake for darwin", "12759441213": "devtools/ymake/bin/ymake for darwin", + "12836461311": "devtools/ymake/bin/ymake for darwin", "5766172292": "devtools/ymake/bin/ymake for darwin", "5805431504": "devtools/ymake/bin/ymake for darwin", "5829027626": "devtools/ymake/bin/ymake for darwin", @@ -3593,6 +3601,7 @@ "12592466452": "devtools/ymake/bin/ymake for darwin-arm64", "12675293631": "devtools/ymake/bin/ymake for darwin-arm64", "12759440320": "devtools/ymake/bin/ymake for darwin-arm64", + "12836459950": "devtools/ymake/bin/ymake for darwin-arm64", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -3737,6 +3746,7 @@ "12592469929": "devtools/ymake/bin/ymake for linux", "12675298739": "devtools/ymake/bin/ymake for linux", "12759442670": "devtools/ymake/bin/ymake for linux", + "12836463794": "devtools/ymake/bin/ymake for linux", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -3881,6 +3891,7 @@ "12592464955": "devtools/ymake/bin/ymake for linux-aarch64", "12675292272": "devtools/ymake/bin/ymake for linux-aarch64", "12759439073": "devtools/ymake/bin/ymake for linux-aarch64", + "12836458633": "devtools/ymake/bin/ymake for linux-aarch64", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -4025,6 +4036,7 @@ "12592468857": "devtools/ymake/bin/ymake for win32", "12675297027": "devtools/ymake/bin/ymake for win32", "12759441947": "devtools/ymake/bin/ymake for win32", + "12836462531": "devtools/ymake/bin/ymake for win32", "8270821739": "devtools/ymake/bin/ymake for win32", "8295446553": "devtools/ymake/bin/ymake for win32", "8326170338": "devtools/ymake/bin/ymake for win32", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 45904018c0f..691b5b613d5 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807476180) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845357942) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807473502) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845356994) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807481043) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845359902) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807470623) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845355796) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807478544) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845358723) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index f6061502b1c..5411b2969fb 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807484254) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845362044) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807482262) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845360921) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807488115) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845364131) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807480382) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845360238) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807486192) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845363322) ENDIF() diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 48d3d1f862e..1b0f8d6c02d 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -14,7 +14,7 @@ GO_FAKEID=2025-11-05 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 IWYU_FAKEID=2025-01-08 -CYTHON_FAKEID=2026-06-01 +CYTHON_FAKEID=2026-07-18 JAVA_FAKEID=13086370 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 diff --git a/contrib/libs/backtrace/dwarf.c b/contrib/libs/backtrace/dwarf.c index 58f6e4e1e2b..7a99eabb7c9 100644 --- a/contrib/libs/backtrace/dwarf.c +++ b/contrib/libs/backtrace/dwarf.c @@ -2026,6 +2026,113 @@ read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset, return 0; } +/* A cache of abbreviation tables, keyed by their offset in the + .debug_abbrev section. Compilation units may share abbrev tables: + tools that rewrite debug info (e.g. dwz, llvm-bolt + --update-debug-sections) deduplicate identical tables, which can + leave many thousands of units referring to one table. Without a + cache we would parse and allocate that table anew for every unit, + multiplying memory usage by orders of magnitude. The cache owns + the parsed abbrevs; units hold shallow copies. */ + +struct abbrevs_cache_entry +{ + /* The offset of the table in the .debug_abbrev section. */ + uint64_t abbrev_offset; + /* The parsed table. */ + struct abbrevs abbrevs; +}; + +/* A growable vector of abbrevs_cache_entry, sorted by + abbrev_offset. */ + +struct abbrevs_cache +{ + /* Memory for the entries. */ + struct backtrace_vector vec; + /* The number of entries. */ + size_t count; +}; + +/* Read the abbreviations at ABBREV_OFFSET, using CACHE. On a cache + miss read them with read_abbrevs and add them to the cache. The + abbrevs stored in *ABBREVS are owned by the cache and may be shared + with other units; the caller must not free them. They stay valid + until free_abbrevs_cache. Returns 1 on success, 0 on failure. */ + +static int +read_abbrevs_cached (struct backtrace_state *state, uint64_t abbrev_offset, + const unsigned char *dwarf_abbrev, + size_t dwarf_abbrev_size, int is_bigendian, + backtrace_error_callback error_callback, void *data, + struct abbrevs_cache *cache, struct abbrevs *abbrevs) +{ + struct abbrevs_cache_entry *entries; + size_t lo; + size_t hi; + struct abbrevs parsed; + + /* Binary search for ABBREV_OFFSET; if it is not present, LO is the + insertion point. */ + entries = (struct abbrevs_cache_entry *) cache->vec.base; + lo = 0; + hi = cache->count; + while (lo < hi) + { + size_t mid; + + mid = lo + (hi - lo) / 2; + if (entries[mid].abbrev_offset < abbrev_offset) + lo = mid + 1; + else if (entries[mid].abbrev_offset > abbrev_offset) + hi = mid; + else + { + *abbrevs = entries[mid].abbrevs; + return 1; + } + } + + if (!read_abbrevs (state, abbrev_offset, dwarf_abbrev, dwarf_abbrev_size, + is_bigendian, error_callback, data, &parsed)) + return 0; + + if (backtrace_vector_grow (state, sizeof (struct abbrevs_cache_entry), + error_callback, data, &cache->vec) == NULL) + { + free_abbrevs (state, &parsed, error_callback, data); + return 0; + } + + /* Growing the vector may have moved the entries. */ + entries = (struct abbrevs_cache_entry *) cache->vec.base; + memmove (entries + lo + 1, entries + lo, + (cache->count - lo) * sizeof (struct abbrevs_cache_entry)); + entries[lo].abbrev_offset = abbrev_offset; + entries[lo].abbrevs = parsed; + ++cache->count; + + *abbrevs = parsed; + return 1; +} + +/* Free the abbrevs held by CACHE. */ + +static void +free_abbrevs_cache (struct backtrace_state *state, + struct abbrevs_cache *cache, + backtrace_error_callback error_callback, void *data) +{ + struct abbrevs_cache_entry *entries; + size_t i; + + entries = (struct abbrevs_cache_entry *) cache->vec.base; + for (i = 0; i < cache->count; i++) + free_abbrevs (state, &entries[i].abbrevs, error_callback, data); + backtrace_vector_free (state, &cache->vec, error_callback, data); + cache->count = 0; +} + /* Return the abbrev information for an abbrev code. */ static const struct abbrev * @@ -2645,6 +2752,7 @@ build_address_map (struct backtrace_state *state, struct unit **pu; size_t unit_offset = 0; struct unit_addrs *pa; + struct abbrevs_cache abbrevs_cache; memset (&addrs->vec, 0, sizeof addrs->vec); memset (&unit_vec->vec, 0, sizeof unit_vec->vec); @@ -2666,6 +2774,7 @@ build_address_map (struct backtrace_state *state, memset (&units, 0, sizeof units); units_count = 0; + memset (&abbrevs_cache, 0, sizeof abbrevs_cache); while (info.left > 0) { @@ -2732,10 +2841,11 @@ build_address_map (struct backtrace_state *state, memset (&u->abbrevs, 0, sizeof u->abbrevs); abbrev_offset = read_offset (&unit_buf, is_dwarf64); - if (!read_abbrevs (state, abbrev_offset, - dwarf_sections->data[DEBUG_ABBREV], - dwarf_sections->size[DEBUG_ABBREV], - is_bigendian, error_callback, data, &u->abbrevs)) + if (!read_abbrevs_cached (state, abbrev_offset, + dwarf_sections->data[DEBUG_ABBREV], + dwarf_sections->size[DEBUG_ABBREV], + is_bigendian, error_callback, data, + &abbrevs_cache, &u->abbrevs)) goto fail; if (version < 5) @@ -2799,19 +2909,22 @@ build_address_map (struct backtrace_state *state, pa->high = pa->low; pa->u = NULL; + /* The units hold shallow copies of the cached abbrevs; the abbrevs + themselves are intentionally never freed after this point. + Release just the cache bookkeeping vector. */ + backtrace_vector_free (state, &abbrevs_cache.vec, error_callback, data); + unit_vec->vec = units; unit_vec->count = units_count; return 1; fail: + free_abbrevs_cache (state, &abbrevs_cache, error_callback, data); if (units_count > 0) { pu = (struct unit **) units.base; for (i = 0; i < units_count; i++) - { - free_abbrevs (state, &pu[i]->abbrevs, error_callback, data); - backtrace_free (state, pu[i], sizeof **pu, error_callback, data); - } + backtrace_free (state, pu[i], sizeof **pu, error_callback, data); backtrace_vector_free (state, &units, error_callback, data); } if (addrs->count > 0) diff --git a/contrib/libs/backtrace/patches/abbrev-cache.patch b/contrib/libs/backtrace/patches/abbrev-cache.patch new file mode 100644 index 00000000000..2978cbdfc16 --- /dev/null +++ b/contrib/libs/backtrace/patches/abbrev-cache.patch @@ -0,0 +1,175 @@ +--- a/dwarf.c ++++ b/dwarf.c +@@ -2026,6 +2026,113 @@ read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset, + return 0; + } + ++/* A cache of abbreviation tables, keyed by their offset in the ++ .debug_abbrev section. Compilation units may share abbrev tables: ++ tools that rewrite debug info (e.g. dwz, llvm-bolt ++ --update-debug-sections) deduplicate identical tables, which can ++ leave many thousands of units referring to one table. Without a ++ cache we would parse and allocate that table anew for every unit, ++ multiplying memory usage by orders of magnitude. The cache owns ++ the parsed abbrevs; units hold shallow copies. */ ++ ++struct abbrevs_cache_entry ++{ ++ /* The offset of the table in the .debug_abbrev section. */ ++ uint64_t abbrev_offset; ++ /* The parsed table. */ ++ struct abbrevs abbrevs; ++}; ++ ++/* A growable vector of abbrevs_cache_entry, sorted by ++ abbrev_offset. */ ++ ++struct abbrevs_cache ++{ ++ /* Memory for the entries. */ ++ struct backtrace_vector vec; ++ /* The number of entries. */ ++ size_t count; ++}; ++ ++/* Read the abbreviations at ABBREV_OFFSET, using CACHE. On a cache ++ miss read them with read_abbrevs and add them to the cache. The ++ abbrevs stored in *ABBREVS are owned by the cache and may be shared ++ with other units; the caller must not free them. They stay valid ++ until free_abbrevs_cache. Returns 1 on success, 0 on failure. */ ++ ++static int ++read_abbrevs_cached (struct backtrace_state *state, uint64_t abbrev_offset, ++ const unsigned char *dwarf_abbrev, ++ size_t dwarf_abbrev_size, int is_bigendian, ++ backtrace_error_callback error_callback, void *data, ++ struct abbrevs_cache *cache, struct abbrevs *abbrevs) ++{ ++ struct abbrevs_cache_entry *entries; ++ size_t lo; ++ size_t hi; ++ struct abbrevs parsed; ++ ++ /* Binary search for ABBREV_OFFSET; if it is not present, LO is the ++ insertion point. */ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ lo = 0; ++ hi = cache->count; ++ while (lo < hi) ++ { ++ size_t mid; ++ ++ mid = lo + (hi - lo) / 2; ++ if (entries[mid].abbrev_offset < abbrev_offset) ++ lo = mid + 1; ++ else if (entries[mid].abbrev_offset > abbrev_offset) ++ hi = mid; ++ else ++ { ++ *abbrevs = entries[mid].abbrevs; ++ return 1; ++ } ++ } ++ ++ if (!read_abbrevs (state, abbrev_offset, dwarf_abbrev, dwarf_abbrev_size, ++ is_bigendian, error_callback, data, &parsed)) ++ return 0; ++ ++ if (backtrace_vector_grow (state, sizeof (struct abbrevs_cache_entry), ++ error_callback, data, &cache->vec) == NULL) ++ { ++ free_abbrevs (state, &parsed, error_callback, data); ++ return 0; ++ } ++ ++ /* Growing the vector may have moved the entries. */ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ memmove (entries + lo + 1, entries + lo, ++ (cache->count - lo) * sizeof (struct abbrevs_cache_entry)); ++ entries[lo].abbrev_offset = abbrev_offset; ++ entries[lo].abbrevs = parsed; ++ ++cache->count; ++ ++ *abbrevs = parsed; ++ return 1; ++} ++ ++/* Free the abbrevs held by CACHE. */ ++ ++static void ++free_abbrevs_cache (struct backtrace_state *state, ++ struct abbrevs_cache *cache, ++ backtrace_error_callback error_callback, void *data) ++{ ++ struct abbrevs_cache_entry *entries; ++ size_t i; ++ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ for (i = 0; i < cache->count; i++) ++ free_abbrevs (state, &entries[i].abbrevs, error_callback, data); ++ backtrace_vector_free (state, &cache->vec, error_callback, data); ++ cache->count = 0; ++} ++ + /* Return the abbrev information for an abbrev code. */ + + static const struct abbrev * +@@ -2645,6 +2752,7 @@ build_address_map (struct backtrace_state *state, + struct unit **pu; + size_t unit_offset = 0; + struct unit_addrs *pa; ++ struct abbrevs_cache abbrevs_cache; + + memset (&addrs->vec, 0, sizeof addrs->vec); + memset (&unit_vec->vec, 0, sizeof unit_vec->vec); +@@ -2666,6 +2774,7 @@ build_address_map (struct backtrace_state *state, + + memset (&units, 0, sizeof units); + units_count = 0; ++ memset (&abbrevs_cache, 0, sizeof abbrevs_cache); + + while (info.left > 0) + { +@@ -2732,10 +2841,11 @@ build_address_map (struct backtrace_state *state, + + memset (&u->abbrevs, 0, sizeof u->abbrevs); + abbrev_offset = read_offset (&unit_buf, is_dwarf64); +- if (!read_abbrevs (state, abbrev_offset, +- dwarf_sections->data[DEBUG_ABBREV], +- dwarf_sections->size[DEBUG_ABBREV], +- is_bigendian, error_callback, data, &u->abbrevs)) ++ if (!read_abbrevs_cached (state, abbrev_offset, ++ dwarf_sections->data[DEBUG_ABBREV], ++ dwarf_sections->size[DEBUG_ABBREV], ++ is_bigendian, error_callback, data, ++ &abbrevs_cache, &u->abbrevs)) + goto fail; + + if (version < 5) +@@ -2799,19 +2909,22 @@ build_address_map (struct backtrace_state *state, + pa->high = pa->low; + pa->u = NULL; + ++ /* The units hold shallow copies of the cached abbrevs; the abbrevs ++ themselves are intentionally never freed after this point. ++ Release just the cache bookkeeping vector. */ ++ backtrace_vector_free (state, &abbrevs_cache.vec, error_callback, data); ++ + unit_vec->vec = units; + unit_vec->count = units_count; + return 1; + + fail: ++ free_abbrevs_cache (state, &abbrevs_cache, error_callback, data); + if (units_count > 0) + { + pu = (struct unit **) units.base; + for (i = 0; i < units_count; i++) +- { +- free_abbrevs (state, &pu[i]->abbrevs, error_callback, data); +- backtrace_free (state, pu[i], sizeof **pu, error_callback, data); +- } ++ backtrace_free (state, pu[i], sizeof **pu, error_callback, data); + backtrace_vector_free (state, &units, error_callback, data); + } + if (addrs->count > 0) diff --git a/contrib/libs/expat/.yandex_meta/__init__.py b/contrib/libs/expat/.yandex_meta/__init__.py index 8344f573151..372eabf2f4f 100644 --- a/contrib/libs/expat/.yandex_meta/__init__.py +++ b/contrib/libs/expat/.yandex_meta/__init__.py @@ -42,7 +42,10 @@ def post_build(self): ), ), ) - expat.PEERDIR.add("contrib/libs/libc_compat") + expat.PEERDIR |= { + "contrib/libs/libc_compat", + "library/cpp/sanitizer/include", + } expat = GNUMakeNixProject( diff --git a/contrib/libs/expat/.yandex_meta/devtools.copyrights.report b/contrib/libs/expat/.yandex_meta/devtools.copyrights.report index 072776c75e9..b3280199fc1 100644 --- a/contrib/libs/expat/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/expat/.yandex_meta/devtools.copyrights.report @@ -48,7 +48,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 12db9abedf3e0b98a87d6f4ec1377a19 BELONGS ya.make @@ -61,9 +61,9 @@ BELONGS ya.make expat.h [9:23] expat_external.h [9:19] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 16cc365f87360f9e773cd8812039915f BELONGS ya.make @@ -73,8 +73,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 17566be0ee85deadbd5b2fcedc8b66a9 BELONGS ya.make @@ -84,7 +84,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 1916cbefc2e0a780a3d503ba26f3780a BELONGS ya.make @@ -107,7 +107,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 2309e9d42c2779decb776323d1f5dabd BELONGS ya.make @@ -117,8 +117,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 262c58e3a627f5cee77a882379e1364f BELONGS ya.make @@ -139,7 +139,19 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] + +KEEP COPYRIGHT_SERVICE_LABEL 2d8e554003b116aae13ce1d358a8a9b9 +BELONGS ya.make + License text: + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xcsinc.c [9:10] KEEP COPYRIGHT_SERVICE_LABEL 338b8ad8ee9b8449a90a88a0559aefd9 BELONGS ya.make @@ -149,7 +161,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 347b7c5a6d0cfaa5712b5fb2bfd1a69a BELONGS ya.make @@ -180,7 +192,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 4010f67351b9e656cc500aa367c0c393 BELONGS ya.make @@ -190,9 +202,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 4588cbaabde7eebe28cc29740402f156 BELONGS ya.make @@ -202,8 +214,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 477adc1652fe6ac004ea66fbbcc0bb19 BELONGS ya.make @@ -214,8 +226,8 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_getrandom.c [9:13] - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 4ffe0aad4a07d95c6b3c7d06e1dfbead BELONGS ya.make @@ -225,7 +237,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 50da2a76d12ee3df6d928d81ca59a715 BELONGS ya.make @@ -245,7 +257,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 5619621de1e7b76391ee5fff9c1b1f91 BELONGS ya.make @@ -255,7 +267,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 576486d7fb5eeab148dcf39027acec46 BELONGS ya.make @@ -265,7 +277,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 581b53ae6f0fb8a0cc30c73b46bc3441 BELONGS ya.make @@ -308,7 +320,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 5fe78d330bca84f8e39670e48c92ba39 BELONGS ya.make @@ -319,7 +331,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/internal.h [28:36] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 60e5da68428d71447ddd4bd7750453bb BELONGS ya.make @@ -329,7 +341,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 610290a5946ae0c99de2de99646ba086 BELONGS ya.make @@ -349,7 +361,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 6451d5e490271b354ad3b567c7a03423 BELONGS ya.make @@ -360,7 +372,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_rand_s.c [9:14] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 647bb74c2d45ce0423d4c9a6b28bddac BELONGS ya.make @@ -380,7 +392,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 66d3ecc8836b1f9801bfad98582d2a32 BELONGS ya.make @@ -390,7 +402,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 7588a9020c401c85f6b542a5629c0c92 BELONGS ya.make @@ -410,7 +422,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 7c09099ef5f35bf3be4611e6cbb14510 BELONGS ya.make @@ -420,7 +432,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 7d65e46a306fed19dd12ba56b69d764c BELONGS ya.make @@ -440,9 +452,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL 868bf74335c8c4c3a6c576a3120e708c @@ -484,12 +496,12 @@ BELONGS ya.make lib/nametab.h [9:11] lib/utf8tab.h [9:13] lib/winconfig.h [9:14] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] lib/xmlrole.h [9:14] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_impl.h [9:12] lib/xmltok_ns.c [9:16] @@ -503,7 +515,7 @@ BELONGS ya.make Files with this license: lib/winconfig.h [9:14] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL 962bda2bd5ce73f8fa7de2620a3507f9 @@ -516,6 +528,16 @@ BELONGS ya.make Files with this license: lib/xmltok.h [9:15] +KEEP COPYRIGHT_SERVICE_LABEL 9842d67f5d734f4ca2f5f6c1c52393fe +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xmlparse.c [9:53] + KEEP COPYRIGHT_SERVICE_LABEL 99dbe0c5cacb0e3d1bc5b15bef337ea4 BELONGS ya.make Note: matched license text is too long. Read it in the source files. @@ -527,10 +549,10 @@ BELONGS ya.make lib/internal.h [28:36] lib/random_rand_s.c [9:14] lib/random_rand_s.h [9:11] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 9b348497698951665d1e43c1d376a510 BELONGS ya.make @@ -540,7 +562,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 9fdb85dcaaf74d518b27233b48fffa52 BELONGS ya.make @@ -552,7 +574,7 @@ BELONGS ya.make Files with this license: expat.h [9:23] expat_external.h [9:19] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL a0fdd1392c0b9b2558b9ccfe44592143 BELONGS ya.make @@ -574,7 +596,7 @@ BELONGS ya.make Files with this license: expat.h [9:23] lib/internal.h [28:36] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL a5c66b2d6998a1fbabcc4f2885ad5707 BELONGS ya.make @@ -584,7 +606,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL a6a07a1f1768704011ca547d57a193f2 BELONGS ya.make @@ -594,7 +616,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL aa19fc04b7b0134a0a4b71a6c182c1ae BELONGS ya.make @@ -605,7 +627,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL aafe06df8255f48781ac9d4e96e1ea4e BELONGS ya.make @@ -616,9 +638,9 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL abbaf6167a20f6ba0eee322937821c15 BELONGS ya.make @@ -628,7 +650,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ac721fcd634b3e5674a847f5ed2f1c8e BELONGS ya.make @@ -638,7 +660,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL acbc3573ff7fdf431f94f6aea99de1e0 BELONGS ya.make @@ -648,7 +670,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ad31071eb61955b9cdd9e0a463d6714f BELONGS ya.make @@ -658,7 +680,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL b667d591a50b1e424d66b58f6e0ff4a1 BELONGS ya.make @@ -668,7 +690,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL b927d5afef47e3ae01d339429012d6a7 BELONGS ya.make @@ -719,12 +741,12 @@ BELONGS ya.make lib/iasciitab.h [9:13] lib/latin1tab.h [9:13] lib/utf8tab.h [9:13] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] lib/xmlrole.h [9:14] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_impl.h [9:12] lib/xmltok_ns.c [9:16] @@ -751,7 +773,22 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] + +KEEP COPYRIGHT_SERVICE_LABEL d363fbc5cd02e7b4ee3c5c0d9fca8733 +BELONGS ya.make + License text: + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/fallthrough.h [9:10] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL d548c6beaeae204247905b60d5feff91 BELONGS ya.make @@ -761,8 +798,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL d9ac29affff64c14b3c82b78ed157a77 BELONGS ya.make @@ -772,9 +809,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL dd3c5623e58aa85a367a6638299f50f3 BELONGS ya.make @@ -784,7 +821,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL dd3e49797101e1ae2e966f0ebdd58da4 BELONGS ya.make @@ -817,10 +854,10 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_arc4random.h [9:10] - lib/random_arc4random_buf.c [9:10] + lib/random_arc4random_buf.c [9:11] lib/random_arc4random_buf.h [9:10] lib/random_dev_urandom.h [9:10] - lib/random_getentropy.c [9:10] + lib/random_getentropy.c [9:11] lib/random_getentropy.h [9:10] lib/random_getrandom.h [9:10] lib/random_rand_s.h [9:11] @@ -833,7 +870,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e2f8f506923a26765706e1ad8cc8edc6 BELONGS ya.make @@ -843,7 +880,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e3d6c1b6030b59aad9996cc0a9efeda5 BELONGS ya.make @@ -853,7 +890,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e526eb605bd716769dd1ed15786af827 BELONGS ya.make @@ -874,8 +911,11 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat_external.h [9:19] + lib/memory_sanitizer.h [9:10] lib/random_arc4random.c [9:11] + lib/random_arc4random_buf.c [9:11] lib/random_dev_urandom.c [9:11] + lib/random_getentropy.c [9:11] lib/random_getrandom.c [9:13] lib/random_rand_s.c [9:14] @@ -888,7 +928,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_getrandom.c [9:13] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ef0dda0153a00710149f327147a79b7f BELONGS ya.make @@ -939,7 +979,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL f792720cb266393d3d3fb2199660c4ea BELONGS ya.make @@ -949,9 +989,19 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] +KEEP COPYRIGHT_SERVICE_LABEL fddb9185e786dd5d20eac5402cf4f745 +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xmlparse.c [9:53] + KEEP COPYRIGHT_SERVICE_LABEL fea018c6e4e19bc6bd4ac263c015567a BELONGS ya.make Note: matched license text is too long. Read it in the source files. @@ -967,7 +1017,7 @@ BELONGS ya.make lib/utf8tab.h [9:13] lib/xmlrole.h [9:14] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL ff4131ca4bcbb6079f40a140651275ac @@ -978,7 +1028,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ffabe442e64111946b696c0384a52a77 BELONGS ya.make diff --git a/contrib/libs/expat/.yandex_meta/devtools.licenses.report b/contrib/libs/expat/.yandex_meta/devtools.licenses.report index 0402a3133b7..7d0b620fe8f 100644 --- a/contrib/libs/expat/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/expat/.yandex_meta/devtools.licenses.report @@ -42,7 +42,7 @@ BELONGS ya.make KEEP MIT 6bb6514a1d779748b76a73215a89ae66 BELONGS ya.make -FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at line 35, lib/ascii.h at line 30, lib/asciitab.h at line 29, lib/iasciitab.h at line 29, lib/internal.h at line 52, lib/latin1tab.h at line 29, lib/random_arc4random.c at line 27, lib/random_arc4random.h at line 26, lib/random_arc4random_buf.c at line 26, lib/random_arc4random_buf.h at line 26, lib/random_dev_urandom.c at line 27, lib/random_dev_urandom.h at line 26, lib/random_getentropy.c at line 26, lib/random_getentropy.h at line 26, lib/random_getrandom.c at line 29, lib/random_getrandom.h at line 26, lib/random_rand_s.c at line 30, lib/random_rand_s.h at line 27, lib/utf8tab.h at line 29, lib/winconfig.h at line 30, lib/xmlparse.c at line 66, lib/xmlrole.c at line 36, lib/xmlrole.h at line 30, lib/xmltok.c at line 44, lib/xmltok.h at line 31, lib/xmltok_impl.c at line 36, lib/xmltok_impl.h at line 28, lib/xmltok_ns.c at line 32 +FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at line 35, lib/ascii.h at line 30, lib/asciitab.h at line 29, lib/fallthrough.h at line 26, lib/iasciitab.h at line 29, lib/internal.h at line 52, lib/latin1tab.h at line 29, lib/memory_sanitizer.h at line 26, lib/random_arc4random.c at line 27, lib/random_arc4random.h at line 26, lib/random_arc4random_buf.c at line 27, lib/random_arc4random_buf.h at line 26, lib/random_dev_urandom.c at line 27, lib/random_dev_urandom.h at line 26, lib/random_getentropy.c at line 27, lib/random_getentropy.h at line 26, lib/random_getrandom.c at line 29, lib/random_getrandom.h at line 26, lib/random_rand_s.c at line 30, lib/random_rand_s.h at line 27, lib/utf8tab.h at line 29, lib/winconfig.h at line 30, lib/xcsinc.c at line 26, lib/xmlparse.c at line 69, lib/xmlrole.c at line 36, lib/xmlrole.h at line 30, lib/xmltok.c at line 45, lib/xmltok.h at line 31, lib/xmltok_impl.c at line 37, lib/xmltok_impl.h at line 28, lib/xmltok_ns.c at line 32 Note: matched license text is too long. Read it in the source files. Scancode info: Original SPDX id: MIT @@ -54,16 +54,18 @@ FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at lin expat_external.h [21:38] lib/ascii.h [16:33] lib/asciitab.h [15:32] + lib/fallthrough.h [12:29] lib/iasciitab.h [15:32] lib/internal.h [38:55] lib/latin1tab.h [15:32] + lib/memory_sanitizer.h [12:29] lib/random_arc4random.c [13:30] lib/random_arc4random.h [12:29] - lib/random_arc4random_buf.c [12:29] + lib/random_arc4random_buf.c [13:30] lib/random_arc4random_buf.h [12:29] lib/random_dev_urandom.c [13:30] lib/random_dev_urandom.h [12:29] - lib/random_getentropy.c [12:29] + lib/random_getentropy.c [13:30] lib/random_getentropy.h [12:29] lib/random_getrandom.c [15:32] lib/random_getrandom.h [12:29] @@ -71,12 +73,13 @@ FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at lin lib/random_rand_s.h [13:30] lib/utf8tab.h [15:32] lib/winconfig.h [16:33] - lib/xmlparse.c [52:69] + lib/xcsinc.c [12:29] + lib/xmlparse.c [55:72] lib/xmlrole.c [22:39] lib/xmlrole.h [16:33] - lib/xmltok.c [30:47] + lib/xmltok.c [31:48] lib/xmltok.h [17:34] - lib/xmltok_impl.c [22:39] + lib/xmltok_impl.c [23:40] lib/xmltok_impl.h [14:31] lib/xmltok_ns.c [18:35] @@ -106,17 +109,19 @@ BELONGS ya.make expat_external.h [19:19] lib/ascii.h [14:14] lib/asciitab.h [13:13] + lib/fallthrough.h [10:10] lib/iasciitab.h [13:13] lib/internal.h [36:36] lib/latin1tab.h [13:13] + lib/memory_sanitizer.h [10:10] lib/nametab.h [11:11] lib/random_arc4random.c [11:11] lib/random_arc4random.h [10:10] - lib/random_arc4random_buf.c [10:10] + lib/random_arc4random_buf.c [11:11] lib/random_arc4random_buf.h [10:10] lib/random_dev_urandom.c [11:11] lib/random_dev_urandom.h [10:10] - lib/random_getentropy.c [10:10] + lib/random_getentropy.c [11:11] lib/random_getentropy.h [10:10] lib/random_getrandom.c [13:13] lib/random_getrandom.h [10:10] @@ -124,12 +129,13 @@ BELONGS ya.make lib/random_rand_s.h [11:11] lib/utf8tab.h [13:13] lib/winconfig.h [14:14] - lib/xmlparse.c [50:50] + lib/xcsinc.c [10:10] + lib/xmlparse.c [53:53] lib/xmlrole.c [20:20] lib/xmlrole.h [14:14] - lib/xmltok.c [28:28] + lib/xmltok.c [29:29] lib/xmltok.h [15:15] - lib/xmltok_impl.c [20:20] + lib/xmltok_impl.c [21:21] lib/xmltok_impl.h [12:12] lib/xmltok_ns.c [16:16] diff --git a/contrib/libs/expat/.yandex_meta/licenses.list.txt b/contrib/libs/expat/.yandex_meta/licenses.list.txt index 8b7e30d3de9..687cd921a14 100644 --- a/contrib/libs/expat/.yandex_meta/licenses.list.txt +++ b/contrib/libs/expat/.yandex_meta/licenses.list.txt @@ -76,6 +76,9 @@ Copyright (c) 2026 Rosen Penev <[email protected]> Copyright (c) 2026 Francesco Bertolaccini Copyright (c) 2026 Christian Ng <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> + Copyright (c) 2026 Kartik Kenchi <[email protected]> + Copyright (c) 2026 Haris Hussain <[email protected]> Licensed under the MIT license: @@ -99,6 +102,7 @@ Copyright (c) 2022 Sean McBride <[email protected]> Copyright (c) 2023 Hanno Böck <[email protected]> Copyright (c) 2025 Alfonso Gregory <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: @@ -124,6 +128,7 @@ Copyright (c) 2019 David Loffredo <[email protected]> Copyright (c) 2020 Boris Kolpackov <[email protected]> Copyright (c) 2022 Martin Ettl <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: @@ -223,6 +228,16 @@ ====================COPYRIGHT==================== + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + + +====================COPYRIGHT==================== + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + + +====================COPYRIGHT==================== Copyright (c) 2026 Sebastian Pipping <[email protected]> Licensed under the MIT license: diff --git a/contrib/libs/expat/.yandex_meta/override.nix b/contrib/libs/expat/.yandex_meta/override.nix index 19266e058c2..ae1a5b09b61 100644 --- a/contrib/libs/expat/.yandex_meta/override.nix +++ b/contrib/libs/expat/.yandex_meta/override.nix @@ -1,12 +1,12 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "2.8.1"; + version = "2.8.2"; versionTag = "R_${lib.replaceStrings ["."] ["_"] version}"; src = fetchFromGitHub { owner = "libexpat"; repo = "libexpat"; rev = "${versionTag}"; - hash = "sha256-XHQN6Wd29Furdu8VB1UokZMmIFKq4sCG+Aa8/KfSoi4="; + hash = "sha256-YNrsTYgY4gyRDpYPZPmkCTOF1eKA7rCcyfS0jR/SItY="; }; nativeBuildInputs = [ autoreconfHook ]; diff --git a/contrib/libs/expat/Changes b/contrib/libs/expat/Changes index 313b3dc60cd..6a1b89160fb 100644 --- a/contrib/libs/expat/Changes +++ b/contrib/libs/expat/Changes @@ -29,6 +29,94 @@ !! THANK YOU! Sebastian Pipping -- Berlin, 2026-03-17 !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Release 2.8.2 Thu June 25 2026 + Security fixes: + #1246 CVE-2026-50219 -- Disallow calls to functions + `XML_GetBuffer`, `XML_Parse`, `XML_ParseBuffer`, + `XML_ParserFree`, `XML_ParserReset` to guard e.g. + Expat bindings from memory corruption; + this CPython issue is related: + https://github.com/python/cpython/issues/146169 + #1267 CVE-2026-56131 -- Protect XML_ResumeParser from being called + from a handler, plugging a hole in the fix + to CVE-2026-50219 + #1272 CVE-2026-56132 -- Fix out-of-bound scaffolding index store + in `doProlog` + #1229 #1232 CVE-2026-56403 -- Integer overflow in `storeAtts` + #1249 CVE-2026-56404 -- Integer overflow in `addBinding` + #1251 CVE-2026-56405 -- Integer overflow in `getAttributeId` + #1255 CVE-2026-56406 -- Integer overflow in `XML_ParseBuffer` + #1262 CVE-2026-56407 -- Integer overflow in `textLen` handling + #565 CVE-2026-56408 -- Integer overflow in `copyString` + (commit 16e2efd867ea8567ffa012210b52ef5918e20817) + #1259 CVE-2026-56409 -- xmlwf: Integer overflow in output path join + #1252 CVE-2026-56410 -- xmlwf: Integer overflow in + `resolveSystemId` + #1263 CVE-2026-56411 -- xmlwf: Integer overflow in notation list + allocation + #1278 CVE-2026-56412 -- Guard XML_TOK_DATA_CHARS handler calls in + `doCdataSection`, plugging a hole in the fix to + CVE-2026-50219 + + Bug fixes: + #1260 xmlwf: Escape names and base URI in meta output + #1266 xmlwf: Pick a safe quote for notation system and public IDs + + Other changes: + #1257 CMake|Autotools: Stop using /dev/urandom by default + #1244 #1254 CMake: Fix guard for Unix sources of entropy + #1183 #1270 CMake|Windows: Add missing export for symbol + `XML_SetHashSalt16Bytes` + #1236 CMake: Mark option EXPAT_OSSFUZZ_BUILD as advanced + #1283 Limit output indentation for EXPAT_ENTITY_DEBUG=1 and + allow unlimited indentation via EXPAT_ENTITY_DEBUG=2 + #565 Replace some loops by use of `memcpy`, `strlen`, `wcslen` + #1220 lib: Use a size_t for group sizes + #1221 lib: Fix too-conservative integer overflow check when + appending raw name + #1222 lib: Simplify attribute allocation/management logic + #1224 Update fallthrough annotations to satisfy Clang and GCC + #1226 lib: Remove unnecessary void * casts in random code + #1228 lib: Reduce scope of locals in storeAtts + #1230 lib: Count attributes with size_t variables + #1238 Minor get-buffer improvements + #1239 #1240 lib|tests: Include header expat_config.h first + #1241 lib: Shrink size of XML_GetBuffer + #1242 lib: Remove a legacy comment + #1243 lib: XML_ParserReset: Extract repeated linked-list move logic + #1243 lib: Unify entity free lists + #1247 lib: Fix use of '0' as boolean literal + #1248 lib: Make XML_Index overflow check more intuitive + #1256 lib: Use size_t for counting string/URI lengths + #1258 lib: XML_GetInputContext: Remove use of 0 for NULL + #1261 Comment typo fixes + #1275 Teach Memory Sanitizer semantics of randomization functions + #1276 #1281 Version info bumped from 13:1:12 (libexpat*.so.1.12.1) + to 13:2:12 (libexpat*.so.1.12.2); see https://verbump.de/ + for what these numbers do + + Infrastructure: + #1231 perl-integration.yml: Bump to XML::Parser 2.59 + #1237 emscripten.yml: Bump from Ubuntu 22.04 to 24.04 + #1183 #1271 windows-build.yml: Cover completeness of file + libexpat.def.cmake + #1274 linux.yml: Make llvm-symbolizer available in CI + + Special thanks to: + Alessandro Gario + Asher Darden + Christoph Reiter + Haris Hussain + Matthew Fernandez + Kartik Kenchi + Nick Begg + Sajin S + Yousef Shanableh + and + Anthropic + Astra Security + Trail of Bits + Release 2.8.1 Sun May 10 2026 Security fixes: #1216 CVE-2026-45186 -- Fix quadratic runtime from attribute name @@ -1820,7 +1908,7 @@ Release 2.1.0 Sat March 24 2012 - Bug Fixes: #1742315: Harmful XML_ParserCreateNS suggestion. #1785430: Expat build fails on linux-amd64 with gcc version>=4.1 -O3. - #1983953, 2517952, 2517962, 2649838: + #1983953, 2517952, 2517962, 2649838: Build modifications using autoreconf instead of buildconf.sh. #2815947, #2884086: OBJEXT and EXEEXT support while building. #2517938: xmlwf should return non-zero exit status if not well-formed. @@ -1846,7 +1934,7 @@ Release 2.1.0 Sat March 24 2012 See bug #2990652 and patch #3312568. Added run-benchmark target to Makefile.in - relies on testdata module present in the same relative location as in the repository. - + Release 2.0.1 Tue June 5 2007 - Fixed bugs #1515266, #1515600: The character data handler's calling of XML_StopParser() was not handled properly; if the parser was diff --git a/contrib/libs/expat/README.md b/contrib/libs/expat/README.md index 7a5ed463b6d..5ad5e1e85af 100644 --- a/contrib/libs/expat/README.md +++ b/contrib/libs/expat/README.md @@ -11,7 +11,7 @@ > at the top of the `Changes` file. -# Expat, Release 2.8.1 +# Expat, Release 2.8.2 This is Expat, a C99 library for parsing [XML 1.0 Fourth Edition](https://www.w3.org/TR/2006/REC-xml-20060816/), started by @@ -286,9 +286,6 @@ EXPAT_ENABLE_INSTALL:BOOL=ON // Use /MT flag (static CRT) when compiling in MSVC EXPAT_MSVC_STATIC_CRT:BOOL=OFF -// Build fuzzers via OSS-Fuzz for the expat library -EXPAT_OSSFUZZ_BUILD:BOOL=OFF - // Build a shared expat library EXPAT_SHARED_LIBS:BOOL=ON diff --git a/contrib/libs/expat/expat.h b/contrib/libs/expat/expat.h index ec3f58544cb..c493c70441c 100644 --- a/contrib/libs/expat/expat.h +++ b/contrib/libs/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 1 +# define XML_MICRO_VERSION 2 # ifdef __cplusplus } diff --git a/contrib/libs/expat/expat_config-linux.h b/contrib/libs/expat/expat_config-linux.h index c97ae4de7e6..2a91d46e67e 100644 --- a/contrib/libs/expat/expat_config-linux.h +++ b/contrib/libs/expat/expat_config-linux.h @@ -83,7 +83,7 @@ #define PACKAGE_NAME "expat" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "expat 2.8.1" +#define PACKAGE_STRING "expat 2.8.2" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "expat" @@ -92,7 +92,7 @@ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "2.8.1" +#define PACKAGE_VERSION "2.8.2" /* Define to 1 if all of the C89 standard headers exist (not just the ones required in a freestanding environment). This macro is provided for @@ -100,7 +100,7 @@ #define STDC_HEADERS 1 /* Version number of package */ -#define VERSION "2.8.1" +#define VERSION "2.8.2" /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ @@ -123,7 +123,7 @@ #define XML_CONTEXT_BYTES 1024 /* Define to include code reading entropy from `/dev/urandom'. */ -#define XML_DEV_URANDOM 1 +/* #undef XML_DEV_URANDOM */ /* Define to make parameter entity parsing functionality available. */ #define XML_DTD 1 diff --git a/contrib/libs/expat/lib/fallthrough.h b/contrib/libs/expat/lib/fallthrough.h new file mode 100644 index 00000000000..707dbdd44bf --- /dev/null +++ b/contrib/libs/expat/lib/fallthrough.h @@ -0,0 +1,49 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/contrib/libs/expat/lib/memory_sanitizer.h b/contrib/libs/expat/lib/memory_sanitizer.h new file mode 100644 index 00000000000..a8a8006ccde --- /dev/null +++ b/contrib/libs/expat/lib/memory_sanitizer.h @@ -0,0 +1,51 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Matthew Fernandez <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(MEMORY_SANITIZER_H) +# define MEMORY_SANITIZER_H 1 + +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include <sanitizer/msan_interface.h> + +// inform Memory Sanitizer that [base, base + extent) is now initialized +# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent)) + +# endif +# endif + +# if ! defined(MSAN_UNPOISON) +# define MSAN_UNPOISON(base, extent) \ + do { \ + } while (0) +# endif + +#endif // ! defined(MEMORY_SANITIZER_H) diff --git a/contrib/libs/expat/lib/random_arc4random_buf.c b/contrib/libs/expat/lib/random_arc4random_buf.c index 1476f07ba1a..5212b9f192c 100644 --- a/contrib/libs/expat/lib/random_arc4random_buf.c +++ b/contrib/libs/expat/lib/random_arc4random_buf.c @@ -7,6 +7,7 @@ |_| XML parser Copyright (c) 2026 Sebastian Pipping <[email protected]> + Copyright (c) 2026 Matthew Fernandez <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,9 +36,12 @@ # define _DEFAULT_SOURCE 1 /* for glibc */ #endif +#include "memory_sanitizer.h" #include <stdlib.h> // for arc4random_buf void writeRandomBytes_arc4random_buf(void *target, size_t count) { arc4random_buf(target, count); + // MSan does not understand `arc4random_buf`, so explain its effects + MSAN_UNPOISON(target, count); } diff --git a/contrib/libs/expat/lib/random_dev_urandom.c b/contrib/libs/expat/lib/random_dev_urandom.c index e508fe9f3e3..acae6ba4b04 100644 --- a/contrib/libs/expat/lib/random_dev_urandom.c +++ b/contrib/libs/expat/lib/random_dev_urandom.c @@ -49,11 +49,11 @@ writeRandomBytes_dev_urandom(void *target, size_t count) { const int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); if (fd < 0) { - return 0; + return false; } do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); + void *const currentTarget = (char *)target + bytesWrittenTotal; const size_t bytesToWrite = count - bytesWrittenTotal; errno = 0; diff --git a/contrib/libs/expat/lib/random_getentropy.c b/contrib/libs/expat/lib/random_getentropy.c index b4be70ed73c..5cea2d8838b 100644 --- a/contrib/libs/expat/lib/random_getentropy.c +++ b/contrib/libs/expat/lib/random_getentropy.c @@ -7,6 +7,7 @@ |_| XML parser Copyright (c) 2026 Sebastian Pipping <[email protected]> + Copyright (c) 2026 Matthew Fernandez <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -45,10 +46,15 @@ # include <unistd.h> #endif // ! defined(__APPLE__) +#include "memory_sanitizer.h" #include <errno.h> bool writeRandomBytes_getentropy(void *target, size_t count) { errno = 0; - return getentropy(target, count) == 0; + const bool success = getentropy(target, count); + // MSan does not understand `getentropy`, so explain its effects + if (success) + MSAN_UNPOISON(target, count); + return success; } diff --git a/contrib/libs/expat/lib/random_getrandom.c b/contrib/libs/expat/lib/random_getrandom.c index e5724a211cc..72e2cc7093b 100644 --- a/contrib/libs/expat/lib/random_getrandom.c +++ b/contrib/libs/expat/lib/random_getrandom.c @@ -32,10 +32,10 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "random_getrandom.h" - #include "expat_config.h" // for HAVE_GETRANDOM, HAVE_SYSCALL_GETRANDOM +#include "random_getrandom.h" + #if defined(HAVE_GETRANDOM) # include <sys/random.h> /* getrandom */ #endif @@ -52,6 +52,7 @@ # define GRND_NONBLOCK 0x0001 #endif /* defined(GRND_NONBLOCK) */ +#include "memory_sanitizer.h" #include <assert.h> #include <errno.h> #include <limits.h> // for INT_MAX @@ -64,7 +65,7 @@ writeRandomBytes_getrandom_nonblock(void *target, size_t count) { const unsigned int getrandomFlags = GRND_NONBLOCK; do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); + void *const currentTarget = (char *)target + bytesWrittenTotal; const size_t bytesToWrite = count - bytesWrittenTotal; assert(bytesToWrite <= INT_MAX); @@ -74,9 +75,13 @@ writeRandomBytes_getrandom_nonblock(void *target, size_t count) { const int bytesWrittenMore = #if defined(HAVE_GETRANDOM) (int)getrandom(currentTarget, bytesToWrite, getrandomFlags); + // MSan understands `getrandom`, so does not need extra guidance #else (int)syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); + // MSan does not understand `syscall`, so explain its effects + if (bytesWrittenMore > 0) + MSAN_UNPOISON(currentTarget, bytesWrittenMore); #endif if (bytesWrittenMore > 0) { diff --git a/contrib/libs/expat/lib/siphash.h b/contrib/libs/expat/lib/siphash.h index 04f6f74585b..be216b4006d 100644 --- a/contrib/libs/expat/lib/siphash.h +++ b/contrib/libs/expat/lib/siphash.h @@ -100,6 +100,7 @@ #include <stddef.h> /* size_t */ #include <stdint.h> /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" /* * Workaround to not require a C++11 compiler for using ULL suffix @@ -234,25 +235,25 @@ sip24_final(struct siphash *H) { switch (left) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/contrib/libs/expat/lib/xcsinc.c b/contrib/libs/expat/lib/xcsinc.c new file mode 100644 index 00000000000..3597c2480bc --- /dev/null +++ b/contrib/libs/expat/lib/xcsinc.c @@ -0,0 +1,48 @@ +/* This file is included from other .c files! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +static size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} diff --git a/contrib/libs/expat/lib/xmlparse.c b/contrib/libs/expat/lib/xmlparse.c index 890a9389f15..60f6cc70054 100644 --- a/contrib/libs/expat/lib/xmlparse.c +++ b/contrib/libs/expat/lib/xmlparse.c @@ -1,4 +1,4 @@ -/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) +/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -47,6 +47,9 @@ Copyright (c) 2026 Rosen Penev <[email protected]> Copyright (c) 2026 Francesco Bertolaccini Copyright (c) 2026 Christian Ng <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> + Copyright (c) 2026 Kartik Kenchi <[email protected]> + Copyright (c) 2026 Haris Hussain <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -90,7 +93,7 @@ #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> -#include <limits.h> /* INT_MAX, UINT_MAX */ +#include <limits.h> /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ #include <stdio.h> /* fprintf */ #include <stdlib.h> /* getenv */ #include <stdint.h> /* SIZE_MAX, uintptr_t */ @@ -114,6 +117,7 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" +#include "xcsinc.c" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -156,7 +160,7 @@ * Windows >=Vista (rand_s): _WIN32. \ \ If you insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -207,6 +211,12 @@ typedef char ICHAR; #endif +#ifdef XML_LARGE_SIZE +# define XML_INDEX_MAX LLONG_MAX +#else +# define XML_INDEX_MAX LONG_MAX +#endif + /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -270,8 +280,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -283,9 +293,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -384,8 +394,8 @@ typedef struct { const XML_Char *name; PREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; @@ -417,6 +427,7 @@ typedef struct { unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; enum EntityType { @@ -588,6 +599,7 @@ static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); +static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s); static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, @@ -642,16 +654,40 @@ static XML_Parser getRootParserOf(XML_Parser parser, static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -730,11 +766,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; OPEN_INTERNAL_ENTITY *m_openAttributeEntities; - OPEN_INTERNAL_ENTITY *m_freeAttributeEntities; OPEN_INTERNAL_ENTITY *m_openValueEntities; - OPEN_INTERNAL_ENTITY *m_freeValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -754,7 +788,7 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; @@ -768,7 +802,7 @@ struct XML_ParserStruct { STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -785,6 +819,7 @@ struct XML_ParserStruct { ENTITY_STATS m_entity_stats; #endif XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; #if XML_GE == 1 @@ -1128,6 +1163,23 @@ generate_hash_secret_salt(void) { #endif } +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1364,9 +1416,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; - parser->m_freeAttributeEntities = NULL; - parser->m_freeValueEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1482,6 +1532,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_parsingStatus.parsing = XML_INITIALIZED; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; @@ -1515,12 +1566,22 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) @@ -1535,32 +1596,14 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } - /* move m_openAttributeEntities to m_freeAttributeEntities (i.e. same task but - * for attributes) */ - openEntityList = parser->m_openAttributeEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; - } - /* move m_openValueEntities to m_freeValueEntities (i.e. same task but - * for value entities) */ - openEntityList = parser->m_openValueEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) @@ -1712,11 +1755,6 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); @@ -1806,8 +1844,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; @@ -1825,48 +1862,35 @@ XML_ParserFree(XML_Parser parser) { destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openAttributeEntities and m_freeAttributeEntities */ - entityList = parser->m_openAttributeEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeAttributeEntities == NULL) - break; - entityList = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = NULL; - } - openEntity = entityList; + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openValueEntities and m_freeValueEntities */ - entityList = parser->m_openValueEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeValueEntities == NULL) - break; - entityList = parser->m_freeValueEntities; - parser->m_freeValueEntities = NULL; - } - openEntity = entityList; + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -2263,6 +2287,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2275,7 +2301,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -2286,7 +2312,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2317,7 +2343,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2372,7 +2398,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2398,11 +2424,19 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + // Detect and avoid integer overflow + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2428,6 +2462,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2438,9 +2473,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2461,9 +2519,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2473,7 +2528,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2481,16 +2538,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2507,7 +2564,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2522,49 +2578,12 @@ XML_GetBuffer(XML_Parser parser, int len) { } // NOTE: We are avoiding MALLOC(..) here to leave limiting // the input size to the application using Expat. - newBuf = parser->m_mem.malloc_fcn(bufferSize); + char *const newBuf = parser->m_mem.malloc_fcn(bufferSize); if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2617,7 +2636,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2643,6 +2662,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2704,7 +2724,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } XML_Size XMLCALL @@ -3063,7 +3083,7 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; bufSize = nameLen + rawNameLen; if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { @@ -3190,7 +3210,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default: start = next; } @@ -3269,7 +3289,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3322,9 +3344,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3344,9 +3368,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3357,10 +3383,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3374,9 +3402,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, entity->open = XML_FALSE; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -3384,7 +3415,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -3416,11 +3446,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *fromPtr = tag->rawName; toPtr = tag->buf.str; for (;;) { - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - tag->buf.str); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; @@ -3445,16 +3474,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3474,14 +3504,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3539,14 +3573,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3569,8 +3608,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3579,32 +3620,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3624,13 +3671,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3655,16 +3707,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3702,7 +3759,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -3722,8 +3779,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3747,20 +3807,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3775,75 +3829,71 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = REALLOC(parser, parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; #ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -# endif - temp2 = REALLOC(parser, parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); + XML_AttrInfo *const temp2 = REALLOC( + parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); if (temp2 == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_attInfo = temp2; #endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + } } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; #ifdef XML_ATTR_INFO XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; @@ -3876,13 +3926,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { + for (size_t j = 0; j < nDefaultAtts; j++) { if (attId == elementType->defaultAtts[j].id) { isCdata = elementType->defaultAtts[j].isCdata; break; @@ -3891,7 +3939,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3929,7 +3977,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3938,7 +3986,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3964,7 +4012,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; @@ -3979,7 +4027,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4007,7 +4054,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - temp = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -4028,9 +4076,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -4038,7 +4083,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -4046,7 +4092,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -4056,15 +4102,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < (unsigned int)b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -4073,12 +4116,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -4106,10 +4150,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -4152,48 +4195,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - while (binding->prefix->name[prefixLen++]) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -4201,10 +4232,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -4339,7 +4374,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -4347,15 +4382,14 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -4374,6 +4408,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4414,25 +4452,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif XML_Char *temp = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); @@ -4448,18 +4482,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { - return XML_ERROR_NO_MEMORY; - } -#endif b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { @@ -4483,9 +4509,12 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } @@ -4547,15 +4576,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4566,7 +4600,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4579,16 +4615,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4633,7 +4674,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -4827,8 +4868,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4878,8 +4921,11 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { @@ -5256,9 +5302,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_DOCTYPE_INTERNAL_SUBSET: if (parser->m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -5297,7 +5345,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -5325,9 +5373,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5354,14 +5404,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_useForeignDTD) entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5374,7 +5432,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5394,14 +5454,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5494,10 +5562,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5532,10 +5602,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5550,16 +5622,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5577,10 +5655,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5611,9 +5691,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5625,7 +5709,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5656,10 +5740,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5672,17 +5758,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5789,9 +5879,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5799,9 +5891,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5835,41 +5929,18 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - dtd->scaffIndex = new_scaff_index; + char *const new_connector = REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { @@ -5884,6 +5955,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5964,7 +6050,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5985,9 +6073,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6005,9 +6096,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -6042,8 +6137,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } dtd->in_eldecl = XML_FALSE; @@ -6087,9 +6184,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - while (name[nameLen++]) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -6125,8 +6220,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } dtd->in_eldecl = XML_FALSE; dtd->contentStringLen = 0; @@ -6188,7 +6285,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -6268,7 +6365,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; parser->m_eventPtr = s = next; } @@ -6278,20 +6375,18 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, enum EntityType type) { - OPEN_INTERNAL_ENTITY *openEntity, **openEntityList, **freeEntityList; + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; switch (type) { case ENTITY_INTERNAL: parser->m_processor = internalEntityProcessor; openEntityList = &parser->m_openInternalEntities; - freeEntityList = &parser->m_freeInternalEntities; break; case ENTITY_ATTRIBUTE: openEntityList = &parser->m_openAttributeEntities; - freeEntityList = &parser->m_freeAttributeEntities; break; case ENTITY_VALUE: openEntityList = &parser->m_openValueEntities; - freeEntityList = &parser->m_freeValueEntities; break; /* default case serves merely as a safety net in case of a * wrong entityType. Therefore we exclude the following lines @@ -6408,8 +6503,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; if (parser->m_openInternalEntities == NULL) { parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; @@ -6485,8 +6580,8 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6539,7 +6634,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6559,10 +6653,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6570,7 +6663,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6624,8 +6717,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6758,8 +6854,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6775,9 +6874,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6813,17 +6915,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6841,12 +6941,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6943,8 +7040,8 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6974,6 +7071,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -7026,7 +7128,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7046,7 +7150,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7087,15 +7193,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -7106,48 +7217,34 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttsNames), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts - = MALLOC(parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; @@ -7156,8 +7253,8 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + NAMED *const nameAddedOrFound + = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); if (! nameAddedOrFound) return 0; @@ -7230,13 +7327,14 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, @@ -7264,46 +7362,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -7318,23 +7409,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -7342,9 +7431,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -7466,6 +7554,7 @@ dtdCreate(XML_Parser parser) { p->in_eldecl = XML_FALSE; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -7487,8 +7576,7 @@ dtdReset(DTD *p, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD @@ -7507,6 +7595,7 @@ dtdReset(DTD *p, XML_Parser parser) { FREE(parser, p->scaffIndex); p->scaffIndex = NULL; + p->scaffIndexSize = 0; FREE(parser, p->scaffold); p->scaffold = NULL; @@ -7529,8 +7618,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -7609,7 +7697,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7627,15 +7714,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableInit(&(newE->defaultAttsNames), parser); if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7649,7 +7731,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( oldParser, &(newDtd->attributeIds), attributeName, 0); @@ -7662,8 +7744,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), + attributeName, sizeof(NAMED)); if (! nameAddedOrFound) { return 0; } @@ -7693,6 +7775,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7764,18 +7847,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, static XML_Bool FASTCALL keyeq(KEY s1, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +# else for (; *s1 == *s2; s1++, s2++) if (*s1 == 0) return XML_TRUE; return XML_FALSE; +# endif +#else + return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +#endif } static size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void @@ -7993,10 +8081,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8035,10 +8121,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8046,11 +8130,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -8059,9 +8140,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -8196,6 +8276,19 @@ poolGrow(STRING_POOL *pool) { return XML_TRUE; } +static bool FASTCALL +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + static int FASTCALL nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -8203,18 +8296,14 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX + /* Detect and prevent integer overflow. */ if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } @@ -8375,12 +8464,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -8427,22 +8525,24 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, static XML_Char * copyString(const XML_Char *s, XML_Parser parser) { - size_t charsRequired = 0; - XML_Char *result; - /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = MALLOC(parser, charsRequired * sizeof(XML_Char)); + XML_Char *const result = MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -8609,15 +8709,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - ((int)rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void diff --git a/contrib/libs/expat/lib/xmltok.c b/contrib/libs/expat/lib/xmltok.c index 32cd5f147e9..387c6e44a25 100644 --- a/contrib/libs/expat/lib/xmltok.c +++ b/contrib/libs/expat/lib/xmltok.c @@ -25,6 +25,7 @@ Copyright (c) 2022 Sean McBride <[email protected]> Copyright (c) 2023 Hanno Böck <[email protected]> Copyright (c) 2025 Alfonso Gregory <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -58,6 +59,7 @@ #endif #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -641,7 +643,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -1557,7 +1559,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; diff --git a/contrib/libs/expat/lib/xmltok_impl.c b/contrib/libs/expat/lib/xmltok_impl.c index 239a2d06c45..eae11bdd968 100644 --- a/contrib/libs/expat/lib/xmltok_impl.c +++ b/contrib/libs/expat/lib/xmltok_impl.c @@ -17,6 +17,7 @@ Copyright (c) 2019 David Loffredo <[email protected]> Copyright (c) 2020 Boris Kolpackov <[email protected]> Copyright (c) 2022 Martin Ettl <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -83,7 +84,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +113,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -209,7 +210,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_CR: case BT_LF: @@ -323,7 +324,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr + MINBPC(enc); return tok; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -615,7 +616,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_EQUALS: { int open; # ifdef XML_NS @@ -898,7 +899,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -1059,7 +1060,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1075,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1190,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1484,7 +1485,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ diff --git a/contrib/libs/expat/ya.make b/contrib/libs/expat/ya.make index cdd5cc2d3e5..0fd2f760564 100644 --- a/contrib/libs/expat/ya.make +++ b/contrib/libs/expat/ya.make @@ -10,12 +10,13 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2.8.1) +VERSION(2.8.2) -ORIGINAL_SOURCE(https://github.com/libexpat/libexpat/archive/R_2_8_1.tar.gz) +ORIGINAL_SOURCE(https://github.com/libexpat/libexpat/archive/R_2_8_2.tar.gz) PEERDIR( contrib/libs/libc_compat + library/cpp/sanitizer/include ) ADDINCL( diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report index 93fa9c4becc..f87d3c05433 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report @@ -392,7 +392,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: LICENSE.md [32:33] - tif_lzw.c [1445:1446] + tif_lzw.c [1463:1464] KEEP COPYRIGHT_SERVICE_LABEL ef9e24bc2ee738c92b1a0aae6c46127d BELONGS ya.make diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report index 358bb3da041..a91c6140116 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report @@ -61,7 +61,7 @@ BELONGS ya.make Match type : TEXT Links : https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/bsla.LICENSE Files with this license: - tif_lzw.c [1452:1462] + tif_lzw.c [1470:1480] KEEP BSLA 99516a4e1b8d6f6054da2439715a8bda BELONGS ya.make @@ -144,7 +144,6 @@ BELONGS ya.make tif_version.c [5:22] tif_warning.c [5:22] tif_webp.c [5:22] - tif_win32.c [5:22] tif_write.c [5:22] tif_zip.c [5:22] tif_zstd.c [5:22] @@ -164,3 +163,14 @@ BELONGS ya.make Files with this license: LICENSE.md [6:23] README.md [52:69] + +KEEP libtiff eb46110bca2ca592d93347fc2a74357a +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: libtiff + Score : 93.33 + Match type : TEXT + Links : https://fedoraproject.org/wiki/Licensing/libtiff, https://spdx.org/licenses/libtiff + Files with this license: + tif_win32.c [5:23] diff --git a/contrib/libs/libtiff/.yandex_meta/licenses.list.txt b/contrib/libs/libtiff/.yandex_meta/licenses.list.txt index ca337b4f57f..054ff5c4672 100644 --- a/contrib/libs/libtiff/.yandex_meta/licenses.list.txt +++ b/contrib/libs/libtiff/.yandex_meta/licenses.list.txt @@ -215,6 +215,28 @@ Silicon Graphics has seen fit to allow us to give this work away. It ====================libtiff==================== + * Permission to use, copy, modify, distribute, and sell this software and + * its documentation for any purpose is hereby granted without fee, provided + * that (i) the above copyright notices and this permission notice appear in + * that (i) the above copyright notices and this permission notice appear in + * all copies of the software and related documentation, and (ii) the names of + * Sam Leffler and Silicon Graphics may not be used in any advertising or + * publicity relating to the software without the specific, prior written + * permission of Sam Leffler and Silicon Graphics. + * + * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, + * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY + * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + * + * IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR + * ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, + * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF + * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THIS SOFTWARE. + + +====================libtiff==================== Permission to use, copy, modify, distribute, and sell this software and its documentation for any purpose is hereby granted without fee, provided that (i) the above copyright notices and this permission notice appear in diff --git a/contrib/libs/libtiff/.yandex_meta/override.nix b/contrib/libs/libtiff/.yandex_meta/override.nix index 9ce9dd2f515..0a61f01b1fe 100644 --- a/contrib/libs/libtiff/.yandex_meta/override.nix +++ b/contrib/libs/libtiff/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; rec { - version = "4.7.1"; + version = "4.7.2"; src = fetchFromGitLab { owner = "libtiff"; repo = "libtiff"; rev = "v${version}"; - hash = "sha256-UiC6s86i7UavW86EKm74oPVlEacvoKmwW7KETjpnNaI="; + hash = "sha256-60Lpg5WRfWMzlOoOUA+C6KLlYIZ+3BjXidOVqv4M2GA="; }; patches = []; diff --git a/contrib/libs/libtiff/README_for_libtiff_developpers.md b/contrib/libs/libtiff/README_for_libtiff_developpers.md new file mode 100644 index 00000000000..6f9b6e4a0bd --- /dev/null +++ b/contrib/libs/libtiff/README_for_libtiff_developpers.md @@ -0,0 +1,149 @@ +README for libtiff developpers +============================== + +## About TIFFGetMaxCompressionRatio() + +That function was introduced in libtiff 4.7.2 per https://gitlab.com/libtiff/libtiff/-/merge_requests/872 + +The estimation of the maximum compression ratio per codec has been performed by +generated a number of test files using [gdal raster create](https://gdal.org/en/stable/programs/gdal_raster_create.html), +with 0-byte content, for different image sizes, and guessing the compression ratio pattern. + +```console +gdal raster create --size 1024,1024 black_1024_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count=3 --size 1024,1024 black_1024_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count=3 --size 1024,1024 black_1024_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 1024,1024 black_1024_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count 3 --size 1024,1024 black_1024_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=1024 --overwrite + +gdal raster create --size 4096,4096 black_4096_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count=3 --size 4096,4096 black_4096_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count=3 --size 4096,4096 black_4096_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 4096,4096 black_4096_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count 3 --size 4096,4096 black_4096_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=4096 --overwrite + +gdal raster create --size 16383,16383 black_16383_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count=3 --size 16383,16383 black_16383_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count=3 --size 16383,16383 black_16383_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 16383,16383 black_16383_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count 4 --size 16383,16383 black_16383_rgba_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count 3 --size 16383,16383 black_16383_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=16383 --overwrite + +gdal raster create --size 65536,65536 black_65536_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=65536 --overwrite +``` + +and then extracted the compression ratios with: + +```console +for i in black*.tif; do bytecounts=$(tiffdump $i | grep StripByteCounts | sed "s/StripByteCounts (279) LONG (4) 1<//" | sed "s/>//"); python3 -c "import math; uncompressed_size = 1024 * 1024 if '1024' in '$i' else 4096 * 4096 if '4096' in '$i' else 65536 * 65536 if '65536' in '$i' else 200000 * 200000 if '200000' in '$i' else 16363 * 16383; uncompressed_size = uncompressed_size * 4 if 'rgba' in '$i' else uncompressed_size * 3 if 'rgb' in '$i' or 'ycbcr' in '$i' else uncompressed_size; uncompressed_size = uncompressed_size / 8 if 'rle' in '$i' or 'fax' in '$i' else uncompressed_size; print('$i', math.ceil(uncompressed_size / ${bytecounts}))"; done +``` + +which outputs + +```console +black_1024_deflate.tif 975 +black_1024_fax3.tif 36 +black_1024_fax4.tif 1001 +black_1024_jpg.tif 252 +black_1024_jxl.tif 5323 +black_1024_lerc_deflate.tif 23832 +black_1024_lerc.tif 14980 +black_1024_lerc_zstd.tif 18397 +black_1024_lzma.tif 3800 +black_1024_lzw.tif 562 +black_1024_packbits.tif 64 +black_1024_rgb_jpg.tif 255 +black_1024_rgb_jxl.tif 16216 +black_1024_rgb_webp.tif 1634 +black_1024_rle.tif 43 +black_1024_ycbcr_jpg.tif 502 +black_1024_zstd.tif 21400 +black_16383_deflate.tif 989 +black_16383_fax3.tif 163 +black_16383_fax4.tif 16339 +black_16383_jpg.tif 256 +black_16383_jxl.tif 45360 +black_16383_lerc_deflate.tif 5703725 +black_16383_lerc.tif 3829644 +black_16383_lerc_zstd.tif 4394673 +black_16383_lzma.tif 6846 +black_16383_lzw.tif 1353 +black_16383_packbits.tif 64 +black_16383_rgba_webp_lossless.tif 104067 +black_16383_rgba_webp.tif 2199 +black_16383_rgb_jpg.tif 256 +black_16383_rgb_jxl.tif 136148 +black_16383_rgb_webp_lossless.tif 78050 +black_16383_rgb_webp.tif 1685 +black_16383_rle.tif 171 +black_16383_ycbcr_jpg.tif 512 +black_16383_zstd.tif 32669 +black_4096_deflate.tif 989 +black_4096_fax3.tif 100 +black_4096_fax4.tif 4073 +black_4096_jpg.tif 256 +black_4096_jxl_rgba.tif 90934 +black_4096_jxl.tif 25079 +black_4096_lerc_deflate.tif 381301 +black_4096_lerc.tif 239675 +black_4096_lerc_zstd.tif 294338 +black_4096_lzma.tif 6534 +black_4096_lzw.tif 1243 +black_4096_packbits.tif 64 +black_4096_rgb_jpg.tif 256 +black_4096_rgb_jxl.tif 75574 +black_4096_rgb_webp.tif 1684 +black_4096_rle.tif 128 +black_4096_ycbcr_jpg.tif 512 +black_4096_zstd.tif 31715 +black_65536_deflate.tif 990 +black_65536_fax3.tif 200 +black_65536_fax4.tif 65513 +black_65536_lzma.tif 6874 +black_65536_lzw.tif 1362 +black_65536_rle.tif 205 +black_65536_zstd.tif 32764 +``` + diff --git a/contrib/libs/libtiff/RELEASE-DATE b/contrib/libs/libtiff/RELEASE-DATE index cc3bc45669d..4fff66f6b5d 100644 --- a/contrib/libs/libtiff/RELEASE-DATE +++ b/contrib/libs/libtiff/RELEASE-DATE @@ -1 +1 @@ -20250911 +20260627 diff --git a/contrib/libs/libtiff/libtiff.map b/contrib/libs/libtiff/libtiff.map index 45641920874..b8a37705eb6 100644 --- a/contrib/libs/libtiff/libtiff.map +++ b/contrib/libs/libtiff/libtiff.map @@ -222,3 +222,15 @@ LIBTIFF_4.6.1 { LIBTIFF_4.7.1 { TIFFOpenOptionsSetWarnAboutUnknownTags; } LIBTIFF_4.6.1; + +LIBTIFF_4.7.2 { + _TIFFAdd64; + _TIFFAddSSize; + _TIFFCastUInt64ToSSize; + _TIFFCastUInt64ToUInt32; + _TIFFComputeBitOffset; + _TIFFComputeRowOffset; + _TIFFComputeRowSize64; + _TIFFMultiplySSize; + TIFFGetMaxCompressionRatio; +} LIBTIFF_4.7.1; diff --git a/contrib/libs/libtiff/tif_aux.c b/contrib/libs/libtiff/tif_aux.c index 31b240070c6..1b2db768670 100644 --- a/contrib/libs/libtiff/tif_aux.c +++ b/contrib/libs/libtiff/tif_aux.c @@ -56,6 +56,21 @@ uint64_t _TIFFMultiply64(TIFF *tif, uint64_t first, uint64_t second, return first * second; } +uint64_t _TIFFAdd64(TIFF *tif, uint64_t first, uint64_t second, + const char *where) +{ + if (first > UINT64_MAX - second) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, "Integer overflow in %s", where); + } + return 0; + } + + return first + second; +} + tmsize_t _TIFFMultiplySSize(TIFF *tif, tmsize_t first, tmsize_t second, const char *where) { @@ -81,6 +96,30 @@ tmsize_t _TIFFMultiplySSize(TIFF *tif, tmsize_t first, tmsize_t second, return first * second; } +tmsize_t _TIFFAddSSize(TIFF *tif, tmsize_t first, tmsize_t second, + const char *where) +{ + if (first < 0 || second < 0) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, + "Invalid argument to _TIFFAddSSize() in %s", where); + } + return 0; + } + + if (first > TIFF_TMSIZE_T_MAX - second) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, "Integer overflow in %s", where); + } + return 0; + } + return first + second; +} + tmsize_t _TIFFCastUInt64ToSSize(TIFF *tif, uint64_t val, const char *module) { if (val > (uint64_t)TIFF_TMSIZE_T_MAX) @@ -94,6 +133,55 @@ tmsize_t _TIFFCastUInt64ToSSize(TIFF *tif, uint64_t val, const char *module) return (tmsize_t)val; } +uint32_t _TIFFCastUInt64ToUInt32(TIFF *tif, uint64_t val, const char *module) +{ + if (val > UINT32_MAX) + { + if (tif != NULL && module != NULL) + { + TIFFErrorExtR(tif, module, "Integer overflow"); + } + return 0; + } + return (uint32_t)val; +} + +tmsize_t _TIFFComputeRowOffset(TIFF *tif, tmsize_t rowstride, uint32_t row, + const char *where) +{ + if (row == 0) + return 0; + return _TIFFMultiplySSize(tif, rowstride, (tmsize_t)row, where); +} + +uint64_t _TIFFComputeBitOffset(TIFF *tif, uint32_t col, uint16_t spp, + uint16_t bps, const char *where) +{ + uint64_t samples = _TIFFMultiply64(tif, col, spp, where); + if (samples == 0 && col != 0) + return 0; + return _TIFFMultiply64(tif, samples, bps, where); +} + +/* + * Returns 0 on overflow or invalid zero-sized row inputs. Callers that + * intentionally allow empty rows should not use this helper directly. + */ +uint64_t _TIFFComputeRowSize64(TIFF *tif, uint32_t width, uint16_t spp, + uint16_t bps, const char *where) +{ + uint64_t samples = _TIFFMultiply64(tif, width, spp, where); + uint64_t bits; + if (samples == 0) + return 0; + + bits = _TIFFMultiply64(tif, samples, bps, where); + if (bits == 0) + return 0; + + return TIFFhowmany8_64(bits); +} + void *_TIFFCheckRealloc(TIFF *tif, void *buffer, tmsize_t nmemb, tmsize_t elem_size, const char *what) { @@ -137,8 +225,8 @@ static int TIFFDefaultTransferFunction(TIFF *tif, TIFFDirectory *td) if (td->td_bitspersample > 24) return 0; - n = ((tmsize_t)1) << td->td_bitspersample; - nbytes = n * sizeof(uint16_t); + n = (tmsize_t)(1ULL << td->td_bitspersample); + nbytes = (tmsize_t)((uint64_t)n * sizeof(uint16_t)); tf[0] = (uint16_t *)_TIFFmallocExt(tif, nbytes); if (tf[0] == NULL) return 0; @@ -186,10 +274,10 @@ static int TIFFDefaultRefBlackWhite(TIFF *tif, TIFFDirectory *td) * YCbCr (Class Y) images must have the ReferenceBlackWhite * tag set. Fix the broken images, which lacks that tag. */ - td->td_refblackwhite[0] = 0.0F; + td->td_refblackwhite[0] = 0.0f; td->td_refblackwhite[1] = td->td_refblackwhite[3] = - td->td_refblackwhite[5] = 255.0F; - td->td_refblackwhite[2] = td->td_refblackwhite[4] = 128.0F; + td->td_refblackwhite[5] = 255.0f; + td->td_refblackwhite[2] = td->td_refblackwhite[4] = 128.0f; } else { @@ -199,8 +287,11 @@ static int TIFFDefaultRefBlackWhite(TIFF *tif, TIFFDirectory *td) for (i = 0; i < 3; i++) { td->td_refblackwhite[2 * i + 0] = 0; - td->td_refblackwhite[2 * i + 1] = - (float)((1L << td->td_bitspersample) - 1L); + if (td->td_bitspersample < 64) + td->td_refblackwhite[2 * i + 1] = + (float)((1ULL << td->td_bitspersample) - 1ULL); + else + td->td_refblackwhite[2 * i + 1] = (float)UINT64_MAX; } } return 1; @@ -258,8 +349,8 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) * 65535 even if td_bitspersamle is > 16 */ if (td->td_bitspersample <= 16) { - maxsamplevalue = (1 << td->td_bitspersample) - - 1; /* 2**(BitsPerSample) - 1 */ + maxsamplevalue = (uint16_t)((1U << td->td_bitspersample) - + 1); /* 2**(BitsPerSample) - 1 */ } else { @@ -295,7 +386,11 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) } case TIFFTAG_DOTRANGE: *va_arg(ap, uint16_t *) = 0; - *va_arg(ap, uint16_t *) = (1 << td->td_bitspersample) - 1; + if (td->td_bitspersample <= 16) + *va_arg(ap, uint16_t *) = + (uint16_t)((1U << td->td_bitspersample) - 1); + else + *va_arg(ap, uint16_t *) = 65535; return (1); case TIFFTAG_INKSET: *va_arg(ap, uint16_t *) = INKSET_CMYK; @@ -309,14 +404,14 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) return (1); case TIFFTAG_MATTEING: *va_arg(ap, uint16_t *) = - (td->td_extrasamples == 1 && + (td->td_extrasamples == 1 && td->td_sampleinfo && td->td_sampleinfo[0] == EXTRASAMPLE_ASSOCALPHA); return (1); case TIFFTAG_TILEDEPTH: *va_arg(ap, uint32_t *) = td->td_tiledepth; return (1); case TIFFTAG_DATATYPE: - *va_arg(ap, uint16_t *) = td->td_sampleformat - 1; + *va_arg(ap, uint16_t *) = (uint16_t)(td->td_sampleformat - 1); return (1); case TIFFTAG_SAMPLEFORMAT: *va_arg(ap, uint16_t *) = td->td_sampleformat; @@ -369,6 +464,8 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) return (0); *va_arg(ap, const float **) = td->td_refblackwhite; return (1); + default: + break; } return 0; } @@ -390,9 +487,9 @@ int TIFFGetFieldDefaulted(TIFF *tif, uint32_t tag, ...) float _TIFFClampDoubleToFloat(double val) { - if (val > FLT_MAX) + if (val > (double)FLT_MAX) return FLT_MAX; - if (val < -FLT_MAX) + if (val < -(double)FLT_MAX) return -FLT_MAX; return (float)val; } @@ -401,7 +498,7 @@ uint32_t _TIFFClampDoubleToUInt32(double val) { if (val < 0) return 0; - if (val > 0xFFFFFFFFU || val != val) + if (val > 0xFFFFFFFFU || isnan(val)) return 0xFFFFFFFFU; return (uint32_t)val; } diff --git a/contrib/libs/libtiff/tif_close.c b/contrib/libs/libtiff/tif_close.c index d498010740c..e60aba7113d 100644 --- a/contrib/libs/libtiff/tif_close.c +++ b/contrib/libs/libtiff/tif_close.c @@ -88,7 +88,7 @@ void TIFFCleanup(TIFF *tif) */ TIFFFieldIsAnonymous(fld)) { - _TIFFfreeExt(tif, fld->field_name); + _TIFFfreeExt(tif, (void *)fld->field_name); _TIFFfreeExt(tif, fld); } } @@ -121,7 +121,7 @@ void TIFFCleanup(TIFF *tif) } /************************************************************************/ -/* _TIFFCleanupIFDOffsetAndNumberMaps() */ +/* _TIFFCleanupIFDOffsetAndNumberMaps() */ /************************************************************************/ void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif) @@ -139,7 +139,7 @@ void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif) } /************************************************************************/ -/* TIFFClose() */ +/* TIFFClose() */ /************************************************************************/ /** diff --git a/contrib/libs/libtiff/tif_color.c b/contrib/libs/libtiff/tif_color.c index a52fdacba59..4a17914a44c 100644 --- a/contrib/libs/libtiff/tif_color.c +++ b/contrib/libs/libtiff/tif_color.c @@ -55,34 +55,34 @@ void TIFFCIELabToXYZ(TIFFCIELabToRGB *cielab, uint32_t l, int32_t a, int32_t b, void TIFFCIELab16ToXYZ(TIFFCIELabToRGB *cielab, uint32_t l, int32_t a, int32_t b, float *X, float *Y, float *Z) { - float L = (float)l * 100.0F / 65535.0F; + float L = (float)l * 100.0f / 65535.0f; float cby, tmp; - if (L < 8.856F) + if (L < 8.856f) { - *Y = (L * cielab->Y0) / 903.292F; - cby = 7.787F * (*Y / cielab->Y0) + 16.0F / 116.0F; + *Y = (L * cielab->Y0) / 903.292f; + cby = 7.787f * (*Y / cielab->Y0) + 16.0f / 116.0f; } else { - cby = (L + 16.0F) / 116.0F; + cby = (L + 16.0f) / 116.0f; *Y = cielab->Y0 * cby * cby * cby; } - tmp = (float)a / 256.0F / 500.0F + cby; - if (tmp < 0.2069F) - *X = cielab->X0 * (tmp - 0.13793F) / 7.787F; + tmp = (float)a / 256.0f / 500.0f + cby; + if (tmp < 0.2069f) + *X = cielab->X0 * (tmp - 0.13793f) / 7.787f; else *X = cielab->X0 * tmp * tmp * tmp; - tmp = cby - (float)b / 256.0F / 200.0F; - if (tmp < 0.2069F) - *Z = cielab->Z0 * (tmp - 0.13793F) / 7.787F; + tmp = cby - (float)b / 256.0f / 200.0f; + if (tmp < 0.2069f) + *Z = cielab->Z0 * (tmp - 0.13793f) / 7.787f; else *Z = cielab->Z0 * tmp * tmp * tmp; } -#define RINT(R) ((uint32_t)((R) > 0 ? ((R) + 0.5) : ((R)-0.5))) +#define RINT(R) ((uint32_t)((R) > 0 ? ((R) + 0.5f) : ((R) - 0.5f))) /* * Convert color value from the XYZ space to RGB. */ @@ -143,32 +143,32 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, _TIFFmemcpy(&cielab->display, display, sizeof(TIFFDisplay)); /* Red */ - dfGamma = 1.0 / cielab->display.d_gammaR; + dfGamma = 1.0 / (double)cielab->display.d_gammaR; cielab->rstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yr2r[i] = cielab->display.d_Vrwr * + cielab->Yr2r[i] = (float)cielab->display.d_Vrwr * ((float)pow((double)i / cielab->range, dfGamma)); } /* Green */ - dfGamma = 1.0 / cielab->display.d_gammaG; + dfGamma = 1.0 / (double)cielab->display.d_gammaG; cielab->gstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yg2g[i] = cielab->display.d_Vrwg * + cielab->Yg2g[i] = (float)cielab->display.d_Vrwg * ((float)pow((double)i / cielab->range, dfGamma)); } /* Blue */ - dfGamma = 1.0 / cielab->display.d_gammaB; + dfGamma = 1.0 / (double)cielab->display.d_gammaB; cielab->bstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yb2b[i] = cielab->display.d_Vrwb * + cielab->Yb2b[i] = (float)cielab->display.d_Vrwb * ((float)pow((double)i / cielab->range, dfGamma)); } @@ -186,11 +186,11 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, * see below for more information on how it works. */ #define SHIFT 16 -#define FIX(x) ((int32_t)((x) * (1L << SHIFT) + 0.5)) +#define FIX(x) ((int32_t)((double)(x) * (1L << SHIFT) + 0.5)) #define ONE_HALF ((int32_t)(1 << (SHIFT - 1))) #define Code2V(c, RB, RW, CR) \ - ((((c) - (int32_t)(RB)) * (float)(CR)) / \ - (float)(((RW) - (RB) != 0) ? ((RW) - (RB)) : 1)) + (((float)((c) - (int32_t)(RB)) * (float)(CR)) / \ + ((!TIFF_FLOAT_EQ((RW), (RB))) ? ((RW) - (RB)) : 1.0f)) /* !((f)>=(min)) written that way to deal with NaN */ #define CLAMP(f, min, max) \ ((!((f) >= (min))) ? (min) : (f) > (max) ? (max) : (f)) @@ -207,12 +207,12 @@ void TIFFYCbCrtoRGB(TIFFYCbCrToRGB *ycbcr, uint32_t Y, int32_t Cb, int32_t Cr, Cr = CLAMP(Cr, 0, 255); i = ycbcr->Y_tab[Y] + ycbcr->Cr_r_tab[Cr]; - *r = CLAMP(i, 0, 255); + *r = (uint32_t)CLAMP(i, 0, 255); i = ycbcr->Y_tab[Y] + - (int)((ycbcr->Cb_g_tab[Cb] + ycbcr->Cr_g_tab[Cr]) >> SHIFT); - *g = CLAMP(i, 0, 255); + ((ycbcr->Cb_g_tab[Cb] + ycbcr->Cr_g_tab[Cr]) >> SHIFT); + *g = (uint32_t)CLAMP(i, 0, 255); i = ycbcr->Y_tab[Y] + ycbcr->Cb_b_tab[Cb]; - *b = CLAMP(i, 0, 255); + *b = (uint32_t)CLAMP(i, 0, 255); } /* Clamp function for sanitization purposes. Normally clamping should not */ @@ -258,8 +258,7 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) #define LumaBlue luma[2] clamptab = - (TIFFRGBValue *)((uint8_t *)ycbcr + - TIFFroundup_32(sizeof(TIFFYCbCrToRGB), sizeof(long))); + (uint8_t *)ycbcr + TIFFroundup_32(sizeof(TIFFYCbCrToRGB), sizeof(long)); _TIFFmemset(clamptab, 0, 256); /* v < 0 => 0 */ ycbcr->clamptab = (clamptab += 256); for (i = 0; i < 256; i++) @@ -273,13 +272,13 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) { float f1 = 2 - 2 * LumaRed; - int32_t D1 = FIX(CLAMP(f1, 0.0F, 2.0F)); + int32_t D1 = FIX(CLAMP(f1, 0.0f, 2.0f)); float f2 = LumaRed * f1 / LumaGreen; - int32_t D2 = -FIX(CLAMP(f2, 0.0F, 2.0F)); + int32_t D2 = -FIX(CLAMP(f2, 0.0f, 2.0f)); float f3 = 2 - 2 * LumaBlue; - int32_t D3 = FIX(CLAMP(f3, 0.0F, 2.0F)); + int32_t D3 = FIX(CLAMP(f3, 0.0f, 2.0f)); float f4 = LumaBlue * f3 / LumaGreen; - int32_t D4 = -FIX(CLAMP(f4, 0.0F, 2.0F)); + int32_t D4 = -FIX(CLAMP(f4, 0.0f, 2.0f)); int x; #undef LumaBlue @@ -295,12 +294,12 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) */ for (i = 0, x = -128; i < 256; i++, x++) { - int32_t Cr = (int32_t)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F, - refBlackWhite[5] - 128.0F, 127), - -128.0F * 32, 128.0F * 32); - int32_t Cb = (int32_t)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F, - refBlackWhite[3] - 128.0F, 127), - -128.0F * 32, 128.0F * 32); + int32_t Cr = (int32_t)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0f, + refBlackWhite[5] - 128.0f, 127), + -128.0f * 32, 128.0f * 32); + int32_t Cb = (int32_t)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0f, + refBlackWhite[3] - 128.0f, 127), + -128.0f * 32, 128.0f * 32); ycbcr->Cr_r_tab[i] = (int32_t)((D1 * Cr + ONE_HALF) >> SHIFT); ycbcr->Cb_b_tab[i] = (int32_t)((D3 * Cb + ONE_HALF) >> SHIFT); @@ -308,7 +307,7 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) ycbcr->Cb_g_tab[i] = D4 * Cb + ONE_HALF; ycbcr->Y_tab[i] = (int32_t)CLAMPw( Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255), - -128.0F * 32, 128.0F * 32); + -128.0f * 32, 128.0f * 32); } } diff --git a/contrib/libs/libtiff/tif_compress.c b/contrib/libs/libtiff/tif_compress.c index c6e17d3e114..3002083f12e 100644 --- a/contrib/libs/libtiff/tif_compress.c +++ b/contrib/libs/libtiff/tif_compress.c @@ -139,6 +139,18 @@ static int _TIFFtrue(TIFF *tif) } static void _TIFFvoid(TIFF *tif) { (void)tif; } +static uint64_t _TIFFDefaultGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + return 0; /* unknown */ +} + +static uint64_t _TIFFGetMaxCompressionRatioOne(TIFF *tif) +{ + (void)tif; + return 1; /* no compression */ +} + void _TIFFSetDefaultCompressionState(TIFF *tif) { tif->tif_fixuptags = _TIFFNoFixupTags; @@ -160,6 +172,7 @@ void _TIFFSetDefaultCompressionState(TIFF *tif) tif->tif_cleanup = _TIFFvoid; tif->tif_defstripsize = _TIFFDefaultStripSize; tif->tif_deftilesize = _TIFFDefaultTileSize; + tif->tif_getmaxcompressionratio = _TIFFDefaultGetMaxCompressionRatio; tif->tif_flags &= ~(TIFF_NOBITREV | TIFF_NOREADRAW); } @@ -168,6 +181,8 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme) const TIFFCodec *c = TIFFFindCODEC((uint16_t)scheme); _TIFFSetDefaultCompressionState(tif); + if (scheme == COMPRESSION_NONE) + tif->tif_getmaxcompressionratio = _TIFFGetMaxCompressionRatioOne; /* * Don't treat an unknown compression scheme as an error. * This permits applications to open files with data that @@ -177,6 +192,13 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme) return (c ? (*c->init)(tif, scheme) : 1); } +uint64_t TIFFGetMaxCompressionRatio(TIFF *tif) +{ + if (tif->tif_getmaxcompressionratio) + return tif->tif_getmaxcompressionratio(tif); + return 0; +} + /* * Other compression schemes may be registered. Registered * schemes can also override the builtin versions provided @@ -200,7 +222,7 @@ const TIFFCodec *TIFFFindCODEC(uint16_t scheme) for (c = _TIFFBuiltinCODECS; c->name; c++) if (c->scheme == scheme) return (c); - return ((const TIFFCodec *)0); + return NULL; } TIFFCodec *TIFFRegisterCODEC(uint16_t scheme, const char *name, @@ -212,9 +234,11 @@ TIFFCodec *TIFFRegisterCODEC(uint16_t scheme, const char *name, if (cd != NULL) { + char *codec_name; cd->info = (TIFFCodec *)((uint8_t *)cd + sizeof(codec_t)); - cd->info->name = (char *)((uint8_t *)cd->info + sizeof(TIFFCodec)); - strcpy(cd->info->name, name); + codec_name = (char *)((uint8_t *)cd->info + sizeof(TIFFCodec)); + strcpy(codec_name, name); + cd->info->name = codec_name; cd->info->scheme = scheme; cd->info->init = init; cd->next = registeredCODECS; @@ -247,7 +271,7 @@ void TIFFUnRegisterCODEC(TIFFCodec *c) } /************************************************************************/ -/* TIFFGetConfisuredCODECs() */ +/* TIFFGetConfiguredCODECs() */ /************************************************************************/ /** @@ -258,7 +282,7 @@ void TIFFUnRegisterCODEC(TIFFCodec *c) * or NULL if function failed. */ -TIFFCodec *TIFFGetConfiguredCODECs() +TIFFCodec *TIFFGetConfiguredCODECs(void) { int i = 1; codec_t *cd; @@ -268,8 +292,8 @@ TIFFCodec *TIFFGetConfiguredCODECs() for (cd = registeredCODECS; cd; cd = cd->next) { - new_codecs = - (TIFFCodec *)_TIFFreallocExt(NULL, codecs, i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); @@ -283,21 +307,21 @@ TIFFCodec *TIFFGetConfiguredCODECs() { if (TIFFIsCODECConfigured(c->scheme)) { - new_codecs = (TIFFCodec *)_TIFFreallocExt(NULL, codecs, - i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); return NULL; } codecs = new_codecs; - _TIFFmemcpy(codecs + i - 1, (const void *)c, sizeof(TIFFCodec)); + _TIFFmemcpy(codecs + i - 1, c, sizeof(TIFFCodec)); i++; } } - new_codecs = - (TIFFCodec *)_TIFFreallocExt(NULL, codecs, i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); diff --git a/contrib/libs/libtiff/tif_config.h b/contrib/libs/libtiff/tif_config.h index a4c90a65fbf..693d661d991 100644 --- a/contrib/libs/libtiff/tif_config.h +++ b/contrib/libs/libtiff/tif_config.h @@ -133,15 +133,7 @@ /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* #undef WORDS_BIGENDIAN */ -# endif -#endif +#define WORDS_BIGENDIAN 0 #if !defined(__MINGW32__) # define TIFF_SIZE_FORMAT "zu" diff --git a/contrib/libs/libtiff/tif_dir.c b/contrib/libs/libtiff/tif_dir.c index 7421c738db2..999a2452f33 100644 --- a/contrib/libs/libtiff/tif_dir.c +++ b/contrib/libs/libtiff/tif_dir.c @@ -31,6 +31,7 @@ #include "tiffiop.h" #include <float.h> /*--: for Rational2Double */ #include <limits.h> +#include <math.h> /* * These are used in the backwards compatibility code... @@ -50,7 +51,8 @@ static void setByteArray(TIFF *tif, void **vpp, const void *vp, size_t nmemb, } if (vp) { - tmsize_t bytes = _TIFFMultiplySSize(NULL, nmemb, elem_size, NULL); + tmsize_t bytes = _TIFFMultiplySSize(NULL, (tmsize_t)nmemb, + (tmsize_t)elem_size, NULL); if (bytes) *vpp = (void *)_TIFFmallocExt(tif, bytes); if (*vpp) @@ -121,7 +123,8 @@ static void setDoubleArrayOneValue(TIFF *tif, double **vpp, double value, { if (*vpp) _TIFFfreeExt(tif, *vpp); - *vpp = _TIFFmallocExt(tif, nmemb * sizeof(double)); + *vpp = (double *)_TIFFmallocExt(tif, + (tmsize_t)nmemb * (tmsize_t)sizeof(double)); if (*vpp) { while (nmemb--) @@ -294,7 +297,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) /* * Setup new compression routine state. */ - if ((status = TIFFSetCompressionScheme(tif, v)) != 0) + if ((status = TIFFSetCompressionScheme(tif, (int)v)) != 0) td->td_compression = (uint16_t)v; else status = 0; @@ -402,13 +405,13 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) break; case TIFFTAG_XRESOLUTION: dblval = va_arg(ap, double); - if (dblval != dblval || dblval < 0) + if (isnan(dblval) || dblval < 0) goto badvaluedouble; td->td_xresolution = _TIFFClampDoubleToFloat(dblval); break; case TIFFTAG_YRESOLUTION: dblval = va_arg(ap, double); - if (dblval != dblval || dblval < 0) + if (isnan(dblval) || dblval < 0) goto badvaluedouble; td->td_yresolution = _TIFFClampDoubleToFloat(dblval); break; @@ -439,7 +442,12 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) td->td_halftonehints[1] = (uint16_t)va_arg(ap, uint16_vap); break; case TIFFTAG_COLORMAP: - v32 = (uint32_t)(1L << td->td_bitspersample); + if (td->td_bitspersample >= 32) + { + v = td->td_bitspersample; + goto badvalue; + } + v32 = 1U << td->td_bitspersample; _TIFFsetShortArrayExt(tif, &td->td_colormap[0], va_arg(ap, uint16_t *), v32); _TIFFsetShortArrayExt(tif, &td->td_colormap[1], @@ -557,11 +565,17 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) case TIFFTAG_TRANSFERFUNCTION: { uint32_t i; + uint32_t count; + if (td->td_bitspersample >= 32) + { + v = td->td_bitspersample; + goto badvalue; + } + count = 1U << td->td_bitspersample; v = (td->td_samplesperpixel - td->td_extrasamples) > 1 ? 3 : 1; for (i = 0; i < v; i++) _TIFFsetShortArrayExt(tif, &td->td_transferfunction[i], - va_arg(ap, uint16_t *), - 1U << td->td_bitspersample); + va_arg(ap, uint16_t *), count); break; } case TIFFTAG_REFERENCEBLACKWHITE: @@ -579,7 +593,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (ninksinstring > 0) { _TIFFsetNString(tif, &td->td_inknames, s, v); - td->td_inknameslen = v; + td->td_inknameslen = (int)v; /* Set NumberOfInks to the value ninksinstring */ if (TIFFFieldSet(tif, FIELD_NUMBEROFINKS)) { @@ -725,7 +739,8 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) new_customValues = (TIFFTagValue *)_TIFFreallocExt( tif, td->td_customValues, - sizeof(TIFFTagValue) * (td->td_customValueCount + 1)); + (tmsize_t)(sizeof(TIFFTagValue) * + (size_t)(td->td_customValueCount + 1))); if (!new_customValues) { TIFFErrorExtR(tif, module, @@ -754,7 +769,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (tv_size == 0) { status = 0; - TIFFErrorExtR(tif, module, "%s: Bad field type %d for \"%s\"", + TIFFErrorExtR(tif, module, "%s: Bad field type %u for \"%s\"", tif->tif_name, fip->field_type, fip->field_name); goto end; } @@ -784,7 +799,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) } ma = (uint32_t)len; } - tv->count = ma; + tv->count = (int)ma; setByteArray(tif, &tv->value, mb, ma, 1); } else @@ -792,9 +807,9 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (fip->field_passcount) { if (fip->field_writecount == TIFF_VARIABLE2) - tv->count = (uint32_t)va_arg(ap, uint32_t); + tv->count = (int)va_arg(ap, uint32_t); else - tv->count = (int)va_arg(ap, int); + tv->count = va_arg(ap, int); } else if (fip->field_writecount == TIFF_VARIABLE || fip->field_writecount == TIFF_VARIABLE2) @@ -808,7 +823,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) { TIFFWarningExtR(tif, module, "%s: Null count for \"%s\" (type " - "%d, writecount %d, passcount %d)", + "%u, writecount %d, passcount %d)", tif->tif_name, fip->field_name, fip->field_type, fip->field_writecount, fip->field_passcount); @@ -845,11 +860,12 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) * 4 or 8 according to fip->set_get_field_type! */ _TIFFmemcpy(tv->value, va_arg(ap, void *), tv->count * tv_size); - /* Test here for too big values for LONG8, SLONG8 in + /* Test here for too big values for LONG8, IFD8, SLONG8 in * ClassicTIFF and delete custom field from custom list */ if (!(tif->tif_flags & TIFF_BIGTIFF)) { - if (tv->info->field_type == TIFF_LONG8) + if (tv->info->field_type == TIFF_LONG8 || + tv->info->field_type == TIFF_IFD8) { uint64_t *pui64 = (uint64_t *)tv->value; for (int i = 0; i < tv->count; i++) @@ -858,12 +874,15 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) { TIFFErrorExtR( tif, module, - "%s: Bad LONG8 value %" PRIu64 + "%s: Bad %s value %" PRIu64 " at %d. array position for \"%s\" tag " - "%d in ClassicTIFF. Tag won't be " + "%u in ClassicTIFF. Tag won't be " "written to file", - tif->tif_name, pui64[i], i, - fip->field_name, tag); + tif->tif_name, + (tv->info->field_type == TIFF_LONG8 + ? "LONG8" + : "IFD8"), + pui64[i], i, fip->field_name, tag); goto badvalueifd8long8; } } @@ -880,7 +899,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) tif, module, "%s: Bad SLONG8 value %" PRIi64 " at %d. array position for \"%s\" tag " - "%d in ClassicTIFF. Tag won't be " + "%u in ClassicTIFF. Tag won't be " "written to file", tif->tif_name, pi64[i], i, fip->field_name, tag); @@ -948,7 +967,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFErrorExtR( tif, module, "%s: Bad LONG8 or IFD8 value %" PRIu64 - " for \"%s\" tag %d in ClassicTIFF. Tag " + " for \"%s\" tag %u in ClassicTIFF. Tag " "won't be written to file", tif->tif_name, v2, fip->field_name, tag); goto badvalueifd8long8; @@ -967,7 +986,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFErrorExtR( tif, module, "%s: Bad SLONG8 value %" PRIi64 - " for \"%s\" tag %d in ClassicTIFF. Tag " + " for \"%s\" tag %u in ClassicTIFF. Tag " "won't be written to file", tif->tif_name, v2, fip->field_name, tag); goto badvalueifd8long8; @@ -1018,6 +1037,8 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) _TIFFmemcpy(val, &v2, tv_size); } break; + case TIFF_NOTYPE: + case TIFF_ASCII: default: _TIFFmemset(val, 0, tv_size); status = 0; @@ -1345,7 +1366,7 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) break; case TIFFTAG_MATTEING: *va_arg(ap, uint16_t *) = - (td->td_extrasamples == 1 && + (td->td_extrasamples == 1 && td->td_sampleinfo && td->td_sampleinfo[0] == EXTRASAMPLE_ASSOCALPHA); break; case TIFFTAG_EXTRASAMPLES: @@ -1376,6 +1397,8 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) case SAMPLEFORMAT_VOID: *va_arg(ap, uint16_t *) = DATATYPE_VOID; break; + default: + break; } break; case TIFFTAG_SAMPLEFORMAT: @@ -1562,6 +1585,8 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) *va_arg(ap, double *) = *(double *)val; ret_val = 1; break; + case TIFF_NOTYPE: + case TIFF_ASCII: default: ret_val = 0; break; @@ -1614,6 +1639,21 @@ int TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) } /* + * Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. + */ +void _TIFFResetTifDirAndInitStrileCounters(TIFFDirectory *td) +{ + _TIFFmemset(td, 0, sizeof(*td)); + td->td_curstrip = NOSTRIP; /* invalid strip = NOSTRIP */ + td->td_row = (uint32_t)-1; /* read/write pre-increment */ + td->td_col = (uint32_t)-1; /* read/write pre-increment */ + td->td_scanlinesize = 0; /* initialize to zero */ + td->td_curtile = NOTILE; /* invalid tile = NOTILE */ + td->td_tilesize = (tmsize_t)-1; /* invalidate tilezize */ +} + +/* * Release storage associated with a directory. */ void TIFFFreeDirectory(TIFF *tif) @@ -1664,6 +1704,7 @@ void TIFFFreeDirectory(TIFF *tif) tif->tif_dir.td_dirdatasize_Noffsets = 0; } tif->tif_dir.td_iswrittentofile = FALSE; + /* Note: tif->tif_dir structure is set to zero in TIFFDefaultDirectory() */ } #undef CleanupField @@ -1694,10 +1735,7 @@ int TIFFCreateDirectory(TIFF *tif) tif->tif_diroff = 0; tif->tif_nextdiroff = 0; tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; tif->tif_dir.td_iswrittentofile = FALSE; - return 0; } @@ -1717,15 +1755,12 @@ int TIFFCreateCustomDirectory(TIFF *tif, const TIFFFieldArray *infoarray) tif->tif_diroff = 0; tif->tif_nextdiroff = 0; tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; /* invalidate directory index */ tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; /* invalidate IFD loop lists */ _TIFFCleanupIFDOffsetAndNumberMaps(tif); /* To be able to return from SubIFD or custom-IFD to main-IFD */ tif->tif_setdirectory_force_absolute = TRUE; - return 0; } @@ -1751,13 +1786,14 @@ int TIFFCreateGPSDirectory(TIFF *tif) */ int TIFFDefaultDirectory(TIFF *tif) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; const TIFFFieldArray *tiffFieldArray; tiffFieldArray = _TIFFGetFields(); _TIFFSetupFields(tif, tiffFieldArray); - - _TIFFmemset(td, 0, sizeof(*td)); + /* Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. */ + _TIFFResetTifDirAndInitStrileCounters(td); td->td_fillorder = FILLORDER_MSB2LSB; td->td_bitspersample = 1; td->td_threshholding = THRESHHOLD_BILEVEL; @@ -1860,10 +1896,8 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, tmsize_t poffa, poffb, poffc, poffd; uint16_t dircount; uint32_t nextdir32; - poffa = (tmsize_t)poff; - poffb = poffa + sizeof(uint16_t); - if (((uint64_t)poffa != poff) || (poffb < poffa) || - (poffb < (tmsize_t)sizeof(uint16_t)) || (poffb > tif->tif_size)) + if (poff > (uint64_t)TIFF_TMSIZE_T_MAX - sizeof(uint16_t) || + poff > (uint64_t)tif->tif_size - sizeof(uint16_t)) { TIFFErrorExtR(tif, module, "%s:%d: %s: Error fetching directory count", @@ -1871,13 +1905,20 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, *nextdiroff = 0; return (0); } + poffa = (tmsize_t)poff; + poffb = poffa + (tmsize_t)sizeof(uint16_t); _TIFFmemcpy(&dircount, tif->tif_base + poffa, sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); + if (poffb > + TIFF_TMSIZE_T_MAX - dircount * 12 - (tmsize_t)sizeof(uint32_t)) + { + TIFFErrorExtR(tif, module, "Error fetching directory link"); + return (0); + } poffc = poffb + dircount * 12; - poffd = poffc + sizeof(uint32_t); - if ((poffc < poffb) || (poffc < dircount * 12) || (poffd < poffc) || - (poffd < (tmsize_t)sizeof(uint32_t)) || (poffd > tif->tif_size)) + poffd = poffc + (tmsize_t)sizeof(uint32_t); + if (poffd > tif->tif_size) { TIFFErrorExtR(tif, module, "Error fetching directory link"); return (0); @@ -1893,7 +1934,6 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, { tmsize_t poffa, poffb, poffc, poffd; uint64_t dircount64; - uint16_t dircount16; if (poff > (uint64_t)TIFF_TMSIZE_T_MAX - sizeof(uint64_t)) { TIFFErrorExtR(tif, module, @@ -1902,7 +1942,7 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, return (0); } poffa = (tmsize_t)poff; - poffb = poffa + sizeof(uint64_t); + poffb = poffa + (tmsize_t)sizeof(uint64_t); if (poffb > tif->tif_size) { TIFFErrorExtR(tif, module, @@ -1919,15 +1959,14 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, "Sanity check on directory count failed"); return (0); } - dircount16 = (uint16_t)dircount64; - if (poffb > TIFF_TMSIZE_T_MAX - (tmsize_t)(dircount16 * 20) - + if (poffb > TIFF_TMSIZE_T_MAX - (tmsize_t)(dircount64 * 20) - (tmsize_t)sizeof(uint64_t)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); return (0); } - poffc = poffb + dircount16 * 20; - poffd = poffc + sizeof(uint64_t); + poffc = poffb + (tmsize_t)(dircount64 * 20); + poffd = poffc + (tmsize_t)sizeof(uint64_t); if (poffd > tif->tif_size) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -1957,9 +1996,9 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); if (off != NULL) - *off = TIFFSeekFile(tif, dircount * 12, SEEK_CUR); + *off = TIFFSeekFile(tif, dircount * 12U, SEEK_CUR); else - (void)TIFFSeekFile(tif, dircount * 12, SEEK_CUR); + (void)TIFFSeekFile(tif, dircount * 12U, SEEK_CUR); if (!ReadOK(tif, &nextdir32, sizeof(uint32_t))) { TIFFErrorExtR(tif, module, "%s: Error fetching directory link", @@ -1973,7 +2012,6 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, else { uint64_t dircount64; - uint16_t dircount16; if (!SeekOK(tif, *nextdiroff) || !ReadOK(tif, &dircount64, sizeof(uint64_t))) { @@ -1991,11 +2029,10 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, __FILE__, __LINE__, tif->tif_name); return (0); } - dircount16 = (uint16_t)dircount64; if (off != NULL) - *off = TIFFSeekFile(tif, dircount16 * 20, SEEK_CUR); + *off = TIFFSeekFile(tif, dircount64 * 20, SEEK_CUR); else - (void)TIFFSeekFile(tif, dircount16 * 20, SEEK_CUR); + (void)TIFFSeekFile(tif, dircount64 * 20, SEEK_CUR); if (!ReadOK(tif, nextdiroff, sizeof(uint64_t))) { TIFFErrorExtR(tif, module, "%s: Error fetching directory link", @@ -2354,8 +2391,6 @@ int TIFFUnlinkDirectory(TIFF *tif, tdir_t dirn) tif->tif_nextdiroff = 0; /* next write must be at end */ tif->tif_lastdiroff = 0; /* will be updated on next link */ tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; if (tif->tif_curdircount > 0) tif->tif_curdircount--; diff --git a/contrib/libs/libtiff/tif_dir.h b/contrib/libs/libtiff/tif_dir.h index 44270f70970..aed97222288 100644 --- a/contrib/libs/libtiff/tif_dir.h +++ b/contrib/libs/libtiff/tif_dir.h @@ -104,6 +104,16 @@ typedef struct uint16_t td_halftonehints[2]; uint16_t td_extrasamples; uint16_t *td_sampleinfo; + /* strip support */ + uint32_t td_row; /* current scanline */ + uint32_t td_curstrip; /* current strip for read/write */ + tmsize_t td_scanlinesize; /* # of bytes in a scanline */ +#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ + /* tiling support */ + uint32_t td_col; /* current column (offset by row too) */ + uint32_t td_curtile; /* current tile for read/write */ + tmsize_t td_tilesize; /* # of bytes in a tile */ +#define NOTILE ((uint32_t)(-1)) /* undefined state */ /* even though the name is misleading, td_stripsperimage is the number * of striles (=strips or tiles) per plane, and td_nstrips the total * number of striles */ @@ -164,6 +174,8 @@ typedef struct entries. */ } TIFFDirectory; +extern void _TIFFResetTifDirAndInitStrileCounters(TIFFDirectory *td); + /* * Field flags used to indicate fields that have been set in a directory, and * to reference fields when manipulating a directory. @@ -231,7 +243,7 @@ typedef struct #define FIELD_LAST (32 * FIELDSET_ITEMS - 1) -#define BITn(n) (((uint32_t)1L) << ((n)&0x1f)) +#define BITn(n) (1U << ((n) & 0x1f)) #define BITFIELDn(tif, n) ((tif)->tif_dir.td_fieldsset[(n) / 32]) #define TIFFFieldSet(tif, field) (BITFIELDn(tif, field) & BITn(field)) #define TIFFSetFieldBit(tif, field) (BITFIELDn(tif, field) |= BITn(field)) @@ -340,7 +352,7 @@ extern "C" unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ - char *field_name; /* ASCII name */ + const char *field_name; /* ASCII name */ TIFFFieldArray *field_subfields; /* if field points to child ifds, child ifd field definition array */ }; diff --git a/contrib/libs/libtiff/tif_dirinfo.c b/contrib/libs/libtiff/tif_dirinfo.c index c038497cf49..9218661b723 100644 --- a/contrib/libs/libtiff/tif_dirinfo.c +++ b/contrib/libs/libtiff/tif_dirinfo.c @@ -40,17 +40,47 @@ * values accordingly. */ -/* const object should be initialized */ -#ifdef _MSC_VER +/* Forward declarations - definition follows after field arrays. + * Note: In C, we can forward declare static const objects and define them + * later. In C++, we need extern for the declaration, then define without + * extern. Since these are only used within this file via pointers, we use a + * workaround that works in both C and C++: declare as extern here, define as + * static later, but actually we need a different approach for C++ + * compatibility. + * + * For C/C++ compatibility, we define a simple struct that holds the pointer + * and initialize it after the arrays are defined. + */ +#ifdef __cplusplus +/* C++ doesn't allow forward declaration of const objects, so we use extern */ +extern const TIFFFieldArray tiffFieldArray; +extern const TIFFFieldArray exifFieldArray; +extern const TIFFFieldArray gpsFieldArray; +#else +/* C allows forward declaration of const objects, but C++ doesn't. + * We disable the C++-compat warning for this section since these circular + * dependencies are unavoidable with static initialization. */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wc++-compat" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++-compat" +#elif defined(_MSC_VER) #pragma warning(push) #pragma warning(disable : 4132) #endif static const TIFFFieldArray tiffFieldArray; static const TIFFFieldArray exifFieldArray; static const TIFFFieldArray gpsFieldArray; -#ifdef _MSC_VER +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) #pragma warning(pop) #endif +#endif /*--: Rational2Double: -- * The Rational2Double upgraded libtiff functionality allows the definition and * achievement of true double-precision accuracy for TIFF tags of RATIONAL type @@ -152,9 +182,9 @@ static const TIFFField tiffFields[] = { /*--: EXIFIFD and GPSIFD specified as TIFF_LONG by Aware-Systems and not TIFF_IFD8 as in original LibTiff. However, for IFD-like tags, * libtiff uses the data type TIFF_IFD8 in tiffFields[]-tag definition combined with a special handling procedure in order to write either * a 32-bit value and the TIFF_IFD type-id into ClassicTIFF files or a 64-bit value and the TIFF_IFD8 type-id into BigTIFF files. */ - {TIFFTAG_EXIFIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_IFD8, FIELD_CUSTOM, 1, 0, "EXIFIFDOffset", (TIFFFieldArray *)&exifFieldArray}, + {TIFFTAG_EXIFIFD, 1, 1, TIFF_LONG8, 0, TIFF_SETGET_UINT64, FIELD_CUSTOM, 1, 0, "EXIFIFDOffset", (TIFFFieldArray *)&exifFieldArray}, {TIFFTAG_ICCPROFILE, -3, -3, TIFF_UNDEFINED, 0, TIFF_SETGET_C32_UINT8, FIELD_CUSTOM, 1, 1, "ICC Profile", NULL}, - {TIFFTAG_GPSIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_IFD8, FIELD_CUSTOM, 1, 0, "GPSIFDOffset", (TIFFFieldArray *)&gpsFieldArray}, + {TIFFTAG_GPSIFD, 1, 1, TIFF_LONG8, 0, TIFF_SETGET_UINT64, FIELD_CUSTOM, 1, 0, "GPSIFDOffset", (TIFFFieldArray *)&gpsFieldArray}, {TIFFTAG_FAXRECVPARAMS, 1, 1, TIFF_LONG, 0, TIFF_SETGET_UINT32, FIELD_CUSTOM, TRUE, FALSE, "FaxRecvParams", NULL}, {TIFFTAG_FAXSUBADDRESS, -1, -1, TIFF_ASCII, 0, TIFF_SETGET_ASCII, FIELD_CUSTOM, TRUE, FALSE, "FaxSubAddress", NULL}, {TIFFTAG_FAXRECVTIME, 1, 1, TIFF_LONG, 0, TIFF_SETGET_UINT32, FIELD_CUSTOM, TRUE, FALSE, "FaxRecvTime", NULL}, @@ -479,12 +509,36 @@ static const TIFFField gpsFields[] = { {GPSTAG_GPSHPOSITIONINGERROR, 1, 1, TIFF_RATIONAL, 0, TIFF_SETGET_DOUBLE, FIELD_CUSTOM, 1, 0, "HorizontalPositioningError", NULL}}; /* clang-format on */ /* was off for better readability of tag comments */ +#ifdef __cplusplus +/* In C++, the forward declaration used extern, so definitions must not be + * static */ +const TIFFFieldArray tiffFieldArray = { + tfiatImage, 0, TIFFArrayCount(tiffFields), (TIFFField *)tiffFields}; +const TIFFFieldArray exifFieldArray = {tfiatExif, 0, TIFFArrayCount(exifFields), + (TIFFField *)exifFields}; +const TIFFFieldArray gpsFieldArray = {tfiatGps, 0, TIFFArrayCount(gpsFields), + (TIFFField *)gpsFields}; +#else +/* Suppress C++-compat warning for the definitions as well */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wc++-compat" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++-compat" +#endif static const TIFFFieldArray tiffFieldArray = { tfiatImage, 0, TIFFArrayCount(tiffFields), (TIFFField *)tiffFields}; static const TIFFFieldArray exifFieldArray = { tfiatExif, 0, TIFFArrayCount(exifFields), (TIFFField *)exifFields}; static const TIFFFieldArray gpsFieldArray = { tfiatGps, 0, TIFFArrayCount(gpsFields), (TIFFField *)gpsFields}; +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif +#endif /* * We have our own local lfind() equivalent to avoid subtle differences @@ -523,7 +577,7 @@ void _TIFFSetupFields(TIFF *tif, const TIFFFieldArray *fieldarray) { if (fld->field_bit == FIELD_CUSTOM && TIFFFieldIsAnonymous(fld)) { - _TIFFfreeExt(tif, fld->field_name); + _TIFFfreeExt(tif, (void *)fld->field_name); /* caution: tif_fields[i] must not be the beginning of a * fields-array. Otherwise the following tags are also freed * with the first free(). @@ -579,22 +633,26 @@ int _TIFFMergeFields(TIFF *tif, const TIFFField info[], uint32_t n) tif->tif_foundfield = NULL; + TIFFField **tif_newfields = NULL; + if (tif->tif_fields && tif->tif_nfields > 0) { - tif->tif_fields = (TIFFField **)_TIFFCheckRealloc( - tif, tif->tif_fields, (tif->tif_nfields + n), sizeof(TIFFField *), - reason); + tif_newfields = (TIFFField **)_TIFFCheckRealloc( + tif, tif->tif_fields, (tmsize_t)tif->tif_nfields + n, + (tmsize_t)sizeof(TIFFField *), reason); } else { - tif->tif_fields = + tif_newfields = (TIFFField **)_TIFFCheckMalloc(tif, n, sizeof(TIFFField *), reason); } - if (!tif->tif_fields) + if (!tif_newfields) { + tif->tif_nfields = 0; TIFFErrorExtR(tif, module, "Failed to allocate fields array"); return 0; } + tif->tif_fields = tif_newfields; /* tp = tif->tif_fields + tif->tif_nfields; */ for (i = 0; i < n; i++) @@ -612,7 +670,7 @@ int _TIFFMergeFields(TIFF *tif, const TIFFField info[], uint32_t n) /* Sort the field info by tag number */ qsort(tif->tif_fields, tif->tif_nfields, sizeof(TIFFField *), tagCompare); - return n; + return (int)n; } void _TIFFPrintFieldInfo(TIFF *tif, FILE *fd) @@ -623,7 +681,7 @@ void _TIFFPrintFieldInfo(TIFF *tif, FILE *fd) for (i = 0; i < tif->tif_nfields; i++) { const TIFFField *fip = tif->tif_fields[i]; - fprintf(fd, "field[%2d] %5lu, %2d, %2d, %d, %2d, %5s, %5s, %s\n", + fprintf(fd, "field[%2d] %5lu, %2d, %2d, %u, %2d, %5s, %5s, %s\n", (int)i, (unsigned long)fip->field_tag, fip->field_readcount, fip->field_writecount, fip->field_type, fip->field_bit, fip->field_oktochange ? "TRUE" : "FALSE", @@ -787,6 +845,34 @@ int TIFFFieldSetGetCountSize(const TIFFField *fip) case TIFF_SETGET_C32_DOUBLE: case TIFF_SETGET_C32_IFD8: return 4; + case TIFF_SETGET_UNDEFINED: + case TIFF_SETGET_ASCII: + case TIFF_SETGET_UINT8: + case TIFF_SETGET_SINT8: + case TIFF_SETGET_UINT16: + case TIFF_SETGET_SINT16: + case TIFF_SETGET_UINT32: + case TIFF_SETGET_SINT32: + case TIFF_SETGET_UINT64: + case TIFF_SETGET_SINT64: + case TIFF_SETGET_FLOAT: + case TIFF_SETGET_DOUBLE: + case TIFF_SETGET_IFD8: + case TIFF_SETGET_INT: + case TIFF_SETGET_UINT16_PAIR: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_C0_UINT8: + case TIFF_SETGET_C0_SINT8: + case TIFF_SETGET_C0_UINT16: + case TIFF_SETGET_C0_SINT16: + case TIFF_SETGET_C0_UINT32: + case TIFF_SETGET_C0_SINT32: + case TIFF_SETGET_C0_UINT64: + case TIFF_SETGET_C0_SINT64: + case TIFF_SETGET_C0_FLOAT: + case TIFF_SETGET_C0_DOUBLE: + case TIFF_SETGET_C0_IFD8: + case TIFF_SETGET_OTHER: default: return 0; } @@ -794,7 +880,8 @@ int TIFFFieldSetGetCountSize(const TIFFField *fip) const TIFFField *TIFFFindField(TIFF *tif, uint32_t tag, TIFFDataType dt) { - TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, 0, 0, 0, 0, NULL, NULL}; + TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, TIFF_SETGET_UNDEFINED, + 0, 0, 0, NULL, NULL}; TIFFField *pkey = &key; const TIFFField **ret; if (tif->tif_foundfield && tif->tif_foundfield->field_tag == tag && @@ -818,7 +905,8 @@ const TIFFField *TIFFFindField(TIFF *tif, uint32_t tag, TIFFDataType dt) static const TIFFField *_TIFFFindFieldByName(TIFF *tif, const char *field_name, TIFFDataType dt) { - TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, 0, 0, 0, 0, NULL, NULL}; + TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, TIFF_SETGET_UNDEFINED, + 0, 0, 0, NULL, NULL}; TIFFField *pkey = &key; const TIFFField **ret; if (tif->tif_foundfield && @@ -876,7 +964,10 @@ int TIFFFieldReadCount(const TIFFField *fip) { return fip->field_readcount; } int TIFFFieldWriteCount(const TIFFField *fip) { return fip->field_writecount; } -int TIFFFieldIsAnonymous(const TIFFField *fip) { return fip->field_anonymous; } +int TIFFFieldIsAnonymous(const TIFFField *fip) +{ + return (int)fip->field_anonymous; +} const TIFFField *_TIFFFindOrRegisterField(TIFF *tif, uint32_t tag, TIFFDataType dt) @@ -954,6 +1045,7 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, case TIFF_SLONG8: fld->set_get_field_type = TIFF_SETGET_C32_SINT64; break; + case TIFF_NOTYPE: default: fld->set_get_field_type = TIFF_SETGET_UNDEFINED; break; @@ -961,8 +1053,8 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, fld->field_bit = FIELD_CUSTOM; fld->field_oktochange = TRUE; fld->field_passcount = TRUE; - fld->field_name = (char *)_TIFFmallocExt(tif, 32); - if (fld->field_name == NULL) + char *field_name_buf = (char *)_TIFFmallocExt(tif, 32); + if (field_name_buf == NULL) { _TIFFfreeExt(tif, fld); return NULL; @@ -975,7 +1067,8 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, * Update: * This special sign is replaced by fld->field_anonymous flag. */ - (void)snprintf(fld->field_name, 32, "Tag %d", (int)tag); + (void)snprintf(field_name_buf, 32, "Tag %d", (int)tag); + fld->field_name = field_name_buf; return fld; } @@ -1025,6 +1118,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_UINT64; case TIFF_SLONG8: return TIFF_SETGET_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1062,6 +1156,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C0_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C0_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1099,6 +1194,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C16_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C16_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1136,6 +1232,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C32_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C32_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1152,22 +1249,25 @@ int TIFFMergeFieldInfo(TIFF *tif, const TIFFFieldInfo info[], uint32_t n) size_t nfields; uint32_t i; + TIFFFieldArray *tif_newfieldscompat = NULL; + if (tif->tif_nfieldscompat > 0) { - tif->tif_fieldscompat = (TIFFFieldArray *)_TIFFCheckRealloc( - tif, tif->tif_fieldscompat, tif->tif_nfieldscompat + 1, - sizeof(TIFFFieldArray), reason); + tif_newfieldscompat = (TIFFFieldArray *)_TIFFCheckRealloc( + tif, tif->tif_fieldscompat, (tmsize_t)tif->tif_nfieldscompat + 1, + (tmsize_t)sizeof(TIFFFieldArray), reason); } else { - tif->tif_fieldscompat = (TIFFFieldArray *)_TIFFCheckMalloc( + tif_newfieldscompat = (TIFFFieldArray *)_TIFFCheckMalloc( tif, 1, sizeof(TIFFFieldArray), reason); } - if (!tif->tif_fieldscompat) + if (!tif_newfieldscompat) { TIFFErrorExtR(tif, module, "Failed to allocate fields array"); return -1; } + tif->tif_fieldscompat = tif_newfieldscompat; nfields = tif->tif_nfieldscompat++; tif->tif_fieldscompat[nfields].type = tfiatOther; @@ -1309,6 +1409,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) case TIFFTAG_JPEGPROC: case TIFFTAG_JPEGRESTARTINTERVAL: return 1; + default: + break; } break; case COMPRESSION_CCITTRLE: @@ -1329,6 +1431,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4) return 1; break; + default: + break; } break; case COMPRESSION_JBIG: @@ -1359,6 +1463,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) if (tag == TIFFTAG_LERC_PARAMETERS) return 1; break; + default: + break; } return 0; } diff --git a/contrib/libs/libtiff/tif_dirread.c b/contrib/libs/libtiff/tif_dirread.c index 0182f9b4194..50803caf4fd 100644 --- a/contrib/libs/libtiff/tif_dirread.c +++ b/contrib/libs/libtiff/tif_dirread.c @@ -597,7 +597,7 @@ TIFFReadDirEntrySshort(TIFF *tif, TIFFDirEntry *direntry, int16_t *value) err = TIFFReadDirEntryCheckRangeSshortShort(m); if (err != TIFFReadDirEntryErrOk) return (err); - *value = (uint16_t)m; + *value = (int16_t)m; return (TIFFReadDirEntryErrOk); } case TIFF_SSHORT: @@ -695,6 +695,7 @@ TIFFReadDirEntryLong(TIFF *tif, TIFFDirEntry *direntry, uint32_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG: + case TIFF_IFD: TIFFReadDirEntryCheckedLong(tif, direntry, value); return (TIFFReadDirEntryErrOk); case TIFF_SLONG: @@ -708,6 +709,7 @@ TIFFReadDirEntryLong(TIFF *tif, TIFFDirEntry *direntry, uint32_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG8: + case TIFF_IFD8: { uint64_t m; err = TIFFReadDirEntryCheckedLong8(tif, direntry, &m); @@ -857,6 +859,7 @@ TIFFReadDirEntryLong8(TIFF *tif, TIFFDirEntry *direntry, uint64_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG: + case TIFF_IFD: { uint32_t m; TIFFReadDirEntryCheckedLong(tif, direntry, &m); @@ -874,6 +877,7 @@ TIFFReadDirEntryLong8(TIFF *tif, TIFFDirEntry *direntry, uint64_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG8: + case TIFF_IFD8: err = TIFFReadDirEntryCheckedLong8(tif, direntry, value); return (err); case TIFF_SLONG8: @@ -1058,7 +1062,7 @@ TIFFReadDirEntryFloat(TIFF *tif, TIFFDirEntry *direntry, float *value) err = TIFFReadDirEntryCheckedDouble(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk) return (err); - if ((m > FLT_MAX) || (m < -FLT_MAX)) + if ((m > (double)FLT_MAX) || (m < -(double)FLT_MAX)) return (TIFFReadDirEntryErrRange); *value = (float)m; return (TIFFReadDirEntryErrOk); @@ -1237,6 +1241,8 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryDataAndRealloc(TIFF *tif, *pdest = new_dest; bytes_read = TIFFReadFile(tif, (char *)*pdest + already_read, to_read); + if (bytes_read < 0) + return TIFFReadDirEntryErrIo; already_read += bytes_read; if (bytes_read != to_read) { @@ -1260,7 +1266,7 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, void *data; uint64_t target_count64; int original_datasize_clamped; - typesize = TIFFDataWidth(direntry->tdir_type); + typesize = TIFFDataWidth((TIFFDataType)direntry->tdir_type); target_count64 = (direntry->tdir_count > maxcount) ? maxcount : direntry->tdir_count; @@ -1284,13 +1290,13 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, * in either the current data type or the dest data type. This also * avoids problems with overflow of tmsize_t on 32bit systems. */ - if ((uint64_t)(MAX_SIZE_TAG_DATA / typesize) < target_count64) + if ((uint64_t)(MAX_SIZE_TAG_DATA / (unsigned int)typesize) < target_count64) return (TIFFReadDirEntryErrSizesan); if ((uint64_t)(MAX_SIZE_TAG_DATA / desttypesize) < target_count64) return (TIFFReadDirEntryErrSizesan); *count = (uint32_t)target_count64; - datasize = (*count) * typesize; + datasize = (uint32_t)(*count) * (unsigned int)typesize; assert((tmsize_t)datasize > 0); if (datasize > 100 * 1024 * 1024) @@ -1440,6 +1446,8 @@ TIFFReadDirEntryByteArray(TIFF *tif, TIFFDirEntry *direntry, uint8_t **value) *value = (uint8_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint8_t *)_TIFFmallocExt(tif, count); if (data == 0) @@ -1557,6 +1565,8 @@ TIFFReadDirEntryByteArray(TIFF *tif, TIFFDirEntry *direntry, uint8_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1620,6 +1630,8 @@ TIFFReadDirEntrySbyteArray(TIFF *tif, TIFFDirEntry *direntry, int8_t **value) case TIFF_SBYTE: *value = (int8_t *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } data = (int8_t *)_TIFFmallocExt(tif, count); if (data == 0) @@ -1737,6 +1749,8 @@ TIFFReadDirEntrySbyteArray(TIFF *tif, TIFFDirEntry *direntry, int8_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1802,6 +1816,8 @@ TIFFReadDirEntryShortArray(TIFF *tif, TIFFDirEntry *direntry, uint16_t **value) *value = (uint16_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint16_t *)_TIFFmallocExt(tif, count * 2); if (data == 0) @@ -1910,6 +1926,8 @@ TIFFReadDirEntryShortArray(TIFF *tif, TIFFDirEntry *direntry, uint16_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1975,6 +1993,8 @@ TIFFReadDirEntrySshortArray(TIFF *tif, TIFFDirEntry *direntry, int16_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfShort((uint16_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int16_t *)_TIFFmallocExt(tif, count * 2); if (data == 0) @@ -2078,6 +2098,8 @@ TIFFReadDirEntrySshortArray(TIFF *tif, TIFFDirEntry *direntry, int16_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2106,6 +2128,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) case TIFF_SLONG: case TIFF_LONG8: case TIFF_SLONG8: + case TIFF_IFD: + case TIFF_IFD8: break; default: return (TIFFReadDirEntryErrType); @@ -2119,6 +2143,7 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) switch (direntry->tdir_type) { case TIFF_LONG: + case TIFF_IFD: *value = (uint32_t *)origdata; if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(*value, count); @@ -2143,6 +2168,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) *value = (uint32_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint32_t *)_TIFFmallocExt(tif, count * 4); if (data == 0) @@ -2213,6 +2240,7 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) } break; case TIFF_LONG8: + case TIFF_IFD8: { uint64_t *ma; uint32_t *mb; @@ -2248,6 +2276,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2313,6 +2343,8 @@ TIFFReadDirEntrySlongArray(TIFF *tif, TIFFDirEntry *direntry, int32_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int32_t *)_TIFFmallocExt(tif, count * 4); if (data == 0) @@ -2410,6 +2442,8 @@ TIFFReadDirEntrySlongArray(TIFF *tif, TIFFDirEntry *direntry, int32_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2439,6 +2473,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, case TIFF_SLONG: case TIFF_LONG8: case TIFF_SLONG8: + case TIFF_IFD: + case TIFF_IFD8: break; default: return (TIFFReadDirEntryErrType); @@ -2453,6 +2489,7 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, switch (direntry->tdir_type) { case TIFF_LONG8: + case TIFF_IFD8: *value = (uint64_t *)origdata; if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(*value, count); @@ -2477,6 +2514,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, *value = (uint64_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -2547,6 +2586,7 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, } break; case TIFF_LONG: + case TIFF_IFD: { uint32_t *ma; uint64_t *mb; @@ -2579,6 +2619,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2651,6 +2693,8 @@ TIFFReadDirEntrySlong8Array(TIFF *tif, TIFFDirEntry *direntry, int64_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8((uint64_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -2742,6 +2786,8 @@ TIFFReadDirEntrySlong8Array(TIFF *tif, TIFFDirEntry *direntry, int64_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -2787,8 +2833,11 @@ TIFFReadDirEntryFloatArray(TIFF *tif, TIFFDirEntry *direntry, float **value) TIFFCvtIEEEFloatToNative(tif, count, (float *)origdata); *value = (float *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } - data = (float *)_TIFFmallocExt(tif, count * sizeof(float)); + data = + (float *)_TIFFmallocExt(tif, (tmsize_t)((size_t)count * sizeof(float))); if (data == 0) { _TIFFfreeExt(tif, origdata); @@ -2970,14 +3019,16 @@ TIFFReadDirEntryFloatArray(TIFF *tif, TIFFDirEntry *direntry, float **value) for (n = 0; n < count; n++) { double val = *ma++; - if (val > FLT_MAX) - val = FLT_MAX; - else if (val < -FLT_MAX) - val = -FLT_MAX; + if (val > (double)FLT_MAX) + val = (double)FLT_MAX; + else if (val < -(double)FLT_MAX) + val = -(double)FLT_MAX; *mb++ = (float)val; } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3023,8 +3074,11 @@ TIFFReadDirEntryDoubleArray(TIFF *tif, TIFFDirEntry *direntry, double **value) TIFFCvtIEEEDoubleToNative(tif, count, (double *)origdata); *value = (double *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } - data = (double *)_TIFFmallocExt(tif, count * sizeof(double)); + data = (double *)_TIFFmallocExt(tif, + (tmsize_t)((size_t)count * sizeof(double))); if (data == 0) { _TIFFfreeExt(tif, origdata); @@ -3207,6 +3261,8 @@ TIFFReadDirEntryDoubleArray(TIFF *tif, TIFFDirEntry *direntry, double **value) *mb++ = (double)(*ma++); } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3244,6 +3300,8 @@ TIFFReadDirEntryIfd8Array(TIFF *tif, TIFFDirEntry *direntry, uint64_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(*value, count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (uint64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -3269,6 +3327,8 @@ TIFFReadDirEntryIfd8Array(TIFF *tif, TIFFDirEntry *direntry, uint64_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3283,13 +3343,42 @@ TIFFReadDirEntryPersampleShort(TIFF *tif, TIFFDirEntry *direntry, uint16_t *m; uint16_t *na; uint16_t nb; - if (direntry->tdir_count < (uint64_t)tif->tif_dir.td_samplesperpixel) - return (TIFFReadDirEntryErrCount); + if (direntry->tdir_count != (uint64_t)tif->tif_dir.td_samplesperpixel) + { + const TIFFField *fip = TIFFFieldWithTag(tif, direntry->tdir_tag); + if (direntry->tdir_count == 0) + { + return TIFFReadDirEntryErrCount; + } + else if (direntry->tdir_count < + (uint64_t)tif->tif_dir.td_samplesperpixel) + { + TIFFWarningExtR( + tif, "TIFFReadDirEntryPersampleShort", + "Tag %s entry count is %" PRIu64 + " , whereas it should be SamplesPerPixel=%d. Assuming that " + "missing entries are all at the value of the first one", + fip ? fip->field_name : "unknown tagname", direntry->tdir_count, + tif->tif_dir.td_samplesperpixel); + } + else + { + TIFFWarningExtR(tif, "TIFFReadDirEntryPersampleShort", + "Tag %s entry count is %" PRIu64 + " , whereas it should be SamplesPerPixel=%d. " + "Ignoring extra entries", + fip ? fip->field_name : "unknown tagname", + direntry->tdir_count, + tif->tif_dir.td_samplesperpixel); + } + } err = TIFFReadDirEntryShortArray(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk || m == NULL) return (err); na = m; nb = tif->tif_dir.td_samplesperpixel; + if (direntry->tdir_count < nb) + nb = (uint16_t)direntry->tdir_count; *value = *na++; nb--; while (nb > 0) @@ -3908,7 +3997,7 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryData(TIFF *tif, uint64_t offset, { return TIFFReadDirEntryErrIo; } - mb = ma + size; + mb = (uint64_t)ma + (uint64_t)size; if (mb > (uint64_t)tif->tif_size) return (TIFFReadDirEntryErrIo); _TIFFmemcpy(dest, tif->tif_base + ma, size); @@ -3955,6 +4044,7 @@ static void TIFFReadDirEntryOutputErr(TIFF *tif, enum TIFFReadDirEntryErr err, TIFFErrorExtR(tif, module, "Out of memory reading of \"%s\"", tagname); break; + case TIFFReadDirEntryErrOk: default: assert(0); /* we should never get here */ break; @@ -4001,6 +4091,7 @@ static void TIFFReadDirEntryOutputErr(TIFF *tif, enum TIFFReadDirEntryErr err, "Out of memory reading of \"%s\"; tag ignored", tagname); break; + case TIFFReadDirEntryErrOk: default: assert(0); /* we should never get here */ break; @@ -4086,7 +4177,8 @@ static int ByteCountLooksBad(TIFF *tif) */ static bool EvaluateIFDdatasizeReading(TIFF *tif, TIFFDirEntry *dp) { - const uint64_t data_width = TIFFDataWidth(dp->tdir_type); + const uint64_t data_width = + (uint64_t)TIFFDataWidth((TIFFDataType)dp->tdir_type); if (data_width != 0 && dp->tdir_count > UINT64_MAX / data_width) { TIFFErrorExtR(tif, "EvaluateIFDdatasizeReading", @@ -4173,9 +4265,9 @@ static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) /* Get offset of end of IFD entry space. */ uint64_t IFDendoffset; if (!(tif->tif_flags & TIFF_BIGTIFF)) - IFDendoffset = tif->tif_diroff + 2 + dircount * 12 + 4; + IFDendoffset = tif->tif_diroff + 2 + (uint64_t)dircount * 12 + 4; else - IFDendoffset = tif->tif_diroff + 8 + dircount * 20 + 8; + IFDendoffset = tif->tif_diroff + 8 + (uint64_t)dircount * 20 + 8; /* Check which offsets are right behind IFD entries. However, LibTIFF * increments the writing address for every external data to an even offset. @@ -4224,9 +4316,9 @@ static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) /* Finally, add the size of the IFD tag entries themselves. */ if (!(tif->tif_flags & TIFF_BIGTIFF)) - tif->tif_dir.td_dirdatasize_read = 2 + dircount * 12 + 4 + size; + tif->tif_dir.td_dirdatasize_read = 2U + dircount * 12U + 4U + size; else - tif->tif_dir.td_dirdatasize_read = 8 + dircount * 20 + 8 + size; + tif->tif_dir.td_dirdatasize_read = 8U + dircount * 20U + 8U + size; } /*-- CalcFinalIFDdatasizeReading() --*/ /* @@ -4298,7 +4390,8 @@ int TIFFReadDirectory(TIFF *tif) { TIFFDirEntry *na; uint16_t nb; - for (na = ma + 1, nb = mb + 1; nb < dircount; na++, nb++) + for (na = ma + 1, nb = (uint16_t)(mb + 1); nb < dircount; + na++, nb++) { if (ma->tdir_tag == na->tdir_tag) { @@ -4312,7 +4405,15 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_flags &= ~TIFF_BUF4WRITE; /* reset before new dir */ tif->tif_flags &= ~TIFF_CHOPPEDUPARRAYS; - /* free any old stuff and reinit */ + /* When changing directory, in deferred strile loading mode, we must also + * unset the TIFF_LAZYSTRILELOAD_DONE bit if it was initially set, + * to make sure the strile offset/bytecount are read again (when they fit + * in the tag data area). + */ + tif->tif_flags &= ~TIFF_LAZYSTRILELOAD_DONE; + + /* Free any old stuff and reinit i/o and other parameters within + * TIFFDefaultDirectory() since we are starting on a new directory. */ TIFFFreeDirectory(tif); TIFFDefaultDirectory(tif); @@ -4325,7 +4426,8 @@ int TIFFReadDirectory(TIFF *tif) * checking. Note: Counter are reset within TIFFFreeDirectory(). */ tif->tif_dir.td_dirdatasize_offsets = (TIFFEntryOffsetAndLength *)_TIFFmallocExt( - tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + tif, + (tmsize_t)((size_t)dircount * sizeof(TIFFEntryOffsetAndLength))); if (tif->tif_dir.td_dirdatasize_offsets == NULL) { TIFFErrorExtR( @@ -4787,8 +4889,8 @@ int TIFFReadDirectory(TIFF *tif) (void)TIFFFetchNormalTag(tif, dp, TRUE); break; } /* -- switch (dp->tdir_tag) -- */ - } /* -- if (!dp->tdir_ignore) */ - } /* -- for-loop -- */ + } /* -- if (!dp->tdir_ignore) */ + } /* -- for-loop -- */ /* Evaluate final IFD data size. */ CalcFinalIFDdatasizeReading(tif, dircount); @@ -5097,7 +5199,7 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_dir.td_maxsamplevalue = 0xFFFF; else tif->tif_dir.td_maxsamplevalue = - (uint16_t)((1L << tif->tif_dir.td_bitspersample) - 1); + (uint16_t)((1 << tif->tif_dir.td_bitspersample) - 1); } #ifdef STRIPBYTECOUNTSORTED_UNUSED @@ -5163,24 +5265,18 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_flags &= ~TIFF_DIRTYSTRIP; /* - * Reinitialize i/o since we are starting on a new directory. + * Reinitialize some further i/o since we are starting on a new directory. */ - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; - tif->tif_col = (uint32_t)-1; - tif->tif_curtile = (uint32_t)-1; - tif->tif_tilesize = (tmsize_t)-1; - - tif->tif_scanlinesize = TIFFScanlineSize(tif); - if (!tif->tif_scanlinesize) + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); + if (!tif->tif_dir.td_scanlinesize) { TIFFWarningExtR(tif, module, "Cannot handle zero scanline size"); } if (isTiled(tif)) { - tif->tif_tilesize = TIFFTileSize(tif); - if (!tif->tif_tilesize) + tif->tif_dir.td_tilesize = TIFFTileSize(tif); + if (!tif->tif_dir.td_tilesize) { TIFFWarningExtR(tif, module, "Cannot handle zero tile size"); } @@ -5216,7 +5312,7 @@ static void TIFFReadDirectoryCheckOrder(TIFF *tif, TIFFDirEntry *dir, "ascending order"); break; } - m = o->tdir_tag + 1; + m = o->tdir_tag + 1U; } } @@ -5264,7 +5360,7 @@ static void TIFFReadDirectoryFindFieldInfo(TIFF *tif, uint16_t tagid, break; mb--; } - *fii = mb; + *fii = (uint32_t)mb; } /* @@ -5304,7 +5400,8 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, { TIFFDirEntry *na; uint16_t nb; - for (na = ma + 1, nb = mb + 1; nb < dircount; na++, nb++) + for (na = ma + 1, nb = (uint16_t)(mb + 1); nb < dircount; + na++, nb++) { if (ma->tdir_tag == na->tdir_tag) { @@ -5329,7 +5426,8 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, * checking. Note: Counter are reset within TIFFFreeDirectory(). */ tif->tif_dir.td_dirdatasize_offsets = (TIFFEntryOffsetAndLength *)_TIFFmallocExt( - tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + tif, + (tmsize_t)((size_t)dircount * sizeof(TIFFEntryOffsetAndLength))); if (tif->tif_dir.td_dirdatasize_offsets == NULL) { TIFFErrorExtR( @@ -5511,15 +5609,17 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, uint64_t space; uint16_t n; if (!(tif->tif_flags & TIFF_BIGTIFF)) - space = sizeof(TIFFHeaderClassic) + 2 + dircount * 12 + 4; + space = sizeof(TIFFHeaderClassic) + 2 + + (unsigned long)dircount * 12 + 4; else - space = sizeof(TIFFHeaderBig) + 8 + dircount * 20 + 8; + space = + sizeof(TIFFHeaderBig) + 8 + (unsigned long)dircount * 20 + 8; /* calculate amount of space used by indirect values */ for (dp = dir, n = dircount; n > 0; n--, dp++) { uint32_t typewidth; uint64_t datasize; - typewidth = TIFFDataWidth((TIFFDataType)dp->tdir_type); + typewidth = (uint32_t)TIFFDataWidth((TIFFDataType)dp->tdir_type); if (typewidth == 0) { TIFFErrorExtR( @@ -5782,7 +5882,7 @@ int _TIFFCheckDirNumberAndOffset(TIFF *tif, tdir_t dirn, uint64_t diroff) { TIFFErrorExtR(tif, "_TIFFCheckDirNumberAndOffset", "Cannot handle more than %u TIFF directories", - TIFF_MAX_DIR_COUNT); + (unsigned)TIFF_MAX_DIR_COUNT); return 0; } @@ -6044,7 +6144,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, "to read TIFF directory"); if (origdir == NULL) return 0; - if (!ReadOK(tif, origdir, (tmsize_t)(dircount16 * dirsize))) + if (!ReadOK(tif, origdir, (tmsize_t)dircount16 * dirsize)) { TIFFErrorExtR(tif, module, "%.100s: Can not read TIFF directory", tif->tif_name); @@ -6097,9 +6197,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, */ if (!(tif->tif_flags & TIFF_BIGTIFF)) { - m = off + sizeof(uint16_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint16_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint16_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint16_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory count"); return 0; @@ -6108,7 +6208,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, { _TIFFmemcpy(&dircount16, tif->tif_base + off, sizeof(uint16_t)); } - off += sizeof(uint16_t); + off = (tmsize_t)((uint64_t)off + sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount16); if (dircount16 > 4096) @@ -6123,9 +6223,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, else { uint64_t dircount64; - m = off + sizeof(uint64_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint64_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint64_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory count"); return 0; @@ -6134,7 +6234,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, { _TIFFmemcpy(&dircount64, tif->tif_base + off, sizeof(uint64_t)); } - off += sizeof(uint64_t); + off = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8(&dircount64); if (dircount64 > 4096) @@ -6173,7 +6273,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, if (origdir == NULL) return 0; m = off + dircount16 * dirsize; - if ((m < off) || (m < (tmsize_t)(dircount16 * dirsize)) || + if ((m < off) || (m < (tmsize_t)dircount16 * (tmsize_t)dirsize) || (m > tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory"); @@ -6182,7 +6282,8 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, } else { - _TIFFmemcpy(origdir, tif->tif_base + off, dircount16 * dirsize); + _TIFFmemcpy(origdir, tif->tif_base + off, + (tmsize_t)dircount16 * dirsize); } if (nextdiroff) { @@ -6190,9 +6291,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, if (!(tif->tif_flags & TIFF_BIGTIFF)) { uint32_t nextdiroff32; - m = off + sizeof(uint32_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint32_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint32_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint32_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) nextdiroff32 = 0; else _TIFFmemcpy(&nextdiroff32, tif->tif_base + off, @@ -6203,9 +6304,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, } else { - m = off + sizeof(uint64_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint64_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint64_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) *nextdiroff = 0; else _TIFFmemcpy(nextdiroff, tif->tif_base + off, @@ -6351,8 +6452,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) /* TIFFReadDirEntryArrayWithLimit() ensures this can't be * larger than MAX_SIZE_TAG_DATA */ assert((uint32_t)dp->tdir_count + 1 == dp->tdir_count + 1); - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -6494,7 +6595,7 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) { if (!EvaluateIFDdatasizeReading(tif, dp)) return 0; - if (!TIFFSetField(tif, dp->tdir_tag, data)) + if (!TIFFSetField(tif, dp->tdir_tag, (double)data)) return (0); } } @@ -6906,6 +7007,41 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } } break; + case TIFF_SETGET_C0_IFD8: + { + uint64_t *data; + assert(fip->field_readcount >= 1); + assert(fip->field_passcount == 0); + if (dp->tdir_count != (uint64_t)fip->field_readcount) + { + TIFFWarningExtR(tif, module, + "incorrect count for field \"%s\", expected " + "%d, got %" PRIu64, + fip->field_name, (int)fip->field_readcount, + dp->tdir_count); + return (0); + } + else + { + err = TIFFReadDirEntryIfd8Array(tif, dp, &data); + if (err == TIFFReadDirEntryErrOk) + { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } + int m; + m = TIFFSetField(tif, dp->tdir_tag, data); + if (data != 0) + _TIFFfreeExt(tif, data); + if (!m) + return (0); + } + } + } + break; case TIFF_SETGET_C16_ASCII: { uint8_t *data; @@ -6934,8 +7070,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) "byte. Forcing it to be null", fip->field_name); /* Enlarge buffer and add terminating null. */ - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -7305,8 +7441,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) "in null byte. Forcing it to be null", fip->field_name); /* Enlarge buffer and add terminating null. */ - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -7620,6 +7756,11 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } } break; + case TIFF_SETGET_INT: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_OTHER: + assert(0); /* these should not arrive here */ + break; default: assert(0); /* we should never get here */ break; @@ -7696,9 +7837,10 @@ static int TIFFFetchStripThing(TIFF *tif, TIFFDirEntry *dir, uint32_t nstrips, } if (dir->tdir_count) _TIFFmemcpy(resizeddata, data, - (uint32_t)dir->tdir_count * sizeof(uint64_t)); + (tmsize_t)((size_t)dir->tdir_count * sizeof(uint64_t))); _TIFFmemset(resizeddata + (uint32_t)dir->tdir_count, 0, - (nstrips - (uint32_t)dir->tdir_count) * sizeof(uint64_t)); + (tmsize_t)((size_t)(nstrips - (uint32_t)dir->tdir_count) * + sizeof(uint64_t))); _TIFFfreeExt(tif, data); data = resizeddata; } @@ -7862,7 +8004,7 @@ static void allocChoppedUpStripArrays(TIFF *tif, uint32_t nstrips, */ static void ChopUpSingleUncompressedStrip(TIFF *tif) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; uint64_t bytecount; uint64_t offset; uint32_t rowblock; @@ -7877,7 +8019,7 @@ static void ChopUpSingleUncompressedStrip(TIFF *tif) /* later ( StripOffsets and StripByteCounts improperly filled) . */ if (bytecount == 0 && tif->tif_mode != O_RDONLY) return; - offset = TIFFGetStrileByteCount(tif, 0); + offset = TIFFGetStrileOffset(tif, 0); assert(td->td_planarconfig == PLANARCONFIG_CONTIG); if ((td->td_photometric == PHOTOMETRIC_YCBCR) && (!isUpSampled(tif))) rowblock = td->td_ycbcrsubsampling[1]; @@ -8028,7 +8170,7 @@ static void TryChopUpUncompressedBigTiff(TIFF *tif) TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW static uint64_t _TIFFUnsanitizedAddUInt64AndInt(uint64_t a, int b) { - return a + b; + return a + (uint64_t)b; } /* Read the value of [Strip|Tile]Offset or [Strip|Tile]ByteCount around @@ -8107,7 +8249,7 @@ static int _TIFFPartialReadStripArray(TIFF *tif, TIFFDirEntry *dirent, panVals[strile] = 0; return 0; } - nOffset = nBaseOffset + sizeofval * strile; + nOffset = nBaseOffset + (uint64_t)sizeofval * (uint64_t)strile; nOffsetStartPage = (nOffset / IO_CACHE_PAGE_SIZE) * IO_CACHE_PAGE_SIZE; nOffsetEndPage = nOffsetStartPage + IO_CACHE_PAGE_SIZE; @@ -8115,7 +8257,7 @@ static int _TIFFPartialReadStripArray(TIFF *tif, TIFFDirEntry *dirent, nOffsetEndPage += IO_CACHE_PAGE_SIZE; #undef IO_CACHE_PAGE_SIZE - nLastStripOffset = nBaseOffset + arraySize * sizeofval; + nLastStripOffset = nBaseOffset + (uint64_t)arraySize * sizeofval; if (nLastStripOffset < nOffsetEndPage) nOffsetEndPage = nLastStripOffset; if (nOffsetStartPage >= nOffsetEndPage) @@ -8250,9 +8392,9 @@ static int _TIFFFetchStrileValue(TIFF *tif, uint32_t strile, } #endif offsetArray = (uint64_t *)(_TIFFreallocExt(tif, td->td_stripoffset_p, - nArraySize)); + (tmsize_t)nArraySize)); bytecountArray = (uint64_t *)(_TIFFreallocExt( - tif, td->td_stripbytecount_p, nArraySize)); + tif, td->td_stripbytecount_p, (tmsize_t)nArraySize)); if (offsetArray) td->td_stripoffset_p = offsetArray; if (bytecountArray) @@ -8286,7 +8428,7 @@ static int _TIFFFetchStrileValue(TIFF *tif, uint32_t strile, if (~((*parray)[strile]) == 0) { - if (!_TIFFPartialReadStripArray(tif, dirent, strile, *parray)) + if (!_TIFFPartialReadStripArray(tif, dirent, (int)strile, *parray)) { (*parray)[strile] = 0; return 0; @@ -8304,10 +8446,26 @@ static uint64_t _TIFFGetStrileOffsetOrByteCountValue(TIFF *tif, uint32_t strile, TIFFDirectory *td = &tif->tif_dir; if (pbErr) *pbErr = 0; + + /* Avoid the "dirent->tdir_count <= 4" code path for one of + * StripOffsets/StripByteCounts, and the other code path for the other one, + * which will lead to inconsistencies and potential out-of-bounds reads. + */ + if ((td->td_stripoffset_entry.tdir_count <= 4) != + (td->td_stripbytecount_entry.tdir_count <= 4)) + { + TIFFErrorExtR(tif, "_TIFFGetStrileOffsetOrByteCountValue", + "Inconsistent directory count between StripOffsets and " + "StripByteCounts"); + if (pbErr) + *pbErr = 1; + return 0; + } + if ((tif->tif_flags & TIFF_DEFERSTRILELOAD) && !(tif->tif_flags & TIFF_CHOPPEDUPARRAYS)) { - if (!(tif->tif_flags & TIFF_LAZYSTRILELOAD) || + if (!(tif->tif_flags & TIFF_LAZYSTRILELOAD_ASKED) || /* If the values may fit in the toff_long/toff_long8 member */ /* then use _TIFFFillStriles to simplify _TIFFFetchStrileValue */ dirent->tdir_count <= 4) @@ -8378,7 +8536,7 @@ int _TIFFFillStriles(TIFF *tif) { return _TIFFFillStrilesInternal(tif, 1); } static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; int return_value = 1; /* Do not do anything if TIFF_DEFERSTRILELOAD is not set */ @@ -8386,7 +8544,8 @@ static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) (tif->tif_flags & TIFF_CHOPPEDUPARRAYS) != 0) return 1; - if (tif->tif_flags & TIFF_LAZYSTRILELOAD) + if ((tif->tif_flags & TIFF_LAZYSTRILELOAD_ASKED) && + !(tif->tif_flags & TIFF_LAZYSTRILELOAD_DONE)) { /* In case of lazy loading, reload completely the arrays */ _TIFFfreeExt(tif, td->td_stripoffset_p); @@ -8394,7 +8553,7 @@ static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) td->td_stripoffset_p = NULL; td->td_stripbytecount_p = NULL; td->td_stripoffsetbyteallocsize = 0; - tif->tif_flags &= ~TIFF_LAZYSTRILELOAD; + tif->tif_flags |= TIFF_LAZYSTRILELOAD_DONE; } /* If stripoffset array is already loaded, exit with success */ diff --git a/contrib/libs/libtiff/tif_dirwrite.c b/contrib/libs/libtiff/tif_dirwrite.c index bc5ada0f2b5..5da087e2954 100644 --- a/contrib/libs/libtiff/tif_dirwrite.c +++ b/contrib/libs/libtiff/tif_dirwrite.c @@ -365,7 +365,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 4)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -377,7 +377,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint32_t m; m = 0; - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!WriteOK(tif, &m, 4)) { @@ -417,7 +417,6 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, while (1) { uint64_t dircount64; - uint16_t dircount; uint64_t nextnextdir; if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount64, 8)) @@ -435,8 +434,8 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, "corrupt TIFF"); return (0); } - dircount = (uint16_t)dircount64; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, + SEEK_SET); if (!ReadOK(tif, &nextnextdir, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -448,7 +447,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint64_t m; m = 0; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, SEEK_SET); if (!WriteOK(tif, &m, 8)) { @@ -592,24 +591,24 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_RESOLUTION)) { - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_XRESOLUTION, - tif->tif_dir.td_xresolution)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_XRESOLUTION, + (double)tif->tif_dir.td_xresolution)) goto bad; - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_YRESOLUTION, - tif->tif_dir.td_yresolution)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_YRESOLUTION, + (double)tif->tif_dir.td_yresolution)) goto bad; } if (TIFFFieldSet(tif, FIELD_POSITION)) { - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_XPOSITION, - tif->tif_dir.td_xposition)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_XPOSITION, + (double)tif->tif_dir.td_xposition)) goto bad; - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_YPOSITION, - tif->tif_dir.td_yposition)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_YPOSITION, + (double)tif->tif_dir.td_yposition)) goto bad; } if (TIFFFieldSet(tif, FIELD_SUBFILETYPE)) @@ -712,6 +711,14 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_STRIPBYTECOUNTS)) { + /* Check td_stripbytecount_p for NULL pointer (bug#749) */ + if (tif->tif_dir.td_stripbytecount_p == NULL) + { + TIFFErrorExtR( + tif, module, + "StripByteCount array is not set, pointer is NULL"); + goto bad; + } if (!isTiled(tif)) { if (!TIFFWriteDirectoryTagLongLong8Array( @@ -731,6 +738,14 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_STRIPOFFSETS)) { + /* Check td_stripoffset_p for NULL pointer (bug#749) */ + if (tif->tif_dir.td_stripoffset_p == NULL) + { + TIFFErrorExtR( + tif, module, + "StripByteOffset array is not set, pointer is NULL"); + goto bad; + } if (!isTiled(tif)) { /* td_stripoffset_p might be NULL in an odd OJPEG case. See @@ -851,7 +866,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagAscii( tif, &ndir, dir, TIFFTAG_INKNAMES, - tif->tif_dir.td_inknameslen, tif->tif_dir.td_inknames)) + (uint32_t)tif->tif_dir.td_inknameslen, + tif->tif_dir.td_inknames)) goto bad; } if (TIFFFieldSet(tif, FIELD_NUMBEROFINKS)) @@ -932,10 +948,58 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, TIFFGetField(tif, o->field_tag, &pa, &pb); if (!TIFFWriteDirectoryTagUndefinedArray( tif, &ndir, dir, (uint16_t)o->field_tag, - pa, pb)) + pa, (uint8_t *)pb)) goto bad; } break; + case TIFF_SETGET_UNDEFINED: + case TIFF_SETGET_SINT8: + case TIFF_SETGET_SINT16: + case TIFF_SETGET_SINT32: + case TIFF_SETGET_UINT64: + case TIFF_SETGET_SINT64: + case TIFF_SETGET_FLOAT: + case TIFF_SETGET_DOUBLE: + case TIFF_SETGET_IFD8: + case TIFF_SETGET_INT: + case TIFF_SETGET_UINT16_PAIR: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_C0_UINT8: + case TIFF_SETGET_C0_SINT8: + case TIFF_SETGET_C0_UINT16: + case TIFF_SETGET_C0_SINT16: + case TIFF_SETGET_C0_UINT32: + case TIFF_SETGET_C0_SINT32: + case TIFF_SETGET_C0_UINT64: + case TIFF_SETGET_C0_SINT64: + case TIFF_SETGET_C0_FLOAT: + case TIFF_SETGET_C0_DOUBLE: + case TIFF_SETGET_C0_IFD8: + case TIFF_SETGET_C16_ASCII: + case TIFF_SETGET_C16_UINT8: + case TIFF_SETGET_C16_SINT8: + case TIFF_SETGET_C16_UINT16: + case TIFF_SETGET_C16_SINT16: + case TIFF_SETGET_C16_UINT32: + case TIFF_SETGET_C16_SINT32: + case TIFF_SETGET_C16_UINT64: + case TIFF_SETGET_C16_SINT64: + case TIFF_SETGET_C16_FLOAT: + case TIFF_SETGET_C16_DOUBLE: + case TIFF_SETGET_C16_IFD8: + case TIFF_SETGET_C32_ASCII: + case TIFF_SETGET_C32_SINT8: + case TIFF_SETGET_C32_UINT16: + case TIFF_SETGET_C32_SINT16: + case TIFF_SETGET_C32_UINT32: + case TIFF_SETGET_C32_SINT32: + case TIFF_SETGET_C32_UINT64: + case TIFF_SETGET_C32_SINT64: + case TIFF_SETGET_C32_FLOAT: + case TIFF_SETGET_C32_DOUBLE: + case TIFF_SETGET_C32_IFD8: + case TIFF_SETGET_UINT8: + case TIFF_SETGET_OTHER: default: TIFFErrorExtR( tif, module, @@ -954,67 +1018,67 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint16_t tag = (uint16_t)tif->tif_dir.td_customValues[m].info->field_tag; - uint32_t count = tif->tif_dir.td_customValues[m].count; + uint32_t count = (uint32_t)tif->tif_dir.td_customValues[m].count; switch (tif->tif_dir.td_customValues[m].info->field_type) { case TIFF_ASCII: if (!TIFFWriteDirectoryTagAscii( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (char *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_UNDEFINED: if (!TIFFWriteDirectoryTagUndefinedArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_BYTE: if (!TIFFWriteDirectoryTagByteArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SBYTE: if (!TIFFWriteDirectoryTagSbyteArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SHORT: if (!TIFFWriteDirectoryTagShortArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint16_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SSHORT: if (!TIFFWriteDirectoryTagSshortArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int16_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_LONG: if (!TIFFWriteDirectoryTagLongArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SLONG: if (!TIFFWriteDirectoryTagSlongArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_LONG8: if (!TIFFWriteDirectoryTagLong8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SLONG8: if (!TIFFWriteDirectoryTagSlong8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_RATIONAL: @@ -1029,7 +1093,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagRationalDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m] + .value)) goto bad; } else @@ -1037,7 +1102,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, /*-- default should be tv_size == 4 */ if (!TIFFWriteDirectoryTagRationalArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; /*-- ToDo: After Testing, this should be removed and * tv_size==4 should be set as default. */ @@ -1064,7 +1129,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagSrationalDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m] + .value)) goto bad; } else @@ -1072,7 +1138,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, /*-- default should be tv_size == 4 */ if (!TIFFWriteDirectoryTagSrationalArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; /*-- ToDo: After Testing, this should be removed and * tv_size==4 should be set as default. */ @@ -1090,27 +1156,28 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, case TIFF_FLOAT: if (!TIFFWriteDirectoryTagFloatArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_DOUBLE: if (!TIFFWriteDirectoryTagDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_IFD: if (!TIFFWriteDirectoryTagIfdArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_IFD8: if (!TIFFWriteDirectoryTagIfdIfd8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; + case TIFF_NOTYPE: default: assert(0); /* we should never get here */ break; @@ -1128,7 +1195,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, tif->tif_dir.td_dirdatasize_write += 8 + ndir * 20 + 8; /* Setup a new directory within first pass. */ - dir = _TIFFmallocExt(tif, ndir * sizeof(TIFFDirEntry)); + dir = (TIFFDirEntry *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)ndir * sizeof(TIFFDirEntry))); if (dir == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1238,7 +1306,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, uint8_t *n; uint32_t nTmp; TIFFDirEntry *o; - n = dirmem; + n = (uint8_t *)dirmem; *(uint16_t *)n = (uint16_t)ndir; if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort((uint16_t *)n); @@ -1274,7 +1342,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint8_t *n; TIFFDirEntry *o; - n = dirmem; + n = (uint8_t *)dirmem; *(uint64_t *)n = ndir; if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8((uint64_t *)n); @@ -1405,7 +1473,7 @@ static int8_t TIFFClampDoubleToInt8(double val) { if (val > 127) return 127; - if (val < -128 || val != val) + if (val < -128 || isnan(val)) return -128; return (int8_t)val; } @@ -1414,7 +1482,7 @@ static int16_t TIFFClampDoubleToInt16(double val) { if (val > 32767) return 32767; - if (val < -32768 || val != val) + if (val < -32768 || isnan(val)) return -32768; return (int16_t)val; } @@ -1423,7 +1491,7 @@ static int32_t TIFFClampDoubleToInt32(double val) { if (val > 0x7FFFFFFF) return 0x7FFFFFFF; - if (val < -0x7FFFFFFF - 1 || val != val) + if (val < -0x7FFFFFFF - 1 || isnan(val)) return -0x7FFFFFFF - 1; return (int32_t)val; } @@ -1432,7 +1500,7 @@ static uint8_t TIFFClampDoubleToUInt8(double val) { if (val < 0) return 0; - if (val > 255 || val != val) + if (val > 255 || isnan(val)) return 255; return (uint8_t)val; } @@ -1441,7 +1509,7 @@ static uint16_t TIFFClampDoubleToUInt16(double val) { if (val < 0) return 0; - if (val > 65535 || val != val) + if (val > 65535 || isnan(val)) return 65535; return (uint16_t)val; } @@ -1450,7 +1518,7 @@ static uint32_t TIFFClampDoubleToUInt32(double val) { if (val < 0) return 0; - if (val > 0xFFFFFFFFU || val != val) + if (val > 0xFFFFFFFFU || isnan(val)) return 0xFFFFFFFFU; return (uint32_t)val; } @@ -1464,7 +1532,7 @@ static int TIFFWriteDirectoryTagSampleformatArray(TIFF *tif, uint32_t *ndir, void *conv; uint32_t i; int ok; - conv = _TIFFmallocExt(tif, count * sizeof(double)); + conv = _TIFFmallocExt(tif, (tmsize_t)((size_t)count * sizeof(double))); if (conv == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1603,7 +1671,9 @@ static int TIFFWriteDirectoryTagShortPerSample(TIFF *tif, uint32_t *ndir, return (TIFFWriteDirectoryTagCheckedShortArray( tif, ndir, dir, tag, tif->tif_dir.td_samplesperpixel, NULL)); } - m = _TIFFmallocExt(tif, tif->tif_dir.td_samplesperpixel * sizeof(uint16_t)); + m = (uint16_t *)_TIFFmallocExt( + tif, + (tmsize_t)((size_t)tif->tif_dir.td_samplesperpixel * sizeof(uint16_t))); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1664,14 +1734,6 @@ static int TIFFWriteDirectoryTagLong8Array(TIFF *tif, uint32_t *ndir, uint32_t *q; int o; - /* is this just a counting pass? */ - if (dir == NULL) - { - /* only evaluate IFD data size and inc. ndir */ - return (TIFFWriteDirectoryTagCheckedLong8Array(tif, ndir, dir, tag, - count, value)); - } - /* We always write Long8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) return (TIFFWriteDirectoryTagCheckedLong8Array(tif, ndir, dir, tag, @@ -1681,7 +1743,8 @@ static int TIFFWriteDirectoryTagLong8Array(TIFF *tif, uint32_t *ndir, ** For classic tiff we want to verify everything is in range for long ** and convert to long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1725,13 +1788,6 @@ static int TIFFWriteDirectoryTagSlong8Array(TIFF *tif, uint32_t *ndir, int32_t *q; int o; - /* is this just a counting pass? */ - if (dir == NULL) - { - /* only evaluate IFD data size and inc. ndir */ - return (TIFFWriteDirectoryTagCheckedSlong8Array(tif, ndir, dir, tag, - count, value)); - } /* We always write SLong8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) return (TIFFWriteDirectoryTagCheckedSlong8Array(tif, ndir, dir, tag, @@ -1741,7 +1797,8 @@ static int TIFFWriteDirectoryTagSlong8Array(TIFF *tif, uint32_t *ndir, ** For classic tiff we want to verify everything is in range for signed-long ** and convert to signed-long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (int32_t *)_TIFFmallocExt(tif, + (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1960,7 +2017,8 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, ** and convert to long format. */ - uint32_t *p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + uint32_t *p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); uint32_t *q; uint64_t *ma; uint32_t mb; @@ -1990,7 +2048,8 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, } else { - uint16_t *p = _TIFFmallocExt(tif, count * sizeof(uint16_t)); + uint16_t *p = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint16_t))); uint16_t *q; uint64_t *ma; uint32_t mb; @@ -2050,7 +2109,8 @@ static int TIFFWriteDirectoryTagIfdIfd8Array(TIFF *tif, uint32_t *ndir, ** and convert to long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2101,26 +2161,45 @@ static int TIFFWriteDirectoryTagColormap(TIFF *tif, uint32_t *ndir, { static const char module[] = "TIFFWriteDirectoryTagColormap"; uint32_t m; + uint32_t count; + uint64_t count64; + tmsize_t total_values; + tmsize_t plane_bytes; uint16_t *n; int o; - m = (1 << tif->tif_dir.td_bitspersample); + if (tif->tif_dir.td_bitspersample >= 32) + { + TIFFErrorExtR(tif, module, "BitsPerSample too large for Colormap"); + return (0); + } + m = 1U << tif->tif_dir.td_bitspersample; + count64 = _TIFFMultiply64(tif, 3U, m, module); + if (count64 == 0) + return (0); + count = _TIFFCastUInt64ToUInt32(tif, count64, module); + total_values = _TIFFCastUInt64ToSSize(tif, count64, module); + plane_bytes = _TIFFCastUInt64ToSSize( + tif, _TIFFMultiply64(tif, m, sizeof(uint16_t), module), module); + if (count == 0 || total_values == 0 || plane_bytes == 0) + return (0); if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - EvaluateIFDdatasizeWrite(tif, 3 * m, sizeof(uint16_t), ndir); + EvaluateIFDdatasizeWrite(tif, count, sizeof(uint16_t), ndir); return 1; } - n = _TIFFmallocExt(tif, 3 * m * sizeof(uint16_t)); + n = (uint16_t *)_TIFFCheckMalloc(tif, total_values, sizeof(uint16_t), + module); if (n == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); return (0); } - _TIFFmemcpy(&n[0], tif->tif_dir.td_colormap[0], m * sizeof(uint16_t)); - _TIFFmemcpy(&n[m], tif->tif_dir.td_colormap[1], m * sizeof(uint16_t)); - _TIFFmemcpy(&n[2 * m], tif->tif_dir.td_colormap[2], m * sizeof(uint16_t)); + _TIFFmemcpy(&n[0], tif->tif_dir.td_colormap[0], plane_bytes); + _TIFFmemcpy(&n[m], tif->tif_dir.td_colormap[1], plane_bytes); + _TIFFmemcpy(&n[2 * m], tif->tif_dir.td_colormap[2], plane_bytes); o = TIFFWriteDirectoryTagCheckedShortArray(tif, ndir, dir, TIFFTAG_COLORMAP, - 3 * m, n); + count, n); _TIFFfreeExt(tif, n); return (o); } @@ -2130,13 +2209,27 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, { static const char module[] = "TIFFWriteDirectoryTagTransferfunction"; uint32_t m; + uint32_t count; + uint64_t count64; + tmsize_t total_values; + tmsize_t plane_bytes; uint16_t n; uint16_t *o; int p; /* TIFFTAG_TRANSFERFUNCTION expects (1 or 3) pointer to arrays with - * (1 << BitsPerSample) * uint16_t values. + * 2**BitsPerSample uint16_t values. */ - m = (1 << tif->tif_dir.td_bitspersample); + if (tif->tif_dir.td_bitspersample >= 32) + { + TIFFErrorExtR(tif, module, + "BitsPerSample too large for TransferFunction"); + return (0); + } + m = 1U << tif->tif_dir.td_bitspersample; + plane_bytes = _TIFFCastUInt64ToSSize( + tif, _TIFFMultiply64(tif, m, sizeof(uint16_t), module), module); + if (plane_bytes == 0) + return (0); /* clang-format off */ n = (tif->tif_dir.td_samplesperpixel - tif->tif_dir.td_extrasamples) > 1 ? 3 : 1; /* clang-format on */ @@ -2161,35 +2254,39 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, if (n == 3) { if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], - tif->tif_dir.td_transferfunction[2], - m * sizeof(uint16_t)) && + tif->tif_dir.td_transferfunction[2], plane_bytes) && !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], - tif->tif_dir.td_transferfunction[1], - m * sizeof(uint16_t))) + tif->tif_dir.td_transferfunction[1], plane_bytes)) n = 1; } + count64 = _TIFFMultiply64(tif, n, m, module); + if (count64 == 0) + return (0); + count = _TIFFCastUInt64ToUInt32(tif, count64, module); + total_values = _TIFFCastUInt64ToSSize(tif, count64, module); + if (count == 0 || total_values == 0) + return (0); if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - EvaluateIFDdatasizeWrite(tif, n * m, 2, ndir); + EvaluateIFDdatasizeWrite(tif, count, 2, ndir); return 1; } - o = _TIFFmallocExt(tif, n * m * sizeof(uint16_t)); + o = (uint16_t *)_TIFFCheckMalloc(tif, total_values, sizeof(uint16_t), + module); if (o == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); return (0); } - _TIFFmemcpy(&o[0], tif->tif_dir.td_transferfunction[0], - m * sizeof(uint16_t)); + _TIFFmemcpy(&o[0], tif->tif_dir.td_transferfunction[0], plane_bytes); if (n > 1) - _TIFFmemcpy(&o[m], tif->tif_dir.td_transferfunction[1], - m * sizeof(uint16_t)); + _TIFFmemcpy(&o[m], tif->tif_dir.td_transferfunction[1], plane_bytes); if (n > 2) _TIFFmemcpy(&o[2 * m], tif->tif_dir.td_transferfunction[2], - m * sizeof(uint16_t)); + plane_bytes); p = TIFFWriteDirectoryTagCheckedShortArray( - tif, ndir, dir, TIFFTAG_TRANSFERFUNCTION, n * m, o); + tif, ndir, dir, TIFFTAG_TRANSFERFUNCTION, count, o); _TIFFfreeExt(tif, o); return (p); } @@ -2209,7 +2306,9 @@ static int TIFFWriteDirectoryTagSubifd(TIFF *tif, uint32_t *ndir, uint64_t *pa; uint32_t *pb; uint16_t p; - o = _TIFFmallocExt(tif, tif->tif_dir.td_nsubifd * sizeof(uint32_t)); + o = (uint32_t *)_TIFFmallocExt( + tif, + (tmsize_t)((size_t)tif->tif_dir.td_nsubifd * sizeof(uint32_t))); if (o == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2494,7 +2593,7 @@ static int TIFFWriteDirectoryTagCheckedRational(TIFF *tif, uint32_t *ndir, TIFFErrorExtR(tif, module, "Negative value is illegal"); return 0; } - else if (value != value) + else if (isnan(value)) { TIFFErrorExtR(tif, module, "Not-a-number value is illegal"); return 0; @@ -2537,7 +2636,8 @@ static int TIFFWriteDirectoryTagCheckedRationalArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); + m = (uint32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(uint32_t), + "for rational array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2545,7 +2645,7 @@ static int TIFFWriteDirectoryTagCheckedRationalArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToRational(*na, &nb[0], &nb[1]); + DoubleToRational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(m, count * 2); @@ -2573,7 +2673,8 @@ static int TIFFWriteDirectoryTagCheckedSrationalArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); + m = (int32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(int32_t), + "for srational array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2581,7 +2682,7 @@ static int TIFFWriteDirectoryTagCheckedSrationalArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToSrational(*na, &nb[0], &nb[1]); + DoubleToSrational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)m, count * 2); @@ -2610,7 +2711,8 @@ TIFFWriteDirectoryTagCheckedRationalDoubleArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); + m = (uint32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(uint32_t), + "for rational double array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2618,7 +2720,7 @@ TIFFWriteDirectoryTagCheckedRationalDoubleArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToRational(*na, &nb[0], &nb[1]); + DoubleToRational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(m, count * 2); @@ -2645,7 +2747,8 @@ static int TIFFWriteDirectoryTagCheckedSrationalDoubleArray( EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); + m = (int32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(int32_t), + "for srational double array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2653,7 +2756,7 @@ static int TIFFWriteDirectoryTagCheckedSrationalDoubleArray( } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToSrational(*na, &nb[0], &nb[1]); + DoubleToSrational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)m, count * 2); @@ -2744,7 +2847,8 @@ static void ToRationalEuclideanGCD(double value, int blnUseSignedRange, *the double-value of it reaches an integer number without fractional part. */ bigDenom = 1; - while ((value != floor(value)) && (value < fMax) && (bigDenom < nMax)) + while ((!TIFF_DOUBLE_EQ(value, floor(value))) && (value < fMax) && + (bigDenom < nMax)) { bigDenom <<= 1; value *= 2; @@ -2836,7 +2940,7 @@ static void DoubleToRational(double value, uint32_t *num, uint32_t *denom) return; } /*-- Check for easy integer numbers -- */ - if (value == (uint32_t)(value)) + if (TIFF_DOUBLE_EQ(value, (double)(uint32_t)value)) { *num = (uint32_t)value; *denom = 1; @@ -2911,7 +3015,7 @@ static void DoubleToSrational(double value, int32_t *num, int32_t *denom) return; } /*-- Check for easy numbers -- */ - if (value == (int32_t)(value)) + if (TIFF_DOUBLE_EQ(value, (double)(int32_t)value)) { *num = (int32_t)(neg * value); *denom = 1; @@ -3231,6 +3335,14 @@ static int TIFFLinkDirectory(TIFF *tif) uint16_t dircount; uint32_t nextnextdir; + /* Update IDF loop list and check for IFD loop. + * ndir is IFD ID plus one. */ + if (!_TIFFCheckDirNumberAndOffset(tif, ndir - 1, nextdir)) + { + TIFFErrorExtR(tif, module, "Error IFD loop detected"); + return 0; /* bad offset (IFD looping or more than + TIFF_MAX_DIR_COUNT IFDs) */ + } if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount, 2)) { TIFFErrorExtR(tif, module, "Error fetching directory count"); @@ -3238,7 +3350,7 @@ static int TIFFLinkDirectory(TIFF *tif) } if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 4)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -3248,7 +3360,7 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFSwabLong(&nextnextdir); if (nextnextdir == 0) { - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!WriteOK(tif, &m, 4)) { TIFFErrorExtR(tif, module, "Error writing directory link"); @@ -3306,9 +3418,15 @@ static int TIFFLinkDirectory(TIFF *tif) while (1) { uint64_t dircount64; - uint16_t dircount; uint64_t nextnextdir; + /* Update IDF loop list and check for IFD loop. */ + if (!_TIFFCheckDirNumberAndOffset(tif, ndir - 1, nextdir)) + { + TIFFErrorExtR(tif, module, "Error IFD loop detected"); + return 0; /* bad offset (IFD looping or more than + TIFF_MAX_DIR_COUNT IFDs) */ + } if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount64, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory count"); @@ -3323,8 +3441,7 @@ static int TIFFLinkDirectory(TIFF *tif) "likely corrupt TIFF"); return (0); } - dircount = (uint16_t)dircount64; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -3334,7 +3451,8 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFSwabLong8(&nextnextdir); if (nextnextdir == 0) { - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, + SEEK_SET); if (!WriteOK(tif, &m, 8)) { TIFFErrorExtR(tif, module, "Error writing directory link"); @@ -3475,7 +3593,7 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (entry_tag == tag) break; - read_offset += dirsize; + read_offset += (uint64_t)dirsize; } if (entry_tag != tag) @@ -3584,13 +3702,13 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (in_datatype == TIFF_LONG8 && (entry_type == TIFF_SHORT || entry_type == TIFF_LONG || entry_type == TIFF_LONG8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else if (in_datatype == TIFF_SLONG8 && (entry_type == TIFF_SLONG || entry_type == TIFF_SLONG8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else if (in_datatype == TIFF_IFD8 && (entry_type == TIFF_IFD || entry_type == TIFF_IFD8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else datatype = in_datatype; } @@ -3605,7 +3723,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, return 0; if (datatype == in_datatype) - memcpy(buf_to_write, data, count * TIFFDataWidth(datatype)); + memcpy(buf_to_write, data, + (size_t)count * (size_t)TIFFDataWidth(datatype)); else if (datatype == TIFF_SLONG && in_datatype == TIFF_SLONG8) { tmsize_t i; @@ -3698,8 +3817,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, tif->tif_dir.td_stripoffset_entry.tdir_type == 0 && tif->tif_dir.td_stripoffset_entry.tdir_offset.toff_long8 == 0) { - tif->tif_dir.td_stripoffset_entry.tdir_type = datatype; - tif->tif_dir.td_stripoffset_entry.tdir_count = count; + tif->tif_dir.td_stripoffset_entry.tdir_type = (uint16_t)datatype; + tif->tif_dir.td_stripoffset_entry.tdir_count = (uint64_t)count; } else if ((tag == TIFFTAG_TILEBYTECOUNTS || tag == TIFFTAG_STRIPBYTECOUNTS) && @@ -3707,8 +3826,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, tif->tif_dir.td_stripbytecount_entry.tdir_type == 0 && tif->tif_dir.td_stripbytecount_entry.tdir_offset.toff_long8 == 0) { - tif->tif_dir.td_stripbytecount_entry.tdir_type = datatype; - tif->tif_dir.td_stripbytecount_entry.tdir_count = count; + tif->tif_dir.td_stripbytecount_entry.tdir_type = (uint16_t)datatype; + tif->tif_dir.td_stripbytecount_entry.tdir_count = (uint64_t)count; } /* -------------------------------------------------------------------- */ @@ -3756,13 +3875,14 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (count * TIFFDataWidth(datatype) == 4) { uint32_t value; - memcpy(&value, buf_to_write, count * TIFFDataWidth(datatype)); + memcpy(&value, buf_to_write, + (size_t)count * (size_t)TIFFDataWidth(datatype)); entry_offset = value; } else { memcpy(&entry_offset, buf_to_write, - count * TIFFDataWidth(datatype)); + (size_t)count * (size_t)TIFFDataWidth(datatype)); } } @@ -3772,7 +3892,7 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, /* -------------------------------------------------------------------- */ /* Adjust the directory entry. */ /* -------------------------------------------------------------------- */ - entry_type = datatype; + entry_type = (uint16_t)datatype; entry_count = (uint64_t)count; memcpy(direntry_raw + 2, &entry_type, sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) diff --git a/contrib/libs/libtiff/tif_dumpmode.c b/contrib/libs/libtiff/tif_dumpmode.c index 267d5d2d7a5..4e3133f4192 100644 --- a/contrib/libs/libtiff/tif_dumpmode.c +++ b/contrib/libs/libtiff/tif_dumpmode.c @@ -80,7 +80,7 @@ static int DumpModeDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) "Not enough data for scanline %" PRIu32 ", expected a request for at most %" TIFF_SSIZE_FORMAT " bytes, got a request for %" TIFF_SSIZE_FORMAT " bytes", - tif->tif_row, tif->tif_rawcc, cc); + tif->tif_dir.td_row, tif->tif_rawcc, cc); return (0); } /* @@ -99,8 +99,23 @@ static int DumpModeDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) */ static int DumpModeSeek(TIFF *tif, uint32_t nrows) { - tif->tif_rawcp += nrows * tif->tif_scanlinesize; - tif->tif_rawcc -= nrows * tif->tif_scanlinesize; + tmsize_t seek_size; + if (nrows > 0 && + tif->tif_dir.td_scanlinesize > (tmsize_t)(TIFF_TMSIZE_T_MAX / nrows)) + { + TIFFErrorExtR(tif, "DumpModeSeek", + "Integer overflow computing seek size"); + return (0); + } + seek_size = (tmsize_t)nrows * tif->tif_dir.td_scanlinesize; + if (seek_size > tif->tif_rawcc) + { + TIFFErrorExtR(tif, "DumpModeSeek", + "Seek beyond end of raw data buffer"); + return (0); + } + tif->tif_rawcp += seek_size; + tif->tif_rawcc -= seek_size; return (1); } diff --git a/contrib/libs/libtiff/tif_fax3.c b/contrib/libs/libtiff/tif_fax3.c index 34e1c7882f7..06b9508df58 100644 --- a/contrib/libs/libtiff/tif_fax3.c +++ b/contrib/libs/libtiff/tif_fax3.c @@ -123,7 +123,7 @@ typedef struct static const char module[] = mod; \ Fax3CodecState *sp = DecoderState(tif); \ int a0; /* reference element */ \ - int lastx = sp->b.rowpixels; /* last element in row */ \ + int lastx = (int)sp->b.rowpixels; /* last element in row */ \ uint32_t BitAcc; /* bit accumulator */ \ int BitsAvail; /* # valid bits in BitAcc */ \ int RunLength; /* length of current run */ \ @@ -209,61 +209,68 @@ static int Fax3PreDecode(TIFF *tif, uint16_t s) static void Fax3Unexpected(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFErrorExtR(tif, module, - "Bad code word at line %" PRIu32 " of %s %" PRIu32 - " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFErrorExtR( + tif, module, + "Bad code word at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define unexpected(table, a0) \ do \ { \ - Fax3Unexpected(module, tif, sp->line, a0); \ + Fax3Unexpected(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ ++sp->unexpectedReachedCount; \ } while (0) static void Fax3Extension(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFErrorExtR(tif, module, - "Uncompressed data (not supported) at line %" PRIu32 - " of %s %" PRIu32 " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFErrorExtR( + tif, module, + "Uncompressed data (not supported) at line %" PRIu32 " of %s %" PRIu32 + " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } -#define extension(a0) Fax3Extension(module, tif, sp->line, a0) +#define extension(a0) \ + Fax3Extension(module, tif, (uint32_t)sp->line, (uint32_t)(a0)) static void Fax3BadLength(const char *module, TIFF *tif, uint32_t line, uint32_t a0, uint32_t lastx) { - TIFFWarningExtR(tif, module, - "%s at line %" PRIu32 " of %s %" PRIu32 " (got %" PRIu32 - ", expected %" PRIu32 ")", - a0 < lastx ? "Premature EOL" : "Line length mismatch", line, - isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0, - lastx); + TIFFWarningExtR( + tif, module, + "%s at line %" PRIu32 " of %s %" PRIu32 " (got %" PRIu32 + ", expected %" PRIu32 ")", + a0 < lastx ? "Premature EOL" : "Line length mismatch", line, + isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), a0, + lastx); } #define badlength(a0, lastx) \ do \ { \ - Fax3BadLength(module, tif, sp->line, a0, lastx); \ + Fax3BadLength(module, tif, (uint32_t)sp->line, (uint32_t)(a0), \ + (uint32_t)(lastx)); \ ++sp->eolReachedCount; \ } while (0) static void Fax3PrematureEOF(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFWarningExtR(tif, module, - "Premature EOF at line %" PRIu32 " of %s %" PRIu32 - " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFWarningExtR( + tif, module, + "Premature EOF at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define prematureEOF(a0) \ do \ { \ - Fax3PrematureEOF(module, tif, sp->line, a0); \ + Fax3PrematureEOF(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ ++sp->eofReachedCount; \ } while (0) @@ -275,16 +282,15 @@ static void Fax3TryG3WithoutEOL(const char *module, TIFF *tif, uint32_t line, "Try to decode (read) fax Group 3 data without EOL at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 "). Please check result", line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define tryG3WithoutEOL(a0) \ do \ { \ - Fax3TryG3WithoutEOL(module, tif, sp->line, a0); \ + Fax3TryG3WithoutEOL(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ } while (0) -#define Nop - static int CheckReachedCounters(TIFF *tif, const char *module, Fax3CodecState *sp) { @@ -344,12 +350,12 @@ RETRY_WITHOUT_EOL_1D: pa = thisrun; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %" PRIu32 "\n", tif->tif_row); + printf("-------------------- %" PRIu32 "\n", tif->tif_dir.td_row); fflush(stdout); #endif SYNC_EOL(EOF1D, RETRY_WITHOUT_EOL_1D); EXPAND1D(EOF1Da); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); buf += sp->b.rowbytes; occ -= sp->b.rowbytes; sp->line++; @@ -357,7 +363,7 @@ RETRY_WITHOUT_EOL_1D: EOF1D: /* premature EOF */ CLEANUP_RUNS(); EOF1Da: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -404,16 +410,16 @@ RETRY_WITHOUT_EOL_2D: ClrBits(1); #ifdef FAX3_DEBUG printf(" %s\n-------------------- %" PRIu32 "\n", is1D ? "1D" : "2D", - tif->tif_row); + tif->tif_dir.td_row); fflush(stdout); #endif pb = sp->refruns; - b1 = *pb++; + b1 = (int)*pb++; if (is1D) EXPAND1D(EOF2Da); else EXPAND2D(EOF2Da); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); if (pa < thisrun + sp->nruns) { SETVALUE(0); /* imaginary change for reference */ @@ -426,7 +432,7 @@ RETRY_WITHOUT_EOL_2D: EOF2D: /* premature EOF */ CLEANUP_RUNS(); EOF2Da: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -479,12 +485,12 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, { if (bx) { /* align to byte boundary */ - *cp++ &= 0xff << (8 - bx); + *cp++ &= (unsigned char)(0xff << (8 - bx)); run -= 8 - bx; } - if ((n = run >> 3) != 0) + if ((n = (int32_t)(run >> 3)) != 0) { /* multiple bytes to fill */ - if ((n / sizeof(int64_t)) > 1) + if (((size_t)n / sizeof(int64_t)) > 1) { /* * Align to int64_tword boundary and fill. @@ -492,8 +498,8 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, for (; n && !isAligned(cp, int64_t); n--) *cp++ = 0x00; lp = (int64_t *)cp; - nw = (int32_t)(n / sizeof(int64_t)); - n -= nw * sizeof(int64_t); + nw = (int32_t)((size_t)n / sizeof(int64_t)); + n -= (int32_t)((size_t)nw * sizeof(int64_t)); do { *lp++ = 0L; @@ -504,10 +510,10 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, run &= 7; } if (run) - cp[0] &= 0xff >> run; + cp[0] &= (unsigned char)(0xff >> run); } else - cp[0] &= ~(_fillmasks[run] >> bx); + cp[0] &= (unsigned char)~(_fillmasks[run] >> bx); x += runs[0]; } run = runs[1]; @@ -521,12 +527,12 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, { if (bx) { /* align to byte boundary */ - *cp++ |= 0xff >> bx; + *cp++ |= (unsigned char)(0xff >> bx); run -= 8 - bx; } - if ((n = run >> 3) != 0) + if ((n = (int32_t)(run >> 3)) != 0) { /* multiple bytes to fill */ - if ((n / sizeof(int64_t)) > 1) + if (((size_t)n / sizeof(int64_t)) > 1) { /* * Align to int64_t boundary and fill. @@ -534,8 +540,8 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, for (; n && !isAligned(cp, int64_t); n--) *cp++ = 0xff; lp = (int64_t *)cp; - nw = (int32_t)(n / sizeof(int64_t)); - n -= nw * sizeof(int64_t); + nw = (int32_t)((size_t)n / sizeof(int64_t)); + n -= (int32_t)((size_t)nw * sizeof(int64_t)); do { *lp++ = -1L; @@ -550,7 +556,7 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, cp[0] = (unsigned char)((cp[0] | (0xff00 >> run)) & 0xff); } else - cp[0] |= _fillmasks[run] >> bx; + cp[0] |= (unsigned char)(_fillmasks[run] >> bx); x += runs[1]; } } @@ -726,12 +732,13 @@ static const int _msbmask[9] = {0x00, 0x01, 0x03, 0x07, 0x0f, { \ while (length > bit) \ { \ - data |= bits >> (length - bit); \ + data |= (int)((unsigned int)bits >> (length - bit)); \ length -= bit; \ _FlushBits(tif); \ } \ assert(length < 9); \ - data |= (bits & _msbmask[length]) << (bit - length); \ + data |= (int)(((unsigned int)bits & (unsigned int)_msbmask[length]) \ + << (unsigned int)(bit - length)); \ bit -= length; \ if (bit == 0) \ _FlushBits(tif); \ @@ -745,13 +752,13 @@ static const int _msbmask[9] = {0x00, 0x01, 0x03, 0x07, 0x0f, static int Fax3PutBits(TIFF *tif, unsigned int bits, unsigned int length) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; _PutBits(tif, bits, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -782,8 +789,8 @@ static int Fax3PutBits(TIFF *tif, unsigned int bits, unsigned int length) static int putspan(TIFF *tif, int32_t span, const tableentry *tab) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; unsigned int code, length; while (span >= 2624) @@ -816,8 +823,8 @@ static int putspan(TIFF *tif, int32_t span, const tableentry *tab) #endif _PutBits(tif, code, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -831,8 +838,8 @@ static int putspan(TIFF *tif, int32_t span, const tableentry *tab) static int Fax3PutEOL(TIFF *tif) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; unsigned int code, length, tparm; if (sp->b.groupoptions & GROUP3OPT_FILLBITS) @@ -849,7 +856,7 @@ static int Fax3PutEOL(TIFF *tif) align = sp->bit + (8 - align); else align = sp->bit - align; - tparm = align; + tparm = (unsigned int)align; _PutBits(tif, 0, tparm); } } @@ -862,8 +869,8 @@ static int Fax3PutEOL(TIFF *tif) } _PutBits(tif, code, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -991,10 +998,10 @@ static inline int32_t find0span(unsigned char *bp, int32_t bs, int32_t be) bp++; } lp = (int64_t *)bp; - while ((bits >= (int32_t)(8 * sizeof(int64_t))) && (0 == *lp)) + while ((bits >= (int32_t)(size_t)(8 * sizeof(int64_t))) && (0 == *lp)) { - span += 8 * sizeof(int64_t); - bits -= 8 * sizeof(int64_t); + span += (int32_t)(size_t)(8 * sizeof(int64_t)); + bits -= (int32_t)(size_t)(8 * sizeof(int64_t)); lp++; } bp = (unsigned char *)lp; @@ -1059,11 +1066,11 @@ static inline int32_t find1span(unsigned char *bp, int32_t bs, int32_t be) bp++; } lp = (int64_t *)bp; - while ((bits >= (int32_t)(8 * sizeof(int64_t))) && + while ((bits >= (int32_t)(size_t)(8 * sizeof(int64_t))) && (~((uint64_t)0) == (uint64_t)*lp)) { - span += 8 * sizeof(int64_t); - bits -= 8 * sizeof(int64_t); + span += (int32_t)(size_t)(8 * sizeof(int64_t)); + bits -= (int32_t)(size_t)(8 * sizeof(int64_t)); lp++; } bp = (unsigned char *)lp; @@ -1118,16 +1125,16 @@ static int Fax3Encode1DRow(TIFF *tif, unsigned char *bp, uint32_t bits) for (;;) { - span = find0span(bp, bs, bits); /* white span */ + span = find0span(bp, (int32_t)bs, (int32_t)bits); /* white span */ if (!putspan(tif, span, TIFFFaxWhiteCodes)) return 0; - bs += span; + bs += (uint32_t)span; if (bs >= bits) break; - span = find1span(bp, bs, bits); /* black span */ + span = find1span(bp, (int32_t)bs, (int32_t)bits); /* black span */ if (!putspan(tif, span, TIFFFaxBlackCodes)) return 0; - bs += span; + bs += (uint32_t)span; if (bs >= bits) break; } @@ -1161,15 +1168,20 @@ static const tableentry vcodes[7] = { static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, uint32_t bits) { -#define PIXEL(buf, ix) ((((buf)[(ix) >> 3]) >> (7 - ((ix)&7))) & 1) +#define PIXEL(buf, ix) ((((buf)[(ix) >> 3]) >> (7 - ((ix) & 7))) & 1) uint32_t a0 = 0; - uint32_t a1 = (PIXEL(bp, 0) != 0 ? 0 : finddiff(bp, 0, bits, 0)); - uint32_t b1 = (PIXEL(rp, 0) != 0 ? 0 : finddiff(rp, 0, bits, 0)); + uint32_t a1 = (PIXEL(bp, 0) != 0 + ? 0 + : (uint32_t)finddiff(bp, (int32_t)0, (int32_t)bits, 0)); + uint32_t b1 = (PIXEL(rp, 0) != 0 + ? 0 + : (uint32_t)finddiff(rp, (int32_t)0, (int32_t)bits, 0)); uint32_t a2, b2; for (;;) { - b2 = finddiff2(rp, b1, bits, PIXEL(rp, b1)); + b2 = (uint32_t)finddiff2(rp, (int32_t)b1, (int32_t)bits, + (int32_t)PIXEL(rp, b1)); if (b2 >= a1) { /* Naive computation triggers @@ -1182,21 +1194,22 @@ static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, : 0x7FFFFFFF; if (!(-3 <= d && d <= 3)) { /* horizontal mode */ - a2 = finddiff2(bp, a1, bits, PIXEL(bp, a1)); + a2 = (uint32_t)finddiff2(bp, (int32_t)a1, (int32_t)bits, + (int32_t)PIXEL(bp, a1)); if (!putcode(tif, &horizcode)) return 0; if (a0 + a1 == 0 || PIXEL(bp, a0) == 0) { - if (!putspan(tif, a1 - a0, TIFFFaxWhiteCodes)) + if (!putspan(tif, (int32_t)(a1 - a0), TIFFFaxWhiteCodes)) return 0; - if (!putspan(tif, a2 - a1, TIFFFaxBlackCodes)) + if (!putspan(tif, (int32_t)(a2 - a1), TIFFFaxBlackCodes)) return 0; } else { - if (!putspan(tif, a1 - a0, TIFFFaxBlackCodes)) + if (!putspan(tif, (int32_t)(a1 - a0), TIFFFaxBlackCodes)) return 0; - if (!putspan(tif, a2 - a1, TIFFFaxWhiteCodes)) + if (!putspan(tif, (int32_t)(a2 - a1), TIFFFaxWhiteCodes)) return 0; } a0 = a2; @@ -1216,9 +1229,12 @@ static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, } if (a0 >= bits) break; - a1 = finddiff(bp, a0, bits, PIXEL(bp, a0)); - b1 = finddiff(rp, a0, bits, !PIXEL(bp, a0)); - b1 = finddiff(rp, b1, bits, PIXEL(bp, a0)); + a1 = (uint32_t)finddiff(bp, (int32_t)a0, (int32_t)bits, + (int32_t)PIXEL(bp, a0)); + b1 = (uint32_t)finddiff(rp, (int32_t)a0, (int32_t)bits, + (int32_t)!PIXEL(bp, a0)); + b1 = (uint32_t)finddiff(rp, (int32_t)b1, (int32_t)bits, + (int32_t)PIXEL(bp, a0)); } return (1); #undef PIXEL @@ -1487,6 +1503,8 @@ static void Fax3PrintDir(TIFF *tif, FILE *fd, long flags) case CLEANFAXDATA_UNCLEAN: fprintf(fd, " uncorrected errors"); break; + default: + break; } fprintf(fd, " (%" PRIu16 " = 0x%" PRIx16 ")\n", sp->cleanfaxdata, sp->cleanfaxdata); @@ -1500,6 +1518,22 @@ static void Fax3PrintDir(TIFF *tif, FILE *fd, long flags) (*sp->printdir)(tif, fd, flags); } +static uint64_t Fax3GetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 36 */ + /* 4096x4096: 100 */ + /* 16383x16383: 163 */ + /* 65536x65536: 200 */ + /* 200000x200000: 208 */ + + return 250; +} + static int InitCCITTFax3(TIFF *tif) { static const char module[] = "InitCCITTFax3"; @@ -1564,6 +1598,7 @@ static int InitCCITTFax3(TIFF *tif) tif->tif_encodetile = Fax3Encode; tif->tif_close = Fax3Close; tif->tif_cleanup = Fax3Cleanup; + tif->tif_getmaxcompressionratio = Fax3GetMaxCompressionRatio; return (1); } @@ -1626,10 +1661,10 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) RunLength = 0; pa = thisrun = sp->curruns; pb = sp->refruns; - b1 = *pb++; + b1 = (int)*pb++; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %d\n", tif->tif_row); + printf("-------------------- %d\n", tif->tif_dir.td_row); fflush(stdout); #endif EXPAND2D(EOFG4); @@ -1643,7 +1678,7 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) occ, lastx); return -1; } - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); SETVALUE(0); /* imaginary change for reference */ SWAP(uint32_t *, sp->curruns, sp->refruns); buf += sp->b.rowbytes; @@ -1666,7 +1701,7 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) occ, lastx); return -1; } - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (sp->line != start ? 1 @@ -1713,6 +1748,16 @@ static int Fax4PostEncode(TIFF *tif) return (1); } +static uint64_t Fax4GetMaxCompressionRatio(TIFF *tif) +{ + /* FAX4 can compress up to almost one byte per line, so the compression + * ratio can be up to the tile/strip width. + * See README_for_libtiff_developpers.md for raw data + */ + return isTiled(tif) ? tif->tif_dir.td_tilewidth + : tif->tif_dir.td_imagewidth; +} + int TIFFInitCCITTFax4(TIFF *tif, int scheme) { (void)scheme; @@ -1735,6 +1780,7 @@ int TIFFInitCCITTFax4(TIFF *tif, int scheme) tif->tif_encodestrip = Fax4Encode; tif->tif_encodetile = Fax4Encode; tif->tif_postencode = Fax4PostEncode; + tif->tif_getmaxcompressionratio = Fax4GetMaxCompressionRatio; /* * Suppress RTC at the end of each strip. */ @@ -1773,11 +1819,11 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) pa = thisrun; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %" PRIu32 "\n", tif->tif_row); + printf("-------------------- %" PRIu32 "\n", tif->tif_dir.td_row); fflush(stdout); #endif EXPAND1D(EOFRLE); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); /* * Cleanup at the end of the row. */ @@ -1798,7 +1844,7 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) sp->line++; continue; EOFRLE: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -1806,6 +1852,21 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) return (1); } +static uint64_t Fax3RLEGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 43 */ + /* 4096x4096: 128 */ + /* 16383x16383: 171 */ + /* 65536x65536: 205 */ + /* 200000x200000: 211 */ + + return 250; +} + int TIFFInitCCITTRLE(TIFF *tif, int scheme) { (void)scheme; @@ -1814,6 +1875,7 @@ int TIFFInitCCITTRLE(TIFF *tif, int scheme) tif->tif_decoderow = Fax3DecodeRLE; tif->tif_decodestrip = Fax3DecodeRLE; tif->tif_decodetile = Fax3DecodeRLE; + tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio; /* * Suppress RTC+EOLs when encoding and byte-align data. */ @@ -1832,6 +1894,7 @@ int TIFFInitCCITTRLEW(TIFF *tif, int scheme) tif->tif_decoderow = Fax3DecodeRLE; tif->tif_decodestrip = Fax3DecodeRLE; tif->tif_decodetile = Fax3DecodeRLE; + tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio; /* * Suppress RTC+EOLs when encoding and word-align data. */ diff --git a/contrib/libs/libtiff/tif_fax3.h b/contrib/libs/libtiff/tif_fax3.h index c3100ce2c6c..60d709c7e5c 100644 --- a/contrib/libs/libtiff/tif_fax3.h +++ b/contrib/libs/libtiff/tif_fax3.h @@ -267,14 +267,14 @@ static const char *StateNames[] = { { \ if (pa >= thisrun + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, "Buffer overflow at line %u of %s %u", \ + TIFFErrorExtR(tif, module, "Buffer overflow at line %d of %s %u", \ sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - *pa++ = RunLength + (x); \ - a0 += (x); \ + *pa++ = (uint32_t)((uint32_t)RunLength + (uint32_t)(x)); \ + a0 += (int)(uint32_t)(x); \ RunLength = 0; \ } while (0) #endif @@ -334,20 +334,20 @@ static const char *StateNames[] = { { \ if (RunLength) \ SETVALUE(0); \ - if (a0 != lastx) \ + if (a0 != (int)lastx) \ { \ badlength(a0, lastx); \ - while (a0 > lastx && pa > thisrun) \ - a0 -= *--pa; \ - if (a0 < lastx) \ + while (a0 > (int)lastx && pa > thisrun) \ + a0 -= (int)*--pa; \ + if (a0 < (int)lastx) \ { \ if (a0 < 0) \ a0 = 0; \ if ((pa - thisrun) & 1) \ SETVALUE(0); \ - SETVALUE(lastx - a0); \ + SETVALUE((uint32_t)((int)lastx - a0)); \ } \ - else if (a0 > lastx) \ + else if (a0 > (int)lastx) \ { \ SETVALUE(lastx); \ SETVALUE(0); \ @@ -385,8 +385,9 @@ static const char *StateNames[] = { goto doneWhite1d; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = \ + (int)((uint32_t)RunLength + TabEnt->Param); \ break; \ default: \ unexpected("WhiteTable", a0); \ @@ -394,7 +395,7 @@ static const char *StateNames[] = { } \ } \ doneWhite1d: \ - if (a0 >= lastx) \ + if (a0 >= (int)lastx) \ goto done1d; \ for (;;) \ { \ @@ -409,8 +410,8 @@ static const char *StateNames[] = { goto doneBlack1d; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 += (int)TabEnt->Param; \ + RunLength += (int)TabEnt->Param; \ break; \ default: \ unexpected("BlackTable", a0); \ @@ -418,7 +419,7 @@ static const char *StateNames[] = { } \ } \ doneBlack1d: \ - if (a0 >= lastx) \ + if (a0 >= (int)lastx) \ goto done1d; \ if (*(pa - 1) == 0 && *(pa - 2) == 0) \ pa -= 2; \ @@ -439,17 +440,18 @@ static const char *StateNames[] = { do \ { \ if (pa != thisrun) \ - while (b1 <= a0 && b1 < lastx) \ + while (b1 <= a0 && b1 < (int)lastx) \ { \ if (pb + 1 >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR( \ - tif, module, "Buffer overflow at line %u of %s %u", \ - sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip); \ + TIFFErrorExtR(tif, module, \ + "Buffer overflow at line %d of %s %u", \ + sp->line, isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += pb[0] + pb[1]; \ + b1 += (int)(pb[0] + pb[1]); \ pb += 2; \ } \ } while (0) @@ -460,14 +462,15 @@ static const char *StateNames[] = { #define EXPAND2D(eoflab) \ do \ { \ - while (a0 < lastx) \ + while (a0 < (int)lastx) \ { \ if (pa >= thisrun + sp->nruns) \ { \ - TIFFErrorExtR( \ - tif, module, "Buffer overflow at line %u of %s %u", \ - sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip); \ + TIFFErrorExtR(tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ LOOKUP8(7, TIFFFaxMainTable, eof2d); \ @@ -477,18 +480,19 @@ static const char *StateNames[] = { CHECK_b1; \ if (pb + 1 >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ - RunLength += b1 - a0; \ + b1 = b1 + (int)*pb++; \ + RunLength = \ + (int)((uint32_t)RunLength + (uint32_t)(b1 - a0)); \ a0 = b1; \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_Horiz: \ if ((pa - thisrun) & 1) \ @@ -503,8 +507,9 @@ static const char *StateNames[] = { goto doneWhite2da; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badBlack2d; \ @@ -521,8 +526,9 @@ static const char *StateNames[] = { goto doneBlack2da; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badWhite2d; \ @@ -542,8 +548,9 @@ static const char *StateNames[] = { goto doneWhite2db; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badWhite2d; \ @@ -560,8 +567,9 @@ static const char *StateNames[] = { goto doneBlack2db; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badBlack2d; \ @@ -576,47 +584,47 @@ static const char *StateNames[] = { SETVALUE(b1 - a0); \ if (pb >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_VR: \ CHECK_b1; \ - SETVALUE(b1 - a0 + TabEnt->Param); \ + SETVALUE((int)((uint32_t)(b1 - a0) + TabEnt->Param)); \ if (pb >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_VL: \ CHECK_b1; \ - if (b1 < (int)(a0 + TabEnt->Param)) \ + if (b1 < (int)((uint32_t)a0 + TabEnt->Param)) \ { \ unexpected("VL", a0); \ goto eol2d; \ } \ - SETVALUE(b1 - a0 - TabEnt->Param); \ - b1 -= *--pb; \ + SETVALUE((int)((uint32_t)(b1 - a0) - TabEnt->Param)); \ + b1 = b1 - (int)*--pb; \ break; \ case S_Ext: \ - *pa++ = lastx - a0; \ + *pa++ = (uint32_t)((int)lastx - a0); \ extension(a0); \ goto eol2d; \ case S_EOL: \ - *pa++ = lastx - a0; \ + *pa++ = (uint32_t)((int)lastx - a0); \ NeedBits8(4, eof2d); \ if (GetBits(4)) \ unexpected("EOL", a0); \ diff --git a/contrib/libs/libtiff/tif_getimage.c b/contrib/libs/libtiff/tif_getimage.c index 9d4259ef831..4a759a65482 100644 --- a/contrib/libs/libtiff/tif_getimage.c +++ b/contrib/libs/libtiff/tif_getimage.c @@ -29,6 +29,7 @@ */ #include "tiffiop.h" #include <limits.h> +#include <math.h> #include <stdio.h> static int gtTileContig(TIFFRGBAImage *, uint32_t *, uint32_t, uint32_t); @@ -57,21 +58,21 @@ static const char photoTag[] = "PhotometricInterpretation"; static const TIFFDisplay display_sRGB = { {/* XYZ -> luminance matrix */ - {3.2410F, -1.5374F, -0.4986F}, - {-0.9692F, 1.8760F, 0.0416F}, - {0.0556F, -0.2040F, 1.0570F}}, - 100.0F, - 100.0F, - 100.0F, /* Light o/p for reference white */ + {3.2410f, -1.5374f, -0.4986f}, + {-0.9692f, 1.8760f, 0.0416f}, + {0.0556f, -0.2040f, 1.0570f}}, + 100.0f, + 100.0f, + 100.0f, /* Light o/p for reference white */ 255, 255, 255, /* Pixel values for ref. white */ - 1.0F, - 1.0F, - 1.0F, /* Residual light o/p for black pixel */ - 2.4F, - 2.4F, - 2.4F, /* Gamma values for the three guns */ + 1.0f, + 1.0f, + 1.0f, /* Residual light o/p for black pixel */ + 2.4f, + 2.4f, + 2.4f, /* Gamma values for the three guns */ }; /* @@ -316,7 +317,7 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, uint16_t compress; int colorchannels; uint16_t *red_orig, *green_orig, *blue_orig; - int n_color; + uint32_t n_color; if (!TIFFRGBAImageOK(tif, emsg)) return 0; @@ -372,6 +373,8 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, case EXTRASAMPLE_UNASSALPHA: /* data is not pre-multiplied */ img->alpha = sampleinfo[0]; break; + default: + break; } } @@ -421,13 +424,13 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, } /* copy the colormaps so we can modify them */ - n_color = (1U << img->bitspersample); - img->redcmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); - img->greencmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); - img->bluecmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); + n_color = (uint32_t)(1U << img->bitspersample); + img->redcmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); + img->greencmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); + img->bluecmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); if (!img->redcmap || !img->greencmap || !img->bluecmap) { snprintf(emsg, EMSG_BUF_SIZE, @@ -604,7 +607,7 @@ int TIFFRGBAImageGet(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, * Width is checked in img->get() function individually. */ if (0 <= img->row_offset && (uint32_t)img->row_offset < img->height) { - uint32_t hx = img->height - img->row_offset; + uint32_t hx = img->height - (uint32_t)img->row_offset; if (h > hx) { /* Adapt parameters to read only available lines and put image @@ -617,7 +620,7 @@ int TIFFRGBAImageGet(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), "Error in TIFFRGBAImageGet: row offset %d exceeds " - "image height %d", + "image height %u", img->row_offset, img->height); return 0; } @@ -751,13 +754,13 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, if (0 <= img->col_offset && (uint32_t)img->col_offset < img->width) { - wmin = TIFFmin(w, img->width - img->col_offset); + wmin = TIFFmin(w, img->width - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtTileContig: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, img->width); return 0; } @@ -785,7 +788,7 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } else { - if (tw > ((int64_t)INT_MAX + w)) + if (tw > ((int64_t)INT_MAX + w) || w > ((int64_t)INT_MAX + tw)) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s", "unsupported tile size (too wide)"); @@ -804,9 +807,9 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* * Leftmost tile is clipped on left side if col_offset > 0. */ - leftmost_fromskew = img->col_offset % tw; - leftmost_tw = tw - leftmost_fromskew; - int64_t skew_i64 = (int64_t)toskew + leftmost_fromskew; + leftmost_fromskew = (int32_t)((uint32_t)img->col_offset % tw); + leftmost_tw = (uint32_t)((int32_t)tw - leftmost_fromskew); + int64_t skew_i64 = (int64_t)toskew + (int64_t)(int32_t)leftmost_fromskew; if (skew_i64 > INT_MAX || skew_i64 < INT_MIN) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s %" PRId64, "Invalid skew", @@ -816,34 +819,36 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, leftmost_toskew = (int32_t)skew_i64; for (row = 0; ret != 0 && row < h; row += nrow) { - rowstoread = th - (row + img->row_offset) % th; + rowstoread = th - ((uint32_t)row + (uint32_t)img->row_offset) % th; nrow = (row + rowstoread > h ? h - row : rowstoread); fromskew = leftmost_fromskew; this_tw = leftmost_tw; this_toskew = leftmost_toskew; tocol = 0; - col = img->col_offset; + col = (uint32_t)img->col_offset; /* wmin: only write imagewidth if raster is bigger. */ while (tocol < wmin) { if (_TIFFReadTileAndAllocBuffer(tif, (void **)&buf, bufsize, col, - row + img->row_offset, 0, - 0) == (tmsize_t)(-1) && + (uint32_t)row + + (uint32_t)img->row_offset, + 0, 0) == (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; break; } - pos = ((row + img->row_offset) % th) * TIFFTileRowSize(tif) + + pos = (((uint32_t)row + (uint32_t)img->row_offset) % th) * + TIFFTileRowSize(tif) + ((tmsize_t)fromskew * img->samplesperpixel); if (tocol + this_tw > wmin) { /* * Rightmost tile is clipped on right side. */ - fromskew = tw - (wmin - tocol); - this_tw = tw - fromskew; - this_toskew = toskew + fromskew; + fromskew = (int32_t)(tw - (wmin - tocol)); + this_tw = (uint32_t)((int32_t)tw - fromskew); + this_toskew = (int32_t)(toskew + (int32_t)(uint32_t)fromskew); } tmsize_t roffset = (tmsize_t)y * w + tocol; (*put)(img, raster + roffset, tocol, y, this_tw, nrow, fromskew, @@ -859,7 +864,8 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, this_toskew = toskew; } - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } _TIFFfreeExt(img->tif, buf); @@ -869,7 +875,7 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -924,13 +930,13 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < img->width) { - wmin = TIFFmin(w, img->width - img->col_offset); + wmin = TIFFmin(w, img->width - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtTileSeparate: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, img->width); return 0; } @@ -960,7 +966,7 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } else { - if (tw > ((int64_t)INT_MAX + w)) + if (tw > ((int64_t)INT_MAX + w) || w > ((int64_t)INT_MAX + tw)) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s", "unsupported tile size (too wide)"); @@ -992,9 +998,9 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* * Leftmost tile is clipped on left side if col_offset > 0. */ - leftmost_fromskew = img->col_offset % tw; - leftmost_tw = tw - leftmost_fromskew; - int64_t skew_i64 = (int64_t)toskew + leftmost_fromskew; + leftmost_fromskew = (int32_t)((uint32_t)img->col_offset % tw); + leftmost_tw = (uint32_t)((int32_t)tw - leftmost_fromskew); + int64_t skew_i64 = (int64_t)toskew + (int64_t)(int32_t)leftmost_fromskew; if (skew_i64 > INT_MAX || skew_i64 < INT_MIN) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s %" PRId64, "Invalid skew", @@ -1004,21 +1010,22 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, leftmost_toskew = (int32_t)skew_i64; for (row = 0; ret != 0 && row < h; row += nrow) { - rowstoread = th - (row + img->row_offset) % th; + rowstoread = th - ((uint32_t)row + (uint32_t)img->row_offset) % th; nrow = (row + rowstoread > h ? h - row : rowstoread); fromskew = leftmost_fromskew; this_tw = leftmost_tw; this_toskew = leftmost_toskew; tocol = 0; - col = img->col_offset; + col = (uint32_t)img->col_offset; /* wmin: only write imagewidth if raster is bigger. */ while (tocol < wmin) { if (buf == NULL) { - if (_TIFFReadTileAndAllocBuffer(tif, (void **)&buf, bufsize, - col, row + img->row_offset, 0, - 0) == (tmsize_t)(-1) && + if (_TIFFReadTileAndAllocBuffer( + tif, (void **)&buf, bufsize, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + 0) == (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; @@ -1037,32 +1044,36 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, pa = (alpha ? (p2 + tilesize) : NULL); } } - else if (TIFFReadTile(tif, p0, col, row + img->row_offset, 0, 0) == - (tmsize_t)(-1) && + else if (TIFFReadTile(tif, p0, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + 0) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (colorchannels > 1 && - TIFFReadTile(tif, p1, col, row + img->row_offset, 0, 1) == - (tmsize_t)(-1) && + TIFFReadTile(tif, p1, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)1) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (colorchannels > 1 && - TIFFReadTile(tif, p2, col, row + img->row_offset, 0, 2) == - (tmsize_t)(-1) && + TIFFReadTile(tif, p2, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)2) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (alpha && - TIFFReadTile(tif, pa, col, row + img->row_offset, 0, - colorchannels) == (tmsize_t)(-1) && + TIFFReadTile(tif, pa, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)colorchannels) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; @@ -1071,16 +1082,17 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* For SEPARATE the pos-offset is per sample and should not be * multiplied by img->samplesperpixel. */ - pos = ((row + img->row_offset) % th) * TIFFTileRowSize(tif) + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % th) * + TIFFTileRowSize(tif) + (tmsize_t)fromskew; if (tocol + this_tw > wmin) { /* * Rightmost tile is clipped on right side. */ - fromskew = tw - (wmin - tocol); - this_tw = tw - fromskew; - this_toskew = toskew + fromskew; + fromskew = (int32_t)(tw - (wmin - tocol)); + this_tw = (uint32_t)((int32_t)tw - fromskew); + this_toskew = (int32_t)(toskew + (int32_t)(uint32_t)fromskew); } tmsize_t roffset = (tmsize_t)y * w + tocol; (*put)(img, raster + roffset, tocol, y, this_tw, nrow, fromskew, @@ -1097,7 +1109,8 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, this_toskew = toskew; } - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1106,7 +1119,7 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1155,13 +1168,13 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < imagewidth) { - wmin = TIFFmin(w, imagewidth - img->col_offset); + wmin = TIFFmin(w, imagewidth - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtStripContig: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, imagewidth); return 0; } @@ -1195,7 +1208,7 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, y = 0; /* Skew forward to the end of the raster width of the row currently * copied. */ - toskew = w - wmin; + toskew = (int32_t)(w - wmin); } TIFFGetFieldDefaulted(tif, TIFFTAG_ROWSPERSTRIP, &rowsperstrip); @@ -1206,16 +1219,18 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } scanline = TIFFScanlineSize(tif); - fromskew = (w < imagewidth ? imagewidth - w : 0); + fromskew = (int32_t)(w < imagewidth ? imagewidth - w : 0); for (row = 0; row < h; row += nrow) { uint32_t temp; - rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip; + rowstoread = rowsperstrip - + (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip; nrow = (row + rowstoread > h ? h - row : rowstoread); nrowsub = nrow; if ((nrowsub % subsamplingver) != 0) nrowsub += subsamplingver - nrowsub % subsamplingver; - temp = (row + img->row_offset) % rowsperstrip + nrowsub; + temp = + (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip + nrowsub; if (scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline)) { TIFFErrorExtR(tif, TIFFFileName(tif), @@ -1223,21 +1238,27 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, return 0; } if (_TIFFReadEncodedStripAndAllocBuffer( - tif, TIFFComputeStrip(tif, row + img->row_offset, 0), + tif, + TIFFComputeStrip(tif, + (uint32_t)((int32_t)row + img->row_offset), 0), (void **)(&buf), maxstripsize, - temp * scanline) == (tmsize_t)(-1) && + (tmsize_t)((size_t)temp * (size_t)scanline)) == + (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; break; } - pos = ((row + img->row_offset) % rowsperstrip) * scanline + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % + rowsperstrip) * + scanline + ((tmsize_t)img->col_offset * img->samplesperpixel); tmsize_t roffset = (tmsize_t)y * w; (*put)(img, raster + roffset, 0, y, wmin, nrow, fromskew, toskew, buf + pos); - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1250,7 +1271,7 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1300,13 +1321,13 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < imagewidth) { - wmin = TIFFmin(w, imagewidth - img->col_offset); + wmin = TIFFmin(w, imagewidth - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtStripSeparate: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, imagewidth); return 0; } @@ -1337,7 +1358,7 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, y = 0; /* Skew forward to the end of the raster width of the row currently * written. */ - toskew = w - wmin; + toskew = (int32_t)(w - wmin); } switch (img->photometric) @@ -1361,14 +1382,16 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } scanline = TIFFScanlineSize(tif); - fromskew = (w < imagewidth ? imagewidth - w : 0); + fromskew = (int32_t)(w < imagewidth ? imagewidth - w : 0); for (row = 0; row < h; row += nrow) { uint32_t temp; - rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip; + rowstoread = + rowsperstrip - + ((uint32_t)((int32_t)row + img->row_offset) % rowsperstrip); nrow = (row + rowstoread > h ? h - row : rowstoread); - offset_row = row + img->row_offset; - temp = (row + img->row_offset) % rowsperstrip + nrow; + offset_row = (uint32_t)((int32_t)row + img->row_offset); + temp = (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip + nrow; if (scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline)) { TIFFErrorExtR(tif, TIFFFileName(tif), @@ -1435,12 +1458,15 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* For SEPARATE the pos-offset is per sample and should not be * multiplied by img->samplesperpixel. */ - pos = ((row + img->row_offset) % rowsperstrip) * scanline + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % + rowsperstrip) * + scanline + (tmsize_t)img->col_offset; tmsize_t roffset = (tmsize_t)y * w; (*put)(img, raster + roffset, 0, y, wmin, nrow, fromskew, toskew, p0 + pos, p1 + pos, p2 + pos, (alpha ? (pa + pos) : NULL)); - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1449,7 +1475,7 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1505,6 +1531,9 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, op; /*-fallthrough*/ \ case 1: \ op; \ + break; \ + default: \ + break; \ } #define CASE4(x, op) \ switch (x) \ @@ -1515,6 +1544,9 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, op; /*-fallthrough*/ \ case 1: \ op; \ + break; \ + default: \ + break; \ } #define NOP @@ -1581,13 +1613,6 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, #define PACK4(r, g, b, a) \ ((uint32_t)(r) | ((uint32_t)(g) << 8) | ((uint32_t)(b) << 16) | \ ((uint32_t)(a) << 24)) -#define W2B(v) (((v) >> 8) & 0xff) -/* TODO: PACKW should have be made redundant in favor of Bitdepth16To8 LUT */ -#define PACKW(r, g, b) \ - ((uint32_t)W2B(r) | ((uint32_t)W2B(g) << 8) | ((uint32_t)W2B(b) << 16) | A1) -#define PACKW4(r, g, b, a) \ - ((uint32_t)W2B(r) | ((uint32_t)W2B(g) << 8) | ((uint32_t)W2B(b) << 16) | \ - ((uint32_t)W2B(a) << 24)) #define DECLAREContigPutFunc(name) \ static void name(TIFFRGBAImage *img, uint32_t *cp, uint32_t x, uint32_t y, \ @@ -1706,7 +1731,7 @@ DECLAREContigPutFunc(putagreytile) { for (x = w; x > 0; --x) { - *cp++ = BWmap[*pp][0] & ((uint32_t) * (pp + 1) << 24 | ~A1); + *cp++ = BWmap[*pp][0] & ((uint32_t)*(pp + 1) << 24 | ~A1); pp += samplesperpixel; } cp += toskew; @@ -1721,6 +1746,9 @@ DECLAREContigPutFunc(put16bitbwtile) { int samplesperpixel = img->samplesperpixel; uint32_t **BWmap = img->BWmap; + /* Convert pixel skew to byte skew (16-bit samples) */ + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)(2 * samplesperpixel); (void)y; for (; h > 0; --h) @@ -1736,7 +1764,7 @@ DECLAREContigPutFunc(put16bitbwtile) wp += samplesperpixel; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1806,13 +1834,14 @@ DECLAREContigPutFunc(putRGBcontig8bittile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { UNROLL8(w, NOP, *cp++ = PACK(pp[0], pp[1], pp[2]); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1826,13 +1855,14 @@ DECLAREContigPutFunc(putRGBAAcontig8bittile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { UNROLL8(w, NOP, *cp++ = PACK4(pp[0], pp[1], pp[2], pp[3]); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1844,7 +1874,8 @@ DECLAREContigPutFunc(putRGBUAcontig8bittile) { int samplesperpixel = img->samplesperpixel; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { uint32_t r, g, b, a; @@ -1860,7 +1891,7 @@ DECLAREContigPutFunc(putRGBUAcontig8bittile) pp += samplesperpixel; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1872,7 +1903,8 @@ DECLAREContigPutFunc(putRGBcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -1882,7 +1914,7 @@ DECLAREContigPutFunc(putRGBcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1895,7 +1927,8 @@ DECLAREContigPutFunc(putRGBAAcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -1905,7 +1938,7 @@ DECLAREContigPutFunc(putRGBAAcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1918,7 +1951,8 @@ DECLAREContigPutFunc(putRGBUAcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { uint32_t r, g, b, a; @@ -1934,7 +1968,7 @@ DECLAREContigPutFunc(putRGBUAcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1950,14 +1984,17 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKtile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { - UNROLL8(w, NOP, k = 255 - pp[3]; r = (k * (255 - pp[0])) / 255; - g = (k * (255 - pp[1])) / 255; b = (k * (255 - pp[2])) / 255; + UNROLL8(w, NOP, k = (uint16_t)(255 - pp[3]); + r = (uint16_t)((k * (uint16_t)(255 - pp[0])) / 255); + g = (uint16_t)((k * (uint16_t)(255 - pp[1])) / 255); + b = (uint16_t)((k * (uint16_t)(255 - pp[2])) / 255); *cp++ = PACK(r, g, b); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1970,22 +2007,23 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKMaptile) { int samplesperpixel = img->samplesperpixel; TIFFRGBValue *Map = img->Map; - uint16_t r, g, b, k; + unsigned int r, g, b, k; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) { - k = 255 - pp[3]; - r = (k * (255 - pp[0])) / 255; - g = (k * (255 - pp[1])) / 255; - b = (k * (255 - pp[2])) / 255; + k = 255U - pp[3]; + r = (k * (255U - pp[0])) / 255U; + g = (k * (255U - pp[1])) / 255U; + b = (k * (255U - pp[2])) / 255U; *cp++ = PACK(Map[r], Map[g], Map[b]); pp += samplesperpixel; } - pp += fromskew; + pp += fromskewLocal; cp += toskew; } } @@ -2038,13 +2076,13 @@ DECLARESepPutFunc(putCMYKseparate8bittile) (void)y; for (; h > 0; --h) { - uint32_t rv, gv, bv, kv; + unsigned int rv, gv, bv, kv; for (x = w; x > 0; --x) { - kv = 255 - *a++; - rv = (kv * (255 - *r++)) / 255; - gv = (kv * (255 - *g++)) / 255; - bv = (kv * (255 - *b++)) / 255; + kv = 255U - *a++; + rv = (kv * (255U - *r++)) / 255U; + gv = (kv * (255U - *g++)) / 255U; + bv = (kv * (255U - *b++)) / 255U; *cp++ = PACK4(rv, gv, bv, 255); } SKEW4(r, g, b, a, fromskew); @@ -2156,7 +2194,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab8) float X, Y, Z; uint32_t r, g, b; (void)y; - fromskew *= 3; + const tmsize_t fromskewLocal = (tmsize_t)fromskew * (tmsize_t)3; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -2168,7 +2206,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab8) pp += 3; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -2181,7 +2219,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) uint32_t r, g, b; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= 3; + const tmsize_t fromskewLocal = (tmsize_t)fromskew * (tmsize_t)3; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -2193,7 +2231,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) wp += 3; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -2204,7 +2242,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) #define YCbCrtoRGB(dst, Y) \ { \ uint32_t r, g, b; \ - TIFFYCbCrtoRGB(img->ycbcr, (Y), Cb, Cr, &r, &g, &b); \ + TIFFYCbCrtoRGB(img->ycbcr, (Y), (int32_t)Cb, (int32_t)Cr, &r, &g, &b); \ dst = PACK(r, g, b); \ } @@ -2216,11 +2254,12 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) uint32_t *cp1 = cp + w + toskew; uint32_t *cp2 = cp1 + w + toskew; uint32_t *cp3 = cp2 + w + toskew; - int32_t incr = 3 * w + 4 * toskew; + const tmsize_t incr = 3 * (tmsize_t)w + 4 * (tmsize_t)toskew; (void)y; /* adjust fromskew */ - fromskew = (fromskew / 4) * (4 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 4 + 2); if ((h & 3) == 0 && (w & 3) == 0) { for (; h >= 4; h -= 4) @@ -2258,7 +2297,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) cp1 += incr; cp2 += incr; cp3 += incr; - pp += fromskew; + pp += fromskewLocal; } } else @@ -2282,7 +2321,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[3], pp[7]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[3], pp[3]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 3: switch (h) { @@ -2294,7 +2333,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[2], pp[6]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[2], pp[2]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 2: switch (h) { @@ -2306,7 +2345,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[1], pp[5]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[1], pp[1]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 1: switch (h) { @@ -2318,7 +2357,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[0], pp[4]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[0], pp[0]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ } if (x < 4) { @@ -2345,7 +2384,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) cp1 += incr; cp2 += incr; cp3 += incr; - pp += fromskew; + pp += fromskewLocal; } } } @@ -2356,10 +2395,11 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) DECLAREContigPutFunc(putcontig8bitYCbCr42tile) { uint32_t *cp1 = cp + w + toskew; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 4) * (4 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 2 + 2); if ((w & 3) == 0 && (h & 1) == 0) { for (; h >= 2; h -= 2) @@ -2385,7 +2425,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) } while (--x); cp += incr; cp1 += incr; - pp += fromskew; + pp += fromskewLocal; } } else @@ -2405,7 +2445,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[3], pp[7]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[3], pp[3]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 3: switch (h) { @@ -2413,7 +2453,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[2], pp[6]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[2], pp[2]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 2: switch (h) { @@ -2421,7 +2461,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[1], pp[5]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[1], pp[1]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 1: switch (h) { @@ -2429,7 +2469,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[0], pp[4]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[0], pp[0]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ } if (x < 4) { @@ -2450,7 +2490,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) h -= 2; cp += incr; cp1 += incr; - pp += fromskew; + pp += fromskewLocal; } } } @@ -2461,7 +2501,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) DECLAREContigPutFunc(putcontig8bitYCbCr41tile) { (void)y; - fromskew = (fromskew / 4) * (4 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 1 + 2); do { x = w >> 2; @@ -2495,6 +2536,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) YCbCrtoRGB(cp[0], pp[0]); /*-fallthrough*/ case 0: break; + default: + break; } cp += (w & 3); @@ -2502,7 +2545,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2512,9 +2555,10 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) DECLAREContigPutFunc(putcontig8bitYCbCr22tile) { uint32_t *cp2; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 2) * (2 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 2) * (tmsize_t)(2 * 2 + 2); cp2 = cp + w + toskew; while (h >= 2) { @@ -2544,7 +2588,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr22tile) } cp += incr; cp2 += incr; - pp += fromskew; + pp += fromskewLocal; h -= 2; } if (h == 1) @@ -2576,7 +2620,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr22tile) DECLAREContigPutFunc(putcontig8bitYCbCr21tile) { (void)y; - fromskew = (fromskew / 2) * (2 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 2) * (tmsize_t)(2 * 1 + 2); do { x = w >> 1; @@ -2605,7 +2650,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile) } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2615,9 +2660,10 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile) DECLAREContigPutFunc(putcontig8bitYCbCr12tile) { uint32_t *cp2; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 1) * (1 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 1) * (tmsize_t)(1 * 2 + 2); cp2 = cp + w + toskew; while (h >= 2) { @@ -2634,7 +2680,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr12tile) } while (--x); cp += incr; cp2 += incr; - pp += fromskew; + pp += fromskewLocal; h -= 2; } if (h == 1) @@ -2657,7 +2703,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr12tile) DECLAREContigPutFunc(putcontig8bitYCbCr11tile) { (void)y; - fromskew = (fromskew / 1) * (1 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 1) * (tmsize_t)(1 * 1 + 2); do { x = w; /* was x = w>>1; patched 2000/09/25 [email protected] */ @@ -2671,7 +2718,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr11tile) pp += 3; } while (--x); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2730,8 +2777,8 @@ static int initYCbCrConversion(TIFFRGBAImage *img) /* Do some validation to avoid later issues. Detect NaN for now */ /* and also if lumaGreen is zero since we divide by it later */ - if (luma[0] != luma[0] || luma[1] != luma[1] || luma[1] == 0.0 || - luma[2] != luma[2]) + if (isnan(luma[0]) || isnan(luma[1]) || TIFF_FLOAT_EQ(luma[1], 0.0f) || + isnan(luma[2])) { TIFFErrorExtR(img->tif, module, "Invalid values for YCbCrCoefficients tag"); @@ -2763,7 +2810,7 @@ static tileContigRoutine initCIELabConversion(TIFFRGBAImage *img) float refWhite[3]; TIFFGetFieldDefaulted(img->tif, TIFFTAG_WHITEPOINT, &whitePoint); - if (whitePoint[1] == 0.0f) + if (TIFF_FLOAT_EQ(whitePoint[1], 0.0f)) { TIFFErrorExtR(img->tif, module, "Invalid value for WhitePoint tag."); return NULL; @@ -2781,10 +2828,10 @@ static tileContigRoutine initCIELabConversion(TIFFRGBAImage *img) } } - refWhite[1] = 100.0F; + refWhite[1] = 100.0f; refWhite[0] = whitePoint[0] / whitePoint[1] * refWhite[1]; refWhite[2] = - (1.0F - whitePoint[0] - whitePoint[1]) / whitePoint[1] * refWhite[1]; + (1.0f - whitePoint[0] - whitePoint[1]) / whitePoint[1] * refWhite[1]; if (TIFFCIELabToRGBInit(img->cielab, &display_sRGB, refWhite) < 0) { TIFFErrorExtR(img->tif, module, @@ -2820,7 +2867,8 @@ static int makebwmap(TIFFRGBAImage *img) img->BWmap = (uint32_t **)_TIFFmallocExt( img->tif, - 256 * sizeof(uint32_t *) + (256 * nsamples * sizeof(uint32_t))); + (tmsize_t)((size_t)256 * sizeof(uint32_t *) + + ((size_t)256 * (size_t)nsamples * sizeof(uint32_t)))); if (img->BWmap == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -2861,6 +2909,8 @@ static int makebwmap(TIFFRGBAImage *img) case 16: GREY(i); break; + default: + break; } #undef GREY } @@ -2876,14 +2926,14 @@ static int setupMap(TIFFRGBAImage *img) { int32_t x, range; - range = (int32_t)((1L << img->bitspersample) - 1); + range = (int32_t)((1U << img->bitspersample) - 1U); /* treat 16 bit the same as eight bit */ if (img->bitspersample == 16) - range = (int32_t)255; + range = 255; img->Map = (TIFFRGBValue *)_TIFFmallocExt( - img->tif, (range + 1) * sizeof(TIFFRGBValue)); + img->tif, (tmsize_t)((size_t)(range + 1) * sizeof(TIFFRGBValue))); if (img->Map == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -2966,7 +3016,8 @@ static int makecmap(TIFFRGBAImage *img) img->PALmap = (uint32_t **)_TIFFmallocExt( img->tif, - 256 * sizeof(uint32_t *) + (256 * nsamples * sizeof(uint32_t))); + (tmsize_t)((size_t)256 * sizeof(uint32_t *) + + ((size_t)256 * (size_t)nsamples * sizeof(uint32_t)))); if (img->PALmap == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -3006,6 +3057,8 @@ static int makecmap(TIFFRGBAImage *img) case 8: CMAP(i); break; + default: + break; } #undef CMAP } @@ -3048,6 +3101,8 @@ static int buildMap(TIFFRGBAImage *img) if (img->bitspersample <= 8 && !makecmap(img)) return (0); break; + default: + break; } return (1); } @@ -3096,6 +3151,8 @@ static int PickContigCase(TIFFRGBAImage *img) img->put.contig = putRGBcontig16bittile; } break; + default: + break; } break; case PHOTOMETRIC_SEPARATED: @@ -3127,6 +3184,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 1: img->put.contig = put1bitcmaptile; break; + default: + break; } } break; @@ -3154,6 +3213,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 1: img->put.contig = put1bitbwtile; break; + default: + break; } } break; @@ -3175,6 +3236,19 @@ static int PickContigCase(TIFFRGBAImage *img) uint16_t SubsamplingVer; TIFFGetFieldDefaulted(img->tif, TIFFTAG_YCBCRSUBSAMPLING, &SubsamplingHor, &SubsamplingVer); + /* Validate that the image dimensions are compatible with + the subsampling block. All putcontig8bitYCbCrXYtile routines + assume width >= X and height >= Y. */ + if (img->width < SubsamplingHor || + img->height < SubsamplingVer) + { + TIFFErrorExtR(img->tif, TIFFFileName(img->tif), + "YCbCr subsampling (%u,%u) incompatible " + "with image size %ux%u", + SubsamplingHor, SubsamplingVer, + img->width, img->height); + return (0); + } switch ((SubsamplingHor << 4) | SubsamplingVer) { case 0x44: @@ -3198,6 +3272,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 0x11: img->put.contig = putcontig8bitYCbCr11tile; break; + default: + break; } } } @@ -3209,6 +3285,9 @@ static int PickContigCase(TIFFRGBAImage *img) img->put.contig = initCIELabConversion(img); break; } + break; + default: + break; } return ((img->get != NULL) && (img->put.contig != NULL)); } @@ -3260,6 +3339,8 @@ static int PickSeparateCase(TIFFRGBAImage *img) img->put.separate = putRGBseparate16bittile; } break; + default: + break; } break; case PHOTOMETRIC_SEPARATED: @@ -3284,10 +3365,14 @@ static int PickSeparateCase(TIFFRGBAImage *img) img->put.separate = putseparate8bitYCbCr11tile; break; /* TODO: add other cases here */ + default: + break; } } } break; + default: + break; } return ((img->get != NULL) && (img->put.separate != NULL)); } @@ -3298,7 +3383,7 @@ static int BuildMapUaToAa(TIFFRGBAImage *img) uint8_t *m; uint16_t na, nv; assert(img->UaToAa == NULL); - img->UaToAa = _TIFFmallocExt(img->tif, 65536); + img->UaToAa = (uint8_t *)_TIFFmallocExt(img->tif, 65536); if (img->UaToAa == NULL) { TIFFErrorExtR(img->tif, module, "Out of memory"); @@ -3319,7 +3404,7 @@ static int BuildMapBitdepth16To8(TIFFRGBAImage *img) uint8_t *m; uint32_t n; assert(img->Bitdepth16To8 == NULL); - img->Bitdepth16To8 = _TIFFmallocExt(img->tif, 65536); + img->Bitdepth16To8 = (uint8_t *)_TIFFmallocExt(img->tif, 65536); if (img->Bitdepth16To8 == NULL) { TIFFErrorExtR(img->tif, module, "Out of memory"); @@ -3386,7 +3471,7 @@ int TIFFReadRGBAStripExt(TIFF *tif, uint32_t row, uint32_t *raster, return (0); } - img.row_offset = row; + img.row_offset = (int)row; img.col_offset = 0; if (row + rowsperstrip > img.height) @@ -3497,8 +3582,8 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, * Read the chunk of imagery. */ - img.row_offset = row; - img.col_offset = col; + img.row_offset = (int)row; + img.col_offset = (int)col; ok = TIFFRGBAImageGet(&img, raster, read_xsize, read_ysize); @@ -3520,15 +3605,16 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, memmove(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize, raster + (size_t)(read_ysize - i_row - 1) * read_xsize, read_xsize * sizeof(uint32_t)); - _TIFFmemset(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize + - read_xsize, - 0, sizeof(uint32_t) * (tile_xsize - read_xsize)); + _TIFFmemset( + raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize + read_xsize, + 0, + (tmsize_t)(sizeof(uint32_t) * (size_t)(tile_xsize - read_xsize))); } for (i_row = read_ysize; i_row < tile_ysize; i_row++) { _TIFFmemset(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize, 0, - sizeof(uint32_t) * tile_xsize); + (tmsize_t)(sizeof(uint32_t) * (size_t)tile_xsize)); } return (ok); diff --git a/contrib/libs/libtiff/tif_hash_set.c b/contrib/libs/libtiff/tif_hash_set.c index 81dea3fcf26..7c27191ceb6 100644 --- a/contrib/libs/libtiff/tif_hash_set.c +++ b/contrib/libs/libtiff/tif_hash_set.c @@ -76,7 +76,7 @@ static const int anPrimes[] = { 100663319, 201326611, 402653189, 805306457, 1610612741}; /************************************************************************/ -/* TIFFHashSetHashPointer() */ +/* TIFFHashSetHashPointer() */ /************************************************************************/ /** @@ -93,7 +93,7 @@ static unsigned long TIFFHashSetHashPointer(const void *elt) } /************************************************************************/ -/* TIFFHashSetEqualPointer() */ +/* TIFFHashSetEqualPointer() */ /************************************************************************/ /** @@ -111,7 +111,7 @@ static bool TIFFHashSetEqualPointer(const void *elt1, const void *elt2) } /************************************************************************/ -/* TIFFHashSetNew() */ +/* TIFFHashSetNew() */ /************************************************************************/ /** @@ -164,7 +164,7 @@ TIFFHashSet *TIFFHashSetNew(TIFFHashSetHashFunc fnHashFunc, } /************************************************************************/ -/* TIFFHashSetSize() */ +/* TIFFHashSetSize() */ /************************************************************************/ /** @@ -184,7 +184,7 @@ int TIFFHashSetSize(const TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetGetNewListElt() */ +/* TIFFHashSetGetNewListElt() */ /************************************************************************/ static TIFFList *TIFFHashSetGetNewListElt(TIFFHashSet *set) @@ -202,7 +202,7 @@ static TIFFList *TIFFHashSetGetNewListElt(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetReturnListElt() */ +/* TIFFHashSetReturnListElt() */ /************************************************************************/ static void TIFFHashSetReturnListElt(TIFFHashSet *set, TIFFList *psList) @@ -220,7 +220,7 @@ static void TIFFHashSetReturnListElt(TIFFHashSet *set, TIFFList *psList) } /************************************************************************/ -/* TIFFHashSetClearInternal() */ +/* TIFFHashSetClearInternal() */ /************************************************************************/ static void TIFFHashSetClearInternal(TIFFHashSet *set, bool bFinalize) @@ -246,7 +246,7 @@ static void TIFFHashSetClearInternal(TIFFHashSet *set, bool bFinalize) } /************************************************************************/ -/* TIFFListDestroy() */ +/* TIFFListDestroy() */ /************************************************************************/ /** @@ -270,7 +270,7 @@ static void TIFFListDestroy(TIFFList *psList) } /************************************************************************/ -/* TIFFHashSetDestroy() */ +/* TIFFHashSetDestroy() */ /************************************************************************/ /** @@ -295,7 +295,7 @@ void TIFFHashSetDestroy(TIFFHashSet *set) #ifdef notused /************************************************************************/ -/* TIFFHashSetClear() */ +/* TIFFHashSetClear() */ /************************************************************************/ /** @@ -319,7 +319,7 @@ void TIFFHashSetClear(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetForeach() */ +/* TIFFHashSetForeach() */ /************************************************************************/ /** @@ -360,14 +360,14 @@ void TIFFHashSetForeach(TIFFHashSet *set, TIFFHashSetIterEltFunc fnIterFunc, #endif /************************************************************************/ -/* TIFFHashSetRehash() */ +/* TIFFHashSetRehash() */ /************************************************************************/ static bool TIFFHashSetRehash(TIFFHashSet *set) { int nNewAllocatedSize = anPrimes[set->nIndiceAllocatedSize]; TIFFList **newTabList = - (TIFFList **)(calloc(nNewAllocatedSize, sizeof(TIFFList *))); + (TIFFList **)(calloc((size_t)nNewAllocatedSize, sizeof(TIFFList *))); if (newTabList == NULL) return false; #ifdef HASH_DEBUG @@ -384,7 +384,7 @@ static bool TIFFHashSetRehash(TIFFHashSet *set) while (cur) { const unsigned long nNewHashVal = - set->fnHashFunc(cur->pData) % nNewAllocatedSize; + set->fnHashFunc(cur->pData) % (unsigned long)nNewAllocatedSize; #ifdef HASH_DEBUG if (newTabList[nNewHashVal]) set->nCollisions++; @@ -403,12 +403,13 @@ static bool TIFFHashSetRehash(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetFindPtr() */ +/* TIFFHashSetFindPtr() */ /************************************************************************/ static void **TIFFHashSetFindPtr(TIFFHashSet *set, const void *elt) { - const unsigned long nHashVal = set->fnHashFunc(elt) % set->nAllocatedSize; + const unsigned long nHashVal = + set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize; TIFFList *cur = set->tabList[nHashVal]; while (cur) { @@ -464,7 +465,8 @@ bool TIFFHashSetInsert(TIFFHashSet *set, void *elt) } } - const unsigned long nHashVal = set->fnHashFunc(elt) % set->nAllocatedSize; + const unsigned long nHashVal = + set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize; #ifdef HASH_DEBUG if (set->tabList[nHashVal]) set->nCollisions++; @@ -486,7 +488,7 @@ bool TIFFHashSetInsert(TIFFHashSet *set, void *elt) } /************************************************************************/ -/* TIFFHashSetLookup() */ +/* TIFFHashSetLookup() */ /************************************************************************/ /** @@ -532,7 +534,8 @@ static bool TIFFHashSetRemoveInternal(TIFFHashSet *set, const void *elt, } } - int nHashVal = (int)(set->fnHashFunc(elt) % set->nAllocatedSize); + int nHashVal = + (int)(set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize); TIFFList *cur = set->tabList[nHashVal]; TIFFList *prev = NULL; while (cur) @@ -581,7 +584,7 @@ bool TIFFHashSetRemove(TIFFHashSet *set, const void *elt) #ifdef notused /************************************************************************/ -/* TIFFHashSetRemoveDeferRehash() */ +/* TIFFHashSetRemoveDeferRehash() */ /************************************************************************/ /** diff --git a/contrib/libs/libtiff/tif_jbig.c b/contrib/libs/libtiff/tif_jbig.c index d0fae1f124a..4c35f3c0dcf 100644 --- a/contrib/libs/libtiff/tif_jbig.c +++ b/contrib/libs/libtiff/tif_jbig.c @@ -33,7 +33,14 @@ #include "tiffiop.h" #ifdef JBIG_SUPPORT +#ifdef __cplusplus +extern "C" +{ +#endif #error #include "jbig.h" +#ifdef __cplusplus +} +#endif static int JBIGSetupDecode(TIFF *tif) { @@ -100,7 +107,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) decodedSize = jbg_dec_getsize(&decoder); if ((tmsize_t)decodedSize < size) { - memset(buffer + decodedSize, 0, (size_t)(size - decodedSize)); + memset(buffer + decodedSize, 0, (size_t)(size - (tmsize_t)decodedSize)); TIFFWarningExtR(tif, "JBIG", "Only decoded %lu bytes, whereas %" TIFF_SSIZE_FORMAT " requested", @@ -116,7 +123,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) return 0; } pImage = jbg_dec_getimage(&decoder, 0); - _TIFFmemcpy(buffer, pImage, decodedSize); + _TIFFmemcpy(buffer, pImage, (tmsize_t)decodedSize); jbg_dec_free(&decoder); tif->tif_rawcp += tif->tif_rawcc; diff --git a/contrib/libs/libtiff/tif_jpeg.c b/contrib/libs/libtiff/tif_jpeg.c index aba5f99b7c0..b9a85cb18d9 100644 --- a/contrib/libs/libtiff/tif_jpeg.c +++ b/contrib/libs/libtiff/tif_jpeg.c @@ -22,9 +22,6 @@ * OF THIS SOFTWARE. */ -#define WIN32_LEAN_AND_MEAN -#define VC_EXTRALEAN - #include "tiffiop.h" #include <stdlib.h> @@ -146,20 +143,9 @@ int TIFFJPEGIsFullStripRequired_12(TIFF *tif); #define LONGJMP(jbuf, code) longjmp(jbuf, code) #define JMP_BUF jmp_buf -#ifndef TIFF_jpeg_destination_mgr_defined -#define TIFF_jpeg_destination_mgr_defined -typedef struct jpeg_destination_mgr jpeg_destination_mgr; -#endif - -#ifndef TIFF_jpeg_source_mgr_defined -#define TIFF_jpeg_source_mgr_defined -typedef struct jpeg_source_mgr jpeg_source_mgr; -#endif - -#ifndef TIFF_jpeg_error_mgr_defined -#define TIFF_jpeg_error_mgr_defined -typedef struct jpeg_error_mgr jpeg_error_mgr; -#endif +typedef struct jpeg_destination_mgr tiff_jpeg_destination_mgr; +typedef struct jpeg_source_mgr tiff_jpeg_source_mgr; +typedef struct jpeg_error_mgr tiff_jpeg_error_mgr; /* * State block for each open TIFF file using @@ -174,6 +160,10 @@ typedef struct jpeg_error_mgr jpeg_error_mgr; * so we can safely cast JPEGState* -> jpeg_xxx_struct* * and vice versa! */ +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4324) /* structure padding due to alignment */ +#endif typedef struct { union @@ -184,22 +174,24 @@ typedef struct } cinfo; /* NB: must be first */ int cinfo_initialized; - jpeg_error_mgr err; /* libjpeg error manager */ - JMP_BUF exit_jmpbuf; /* for catching libjpeg failures */ + tiff_jpeg_error_mgr err; /* libjpeg error manager */ + JMP_BUF exit_jmpbuf; /* for catching libjpeg failures */ struct jpeg_progress_mgr progress; /* * The following two members could be a union, but * they're small enough that it's not worth the effort. */ - jpeg_destination_mgr dest; /* data dest for compression */ - jpeg_source_mgr src; /* data source for decompression */ - /* private state */ - TIFF *tif; /* back link needed by some code */ - uint16_t photometric; /* copy of PhotometricInterpretation */ - uint16_t h_sampling; /* luminance sampling factors */ + tiff_jpeg_destination_mgr dest; /* data dest for compression */ + tiff_jpeg_source_mgr src; /* data source for decompression */ + /* private state */ + TIFF *tif; /* back link needed by some code */ + uint16_t photometric; /* copy of PhotometricInterpretation */ + uint16_t h_sampling; /* luminance sampling factors */ uint16_t v_sampling; tmsize_t bytesperline; /* decompressed bytes per scanline */ + uint32_t strile_width; + uint32_t strile_height; /* pointers to intermediate buffers when processing downsampled data */ TIFF_JSAMPARRAY ds_buffer[MAX_COMPONENTS]; int scancount; /* number of "scanlines" accumulated */ @@ -209,6 +201,9 @@ typedef struct int encode_raw_error; } JPEGState; +#ifdef _MSC_VER +#pragma warning(pop) +#endif #define JState(tif) ((JPEGState *)(tif)->tif_data) @@ -559,7 +554,7 @@ static boolean tables_empty_output_buffer(j_compress_ptr cinfo) /* the entire buffer has been filled; enlarge it by 1000 bytes */ newbuf = _TIFFreallocExt(sp->tif, (void *)sp->otherSettings.jpegtables, - (tmsize_t)(sp->otherSettings.jpegtables_length + 1000)); + (tmsize_t)sp->otherSettings.jpegtables_length + 1000); if (newbuf == NULL) ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 100); sp->dest.next_output_byte = @@ -912,7 +907,7 @@ JPEGFixupTagsSubsamplingSec(struct JPEGFixupTagsSubsamplingData *data) return (0); if (n < 2) return (0); - n -= 2; + n = (uint16_t)(n - 2); if (n > 0) JPEGFixupTagsSubsamplingSkip(data, n); } @@ -1016,7 +1011,7 @@ JPEGFixupTagsSubsamplingReadByte(struct JPEGFixupTagsSubsamplingData *data, assert(m < 0x80000000UL); if (TIFFReadFile(data->tif, data->buffer, (tmsize_t)m) != (tmsize_t)m) return (0); - data->buffercurrentbyte = data->buffer; + data->buffercurrentbyte = (uint8_t *)data->buffer; data->bufferbytesleft = m; data->fileoffset += m; data->filebytesleft -= m; @@ -1037,7 +1032,7 @@ JPEGFixupTagsSubsamplingReadWord(struct JPEGFixupTagsSubsamplingData *data, return (0); if (!JPEGFixupTagsSubsamplingReadByte(data, &mb)) return (0); - *result = (ma << 8) | mb; + *result = (uint16_t)((ma << 8) | mb); return (1); } @@ -1159,6 +1154,47 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) return ret; } +static int JPEGComputeStrileWidthHeightBytesPerLine(TIFF *tif, uint16_t s) +{ + JPEGState *sp = JState(tif); + TIFFDirectory *td = &tif->tif_dir; + + /* + * Check image parameters and set decompression parameters. + */ + if (isTiled(tif)) + { + sp->strile_width = td->td_tilewidth; + sp->strile_height = td->td_tilelength; + sp->bytesperline = TIFFTileRowSize(tif); + } + else + { + sp->strile_width = td->td_imagewidth; + sp->strile_height = td->td_imagelength - tif->tif_dir.td_row; + if (sp->strile_height > td->td_rowsperstrip) + sp->strile_height = td->td_rowsperstrip; + sp->bytesperline = TIFFScanlineSize(tif); + } + if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) + { + /* + * For PC 2, scale down the expected strip/tile size + * to match a downsampled component + */ + if (sp->h_sampling == 0 || sp->v_sampling == 0) + { + TIFFErrorExtR(tif, "JPEGComputeStrileWidthHeightBytesPerLine", + "JPEG horizontal or vertical sampling is zero"); + return (0); + } + sp->strile_width = TIFFhowmany_32(sp->strile_width, sp->h_sampling); + sp->strile_height = TIFFhowmany_32(sp->strile_height, sp->v_sampling); + } + + return 1; +} + /* * Set up for decoding a strip or tile. */ @@ -1167,7 +1203,6 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) JPEGState *sp = JState(tif); TIFFDirectory *td = &tif->tif_dir; static const char module[] = "JPEGPreDecode"; - uint32_t segment_width, segment_height; int downsampled_output; int ci; @@ -1193,52 +1228,24 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) return (0); tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; - /* - * Check image parameters and set decompression parameters. - */ - if (isTiled(tif)) - { - segment_width = td->td_tilewidth; - segment_height = td->td_tilelength; - sp->bytesperline = TIFFTileRowSize(tif); - } - else - { - segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; - if (segment_height > td->td_rowsperstrip) - segment_height = td->td_rowsperstrip; - sp->bytesperline = TIFFScanlineSize(tif); - } - if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) - { - /* - * For PC 2, scale down the expected strip/tile size - * to match a downsampled component - */ - if (sp->h_sampling == 0 || sp->v_sampling == 0) - { - TIFFErrorExtR(tif, module, - "JPEG horizontal or vertical sampling is zero"); - return (0); - } - segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); - segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); - } - if (sp->cinfo.d.image_width < segment_width || - sp->cinfo.d.image_height < segment_height) + if (!JPEGComputeStrileWidthHeightBytesPerLine(tif, s)) + return 0; + + if (sp->cinfo.d.image_width < sp->strile_width || + sp->cinfo.d.image_height < sp->strile_height) { TIFFWarningExtR(tif, module, "Improper JPEG strip/tile size, " "expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); } - if (sp->cinfo.d.image_width == segment_width && - sp->cinfo.d.image_height > segment_height && - tif->tif_row + segment_height == td->td_imagelength && !isTiled(tif)) + if (sp->cinfo.d.image_width == sp->strile_width && + sp->cinfo.d.image_height > sp->strile_height && + tif->tif_dir.td_row + sp->strile_height == td->td_imagelength && + !isTiled(tif)) { /* Some files have a last strip, that should be truncated, */ /* but their JPEG codestream has still the maximum strip */ @@ -1247,11 +1254,11 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) TIFFWarningExtR(tif, module, "JPEG strip size exceeds expected dimensions," " expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); } - else if (sp->cinfo.d.image_width > segment_width || - sp->cinfo.d.image_height > segment_height) + else if (sp->cinfo.d.image_width > sp->strile_width || + sp->cinfo.d.image_height > sp->strile_height) { /* * This case could be dangerous, if the strip or tile size has @@ -1262,8 +1269,8 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) TIFFErrorExtR(tif, module, "JPEG strip/tile size exceeds expected dimensions," " expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); return (0); } if (sp->cinfo.d.num_components != @@ -1282,7 +1289,8 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) sp->cinfo.d.data_precision = td->td_bitspersample; sp->cinfo.d.bits_in_jsample = td->td_bitspersample; #else - if (sp->cinfo.d.data_precision != td->td_bitspersample) + if (td->td_bitspersample != BITS_IN_JSAMPLE || + sp->cinfo.d.data_precision != td->td_bitspersample) { TIFFErrorExtR(tif, module, "Improper JPEG data precision"); return (0); @@ -1319,11 +1327,12 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) if (compptr->h_samp_factor > 0 && compptr->v_samp_factor > 0) { nRequiredMemory += - (toff_t)(((compptr->width_in_blocks + - compptr->h_samp_factor - 1) / - compptr->h_samp_factor)) * - ((compptr->height_in_blocks + compptr->v_samp_factor - 1) / - compptr->v_samp_factor) * + (toff_t)((JDIMENSION)(((int)compptr->width_in_blocks + + compptr->h_samp_factor - 1) / + compptr->h_samp_factor)) * + (JDIMENSION)(((int)compptr->height_in_blocks + + compptr->v_samp_factor - 1) / + compptr->v_samp_factor) * sizeof(JBLOCK); } } @@ -1439,7 +1448,7 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) * Decode a chunk of pixels. * "Standard" case: returned data is not downsampled. */ -#if !JPEG_LIB_MK1_OR_12BIT +#if !defined(JPEG_LIB_MK1_OR_12BIT) static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { JPEGState *sp = JState(tif); @@ -1464,7 +1473,18 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline not read"); if (nrows > (tmsize_t)sp->cinfo.d.image_height) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile height. */ + memset(buf, 0, (size_t)cc); nrows = sp->cinfo.d.image_height; + } + else if (sp->strile_width > sp->cinfo.d.image_width) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile width. */ + memset(buf, 0, (size_t)cc); + } /* data is expected to be read in multiples of a scanline */ if (nrows) @@ -1483,7 +1503,7 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) return (0); } - ++tif->tif_row; + ++tif->tif_dir.td_row; buf += sp->bytesperline; cc -= sp->bytesperline; } while (--nrows > 0); @@ -1491,15 +1511,15 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* Update information on consumed data */ tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; /* Close down the decompressor if we've finished the strip or tile. */ return sp->cinfo.d.output_scanline < sp->cinfo.d.output_height || TIFFjpeg_finish_decompress(sp); } -#endif /* !JPEG_LIB_MK1_OR_12BIT */ +#endif /* !defined(JPEG_LIB_MK1_OR_12BIT) */ -#if JPEG_LIB_MK1_OR_12BIT +#if defined(JPEG_LIB_MK1_OR_12BIT) /*ARGSUSED*/ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { @@ -1525,7 +1545,18 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline not read"); if (nrows > (tmsize_t)sp->cinfo.d.image_height) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile height. */ + memset(buf, 0, (size_t)cc); nrows = sp->cinfo.d.image_height; + } + else if (sp->strile_width > sp->cinfo.d.image_width) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile width. */ + memset(buf, 0, (size_t)cc); + } /* data is expected to be read in multiples of a scanline */ if (nrows) @@ -1539,8 +1570,9 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (sp->cinfo.d.data_precision == 12) { line_work_buf = (TIFF_JSAMPROW)_TIFFmallocExt( - tif, sizeof(short) * sp->cinfo.d.output_width * - sp->cinfo.d.num_components); + tif, (tmsize_t)((size_t)sizeof(short) * + (size_t)sp->cinfo.d.output_width * + (size_t)sp->cinfo.d.num_components)); } do @@ -1556,14 +1588,16 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (TIFFjpeg_read_scanlines(sp, &line_work_buf, 1) != 1) { memset(buf, 0, (size_t)cc); + _TIFFfreeExt(tif, line_work_buf); return (0); } if (sp->cinfo.d.data_precision == 12) { - int value_pairs = (sp->cinfo.d.output_width * - sp->cinfo.d.num_components) / - 2; + int value_pairs = + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components) / + 2; int iPair; for (iPair = 0; iPair < value_pairs; iPair++) @@ -1582,18 +1616,19 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) else if (sp->cinfo.d.data_precision == 8) { int value_count = - (sp->cinfo.d.output_width * sp->cinfo.d.num_components); + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components); int iValue; for (iValue = 0; iValue < value_count; iValue++) { ((unsigned char *)buf)[iValue] = - line_work_buf[iValue] & 0xff; + (unsigned char)(line_work_buf[iValue] & 0xff); } } } - ++tif->tif_row; + ++tif->tif_dir.td_row; buf += sp->bytesperline; cc -= sp->bytesperline; } while (--nrows > 0); @@ -1604,13 +1639,13 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* Update information on consumed data */ tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; /* Close down the decompressor if we've finished the strip or tile. */ return sp->cinfo.d.output_scanline < sp->cinfo.d.output_height || TIFFjpeg_finish_decompress(sp); } -#endif /* JPEG_LIB_MK1_OR_12BIT */ +#endif /* defined(JPEG_LIB_MK1_OR_12BIT) */ /*ARGSUSED*/ static int DecodeRowError(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) @@ -1644,8 +1679,16 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* For last strip, limit number of rows to its truncated height */ /* even if the codestream height is larger (which is not compliant, */ /* but that we tolerate) */ - if ((uint32_t)nrows > td->td_imagelength - tif->tif_row && !isTiled(tif)) - nrows = td->td_imagelength - tif->tif_row; + if ((uint32_t)nrows > td->td_imagelength - tif->tif_dir.td_row && + !isTiled(tif)) + nrows = td->td_imagelength - tif->tif_dir.td_row; + + /* The downsampled-data decode loop below only writes up to the JPEG + * codestream dimensions (and, due to the clump stride, may not cover the + * full caller buffer even for compliant files). Zero the whole buffer so + * no uninitialised heap is ever returned, mirroring the guard JPEGDecode + * received in 65759931ab6e (#826). */ + memset(buf, 0, (size_t)cc); #if defined(JPEG_LIB_MK1_OR_12BIT) unsigned short *tmpbuf = NULL; @@ -1660,9 +1703,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) int samples_per_clump = sp->samplesperclump; #if defined(JPEG_LIB_MK1_OR_12BIT) - tmpbuf = _TIFFmallocExt(tif, sizeof(unsigned short) * - sp->cinfo.d.output_width * - sp->cinfo.d.num_components); + tmpbuf = (unsigned short *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)sizeof(unsigned short) * + (size_t)sp->cinfo.d.output_width * + (size_t)sp->cinfo.d.num_components)); if (tmpbuf == NULL) { TIFFErrorExtR(tif, "JPEGDecodeRaw", "Out of memory"); @@ -1755,18 +1799,20 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (sp->cinfo.d.data_precision == 8) { int i = 0; - int len = - sp->cinfo.d.output_width * sp->cinfo.d.num_components; + int len = (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components); for (i = 0; i < len; i++) { - ((unsigned char *)buf)[i] = tmpbuf[i] & 0xff; + ((unsigned char *)buf)[i] = + (unsigned char)(tmpbuf[i] & 0xff); } } else { /* 12-bit */ - int value_pairs = (sp->cinfo.d.output_width * - sp->cinfo.d.num_components) / - 2; + int value_pairs = + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components) / + 2; int iPair; for (iPair = 0; iPair < value_pairs; iPair++) { @@ -1785,7 +1831,7 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) #endif sp->scancount++; - tif->tif_row += sp->v_sampling; + tif->tif_dir.td_row += sp->v_sampling; buf += sp->bytesperline; cc -= sp->bytesperline; @@ -2024,9 +2070,9 @@ static int JPEGSetupEncode(TIFF *tif) if (!TIFFGetField(tif, TIFFTAG_REFERENCEBLACKWHITE, &ref)) { float refbw[6]; - long top = 1L << td->td_bitspersample; + uint32_t top = 1U << td->td_bitspersample; refbw[0] = 0; - refbw[1] = (float)(top - 1L); + refbw[1] = (float)(top - 1); refbw[2] = (float)(top >> 1); refbw[3] = refbw[1]; refbw[4] = refbw[2]; @@ -2074,30 +2120,30 @@ static int JPEGSetupEncode(TIFF *tif) #endif if (isTiled(tif)) { - if ((td->td_tilelength % (sp->v_sampling * DCTSIZE)) != 0) + if ((td->td_tilelength % ((uint32_t)sp->v_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "JPEG tile height must be multiple of %" PRIu32, - (uint32_t)(sp->v_sampling * DCTSIZE)); + (uint32_t)sp->v_sampling * DCTSIZE); return (0); } - if ((td->td_tilewidth % (sp->h_sampling * DCTSIZE)) != 0) + if ((td->td_tilewidth % ((uint32_t)sp->h_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "JPEG tile width must be multiple of %" PRIu32, - (uint32_t)(sp->h_sampling * DCTSIZE)); + (uint32_t)sp->h_sampling * DCTSIZE); return (0); } } else { if (td->td_rowsperstrip < td->td_imagelength && - (td->td_rowsperstrip % (sp->v_sampling * DCTSIZE)) != 0) + (td->td_rowsperstrip % ((uint32_t)sp->v_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "RowsPerStrip must be multiple of %" PRIu32 " for JPEG", - (uint32_t)(sp->v_sampling * DCTSIZE)); + (uint32_t)sp->v_sampling * DCTSIZE); return (0); } } @@ -2153,7 +2199,6 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) JPEGState *sp = JState(tif); TIFFDirectory *td = &tif->tif_dir; static const char module[] = "JPEGPreEncode"; - uint32_t segment_width, segment_height; int downsampled_input; assert(sp != NULL); @@ -2164,47 +2209,20 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) } assert(!sp->cinfo.comm.is_decompressor); - /* - * Set encoding parameters for this strip/tile. - */ - if (isTiled(tif)) - { - segment_width = td->td_tilewidth; - segment_height = td->td_tilelength; - sp->bytesperline = TIFFTileRowSize(tif); - } - else - { - segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; - if (segment_height > td->td_rowsperstrip) - segment_height = td->td_rowsperstrip; - sp->bytesperline = TIFFScanlineSize(tif); - } - if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) - { - /* for PC 2, scale down the strip/tile size - * to match a downsampled component - */ - if (sp->h_sampling == 0 || sp->v_sampling == 0) - { - TIFFErrorExtR(tif, module, - "JPEG horizontal or vertical sampling is zero"); - return (0); - } - segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); - segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); - } - if (segment_width > (uint32_t)JPEG_MAX_DIMENSION || - segment_height > (uint32_t)JPEG_MAX_DIMENSION) + + if (!JPEGComputeStrileWidthHeightBytesPerLine(tif, s)) + return 0; + + if (sp->strile_width > (uint32_t)JPEG_MAX_DIMENSION || + sp->strile_height > (uint32_t)JPEG_MAX_DIMENSION) { TIFFErrorExtR(tif, module, "Strip/tile too large for JPEG. Maximum dimension is %d", (int)JPEG_MAX_DIMENSION); return (0); } - sp->cinfo.c.image_width = segment_width; - sp->cinfo.c.image_height = segment_height; + sp->cinfo.c.image_width = sp->strile_width; + sp->cinfo.c.image_height = sp->strile_height; downsampled_input = FALSE; if (td->td_planarconfig == PLANARCONFIG_CONTIG) { @@ -2276,7 +2294,13 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) /* an existing file */ suppress_huff_table(sp, 0); suppress_huff_table(sp, 1); - sp->cinfo.c.optimize_coding = FALSE; + + /* We want to keep optimize_coding = TRUE for 12-bit JPEG */ + /* See lengthy explanation at + * https://gitlab.com/libtiff/libtiff/-/work_items/773#note_3009836854 + */ + if (sp->cinfo.c.data_precision == 8) + sp->cinfo.c.optimize_coding = FALSE; } else sp->cinfo.c.optimize_coding = TRUE; @@ -2332,13 +2356,15 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline discarded"); /* The last strip will be limited to image size */ - if (!isTiled(tif) && tif->tif_row + nrows > tif->tif_dir.td_imagelength) - nrows = tif->tif_dir.td_imagelength - tif->tif_row; + if (!isTiled(tif) && + tif->tif_dir.td_row + nrows > tif->tif_dir.td_imagelength) + nrows = tif->tif_dir.td_imagelength - tif->tif_dir.td_row; if (sp->cinfo.c.data_precision == 12) { line16_count = (int)((sp->bytesperline * 2) / 3); - line16 = (short *)_TIFFmallocExt(tif, sizeof(short) * line16_count); + line16 = (short *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(short) * (size_t)line16_count)); if (!line16) { TIFFErrorExtR(tif, "JPEGEncode", "Failed to allocate memory"); @@ -2363,8 +2389,10 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) unsigned char *in_ptr = ((unsigned char *)buf) + iPair * 3; TIFF_JSAMPLE *out_ptr = (TIFF_JSAMPLE *)(line16 + iPair * 2); - out_ptr[0] = (in_ptr[0] << 4) | ((in_ptr[1] & 0xf0) >> 4); - out_ptr[1] = ((in_ptr[1] & 0x0f) << 8) | in_ptr[2]; + out_ptr[0] = (TIFF_JSAMPLE)((in_ptr[0] << 4) | + ((in_ptr[1] & 0xf0) >> 4)); + out_ptr[1] = + (TIFF_JSAMPLE)(((in_ptr[1] & 0x0f) << 8) | in_ptr[2]); } } else @@ -2374,7 +2402,7 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (TIFFjpeg_write_scanlines(sp, bufptr, 1) != 1) return (0); if (nrows > 0) - tif->tif_row++; + tif->tif_dir.td_row++; buf += sp->bytesperline; } @@ -2443,8 +2471,9 @@ static int JPEGEncodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { int hsamp = compptr->h_samp_factor; int vsamp = compptr->v_samp_factor; - int padding = (int)(compptr->width_in_blocks * DCTSIZE - - clumps_per_line * hsamp); + int padding = + (int)(compptr->width_in_blocks * DCTSIZE - + (JDIMENSION)clumps_per_line * (JDIMENSION)hsamp); for (ypos = 0; ypos < vsamp; ypos++) { inptr = ((TIFF_JSAMPLE *)buf) + clumpoffset; @@ -2488,7 +2517,7 @@ static int JPEGEncodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) } sp->scancount = 0; } - tif->tif_row += sp->v_sampling; + tif->tif_dir.td_row += sp->v_sampling; buf += bytesperclumpline; nrows -= sp->v_sampling; } @@ -2582,10 +2611,11 @@ static void JPEGResetUpsampled(TIFF *tif) * Must recalculate cached tile size in case sampling state changed. * Should we really be doing this now if image size isn't set? */ - if (tif->tif_tilesize > 0) - tif->tif_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); - if (tif->tif_scanlinesize > 0) - tif->tif_scanlinesize = TIFFScanlineSize(tif); + if (tif->tif_dir.td_tilesize > 0) + tif->tif_dir.td_tilesize = + isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); + if (tif->tif_dir.td_scanlinesize > 0) + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); } static int JPEGVSetField(TIFF *tif, uint32_t tag, va_list ap) @@ -2793,6 +2823,33 @@ static int JPEGInitializeLibJPEG(TIFF *tif, int decompress) return 1; } +static uint64_t JPEGGetMaxCompressionRatio(TIFF *tif) +{ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + const JPEGState *sp = JState(tif); + if ((tif->tif_dir.td_photometric == PHOTOMETRIC_YCBCR) && + (tif->tif_dir.td_planarconfig == PLANARCONFIG_CONTIG) && + (tif->tif_dir.td_samplesperpixel == 3)) + { + if (sp->h_sampling == 2 && sp->v_sampling == 2) + { + if (tif->tif_dir.td_bitspersample == 12) + return 768; + else + return 512; + } + + return 0; /* unknown */ + } + + if (tif->tif_dir.td_bitspersample == 12) + return 384; + else + return 256; +} + /* Common to tif_jpeg.c and tif_jpeg_12.c */ static void TIFFInitJPEGCommon(TIFF *tif) { @@ -2829,6 +2886,7 @@ static void TIFFInitJPEGCommon(TIFF *tif) tif->tif_encoderow = JPEGEncode; tif->tif_encodestrip = JPEGEncode; tif->tif_encodetile = JPEGEncode; + tif->tif_getmaxcompressionratio = JPEGGetMaxCompressionRatio; tif->tif_cleanup = JPEGCleanup; tif->tif_defstripsize = JPEGDefaultStripSize; diff --git a/contrib/libs/libtiff/tif_lerc.c b/contrib/libs/libtiff/tif_lerc.c index 4c7f1b210c3..cdbc6106b01 100644 --- a/contrib/libs/libtiff/tif_lerc.c +++ b/contrib/libs/libtiff/tif_lerc.c @@ -33,6 +33,7 @@ #error #include "Lerc_c_api.h" #include "zlib.h" +#include <math.h> #ifdef ZSTD_SUPPORT #include "zstd.h" #endif @@ -188,7 +189,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) else { sp->segment_width = td->td_imagewidth; - sp->segment_height = td->td_imagelength - tif->tif_row; + sp->segment_height = td->td_imagelength - tif->tif_dir.td_row; if (sp->segment_height > td->td_rowsperstrip) sp->segment_height = td->td_rowsperstrip; } @@ -242,7 +243,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) } if ((td->td_planarconfig == PLANARCONFIG_CONTIG && - td->td_extrasamples > 0 && + td->td_extrasamples > 0 && td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) || (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && @@ -343,7 +344,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (res != LIBDEFLATE_SUCCESS) { TIFFErrorExtR(tif, module, "Decoding error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } assert(lerc_data_sizet == (unsigned int)lerc_data_sizet); @@ -387,7 +388,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) size_t zstd_ret; zstd_ret = ZSTD_decompress(sp->compressed_buffer, sp->compressed_size, - tif->tif_rawcp, tif->tif_rawcc); + tif->tif_rawcp, (size_t)tif->tif_rawcc); if (ZSTD_isError(zstd_ret)) { TIFFErrorExtR(tif, module, "Error in ZSTD_decompress(): %s", @@ -420,6 +421,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) /* LERC info has dim == samplesperpixel - 1, then there is a LERC */ /* mask. */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && + td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1 && infoArray[2] == td->td_samplesperpixel - 1U) @@ -432,7 +434,8 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) use_mask = 1; } - ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1; + ndims = + (int)(td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1); /* Info returned in infoArray is { version, dataType, nDim/nDepth, nCols, nRows, nBands, nValidPixels, blobSize, @@ -440,12 +443,12 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (infoArray[0] != (unsigned)sp->lerc_version) { TIFFWarningExtR(tif, module, - "Unexpected version number: %d. Expected: %d", + "Unexpected version number: %u. Expected: %d", infoArray[0], sp->lerc_version); } if (infoArray[1] != (unsigned)lerc_data_type) { - TIFFErrorExtR(tif, module, "Unexpected dataType: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected dataType: %u. Expected: %d", infoArray[1], lerc_data_type); return 0; } @@ -458,7 +461,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { if (nFoundDims != 1 && nFoundDims != (unsigned)ndims) { - TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: 1 or %d", + TIFFErrorExtR(tif, module, "Unexpected nDim: %u. Expected: 1 or %d", nFoundDims, ndims); return 0; } @@ -467,20 +470,20 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) #endif if (nFoundDims != (unsigned)ndims) { - TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nDim: %u. Expected: %d", nFoundDims, ndims); return 0; } if (infoArray[3] != sp->segment_width) { - TIFFErrorExtR(tif, module, "Unexpected nCols: %d. Expected: %du", + TIFFErrorExtR(tif, module, "Unexpected nCols: %u. Expected: %u", infoArray[3], sp->segment_width); return 0; } if (infoArray[4] != sp->segment_height) { - TIFFErrorExtR(tif, module, "Unexpected nRows: %d. Expected: %u", + TIFFErrorExtR(tif, module, "Unexpected nRows: %u. Expected: %u", infoArray[4], sp->segment_height); return 0; } @@ -504,28 +507,28 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) #endif if (nFoundBands != td->td_samplesperpixel) { - TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nBands: %u. Expected: %d", nFoundBands, td->td_samplesperpixel); return 0; } } else if (nFoundBands != 1) { - TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nBands: %u. Expected: %d", nFoundBands, 1); return 0; } if (infoArray[7] != lerc_data_size) { - TIFFErrorExtR(tif, module, "Unexpected blobSize: %d. Expected: %u", + TIFFErrorExtR(tif, module, "Unexpected blobSize: %u. Expected: %u", infoArray[7], lerc_data_size); return 0; } int nRequestedMasks = use_mask ? 1 : 0; #if LERC_AT_LEAST_VERSION(3, 0, 0) - const int nFoundMasks = infoArray[8]; + const int nFoundMasks = (int)infoArray[8]; if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_samplesperpixel > 1 && nFoundDims == 1) @@ -576,21 +579,22 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; } lerc_ret = lerc_decode(lerc_data, lerc_data_size, nRequestedMasks, - sp->mask_buffer, nFoundDims, sp->segment_width, - sp->segment_height, nFoundBands, lerc_data_type, + sp->mask_buffer, (int)nFoundDims, + (int)sp->segment_width, (int)sp->segment_height, + (int)nFoundBands, (unsigned int)lerc_data_type, sp->uncompressed_buffer_multiband); } else #endif { - lerc_ret = - lerc_decode(lerc_data, lerc_data_size, + lerc_ret = lerc_decode( + lerc_data, lerc_data_size, #if LERC_AT_LEAST_VERSION(3, 0, 0) - nRequestedMasks, + nRequestedMasks, #endif - use_mask ? sp->mask_buffer : NULL, nFoundDims, - sp->segment_width, sp->segment_height, nFoundBands, - lerc_data_type, sp->uncompressed_buffer); + use_mask ? sp->mask_buffer : NULL, (int)nFoundDims, + (int)sp->segment_width, (int)sp->segment_height, (int)nFoundBands, + (unsigned int)lerc_data_type, sp->uncompressed_buffer); } if (lerc_ret != 0) { @@ -601,17 +605,17 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) /* Interleave alpha mask with other samples. */ if (use_mask && GetLercDataType(tif) == 1) { - unsigned src_stride = - (td->td_samplesperpixel - 1) * (td->td_bitspersample / 8); + unsigned src_stride = (unsigned int)((td->td_samplesperpixel - 1) * + (td->td_bitspersample / 8)); unsigned dst_stride = - td->td_samplesperpixel * (td->td_bitspersample / 8); + (unsigned int)(td->td_samplesperpixel * (td->td_bitspersample / 8)); unsigned i = sp->segment_width * sp->segment_height; /* Operate from end to begin to be able to move in place */ while (i > 0 && i > nomask_bands) { i--; sp->uncompressed_buffer[i * dst_stride + td->td_samplesperpixel - - 1] = 255 * sp->mask_buffer[i]; + 1] = (uint8_t)(255 * sp->mask_buffer[i]); memcpy(sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride); } @@ -620,7 +624,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { i--; sp->uncompressed_buffer[i * dst_stride + td->td_samplesperpixel - - 1] = 255 * sp->mask_buffer[i]; + 1] = (uint8_t)(255 * sp->mask_buffer[i]); memmove(sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride); } @@ -649,7 +653,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { if (sp->mask_buffer[i] == 0) @@ -677,7 +681,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { for (int j = 0; j < td->td_samplesperpixel; j++) @@ -704,30 +708,32 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { for (int j = 0; j < td->td_samplesperpixel; j++) { - if (sp->mask_buffer[i + j * nb_pixels] == 0) + if (sp->mask_buffer[i + (unsigned int)j * nb_pixels] == + 0) ((float *)sp->uncompressed_buffer)[k] = nan_float32; else ((float *)sp->uncompressed_buffer)[k] = ((float *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels]; + [i + (unsigned int)j * nb_pixels]; ++k; } } } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { for (int j = 0; j < td->td_samplesperpixel; j++) { - if (sp->mask_buffer[i + j * nb_pixels] == 0) + if (sp->mask_buffer[i + (unsigned int)j * nb_pixels] == + 0) ((double *)sp->uncompressed_buffer)[k] = nan_float64; else ((double *)sp->uncompressed_buffer)[k] = ((double *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels]; + [i + (unsigned int)j * nb_pixels]; ++k; } } @@ -766,8 +772,8 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - memcpy(op, sp->uncompressed_buffer + sp->uncompressed_offset, occ); - sp->uncompressed_offset += (unsigned)occ; + memcpy(op, sp->uncompressed_buffer + sp->uncompressed_offset, (size_t)occ); + sp->uncompressed_offset += (unsigned int)occ; return 1; } @@ -832,8 +838,8 @@ static int LERCEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - memcpy(sp->uncompressed_buffer + sp->uncompressed_offset, bp, cc); - sp->uncompressed_offset += (unsigned)cc; + memcpy(sp->uncompressed_buffer + sp->uncompressed_offset, bp, (size_t)cc); + sp->uncompressed_offset += (unsigned int)cc; return 1; } @@ -863,13 +869,15 @@ static int LERCPostEncode(TIFF *tif) /* Extract alpha mask (if containing only 0 and 255 values, */ /* and compact array of regular bands */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && + td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) { const unsigned dst_stride = - (td->td_samplesperpixel - 1) * (td->td_bitspersample / 8); + (unsigned int)((td->td_samplesperpixel - 1) * + (td->td_bitspersample / 8)); const unsigned src_stride = - td->td_samplesperpixel * (td->td_bitspersample / 8); + (unsigned int)(td->td_samplesperpixel * (td->td_bitspersample / 8)); unsigned i = 0; use_mask = 1; @@ -923,7 +931,7 @@ static int LERCPostEncode(TIFF *tif) { const float val = ((float *)sp->uncompressed_buffer)[k]; ++k; - if (val != val) + if (isnan(val)) { ++count_nan; } @@ -944,7 +952,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const float val = ((float *)sp->uncompressed_buffer)[i]; - if (val != val) + if (isnan(val)) { use_mask = 1; break; @@ -965,7 +973,7 @@ static int LERCPostEncode(TIFF *tif) const double val = ((double *)sp->uncompressed_buffer)[k]; ++k; - if (val != val) + if (isnan(val)) { ++count_nan; } @@ -986,7 +994,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const double val = ((double *)sp->uncompressed_buffer)[i]; - if (val != val) + if (isnan(val)) { use_mask = 1; break; @@ -1025,10 +1033,10 @@ static int LERCPostEncode(TIFF *tif) const float val = ((float *)sp->uncompressed_buffer)[k]; ((float *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels] = val; + [i + (unsigned int)j * nb_pixels] = val; ++k; - sp->mask_buffer[i + j * nb_pixels] = - (val == val) ? 255 : 0; + sp->mask_buffer[i + (unsigned int)j * nb_pixels] = + !isnan(val) ? 255 : 0; } } } @@ -1041,10 +1049,10 @@ static int LERCPostEncode(TIFF *tif) const double val = ((double *)sp->uncompressed_buffer)[k]; ((double *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels] = val; + [i + (unsigned int)j * nb_pixels] = val; ++k; - sp->mask_buffer[i + j * nb_pixels] = - (val == val) ? 255 : 0; + sp->mask_buffer[i + (unsigned int)j * nb_pixels] = + !isnan(val) ? 255 : 0; } } } @@ -1064,7 +1072,7 @@ static int LERCPostEncode(TIFF *tif) { const float val = ((float *)sp->uncompressed_buffer)[i * dst_nbands]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } else @@ -1073,7 +1081,7 @@ static int LERCPostEncode(TIFF *tif) { const double val = ((double *)sp->uncompressed_buffer)[i * dst_nbands]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } } @@ -1084,7 +1092,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const float val = ((float *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } else @@ -1093,7 +1101,7 @@ static int LERCPostEncode(TIFF *tif) { const double val = ((double *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } } @@ -1104,7 +1112,8 @@ static int LERCPostEncode(TIFF *tif) #if LERC_AT_LEAST_VERSION(3, 0, 0) if (mask_count > 1) { - estimated_compressed_size += nb_pixels * mask_count / 8; + estimated_compressed_size += + (unsigned int)(nb_pixels * (unsigned int)mask_count / 8); } #endif @@ -1125,8 +1134,9 @@ static int LERCPostEncode(TIFF *tif) { lerc_ret = lerc_encodeForVersion( sp->uncompressed_buffer_multiband, sp->lerc_version, - GetLercDataType(tif), 1, sp->segment_width, sp->segment_height, - dst_nbands, dst_nbands, sp->mask_buffer, sp->maxzerror, + (unsigned int)GetLercDataType(tif), 1, (int)sp->segment_width, + (int)sp->segment_height, (int)dst_nbands, (int)dst_nbands, + sp->mask_buffer, sp->maxzerror, (unsigned char *)sp->compressed_buffer, sp->compressed_size, &numBytesWritten); } @@ -1134,9 +1144,10 @@ static int LERCPostEncode(TIFF *tif) #endif { lerc_ret = lerc_encodeForVersion( - sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, - sp->segment_width, sp->segment_height, 1, + sp->uncompressed_buffer, sp->lerc_version, + (unsigned int)GetLercDataType(tif), + (int)(td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1), + (int)sp->segment_width, (int)sp->segment_height, 1, #if LERC_AT_LEAST_VERSION(3, 0, 0) use_mask ? 1 : 0, #endif @@ -1179,14 +1190,14 @@ static int LERCPostEncode(TIFF *tif) return 0; } - tif->tif_rawcc = libdeflate_zlib_compress( + tif->tif_rawcc = (tmsize_t)libdeflate_zlib_compress( sp->libdeflate_enc, sp->compressed_buffer, numBytesWritten, sp->uncompressed_buffer, sp->uncompressed_alloc); if (tif->tif_rawcc == 0) { TIFFErrorExtR(tif, module, "Encoder error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } #else @@ -1252,7 +1263,7 @@ static int LERCPostEncode(TIFF *tif) int ret; uint8_t *tif_rawdata_backup = tif->tif_rawdata; tif->tif_rawdata = sp->uncompressed_buffer; - tif->tif_rawcc = zstd_ret; + tif->tif_rawcc = (tmsize_t)zstd_ret; ret = TIFFFlushData1(tif); tif->tif_rawdata = tif_rawdata_backup; if (!ret) @@ -1350,7 +1361,7 @@ static int LERCVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_LERC_PARAMETERS: { - uint32_t count = va_arg(ap, int); + uint32_t count = (uint32_t)va_arg(ap, int); int *params = va_arg(ap, int *); if (count < 2) { @@ -1529,6 +1540,13 @@ int TIFFInitLERC(TIFF *tif, int scheme) #endif tif->tif_cleanup = LERCCleanup; + /* LERC compression ratio can grow to several millions */ + /* eg. 5703725 for Lerc deflate on 16383x16383 array */ + /* or 3829644 for regular Lerc */ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + /* so we don't define tif->tif_getmaxcompressionratio */ + /* Default values for codec-specific fields */ TIFFSetField(tif, TIFFTAG_LERC_VERSION, LERC_VERSION_2_4); TIFFSetField(tif, TIFFTAG_LERC_ADD_COMPRESSION, LERC_ADD_COMPRESSION_NONE); diff --git a/contrib/libs/libtiff/tif_luv.c b/contrib/libs/libtiff/tif_luv.c index ecc32252659..d9bc8fe3105 100644 --- a/contrib/libs/libtiff/tif_luv.c +++ b/contrib/libs/libtiff/tif_luv.c @@ -212,7 +212,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } tp = (int16_t *)sp->tbuf; } - _TIFFmemset((void *)tp, 0, npixels * sizeof(tp[0])); + _TIFFmemset((void *)tp, 0, (tmsize_t)((size_t)npixels * sizeof(tp[0]))); bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; @@ -235,7 +235,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { /* non-run */ rc = *bp++; /* nul is noop */ while (--cc && rc-- && i < npixels) - tp[i++] |= (int16_t)*bp++ << shft; + tp[i++] |= (int16_t)(*bp++ << shft); } } if (i != npixels) @@ -243,7 +243,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); tif->tif_rawcp = (uint8_t *)bp; tif->tif_rawcc = cc; return (0); @@ -290,7 +290,7 @@ static int LogLuvDecode24(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) cc = tif->tif_rawcc; for (i = 0; i < npixels && cc >= 3; i++) { - tp[i] = bp[0] << 16 | bp[1] << 8 | bp[2]; + tp[i] = (uint32_t)bp[0] << 16 | (uint32_t)bp[1] << 8 | bp[2]; bp += 3; cc -= 3; } @@ -301,7 +301,7 @@ static int LogLuvDecode24(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); return (0); } (*sp->tfunc)(sp, op, npixels); @@ -342,7 +342,7 @@ static int LogLuvDecode32(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } tp = (uint32_t *)sp->tbuf; } - _TIFFmemset((void *)tp, 0, npixels * sizeof(tp[0])); + _TIFFmemset((void *)tp, 0, (tmsize_t)((size_t)npixels * sizeof(tp[0]))); bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; @@ -373,7 +373,7 @@ static int LogLuvDecode32(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); tif->tif_rawcp = (uint8_t *)bp; tif->tif_rawcc = cc; return (0); @@ -767,25 +767,21 @@ static int LogLuvEncodeTile(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) #ifndef M_PI #define M_PI 3.14159265358979323846 #endif -#undef log2 /* Conflict with C'99 function */ -#define log2(x) ((1. / M_LN2) * log(x)) -#undef exp2 /* Conflict with C'99 function */ -#define exp2(x) exp(M_LN2 *(x)) #define TIFF_RAND_MAX 32767 // From POSIX.1-2001 as an example of an implementation of rand() -static uint32_t _TIFFRand() +static uint32_t _TIFFRand(void) { static uint32_t nCounter = 0; if (!nCounter) nCounter = (uint32_t)(time(NULL) & UINT32_MAX); ++nCounter; uint32_t nCounterLocal = - (uint32_t)(((uint64_t)(nCounter)*1103515245U + 12345U) & UINT32_MAX); + (uint32_t)(((uint64_t)(nCounter) * 1103515245U + 12345U) & UINT32_MAX); nCounter = nCounterLocal; return (nCounterLocal / 65536U) % (TIFF_RAND_MAX + 1); -}; +} static int tiff_itrunc(double x, int m) { @@ -797,8 +793,7 @@ static int tiff_itrunc(double x, int m) #if !LOGLUV_PUBLIC static #endif - double - LogL16toY(int p16) /* compute luminance from 16-bit LogL */ + double LogL16toY(int p16) /* compute luminance from 16-bit LogL */ { int Le = p16 & 0x7fff; double Y; @@ -812,8 +807,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - LogL16fromY(double Y, int em) /* get 16-bit LogL from Y */ + int LogL16fromY(double Y, int em) /* get 16-bit LogL from Y */ { if (Y >= 1.8371976e19) return (0x7fff); @@ -855,20 +849,22 @@ static void L16fromY(LogLuvState *sp, uint8_t *op, tmsize_t n) float *yp = (float *)op; while (n-- > 0) - *l16++ = (int16_t)(LogL16fromY(*yp++, sp->encode_meth)); + *l16++ = (int16_t)(LogL16fromY((double)*yp++, sp->encode_meth)); } #if !LOGLUV_PUBLIC static #endif - void - XYZtoRGB24(float *xyz, uint8_t *rgb) + void XYZtoRGB24(float *xyz, uint8_t *rgb) { double r, g, b; /* assume CCIR-709 primaries */ - r = 2.690 * xyz[0] + -1.276 * xyz[1] + -0.414 * xyz[2]; - g = -1.022 * xyz[0] + 1.978 * xyz[1] + 0.044 * xyz[2]; - b = 0.061 * xyz[0] + -0.224 * xyz[1] + 1.163 * xyz[2]; + r = 2.690 * (double)xyz[0] + -1.276 * (double)xyz[1] + + -0.414 * (double)xyz[2]; + g = -1.022 * (double)xyz[0] + 1.978 * (double)xyz[1] + + 0.044 * (double)xyz[2]; + b = 0.061 * (double)xyz[0] + -0.224 * (double)xyz[1] + + 1.163 * (double)xyz[2]; /* assume 2.0 gamma for speed */ /* could use integer sqrt approx., but this is probably faster */ rgb[0] = (uint8_t)((r <= 0.) ? 0 : (r >= 1.) ? 255 : (int)(256. * sqrt(r))); @@ -879,8 +875,7 @@ static #if !LOGLUV_PUBLIC static #endif - double - LogL10toY(int p10) /* compute luminance from 10-bit LogL */ + double LogL10toY(int p10) /* compute luminance from 10-bit LogL */ { if (p10 == 0) return (0.); @@ -890,8 +885,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - LogL10fromY(double Y, int em) /* get 10-bit LogL from Y */ + int LogL10fromY(double Y, int em) /* get 10-bit LogL from Y */ { if (Y >= 15.742) return (0x3ff); @@ -903,13 +897,14 @@ static #define NANGLES 100 #define uv2ang(u, v) \ - ((NANGLES * .499999999 / M_PI) * atan2((v)-V_NEU, (u)-U_NEU) + .5 * NANGLES) + ((NANGLES * .499999999 / M_PI) * atan2((v) - V_NEU, (u) - U_NEU) + \ + .5 * NANGLES) static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ { static int oog_table[NANGLES]; static int initialized = 0; - register int i; + int i; if (!initialized) { /* set up perimeter table */ @@ -919,13 +914,14 @@ static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ eps[i] = 2.; for (vi = UV_NVS; vi--;) { - va = UV_VSTART + (vi + .5) * UV_SQSIZ; + va = (double)UV_VSTART + ((double)vi + .5) * (double)UV_SQSIZ; ustep = uv_row[vi].nus - 1; if (vi == UV_NVS - 1 || vi == 0 || ustep <= 0) ustep = 1; for (ui = uv_row[vi].nus - 1; ui >= 0; ui -= ustep) { - ua = uv_row[vi].ustart + (ui + .5) * UV_SQSIZ; + ua = (double)uv_row[vi].ustart + + ((double)ui + .5) * (double)UV_SQSIZ; ang = uv2ang(ua, va); i = (int)ang; epsa = fabs(ang - (i + .5)); @@ -963,27 +959,28 @@ static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ #if !LOGLUV_PUBLIC static #endif - int - uv_encode(double u, double v, int em) /* encode (u',v') coordinates */ + int uv_encode(double u, double v, int em) /* encode (u',v') coordinates */ { unsigned int vi; int ui; /* check for NaN */ - if (u != u || v != v) + if (isnan(u) || isnan(v)) { u = U_NEU; v = V_NEU; } - if (v < UV_VSTART) + if ((double)v < (double)UV_VSTART) return oog_encode(u, v); - vi = tiff_itrunc((v - UV_VSTART) * (1. / UV_SQSIZ), em); + vi = (unsigned int)tiff_itrunc( + ((double)v - (double)UV_VSTART) * (1. / (double)UV_SQSIZ), em); if (vi >= UV_NVS) return oog_encode(u, v); - if (u < uv_row[vi].ustart) + if ((double)u < (double)uv_row[vi].ustart) return oog_encode(u, v); - ui = tiff_itrunc((u - uv_row[vi].ustart) * (1. / UV_SQSIZ), em); + ui = tiff_itrunc( + ((double)u - (double)uv_row[vi].ustart) * (1. / (double)UV_SQSIZ), em); if (ui >= uv_row[vi].nus) return oog_encode(u, v); @@ -993,8 +990,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - uv_decode(double *up, double *vp, int c) /* decode (u',v') index */ + int uv_decode(double *up, double *vp, int c) /* decode (u',v') index */ { unsigned int upper, lower; int ui; @@ -1020,16 +1016,15 @@ static } vi = lower; ui = c - uv_row[vi].ncum; - *up = uv_row[vi].ustart + (ui + .5) * UV_SQSIZ; - *vp = UV_VSTART + (vi + .5) * UV_SQSIZ; + *up = (double)uv_row[vi].ustart + ((double)ui + .5) * (double)UV_SQSIZ; + *vp = (double)UV_VSTART + ((double)vi + .5) * (double)UV_SQSIZ; return (0); } #if !LOGLUV_PUBLIC static #endif - void - LogLuv24toXYZ(uint32_t p, float *XYZ) + void LogLuv24toXYZ(uint32_t p, float *XYZ) { int Ce; double L, u, v, s, x, y; @@ -1059,15 +1054,14 @@ static #if !LOGLUV_PUBLIC static #endif - uint32_t - LogLuv24fromXYZ(float *XYZ, int em) + uint32_t LogLuv24fromXYZ(float *XYZ, int em) { int Le, Ce; double u, v, s; /* encode luminance */ - Le = LogL10fromY(XYZ[1], em); + Le = LogL10fromY((double)XYZ[1], em); /* encode color */ - s = XYZ[0] + 15. * XYZ[1] + 3. * XYZ[2]; + s = (double)XYZ[0] + 15. * (double)XYZ[1] + 3. * (double)XYZ[2]; if (!Le || s <= 0.) { u = U_NEU; @@ -1075,14 +1069,14 @@ static } else { - u = 4. * XYZ[0] / s; - v = 9. * XYZ[1] / s; + u = 4. * (double)XYZ[0] / s; + v = 9. * (double)XYZ[1] / s; } Ce = uv_encode(u, v, em); if (Ce < 0) /* never happens */ Ce = uv_encode(U_NEU, V_NEU, SGILOGENCODE_NODITHER); /* combine encodings */ - return (Le << 14 | Ce); + return (uint32_t)Le << 14 | (uint32_t)Ce; } static void Luv24toXYZ(LogLuvState *sp, uint8_t *op, tmsize_t n) @@ -1113,8 +1107,8 @@ static void Luv24toLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) u = U_NEU; v = V_NEU; } - *luv3++ = (int16_t)(u * (1L << 15)); - *luv3++ = (int16_t)(v * (1L << 15)); + *luv3++ = (int16_t)(u * (1 << 15)); + *luv3++ = (int16_t)(v * (1 << 15)); luv++; } } @@ -1168,7 +1162,7 @@ static void Luv24fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) sp->encode_meth); if (Ce < 0) /* never happens */ Ce = uv_encode(U_NEU, V_NEU, SGILOGENCODE_NODITHER); - *luv++ = (uint32_t)Le << 14 | Ce; + *luv++ = (uint32_t)Le << 14 | (uint32_t)Ce; luv3 += 3; } } @@ -1176,8 +1170,7 @@ static void Luv24fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) #if !LOGLUV_PUBLIC static #endif - void - LogLuv32toXYZ(uint32_t p, float *XYZ) + void LogLuv32toXYZ(uint32_t p, float *XYZ) { double L, u, v, s, x, y; /* decode luminance */ @@ -1202,15 +1195,14 @@ static #if !LOGLUV_PUBLIC static #endif - uint32_t - LogLuv32fromXYZ(float *XYZ, int em) + uint32_t LogLuv32fromXYZ(float *XYZ, int em) { unsigned int Le, ue, ve; double u, v, s; /* encode luminance */ - Le = (unsigned int)LogL16fromY(XYZ[1], em); + Le = (unsigned int)LogL16fromY((double)XYZ[1], em); /* encode color */ - s = XYZ[0] + 15. * XYZ[1] + 3. * XYZ[2]; + s = (double)XYZ[0] + 15. * (double)XYZ[1] + 3. * (double)XYZ[2]; if (!Le || s <= 0.) { u = U_NEU; @@ -1218,19 +1210,19 @@ static } else { - u = 4. * XYZ[0] / s; - v = 9. * XYZ[1] / s; + u = 4. * (double)XYZ[0] / s; + v = 9. * (double)XYZ[1] / s; } if (u <= 0.) ue = 0; else - ue = tiff_itrunc(UVSCALE * u, em); + ue = (unsigned int)tiff_itrunc(UVSCALE * u, em); if (ue > 255) ue = 255; if (v <= 0.) ve = 0; else - ve = tiff_itrunc(UVSCALE * v, em); + ve = (unsigned int)tiff_itrunc(UVSCALE * v, em); if (ve > 255) ve = 255; /* combine encodings */ @@ -1261,8 +1253,8 @@ static void Luv32toLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) *luv3++ = (int16_t)(*luv >> 16); u = 1. / UVSCALE * ((*luv >> 8 & 0xff) + .5); v = 1. / UVSCALE * ((*luv & 0xff) + .5); - *luv3++ = (int16_t)(u * (1L << 15)); - *luv3++ = (int16_t)(v * (1L << 15)); + *luv3++ = (int16_t)(u * (1 << 15)); + *luv3++ = (int16_t)(v * (1 << 15)); luv++; } } @@ -1303,22 +1295,24 @@ static void Luv32fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) { while (n-- > 0) { - *luv++ = (uint32_t)luv3[0] << 16 | - (luv3[1] * (uint32_t)(UVSCALE + .5) >> 7 & 0xff00) | - (luv3[2] * (uint32_t)(UVSCALE + .5) >> 15 & 0xff); + *luv++ = + (uint32_t)luv3[0] << 16 | + ((uint32_t)luv3[1] * (uint32_t)(UVSCALE + .5) >> 7 & 0xff00) | + ((uint32_t)luv3[2] * (uint32_t)(UVSCALE + .5) >> 15 & 0xff); luv3 += 3; } return; } while (n-- > 0) { - *luv++ = - (uint32_t)luv3[0] << 16 | - (tiff_itrunc(luv3[1] * (UVSCALE / (1 << 15)), sp->encode_meth) - << 8 & - 0xff00) | - (tiff_itrunc(luv3[2] * (UVSCALE / (1 << 15)), sp->encode_meth) & - 0xff); + *luv++ = (uint32_t)luv3[0] << 16 | + ((uint32_t)tiff_itrunc(luv3[1] * (UVSCALE / (1 << 15)), + sp->encode_meth) + << 8 & + 0xff00) | + ((uint32_t)tiff_itrunc(luv3[2] * (UVSCALE / (1 << 15)), + sp->encode_meth) & + 0xff); luv3 += 3; } } @@ -1345,6 +1339,8 @@ static int LogL16GuessDataFmt(TIFFDirectory *td) case PACK(1, 8, SAMPLEFORMAT_VOID): case PACK(1, 8, SAMPLEFORMAT_UINT): return (SGILOGDATAFMT_8BIT); + default: + break; } #undef PACK return (SGILOGDATAFMT_UNKNOWN); @@ -1399,7 +1395,7 @@ static int LogL16InitState(TIFF *tif) sp->tbuflen = multiply_ms(td->td_imagewidth, td->td_imagelength); if (multiply_ms(sp->tbuflen, sizeof(int16_t)) == 0 || (sp->tbuf = (uint8_t *)_TIFFmallocExt( - tif, sp->tbuflen * sizeof(int16_t))) == NULL) + tif, (tmsize_t)((size_t)sp->tbuflen * sizeof(int16_t)))) == NULL) { TIFFErrorExtR(tif, module, "No space for SGILog translation buffer"); return (0); @@ -1506,7 +1502,7 @@ static int LogLuvInitState(TIFF *tif) sp->tbuflen = multiply_ms(td->td_imagewidth, td->td_imagelength); if (multiply_ms(sp->tbuflen, sizeof(uint32_t)) == 0 || (sp->tbuf = (uint8_t *)_TIFFmallocExt( - tif, sp->tbuflen * sizeof(uint32_t))) == NULL) + tif, (tmsize_t)((size_t)sp->tbuflen * sizeof(uint32_t)))) == NULL) { TIFFErrorExtR(tif, module, "No space for SGILog translation buffer"); return (0); @@ -1546,6 +1542,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = Luv24toRGB; break; + default: + break; } } else @@ -1562,6 +1560,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = Luv32toRGB; break; + default: + break; } } return (1); @@ -1577,6 +1577,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = L16toGry; break; + default: + break; } return (1); default: @@ -1755,8 +1757,9 @@ static int LogLuvVSetField(TIFF *tif, uint32_t tag, va_list ap) /* * Must recalculate sizes should bits/sample change. */ - tif->tif_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)-1; - tif->tif_scanlinesize = TIFFScanlineSize(tif); + tif->tif_dir.td_tilesize = + isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)-1; + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); return (1); case TIFFTAG_SGILOGENCODE: sp->encode_meth = (int)va_arg(ap, int); diff --git a/contrib/libs/libtiff/tif_lzma.c b/contrib/libs/libtiff/tif_lzma.c index 9834ae2c2c5..7763f61acd9 100644 --- a/contrib/libs/libtiff/tif_lzma.c +++ b/contrib/libs/libtiff/tif_lzma.c @@ -95,6 +95,17 @@ static const char *LZMAStrerror(lzma_ret ret) return "no progress is possible (stream is truncated or corrupt)"; case LZMA_PROG_ERROR: return "programming error"; +#if LZMA_VERSION >= 50040000 /* 5.4.0 */ + case LZMA_SEEK_NEEDED: + case LZMA_RET_INTERNAL1: + case LZMA_RET_INTERNAL2: + case LZMA_RET_INTERNAL3: + case LZMA_RET_INTERNAL4: + case LZMA_RET_INTERNAL5: + case LZMA_RET_INTERNAL6: + case LZMA_RET_INTERNAL7: + case LZMA_RET_INTERNAL8: +#endif default: return "unidentified liblzma error"; } @@ -179,7 +190,7 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "LZMADecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -231,7 +242,7 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { TIFFErrorExtR(tif, module, "Decoding error at scanline %" PRIu32 ", %s", - tif->tif_row, LZMAStrerror(ret)); + tif->tif_dir.td_row, LZMAStrerror(ret)); break; } } while (sp->stream.avail_out > 0); @@ -242,12 +253,12 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" TIFF_SIZE_FORMAT " bytes)", - tif->tif_row, sp->stream.avail_out); + tif->tif_dir.td_row, sp->stream.avail_out); return 0; } tif->tif_rawcp = (uint8_t *)sp->stream.next_in; /* cast away const */ - tif->tif_rawcc = sp->stream.avail_in; + tif->tif_rawcc = (tmsize_t)sp->stream.avail_in; return 1; } @@ -326,7 +337,7 @@ static int LZMAEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { TIFFErrorExtR(tif, module, "Encoding error at scanline %" PRIu32 ", %s", - tif->tif_row, LZMAStrerror(ret)); + tif->tif_dir.td_row, LZMAStrerror(ret)); return 0; } if (sp->stream.avail_out == 0) @@ -365,7 +376,7 @@ static int LZMAPostEncode(TIFF *tif) if ((tmsize_t)sp->stream.avail_out != tif->tif_rawdatasize) { tif->tif_rawcc = - tif->tif_rawdatasize - sp->stream.avail_out; + tif->tif_rawdatasize - (tmsize_t)sp->stream.avail_out; if (!TIFFFlushData1(tif)) return 0; sp->stream.next_out = tif->tif_rawdata; @@ -376,6 +387,27 @@ static int LZMAPostEncode(TIFF *tif) ZIPPreEncode */ } break; + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_BUF_ERROR: + case LZMA_PROG_ERROR: +#if LZMA_VERSION >= 50040000 /* 5.4.0 */ + case LZMA_SEEK_NEEDED: + case LZMA_RET_INTERNAL1: + case LZMA_RET_INTERNAL2: + case LZMA_RET_INTERNAL3: + case LZMA_RET_INTERNAL4: + case LZMA_RET_INTERNAL5: + case LZMA_RET_INTERNAL6: + case LZMA_RET_INTERNAL7: + case LZMA_RET_INTERNAL8: +#endif default: TIFFErrorExtR(tif, module, "Liblzma error: %s", LZMAStrerror(ret)); @@ -416,7 +448,7 @@ static int LZMAVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_LZMAPRESET: sp->preset = (int)va_arg(ap, int); - lzma_lzma_preset(&sp->opt_lzma, sp->preset); + lzma_lzma_preset(&sp->opt_lzma, (uint32_t)sp->preset); if (sp->state & LSTATE_INIT_ENCODE) { lzma_ret ret = @@ -454,6 +486,21 @@ static const TIFFField lzmaFields[] = { FALSE, "LZMA2 Compression Preset", NULL}, }; +static uint64_t LZMAGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 3800 */ + /* 4096x4096: 6534 */ + /* 16383x16383: 6846 */ + /* 65536x65536: 6874 */ + + return 7000; +} + int TIFFInitLZMA(TIFF *tif, int scheme) { static const char module[] = "TIFFInitLZMA"; @@ -506,7 +553,7 @@ int TIFFInitLZMA(TIFF *tif, int scheme) sp->filters[0].id = LZMA_FILTER_DELTA; sp->filters[0].options = &sp->opt_delta; - lzma_lzma_preset(&sp->opt_lzma, sp->preset); + lzma_lzma_preset(&sp->opt_lzma, (uint32_t)sp->preset); sp->filters[1].id = LZMA_FILTER_LZMA2; sp->filters[1].options = &sp->opt_lzma; @@ -529,6 +576,8 @@ int TIFFInitLZMA(TIFF *tif, int scheme) tif->tif_encodestrip = LZMAEncode; tif->tif_encodetile = LZMAEncode; tif->tif_cleanup = LZMACleanup; + tif->tif_getmaxcompressionratio = LZMAGetMaxCompressionRatio; + /* * Setup predictor setup. */ diff --git a/contrib/libs/libtiff/tif_lzw.c b/contrib/libs/libtiff/tif_lzw.c index 4bf845b393d..d9f56a427ba 100644 --- a/contrib/libs/libtiff/tif_lzw.c +++ b/contrib/libs/libtiff/tif_lzw.c @@ -60,7 +60,7 @@ typedef size_t WordType; */ #define LZW_COMPAT /* include backwards compatibility code */ -#define MAXCODE(n) ((1L << (n)) - 1) +#define MAXCODE(n) ((1 << (n)) - 1) /* * The TIFF spec specifies that encoded bit * strings range from 9 to 12 bits. @@ -72,7 +72,7 @@ typedef size_t WordType; #define CODE_EOI 257 /* end-of-information code */ #define CODE_FIRST 258 /* first free code entry */ #define CODE_MAX MAXCODE(BITS_MAX) -#define HSIZE 9001L /* 91% occupancy */ +#define HSIZE 9001 /* 91% occupancy */ #define HSHIFT (13 - 8) #ifdef LZW_COMPAT /* NB: +1024 is for compatibility with old files */ @@ -325,7 +325,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) */ /* Get the next 32 or 64-bit from the input data */ -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN #define GetNextData(nextdata, bp) memcpy(&nextdata, bp, sizeof(nextdata)) #elif SIZEOF_WORDTYPE == 8 #if defined(_M_X64) @@ -372,7 +372,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) nextbits += 8 * SIZEOF_WORDTYPE; \ dec_bitsleft -= 8 * SIZEOF_WORDTYPE; \ code = (WordType)((codetmp | (nextdata >> nextbits)) & \ - nbitsmask); \ + (WordType)nbitsmask); \ break; \ } \ else \ @@ -396,7 +396,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) } \ } \ } \ - code = (WordType)((nextdata >> nextbits) & nbitsmask); \ + code = (WordType)((nextdata >> nextbits) & (WordType)nbitsmask); \ } while (0) static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) @@ -420,7 +420,7 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR(tif, module, "LZWDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -472,7 +472,8 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) } bp = (uint8_t *)tif->tif_rawcp; - sp->dec_bitsleft += (((uint64_t)tif->tif_rawcc - sp->old_tif_rawcc) << 3); + sp->dec_bitsleft += + (((uint64_t)tif->tif_rawcc - (uint64_t)sp->old_tif_rawcc) << 3); uint64_t dec_bitsleft = sp->dec_bitsleft; nbits = sp->lzw_nbits; nextdata = sp->lzw_nextdata; @@ -493,7 +494,7 @@ begin: { WordType code; GetNextCodeLZW(); - codep = dec_codetab + code; + codep = dec_codetab + (unsigned long)code; if (code >= CODE_FIRST) goto code_above_or_equal_to_258; if (code < 256) @@ -508,7 +509,7 @@ code_below_256: goto error_code; free_entp->next = oldcodep; free_entp->firstchar = oldcodep->firstchar; - free_entp->length = oldcodep->length + 1; + free_entp->length = (uint16_t)(oldcodep->length + 1); free_entp->value = (uint8_t)code; free_entp->repeated = (bool)(oldcodep->repeated & (oldcodep->value == code)); @@ -556,7 +557,7 @@ code_above_or_equal_to_258: free_entp->next = oldcodep; free_entp->firstchar = oldcodep->firstchar; - free_entp->length = oldcodep->length + 1; + free_entp->length = (uint16_t)(oldcodep->length + 1); if (++free_entp > maxcodep) { if (++nbits > BITS_MAX) /* should not happen for a conformant encoder */ @@ -673,7 +674,7 @@ code_clear: free_entp = dec_codetab + CODE_FIRST; nbits = BITS_MIN; nbitsmask = MAXCODE(BITS_MIN); - maxcodep = dec_codetab + nbitsmask - 1; + maxcodep = dec_codetab + (unsigned long)nbitsmask - 1; do { GetNextCodeLZW(); @@ -736,7 +737,7 @@ after_loop: TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" PRIu64 " bytes)", - tif->tif_row, (uint64_t)occ); + tif->tif_dir.td_row, (uint64_t)occ); return (0); } return (1); @@ -746,7 +747,7 @@ no_eoi: sp->read_error = 1; TIFFErrorExtR(tif, module, "LZWDecode: Strip %" PRIu32 " not terminated with EOI code", - tif->tif_curstrip); + tif->tif_dir.td_curstrip); return 0; error_code: memset(op, 0, (size_t)occ); @@ -768,13 +769,13 @@ error_code: TIFFWarningExtR(_tif, module, \ "LZWDecode: Strip %" PRIu32 \ " not terminated with EOI code", \ - _tif->tif_curstrip); \ + _tif->tif_dir.td_curstrip); \ _code = CODE_EOI; \ } \ else \ { \ _get(_sp, _bp, _code); \ - dec_bitsleft -= nbits; \ + dec_bitsleft -= (uint64_t)nbits; \ } \ } @@ -790,7 +791,7 @@ error_code: nextdata |= (unsigned long)*(bp)++ << nextbits; \ nextbits += 8; \ } \ - code = (hcode_t)(nextdata & nbitsmask); \ + code = (hcode_t)(nextdata & (unsigned long)nbitsmask); \ nextdata >>= nbits; \ nextbits -= nbits; \ } @@ -858,7 +859,8 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) bp = (uint8_t *)tif->tif_rawcp; - sp->dec_bitsleft += (((uint64_t)tif->tif_rawcc - sp->old_tif_rawcc) << 3); + sp->dec_bitsleft += + (((uint64_t)tif->tif_rawcc - (uint64_t)sp->old_tif_rawcc) << 3); uint64_t dec_bitsleft = sp->dec_bitsleft; nbits = sp->lzw_nbits; @@ -893,7 +895,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR( tif, tif->tif_name, "LZWDecode: Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } *op++ = (uint8_t)code; @@ -911,7 +913,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { TIFFErrorExtR(tif, module, "Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } @@ -921,11 +923,11 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { TIFFErrorExtR(tif, module, "Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } free_entp->firstchar = free_entp->next->firstchar; - free_entp->length = free_entp->next->length + 1; + free_entp->length = (uint16_t)(free_entp->next->length + 1); free_entp->value = (codep < free_entp) ? codep->firstchar : free_entp->firstchar; if (++free_entp > maxcodep) @@ -948,7 +950,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) tif, module, "Wrong length of decoded " "string: data probably corrupted at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } if (codep->length > occ) @@ -1010,7 +1012,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" PRIu64 " bytes)", - tif->tif_row, (uint64_t)occ); + tif->tif_dir.td_row, (uint64_t)occ); return (0); } return (1); @@ -1116,10 +1118,10 @@ static int LZWPreEncode(TIFF *tif, uint16_t s) */ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { - register LZWCodecState *sp = LZWEncoderState(tif); - register long fcode; - register hash_t *hp; - register int h, c; + LZWCodecState *sp = LZWEncoderState(tif); + long fcode; + hash_t *hp; + int h, c; hcode_t ent; long disp; tmsize_t incount, outcount, checkpoint; @@ -1196,7 +1198,7 @@ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) * Avoid pointer arithmetic because of * wraparound problems with segments. */ - if ((h -= disp) < 0) + if ((h -= (int)disp) < 0) h += HSIZE; hp = &sp->enc_hashtab[h]; if (hp->hash == fcode) @@ -1301,7 +1303,7 @@ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) */ static int LZWPostEncode(TIFF *tif) { - register LZWCodecState *sp = LZWEncoderState(tif); + LZWCodecState *sp = LZWEncoderState(tif); uint8_t *op = tif->tif_rawcp; long nextbits = sp->lzw_nextbits; WordType nextdata = sp->lzw_nextdata; @@ -1319,7 +1321,7 @@ static int LZWPostEncode(TIFF *tif) { int free_ent = sp->lzw_free_ent; - PutNextCode(op, sp->enc_oldcode); + PutNextCode(op, (WordType)sp->enc_oldcode); sp->enc_oldcode = (hcode_t)-1; free_ent++; @@ -1357,8 +1359,8 @@ static int LZWPostEncode(TIFF *tif) */ static void cl_hash(LZWCodecState *sp) { - register hash_t *hp = &sp->enc_hashtab[HSIZE - 1]; - register long i = HSIZE - 8; + hash_t *hp = &sp->enc_hashtab[HSIZE - 1]; + long i = HSIZE - 8; do { @@ -1397,6 +1399,21 @@ static void LZWCleanup(TIFF *tif) _TIFFSetDefaultCompressionState(tif); } +static uint64_t LZWGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 562 */ + /* 4096x4096: 1243 */ + /* 16383x16383: 1353 */ + /* 65536x65536: 1362 */ + + return 1400; +} + int TIFFInitLZW(TIFF *tif, int scheme) { static const char module[] = "TIFFInitLZW"; @@ -1430,6 +1447,7 @@ int TIFFInitLZW(TIFF *tif, int scheme) tif->tif_encodestrip = LZWEncode; tif->tif_encodetile = LZWEncode; #endif + tif->tif_getmaxcompressionratio = LZWGetMaxCompressionRatio; tif->tif_cleanup = LZWCleanup; /* * Setup predictor setup. diff --git a/contrib/libs/libtiff/tif_next.c b/contrib/libs/libtiff/tif_next.c index f000574ee74..04ae474b95b 100644 --- a/contrib/libs/libtiff/tif_next.c +++ b/contrib/libs/libtiff/tif_next.c @@ -38,21 +38,22 @@ op[0] = (unsigned char)((v) << 6); \ break; \ case 1: \ - op[0] |= (v) << 4; \ + op[0] |= (unsigned char)((v) << 4); \ break; \ case 2: \ - op[0] |= (v) << 2; \ + op[0] |= (unsigned char)((v) << 2); \ break; \ case 3: \ - *op++ |= (v); \ + *op++ |= (unsigned char)(v); \ op_offset++; \ break; \ + default: \ + break; \ } \ } #define LITERALROW 0x00 #define LITERALSPAN 0x40 -#define WHITE ((1 << 2) - 1) static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { @@ -73,7 +74,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; - scanline = tif->tif_scanlinesize; + scanline = tif->tif_dir.td_scanlinesize; if (occ % scanline) { TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); @@ -146,7 +147,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { TIFFErrorExtR(tif, module, "Invalid data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } if (cc == 0) @@ -163,7 +164,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) return (1); bad: TIFFErrorExtR(tif, module, "Not enough data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } diff --git a/contrib/libs/libtiff/tif_ojpeg.c b/contrib/libs/libtiff/tif_ojpeg.c index 5ef7a6af6fe..e62ec9ea72c 100644 --- a/contrib/libs/libtiff/tif_ojpeg.c +++ b/contrib/libs/libtiff/tif_ojpeg.c @@ -122,9 +122,6 @@ restarting of LibJpeg decoding session. */ -#define WIN32_LEAN_AND_MEAN -#define VC_EXTRALEAN - #include "tiffiop.h" #ifdef OJPEG_SUPPORT @@ -209,18 +206,12 @@ static const TIFFField ojpegFields[] = { #include "jerror.h" #include "jpeglib.h" -#ifndef TIFF_jpeg_source_mgr_defined -#define TIFF_jpeg_source_mgr_defined -typedef struct jpeg_source_mgr jpeg_source_mgr; -#endif +typedef struct jpeg_source_mgr tiff_ojpeg_source_mgr; -#ifndef TIFF_jpeg_error_mgr_defined -#define TIFF_jpeg_error_mgr_defined -typedef struct jpeg_error_mgr jpeg_error_mgr; -#endif +typedef struct jpeg_error_mgr tiff_ojpeg_error_mgr; -typedef struct jpeg_common_struct jpeg_common_struct; -typedef struct jpeg_decompress_struct jpeg_decompress_struct; +typedef struct jpeg_common_struct tiff_ojpeg_common_struct; +typedef struct jpeg_decompress_struct tiff_ojpeg_decompress_struct; typedef enum { @@ -316,9 +307,9 @@ typedef struct uint32_t write_curstrile; uint8_t libjpeg_session_active; uint8_t libjpeg_jpeg_query_style; - jpeg_error_mgr libjpeg_jpeg_error_mgr; - jpeg_decompress_struct libjpeg_jpeg_decompress_struct; - jpeg_source_mgr libjpeg_jpeg_source_mgr; + tiff_ojpeg_error_mgr libjpeg_jpeg_error_mgr; + tiff_ojpeg_decompress_struct libjpeg_jpeg_decompress_struct; + tiff_ojpeg_source_mgr libjpeg_jpeg_source_mgr; uint8_t subsampling_convert_log; uint32_t subsampling_convert_ylinelen; uint32_t subsampling_convert_ylines; @@ -413,46 +404,51 @@ static void OJPEGWriteStreamEoi(TIFF *tif, void **mem, uint32_t *len); #ifdef LIBJPEG_ENCAP_EXTERNAL extern int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); -extern int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo); +extern int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image); extern int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); + tiff_ojpeg_decompress_struct *cinfo); extern int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines); extern int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines); + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines); extern void jpeg_encap_unwind(TIFF *tif); #else static int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *j); -static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *j); +static int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image); static int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); + tiff_ojpeg_decompress_struct *cinfo); static int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines); static int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines); + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines); static void jpeg_encap_unwind(TIFF *tif); #endif -static void OJPEGLibjpegJpegErrorMgrOutputMessage(jpeg_common_struct *cinfo); -static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo); -static void OJPEGLibjpegJpegSourceMgrInitSource(jpeg_decompress_struct *cinfo); +static void +OJPEGLibjpegJpegErrorMgrOutputMessage(tiff_ojpeg_common_struct *cinfo); +static void OJPEGLibjpegJpegErrorMgrErrorExit(tiff_ojpeg_common_struct *cinfo); +static void +OJPEGLibjpegJpegSourceMgrInitSource(tiff_ojpeg_decompress_struct *cinfo); static boolean -OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo); +OJPEGLibjpegJpegSourceMgrFillInputBuffer(tiff_ojpeg_decompress_struct *cinfo); static void -OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrSkipInputData(tiff_ojpeg_decompress_struct *cinfo, long num_bytes); static boolean -OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrResyncToRestart(tiff_ojpeg_decompress_struct *cinfo, int desired); -static void OJPEGLibjpegJpegSourceMgrTermSource(jpeg_decompress_struct *cinfo); +static void +OJPEGLibjpegJpegSourceMgrTermSource(tiff_ojpeg_decompress_struct *cinfo); int TIFFInitOJPEG(TIFF *tif, int scheme) { @@ -473,7 +469,7 @@ int TIFFInitOJPEG(TIFF *tif, int scheme) } /* state block */ - sp = _TIFFmallocExt(tif, sizeof(OJPEGState)); + sp = (OJPEGState *)_TIFFmallocExt(tif, sizeof(OJPEGState)); if (sp == NULL) { TIFFErrorExtR(tif, module, "No space for OJPEG state block"); @@ -724,9 +720,9 @@ static int OJPEGPreDecode(TIFF *tif, uint16_t s) return (0); } if (isTiled(tif)) - m = tif->tif_curtile; + m = tif->tif_dir.td_curtile; else - m = tif->tif_curstrip; + m = tif->tif_dir.td_curstrip; if ((sp->writeheader_done != 0) && ((sp->write_cursample != s) || (sp->write_curstrile > m))) { @@ -797,7 +793,7 @@ static int OJPEGPreDecodeSkipRaw(TIFF *tif) { if (jpeg_read_raw_data_encap(sp, &(sp->libjpeg_jpeg_decompress_struct), sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + (uint32_t)sp->subsampling_ver * 8) == 0) return (0); m -= sp->subsampling_convert_clines; } @@ -805,7 +801,7 @@ static int OJPEGPreDecodeSkipRaw(TIFF *tif) { if (jpeg_read_raw_data_encap(sp, &(sp->libjpeg_jpeg_decompress_struct), sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + (uint32_t)sp->subsampling_ver * 8) == 0) return (0); sp->subsampling_convert_state = m; } @@ -819,7 +815,7 @@ static int OJPEGPreDecodeSkipScanlines(TIFF *tif) uint32_t m; if (sp->skip_buffer == NULL) { - sp->skip_buffer = _TIFFmallocExt(tif, sp->bytes_per_line); + sp->skip_buffer = (uint8_t *)_TIFFmallocExt(tif, sp->bytes_per_line); if (sp->skip_buffer == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -910,10 +906,10 @@ static int OJPEGDecodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc) { if (sp->subsampling_convert_state == 0) { - if (jpeg_read_raw_data_encap(sp, - &(sp->libjpeg_jpeg_decompress_struct), - sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + if (jpeg_read_raw_data_encap( + sp, &(sp->libjpeg_jpeg_decompress_struct), + sp->subsampling_convert_ycbcrimage, + (uint32_t)sp->subsampling_ver * 8) == 0) { sp->error_in_raw_data_decoding = 1; return (0); @@ -1225,7 +1221,7 @@ static int OJPEGReadHeaderInfo(TIFF *tif) TIFFErrorExtR(tif, module, "Invalid subsampling values"); return (0); } - if (sp->strile_length % (sp->subsampling_ver * 8) != 0) + if (sp->strile_length % ((uint32_t)sp->subsampling_ver * 8) != 0) { TIFFErrorExtR(tif, module, "Incompatible vertical subsampling and image " @@ -1233,9 +1229,11 @@ static int OJPEGReadHeaderInfo(TIFF *tif) return (0); } sp->restart_interval = - (uint16_t)(((sp->strile_width + sp->subsampling_hor * 8 - 1) / - (sp->subsampling_hor * 8)) * - (sp->strile_length / (sp->subsampling_ver * 8))); + (uint16_t)(((sp->strile_width + (uint32_t)sp->subsampling_hor * 8 - + 1) / + ((uint32_t)sp->subsampling_hor * 8)) * + (sp->strile_length / + ((uint32_t)sp->subsampling_ver * 8))); } if (OJPEGReadHeaderInfoSec(tif) == 0) return (0); @@ -1358,20 +1356,43 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) /* Check for division by zero. */ if (sp->subsampling_hor == 0 || sp->subsampling_ver == 0) return (0); + /* Check for potential overflow in subsampling_convert_ylinelen + * computation. + */ + if (sp->strile_width > + UINT32_MAX - ((uint32_t)sp->subsampling_hor * 8 - 1)) + return (0); sp->subsampling_convert_ylinelen = - ((sp->strile_width + sp->subsampling_hor * 8 - 1) / - (sp->subsampling_hor * 8) * sp->subsampling_hor * 8); - sp->subsampling_convert_ylines = sp->subsampling_ver * 8; + ((sp->strile_width + (uint32_t)sp->subsampling_hor * 8 - 1) / + ((uint32_t)sp->subsampling_hor * 8) * + ((uint32_t)sp->subsampling_hor * 8)); + sp->subsampling_convert_ylines = (uint32_t)sp->subsampling_ver * 8; sp->subsampling_convert_clinelen = sp->subsampling_convert_ylinelen / sp->subsampling_hor; sp->subsampling_convert_clines = 8; - sp->subsampling_convert_ybuflen = sp->subsampling_convert_ylinelen * - sp->subsampling_convert_ylines; - sp->subsampling_convert_cbuflen = sp->subsampling_convert_clinelen * - sp->subsampling_convert_clines; - sp->subsampling_convert_ycbcrbuflen = - sp->subsampling_convert_ybuflen + - 2 * sp->subsampling_convert_cbuflen; + /* Check for potential overflow in buffer length computations. + * Use 64-bit intermediates to detect uint32_t overflow in + * ylinelen * ylines, clinelen * clines, and their sum. + */ + { + uint64_t ybuflen64 = + (uint64_t)sp->subsampling_convert_ylinelen * + sp->subsampling_convert_ylines; + uint64_t cbuflen64 = + (uint64_t)sp->subsampling_convert_clinelen * + sp->subsampling_convert_clines; + uint64_t ycbcrbuflen64 = ybuflen64 + 2 * cbuflen64; + if (ybuflen64 > UINT32_MAX || cbuflen64 > UINT32_MAX || + ycbcrbuflen64 > UINT32_MAX) + { + TIFFErrorExtR(tif, module, + "Integer overflow in OJPEG buffer size"); + return (0); + } + sp->subsampling_convert_ybuflen = (uint32_t)ybuflen64; + sp->subsampling_convert_cbuflen = (uint32_t)cbuflen64; + sp->subsampling_convert_ycbcrbuflen = (uint32_t)ycbcrbuflen64; + } /* The calloc is not normally necessary, except in some edge/broken * cases */ /* for example for a tiled image of height 1 with a tile height of 1 @@ -1383,8 +1404,8 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) /* Even if this case is allowed (?), its handling is broken because * OJPEGPreDecode() should also likely */ /* reset subsampling_convert_state to 0 when changing tile. */ - sp->subsampling_convert_ycbcrbuf = - _TIFFcallocExt(tif, 1, sp->subsampling_convert_ycbcrbuflen); + sp->subsampling_convert_ycbcrbuf = (uint8_t *)_TIFFcallocExt( + tif, 1, sp->subsampling_convert_ycbcrbuflen); if (sp->subsampling_convert_ycbcrbuf == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1398,8 +1419,9 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) sp->subsampling_convert_ycbcrimagelen = 3 + sp->subsampling_convert_ylines + 2 * sp->subsampling_convert_clines; - sp->subsampling_convert_ycbcrimage = _TIFFmallocExt( - tif, sp->subsampling_convert_ycbcrimagelen * sizeof(uint8_t *)); + sp->subsampling_convert_ycbcrimage = (uint8_t **)_TIFFmallocExt( + tif, (tmsize_t)((size_t)sp->subsampling_convert_ycbcrimagelen * + sizeof(uint8_t *))); if (sp->subsampling_convert_ycbcrimage == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1426,9 +1448,14 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) ((sp->strile_width % sp->subsampling_hor) != 0 ? 1 : 0); sp->subsampling_convert_state = 0; sp->error_in_raw_data_decoding = 0; - sp->bytes_per_line = - sp->subsampling_convert_clinelenout * - (sp->subsampling_ver * sp->subsampling_hor + 2); + + const uint64_t bpl = + (uint64_t)sp->subsampling_convert_clinelenout * + ((uint64_t)sp->subsampling_ver * sp->subsampling_hor + 2); + if (bpl > UINT32_MAX) + return (0); + sp->bytes_per_line = (uint32_t)bpl; + sp->lines_per_strile = sp->strile_length / sp->subsampling_ver + ((sp->strile_length % sp->subsampling_ver) != 0 ? 1 : 0); @@ -1478,7 +1505,8 @@ static void OJPEGLibjpegSessionAbort(TIFF *tif) { OJPEGState *sp = (OJPEGState *)tif->tif_data; assert(sp->libjpeg_session_active != 0); - jpeg_destroy((jpeg_common_struct *)(&(sp->libjpeg_jpeg_decompress_struct))); + jpeg_destroy( + (tiff_ojpeg_common_struct *)(&(sp->libjpeg_jpeg_decompress_struct))); sp->libjpeg_session_active = 0; } @@ -1560,7 +1588,7 @@ static int OJPEGReadHeaderInfoSec(TIFF *tif) return (0); } if (n > 2) - OJPEGReadSkip(sp, n - 2); + OJPEGReadSkip(sp, (uint16_t)(n - 2)); break; case JPEG_MARKER_DRI: if (OJPEGReadHeaderInfoSecStreamDri(tif) == 0) @@ -1603,8 +1631,9 @@ static int OJPEGReadHeaderInfoSec(TIFF *tif) return (0); sp->sof_marker_id = JPEG_MARKER_SOF0; for (o = 0; o < sp->samples_per_pixel; o++) - sp->sof_c[o] = o; - sp->sof_hv[0] = ((sp->subsampling_hor << 4) | sp->subsampling_ver); + sp->sof_c[o] = (uint8_t)o; + sp->sof_hv[0] = + (uint8_t)((sp->subsampling_hor << 4) | sp->subsampling_ver); for (o = 1; o < sp->samples_per_pixel; o++) sp->sof_hv[o] = 17; sp->sof_x = sp->strile_width; @@ -1659,10 +1688,10 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) return (0); } if (sp->subsamplingcorrect != 0) - OJPEGReadSkip(sp, m - 2); + OJPEGReadSkip(sp, (uint16_t)(m - 2)); else { - m -= 2; + m = (uint16_t)(m - 2); do { if (m < 65) @@ -1671,7 +1700,7 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) return (0); } na = sizeof(uint32_t) + 69; - nb = _TIFFmallocExt(tif, na); + nb = (uint8_t *)_TIFFmallocExt(tif, na); if (nb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1697,7 +1726,7 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) if (sp->qtable[o] != 0) _TIFFfreeExt(tif, sp->qtable[o]); sp->qtable[o] = nb; - m -= 65; + m = (uint16_t)(m - 65); } while (m > 0); } return (1); @@ -1725,12 +1754,12 @@ static int OJPEGReadHeaderInfoSecStreamDht(TIFF *tif) } if (sp->subsamplingcorrect != 0) { - OJPEGReadSkip(sp, m - 2); + OJPEGReadSkip(sp, (uint16_t)(m - 2)); } else { - na = sizeof(uint32_t) + 2 + m; - nb = _TIFFmallocExt(tif, na); + na = (uint32_t)(sizeof(uint32_t) + 2 + m); + nb = (uint8_t *)_TIFFmallocExt(tif, na); if (nb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1739,9 +1768,10 @@ static int OJPEGReadHeaderInfoSecStreamDht(TIFF *tif) *(uint32_t *)nb = na; nb[sizeof(uint32_t)] = 255; nb[sizeof(uint32_t) + 1] = JPEG_MARKER_DHT; - nb[sizeof(uint32_t) + 2] = (m >> 8); - nb[sizeof(uint32_t) + 3] = (m & 255); - if (OJPEGReadBlock(sp, m - 2, &nb[sizeof(uint32_t) + 4]) == 0) + nb[sizeof(uint32_t) + 2] = (uint8_t)(m >> 8); + nb[sizeof(uint32_t) + 3] = (uint8_t)(m & 255); + if (OJPEGReadBlock(sp, (uint16_t)(m - 2), &nb[sizeof(uint32_t) + 4]) == + 0) { _TIFFfreeExt(tif, nb); return (0); @@ -1809,7 +1839,7 @@ static int OJPEGReadHeaderInfoSecStreamSof(TIFF *tif, uint8_t marker_id) TIFFErrorExtR(tif, module, "Corrupt SOF marker in JPEG data"); return (0); } - m -= 8; + m = (uint16_t)(m - 8); if (m % 3 != 0) { if (sp->subsamplingcorrect == 0) @@ -2030,7 +2060,7 @@ static int OJPEGReadHeaderInfoSecTablesQTable(TIFF *tif) } } oa = sizeof(uint32_t) + 69; - ob = _TIFFmallocExt(tif, oa); + ob = (uint8_t *)_TIFFmallocExt(tif, oa); if (ob == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2098,8 +2128,8 @@ static int OJPEGReadHeaderInfoSecTablesDcTable(TIFF *tif) q = 0; for (n = 0; n < 16; n++) q += o[n]; - ra = sizeof(uint32_t) + 21 + q; - rb = _TIFFmallocExt(tif, ra); + ra = (uint32_t)(sizeof(uint32_t) + 21 + q); + rb = (uint8_t *)_TIFFmallocExt(tif, ra); if (rb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2122,7 +2152,7 @@ static int OJPEGReadHeaderInfoSecTablesDcTable(TIFF *tif) if (sp->dctable[m] != 0) _TIFFfreeExt(tif, sp->dctable[m]); sp->dctable[m] = rb; - sp->sos_tda[m] = (m << 4); + sp->sos_tda[m] = (uint8_t)(m << 4); } else sp->sos_tda[m] = sp->sos_tda[m - 1]; @@ -2168,8 +2198,8 @@ static int OJPEGReadHeaderInfoSecTablesAcTable(TIFF *tif) q = 0; for (n = 0; n < 16; n++) q += o[n]; - ra = sizeof(uint32_t) + 21 + q; - rb = _TIFFmallocExt(tif, ra); + ra = (uint32_t)(sizeof(uint32_t) + 21 + q); + rb = (uint8_t *)_TIFFmallocExt(tif, ra); if (rb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2220,7 +2250,7 @@ static int OJPEGReadBufferFill(OJPEGState *sp) if ((uint64_t)m > sp->in_buffer_file_togo) m = (uint16_t)sp->in_buffer_file_togo; n = TIFFReadFile(sp->tif, sp->in_buffer, (tmsize_t)m); - if (n == 0) + if (n <= 0) return (0); assert(n > 0); assert(n <= OJPEG_BUFFER); @@ -2286,6 +2316,7 @@ static int OJPEGReadBufferFill(OJPEGState *sp) sp->in_buffer_next_strile++; } break; + case osibsEof: default: return (0); } @@ -2331,7 +2362,7 @@ static int OJPEGReadWord(OJPEGState *sp, uint16_t *word) uint8_t m; if (OJPEGReadByte(sp, &m) == 0) return (0); - *word = (m << 8); + *word = (uint16_t)(m << 8); if (OJPEGReadByte(sp, &m) == 0) return (0); *word |= m; @@ -2345,7 +2376,7 @@ static int OJPEGReadBlock(OJPEGState *sp, uint16_t len, void *mem) uint16_t n; assert(len > 0); mlen = len; - mmem = mem; + mmem = (uint8_t *)mem; do { if (sp->in_buffer_togo == 0) @@ -2359,8 +2390,8 @@ static int OJPEGReadBlock(OJPEGState *sp, uint16_t len, void *mem) n = sp->in_buffer_togo; _TIFFmemcpy(mmem, sp->in_buffer_cur, n); sp->in_buffer_cur += n; - sp->in_buffer_togo -= n; - mlen -= n; + sp->in_buffer_togo = (uint16_t)(sp->in_buffer_togo - n); + mlen = (uint16_t)(mlen - n); mmem += n; } while (mlen > 0); return (1); @@ -2375,8 +2406,8 @@ static void OJPEGReadSkip(OJPEGState *sp, uint16_t len) if (n > sp->in_buffer_togo) n = sp->in_buffer_togo; sp->in_buffer_cur += n; - sp->in_buffer_togo -= n; - m -= n; + sp->in_buffer_togo = (uint16_t)(sp->in_buffer_togo - n); + m = (uint16_t)(m - n); if (m > 0) { assert(sp->in_buffer_togo == 0); @@ -2462,6 +2493,8 @@ static int OJPEGWriteStream(TIFF *tif, void **mem, uint32_t *len) case ososEoi: OJPEGWriteStreamEoi(tif, mem, len); break; + default: + break; } } while (*len == 0); return (1); @@ -2475,7 +2508,7 @@ static void OJPEGWriteStreamSoi(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_SOI; *len = 2; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamQTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2485,9 +2518,10 @@ static void OJPEGWriteStreamQTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->qtable[table_index] != 0) { *mem = (void *)(sp->qtable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->qtable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->qtable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamDcTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2497,9 +2531,10 @@ static void OJPEGWriteStreamDcTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->dctable[table_index] != 0) { *mem = (void *)(sp->dctable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->dctable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->dctable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamAcTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2509,9 +2544,10 @@ static void OJPEGWriteStreamAcTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->actable[table_index] != 0) { *mem = (void *)(sp->actable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->actable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->actable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamDri(TIFF *tif, void **mem, uint32_t *len) @@ -2524,12 +2560,12 @@ static void OJPEGWriteStreamDri(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_DRI; sp->out_buffer[2] = 0; sp->out_buffer[3] = 4; - sp->out_buffer[4] = (sp->restart_interval >> 8); - sp->out_buffer[5] = (sp->restart_interval & 255); + sp->out_buffer[4] = (uint8_t)(sp->restart_interval >> 8); + sp->out_buffer[5] = (uint8_t)(sp->restart_interval & 255); *len = 6; *mem = (void *)sp->out_buffer; } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) @@ -2542,15 +2578,15 @@ static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = sp->sof_marker_id; /* Lf */ sp->out_buffer[2] = 0; - sp->out_buffer[3] = 8 + sp->samples_per_pixel_per_plane * 3; + sp->out_buffer[3] = (uint8_t)(8 + sp->samples_per_pixel_per_plane * 3); /* P */ sp->out_buffer[4] = 8; /* Y */ sp->out_buffer[5] = (uint8_t)(sp->sof_y >> 8); - sp->out_buffer[6] = (sp->sof_y & 255); + sp->out_buffer[6] = (uint8_t)(sp->sof_y & 255); /* X */ sp->out_buffer[7] = (uint8_t)(sp->sof_x >> 8); - sp->out_buffer[8] = (sp->sof_x & 255); + sp->out_buffer[8] = (uint8_t)(sp->sof_x & 255); /* Nf */ sp->out_buffer[9] = sp->samples_per_pixel_per_plane; for (m = 0; m < sp->samples_per_pixel_per_plane; m++) @@ -2564,9 +2600,9 @@ static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[10 + m * 3 + 2] = sp->sof_tq[sp->plane_sample_offset + m]; } - *len = 10 + sp->samples_per_pixel_per_plane * 3; + *len = 10 + (uint32_t)sp->samples_per_pixel_per_plane * 3; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) @@ -2579,7 +2615,7 @@ static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_SOS; /* Ls */ sp->out_buffer[2] = 0; - sp->out_buffer[3] = 6 + sp->samples_per_pixel_per_plane * 2; + sp->out_buffer[3] = (uint8_t)(6 + sp->samples_per_pixel_per_plane * 2); /* Ns */ sp->out_buffer[4] = sp->samples_per_pixel_per_plane; for (m = 0; m < sp->samples_per_pixel_per_plane; m++) @@ -2596,9 +2632,9 @@ static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[5 + sp->samples_per_pixel_per_plane * 2 + 1] = 63; /* Ah and Al */ sp->out_buffer[5 + sp->samples_per_pixel_per_plane * 2 + 2] = 0; - *len = 8 + sp->samples_per_pixel_per_plane * 2; + *len = 8 + (uint32_t)sp->samples_per_pixel_per_plane * 2; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static int OJPEGWriteStreamCompressed(TIFF *tif, void **mem, uint32_t *len) @@ -2626,6 +2662,8 @@ static int OJPEGWriteStreamCompressed(TIFF *tif, void **mem, uint32_t *len) case osibsEof: sp->out_state = ososEoi; break; + case osibsNotSetYet: + case osibsJpegInterchangeFormat: default: break; } @@ -2638,7 +2676,7 @@ static void OJPEGWriteStreamRst(TIFF *tif, void **mem, uint32_t *len) OJPEGState *sp = (OJPEGState *)tif->tif_data; assert(OJPEG_BUFFER >= 2); sp->out_buffer[0] = 255; - sp->out_buffer[1] = JPEG_MARKER_RST0 + sp->restart_index; + sp->out_buffer[1] = (uint8_t)(JPEG_MARKER_RST0 + sp->restart_index); sp->restart_index++; if (sp->restart_index == 8) sp->restart_index = 0; @@ -2659,7 +2697,7 @@ static void OJPEGWriteStreamEoi(TIFF *tif, void **mem, uint32_t *len) #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo) + tiff_ojpeg_decompress_struct *cinfo) { if (SETJMP(sp->exit_jmpbuf)) return 0; @@ -2672,7 +2710,8 @@ static int jpeg_create_decompress_encap(OJPEGState *sp, #endif #ifndef LIBJPEG_ENCAP_EXTERNAL -static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, +static int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image) { if (SETJMP(sp->exit_jmpbuf)) @@ -2687,7 +2726,7 @@ static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo) + tiff_ojpeg_decompress_struct *cinfo) { if (SETJMP(sp->exit_jmpbuf)) return 0; @@ -2701,14 +2740,14 @@ static int jpeg_start_decompress_encap(OJPEGState *sp, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines) { if (SETJMP(sp->exit_jmpbuf)) return 0; else { - jpeg_read_scanlines(cinfo, scanlines, max_lines); + jpeg_read_scanlines(cinfo, (JSAMPARRAY)scanlines, max_lines); return 1; } } @@ -2716,14 +2755,14 @@ static int jpeg_read_scanlines_encap(OJPEGState *sp, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines) + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines) { if (SETJMP(sp->exit_jmpbuf)) return 0; else { - jpeg_read_raw_data(cinfo, data, max_lines); + jpeg_read_raw_data(cinfo, (JSAMPIMAGE)data, max_lines); return 1; } } @@ -2737,14 +2776,15 @@ static void jpeg_encap_unwind(TIFF *tif) } #endif -static void OJPEGLibjpegJpegErrorMgrOutputMessage(jpeg_common_struct *cinfo) +static void +OJPEGLibjpegJpegErrorMgrOutputMessage(tiff_ojpeg_common_struct *cinfo) { char buffer[JMSG_LENGTH_MAX]; (*cinfo->err->format_message)(cinfo, buffer); TIFFWarningExtR(((TIFF *)(cinfo->client_data)), "LibJpeg", "%s", buffer); } -static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo) +static void OJPEGLibjpegJpegErrorMgrErrorExit(tiff_ojpeg_common_struct *cinfo) { char buffer[JMSG_LENGTH_MAX]; (*cinfo->err->format_message)(cinfo, buffer); @@ -2752,13 +2792,14 @@ static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo) jpeg_encap_unwind((TIFF *)(cinfo->client_data)); } -static void OJPEGLibjpegJpegSourceMgrInitSource(jpeg_decompress_struct *cinfo) +static void +OJPEGLibjpegJpegSourceMgrInitSource(tiff_ojpeg_decompress_struct *cinfo) { (void)cinfo; } static boolean -OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo) +OJPEGLibjpegJpegSourceMgrFillInputBuffer(tiff_ojpeg_decompress_struct *cinfo) { TIFF *tif = (TIFF *)cinfo->client_data; OJPEGState *sp = (OJPEGState *)tif->tif_data; @@ -2770,12 +2811,12 @@ OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo) jpeg_encap_unwind(tif); } sp->libjpeg_jpeg_source_mgr.bytes_in_buffer = len; - sp->libjpeg_jpeg_source_mgr.next_input_byte = mem; + sp->libjpeg_jpeg_source_mgr.next_input_byte = (const JOCTET *)mem; return (1); } static void -OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrSkipInputData(tiff_ojpeg_decompress_struct *cinfo, long num_bytes) { TIFF *tif = (TIFF *)cinfo->client_data; @@ -2789,7 +2830,7 @@ OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, #pragma warning(disable : 4702) /* unreachable code */ #endif static boolean -OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrResyncToRestart(tiff_ojpeg_decompress_struct *cinfo, int desired) { TIFF *tif = (TIFF *)cinfo->client_data; @@ -2802,7 +2843,8 @@ OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, #pragma warning(pop) #endif -static void OJPEGLibjpegJpegSourceMgrTermSource(jpeg_decompress_struct *cinfo) +static void +OJPEGLibjpegJpegSourceMgrTermSource(tiff_ojpeg_decompress_struct *cinfo) { (void)cinfo; } diff --git a/contrib/libs/libtiff/tif_open.c b/contrib/libs/libtiff/tif_open.c index 565d13cf694..43cb170286d 100644 --- a/contrib/libs/libtiff/tif_open.c +++ b/contrib/libs/libtiff/tif_open.c @@ -77,7 +77,7 @@ int _TIFFgetMode(TIFFOpenOptions *opts, thandle_t clientdata, const char *mode, return (m); } -TIFFOpenOptions *TIFFOpenOptionsAlloc() +TIFFOpenOptions *TIFFOpenOptionsAlloc(void) { TIFFOpenOptions *opts = (TIFFOpenOptions *)_TIFFcalloc(1, sizeof(TIFFOpenOptions)); @@ -308,6 +308,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, TIFF *tif; int m; const char *cp; + tmsize_t size_to_alloc; /* The following are configuration checks. They should be redundant, but * should not compile to any actual code in an optimised release build @@ -330,7 +331,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, n.a8[0] = 1; n.a8[1] = 0; (void)n; -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN assert(n.a16 == 256); #else assert(n.a16 == 1); @@ -340,7 +341,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, m = _TIFFgetMode(opts, clientdata, mode, module); if (m == -1) goto bad2; - tmsize_t size_to_alloc = (tmsize_t)(sizeof(TIFF) + strlen(name) + 1); + size_to_alloc = (tmsize_t)(sizeof(TIFF) + strlen(name) + 1); if (opts && opts->max_single_mem_alloc > 0 && size_to_alloc > opts->max_single_mem_alloc) { @@ -377,8 +378,6 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; /* non-existent directory */ tif->tif_curdircount = TIFF_NON_EXISTENT_DIR_NUMBER; tif->tif_curoff = 0; - tif->tif_curstrip = (uint32_t)-1; /* invalid strip */ - tif->tif_row = (uint32_t)-1; /* read/write pre-increment */ tif->tif_clientdata = clientdata; tif->tif_readproc = readproc; tif->tif_writeproc = writeproc; @@ -398,6 +397,10 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_warn_about_unknown_tags = opts->warn_about_unknown_tags; } + /* Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. */ + _TIFFResetTifDirAndInitStrileCounters(&tif->tif_dir); + if (!readproc || !writeproc || !seekproc || !closeproc || !sizeproc) { TIFFErrorExtR(tif, module, @@ -480,13 +483,13 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, switch (*cp) { case 'b': -#ifndef WORDS_BIGENDIAN +#if !WORDS_BIGENDIAN if (m & O_CREAT) tif->tif_flags |= TIFF_SWAB; #endif break; case 'l': -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN if ((m & O_CREAT)) tif->tif_flags |= TIFF_SWAB; #endif @@ -536,7 +539,9 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, case 'O': if (m == O_RDONLY) tif->tif_flags |= - (TIFF_LAZYSTRILELOAD | TIFF_DEFERSTRILELOAD); + (TIFF_LAZYSTRILELOAD_ASKED | TIFF_DEFERSTRILELOAD); + break; + default: break; } @@ -561,7 +566,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, /* * Setup header and write. */ -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN tif->tif_header.common.tiff_magic = (tif->tif_flags & TIFF_SWAB) ? TIFF_LITTLEENDIAN : TIFF_BIGENDIAN; #else @@ -654,13 +659,13 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, } if (tif->tif_header.common.tiff_magic == TIFF_BIGENDIAN) { -#ifndef WORDS_BIGENDIAN +#if !WORDS_BIGENDIAN tif->tif_flags |= TIFF_SWAB; #endif } else { -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN tif->tif_flags |= TIFF_SWAB; #endif } @@ -780,6 +785,8 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, if (!TIFFDefaultDirectory(tif)) goto bad; return (tif); + default: + break; } bad: tif->tif_mode = O_RDONLY; /* XXX avoid flush */ @@ -861,7 +868,7 @@ int TIFFIsTiled(TIFF *tif) { return (isTiled(tif)); } /* * Return current row being read/written. */ -uint32_t TIFFCurrentRow(TIFF *tif) { return (tif->tif_row); } +uint32_t TIFFCurrentRow(TIFF *tif) { return (tif->tif_dir.td_row); } /* * Return index of the current directory. @@ -871,12 +878,12 @@ tdir_t TIFFCurrentDirectory(TIFF *tif) { return (tif->tif_curdir); } /* * Return current strip. */ -uint32_t TIFFCurrentStrip(TIFF *tif) { return (tif->tif_curstrip); } +uint32_t TIFFCurrentStrip(TIFF *tif) { return (tif->tif_dir.td_curstrip); } /* * Return current tile. */ -uint32_t TIFFCurrentTile(TIFF *tif) { return (tif->tif_curtile); } +uint32_t TIFFCurrentTile(TIFF *tif) { return (tif->tif_dir.td_curtile); } /* * Return nonzero if the file has byte-swapped data. diff --git a/contrib/libs/libtiff/tif_packbits.c b/contrib/libs/libtiff/tif_packbits.c index d7db9b64ea3..13d0f1d3908 100644 --- a/contrib/libs/libtiff/tif_packbits.c +++ b/contrib/libs/libtiff/tif_packbits.c @@ -194,12 +194,17 @@ static int PackBitsEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) */ if (n == 1 && op[-2] == (uint8_t)-1 && *lastliteral < 126) { - state = (((*lastliteral) += 2) == 127 ? BASE : LITERAL); + state = + (((*lastliteral) = (uint8_t)(*lastliteral + 2)) == 127 + ? BASE + : LITERAL); op[-2] = op[-1]; /* replicate */ } else state = RUN; goto again; + default: + break; } } tif->tif_rawcc += (tmsize_t)(op - tif->tif_rawcp); @@ -306,12 +311,22 @@ static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Not enough data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } return (1); } +static uint64_t PackBitsGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + return 64; +} + int TIFFInitPackBits(TIFF *tif, int scheme) { (void)scheme; @@ -325,6 +340,8 @@ int TIFFInitPackBits(TIFF *tif, int scheme) tif->tif_encodestrip = PackBitsEncodeChunk; tif->tif_encodetile = PackBitsEncodeChunk; #endif + tif->tif_getmaxcompressionratio = PackBitsGetMaxCompressionRatio; + return (1); } #endif /* PACKBITS_SUPPORT */ diff --git a/contrib/libs/libtiff/tif_pixarlog.c b/contrib/libs/libtiff/tif_pixarlog.c index 309054804d3..fa267dfbff2 100644 --- a/contrib/libs/libtiff/tif_pixarlog.c +++ b/contrib/libs/libtiff/tif_pixarlog.c @@ -117,11 +117,11 @@ static float LogK1, LogK2; } while (i > 0); \ } -static void horizontalAccumulateF(uint16_t *wp, int n, int stride, float *op, - float *ToLinearF) +static void horizontalAccumulateF(uint16_t *wp, tmsize_t n, int stride, + float *op, float *ToLinearF) { - register unsigned int cr, cg, cb, ca, mask; - register float t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + float t0, t1, t2, t3; if (n >= stride) { @@ -180,22 +180,22 @@ static void horizontalAccumulateF(uint16_t *wp, int n, int stride, float *op, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinearF[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinearF[*wp & mask]; wp++; op++) n -= stride; } } } } -static void horizontalAccumulate12(uint16_t *wp, int n, int stride, int16_t *op, - float *ToLinearF) +static void horizontalAccumulate12(uint16_t *wp, tmsize_t n, int stride, + int16_t *op, float *ToLinearF) { - register unsigned int cr, cg, cb, ca, mask; - register float t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + float t0, t1, t2, t3; -#define SCALE12 2048.0F -#define CLAMP12(t) (((t) < 3071) ? (uint16_t)(t) : 3071) +#define SCALE12 2048.0f +#define CLAMP12(t) (((t) < 3071) ? (int16_t)(uint16_t)(t) : (int16_t)3071) if (n >= stride) { @@ -255,19 +255,19 @@ static void horizontalAccumulate12(uint16_t *wp, int n, int stride, int16_t *op, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; - t0 = ToLinearF[wp[stride] & mask] * SCALE12; - *op = CLAMP12(t0); wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + t0 = ToLinearF[*wp & mask] * SCALE12; *op = CLAMP12(t0); + wp++; op++) n -= stride; } } } } -static void horizontalAccumulate16(uint16_t *wp, int n, int stride, +static void horizontalAccumulate16(uint16_t *wp, tmsize_t n, int stride, uint16_t *op, uint16_t *ToLinear16) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -312,8 +312,8 @@ static void horizontalAccumulate16(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear16[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear16[*wp & mask]; wp++; op++) n -= stride; } } @@ -324,10 +324,10 @@ static void horizontalAccumulate16(uint16_t *wp, int n, int stride, * Returns the log encoded 11-bit values with the horizontal * differencing undone. */ -static void horizontalAccumulate11(uint16_t *wp, int n, int stride, +static void horizontalAccumulate11(uint16_t *wp, tmsize_t n, int stride, uint16_t *op) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -375,21 +375,22 @@ static void horizontalAccumulate11(uint16_t *wp, int n, int stride, } else { - REPEAT(stride, *op = *wp & mask; wp++; op++) + REPEAT(stride, *op = (uint16_t)(*wp & mask); wp++; op++) n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = *wp & mask; wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = (uint16_t)(*wp & mask); wp++; op++) n -= stride; } } } } -static void horizontalAccumulate8(uint16_t *wp, int n, int stride, +static void horizontalAccumulate8(uint16_t *wp, tmsize_t n, int stride, unsigned char *op, unsigned char *ToLinear8) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -434,20 +435,20 @@ static void horizontalAccumulate8(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear8[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear8[*wp & mask]; wp++; op++) n -= stride; } } } } -static void horizontalAccumulate8abgr(uint16_t *wp, int n, int stride, +static void horizontalAccumulate8abgr(uint16_t *wp, tmsize_t n, int stride, unsigned char *op, unsigned char *ToLinear8) { - register unsigned int cr, cg, cb, ca, mask; - register unsigned char t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + unsigned char t0, t1, t2, t3; if (n >= stride) { @@ -508,8 +509,8 @@ static void horizontalAccumulate8abgr(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear8[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear8[*wp & mask]; wp++; op++) n -= stride; } } @@ -577,7 +578,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) LogK1 = (float)(1. / c); /* if (v >= 2) token = k1*log(v*k2) */ LogK2 = (float)(1. / b); lt2size = (int)(2. / linstep) + 1; - FromLT2 = (uint16_t *)_TIFFmallocExt(tif, lt2size * sizeof(uint16_t)); + FromLT2 = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)lt2size * sizeof(uint16_t))); From14 = (uint16_t *)_TIFFmallocExt(tif, 16384 * sizeof(uint16_t)); From8 = (uint16_t *)_TIFFmallocExt(tif, 256 * sizeof(uint16_t)); ToLinearF = (float *)_TIFFmallocExt(tif, TSIZEP1 * sizeof(float)); @@ -623,16 +625,17 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) for (i = 0; i < TSIZEP1; i++) { - v = ToLinearF[i] * 65535.0 + 0.5; + v = (double)ToLinearF[i] * 65535.0 + 0.5; ToLinear16[i] = (v > 65535.0) ? 65535 : (uint16_t)v; - v = ToLinearF[i] * 255.0 + 0.5; + v = (double)ToLinearF[i] * 255.0 + 0.5; ToLinear8[i] = (v > 255.0) ? 255 : (unsigned char)v; } j = 0; for (i = 0; i < lt2size; i++) { - if ((i * linstep) * (i * linstep) > ToLinearF[j] * ToLinearF[j + 1]) + if ((i * linstep) * (i * linstep) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; FromLT2[i] = (uint16_t)j; } @@ -645,7 +648,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) j = 0; for (i = 0; i < 16384; i++) { - while ((i / 16383.) * (i / 16383.) > ToLinearF[j] * ToLinearF[j + 1]) + while ((i / 16383.) * (i / 16383.) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; From14[i] = (uint16_t)j; } @@ -653,7 +657,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) j = 0; for (i = 0; i < 256; i++) { - while ((i / 255.) * (i / 255.) > ToLinearF[j] * ToLinearF[j + 1]) + while ((i / 255.) * (i / 255.) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; From8[i] = (uint16_t)j; } @@ -708,6 +713,8 @@ static int PixarLogGuessDataFmt(TIFFDirectory *td) if (format == SAMPLEFORMAT_VOID || format == SAMPLEFORMAT_UINT) guess = PIXARLOGDATAFMT_8BIT; break; + default: + break; } return guess; @@ -769,7 +776,8 @@ static int PixarLogSetupDecode(TIFF *tif) multiply_ms(multiply_ms(sp->stride, td->td_imagewidth), strip_height), sizeof(uint16_t)); /* add one more stride in case input ends mid-stride */ - tbuf_size = add_ms(tbuf_size, sizeof(uint16_t) * sp->stride); + tbuf_size = + add_ms(tbuf_size, (tmsize_t)(sizeof(uint16_t) * (size_t)sp->stride)); if (tbuf_size == 0) return (0); /* TODO: this is an error return without error report through TIFFErrorExt */ @@ -838,18 +846,21 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) PixarLogState *sp = PixarLogDecoderState(tif); tmsize_t i; tmsize_t nsamples; - int llen; + tmsize_t llen; uint16_t *up; switch (sp->user_datafmt) { case PIXARLOGDATAFMT_FLOAT: - nsamples = occ / sizeof(float); /* XXX float == 32 bits */ + nsamples = (tmsize_t)((uint64_t)occ / + sizeof(float)); /* XXX float == 32 bits */ break; case PIXARLOGDATAFMT_16BIT: case PIXARLOGDATAFMT_12BITPICIO: case PIXARLOGDATAFMT_11BITLOG: - nsamples = occ / sizeof(uint16_t); /* XXX uint16_t == 16 bits */ + nsamples = + (tmsize_t)((uint64_t)occ / + sizeof(uint16_t)); /* XXX uint16_t == 16 bits */ break; case PIXARLOGDATAFMT_8BIT: case PIXARLOGDATAFMT_8BITABGR: @@ -863,7 +874,55 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - llen = sp->stride * td->td_imagewidth; + llen = (tmsize_t)sp->stride * td->td_imagewidth; + + /* Fix: ABGR with stride=3 expands 3 samples to 4 output bytes per pixel */ + if (sp->user_datafmt == PIXARLOGDATAFMT_8BITABGR && sp->stride == 3) + { + tmsize_t required = (tmsize_t)td->td_imagewidth * 4; + tmsize_t max_rows; + tmsize_t max_nsamples; + + /* + * Ensure at least one expanded output row fits. + */ + if (occ < required) + { + TIFFErrorExtR(tif, module, + "Output buffer too small for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + + /* + * The caller-provided output buffer size must represent a whole + * number of expanded ABGR scanlines. + */ + if (occ % required) + { + TIFFErrorExtR( + tif, module, + "Fractional scanline not supported for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + + /* + * PixarLogDecode() may process multiple rows per call + * (e.g. strip decoding). Limit nsamples so the total + * output written by the loop below never exceeds occ. + */ + max_rows = occ / required; + max_nsamples = max_rows * llen; + + if (nsamples > max_nsamples) + { + TIFFErrorExtR(tif, module, + "Output buffer too small for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + } (void)s; assert(sp != NULL); @@ -876,8 +935,8 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_out = (uInt)(nsamples * sizeof(uint16_t)); - if (sp->stream.avail_out != nsamples * sizeof(uint16_t)) + sp->stream.avail_out = (uInt)((unsigned long)nsamples * sizeof(uint16_t)); + if (sp->stream.avail_out != (unsigned long)nsamples * sizeof(uint16_t)) { TIFFErrorExtR(tif, module, "ZLib cannot deal with buffers this size"); memset(op, 0, (size_t)occ); @@ -899,9 +958,10 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } if (state == Z_DATA_ERROR) { - TIFFErrorExtR( - tif, module, "Decoding error at scanline %" PRIu32 ", %s", - tif->tif_row, sp->stream.msg ? sp->stream.msg : "(null)"); + TIFFErrorExtR(tif, module, + "Decoding error at scanline %" PRIu32 ", %s", + tif->tif_dir.td_row, + sp->stream.msg ? sp->stream.msg : "(null)"); memset(op, 0, (size_t)occ); return (0); } @@ -920,7 +980,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %u bytes)", - tif->tif_row, sp->stream.avail_out); + tif->tif_dir.td_row, sp->stream.avail_out); memset(op, 0, (size_t)occ); return (0); } @@ -941,7 +1001,8 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (nsamples % llen) { TIFFWarningExtR(tif, module, - "stride %d is not a multiple of sample count, " + "stride %" TIFF_SSIZE_FORMAT + " is not a multiple of sample count, " "%" TIFF_SSIZE_FORMAT ", data truncated.", llen, nsamples); nsamples -= nsamples % llen; @@ -954,31 +1015,37 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) case PIXARLOGDATAFMT_FLOAT: horizontalAccumulateF(up, llen, sp->stride, (float *)op, sp->ToLinearF); - op += llen * sizeof(float); + op += (unsigned long)llen * sizeof(float); break; case PIXARLOGDATAFMT_16BIT: horizontalAccumulate16(up, llen, sp->stride, (uint16_t *)op, sp->ToLinear16); - op += llen * sizeof(uint16_t); + op += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_12BITPICIO: horizontalAccumulate12(up, llen, sp->stride, (int16_t *)op, sp->ToLinearF); - op += llen * sizeof(int16_t); + op += (unsigned long)llen * sizeof(int16_t); break; case PIXARLOGDATAFMT_11BITLOG: horizontalAccumulate11(up, llen, sp->stride, (uint16_t *)op); - op += llen * sizeof(uint16_t); + op += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_8BIT: horizontalAccumulate8(up, llen, sp->stride, (unsigned char *)op, sp->ToLinear8); - op += llen * sizeof(unsigned char); + op += (unsigned long)llen * sizeof(unsigned char); break; case PIXARLOGDATAFMT_8BITABGR: horizontalAccumulate8abgr(up, llen, sp->stride, (unsigned char *)op, sp->ToLinear8); - op += llen * sizeof(unsigned char); + + /* For stride == 3 (RGB), horizontalAccumulate8abgr expands to 4 + * bytes/pixel (ABGR) */ + if (sp->stride == 3) + op += (unsigned long)td->td_imagewidth * 4; + else + op += (unsigned long)llen * sizeof(unsigned char); break; default: TIFFErrorExtR(tif, module, "Unsupported bits/sample: %" PRIu16, @@ -1063,17 +1130,18 @@ static int PixarLogPreEncode(TIFF *tif, uint16_t s) return (deflateReset(&sp->stream) == Z_OK); } -static void horizontalDifferenceF(float *ip, int n, int stride, uint16_t *wp, - uint16_t *FromLT2) +static void horizontalDifferenceF(float *ip, tmsize_t n, int stride, + uint16_t *wp, uint16_t *FromLT2) { int32_t r1, g1, b1, a1, r2, g2, b2, a2, mask; float fltsize = Fltsize; #define CLAMP(v) \ - ((v < (float)0.) ? 0 \ - : (v < (float)2.) ? FromLT2[(int)(v * fltsize)] \ - : (v > (float)24.2) ? 2047 \ - : LogK1 * log(v * LogK2) + 0.5) + ((v < (float)0.) ? 0 \ + : (v < (float)2.) ? FromLT2[(int)(v * fltsize)] \ + : (v > (float)24.2) \ + ? 2047 \ + : (double)LogK1 * log((double)v * (double)LogK2) + 0.5) mask = CODE_MASK; if (n >= stride) @@ -1143,10 +1211,10 @@ static void horizontalDifferenceF(float *ip, int n, int stride, uint16_t *wp, } } -static void horizontalDifference16(unsigned short *ip, int n, int stride, +static void horizontalDifference16(unsigned short *ip, tmsize_t n, int stride, unsigned short *wp, uint16_t *From14) { - register int r1, g1, b1, a1, r2, g2, b2, a2, mask; + int r1, g1, b1, a1, r2, g2, b2, a2, mask; /* assumption is unsigned pixel values */ #undef CLAMP @@ -1219,10 +1287,10 @@ static void horizontalDifference16(unsigned short *ip, int n, int stride, } } -static void horizontalDifference8(unsigned char *ip, int n, int stride, +static void horizontalDifference8(unsigned char *ip, tmsize_t n, int stride, unsigned short *wp, uint16_t *From8) { - register int r1, g1, b1, a1, r2, g2, b2, a2, mask; + int r1, g1, b1, a1, r2, g2, b2, a2, mask; #undef CLAMP #define CLAMP(v) (From8[(v)]) @@ -1304,7 +1372,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) PixarLogState *sp = PixarLogEncoderState(tif); tmsize_t i; tmsize_t n; - int llen; + tmsize_t llen; unsigned short *up; (void)s; @@ -1312,12 +1380,14 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) switch (sp->user_datafmt) { case PIXARLOGDATAFMT_FLOAT: - n = cc / sizeof(float); /* XXX float == 32 bits */ + n = (tmsize_t)((unsigned long)cc / + sizeof(float)); /* XXX float == 32 bits */ break; case PIXARLOGDATAFMT_16BIT: case PIXARLOGDATAFMT_12BITPICIO: case PIXARLOGDATAFMT_11BITLOG: - n = cc / sizeof(uint16_t); /* XXX uint16_t == 16 bits */ + n = (tmsize_t)((unsigned long)cc / + sizeof(uint16_t)); /* XXX uint16_t == 16 bits */ break; case PIXARLOGDATAFMT_8BIT: case PIXARLOGDATAFMT_8BITABGR: @@ -1330,7 +1400,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - llen = sp->stride * td->td_imagewidth; + llen = (tmsize_t)sp->stride * td->td_imagewidth; /* Check against the number of elements (of size uint16_t) of sp->tbuf */ if (n > ((tmsize_t)td->td_rowsperstrip * llen)) { @@ -1345,17 +1415,17 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) case PIXARLOGDATAFMT_FLOAT: horizontalDifferenceF((float *)bp, llen, sp->stride, up, sp->FromLT2); - bp += llen * sizeof(float); + bp += (unsigned long)llen * sizeof(float); break; case PIXARLOGDATAFMT_16BIT: horizontalDifference16((uint16_t *)bp, llen, sp->stride, up, sp->From14); - bp += llen * sizeof(uint16_t); + bp += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_8BIT: horizontalDifference8((unsigned char *)bp, llen, sp->stride, up, sp->From8); - bp += llen * sizeof(unsigned char); + bp += (unsigned long)llen * sizeof(unsigned char); break; default: TIFFErrorExtR(tif, module, @@ -1370,8 +1440,8 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_in = (uInt)(n * sizeof(uint16_t)); - if ((sp->stream.avail_in / sizeof(uint16_t)) != (uInt)n) + sp->stream.avail_in = (uInt)((unsigned long)n * sizeof(uint16_t)); + if ((sp->stream.avail_in / sizeof(uint16_t)) != (unsigned long)n) { TIFFErrorExtR(tif, module, "ZLib cannot deal with buffers this size"); return (0); @@ -1562,13 +1632,15 @@ static int PixarLogVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFSetField(tif, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_IEEEFP); break; + default: + break; } /* * Must recalculate sizes should bits/sample change. */ - tif->tif_tilesize = + tif->tif_dir.td_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); - tif->tif_scanlinesize = TIFFScanlineSize(tif); + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); result = 1; /* NB: pseudo tag */ break; default: @@ -1601,6 +1673,16 @@ static const TIFFField pixarlogFields[] = { {TIFFTAG_PIXARLOGQUALITY, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, FIELD_PSEUDO, FALSE, FALSE, "", NULL}}; +static uint64_t PixarLogGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* cf https://zlib.net/zlib_tech.html */ + const uint64_t MAX_DEFLATE_RATIO = 1032; + + /* security margin as I don't understand what this codec does */ + return MAX_DEFLATE_RATIO * (uint64_t)4; +} + int TIFFInitPixarLog(TIFF *tif, int scheme) { static const char module[] = "TIFFInitPixarLog"; @@ -1648,6 +1730,7 @@ int TIFFInitPixarLog(TIFF *tif, int scheme) tif->tif_encodetile = PixarLogEncode; tif->tif_close = PixarLogClose; tif->tif_cleanup = PixarLogCleanup; + tif->tif_getmaxcompressionratio = PixarLogGetMaxCompressionRatio; /* Override SetField so we can handle our private pseudo-tag */ sp->vgetparent = tif->tif_tagmethods.vgetfield; diff --git a/contrib/libs/libtiff/tif_predict.c b/contrib/libs/libtiff/tif_predict.c index 7a6fc4af8b7..18b90c211e1 100644 --- a/contrib/libs/libtiff/tif_predict.c +++ b/contrib/libs/libtiff/tif_predict.c @@ -30,7 +30,7 @@ #include "tif_predict.h" #include "tiffiop.h" -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) #include <emmintrin.h> #endif @@ -151,6 +151,8 @@ static int PredictorSetupDecode(TIFF *tif) case 64: sp->decodepfunc = horAcc64; break; + default: + break; } /* * Override default decoding method with one that does the @@ -247,6 +249,8 @@ static int PredictorSetupEncode(TIFF *tif) case 64: sp->encodepfunc = horDiff64; break; + default: + break; } /* * Override default encoding method with one that does the @@ -590,7 +594,7 @@ static int fpAcc(TIFF *tif, uint8_t *cp0, tmsize_t cc) cp = (uint8_t *)cp0; count = 0; -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) if (bps == 4) { /* Optimization of general case */ @@ -972,7 +976,7 @@ static int PredictorEncodeRow(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) (int64_t)cc); return 0; } - memcpy(working_copy, bp, cc); + memcpy(working_copy, bp, (size_t)cc); if (!(*sp->encodepfunc)(tif, working_copy, cc)) { @@ -1010,7 +1014,7 @@ static int PredictorEncodeTile(TIFF *tif, uint8_t *bp0, tmsize_t cc0, (int64_t)cc0); return 0; } - memcpy(working_copy, bp0, cc0); + memcpy(working_copy, bp0, (size_t)cc0); bp = working_copy; rowsize = sp->rowsize; @@ -1098,8 +1102,10 @@ static void PredictorPrintDir(TIFF *tif, FILE *fd, long flags) case 3: fprintf(fd, "floating point predictor "); break; + default: + break; } - fprintf(fd, "%d (0x%x)\n", sp->predictor, sp->predictor); + fprintf(fd, "%d (0x%x)\n", sp->predictor, (unsigned)sp->predictor); } if (sp->printdir) (*sp->printdir)(tif, fd, flags); diff --git a/contrib/libs/libtiff/tif_print.c b/contrib/libs/libtiff/tif_print.c index addc03ab078..1ea450db95b 100644 --- a/contrib/libs/libtiff/tif_print.c +++ b/contrib/libs/libtiff/tif_print.c @@ -118,10 +118,10 @@ static void _TIFFPrintField(FILE *fd, const TIFFField *fip, if (tv_size == 8) fprintf(fd, "%lf", ((double *)raw_data)[j]); else - fprintf(fd, "%f", ((float *)raw_data)[j]); + fprintf(fd, "%f", (double)((float *)raw_data)[j]); } else if (fip->field_type == TIFF_FLOAT) - fprintf(fd, "%f", ((float *)raw_data)[j]); + fprintf(fd, "%f", (double)((float *)raw_data)[j]); else if (fip->field_type == TIFF_LONG8) fprintf(fd, "%" PRIu64, ((uint64_t *)raw_data)[j]); else if (fip->field_type == TIFF_SLONG8) @@ -193,8 +193,9 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, case TIFFTAG_WHITEPOINT: if (value_count == 2 && fip->field_type == TIFF_RATIONAL) { - fprintf(fd, " White Point: %g-%g\n", ((float *)raw_data)[0], - ((float *)raw_data)[1]); + fprintf(fd, " White Point: %g-%g\n", + (double)((float *)raw_data)[0], + (double)((float *)raw_data)[1]); return 1; } return 0; @@ -232,6 +233,9 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, return 1; } return 0; + + default: + break; } return 0; @@ -244,8 +248,7 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) { TIFFDirectory *td = &tif->tif_dir; - char *sep; - long l, n; + const char *sep; fprintf(fd, "TIFF Directory at offset 0x%" PRIx64 " (%" PRIu64 ")\n", tif->tif_diroff, tif->tif_diroff); @@ -286,8 +289,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) } if (TIFFFieldSet(tif, FIELD_RESOLUTION)) { - fprintf(fd, " Resolution: %g, %g", td->td_xresolution, - td->td_yresolution); + fprintf(fd, " Resolution: %g, %g", (double)td->td_xresolution, + (double)td->td_yresolution); if (TIFFFieldSet(tif, FIELD_RESOLUTIONUNIT)) { switch (td->td_resolutionunit) @@ -310,7 +313,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) fprintf(fd, "\n"); } if (TIFFFieldSet(tif, FIELD_POSITION)) - fprintf(fd, " Position: %g, %g\n", td->td_xposition, td->td_yposition); + fprintf(fd, " Position: %g, %g\n", (double)td->td_xposition, + (double)td->td_yposition); if (TIFFFieldSet(tif, FIELD_BITSPERSAMPLE)) fprintf(fd, " Bits/Sample: %" PRIu16 "\n", td->td_bitspersample); if (TIFFFieldSet(tif, FIELD_SAMPLEFORMAT)) @@ -374,7 +378,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) } } } - if (TIFFFieldSet(tif, FIELD_EXTRASAMPLES) && td->td_extrasamples) + if (TIFFFieldSet(tif, FIELD_EXTRASAMPLES) && td->td_extrasamples && + td->td_sampleinfo) { uint16_t i; fprintf(fd, " Extra Samples: %" PRIu16 "<", td->td_extrasamples); @@ -412,7 +417,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) i > 0 && cp < td->td_inknames + td->td_inknameslen; cp = strchr(cp, '\0') + 1, i--) { - size_t max_chars = td->td_inknameslen - (cp - td->td_inknames); + size_t max_chars = + (size_t)(td->td_inknameslen - (cp - td->td_inknames)); fputs(sep, fd); _TIFFprintAsciiBounded(fd, cp, max_chars); sep = ", "; @@ -548,17 +554,25 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) if (TIFFFieldSet(tif, FIELD_PAGENUMBER)) fprintf(fd, " Page Number: %" PRIu16 "-%" PRIu16 "\n", td->td_pagenumber[0], td->td_pagenumber[1]); - if (TIFFFieldSet(tif, FIELD_COLORMAP)) + if (TIFFFieldSet(tif, FIELD_COLORMAP) && td->td_colormap[0] && + td->td_colormap[1] && td->td_colormap[2]) { fprintf(fd, " Color Map: "); if (flags & TIFFPRINT_COLORMAP) { fprintf(fd, "\n"); - n = 1L << td->td_bitspersample; - for (l = 0; l < n; l++) - fprintf(fd, " %5ld: %5" PRIu16 " %5" PRIu16 " %5" PRIu16 "\n", - l, td->td_colormap[0][l], td->td_colormap[1][l], - td->td_colormap[2][l]); + if (td->td_bitspersample >= 64) + fprintf(fd, " (BitsPerSample too large to print safely)\n"); + else + { + uint64_t n = 1ULL << td->td_bitspersample; + for (uint64_t l = 0u; l < n; l++) + fprintf(fd, + " %5" PRIu64 ": %5" PRIu16 " %5" PRIu16 + " %5" PRIu16 "\n", + l, td->td_colormap[0][l], td->td_colormap[1][l], + td->td_colormap[2][l]); + } } else fprintf(fd, "(present)\n"); @@ -569,26 +583,37 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) fprintf(fd, " Reference Black/White:\n"); for (i = 0; i < 3; i++) fprintf(fd, " %2d: %5g %5g\n", i, - td->td_refblackwhite[2 * i + 0], - td->td_refblackwhite[2 * i + 1]); + (double)td->td_refblackwhite[2 * i + 0], + (double)td->td_refblackwhite[2 * i + 1]); } - if (TIFFFieldSet(tif, FIELD_TRANSFERFUNCTION)) + if (TIFFFieldSet(tif, FIELD_TRANSFERFUNCTION) && + td->td_transferfunction[0] && + ((td->td_samplesperpixel - td->td_extrasamples > 1 && + td->td_transferfunction[1] && td->td_transferfunction[2]) || + td->td_samplesperpixel - td->td_extrasamples <= 1)) { fprintf(fd, " Transfer Function: "); if (flags & TIFFPRINT_CURVES) { fprintf(fd, "\n"); - n = 1L << td->td_bitspersample; - for (l = 0; l < n; l++) + if (td->td_bitspersample >= 64) + fprintf(fd, " (BitsPerSample too large to print safely)\n"); + else { - uint16_t i; - fprintf(fd, " %2ld: %5" PRIu16, l, - td->td_transferfunction[0][l]); - for (i = 1; - i < td->td_samplesperpixel - td->td_extrasamples && i < 3; - i++) - fprintf(fd, " %5" PRIu16, td->td_transferfunction[i][l]); - fputc('\n', fd); + uint64_t n = 1ULL << td->td_bitspersample; + for (uint64_t l = 0; l < n; l++) + { + uint16_t i; + fprintf(fd, " %2" PRIu64 ": %5" PRIu16, l, + td->td_transferfunction[0][l]); + for (i = 1; + i < td->td_samplesperpixel - td->td_extrasamples && + i < 3; + i++) + fprintf(fd, " %5" PRIu16, + td->td_transferfunction[i][l]); + fputc('\n', fd); + } } } else @@ -655,7 +680,7 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) else if (fip->field_readcount == TIFF_SPP) value_count = td->td_samplesperpixel; else - value_count = fip->field_readcount; + value_count = (uint32_t)fip->field_readcount; if (fip->field_tag == TIFFTAG_DOTRANGE && strcmp(fip->field_name, "DotRange") == 0) { @@ -680,7 +705,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) * "set_get_field_type" to determine internal storage size. */ int tv_size = TIFFFieldSetGetSize(fip); - raw_data = _TIFFmallocExt(tif, tv_size * value_count); + raw_data = _TIFFCheckMalloc(tif, value_count, tv_size, + "for tag data"); mem_alloc = 1; if (TIFFGetField(tif, tag, raw_data) != 1) { @@ -744,7 +770,7 @@ static void _TIFFprintAsciiBounded(FILE *fd, const char *cp, size_t max_chars) if (*tp) fprintf(fd, "\\%c", *tp); else - fprintf(fd, "\\%03o", *cp & 0xff); + fprintf(fd, "\\%03o", (unsigned int)(*cp & 0xff)); } } diff --git a/contrib/libs/libtiff/tif_read.c b/contrib/libs/libtiff/tif_read.c index a2bb3046237..d1a4e892168 100644 --- a/contrib/libs/libtiff/tif_read.c +++ b/contrib/libs/libtiff/tif_read.c @@ -27,6 +27,7 @@ * Scanline-oriented Read Support */ #include "tiffiop.h" +#include <limits.h> #include <stdio.h> int TIFFFillStrip(TIFF *tif, uint32_t strip); @@ -39,9 +40,6 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, tmsize_t size, const char *module); -#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ -#define NOTILE ((uint32_t)(-1)) /* undefined state */ - #define INITIAL_THRESHOLD (1024 * 1024) #define THRESHOLD_MULTIPLIER 10 #define MAX_THRESHOLD \ @@ -99,7 +97,9 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, uint8_t *new_rawdata; assert((tif->tif_flags & TIFF_MYBUFFER) != 0); tif->tif_rawdatasize = (tmsize_t)TIFFroundup_64( - (uint64_t)already_read + to_read + rawdata_offset, 1024); + (uint64_t)already_read + (uint64_t)to_read + + (uint64_t)rawdata_offset, + 1024); if (tif->tif_rawdatasize == 0) { TIFFErrorExtR(tif, module, "Invalid buffer size"); @@ -111,7 +111,7 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, { TIFFErrorExtR(tif, module, "No space for data buffer at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); _TIFFfreeExt(tif, tif->tif_rawdata); tif->tif_rawdata = 0; tif->tif_rawdatasize = 0; @@ -127,18 +127,22 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, bytes_read = TIFFReadFile( tif, tif->tif_rawdata + rawdata_offset + already_read, to_read); + if (bytes_read < 0) + /* Treat read errors as short reads before updating offsets. */ + bytes_read = 0; already_read += bytes_read; if (bytes_read != to_read) { - memset(tif->tif_rawdata + rawdata_offset + already_read, 0, - tif->tif_rawdatasize - rawdata_offset - already_read); + memset( + tif->tif_rawdata + rawdata_offset + already_read, 0, + (size_t)(tif->tif_rawdatasize - rawdata_offset - already_read)); if (is_strip) { TIFFErrorExtR(tif, module, "Read error at scanline %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, " "expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, already_read, size); + tif->tif_dir.td_row, already_read, size); } else { @@ -147,8 +151,8 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, ", tile %" PRIu32 "; " "got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT "", - tif->tif_row, tif->tif_col, strip_or_tile, - already_read, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, + strip_or_tile, already_read, size); } return 0; } @@ -160,7 +164,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, int restart) { static const char module[] = "TIFFFillStripPartial"; - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; tmsize_t unused_data; uint64_t read_offset; tmsize_t to_read; @@ -185,7 +189,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, { assert(restart); - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR(tif, module, @@ -214,20 +218,30 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, if (unused_data > 0) { assert((tif->tif_flags & TIFF_BUFFERMMAP) == 0); - memmove(tif->tif_rawdata, tif->tif_rawcp, unused_data); + memmove(tif->tif_rawdata, tif->tif_rawcp, (size_t)unused_data); } /* ** Seek to the point in the file where more data should be read. */ - read_offset = TIFFGetStrileOffset(tif, strip) + tif->tif_rawdataoff + - tif->tif_rawdataloaded; + read_offset = TIFFGetStrileOffset(tif, (uint32_t)strip); + if (read_offset > UINT64_MAX - (uint64_t)tif->tif_rawdataoff || + read_offset + (uint64_t)tif->tif_rawdataoff > + UINT64_MAX - (uint64_t)tif->tif_rawdataloaded) + { + TIFFErrorExtR(tif, module, + "Seek error at scanline %" PRIu32 ", strip %d", + tif->tif_dir.td_row, strip); + return 0; + } + read_offset += + (uint64_t)tif->tif_rawdataoff + (uint64_t)tif->tif_rawdataloaded; if (!SeekOK(tif, read_offset)) { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %d", - tif->tif_row, strip); + tif->tif_dir.td_row, strip); return 0; } @@ -238,11 +252,13 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, to_read = read_ahead_mod - unused_data; else to_read = tif->tif_rawdatasize - unused_data; - if ((uint64_t)to_read > TIFFGetStrileByteCount(tif, strip) - - tif->tif_rawdataoff - tif->tif_rawdataloaded) + if ((uint64_t)to_read > TIFFGetStrileByteCount(tif, (uint32_t)strip) - + (uint64_t)tif->tif_rawdataoff - + (uint64_t)tif->tif_rawdataloaded) { - to_read = (tmsize_t)TIFFGetStrileByteCount(tif, strip) - - tif->tif_rawdataoff - tif->tif_rawdataloaded; + to_read = (tmsize_t)(TIFFGetStrileByteCount(tif, (uint32_t)strip) - + (uint64_t)tif->tif_rawdataoff - + (uint64_t)tif->tif_rawdataloaded); } assert((tif->tif_flags & TIFF_BUFFERMMAP) == 0); @@ -281,16 +297,17 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, /* For JPEG, if there are multiple scans (can generally be known */ /* with the read_ahead used), we need to read the whole strip */ if (tif->tif_dir.td_compression == COMPRESSION_JPEG && - (uint64_t)tif->tif_rawcc < TIFFGetStrileByteCount(tif, strip)) + (uint64_t)tif->tif_rawcc < + TIFFGetStrileByteCount(tif, (uint32_t)strip)) { if (TIFFJPEGIsFullStripRequired(tif)) { - return TIFFFillStrip(tif, strip); + return TIFFFillStrip(tif, (uint32_t)strip); } } #endif - return TIFFStartStrip(tif, strip); + return TIFFStartStrip(tif, (uint32_t)strip); } else { @@ -308,7 +325,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, */ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; uint32_t strip; int whole_strip; tmsize_t read_ahead = 0; @@ -323,8 +340,16 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) td->td_imagelength); return (0); } + if (td->td_rowsperstrip == 0) + { + TIFFErrorExtR(tif, tif->tif_name, + "Cannot compute strip: RowsPerStrip is zero"); + return (0); + } if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, tif->tif_name, @@ -332,16 +357,26 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) sample, td->td_samplesperpixel); return (0); } - strip = (uint32_t)sample * td->td_stripsperimage + - row / td->td_rowsperstrip; + sample_offset = + _TIFFMultiply64(tif, sample, td->td_stripsperimage, "TIFFSeek"); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (0); + strip64 = _TIFFAdd64(tif, sample_offset, row / td->td_rowsperstrip, + "TIFFSeek"); + if (strip64 == 0 && + (sample_offset != 0 || (row / td->td_rowsperstrip) != 0)) + return (0); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, "TIFFSeek"); + if (strip == 0 && strip64 != 0) + return (0); } else strip = row / td->td_rowsperstrip; - /* - * Do we want to treat this strip as one whole chunk or - * read it a few lines at a time? - */ + /* + * Do we want to treat this strip as one whole chunk or + * read it a few lines at a time? + */ #if defined(CHUNKY_STRIP_READ_SUPPORT) whole_strip = TIFFGetStrileByteCount(tif, strip) < 10 || isMapped(tif); if (td->td_compression == COMPRESSION_LERC || @@ -359,14 +394,14 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) /* is some constant value, for example for JPEG tables */ /* coverity[dead_error_line:SUPPRESS] */ - if (tif->tif_scanlinesize < TIFF_TMSIZE_T_MAX / 16 && - tif->tif_scanlinesize * 16 < TIFF_TMSIZE_T_MAX - 5000) + if (tif->tif_dir.td_scanlinesize < TIFF_TMSIZE_T_MAX / 16 && + tif->tif_dir.td_scanlinesize * 16 < TIFF_TMSIZE_T_MAX - 5000) { - read_ahead = tif->tif_scanlinesize * 16 + 5000; + read_ahead = tif->tif_dir.td_scanlinesize * 16 + 5000; } else { - read_ahead = tif->tif_scanlinesize; + read_ahead = tif->tif_dir.td_scanlinesize; } } #else @@ -377,7 +412,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) * If we haven't loaded this strip, do so now, possibly * only reading the first part. */ - if (strip != tif->tif_curstrip) + if (strip != tif->tif_dir.td_curstrip) { /* different strip, refill */ if (whole_strip) @@ -412,7 +447,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) } #endif - if (row < tif->tif_row) + if (row < tif->tif_dir.td_row) { /* * Moving backwards within the same strip: backup @@ -425,7 +460,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) if (tif->tif_rawdataoff != 0) { - if (!TIFFFillStripPartial(tif, strip, read_ahead, 1)) + if (!TIFFFillStripPartial(tif, (int)strip, read_ahead, 1)) return 0; } else @@ -435,7 +470,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) } } - if (row != tif->tif_row) + if (row != tif->tif_dir.td_row) { /* * Seek forward to the desired row. @@ -443,9 +478,9 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) /* TODO: Will this really work with partial buffers? */ - if (!(*tif->tif_seek)(tif, row - tif->tif_row)) + if (!(*tif->tif_seek)(tif, row - tif->tif_dir.td_row)) return (0); - tif->tif_row = row; + tif->tif_dir.td_row = row; } return (1); @@ -462,20 +497,21 @@ int TIFFReadScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* * Decompress desired row into user buffer. */ - e = (*tif->tif_decoderow)(tif, (uint8_t *)buf, tif->tif_scanlinesize, - sample); + e = (*tif->tif_decoderow)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize, sample); /* we are now poised at the beginning of the next row */ - tif->tif_row = row + 1; + tif->tif_dir.td_row = row + 1; if (e) - (*tif->tif_postdecode)(tif, (uint8_t *)buf, tif->tif_scanlinesize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize); } else { /* See TIFFReadEncodedStrip comment regarding TIFFTAG_FAXFILLFUNC. */ if (buf) - memset(buf, 0, (size_t)tif->tif_scanlinesize); + memset(buf, 0, (size_t)tif->tif_dir.td_scanlinesize); } return (e > 0 ? 1 : -1); } @@ -553,9 +589,9 @@ tmsize_t TIFFReadEncodedStrip(TIFF *tif, uint32_t strip, void *buf, if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) - TIFFReverseBits(buf, stripsize); + TIFFReverseBits((uint8_t *)buf, stripsize); - (*tif->tif_postdecode)(tif, buf, stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, stripsize); return (stripsize); } @@ -569,9 +605,9 @@ tmsize_t TIFFReadEncodedStrip(TIFF *tif, uint32_t strip, void *buf, memset(buf, 0, (size_t)stripsize); return ((tmsize_t)(-1)); } - if ((*tif->tif_decodestrip)(tif, buf, stripsize, plane) <= 0) + if ((*tif->tif_decodestrip)(tif, (uint8_t *)buf, stripsize, plane) <= 0) return ((tmsize_t)(-1)); - (*tif->tif_postdecode)(tif, buf, stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, stripsize); return (stripsize); } @@ -603,17 +639,39 @@ tmsize_t _TIFFReadEncodedStripAndAllocBuffer(TIFF *tif, uint32_t strip, if (!TIFFFillStrip(tif, strip)) return ((tmsize_t)(-1)); - *buf = _TIFFmallocExt(tif, bufsizetoalloc); + /* Sanity checks to avoid excessive memory allocation */ + /* Max compression ratio experimentally determined. Might be fragile... + * Only apply this heuristics to situations where the memory allocation + * would be big, to avoid breaking nominal use cases. + */ + if (bufsizetoalloc > 100 * 1024 * 1024) + { + const uint64_t maxCompressionRatio = TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + (uint64_t)tif->tif_rawdatasize < + (uint64_t)this_stripsize / maxCompressionRatio) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Likely invalid strip byte count for strip %u. " + "Uncompressed strip size is %" PRIu64 ", " + "compressed one is %" PRIu64, + strip, (uint64_t)this_stripsize, + (uint64_t)tif->tif_rawdatasize); + return ((tmsize_t)(-1)); + } + } + + *buf = _TIFFcallocExt(tif, 1, bufsizetoalloc); if (*buf == NULL) { TIFFErrorExtR(tif, TIFFFileName(tif), "No space for strip buffer"); return ((tmsize_t)(-1)); } - _TIFFmemset(*buf, 0, bufsizetoalloc); - if ((*tif->tif_decodestrip)(tif, *buf, this_stripsize, plane) <= 0) + if ((*tif->tif_decodestrip)(tif, (uint8_t *)*buf, this_stripsize, plane) <= + 0) return ((tmsize_t)(-1)); - (*tif->tif_postdecode)(tif, *buf, this_stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)*buf, this_stripsize); return (this_stripsize); } @@ -629,7 +687,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %" PRIu32, - tif->tif_row, strip); + tif->tif_dir.td_row, strip); return ((tmsize_t)(-1)); } cc = TIFFReadFile(tif, buf, size); @@ -639,7 +697,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, "Read error at scanline %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, cc, size); + tif->tif_dir.td_row, cc, size); return ((tmsize_t)(-1)); } } @@ -670,7 +728,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, "Read error at scanline %" PRIu32 ", strip %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, strip, n, size); + tif->tif_dir.td_row, strip, n, size); return ((tmsize_t)(-1)); } _TIFFmemcpy(buf, tif->tif_base + ma, size); @@ -691,14 +749,14 @@ static tmsize_t TIFFReadRawStripOrTile2(TIFF *tif, uint32_t strip_or_tile, { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %" PRIu32, - tif->tif_row, strip_or_tile); + tif->tif_dir.td_row, strip_or_tile); } else { - TIFFErrorExtR(tif, module, - "Seek error at row %" PRIu32 ", col %" PRIu32 - ", tile %" PRIu32, - tif->tif_row, tif->tif_col, strip_or_tile); + TIFFErrorExtR( + tif, module, + "Seek error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32, + tif->tif_dir.td_row, tif->tif_dir.td_col, strip_or_tile); } return ((tmsize_t)(-1)); } @@ -774,22 +832,50 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } /* To avoid excessive memory allocations: */ - /* Byte count should normally not be larger than a number of */ - /* times the uncompressed size plus some margin */ - if (bytecount > 1024 * 1024) + const tmsize_t stripsize = TIFFStripSize(tif); + if (stripsize > 0) { - /* 10 and 4096 are just values that could be adjusted. */ - /* Hopefully they are safe enough for all codecs */ - tmsize_t stripsize = TIFFStripSize(tif); - if (stripsize != 0 && (bytecount - 4096) / 10 > (uint64_t)stripsize) + if (bytecount > 1024 * 1024 && + (bytecount - 4096) / 10 > (uint64_t)stripsize) { + /* Byte count should normally not be larger than a number of */ + /* times the uncompressed size plus some margin */ + /* 10 and 4096 are just values that could be adjusted. */ + /* Hopefully they are safe enough for all codecs */ + /* What happens next will depend on whether only the bytecount + */ + /* was corrupted to a large value but the strip/tile data is */ + /* fine. In that situation most codecs should work fine and */ + /* only used part of the tile/strip data. If the strip/tile */ + /* data is corrupted too, then codecs will later error out. */ uint64_t newbytecount = (uint64_t)stripsize * 10 + 4096; - TIFFErrorExtR(tif, module, - "Too large strip byte count %" PRIu64 - ", strip %" PRIu32 ". Limiting to %" PRIu64, - bytecount, strip, newbytecount); + TIFFWarningExtR(tif, module, + "Too large strip byte count %" PRIu64 + ", strip %" PRIu32 ". Limiting to %" PRIu64, + bytecount, strip, newbytecount); bytecount = newbytecount; } + else if (stripsize > 100 * 1024 * 1024) + { + /* Max compression ratio experimentally determined. Might be + * fragile... Only apply this heuristics to situations where the + * memory allocation would be big, to avoid breaking nominal use + * cases. + */ + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + bytecount < (uint64_t)stripsize / maxCompressionRatio) + { + TIFFErrorExtR( + tif, module, + "Likely invalid strip byte count for strip %u. " + "Uncompressed strip size is %" PRIu64 ", " + "compressed one is %" PRIu64, + strip, (uint64_t)stripsize, bytecount); + return 0; + } + } } if (isMapped(tif)) @@ -818,10 +904,10 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) "Read error on strip %" PRIu32 "; " "got %" PRIu64 " bytes, expected %" PRIu64, strip, - NoSanitizeSubUInt64(tif->tif_size, + NoSanitizeSubUInt64((uint64_t)tif->tif_size, TIFFGetStrileOffset(tif, strip)), bytecount); - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; return (0); } } @@ -878,7 +964,7 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } if (bytecountm > tif->tif_rawdatasize) { - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR( @@ -889,7 +975,7 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } if (tif->tif_flags & TIFF_BUFFERMMAP) { - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; tif->tif_rawdata = NULL; tif->tif_rawdatasize = 0; tif->tif_flags &= ~TIFF_BUFFERMMAP; @@ -954,7 +1040,7 @@ tmsize_t TIFFReadEncodedTile(TIFF *tif, uint32_t tile, void *buf, tmsize_t size) { static const char module[] = "TIFFReadEncodedTile"; TIFFDirectory *td = &tif->tif_dir; - tmsize_t tilesize = tif->tif_tilesize; + tmsize_t tilesize = tif->tif_dir.td_tilesize; if (!TIFFCheckRead(tif, 1)) return ((tmsize_t)(-1)); @@ -976,9 +1062,9 @@ tmsize_t TIFFReadEncodedTile(TIFF *tif, uint32_t tile, void *buf, tmsize_t size) if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) - TIFFReverseBits(buf, tilesize); + TIFFReverseBits((uint8_t *)buf, tilesize); - (*tif->tif_postdecode)(tif, buf, tilesize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, tilesize); return (tilesize); } @@ -1032,7 +1118,7 @@ tmsize_t _TIFFReadEncodedTileAndAllocBuffer(TIFF *tif, uint32_t tile, { static const char module[] = "_TIFFReadEncodedTileAndAllocBuffer"; TIFFDirectory *td = &tif->tif_dir; - tmsize_t tilesize = tif->tif_tilesize; + tmsize_t tilesize = tif->tif_dir.td_tilesize; if (*buf != NULL) { @@ -1072,25 +1158,22 @@ tmsize_t _TIFFReadEncodedTileAndAllocBuffer(TIFF *tif, uint32_t tile, * Only apply this heuristics to situations where the memory allocation * would be big, to avoid breaking nominal use cases. */ - const int maxCompressionRatio = - td->td_compression == COMPRESSION_ZSTD ? 33000 - : td->td_compression == COMPRESSION_JXL - ? - /* Evaluated on a 8000x8000 tile */ - 25000 * (td->td_planarconfig == PLANARCONFIG_CONTIG - ? td->td_samplesperpixel - : 1) - : td->td_compression == COMPRESSION_LZMA ? 7000 : 1000; - if (bufsizetoalloc > 100 * 1000 * 1000 && - tif->tif_rawdatasize < tilesize / maxCompressionRatio) + if (bufsizetoalloc > 100 * 1024 * 1024) { - TIFFErrorExtR(tif, TIFFFileName(tif), - "Likely invalid tile byte count for tile %u. " - "Uncompressed tile size is %" PRIu64 ", " - "compressed one is %" PRIu64, - tile, (uint64_t)tilesize, - (uint64_t)tif->tif_rawdatasize); - return ((tmsize_t)(-1)); + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + (uint64_t)tif->tif_rawdatasize < + (uint64_t)tilesize / maxCompressionRatio) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Likely invalid tile byte count for tile %u. " + "Uncompressed tile size is %" PRIu64 ", " + "compressed one is %" PRIu64, + tile, (uint64_t)tilesize, + (uint64_t)tif->tif_rawdatasize); + return ((tmsize_t)(-1)); + } } } @@ -1129,7 +1212,7 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, TIFFErrorExtR(tif, module, "Seek error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32, - tif->tif_row, tif->tif_col, tile); + tif->tif_dir.td_row, tif->tif_dir.td_col, tile); return ((tmsize_t)(-1)); } cc = TIFFReadFile(tif, buf, size); @@ -1139,7 +1222,7 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, "Read error at row %" PRIu32 ", col %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, tif->tif_col, cc, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, cc, size); return ((tmsize_t)(-1)); } } @@ -1162,7 +1245,8 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, "Read error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, tif->tif_col, tile, n, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, tile, n, + size); return ((tmsize_t)(-1)); } _TIFFmemcpy(buf, tif->tif_base + ma, size); @@ -1229,22 +1313,49 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } /* To avoid excessive memory allocations: */ - /* Byte count should normally not be larger than a number of */ - /* times the uncompressed size plus some margin */ - if (bytecount > 1024 * 1024) + const tmsize_t tilesize = TIFFTileSize(tif); + if (tilesize > 0) { - /* 10 and 4096 are just values that could be adjusted. */ - /* Hopefully they are safe enough for all codecs */ - tmsize_t stripsize = TIFFTileSize(tif); - if (stripsize != 0 && (bytecount - 4096) / 10 > (uint64_t)stripsize) + if (bytecount > 1024 * 1024 && + (bytecount - 4096) / 10 > (uint64_t)tilesize) { - uint64_t newbytecount = (uint64_t)stripsize * 10 + 4096; - TIFFErrorExtR(tif, module, - "Too large tile byte count %" PRIu64 - ", tile %" PRIu32 ". Limiting to %" PRIu64, - bytecount, tile, newbytecount); + /* Byte count should normally not be larger than a number of */ + /* times the uncompressed size plus some margin */ + /* 10 and 4096 are just values that could be adjusted. */ + /* Hopefully they are safe enough for all codecs */ + /* What happens next will depend on whether only the bytecount + */ + /* was corrupted to a large value but the strip/tile data is */ + /* fine. In that situation most codecs should work fine and */ + /* only used part of the tile/strip data. If the strip/tile */ + /* data is corrupted too, then codecs will later error out. */ + uint64_t newbytecount = (uint64_t)tilesize * 10 + 4096; + TIFFWarningExtR(tif, module, + "Too large tile byte count %" PRIu64 + ", tile %" PRIu32 ". Limiting to %" PRIu64, + bytecount, tile, newbytecount); bytecount = newbytecount; } + else if (tilesize > 100 * 1024 * 1024) + { + /* Max compression ratio experimentally determined. Might be + * fragile... Only apply this heuristics to situations where the + * memory allocation would be big, to avoid breaking nominal use + * cases. + */ + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + bytecount < (uint64_t)tilesize / maxCompressionRatio) + { + TIFFErrorExtR(tif, module, + "Likely invalid tile byte count for tile %u. " + "Uncompressed tile size is %" PRIu64 ", " + "compressed one is %" PRIu64, + tile, (uint64_t)tilesize, bytecount); + return 0; + } + } } if (isMapped(tif)) @@ -1262,7 +1373,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) TIFFGetStrileOffset(tif, tile) > (uint64_t)tif->tif_size - bytecount) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; return (0); } } @@ -1312,7 +1423,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } if (bytecountm > tif->tif_rawdatasize) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR(tif, module, @@ -1323,7 +1434,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } if (tif->tif_flags & TIFF_BUFFERMMAP) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; tif->tif_rawdata = NULL; tif->tif_rawdatasize = 0; tif->tif_flags &= ~TIFF_BUFFERMMAP; @@ -1410,7 +1521,7 @@ int TIFFReadBufferSetup(TIFF *tif, void *bp, tmsize_t size) { TIFFErrorExtR(tif, module, "No space for data buffer at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); tif->tif_rawdatasize = 0; return (0); } @@ -1431,8 +1542,13 @@ static int TIFFStartStrip(TIFF *tif, uint32_t strip) return (0); tif->tif_flags |= TIFF_CODERSETUP; } - tif->tif_curstrip = strip; - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + if (td->td_stripsperimage == 0) + { + TIFFErrorExtR(tif, "TIFFStartStrip", "Zero strips per image"); + return 0; + } + tif->tif_dir.td_curstrip = strip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; tif->tif_flags &= ~TIFF_BUF4WRITE; if (tif->tif_flags & TIFF_NOREADRAW) @@ -1456,7 +1572,7 @@ static int TIFFStartStrip(TIFF *tif, uint32_t strip) */ /* tif_curstrip, we'd call tif_decoderow() on a possibly invalid */ /* codec state. */ - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; return 0; } return 1; @@ -1478,7 +1594,7 @@ static int TIFFStartTile(TIFF *tif, uint32_t tile) return (0); tif->tif_flags |= TIFF_CODERSETUP; } - tif->tif_curtile = tile; + tif->tif_dir.td_curtile = tile; if (td->td_tilewidth == 0) { TIFFErrorExtR(tif, module, "Zero tilewidth"); @@ -1490,14 +1606,14 @@ static int TIFFStartTile(TIFF *tif, uint32_t tile) TIFFErrorExtR(tif, module, "Zero tiles"); return 0; } - tif->tif_row = (tile % howmany32) * td->td_tilelength; + tif->tif_dir.td_row = (tile % howmany32) * td->td_tilelength; howmany32 = TIFFhowmany_32(td->td_imagelength, td->td_tilelength); if (howmany32 == 0) { TIFFErrorExtR(tif, module, "Zero tiles"); return 0; } - tif->tif_col = (tile % howmany32) * td->td_tilewidth; + tif->tif_dir.td_col = (tile % howmany32) * td->td_tilewidth; tif->tif_flags &= ~TIFF_BUF4WRITE; if (tif->tif_flags & TIFF_NOREADRAW) { @@ -1568,14 +1684,14 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, tif->tif_flags &= ~TIFF_MYBUFFER; tif->tif_flags |= TIFF_BUFFERMMAP; tif->tif_rawdatasize = insize; - tif->tif_rawdata = inbuf; + tif->tif_rawdata = (uint8_t *)inbuf; tif->tif_rawdataoff = 0; tif->tif_rawdataloaded = insize; if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) { - TIFFReverseBits(inbuf, insize); + TIFFReverseBits((uint8_t *)inbuf, insize); } if (TIFFIsTiled(tif)) @@ -1632,13 +1748,13 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) { - TIFFReverseBits(inbuf, insize); + TIFFReverseBits((uint8_t *)inbuf, insize); } tif->tif_flags = (old_tif_flags & (TIFF_MYBUFFER | TIFF_BUFFERMMAP)) | (tif->tif_flags & ~(TIFF_MYBUFFER | TIFF_BUFFERMMAP)); tif->tif_rawdatasize = old_rawdatasize; - tif->tif_rawdata = old_rawdata; + tif->tif_rawdata = (uint8_t *)old_rawdata; tif->tif_rawdataoff = 0; tif->tif_rawdataloaded = 0; diff --git a/contrib/libs/libtiff/tif_strip.c b/contrib/libs/libtiff/tif_strip.c index f13c2b33fba..740de68a21e 100644 --- a/contrib/libs/libtiff/tif_strip.c +++ b/contrib/libs/libtiff/tif_strip.c @@ -47,6 +47,8 @@ uint32_t TIFFComputeStrip(TIFF *tif, uint32_t row, uint16_t sample) strip = row / td->td_rowsperstrip; if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, module, "%lu: Sample out of range, max %lu", @@ -54,7 +56,16 @@ uint32_t TIFFComputeStrip(TIFF *tif, uint32_t row, uint16_t sample) (unsigned long)td->td_samplesperpixel); return (0); } - strip += (uint32_t)sample * td->td_stripsperimage; + sample_offset = _TIFFMultiply64(tif, sample, td->td_stripsperimage, + "TIFFComputeStrip"); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (0); + strip64 = _TIFFAdd64(tif, sample_offset, strip, "TIFFComputeStrip"); + if (strip64 == 0 && (sample_offset != 0 || strip != 0)) + return (0); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, "TIFFComputeStrip"); + if (strip == 0 && strip64 != 0) + return (0); } return (strip); } @@ -83,14 +94,24 @@ uint32_t TIFFNumberOfStrips(TIFF *tif) } /* - * Compute the # bytes in a variable height, row-aligned strip. + * Compute the # bytes in a variable height, row-aligned strip if isStrip is + * TRUE, or in a tile if isStrip is FALSE */ -uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) +uint64_t _TIFFStrileSize64(TIFF *tif, uint32_t nrows, int isStrip) { - static const char module[] = "TIFFVStripSize64"; + static const char module[] = "_TIFFStrileSize64"; TIFFDirectory *td = &tif->tif_dir; - if (nrows == (uint32_t)(-1)) - nrows = td->td_imagelength; + if (isStrip) + { + if (nrows == (uint32_t)(-1)) + nrows = td->td_imagelength; + } + else + { + if (td->td_tilelength == 0 || td->td_tilewidth == 0 || + td->td_tiledepth == 0) + return (0); + } if ((td->td_planarconfig == PLANARCONFIG_CONTIG) && (td->td_photometric == PHOTOMETRIC_YCBCR) && (!isUpSampled(tif))) { @@ -125,9 +146,10 @@ uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) ycbcrsubsampling[0], ycbcrsubsampling[1]); return 0; } - samplingblock_samples = ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; - samplingblocks_hor = - TIFFhowmany_32(td->td_imagewidth, ycbcrsubsampling[0]); + samplingblock_samples = + (uint16_t)(ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2); + const uint32_t width = isStrip ? td->td_imagewidth : td->td_tilewidth; + samplingblocks_hor = TIFFhowmany_32(width, ycbcrsubsampling[0]); samplingblocks_ver = TIFFhowmany_32(nrows, ycbcrsubsampling[1]); samplingrow_samples = _TIFFMultiply64(tif, samplingblocks_hor, samplingblock_samples, module); @@ -137,8 +159,20 @@ uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) _TIFFMultiply64(tif, samplingrow_size, samplingblocks_ver, module)); } else - return (_TIFFMultiply64(tif, nrows, TIFFScanlineSize64(tif), module)); + return (_TIFFMultiply64(tif, nrows, + isStrip ? TIFFScanlineSize64(tif) + : TIFFTileRowSize64(tif), + module)); } + +/* + * Compute the # bytes in a variable height, row-aligned strip. + */ +uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) +{ + return _TIFFStrileSize64(tif, nrows, /* isStrip = */ TRUE); +} + tmsize_t TIFFVStripSize(TIFF *tif, uint32_t nrows) { static const char module[] = "TIFFVStripSize"; @@ -286,7 +320,7 @@ uint64_t TIFFScanlineSize64(TIFF *tif) return 0; } samplingblock_samples = - ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; + (uint16_t)(ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2); samplingblocks_hor = TIFFhowmany_32(td->td_imagewidth, ycbcrsubsampling[0]); samplingrow_samples = _TIFFMultiply64( diff --git a/contrib/libs/libtiff/tif_swab.c b/contrib/libs/libtiff/tif_swab.c index 827b025ce7a..50fe635fbc0 100644 --- a/contrib/libs/libtiff/tif_swab.c +++ b/contrib/libs/libtiff/tif_swab.c @@ -32,7 +32,7 @@ #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabShort) void TIFFSwabShort(uint16_t *wp) { - register unsigned char *cp = (unsigned char *)wp; + unsigned char *cp = (unsigned char *)wp; unsigned char t; assert(sizeof(uint16_t) == 2); t = cp[1]; @@ -44,7 +44,7 @@ void TIFFSwabShort(uint16_t *wp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabLong) void TIFFSwabLong(uint32_t *lp) { - register unsigned char *cp = (unsigned char *)lp; + unsigned char *cp = (unsigned char *)lp; unsigned char t; assert(sizeof(uint32_t) == 4); t = cp[3]; @@ -59,7 +59,7 @@ void TIFFSwabLong(uint32_t *lp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabLong8) void TIFFSwabLong8(uint64_t *lp) { - register unsigned char *cp = (unsigned char *)lp; + unsigned char *cp = (unsigned char *)lp; unsigned char t; assert(sizeof(uint64_t) == 8); t = cp[7]; @@ -78,10 +78,10 @@ void TIFFSwabLong8(uint64_t *lp) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfShort) -void TIFFSwabArrayOfShort(register uint16_t *wp, tmsize_t n) +void TIFFSwabArrayOfShort(uint16_t *wp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint16_t) == 2); /* XXX unroll loop some */ while (n-- > 0) @@ -96,7 +96,7 @@ void TIFFSwabArrayOfShort(register uint16_t *wp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfTriples) -void TIFFSwabArrayOfTriples(register uint8_t *tp, tmsize_t n) +void TIFFSwabArrayOfTriples(uint8_t *tp, tmsize_t n) { unsigned char *cp; unsigned char t; @@ -114,10 +114,10 @@ void TIFFSwabArrayOfTriples(register uint8_t *tp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfLong) -void TIFFSwabArrayOfLong(register uint32_t *lp, tmsize_t n) +void TIFFSwabArrayOfLong(uint32_t *lp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint32_t) == 4); /* XXX unroll loop some */ while (n-- > 0) @@ -135,10 +135,10 @@ void TIFFSwabArrayOfLong(register uint32_t *lp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfLong8) -void TIFFSwabArrayOfLong8(register uint64_t *lp, tmsize_t n) +void TIFFSwabArrayOfLong8(uint64_t *lp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint64_t) == 8); /* XXX unroll loop some */ while (n-- > 0) @@ -164,7 +164,7 @@ void TIFFSwabArrayOfLong8(register uint64_t *lp, tmsize_t n) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabFloat) void TIFFSwabFloat(float *fp) { - register unsigned char *cp = (unsigned char *)fp; + unsigned char *cp = (unsigned char *)fp; unsigned char t; assert(sizeof(float) == 4); t = cp[3]; @@ -177,10 +177,10 @@ void TIFFSwabFloat(float *fp) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfFloat) -void TIFFSwabArrayOfFloat(register float *fp, tmsize_t n) +void TIFFSwabArrayOfFloat(float *fp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(float) == 4); /* XXX unroll loop some */ while (n-- > 0) @@ -200,7 +200,7 @@ void TIFFSwabArrayOfFloat(register float *fp, tmsize_t n) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabDouble) void TIFFSwabDouble(double *dp) { - register unsigned char *cp = (unsigned char *)dp; + unsigned char *cp = (unsigned char *)dp; unsigned char t; assert(sizeof(double) == 8); t = cp[7]; @@ -221,8 +221,8 @@ void TIFFSwabDouble(double *dp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfDouble) void TIFFSwabArrayOfDouble(double *dp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(double) == 8); /* XXX unroll loop some */ while (n-- > 0) diff --git a/contrib/libs/libtiff/tif_thunder.c b/contrib/libs/libtiff/tif_thunder.c index bac0607dbe5..0c688db1ee4 100644 --- a/contrib/libs/libtiff/tif_thunder.c +++ b/contrib/libs/libtiff/tif_thunder.c @@ -41,7 +41,7 @@ * or 3-bit delta values are used, with the deltas packed * into a single byte. */ -#define THUNDER_DATA 0x3f /* mask for 6-bit data */ +// #define THUNDER_DATA 0x3f /* mask for 6-bit data */ #define THUNDER_CODE 0xc0 /* mask for 2-bit code word */ /* code values */ #define THUNDER_RUN 0x00 /* run of pixels w/ encoded count */ @@ -56,11 +56,11 @@ static const int threebitdeltas[8] = {0, 1, 2, 3, 0, -3, -2, -1}; #define SETPIXEL(op, v) \ { \ - lastpixel = (v)&0xf; \ + lastpixel = (v) & 0xf; \ if (npixels < maxpixels) \ { \ if (npixels++ & 1) \ - *op++ |= lastpixel; \ + *op++ |= (uint8_t)lastpixel; \ else \ op[0] = (uint8_t)(lastpixel << 4); \ } \ @@ -85,8 +85,8 @@ static int ThunderSetupDecode(TIFF *tif) static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) { static const char module[] = "ThunderDecode"; - register unsigned char *bp; - register tmsize_t cc; + unsigned char *bp; + tmsize_t cc; unsigned int lastpixel; tmsize_t npixels; uint8_t *op = op0; @@ -112,7 +112,7 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) break; if (npixels & 1) { - op[0] |= lastpixel; + op[0] |= (uint8_t)lastpixel; lastpixel = *op++; npixels++; n--; @@ -150,6 +150,8 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) case THUNDER_RAW: /* raw data */ SETPIXEL(op, n); break; + default: + break; } } tif->tif_rawcp = (uint8_t *)bp; @@ -161,7 +163,7 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) TIFFErrorExtR(tif, module, "%s data at scanline %lu (%" PRIu64 " != %" PRIu64 ")", npixels < maxpixels ? "Not enough" : "Too much", - (unsigned long)tif->tif_row, (uint64_t)npixels, + (unsigned long)tif->tif_dir.td_row, (uint64_t)npixels, (uint64_t)maxpixels); return (0); } @@ -175,7 +177,7 @@ static int ThunderDecodeRow(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) uint8_t *row = buf; (void)s; - if (occ % tif->tif_scanlinesize) + if (occ % tif->tif_dir.td_scanlinesize) { TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return (0); @@ -184,8 +186,8 @@ static int ThunderDecodeRow(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { if (!ThunderDecode(tif, row, tif->tif_dir.td_imagewidth)) return (0); - occ -= tif->tif_scanlinesize; - row += tif->tif_scanlinesize; + occ -= tif->tif_dir.td_scanlinesize; + row += tif->tif_dir.td_scanlinesize; } return (1); } diff --git a/contrib/libs/libtiff/tif_tile.c b/contrib/libs/libtiff/tif_tile.c index f07032f731f..f9076094009 100644 --- a/contrib/libs/libtiff/tif_tile.c +++ b/contrib/libs/libtiff/tif_tile.c @@ -54,12 +54,50 @@ uint32_t TIFFComputeTile(TIFF *tif, uint32_t x, uint32_t y, uint32_t z, uint32_t xpt = TIFFhowmany_32(td->td_imagewidth, dx); uint32_t ypt = TIFFhowmany_32(td->td_imagelength, dy); uint32_t zpt = TIFFhowmany_32(td->td_imagedepth, dz); + uint32_t xpt_ypt = _TIFFMultiply32(tif, xpt, ypt, "TIFFComputeTile"); + uint32_t xpt_ypt_zpt = + _TIFFMultiply32(tif, xpt_ypt, zpt, "TIFFComputeTile"); + uint64_t z_offset; + uint64_t y_offset; + uint64_t tile64; + if ((xpt_ypt == 0 && xpt != 0 && ypt != 0) || + (xpt_ypt_zpt == 0 && xpt_ypt != 0 && zpt != 0)) + return (0); + + z_offset = _TIFFMultiply64(tif, xpt_ypt, z / dz, "TIFFComputeTile"); + y_offset = _TIFFMultiply64(tif, xpt, y / dy, "TIFFComputeTile"); + if ((z_offset == 0 && xpt_ypt != 0 && (z / dz) != 0) || + (y_offset == 0 && xpt != 0 && (y / dy) != 0)) + return (0); + tile64 = _TIFFAdd64(tif, z_offset, y_offset, "TIFFComputeTile"); + if (tile64 == 0 && (z_offset != 0 || y_offset != 0)) + return (0); + tile64 = _TIFFAdd64(tif, tile64, x / dx, "TIFFComputeTile"); + if (tile64 == 0 && (z_offset != 0 || y_offset != 0 || (x / dx) != 0)) + return (0); if (td->td_planarconfig == PLANARCONFIG_SEPARATE) - tile = (xpt * ypt * zpt) * s + (xpt * ypt) * (z / dz) + - xpt * (y / dy) + x / dx; - else - tile = (xpt * ypt) * (z / dz) + xpt * (y / dy) + x / dx; + { + uint64_t sample_offset; + if (s >= td->td_samplesperpixel) + { + TIFFErrorExtR( + tif, "TIFFComputeTile", "%lu: Sample out of range, max %lu", + (unsigned long)s, (unsigned long)td->td_samplesperpixel); + return (0); + } + sample_offset = + _TIFFMultiply64(tif, xpt_ypt_zpt, s, "TIFFComputeTile"); + if (sample_offset == 0 && xpt_ypt_zpt != 0 && s != 0) + return (0); + tile64 = _TIFFAdd64(tif, sample_offset, tile64, "TIFFComputeTile"); + if (tile64 == 0 && (sample_offset != 0 || z_offset != 0 || + y_offset != 0 || (x / dx) != 0)) + return (0); + } + tile = _TIFFCastUInt64ToUInt32(tif, tile64, "TIFFComputeTile"); + if (tile == 0 && tile64 != 0) + return (0); } return (tile); } @@ -187,54 +225,9 @@ tmsize_t TIFFTileRowSize(TIFF *tif) */ uint64_t TIFFVTileSize64(TIFF *tif, uint32_t nrows) { - static const char module[] = "TIFFVTileSize64"; - TIFFDirectory *td = &tif->tif_dir; - if (td->td_tilelength == 0 || td->td_tilewidth == 0 || - td->td_tiledepth == 0) - return (0); - if ((td->td_planarconfig == PLANARCONFIG_CONTIG) && - (td->td_photometric == PHOTOMETRIC_YCBCR) && - (td->td_samplesperpixel == 3) && (!isUpSampled(tif))) - { - /* - * Packed YCbCr data contain one Cb+Cr for every - * HorizontalSampling*VerticalSampling Y values. - * Must also roundup width and height when calculating - * since images that are not a multiple of the - * horizontal/vertical subsampling area include - * YCbCr data for the extended image. - */ - uint16_t ycbcrsubsampling[2]; - uint16_t samplingblock_samples; - uint32_t samplingblocks_hor; - uint32_t samplingblocks_ver; - uint64_t samplingrow_samples; - uint64_t samplingrow_size; - TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, - ycbcrsubsampling + 0, ycbcrsubsampling + 1); - if ((ycbcrsubsampling[0] != 1 && ycbcrsubsampling[0] != 2 && - ycbcrsubsampling[0] != 4) || - (ycbcrsubsampling[1] != 1 && ycbcrsubsampling[1] != 2 && - ycbcrsubsampling[1] != 4)) - { - TIFFErrorExtR(tif, module, "Invalid YCbCr subsampling (%dx%d)", - ycbcrsubsampling[0], ycbcrsubsampling[1]); - return 0; - } - samplingblock_samples = ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; - samplingblocks_hor = - TIFFhowmany_32(td->td_tilewidth, ycbcrsubsampling[0]); - samplingblocks_ver = TIFFhowmany_32(nrows, ycbcrsubsampling[1]); - samplingrow_samples = _TIFFMultiply64(tif, samplingblocks_hor, - samplingblock_samples, module); - samplingrow_size = TIFFhowmany8_64(_TIFFMultiply64( - tif, samplingrow_samples, td->td_bitspersample, module)); - return ( - _TIFFMultiply64(tif, samplingrow_size, samplingblocks_ver, module)); - } - else - return (_TIFFMultiply64(tif, nrows, TIFFTileRowSize64(tif), module)); + return _TIFFStrileSize64(tif, nrows, /* isStrip = */ FALSE); } + tmsize_t TIFFVTileSize(TIFF *tif, uint32_t nrows) { static const char module[] = "TIFFVTileSize"; diff --git a/contrib/libs/libtiff/tif_unix.c b/contrib/libs/libtiff/tif_unix.c index 9e6fc75b080..b02fe294eb3 100644 --- a/contrib/libs/libtiff/tif_unix.c +++ b/contrib/libs/libtiff/tif_unix.c @@ -77,10 +77,10 @@ static tmsize_t _tiffReadProc(thandle_t fd, void *buf, tmsize_t size) return (tmsize_t)-1; } fdh.h = fd; - for (bytes_read = 0; bytes_read < bytes_total; bytes_read += count) + for (bytes_read = 0; bytes_read < bytes_total; bytes_read += (size_t)count) { char *buf_offset = (char *)buf + bytes_read; - size_t io_size = bytes_total - bytes_read; + size_t io_size = (size_t)(bytes_total - bytes_read); if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; /* Below is an obvious false positive of Coverity Scan */ @@ -108,10 +108,11 @@ static tmsize_t _tiffWriteProc(thandle_t fd, void *buf, tmsize_t size) return (tmsize_t)-1; } fdh.h = fd; - for (bytes_written = 0; bytes_written < bytes_total; bytes_written += count) + for (bytes_written = 0; bytes_written < bytes_total; + bytes_written += (size_t)count) { const char *buf_offset = (char *)buf + bytes_written; - size_t io_size = bytes_total - bytes_written; + size_t io_size = (size_t)(bytes_total - bytes_written); if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; /* Below is an obvious false positive of Coverity Scan */ @@ -174,7 +175,7 @@ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) (void *)mmap(0, (size_t)sizem, PROT_READ, MAP_SHARED, fdh.fd, 0); if (*pbase != (void *)-1) { - *psize = (tmsize_t)sizem; + *psize = (toff_t)sizem; return (1); } } @@ -184,7 +185,7 @@ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) static void _tiffUnmapProc(thandle_t fd, void *base, toff_t size) { (void)fd; - (void)munmap(base, (off_t)size); + (void)munmap(base, (size_t)size); } #else /* !HAVE_MMAP */ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) @@ -362,7 +363,8 @@ int _TIFFmemcmp(const void *p1, const void *p2, tmsize_t c) return (memcmp(p1, p2, (size_t)c)); } -static void unixWarningHandler(const char *module, const char *fmt, va_list ap) +static void TIFF_ATTRIBUTE((__format__(__printf__, 2, 0))) + unixWarningHandler(const char *module, const char *fmt, va_list ap) { if (module != NULL) fprintf(stderr, "%s: ", module); @@ -372,7 +374,8 @@ static void unixWarningHandler(const char *module, const char *fmt, va_list ap) } TIFFErrorHandler _TIFFwarningHandler = unixWarningHandler; -static void unixErrorHandler(const char *module, const char *fmt, va_list ap) +static void TIFF_ATTRIBUTE((__format__(__printf__, 2, 0))) + unixErrorHandler(const char *module, const char *fmt, va_list ap) { if (module != NULL) fprintf(stderr, "%s: ", module); diff --git a/contrib/libs/libtiff/tif_webp.c b/contrib/libs/libtiff/tif_webp.c index c06ce163b71..cc8b680c52b 100644 --- a/contrib/libs/libtiff/tif_webp.c +++ b/contrib/libs/libtiff/tif_webp.c @@ -84,15 +84,16 @@ static int TWebPDatasetWriter(const uint8_t *data, size_t data_size, if ((tif->tif_rawcc + (tmsize_t)data_size) > tif->tif_rawdatasize) { - TIFFErrorExtR( - tif, module, "Buffer too small by %" TIFF_SIZE_FORMAT " bytes.", - (size_t)(tif->tif_rawcc + data_size - tif->tif_rawdatasize)); + TIFFErrorExtR(tif, module, + "Buffer too small by %" TIFF_SIZE_FORMAT " bytes.", + (size_t)((uint64_t)tif->tif_rawcc + (uint64_t)data_size - + (uint64_t)tif->tif_rawdatasize)); return 0; } else { - _TIFFmemcpy(tif->tif_rawcp, data, data_size); - tif->tif_rawcc += data_size; + _TIFFmemcpy(tif->tif_rawcp, data, (tmsize_t)data_size); + tif->tif_rawcc += (tmsize_t)data_size; tif->tif_rawcp += data_size; return 1; } @@ -116,7 +117,7 @@ static int TWebPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - memcpy(sp->pBuffer + sp->buffer_offset, bp, cc); + memcpy(sp->pBuffer + sp->buffer_offset, bp, (size_t)cc); sp->buffer_offset += (unsigned)cc; return 1; @@ -141,7 +142,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -158,7 +159,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -246,7 +247,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) sp->pBuffer = NULL; } - sp->pBuffer = _TIFFmallocExt(tif, buffer_size); + sp->pBuffer = (uint8_t *)_TIFFmallocExt(tif, buffer_size); if (!sp->pBuffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); @@ -262,10 +263,10 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) WebPInitDecBuffer(&sp->sDecBuffer); sp->sDecBuffer.is_external_memory = 1; - sp->sDecBuffer.width = segment_width; - sp->sDecBuffer.height = segment_height; + sp->sDecBuffer.width = (int)segment_width; + sp->sDecBuffer.height = (int)segment_height; sp->sDecBuffer.u.RGBA.rgba = decode_whole_strile ? op : sp->pBuffer; - sp->sDecBuffer.u.RGBA.stride = segment_width * sp->nSamples; + sp->sDecBuffer.u.RGBA.stride = (int)(segment_width * sp->nSamples); sp->sDecBuffer.u.RGBA.size = buffer_size; if (sp->nSamples > 3) @@ -298,7 +299,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - status = WebPIAppend(sp->psDecoder, tif->tif_rawcp, tif->tif_rawcc); + status = WebPIAppend(sp->psDecoder, tif->tif_rawcp, (size_t)tif->tif_rawcc); if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { @@ -345,7 +346,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } else { - memcpy(op, buf + (sp->last_y * stride), occ); + memcpy(op, buf + (sp->last_y * stride), (size_t)occ); } tif->tif_rawcp += tif->tif_rawcc; @@ -465,7 +466,7 @@ static int TWebPPreDecode(TIFF *tif, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -601,7 +602,7 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -623,7 +624,7 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) sp->pBuffer = NULL; } - sp->pBuffer = _TIFFmallocExt(tif, sp->buffer_size); + sp->pBuffer = (uint8_t *)_TIFFmallocExt(tif, sp->buffer_size); if (!sp->pBuffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); @@ -631,8 +632,8 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) } sp->buffer_offset = 0; - sp->sPicture.width = segment_width; - sp->sPicture.height = segment_height; + sp->sPicture.width = (int)segment_width; + sp->sPicture.height = (int)segment_height; sp->sPicture.writer = TWebPDatasetWriter; sp->sPicture.custom_ptr = tif; @@ -708,9 +709,11 @@ static int TWebPPostEncode(TIFF *tif) case VP8_ENC_ERROR_USER_ABORT: pszErrorMsg = "User interrupted"; break; + case VP8_ENC_OK: + case VP8_ENC_ERROR_LAST: default: TIFFErrorExtR(tif, module, - "WebPEncode returned an unknown error code: %d", + "WebPEncode returned an unknown error code: %u", sp->sPicture.error_code); pszErrorMsg = "Unknown WebP error type."; break; @@ -776,7 +779,7 @@ static int TWebPVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_WEBP_LEVEL: sp->quality_level = (int)va_arg(ap, int); - if (sp->quality_level <= 0 || sp->quality_level > 100.0f) + if (sp->quality_level <= 0 || sp->quality_level > 100) { TIFFWarningExtR(tif, module, "WEBP_LEVEL should be between 1 and 100"); @@ -844,6 +847,17 @@ static const TIFFField TWebPFields[] = { FIELD_PSEUDO, TRUE, FALSE, "WEBP exact lossless", NULL}, }; +static uint64_t TWebPGetMaxCompressionRatio(TIFF *tif) +{ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* lossy compression: */ + /* return (tif->tif_dir.td_samplesperpixel == 4) ? 2199 : 1685; */ + /* lossless compression: */ + return (tif->tif_dir.td_samplesperpixel == 4) ? 104194 : 78146; +} + int TIFFInitWebP(TIFF *tif, int scheme) { static const char module[] = "TIFFInitWebP"; @@ -907,6 +921,7 @@ int TIFFInitWebP(TIFF *tif, int scheme) tif->tif_encodestrip = TWebPEncode; tif->tif_encodetile = TWebPEncode; tif->tif_cleanup = TWebPCleanup; + tif->tif_getmaxcompressionratio = TWebPGetMaxCompressionRatio; return 1; bad: diff --git a/contrib/libs/libtiff/tif_win32.c b/contrib/libs/libtiff/tif_win32.c index 525cebec76d..4b21ab7d482 100644 --- a/contrib/libs/libtiff/tif_win32.c +++ b/contrib/libs/libtiff/tif_win32.c @@ -5,6 +5,7 @@ * Permission to use, copy, modify, distribute, and sell this software and * its documentation for any purpose is hereby granted without fee, provided * that (i) the above copyright notices and this permission notice appear in + * that (i) the above copyright notices and this permission notice appear in * all copies of the software and related documentation, and (ii) the names of * Sam Leffler and Silicon Graphics may not be used in any advertising or * publicity relating to the software without the specific, prior written @@ -72,7 +73,7 @@ static tmsize_t _tiffReadProc(thandle_t fd, void *buf, tmsize_t size) DWORD o; tmsize_t p; ma = (uint8_t *)buf; - mb = size; + mb = (uint64_t)size; p = 0; while (mb > 0) { @@ -101,7 +102,7 @@ static tmsize_t _tiffWriteProc(thandle_t fd, void *buf, tmsize_t size) DWORD o; tmsize_t p; ma = (uint8_t *)buf; - mb = size; + mb = (uint64_t)size; p = 0; while (mb > 0) { @@ -123,7 +124,7 @@ static uint64_t _tiffSeekProc(thandle_t fd, uint64_t off, int whence) { LARGE_INTEGER offli; DWORD dwMoveMethod; - offli.QuadPart = off; + offli.QuadPart = (LONGLONG)off; switch (whence) { case SEEK_SET: @@ -140,11 +141,11 @@ static uint64_t _tiffSeekProc(thandle_t fd, uint64_t off, int whence) break; } offli.LowPart = - SetFilePointer(fd, offli.LowPart, &offli.HighPart, dwMoveMethod); + SetFilePointer(fd, (LONG)offli.LowPart, &offli.HighPart, dwMoveMethod); if ((offli.LowPart == INVALID_SET_FILE_POINTER) && (GetLastError() != NO_ERROR)) offli.QuadPart = 0; - return (offli.QuadPart); + return ((uint64_t)offli.QuadPart); } static int _tiffCloseProc(thandle_t fd) { return (CloseHandle(fd) ? 0 : -1); } @@ -153,7 +154,7 @@ static uint64_t _tiffSizeProc(thandle_t fd) { LARGE_INTEGER m; if (GetFileSizeEx(fd, &m)) - return (m.QuadPart); + return ((uint64_t)m.QuadPart); else return (0); } @@ -343,7 +344,7 @@ TIFF *TIFFOpenWExt(const wchar_t *name, const char *mode, TIFFOpenOptions *opts) NULL); if (fd == INVALID_HANDLE_VALUE) { - _TIFFErrorEarly(opts, NULL, module, "%S: Cannot open", name); + _TIFFErrorEarly(opts, NULL, module, "%ls: Cannot open", name); return ((TIFF *)0); } diff --git a/contrib/libs/libtiff/tif_write.c b/contrib/libs/libtiff/tif_write.c index 3263853cc0f..614a8ea85a4 100644 --- a/contrib/libs/libtiff/tif_write.c +++ b/contrib/libs/libtiff/tif_write.c @@ -30,7 +30,7 @@ #include "tiffiop.h" #include <stdio.h> -#define STRIPINCR 20 /* expansion factor on strip array */ +#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ #define WRITECHECKSTRIPS(tif, module) \ (((tif)->tif_flags & TIFF_BEENWRITING) || TIFFWriteCheck((tif), 0, module)) @@ -38,7 +38,7 @@ (((tif)->tif_flags & TIFF_BEENWRITING) || TIFFWriteCheck((tif), 1, module)) #define BUFFERCHECK(tif) \ ((((tif)->tif_flags & TIFF_BUFFERSETUP) && tif->tif_rawdata) || \ - TIFFWriteBufferSetup((tif), NULL, (tmsize_t)-1)) + TIFFWriteBufferSetup((tif), NULL, (tmsize_t)(-1))) static int TIFFGrowStrips(TIFF *tif, uint32_t delta, const char *module); static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, @@ -47,7 +47,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) { static const char module[] = "TIFFWriteScanline"; - register TIFFDirectory *td; + TIFFDirectory *td; int status, imagegrew = 0; uint32_t strip; @@ -82,8 +82,16 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* * Calculate strip and check for crossings. */ + if (td->td_rowsperstrip == 0) + { + TIFFErrorExtR(tif, module, + "Cannot compute strip: RowsPerStrip is zero"); + return (-1); + } if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, module, "%lu: Sample out of range, max %lu", @@ -91,7 +99,18 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) (unsigned long)td->td_samplesperpixel); return (-1); } - strip = sample * td->td_stripsperimage + row / td->td_rowsperstrip; + sample_offset = + _TIFFMultiply64(tif, sample, td->td_stripsperimage, module); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (-1); + strip64 = + _TIFFAdd64(tif, sample_offset, row / td->td_rowsperstrip, module); + if (strip64 == 0 && + (sample_offset != 0 || (row / td->td_rowsperstrip) != 0)) + return (-1); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, module); + if (strip == 0 && strip64 != 0) + return (-1); } else strip = row / td->td_rowsperstrip; @@ -104,14 +123,14 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) */ if (strip >= td->td_nstrips && !TIFFGrowStrips(tif, 1, module)) return (-1); - if (strip != tif->tif_curstrip) + if (strip != tif->tif_dir.td_curstrip) { /* * Changing strips -- flush any data present. */ if (!TIFFFlushData(tif)) return (-1); - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* * Watch out for a growing image. The value of strips/image * will initially be 1 (since it can't be deduced until the @@ -125,7 +144,8 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) TIFFErrorExtR(tif, module, "Zero strips per image"); return (-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = + (strip % td->td_stripsperimage) * td->td_rowsperstrip; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { if (!(*tif->tif_setupencode)(tif)) @@ -148,35 +168,35 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) * beginning of a strip (or that we can randomly * access the data -- i.e. no encoding). */ - if (row != tif->tif_row) + if (row != tif->tif_dir.td_row) { - if (row < tif->tif_row) + if (row < tif->tif_dir.td_row) { /* * Moving backwards within the same strip: * backup to the start and then decode * forward (below). */ - tif->tif_row = + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; tif->tif_rawcp = tif->tif_rawdata; } /* * Seek forward to the desired row. */ - if (!(*tif->tif_seek)(tif, row - tif->tif_row)) + if (!(*tif->tif_seek)(tif, row - tif->tif_dir.td_row)) return (-1); - tif->tif_row = row; + tif->tif_dir.td_row = row; } /* swab if needed - note that source buffer will be altered */ - tif->tif_postdecode(tif, (uint8_t *)buf, tif->tif_scanlinesize); + tif->tif_postdecode(tif, (uint8_t *)buf, tif->tif_dir.td_scanlinesize); - status = (*tif->tif_encoderow)(tif, (uint8_t *)buf, tif->tif_scanlinesize, - sample); + status = (*tif->tif_encoderow)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize, sample); /* we are now poised at the beginning of the next row */ - tif->tif_row = row + 1; + tif->tif_dir.td_row = row + 1; return (status); } @@ -188,7 +208,21 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* time if the new compressed tile is bigger than the older one. (GDAL #4771) */ static int _TIFFReserveLargeEnoughWriteBuffer(TIFF *tif, uint32_t strip_or_tile) { + static const char module[] = "_TIFFReserveLargeEnoughWriteBuffer"; TIFFDirectory *td = &tif->tif_dir; + + if (td->td_stripbytecount_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip bytecount array pointer is NULL"); + return 0; + } + + if (strip_or_tile == NOSTRIP || strip_or_tile >= td->td_nstrips) + { + TIFFErrorExtR(tif, module, "Strip/tile number not valid"); + return 0; + } + if (td->td_stripbytecount_p[strip_or_tile] > 0) { /* The +1 is to ensure at least one extra bytes */ @@ -254,7 +288,7 @@ tmsize_t TIFFWriteEncodedStrip(TIFF *tif, uint32_t strip, void *data, tif->tif_flags |= TIFF_BUF4WRITE; - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* this informs TIFFAppendToStrip() we have changed or reset strip */ tif->tif_curoff = 0; @@ -273,7 +307,7 @@ tmsize_t TIFFWriteEncodedStrip(TIFF *tif, uint32_t strip, void *data, return ((tmsize_t)-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { if (!(*tif->tif_setupencode)(tif)) @@ -362,9 +396,9 @@ tmsize_t TIFFWriteRawStrip(TIFF *tif, uint32_t strip, void *data, tmsize_t cc) return ((tmsize_t)-1); } - if (tif->tif_curstrip != strip) + if (tif->tif_dir.td_curstrip != strip) { - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* this informs TIFFAppendToStrip() we have changed or reset strip */ tif->tif_curoff = 0; @@ -375,7 +409,7 @@ tmsize_t TIFFWriteRawStrip(TIFF *tif, uint32_t strip, void *data, tmsize_t cc) TIFFErrorExtR(tif, module, "Zero strips per image"); return ((tmsize_t)-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; return (TIFFAppendToStrip(tif, strip, (uint8_t *)data, cc) ? cc : (tmsize_t)-1); } @@ -437,7 +471,7 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) tif->tif_flags |= TIFF_BUF4WRITE; - tif->tif_curtile = tile; + tif->tif_dir.td_curtile = tile; /* this informs TIFFAppendToStrip() we have changed or reset tile */ tif->tif_curoff = 0; @@ -460,14 +494,14 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) TIFFErrorExtR(tif, module, "Zero tiles"); return ((tmsize_t)(-1)); } - tif->tif_row = (tile % howmany32) * td->td_tilelength; + tif->tif_dir.td_row = (tile % howmany32) * td->td_tilelength; howmany32 = TIFFhowmany_32(td->td_imagewidth, td->td_tilewidth); if (howmany32 == 0) { TIFFErrorExtR(tif, module, "Zero tiles"); return ((tmsize_t)(-1)); } - tif->tif_col = (tile % howmany32) * td->td_tilewidth; + tif->tif_dir.td_col = (tile % howmany32) * td->td_tilewidth; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { @@ -482,8 +516,8 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) * done so that callers can pass in some large number * (e.g. -1) and have the tile size used instead. */ - if (cc < 1 || cc > tif->tif_tilesize) - cc = tif->tif_tilesize; + if (cc < 1 || cc > tif->tif_dir.td_tilesize) + cc = tif->tif_dir.td_tilesize; /* shortcut to avoid an extra memcpy() */ if (td->td_compression == COMPRESSION_NONE) @@ -588,8 +622,10 @@ int TIFFSetupStrips(TIFF *tif) * Place data at the end-of-file * (by setting offsets to zero). */ - _TIFFmemset(td->td_stripoffset_p, 0, td->td_nstrips * sizeof(uint64_t)); - _TIFFmemset(td->td_stripbytecount_p, 0, td->td_nstrips * sizeof(uint64_t)); + _TIFFmemset(td->td_stripoffset_p, 0, + (tmsize_t)((size_t)td->td_nstrips * sizeof(uint64_t))); + _TIFFmemset(td->td_stripbytecount_p, 0, + (tmsize_t)((size_t)td->td_nstrips * sizeof(uint64_t))); TIFFSetFieldBit(tif, FIELD_STRIPOFFSETS); TIFFSetFieldBit(tif, FIELD_STRIPBYTECOUNTS); return (1); @@ -644,14 +680,14 @@ int TIFFWriteCheck(TIFF *tif, int tiles, const char *module) } if (isTiled(tif)) { - tif->tif_tilesize = TIFFTileSize(tif); - if (tif->tif_tilesize == 0) + tif->tif_dir.td_tilesize = TIFFTileSize(tif); + if (tif->tif_dir.td_tilesize == 0) return (0); } else - tif->tif_tilesize = (tmsize_t)(-1); - tif->tif_scanlinesize = TIFFScanlineSize(tif); - if (tif->tif_scanlinesize == 0) + tif->tif_dir.td_tilesize = (tmsize_t)(-1); + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); + if (tif->tif_dir.td_scanlinesize == 0) return (0); tif->tif_flags |= TIFF_BEENWRITING; @@ -689,7 +725,7 @@ int TIFFWriteBufferSetup(TIFF *tif, void *bp, tmsize_t size) } if (size == (tmsize_t)(-1)) { - size = (isTiled(tif) ? tif->tif_tilesize : TIFFStripSize(tif)); + size = (isTiled(tif) ? tif->tif_dir.td_tilesize : TIFFStripSize(tif)); /* Adds 10% margin for cases where compression would expand a bit */ if (size < TIFF_TMSIZE_T_MAX - size / 10) @@ -732,26 +768,31 @@ static int TIFFGrowStrips(TIFF *tif, uint32_t delta, const char *module) assert(td->td_planarconfig == PLANARCONFIG_CONTIG); new_stripoffset = (uint64_t *)_TIFFreallocExt( - tif, td->td_stripoffset_p, (td->td_nstrips + delta) * sizeof(uint64_t)); + tif, td->td_stripoffset_p, + (tmsize_t)(((size_t)td->td_nstrips + (size_t)delta) * + sizeof(uint64_t))); + /* + * Update td_stripoffset_p immediately so the old pointer is not left + * dangling if the second realloc fails. + */ + if (new_stripoffset) + td->td_stripoffset_p = new_stripoffset; new_stripbytecount = (uint64_t *)_TIFFreallocExt( tif, td->td_stripbytecount_p, - (td->td_nstrips + delta) * sizeof(uint64_t)); + (tmsize_t)(((size_t)td->td_nstrips + (size_t)delta) * + sizeof(uint64_t))); + if (new_stripbytecount) + td->td_stripbytecount_p = new_stripbytecount; if (new_stripoffset == NULL || new_stripbytecount == NULL) { - if (new_stripoffset) - _TIFFfreeExt(tif, new_stripoffset); - if (new_stripbytecount) - _TIFFfreeExt(tif, new_stripbytecount); td->td_nstrips = 0; TIFFErrorExtR(tif, module, "No space to expand strip arrays"); return (0); } - td->td_stripoffset_p = new_stripoffset; - td->td_stripbytecount_p = new_stripbytecount; _TIFFmemset(td->td_stripoffset_p + td->td_nstrips, 0, - delta * sizeof(uint64_t)); + (tmsize_t)((size_t)delta * sizeof(uint64_t))); _TIFFmemset(td->td_stripbytecount_p + td->td_nstrips, 0, - delta * sizeof(uint64_t)); + (tmsize_t)((size_t)delta * sizeof(uint64_t))); td->td_nstrips += delta; tif->tif_flags |= TIFF_DIRTYDIRECT; @@ -769,6 +810,23 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, uint64_t m; int64_t old_byte_count = -1; + /* Some security checks */ + if (td->td_stripoffset_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip offset array pointer is NULL"); + return (0); + } + if (td->td_stripbytecount_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip bytecount array pointer is NULL"); + return (0); + } + if (strip == NOSTRIP) + { + TIFFErrorExtR(tif, module, "Strip number not valid (NOSTRIP)"); + return (0); + } + if (tif->tif_curoff == 0) tif->tif_lastvalidoff = 0; @@ -790,7 +848,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, if (!SeekOK(tif, td->td_stripoffset_p[strip])) { TIFFErrorExtR(tif, module, "Seek error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return (0); } @@ -812,11 +870,11 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, /* * We are starting a fresh strip/tile, so set the size to zero. */ - old_byte_count = td->td_stripbytecount_p[strip]; + old_byte_count = (int64_t)td->td_stripbytecount_p[strip]; td->td_stripbytecount_p[strip] = 0; } - m = tif->tif_curoff + cc; + m = tif->tif_curoff + (uint64_t)cc; if (!(tif->tif_flags & TIFF_BIGTIFF)) m = (uint32_t)m; if ((m < tif->tif_curoff) || (m < (uint64_t)cc)) @@ -850,7 +908,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, offsetRead = td->td_stripoffset_p[strip]; offsetWrite = TIFFSeekFile(tif, 0, SEEK_END); - m = offsetWrite + toCopy + cc; + m = offsetWrite + (uint64_t)toCopy + (uint64_t)cc; if (!(tif->tif_flags & TIFF_BIGTIFF) && m != (uint32_t)m) { TIFFErrorExtR(tif, module, "Maximum TIFF file size exceeded"); @@ -872,13 +930,15 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, /* Move data written by previous calls to us at end of file */ while (toCopy > 0) { + tmsize_t chunkSize = + toCopy < (uint64_t)tempSize ? (tmsize_t)toCopy : tempSize; if (!SeekOK(tif, offsetRead)) { TIFFErrorExtR(tif, module, "Seek error"); _TIFFfreeExt(tif, temp); return (0); } - if (!ReadOK(tif, temp, tempSize)) + if (!ReadOK(tif, temp, chunkSize)) { TIFFErrorExtR(tif, module, "Cannot read"); _TIFFfreeExt(tif, temp); @@ -890,32 +950,32 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, _TIFFfreeExt(tif, temp); return (0); } - if (!WriteOK(tif, temp, tempSize)) + if (!WriteOK(tif, temp, chunkSize)) { TIFFErrorExtR(tif, module, "Cannot write"); _TIFFfreeExt(tif, temp); return (0); } - offsetRead += tempSize; - offsetWrite += tempSize; - td->td_stripbytecount_p[strip] += tempSize; - toCopy -= tempSize; + offsetRead += (uint64_t)chunkSize; + offsetWrite += (uint64_t)chunkSize; + td->td_stripbytecount_p[strip] += (uint64_t)chunkSize; + toCopy -= (uint64_t)chunkSize; } _TIFFfreeExt(tif, temp); /* Append the data of this call */ - offsetWrite += cc; + offsetWrite += (uint64_t)cc; m = offsetWrite; } if (!WriteOK(tif, data, cc)) { TIFFErrorExtR(tif, module, "Write error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return (0); } tif->tif_curoff = m; - td->td_stripbytecount_p[strip] += cc; + td->td_stripbytecount_p[strip] += (uint64_t)cc; if ((int64_t)td->td_stripbytecount_p[strip] != old_byte_count) tif->tif_flags |= TIFF_DIRTYSTRIP; @@ -935,9 +995,10 @@ int TIFFFlushData1(TIFF *tif) if (!isFillOrder(tif, tif->tif_dir.td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) TIFFReverseBits((uint8_t *)tif->tif_rawdata, tif->tif_rawcc); - if (!TIFFAppendToStrip( - tif, isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip, - tif->tif_rawdata, tif->tif_rawcc)) + if (!TIFFAppendToStrip(tif, + isTiled(tif) ? tif->tif_dir.td_curtile + : tif->tif_dir.td_curstrip, + tif->tif_rawdata, tif->tif_rawcc)) { /* We update those variables even in case of error since there's */ /* code that doesn't really check the return code of this */ diff --git a/contrib/libs/libtiff/tif_zip.c b/contrib/libs/libtiff/tif_zip.c index 441c7fe57c3..ac55bf0795e 100644 --- a/contrib/libs/libtiff/tif_zip.c +++ b/contrib/libs/libtiff/tif_zip.c @@ -159,7 +159,8 @@ static int ZIPPreDecode(TIFF *tif, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_in = TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawcc); + sp->stream.avail_in = + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawcc); if (inflateReset(&sp->stream) == Z_OK) { sp->read_error = 0; @@ -183,7 +184,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -210,7 +211,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } else { - uint32_t strip_height = td->td_imagelength - tif->tif_row; + uint32_t strip_height = td->td_imagelength - tif->tif_dir.td_row; if (strip_height > td->td_rowsperstrip) strip_height = td->td_rowsperstrip; if (TIFFVStripSize64(tif, strip_height) != (uint64_t)occ) @@ -254,7 +255,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); sp->read_error = 1; return 0; } @@ -275,8 +276,9 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) do { int state; - uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawcc); - uInt avail_out_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(occ); + uInt avail_in_before = + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawcc); + uInt avail_out_before = TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)occ); sp->stream.avail_in = avail_in_before; sp->stream.avail_out = avail_out_before; state = inflate(&sp->stream, Z_PARTIAL_FLUSH); @@ -288,7 +290,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(sp->stream.next_out, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu, %s", - (unsigned long)tif->tif_row, SAFE_MSG(sp)); + (unsigned long)tif->tif_dir.td_row, SAFE_MSG(sp)); sp->read_error = 1; return (0); } @@ -305,7 +307,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %lu (short %" PRIu64 " bytes)", - (unsigned long)tif->tif_row, (uint64_t)occ); + (unsigned long)tif->tif_dir.td_row, (uint64_t)occ); memset(sp->stream.next_out, 0, (size_t)occ); sp->read_error = 1; return (0); @@ -411,7 +413,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) } else { - uint32_t strip_height = td->td_imagelength - tif->tif_row; + uint32_t strip_height = td->td_imagelength - tif->tif_dir.td_row; if (strip_height > td->td_rowsperstrip) strip_height = td->td_rowsperstrip; if (TIFFVStripSize64(tif, strip_height) != (uint64_t)cc) @@ -463,11 +465,11 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) if (nCompressedBytes == 0) { TIFFErrorExtR(tif, module, "Encoder error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } - tif->tif_rawcc = nCompressedBytes; + tif->tif_rawcc = (tmsize_t)nCompressedBytes; if (!TIFFFlushData1(tif)) return 0; @@ -485,7 +487,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) appropriately even before we simplify it */ do { - uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(cc); + uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)cc); sp->stream.avail_in = avail_in_before; if (deflate(&sp->stream, Z_NO_FLUSH) != Z_OK) { @@ -499,7 +501,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; sp->stream.next_out = tif->tif_rawdata; sp->stream.avail_out = - TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawdatasize); + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawdatasize); } cc -= (avail_in_before - sp->stream.avail_in); } while (cc > 0); @@ -682,7 +684,7 @@ static const TIFFField zipFields[] = { static void *TIFF_zalloc(void *opaque, unsigned int items, unsigned int size) { static const char module[] = "TIFF_zalloc"; - TIFF *tif = opaque; + TIFF *tif = (TIFF *)opaque; if (items > ~(size_t)0 / size) { @@ -698,6 +700,13 @@ static void TIFF_zfree(void *opaque, void *ptr) _TIFFfreeExt((TIFF *)opaque, ptr); } +static uint64_t ZIPGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* cf https://zlib.net/zlib_tech.html */ + return 1032; +} + int TIFFInitZIP(TIFF *tif, int scheme) { static const char module[] = "TIFFInitZIP"; @@ -764,6 +773,7 @@ int TIFFInitZIP(TIFF *tif, int scheme) tif->tif_encodestrip = ZIPEncode; tif->tif_encodetile = ZIPEncode; tif->tif_cleanup = ZIPCleanup; + tif->tif_getmaxcompressionratio = ZIPGetMaxCompressionRatio; /* * Setup predictor setup. */ diff --git a/contrib/libs/libtiff/tif_zstd.c b/contrib/libs/libtiff/tif_zstd.c index c828dbf7432..9a0b969e31a 100644 --- a/contrib/libs/libtiff/tif_zstd.c +++ b/contrib/libs/libtiff/tif_zstd.c @@ -159,13 +159,13 @@ static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) memset(op + out_buffer.pos, 0, out_buffer.size - out_buffer.pos); TIFFErrorExtR(tif, module, "Not enough data at scanline %lu (short %lu bytes)", - (unsigned long)tif->tif_row, + (unsigned long)tif->tif_dir.td_row, (unsigned long)((size_t)occ - out_buffer.pos)); return 0; } tif->tif_rawcp += in_buffer.pos; - tif->tif_rawcc -= in_buffer.pos; + tif->tif_rawcc -= (tmsize_t)in_buffer.pos; return 1; } @@ -287,7 +287,7 @@ static int ZSTDPostEncode(TIFF *tif) } if (sp->out_buffer.pos > 0) { - tif->tif_rawcc = sp->out_buffer.pos; + tif->tif_rawcc = (tmsize_t)sp->out_buffer.pos; if (!TIFFFlushData1(tif)) return 0; sp->out_buffer.dst = tif->tif_rawcp; diff --git a/contrib/libs/libtiff/tiffio.h b/contrib/libs/libtiff/tiffio.h index f9c206e32a8..bb8fbde248b 100644 --- a/contrib/libs/libtiff/tiffio.h +++ b/contrib/libs/libtiff/tiffio.h @@ -125,13 +125,13 @@ typedef void *thandle_t; /* client data handle */ */ /* reference white */ -#define D65_X0 (95.0470F) -#define D65_Y0 (100.0F) -#define D65_Z0 (108.8827F) +#define D65_X0 (95.0470f) +#define D65_Y0 (100.0f) +#define D65_Z0 (108.8827f) -#define D50_X0 (96.4250F) -#define D50_Y0 (100.0F) -#define D50_Z0 (82.4680F) +#define D50_X0 (96.4250f) +#define D50_Y0 (100.0f) +#define D50_Z0 (82.4680f) /* Structure for holding information about a display device. */ @@ -254,7 +254,7 @@ struct _TIFFRGBAImage * Macros for extracting components from the * packed ABGR form returned by TIFFReadRGBAImage. */ -#define TIFFGetR(abgr) ((abgr)&0xff) +#define TIFFGetR(abgr) ((abgr) & 0xff) #define TIFFGetG(abgr) (((abgr) >> 8) & 0xff) #define TIFFGetB(abgr) (((abgr) >> 16) & 0xff) #define TIFFGetA(abgr) (((abgr) >> 24) & 0xff) @@ -269,7 +269,7 @@ struct _TIFFRGBAImage typedef int (*TIFFInitMethod)(TIFF *, int); typedef struct { - char *name; + const char *name; uint16_t scheme; TIFFInitMethod init; } TIFFCodec; @@ -584,6 +584,7 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, tmsize_t cc); extern tmsize_t TIFFWriteRawTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc); + extern uint64_t TIFFGetMaxCompressionRatio(TIFF *tif); extern int TIFFDataWidth( TIFFDataType); /* table of tag datatype widths within TIFF file. */ extern void TIFFSetWriteOffset(TIFF *tif, toff_t off); @@ -659,7 +660,7 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ - char *field_name; /* ASCII name */ + const char *field_name; /* ASCII name */ } TIFFFieldInfo; extern int TIFFMergeFieldInfo(TIFF *, const TIFFFieldInfo[], uint32_t); diff --git a/contrib/libs/libtiff/tiffiop.h b/contrib/libs/libtiff/tiffiop.h index 80c04c279be..4f4a29cfa5f 100644 --- a/contrib/libs/libtiff/tiffiop.h +++ b/contrib/libs/libtiff/tiffiop.h @@ -38,6 +38,7 @@ #include <sys/types.h> #endif +#include <math.h> #include <string.h> #ifdef HAVE_ASSERT_H @@ -94,6 +95,7 @@ typedef int (*TIFFSeekMethod)(TIFF *, uint32_t); typedef void (*TIFFPostMethod)(TIFF *tif, uint8_t *buf, tmsize_t size); typedef uint32_t (*TIFFStripMethod)(TIFF *, uint32_t); typedef void (*TIFFTileMethod)(TIFF *, uint32_t *, uint32_t *); +typedef uint64_t (*TIFFGetMaxCompressionRatioMethod)(TIFF *); struct TIFFOffsetAndDirNumber { @@ -143,13 +145,18 @@ struct tiff 0x800000U /* read buffer (tif_rawdata) points into mmap() memory */ #define TIFF_DEFERSTRILELOAD \ 0x1000000U /* defer strip/tile offset/bytecount array loading. */ -#define TIFF_LAZYSTRILELOAD \ - 0x2000000U /* lazy/ondemand loading of strip/tile offset/bytecount values. \ - Only used if TIFF_DEFERSTRILELOAD is set and in read-only \ - mode */ +#define TIFF_LAZYSTRILELOAD_DONE \ + 0x2000000U /* set when lazy/ondemand loading of strip/tile \ + offset/bytecount values has been done. Only used if \ + TIFF_DEFERSTRILELOAD is set and in read-only mode */ #define TIFF_CHOPPEDUPARRAYS \ 0x4000000U /* set when allocChoppedUpStripArrays() has modified strip \ array */ +#define TIFF_LAZYSTRILELOAD_ASKED \ + 0x8000000U /* set when lazy/ondemand loading of strip/tile \ + offset/bytecount values has been requested on opening ('O' \ + flag) */ + uint64_t tif_diroff; /* file offset of current directory */ uint64_t tif_nextdiroff; /* file offset of following directory */ uint64_t tif_lastdiroff; /* file offset of last directory written so far */ @@ -162,7 +169,6 @@ struct tiff tif_customdir; /* custom IFDs are separated from the main ones */ TIFFHeaderUnion tif_header; /* file's header block Classic/BigTIFF union */ uint16_t tif_header_size; /* file's header block and its length */ - uint32_t tif_row; /* current scanline */ /* There are IFDs in the file and an "active" IFD in memory, * from which fields are "set" and "get". @@ -184,7 +190,6 @@ struct tiff * - TIFF_NON_EXISTENT_DIR_NUMBER means 'dont know number of IFDs'. * - 0 means 'empty file opened for writing, but no IFD written yet' */ tdir_t tif_curdircount; - uint32_t tif_curstrip; /* current strip for read/write */ uint64_t tif_curoff; /* current offset for read/write */ uint64_t tif_lastvalidoff; /* last valid offset allowed for rewrite in place. Used only by TIFFAppendToStrip() */ @@ -192,10 +197,6 @@ struct tiff /* SubIFD support */ uint16_t tif_nsubifd; /* remaining subifds to write */ uint64_t tif_subifdoff; /* offset for patching SubIFD link */ - /* tiling support */ - uint32_t tif_col; /* current column (offset by row too) */ - uint32_t tif_curtile; /* current tile for read/write */ - tmsize_t tif_tilesize; /* # of bytes in a tile */ /* compression scheme hooks */ int tif_decodestatus; TIFFBoolMethod tif_fixuptags; /* called in TIFFReadDirectory */ @@ -216,10 +217,10 @@ struct tiff TIFFVoidMethod tif_cleanup; /* cleanup state routine */ TIFFStripMethod tif_defstripsize; /* calculate/constrain strip size */ TIFFTileMethod tif_deftilesize; /* calculate/constrain tile size */ - uint8_t *tif_data; /* compression scheme private data */ + /* returns maximum compression ratio for current compression method */ + TIFFGetMaxCompressionRatioMethod tif_getmaxcompressionratio; + uint8_t *tif_data; /* compression scheme private data */ /* input/output buffering */ - tmsize_t tif_scanlinesize; /* # of bytes in a scanline */ - tmsize_t tif_scanlineskew; /* scanline skew for reading strips */ uint8_t *tif_rawdata; /* raw data buffer */ tmsize_t tif_rawdatasize; /* # of bytes in raw data buffer */ tmsize_t tif_rawdataoff; /* rawdata offset within strip */ @@ -295,13 +296,15 @@ struct TIFFOpenOptions * Default Read/Seek/Write definitions. */ #ifndef ReadOK -#define ReadOK(tif, buf, size) (TIFFReadFile((tif), (buf), (size)) == (size)) +#define ReadOK(tif, buf, size) \ + (TIFFReadFile((tif), (buf), (size)) == (tmsize_t)(size)) #endif #ifndef SeekOK #define SeekOK(tif, off) _TIFFSeekOK(tif, off) #endif #ifndef WriteOK -#define WriteOK(tif, buf, size) (TIFFWriteFile((tif), (buf), (size)) == (size)) +#define WriteOK(tif, buf, size) \ + (TIFFWriteFile((tif), (buf), (size)) == (tmsize_t)(size)) #endif /* NB: the uint32_t casts are to silence certain ANSI-C compilers */ @@ -315,19 +318,19 @@ struct TIFFOpenOptions (((uint32_t)(x) / (uint32_t)(y)) + \ ((((uint32_t)(x) % (uint32_t)(y)) != 0) ? 1 : 0)) #define TIFFhowmany8_32(x) \ - (((x)&0x07) ? ((uint32_t)(x) >> 3) + 1 : (uint32_t)(x) >> 3) -#define TIFFroundup_32(x, y) (TIFFhowmany_32(x, y) * (y)) + (((x) & 0x07) ? ((uint32_t)(x) >> 3) + 1 : (uint32_t)(x) >> 3) +#define TIFFroundup_32(x, y) ((uint32_t)(TIFFhowmany_32(x, y) * (uint32_t)(y))) #define TIFFhowmany_64(x, y) \ ((((uint64_t)(x)) + (((uint64_t)(y)) - 1)) / ((uint64_t)(y))) #define TIFFhowmany8_64(x) \ - (((x)&0x07) ? ((uint64_t)(x) >> 3) + 1 : (uint64_t)(x) >> 3) -#define TIFFroundup_64(x, y) (TIFFhowmany_64(x, y) * (y)) + (((x) & 0x07) ? ((uint64_t)(x) >> 3) + 1 : (uint64_t)(x) >> 3) +#define TIFFroundup_64(x, y) ((uint64_t)(TIFFhowmany_64(x, y) * (uint64_t)(y))) /* Safe multiply which returns zero if there is an *unsigned* integer overflow. * This macro is not safe for *signed* integer types */ #define TIFFSafeMultiply(t, v, m) \ - ((((t)(m) != (t)0) && (((t)(((v) * (m)) / (m))) == (t)(v))) \ - ? (t)((v) * (m)) \ + ((((t)(m) != (t)0) && (((t)((((t)(v)) * ((t)(m))) / ((t)(m)))) == (t)(v))) \ + ? (t)(((t)(v)) * ((t)(m))) \ : (t)0) #define TIFFmax(A, B) ((A) > (B) ? (A) : (B)) @@ -335,6 +338,10 @@ struct TIFFOpenOptions #define TIFFArrayCount(a) (sizeof(a) / sizeof((a)[0])) +/* Float/double equality macros that suppress -Wfloat-equal warnings */ +#define TIFF_FLOAT_EQ(x, y) (!(fabsf((x) - (y)) > 0.0f)) +#define TIFF_DOUBLE_EQ(x, y) (!(fabs((x) - (y)) > 0.0)) + /* Support for large files. @@ -355,7 +362,7 @@ struct TIFFOpenOptions */ #if defined(HAVE_FSEEKO) #define fseek(stream, offset, whence) fseeko(stream, offset, whence) -#define ftell(stream, offset, whence) ftello(stream, offset, whence) +#define ftell(stream) ftello(stream) #endif #endif #if defined(_WIN32) && \ @@ -459,9 +466,18 @@ extern "C" extern uint32_t _TIFFMultiply32(TIFF *, uint32_t, uint32_t, const char *); extern uint64_t _TIFFMultiply64(TIFF *, uint64_t, uint64_t, const char *); + extern uint64_t _TIFFAdd64(TIFF *, uint64_t, uint64_t, const char *); extern tmsize_t _TIFFMultiplySSize(TIFF *, tmsize_t, tmsize_t, const char *); + extern tmsize_t _TIFFAddSSize(TIFF *, tmsize_t, tmsize_t, const char *); extern tmsize_t _TIFFCastUInt64ToSSize(TIFF *, uint64_t, const char *); + extern uint32_t _TIFFCastUInt64ToUInt32(TIFF *, uint64_t, const char *); + extern uint64_t _TIFFComputeRowSize64(TIFF *, uint32_t, uint16_t, uint16_t, + const char *); + extern tmsize_t _TIFFComputeRowOffset(TIFF *, tmsize_t rowstride, uint32_t, + const char *); + extern uint64_t _TIFFComputeBitOffset(TIFF *, uint32_t, uint16_t, uint16_t, + const char *); extern void *_TIFFCheckMalloc(TIFF *, tmsize_t, tmsize_t, const char *); extern void *_TIFFCheckRealloc(TIFF *, void *, tmsize_t, tmsize_t, const char *); @@ -471,6 +487,8 @@ extern "C" extern void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif); + extern uint64_t _TIFFStrileSize64(TIFF *tif, uint32_t nrows, int isStrip); + extern tmsize_t _TIFFReadEncodedStripAndAllocBuffer(TIFF *tif, uint32_t strip, void **buf, diff --git a/contrib/libs/libtiff/tiffvers.h b/contrib/libs/libtiff/tiffvers.h index 5b86ac165d2..57821df8320 100644 --- a/contrib/libs/libtiff/tiffvers.h +++ b/contrib/libs/libtiff/tiffvers.h @@ -8,7 +8,7 @@ * Furthermore, configure_file variables of type "" are * modified by clang-format and won't be substituted by CMake. */ -#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.7.1\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." +#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.7.2\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." /* * This define can be used in code that requires * compilation-related definitions specific to a @@ -16,13 +16,13 @@ * version checking should be done based on the * string returned by TIFFGetVersion. */ -#define TIFFLIB_VERSION 20250911 +#define TIFFLIB_VERSION 20260627 /* The following defines have been added in 4.5.0 */ #define TIFFLIB_MAJOR_VERSION 4 #define TIFFLIB_MINOR_VERSION 7 -#define TIFFLIB_MICRO_VERSION 1 -#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.7.1" +#define TIFFLIB_MICRO_VERSION 2 +#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.7.2" /* Macro added in 4.5.0. Returns TRUE if the current libtiff version is * greater or equal to major.minor.micro diff --git a/contrib/libs/libtiff/ya.make b/contrib/libs/libtiff/ya.make index 3cf767876b6..b1fdf7bf784 100644 --- a/contrib/libs/libtiff/ya.make +++ b/contrib/libs/libtiff/ya.make @@ -10,9 +10,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(4.7.1) +VERSION(4.7.2) -ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.7.1) +ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.7.2) PEERDIR( contrib/libs/libjpeg-turbo diff --git a/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py b/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py index 34a3f3fd550..d07e4060761 100644 --- a/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py +++ b/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py @@ -13,6 +13,7 @@ def post_install(self): GRPC=True, PROTO_NAMESPACE=ArcPath(self.arcdir, GLOBAL=True), PY_NAMESPACE=".", + INCLUDE_TAGS=["TS_PROTO", "TS_PREPARE_DEPS"], ) diff --git a/contrib/libs/opentelemetry-proto/ya.make b/contrib/libs/opentelemetry-proto/ya.make index d1a48612fac..cea01da4ebf 100644 --- a/contrib/libs/opentelemetry-proto/ya.make +++ b/contrib/libs/opentelemetry-proto/ya.make @@ -17,6 +17,11 @@ PROTO_NAMESPACE( contrib/libs/opentelemetry-proto ) +INCLUDE_TAGS( + TS_PREPARE_DEPS + TS_PROTO +) + GRPC() SRCS( diff --git a/contrib/libs/zstd06/common/bitstream.h b/contrib/libs/zstd06/common/bitstream.h index 97fc621579c..d6ff81bfdf4 100644 --- a/contrib/libs/zstd06/common/bitstream.h +++ b/contrib/libs/zstd06/common/bitstream.h @@ -52,6 +52,7 @@ extern "C" { ******************************************/ #include "mem.h" /* unaligned access routines */ #include "error_private.h" /* error codes and messages */ +#include <assert.h> /*========================================= @@ -154,6 +155,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); ****************************************************************/ MEM_STATIC unsigned BIT_highbit32 (register U32 val) { + assert(val != 0); + { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; _BitScanReverse ( &r, val ); @@ -172,6 +175,7 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val) r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; return r; # endif + } } /*===== Local Constants =====*/ diff --git a/contrib/libs/zstd06/common/zstd_internal.h b/contrib/libs/zstd06/common/zstd_internal.h index 2ce2b18cf16..0f73e12dd7f 100644 --- a/contrib/libs/zstd06/common/zstd_internal.h +++ b/contrib/libs/zstd06/common/zstd_internal.h @@ -39,6 +39,7 @@ #include "mem.h" #include "error_private.h" #include "zstd_static.h" +#include <assert.h> /*-************************************* @@ -163,6 +164,8 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) MEM_STATIC unsigned ZSTD_highbit(U32 val) { + assert(val != 0); + { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; _BitScanReverse(&r, val); @@ -181,6 +184,7 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val) r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; return r; # endif + } } diff --git a/contrib/libs/zstd06/compress/zstd_compress.c b/contrib/libs/zstd06/compress/zstd_compress.c index 1bb75c68ccf..d61227ef6c1 100644 --- a/contrib/libs/zstd06/compress/zstd_compress.c +++ b/contrib/libs/zstd06/compress/zstd_compress.c @@ -183,7 +183,8 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { - U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1; + U32 const srcLog = + MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit((U32)(rSize)-1) + 1); if (params->windowLog > srcLog) params->windowLog = srcLog; } } if (params->hashLog > params->windowLog) params->hashLog = params->windowLog; diff --git a/contrib/python/scramp/.dist-info/METADATA b/contrib/python/scramp/.dist-info/METADATA index 3551c560d9d..354f0631bb7 100644 --- a/contrib/python/scramp/.dist-info/METADATA +++ b/contrib/python/scramp/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: scramp -Version: 1.4.11 +Version: 1.4.12 Summary: An implementation of the SCRAM protocol. Project-URL: Homepage, https://codeberg.org/tlocke/scramp Author: The Contributors @@ -54,6 +54,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.12, 2026-07-05](#version-1412-2026-07-05) * [Version 1.4.11, 2026-07-04](#version-1411-2026-07-04) * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) @@ -386,6 +387,11 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.12, 2026-07-05 + +- Check for invalid escape chars in username. + + ### Version 1.4.11, 2026-07-04 - Make sure all errors detected by the server get returned to the client as valid errors. diff --git a/contrib/python/scramp/README.md b/contrib/python/scramp/README.md index 63260c0f265..abd5f1ebf18 100644 --- a/contrib/python/scramp/README.md +++ b/contrib/python/scramp/README.md @@ -33,6 +33,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.12, 2026-07-05](#version-1412-2026-07-05) * [Version 1.4.11, 2026-07-04](#version-1411-2026-07-04) * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) @@ -365,6 +366,11 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.12, 2026-07-05 + +- Check for invalid escape chars in username. + + ### Version 1.4.11, 2026-07-04 - Make sure all errors detected by the server get returned to the client as valid errors. diff --git a/contrib/python/scramp/scramp/core.py b/contrib/python/scramp/scramp/core.py index ba53424634f..3518975852f 100644 --- a/contrib/python/scramp/scramp/core.py +++ b/contrib/python/scramp/scramp/core.py @@ -445,13 +445,31 @@ def _parse_message(msg, desc, *expected_attr_sets): ) +ESCAPE_EQUALS = "=3D" +ESCAPE_COMMA = "=2C" + + +def _username_escape(username): + return username.replace("=", ESCAPE_EQUALS).replace(",", ESCAPE_COMMA) + + +def _username_unescape(username): + for i in (i for i, c in enumerate(username) if c == "="): + if username[i : i + 3] not in (ESCAPE_EQUALS, ESCAPE_COMMA): + raise ScramException( + "An '=' in a username must be followed by '3D', or '2C'", + SERVER_ERROR_INVALID_USERNAME_ENCODING, + ) + return username.replace(ESCAPE_COMMA, ",").replace(ESCAPE_EQUALS, "=") + + def _get_client_first(username, c_nonce, channel_binding, use_binding): try: prepped_u = saslprep(username) except ScramException as e: raise ScramException(e.args[0], SERVER_ERROR_INVALID_USERNAME_ENCODING) - u = prepped_u.replace("=", "=3D").replace(",", "=2C") + u = _username_escape(prepped_u) bare = ",".join((f"n={u}", f"r={c_nonce}")) _, gs2_header = _make_gs2_header(channel_binding, use_binding) @@ -538,7 +556,7 @@ def _set_client_first(client_first, s_nonce, channel_binding, use_binding): c_nonce = msg["r"] nonce = c_nonce + s_nonce - user = msg["n"].replace("=2C", ",").replace("=3D", "=") + user = _username_unescape(msg["n"]) return nonce, user, client_first_bare, upgrade_mechanism diff --git a/contrib/python/scramp/ya.make b/contrib/python/scramp/ya.make index cf5785c2ea7..5b612dfca89 100644 --- a/contrib/python/scramp/ya.make +++ b/contrib/python/scramp/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.4.11) +VERSION(1.4.12) LICENSE(MIT-0) diff --git a/contrib/tools/cython/.dist-info/METADATA b/contrib/tools/cython/.dist-info/METADATA index 7ee86c1f0ca..76194acc38d 100644 --- a/contrib/tools/cython/.dist-info/METADATA +++ b/contrib/tools/cython/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: Cython -Version: 3.2.5 +Version: 3.2.8 Summary: The Cython compiler for writing C extensions in the Python language. Home-page: https://cython.org/ Author: Robert Bradshaw, Stefan Behnel, David Woods, Greg Ewing, et al. @@ -64,82 +64,21 @@ to install an uncompiled (slower) version of Cython with:: .. _Pyrex: https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ -3.2.5 (2026-05-23) +3.2.8 (2026-06-30) ================== Bugs fixed ---------- -* A compile failure was fixed when using the walrus operator inside of try-except. - (Github issue https://github.com/cython/cython/issues/7462) +* Assigning a Python 3.14+ ``.__annotate__`` function to a Cython compiled function no longer + evaluates annotations eagerly. Fixes a regression with ``@functools.wraps()`` in Cython 3.2.6. + Patch by Jelle Zijlstra. (Github issue https://github.com/cython/cython/issues/7767) -* Expressions with side-effects as object argument to ``isinstance()`` could get - evaluated multiple times, e.g. when they use the walrus operator. - (Github issue https://github.com/cython/cython/issues/7670) +* In freethreading Python, the necessary object keep-alive while executing user code in + ``.__dealloc__()`` uses a safer scheme. + Patch by Yaxing Cai. (Github issue https://github.com/cython/cython/issues/7769) -* Several problems generating the shared utility module were resolved, including - a performance regression with memory views. - (Github issues https://github.com/cython/cython/issues/7487, https://github.com/cython/cython/issues/7497, https://github.com/cython/cython/issues/7504, https://github.com/cython/cython/issues/7558) +* The local function state used by ``sys.monitoring`` read from uninitialised memory. + (Github issue https://github.com/cython/cython/issues/7774) -* Some GC and refcounting issues were resolved for Cython functions in the Limited API. - (Github issue https://github.com/cython/cython/issues/7594) - -* Refcounting errors and error handling issues were resolved in some rare error handling cases. - (Github issues https://github.com/cython/cython/issues/7597, https://github.com/cython/cython/issues/7599, https://github.com/cython/cython/issues/7612, https://github.com/cython/cython/issues/7673) - -* Using ``cython.pymutex`` in an extension type with ``cdef`` methods generated - invalid C code missing the required ``PyMutex`` declarations. - (Github issue https://github.com/cython/cython/issues/6995) - -* Calling ``.get_frame()`` on Cython coroutines could crash in freethreading Python. - (Github issue https://github.com/cython/cython/issues/7632) - -* The vectorcall protocol was not used correctly in ``.throw()`` of Cython coroutines - when raising the exception only by type (without value or traceback). - (Github issue https://github.com/cython/cython/issues/7677) - -* A problem with cpdef enums in the Limited API of Python 3.11+ was resolved. - (Github issue https://github.com/cython/cython/issues/7503) - -* Unicode predicates like ``.isdigit()`` are now allowed to fail in the Limited API. - (Github issue https://github.com/cython/cython/issues/7602) - -* Conditional expressions mixing Python float and int object types could accidentally - infer float as the common result type, instead of treating both independently. - -* Using ``sizeof()`` in the size declarations of ``extern`` arrays failed. - (Github issue https://github.com/cython/cython/issues/7451) - -* Enabling profiling generated invalid C code for non-Python return tuples. - (Github issue https://github.com/cython/cython/issues/7580) - -* ``abs()`` on C ``long long`` values could generate invalid C code. - -* When the ``CYTHON_AVOID_BORROWED_REFS`` C macro is enabled, e.g. in GraalPython, - dict iteration could fail to compile. - Patch by Michael Šimáček. (Github issue https://github.com/cython/cython/issues/7631) - -* A C compiler warning about unused functions was resolved. - (Github issue https://github.com/cython/cython/issues/7560) - -* The ``py_safe_call_once()`` C++ mutex helper function in ``libcpp.mutex`` - failed to compile. - (Github issue https://github.com/cython/cython/issues/7585) - -* The ``cpython.array`` declarations were adapted for Python 3.15. Direct access - to the C struct of the ``array.array`` data type might require user code changes. - (Github issue https://github.com/cython/cython/issues/7659) - -* Spaces in file paths written to the ``depfile`` were not escaped. - Patch by Loïc Estève. (Github issue https://github.com/cython/cython/issues/7423) - -* Cython's cache failed to restore multi-file results. - (Github issue https://github.com/cython/cython/issues/7559) - -* Using Tempita from its command line failed with a name error. - (Github issue https://github.com/cython/cython/issues/7567) - -Other changes -------------- - -* The known builtin types were updated for Py3.15. +* The ``.ag_running`` flag attribute of async generators was always false on big-endian systems. diff --git a/contrib/tools/cython/.yandex_meta/override.nix b/contrib/tools/cython/.yandex_meta/override.nix index aea78547bf9..fd901b25082 100644 --- a/contrib/tools/cython/.yandex_meta/override.nix +++ b/contrib/tools/cython/.yandex_meta/override.nix @@ -1,10 +1,10 @@ pkgs: attrs: with pkgs; with pkgs.python311.pkgs; with attrs; rec { - version = "3.2.5"; + version = "3.2.8"; src = fetchPypi { pname = "cython"; inherit version; - hash = "sha256-PdQuTPNq0V8mW9/sIzfMAMaIyOttN0/9E7sZQ3wnu6E="; + hash = "sha256-9PI6VrJSIaBvkYF/6PMRSri0ik+scxh9u2S8LEqHlh8="; }; patches = []; diff --git a/contrib/tools/cython/CHANGES.rst b/contrib/tools/cython/CHANGES.rst index a495a987b4b..b97a2aeb4dc 100644 --- a/contrib/tools/cython/CHANGES.rst +++ b/contrib/tools/cython/CHANGES.rst @@ -2,6 +2,55 @@ Cython Changelog ================ +3.2.8 (2026-06-30) +================== + +Bugs fixed +---------- + +* Assigning a Python 3.14+ ``.__annotate__`` function to a Cython compiled function no longer + evaluates annotations eagerly. Fixes a regression with ``@functools.wraps()`` in Cython 3.2.6. + Patch by Jelle Zijlstra. (Github issue :issue:`7767`) + +* In freethreading Python, the necessary object keep-alive while executing user code in + ``.__dealloc__()`` uses a safer scheme. + Patch by Yaxing Cai. (Github issue :issue:`7769`) + +* The local function state used by ``sys.monitoring`` read from uninitialised memory. + (Github issue :issue:`7774`) + +* The ``.ag_running`` flag attribute of async generators was always false on big-endian systems. + + +3.2.6 (2026-06-24) +================== + +Bugs fixed +---------- + +* ``@functools.wraps()`` was broken in Py3.14+ for Cython compiled functions. + (Github issue :issue:`7675`) + +* A double-free in the t-string code was fixed. + (Github issue :issue:`7712`) + +* The ``-`` operator declarations for iterators in ``libcpp.vector`` we corrected. + Patch by Vadim Markovtsev. (Github issue :issue:`7717`) + +* The shared utility code module no longer uses a temporary file path that + changed the C code on each generation. + (Github issue :issue:`7723`) + +* On 32 bit platforms, cached constants are no longer made immortal during module import. + (Github issue :issue:`7744`) + +Other changes +------------- + +* Binary wheels are now built with ``-DNDEBUG`` to discard runtime assertions from CPython's + inline functions. + + 3.2.5 (2026-05-23) ================== diff --git a/contrib/tools/cython/Cython/Build/Dependencies.py b/contrib/tools/cython/Cython/Build/Dependencies.py index 3726a72b529..d45e53d832b 100644 --- a/contrib/tools/cython/Cython/Build/Dependencies.py +++ b/contrib/tools/cython/Cython/Build/Dependencies.py @@ -52,7 +52,7 @@ def extended_iglob(pattern): # because '/' is generally common for relative paths. if '**/' in pattern or os.sep == '\\' and '**\\' in pattern: seen = set() - first, rest = re.split(r'\*\*[%s]' % ('/\\\\' if os.sep == '\\' else '/'), pattern, 1) + first, rest = re.split(r'\*\*[%s]' % ('/\\\\' if os.sep == '\\' else '/'), pattern, maxsplit=1) if first: first = iglob(first + os.sep) else: diff --git a/contrib/tools/cython/Cython/Build/SharedModule.py b/contrib/tools/cython/Cython/Build/SharedModule.py index fed6263cd89..45da4b46238 100644 --- a/contrib/tools/cython/Cython/Build/SharedModule.py +++ b/contrib/tools/cython/Cython/Build/SharedModule.py @@ -1,13 +1,10 @@ import os -import re -import shutil -import tempfile from Cython.Compiler import ( MemoryView, Code, Options, Pipeline, Errors, Main, Symtab ) from Cython.Compiler.StringEncoding import EncodedString -from Cython.Compiler.Scanning import FileSourceDescriptor +from Cython.Compiler.Scanning import SharedUtilitySourceDescriptor def create_shared_library_pipeline(context, scope, options, result): @@ -72,23 +69,17 @@ def generate_shared_module(options): Errors.open_listing_file(None) dest_c_file = options.shared_c_file_path + pyx_file = os.path.splitext(dest_c_file)[0] + '.pyx' module_name = os.path.splitext(os.path.basename(dest_c_file))[0] context = Main.Context.from_options(options) scope = Symtab.ModuleScope('MemoryView', parent_module = None, context = context, is_package=False) - with tempfile.TemporaryDirectory() as tmpdirname: - pyx_file = os.path.join(tmpdirname, f'{module_name}.pyx') - c_file = os.path.join(tmpdirname, f'{module_name}.c') - with open(pyx_file, 'w'): - pass - source_desc = FileSourceDescriptor(pyx_file) - comp_src = Main.CompilationSource(source_desc, EncodedString(module_name), os.getcwd()) - result = Main.create_default_resultobj(comp_src, options) + source_desc = SharedUtilitySourceDescriptor(pyx_file) + comp_src = Main.CompilationSource(source_desc, EncodedString(module_name), os.getcwd()) + result = Main.create_default_resultobj(comp_src, options) - pipeline = create_shared_library_pipeline(context, scope, options, result) - err, enddata = Pipeline.run_pipeline(pipeline, comp_src) - if err is None: - shutil.copy(c_file, dest_c_file) + pipeline = create_shared_library_pipeline(context, scope, options, result) + err, enddata = Pipeline.run_pipeline(pipeline, comp_src) return err, enddata diff --git a/contrib/tools/cython/Cython/Compiler/Code.py b/contrib/tools/cython/Cython/Compiler/Code.py index 630683bf166..ed783164ff8 100644 --- a/contrib/tools/cython/Cython/Compiler/Code.py +++ b/contrib/tools/cython/Cython/Compiler/Code.py @@ -2429,19 +2429,26 @@ class GlobalState: writer.putln("{") writer.putln(f"PyObject **table = {array_cname};") writer.putln(f"for (Py_ssize_t i=0; i<{constant_count}; ++i) {{") - writer.putln("#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING") + writer.putln("#if PY_VERSION_HEX >= 0x030F0000") + writer.putln("PyUnstable_SetImmortal(table[i]);") + writer.putln("#elif CYTHON_COMPILING_IN_CPYTHON_FREETHREADING") # We don't want to set the refcount on shared constants (e.g. cached integers) # because setting the refcount isn't thread-safe. The chances are that most of the constants # that this applies to are already immortal though so that isn't a great loss. + # Overflow, e.g. on 32 bit systems. + writer.putln("if ((PY_SSIZE_T_MAX <= _Py_IMMORTAL_REFCNT_LOCAL)) break;") writer.putln("#if PY_VERSION_HEX < 0x030E0000") writer.putln("if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1)") writer.putln("#else") writer.putln("if (PyUnstable_Object_IsUniquelyReferenced(table[i]))") writer.putln("#endif") writer.putln("{") - writer.putln("Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL);") + # Go one higher than we think we need to because of a bug in SET_REFCNT check in CPython + writer.putln("Py_SET_REFCNT(table[i], ((Py_ssize_t)_Py_IMMORTAL_REFCNT_LOCAL + 1));") writer.putln("}") writer.putln("#else") + # Overflow, e.g. on 32 bit systems. + writer.putln("if ((PY_SSIZE_T_MAX < _Py_IMMORTAL_INITIAL_REFCNT)) break;") writer.putln("Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT);") writer.putln("#endif") writer.putln("}") # for() diff --git a/contrib/tools/cython/Cython/Compiler/ModuleNode.py b/contrib/tools/cython/Cython/Compiler/ModuleNode.py index eb683f8249d..a14f3e9e2ee 100644 --- a/contrib/tools/cython/Cython/Compiler/ModuleNode.py +++ b/contrib/tools/cython/Cython/Compiler/ModuleNode.py @@ -2020,14 +2020,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry or not entry.is_special: return + code.globalstate.use_utility_code( + UtilityCode.load_cached("DeallocKeepAlive", "ExtensionTypes.c")) code.putln("{") code.putln("PyObject *etype, *eval, *etb;") code.putln("PyErr_Fetch(&etype, &eval, &etb);") - # increase the refcount while we are calling into user code - # to prevent recursive deallocation - code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1);") + # Keep the object alive while we are calling into user code, to prevent + # the user code from triggering recursive deallocation. On free-threading + # this must not use Py_SET_REFCNT() (see DeallocKeepAlive in ExtensionTypes.c). + code.putln("__Pyx_DeallocKeepAliveBegin(o);") code.putln("%s(o);" % entry.func_cname) - code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1);") + code.putln("__Pyx_DeallocKeepAliveEnd(o);") code.putln("PyErr_Restore(etype, eval, etb);") code.putln("}") diff --git a/contrib/tools/cython/Cython/Compiler/Scanning.py b/contrib/tools/cython/Cython/Compiler/Scanning.py index d63c1263f27..d843960dd17 100644 --- a/contrib/tools/cython/Cython/Compiler/Scanning.py +++ b/contrib/tools/cython/Cython/Compiler/Scanning.py @@ -278,6 +278,15 @@ class StringSourceDescriptor(SourceDescriptor): return "<StringSourceDescriptor:%s>" % self.name +class SharedUtilitySourceDescriptor(FileSourceDescriptor): + """ + A specialized source descriptor for shared utility code only. Not part of public API. + """ + + def get_file_object(self, encoding=None, error_handling=None): + from io import StringIO + return StringIO('') + #------------------------------------------------------------------ class PyrexScanner(Scanner): diff --git a/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd b/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd index f7f75928640..14f453ba682 100644 --- a/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd +++ b/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd @@ -79,8 +79,8 @@ cdef extern from "<vector>" namespace "std" nogil: reverse_iterator operator--(int) reverse_iterator operator+(size_type) reverse_iterator operator-(size_type) - difference_type operator-(iterator) - difference_type operator-(const_iterator) + difference_type operator-(reverse_iterator) + difference_type operator-(const_reverse_iterator) bint operator==(reverse_iterator) bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) @@ -104,8 +104,8 @@ cdef extern from "<vector>" namespace "std" nogil: const_reverse_iterator operator--(int) const_reverse_iterator operator+(size_type) const_reverse_iterator operator-(size_type) - difference_type operator-(iterator) - difference_type operator-(const_iterator) + difference_type operator-(reverse_iterator) + difference_type operator-(const_reverse_iterator) bint operator==(reverse_iterator) bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) diff --git a/contrib/tools/cython/Cython/Shadow.py b/contrib/tools/cython/Cython/Shadow.py index 8339642e91b..1ec77baac3a 100644 --- a/contrib/tools/cython/Cython/Shadow.py +++ b/contrib/tools/cython/Cython/Shadow.py @@ -1,7 +1,7 @@ # cython.* namespace for pure mode. # Possible version formats: "3.1.0", "3.1.0a1", "3.1.0a1.dev0" -__version__ = "3.2.5" +__version__ = "3.2.8" # BEGIN shameless copy from Cython/minivect/minitypes.py diff --git a/contrib/tools/cython/Cython/Utility/AsyncGen.c b/contrib/tools/cython/Cython/Utility/AsyncGen.c index 4999487d91d..c2b9fe1d449 100644 --- a/contrib/tools/cython/Cython/Utility/AsyncGen.c +++ b/contrib/tools/cython/Cython/Utility/AsyncGen.c @@ -346,11 +346,19 @@ __Pyx_async_gen_self_method(PyObject *g, PyObject *arg) { } +static PyObject * +__Pyx_async_gen_get_running(__pyx_PyAsyncGenObject *o, void *context) { + CYTHON_UNUSED_VAR(context); + return __Pyx_NewRef(o->ag_running_async ? Py_True : Py_False); +} + + static PyGetSetDef __Pyx_async_gen_getsetlist[] = { {"__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, PyDoc_STR("name of the async generator"), 0}, {"__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, PyDoc_STR("qualified name of the async generator"), 0}, + {"ag_running", (getter)__Pyx_async_gen_get_running, NULL, NULL, 0}, //REMOVED: {(char*) "ag_await", (getter)coro_get_cr_await, NULL, //REMOVED: (char*) PyDoc_STR("object being awaited on, or None")}, {0, 0, 0, 0, 0} /* Sentinel */ @@ -358,7 +366,6 @@ static PyGetSetDef __Pyx_async_gen_getsetlist[] = { static PyMemberDef __Pyx_async_gen_memberlist[] = { //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, - {"ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, //ADDED: "ag_await" {"ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, diff --git a/contrib/tools/cython/Cython/Utility/Coroutine.c b/contrib/tools/cython/Cython/Utility/Coroutine.c index 7d65bc73fe9..a9ee81d4f38 100644 --- a/contrib/tools/cython/Cython/Utility/Coroutine.c +++ b/contrib/tools/cython/Cython/Utility/Coroutine.c @@ -1654,6 +1654,10 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( Py_XINCREF(code); gen->gi_code = code; gen->gi_frame = NULL; +#if CYTHON_USE_SYS_MONITORING && (CYTHON_PROFILE || CYTHON_TRACE) + memset(gen->$monitoring_states_cname, 0, sizeof(gen->$monitoring_states_cname)); + gen->$monitoring_version_cname = 0; +#endif PyObject_GC_Track(gen); return gen; diff --git a/contrib/tools/cython/Cython/Utility/CythonFunction.c b/contrib/tools/cython/Cython/Utility/CythonFunction.c index 38e2f1cfc75..cbb7f743197 100644 --- a/contrib/tools/cython/Cython/Utility/CythonFunction.c +++ b/contrib/tools/cython/Cython/Utility/CythonFunction.c @@ -134,6 +134,7 @@ static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject //@requires: ModuleSetupCode.c::IncludeStructmemberH //@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: ObjectHandling.c::CachedMethodType +//@requires: ObjectHandling.c::PyObjectCallOneArg //@requires: ExtensionTypes.c::CallTypeTraverse //@requires: Synchronization.c::CriticalSections //@substitute: naming @@ -461,6 +462,8 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { return result; } +static int __Pyx_CyFunction_set_annotate_in_dict_if_exists(PyObject *op_in, PyObject *value); /*proto*/ + static int __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { CYTHON_UNUSED_VAR(context); @@ -475,9 +478,59 @@ __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, vo __Pyx_BEGIN_CRITICAL_SECTION(op); __Pyx_Py_XDECREF_SET(op->func_annotations, value); __Pyx_END_CRITICAL_SECTION(); + if (unlikely(__Pyx_CyFunction_set_annotate_in_dict_if_exists((PyObject*) op, Py_None) < 0)) return -1; return 0; } +static int +__Pyx_CyFunction_get_dict_if_exists(PyObject *op_in, PyObject **dict) { + /* Return 1 if the function dict exists, 0 otherwise. This cannot fail: + * _PyObject_GetDictPtr() may clear errors internally, but never reports them. */ +#if CYTHON_COMPILING_IN_PYPY + *dict = PyObject_GenericGetDict(op_in, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030C0000 + *dict = ((__pyx_CyFunctionObject*) op_in)->func_dict; +#else + PyObject **dictptr = _PyObject_GetDictPtr(op_in); + *dict = likely(dictptr) ? *dictptr : NULL; +#endif + return *dict ? 1 : 0; +} + +static int +__Pyx_CyFunction_get_annotate_from_dict_if_exists(PyObject *op_in, PyObject **annotate) { + PyObject *dict; + int dict_found; + *annotate = NULL; + dict_found = __Pyx_CyFunction_get_dict_if_exists(op_in, &dict); + if (!dict_found) return 0; + return __Pyx_PyDict_GetItemRef(dict, PYIDENT("__annotate__"), annotate); +} + +static int +__Pyx_CyFunction_set_annotate_in_dict_if_exists(PyObject *op_in, PyObject *value) { + PyObject *dict; + int dict_found; + dict_found = __Pyx_CyFunction_get_dict_if_exists(op_in, &dict); + if (!dict_found) return 0; + return PyDict_SetItem(dict, PYIDENT("__annotate__"), value); +} + +static int +__Pyx_CyFunction_set_annotate_in_dict(PyObject *op_in, PyObject *value) { + PyObject *dict; + int result; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = __Pyx_CyFunction_get_dict((__pyx_CyFunctionObject*) op_in, NULL); +#else + dict = PyObject_GenericGetDict(op_in, NULL); +#endif + if (unlikely(!dict)) return -1; + result = PyDict_SetItem(dict, PYIDENT("__annotate__"), value); + Py_DECREF(dict); + return result; +} + static PyObject * __Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { PyObject* result = op->func_annotations; @@ -491,15 +544,100 @@ __Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { - PyObject *result; +__Pyx_CyFunction_get_annotations(PyObject *op_in, void *context) { + PyObject *annotate = NULL; + PyObject *result = NULL; + __pyx_CyFunctionObject *op = (__pyx_CyFunctionObject*) op_in; CYTHON_UNUSED_VAR(context); - __Pyx_BEGIN_CRITICAL_SECTION(op); - result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + result = __Pyx_XNewRef(op->func_annotations); + __Pyx_END_CRITICAL_SECTION(); + if (result) return result; + + if (unlikely(__Pyx_CyFunction_get_annotate_from_dict_if_exists(op_in, &annotate) < 0)) return NULL; + if (!annotate || annotate == Py_None) { + Py_XDECREF(annotate); + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + + PyObject *format = PyLong_FromLong(1L); // annotationlib.Format.VALUE + if (likely(format)) { + result = __Pyx_PyObject_CallOneArg(annotate, format); + Py_DECREF(format); + } + Py_DECREF(annotate); + if (unlikely(!result)) return NULL; + if (unlikely(!PyDict_Check(result))) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must return a dict"); + Py_DECREF(result); + return NULL; + } + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + __Pyx_Py_XDECREF_SET(op->func_annotations, __Pyx_NewRef(result)); __Pyx_END_CRITICAL_SECTION(); return result; } +static PyObject *__Pyx_CyFunction_annotate_impl(PyObject *self, PyObject *args) { + // The signature of this function is 0 or 1 args, where arg should be an int (if passed). + // However, for this basic placeholder implementation, we don't care and don't check it. + CYTHON_UNUSED_VAR(args); + if (unlikely(!self)) { + PyErr_SetString(PyExc_SystemError, "cython __annotate__ called without 'self' argument"); + } + Py_XINCREF(self); + return self; +} + +static PyMethodDef __Pyx_CyFunction_annotate_method = { + "__annotate__", + (PyCFunction)(void (*)(void))__Pyx_CyFunction_annotate_impl, + METH_VARARGS, + "Placeholder __annotate__ function to allow 'functools.wraps' to work " + "on Cython functions." +}; + +// We don't yet support PEP649 properly, so __annotate__ is +// implemented in a minimal way, just so that functools.wraps works. +static PyObject * +__Pyx_CyFunction_get_annotate(PyObject *op_in, void *context) { + PyObject *annotate = NULL; + CYTHON_UNUSED_VAR(context); + if (unlikely(__Pyx_CyFunction_get_annotate_from_dict_if_exists(op_in, &annotate) < 0)) return NULL; + if (annotate) return annotate; + + PyObject *annotations = __Pyx_CyFunction_get_annotations(op_in, NULL); + if (unlikely(!annotations)) return NULL; + PyObject *method = PyCFunction_New( + &__Pyx_CyFunction_annotate_method, + annotations); + Py_DECREF(annotations); + return method; +} + +static int +__Pyx_CyFunction_set_annotate(PyObject *op_in, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "__annotate__ cannot be deleted"); + return -1; + } + if (unlikely(value != Py_None && !PyCallable_Check(value))) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + if (value != Py_None) { + __pyx_CyFunctionObject *op = (__pyx_CyFunctionObject*) op_in; + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + Py_CLEAR(op->func_annotations); + __Pyx_END_CRITICAL_SECTION(); + } + return __Pyx_CyFunction_set_annotate_in_dict(op_in, value); +} + static PyObject * __Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; @@ -649,6 +787,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"__annotate__", (getter)__Pyx_CyFunction_get_annotate, (setter)__Pyx_CyFunction_set_annotate, 0, 0}, {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, // {"__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, #if CYTHON_COMPILING_IN_LIMITED_API diff --git a/contrib/tools/cython/Cython/Utility/ExtensionTypes.c b/contrib/tools/cython/Cython/Utility/ExtensionTypes.c index 1e89372fd71..131adb6432c 100644 --- a/contrib/tools/cython/Cython/Utility/ExtensionTypes.c +++ b/contrib/tools/cython/Cython/Utility/ExtensionTypes.c @@ -402,6 +402,34 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { #define __Pyx_TRASHCAN_END #endif +/////////////// DeallocKeepAlive.proto /////////////// + +// Keep `o` alive across a call into user __dealloc__ code, to prevent the user +// code from triggering recursive deallocation, then drop the temporary reference. +// +// On the GIL build this is a plain refcount bump. On free-threading we must NOT +// use Py_SET_REFCNT(o, 1): the object reaching tp_dealloc is unowned and merged +// (ob_ref_shared == _Py_REF_MERGED), so Py_SET_REFCNT takes the unowned branch and +// writes ob_ref_shared = _Py_REF_SHARED(1, _Py_REF_MERGED), i.e. "merged, shared +// count 1" -- indistinguishable from a live object. A concurrent +// PyUnstable_TryIncRef() would then succeed and resurrect the dying object +// (use-after-free). Instead we mirror CPython's _PyObject_ResurrectStart/End: +// re-take ownership and keep the *shared* refcount at the refuse sentinel (0), so +// the bump is invisible to other threads and TryIncRef keeps refusing. + +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_DeallocKeepAliveBegin(o) do { \ + _Py_atomic_store_uintptr_relaxed(&(o)->ob_tid, _Py_ThreadId()); \ + _Py_atomic_store_uint32_relaxed(&(o)->ob_ref_local, 1); \ + _Py_atomic_store_ssize_relaxed(&(o)->ob_ref_shared, 0); \ + } while (0) +#define __Pyx_DeallocKeepAliveEnd(o) \ + _Py_atomic_store_uint32_relaxed(&(o)->ob_ref_local, 0) +#else +#define __Pyx_DeallocKeepAliveBegin(o) __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1) +#define __Pyx_DeallocKeepAliveEnd(o) __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1) +#endif + /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); diff --git a/contrib/tools/cython/Cython/Utility/TString.c b/contrib/tools/cython/Cython/Utility/TString.c index a18cf4ac01c..e27c755d22f 100644 --- a/contrib/tools/cython/Cython/Utility/TString.c +++ b/contrib/tools/cython/Cython/Utility/TString.c @@ -301,8 +301,8 @@ static PyObject* __Pyx_MakeTemplateLibTemplate(PyObject *strings, PyObject *inte if (unlikely(__Pyx_VectorcallBuilder_AddArg(PYIDENT("interpolations"), interpolations, kwargs_builder, args, 1)<0)) goto failed_shortcut; result = __Pyx_Object_Vectorcall_CallFromBuilder(tp, args, 0, kwargs_builder); - Py_DECREF(kwargs_builder); if (result) { + Py_DECREF(kwargs_builder); Py_DECREF(tp); return result; } diff --git a/contrib/tools/cython/cython.py b/contrib/tools/cython/cython.py index 6f4880a93cc..159014bd193 100755 --- a/contrib/tools/cython/cython.py +++ b/contrib/tools/cython/cython.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Change content of this file to change uids for cython programs - cython 3.2.5 r0 +# Change content of this file to change uids for cython programs - cython 3.2.8 r0 # # Cython -- Main Program, generic diff --git a/contrib/tools/cython/ya.make b/contrib/tools/cython/ya.make index 61e66dbe8d3..fd08615635d 100644 --- a/contrib/tools/cython/ya.make +++ b/contrib/tools/cython/ya.make @@ -11,9 +11,9 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:python-contrib) -VERSION(3.2.5) +VERSION(3.2.8) -ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.2.5.tar.gz) +ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.2.8.tar.gz) NO_LINT() diff --git a/library/cpp/unified_agent_client/client_impl.cpp b/library/cpp/unified_agent_client/client_impl.cpp index ffe16d8da37..dc4241344ff 100644 --- a/library/cpp/unified_agent_client/client_impl.cpp +++ b/library/cpp/unified_agent_client/client_impl.cpp @@ -8,6 +8,7 @@ #include <contrib/libs/grpc/src/core/lib/iomgr/executor.h> #include <util/charset/utf8.h> +#include <util/generic/scope.h> #include <util/generic/size_literals.h> #include <util/system/env.h> @@ -513,14 +514,22 @@ namespace NUnifiedAgent::NPrivate { } void TClientSession::CheckGrpcCallInactivity() { - with_lock(Lock) { - if (Closed || !Started) { - return; - } - const auto timeout = Client->GetParameters().GrpcCallInactivityTimeout; - if (timeout == TDuration::Zero()) { - return; + if (!Lock.TryAcquire()) { + TSpinWait sw; + + while (Lock.IsLocked() || !Lock.TryAcquire()) { + if (ForkInProgress.load()) { + return; + } + sw.Sleep(); } + } + Y_DEFER { + Lock.Release(); + }; + + const auto timeout = Client->GetParameters().GrpcCallInactivityTimeout; + if (!Closed && Started && timeout != TDuration::Zero()) { if (NegotiatedProtocol.Defined() && *NegotiatedProtocol > 0 && ActiveGrpcCall && !CloseStarted && diff --git a/util/string/strip_ut.cpp b/util/string/strip_ut.cpp index 1d47e21e23c..7236ca2ebf8 100644 --- a/util/string/strip_ut.cpp +++ b/util/string/strip_ut.cpp @@ -8,19 +8,19 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { Y_UNIT_TEST(TestConstexprStrip) { - static constexpr std::string_view stripped = StripString<std::string_view>(" \n\t abc def \r\n "); + constexpr std::string_view stripped = StripString<std::string_view>(" \n\t abc def \r\n "); static_assert(stripped == "abc def"); - static constexpr std::string_view strippedLeft = StripStringLeft(std::string_view(" \n abc \n ")); + constexpr std::string_view strippedLeft = StripStringLeft(std::string_view(" \n abc \n ")); static_assert(strippedLeft == "abc \n "); - static constexpr std::string_view strippedRight = StripStringRight(std::string_view(" \n abc \n ")); + constexpr std::string_view strippedRight = StripStringRight(std::string_view(" \n abc \n ")); static_assert(strippedRight == " \n abc"); - static constexpr std::string_view empty = StripString<std::string_view>(" \n\t\r "); + constexpr std::string_view empty = StripString<std::string_view>(" \n\t\r "); static_assert(empty.empty()); - static constexpr std::string_view unchanged = StripString<std::string_view>("abc"); + constexpr std::string_view unchanged = StripString<std::string_view>("abc"); static_assert(unchanged == "abc"); UNIT_ASSERT_VALUES_EQUAL(stripped, "abc def"); @@ -29,13 +29,13 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { } Y_UNIT_TEST(TestConstexprCustomStrip) { - static constexpr std::string_view stripped = StripString(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view stripped = StripString(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(stripped == "abc"); - static constexpr std::string_view strippedLeft = StripStringLeft(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view strippedLeft = StripStringLeft(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(strippedLeft == "abc//"); - static constexpr std::string_view strippedRight = StripStringRight(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view strippedRight = StripStringRight(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(strippedRight == "//abc"); UNIT_ASSERT_VALUES_EQUAL(stripped, "abc"); @@ -47,7 +47,7 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { TStringBuf StripRightRes; TStringBuf StripRes; }; - static constexpr TStripTest StripTests[] = { + constexpr TStripTest StripTests[] = { {" 012 ", "012 ", " 012", "012"}, {" 012", "012", " 012", "012"}, {"012\t\t", "012\t\t", "012", "012"}, diff --git a/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go b/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go index 28cd99c7f3f..b332122033b 100644 --- a/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go +++ b/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go @@ -2,24 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* -Package pbkdf2 implements the key derivation function PBKDF2 as defined in RFC -2898 / PKCS #5 v2.0. - -A key derivation function is useful when encrypting data based on a password -or any other not-fully-random data. It uses a pseudorandom function to derive -a secure encryption key based on the password. - -While v2.0 of the standard defines only one pseudorandom function to use, -HMAC-SHA1, the drafted v2.1 specification allows use of all five FIPS Approved -Hash Functions SHA-1, SHA-224, SHA-256, SHA-384 and SHA-512 for HMAC. To -choose, you can pass the `New` functions from the different SHA packages to -pbkdf2.Key. -*/ +// Package pbkdf2 implements the key derivation function PBKDF2 as defined in +// RFC 8018 (PKCS #5 v2.1). +// +// This package is a wrapper for the PBKDF2 implementation in the +// [crypto/pbkdf2] package. It is [frozen] and is not accepting new features. +// +// [frozen]: https://go.dev/wiki/Frozen package pbkdf2 import ( - "crypto/hmac" + "crypto/pbkdf2" "hash" ) @@ -27,51 +20,11 @@ import ( // []byte of length keylen that can be used as cryptographic key. The key is // derived based on the method described as PBKDF2 with the HMAC variant using // the supplied hash function. -// -// For example, to use a HMAC-SHA-1 based PBKDF2 key derivation function, you -// can get a derived key for e.g. AES-256 (which needs a 32-byte key) by -// doing: -// -// dk := pbkdf2.Key([]byte("some password"), salt, 4096, 32, sha1.New) -// -// Remember to get a good random salt. At least 8 bytes is recommended by the -// RFC. -// -// Using a higher iteration count will increase the cost of an exhaustive -// search but will also make derivation proportionally slower. func Key(password, salt []byte, iter, keyLen int, h func() hash.Hash) []byte { - prf := hmac.New(h, password) - hashLen := prf.Size() - numBlocks := (keyLen + hashLen - 1) / hashLen - - var buf [4]byte - dk := make([]byte, 0, numBlocks*hashLen) - U := make([]byte, hashLen) - for block := 1; block <= numBlocks; block++ { - // N.B.: || means concatenation, ^ means XOR - // for each block T_i = U_1 ^ U_2 ^ ... ^ U_iter - // U_1 = PRF(password, salt || uint(i)) - prf.Reset() - prf.Write(salt) - buf[0] = byte(block >> 24) - buf[1] = byte(block >> 16) - buf[2] = byte(block >> 8) - buf[3] = byte(block) - prf.Write(buf[:4]) - dk = prf.Sum(dk) - T := dk[len(dk)-hashLen:] - copy(U, T) - - // U_n = PRF(password, U_(n-1)) - for n := 2; n <= iter; n++ { - prf.Reset() - prf.Write(U) - U = U[:0] - U = prf.Sum(U) - for x := range U { - T[x] ^= U[x] - } - } + out, err := pbkdf2.Key(h, string(password), salt, iter, keyLen) + if err != nil { + // FIPS 140 enforcement, or an invalid key length. + panic(err) } - return dk[:keyLen] + return out } diff --git a/vendor/golang.org/x/crypto/pbkdf2/ya.make b/vendor/golang.org/x/crypto/pbkdf2/ya.make index 93efe6e8d08..bf485573593 100644 --- a/vendor/golang.org/x/crypto/pbkdf2/ya.make +++ b/vendor/golang.org/x/crypto/pbkdf2/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.49.0) +VERSION(v0.53.0) SRCS( pbkdf2.go diff --git a/vendor/golang.org/x/crypto/scrypt/ya.make b/vendor/golang.org/x/crypto/scrypt/ya.make index e5ef97d77b2..aab6270377c 100644 --- a/vendor/golang.org/x/crypto/scrypt/ya.make +++ b/vendor/golang.org/x/crypto/scrypt/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.49.0) +VERSION(v0.53.0) SRCS( scrypt.go @@ -46,33 +46,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "win32": { - "md5": "9dee86473a0e4b0c0a0d963f419dbf33", + "md5": "91a336f13e306b42992ec43c721762ac", "urls": [ - f"{REGISTRY_ENDPOINT}/12807581301" + f"{REGISTRY_ENDPOINT}/12845376268" ] }, "darwin": { - "md5": "e5067a1d83c2dd418be0faf32c2ab354", + "md5": "7113969185c3d148b03d485d7a28b44e", "urls": [ - f"{REGISTRY_ENDPOINT}/12807578337" + f"{REGISTRY_ENDPOINT}/12845375549" ] }, "darwin-arm64": { - "md5": "732315791641f6d8dec06aad3ac70427", + "md5": "55cc64b49e2fbf9a6075661ff9d1285e", "urls": [ - f"{REGISTRY_ENDPOINT}/12807575686" + f"{REGISTRY_ENDPOINT}/12845374717" ] }, "linux-aarch64": { - "md5": "9ac4edc8c328aff5bbb941668634ab70", + "md5": "dc46e90abbe7bf7c29a460ac5d09fd54", "urls": [ - f"{REGISTRY_ENDPOINT}/12807572366" + f"{REGISTRY_ENDPOINT}/12845373877" ] }, "linux": { - "md5": "01654ceac32bcf11ad928ce31db2cc42", + "md5": "c87e561b968477340705066c5b22370b", "urls": [ - f"{REGISTRY_ENDPOINT}/12807584185" + f"{REGISTRY_ENDPOINT}/12845377052" ] } } diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 750fee9746d..9a6f13caeab 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -201,6 +201,7 @@ struct TKikimrEvents : TEvents { ES_SET_COLUMN_CONSTRAINT = 4278, ES_EXTERNAL_IDP_PROVIDER = 4279, ES_PQ_DEFERRED_PUBLISH = 4280, + ES_WORKLOAD_MANAGER = 4281, }; }; diff --git a/ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h b/ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h new file mode 100644 index 00000000000..858cb89bd4a --- /dev/null +++ b/ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h @@ -0,0 +1,61 @@ +#pragma once + +#include <ydb/core/protos/blobstorage_config.pb.h> + +#include <util/generic/string.h> +#include <util/string/builder.h> + +#include <optional> + +namespace NKikimr { + +inline std::optional<TString> ValidateInferPDiskSlotCountSettingsForDriveType( + const NKikimrBlobStorage::TInferPDiskSlotCountForDriveTypeSettings& settings, + TStringBuf context) { + const bool hasSlotSize = settings.HasSlotSize() && settings.GetSlotSize(); + const bool hasUnitSize = settings.HasUnitSize() && settings.GetUnitSize(); + const bool hasMaxSlots = settings.HasMaxSlots() && settings.GetMaxSlots(); + if (hasSlotSize && hasUnitSize) { + return TStringBuilder() << context + << " SlotSize is mutually exclusive with UnitSize" + << " in InferPDiskSlotCountSettings"; + } + + if ((hasSlotSize || hasUnitSize) && !hasMaxSlots) { + return TStringBuilder() << context + << " MaxSlots is mandatory with SlotSize or UnitSize" + << " in InferPDiskSlotCountSettings"; + } + + if (hasMaxSlots && !hasSlotSize && !hasUnitSize) { + return TStringBuilder() << context + << " MaxSlots requires SlotSize or UnitSize" + << " in InferPDiskSlotCountSettings"; + } + + return {}; +} + +inline std::optional<TString> ValidateInferPDiskSlotCountSettings( + const NKikimrBlobStorage::TInferPDiskSlotCountSettings& settings, + TStringBuf context) { + if (settings.HasRot()) { + const TString rotContext = TStringBuilder() << context << ".Rot"; + if (auto error = ValidateInferPDiskSlotCountSettingsForDriveType( + settings.GetRot(), rotContext)) { + return error; + } + } + + if (settings.HasSsd()) { + const TString ssdContext = TStringBuilder() << context << ".Ssd"; + if (auto error = ValidateInferPDiskSlotCountSettingsForDriveType( + settings.GetSsd(), ssdContext)) { + return error; + } + } + + return {}; +} + +} // NKikimr diff --git a/ydb/core/blobstorage/base/pdisk_config_validation.h b/ydb/core/blobstorage/base/pdisk_config_validation.h new file mode 100644 index 00000000000..41f58be7b8c --- /dev/null +++ b/ydb/core/blobstorage/base/pdisk_config_validation.h @@ -0,0 +1,41 @@ +#pragma once + +#include <ydb/core/protos/blobstorage_pdisk_config.pb.h> + +#include <util/generic/string.h> +#include <util/string/builder.h> + +#include <optional> + +namespace NKikimr { + +inline std::optional<TString> ValidatePDiskConfig( + const NKikimrBlobStorage::TPDiskConfig& config, + TStringBuf context) { + const bool hasExpectedSlotCount = config.HasExpectedSlotCount() && config.GetExpectedSlotCount(); + const bool hasSlotSizeInUnits = config.HasSlotSizeInUnits() && config.GetSlotSizeInUnits(); + const bool hasExpectedSlotSize = config.HasExpectedSlotSize() && config.GetExpectedSlotSize(); + const bool hasMaxSlots = config.HasMaxSlots() && config.GetMaxSlots(); + + if (hasExpectedSlotSize && (hasExpectedSlotCount || hasSlotSizeInUnits)) { + return TStringBuilder() << context + << " ExpectedSlotSize is mutually exclusive with ExpectedSlotCount and SlotSizeInUnits" + << " in PDiskConfig"; + } + + if (hasExpectedSlotSize && !hasMaxSlots) { + return TStringBuilder() << context + << " ExpectedSlotSize requires MaxSlots" + << " in PDiskConfig"; + } + + if (hasMaxSlots && !hasExpectedSlotSize) { + return TStringBuilder() << context + << " MaxSlots requires ExpectedSlotSize" + << " in PDiskConfig"; + } + + return {}; +} + +} // namespace NKikimr diff --git a/ydb/core/blobstorage/base/ya.make b/ydb/core/blobstorage/base/ya.make index b7fc88bd39d..859f182c16e 100644 --- a/ydb/core/blobstorage/base/ya.make +++ b/ydb/core/blobstorage/base/ya.make @@ -13,6 +13,7 @@ SRCS( blobstorage_events.cpp blobstorage_events.h blobstorage_host_record.h + infer_pdisk_slot_count_settings.h blobstorage_oos_defs.h blobstorage_vdiskid.cpp blobstorage_vdiskid.h diff --git a/ydb/core/blobstorage/ddisk/ddisk.cpp b/ydb/core/blobstorage/ddisk/ddisk.cpp index 1e06aa6877f..de3e909476c 100644 --- a/ydb/core/blobstorage/ddisk/ddisk.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk.cpp @@ -1,10 +1,5 @@ #include "ddisk.h" -#include <util/generic/utility.h> - -#define XXH_INLINE_ALL -#include <contrib/libs/xxhash/xxhash.h> - namespace NKikimr::NDDisk { namespace { @@ -33,45 +28,6 @@ ui32 AddPayloadThenChecksumImpl(TEvent& ev, TRope&& rope) { } // anonymous -ui64 CalculateBlockChecksum(TRope::TConstIterator it, size_t numBytes) { - Y_ABORT_UNLESS(numBytes > 0); - Y_ABORT_UNLESS((numBytes & (MinSectorSize - 1)) == 0); - - // Fast path: the block is fully contiguous (always true for TEvWrite payloads, and common for PB - // ones), so a single one-shot hash call avoids the XXH3 streaming state overhead. - if (it.Valid() && it.ContiguousSize() >= numBytes) { - return XXH3_64bits(it.ContiguousData(), numBytes); - } - - XXH3_state_t state; - XXH3_64bits_reset(&state); - for (; numBytes && it.Valid(); it.AdvanceToNextContiguousBlock()) { - const size_t n = Min(numBytes, it.ContiguousSize()); - XXH3_64bits_update(&state, it.ContiguousData(), n); - numBytes -= n; - } - // The iterator must not run out before numBytes bytes are consumed: otherwise the digest would - // silently cover fewer bytes than the caller asked for, which is worse than crashing. - Y_ABORT_UNLESS(numBytes == 0); - return XXH3_64bits_digest(&state); -} - -std::vector<ui64> CalculatePayloadChecksums(const TRope& payload) { - Y_ABORT_UNLESS(payload.size() > 0); - Y_ABORT_UNLESS((payload.size() & (MinSectorSize - 1)) == 0); - - std::vector<ui64> checksums; - checksums.reserve(payload.size() / MinSectorSize); - - auto it = payload.Begin(); - for (size_t offset = 0; offset < payload.size(); offset += MinSectorSize) { - checksums.push_back(CalculateBlockChecksum(it, MinSectorSize)); - it += MinSectorSize; - } - - return checksums; -} - ui32 TEvWrite::AddPayloadThenChecksum(TRope&& rope) { return AddPayloadThenChecksumImpl(*this, std::move(rope)); } diff --git a/ydb/core/blobstorage/ddisk/ddisk.h b/ydb/core/blobstorage/ddisk/ddisk.h index f55dcc5c91d..3521d7ae841 100644 --- a/ydb/core/blobstorage/ddisk/ddisk.h +++ b/ydb/core/blobstorage/ddisk/ddisk.h @@ -2,6 +2,7 @@ #include "defs.h" #include "ddisk_config.h" +#include "ddisk_checksums.h" #include <ydb/core/base/events.h> @@ -18,16 +19,7 @@ namespace NKikimr::NDDisk { constexpr size_t MinSectorSize = 4096; constexpr size_t DataAlignment = MinSectorSize; - // Computes an XXH3-64 checksum over exactly numBytes bytes starting at it. The iterator is passed by - // value, so the caller's own iterator is not advanced. This is a raw data checksum with no identity - // or salt mixed in (unlike TDDiskActor::CalculateChecksum, which additionally mixes in - // PersistentBufferUniqueId for on-disk PB sector integrity - a value senders cannot know, so it - // cannot be used for a sender-computed, wire-level checksum like this one). - ui64 CalculateBlockChecksum(TRope::TConstIterator it, size_t numBytes); - - // Splits payload into MinSectorSize (4 KiB) blocks and computes a checksum for each block, in order. - // payload.size() must be a non-zero multiple of MinSectorSize. - std::vector<ui64> CalculatePayloadChecksums(const TRope& payload); + static_assert(MinSectorSize == IntegrityUnitSize); struct TEv { enum { @@ -639,7 +631,7 @@ struct TPersistentBufferFormat { TEvReadPersistentBufferResult(NKikimrBlobStorage::NDDisk::TReplyStatus::E status, const std::optional<TString>& errorReason = std::nullopt, ui64 vChunkIndex = 0, ui32 offsetInBytes = 0, ui32 sizeInBytes = 0, - TRope data = {}) { + TRope data = {}, const std::vector<ui64>& checksums = {}) { Record.SetStatus(status); if (errorReason) { Record.SetErrorReason(*errorReason); @@ -649,6 +641,12 @@ struct TPersistentBufferFormat { Record.SetOffsetInBytes(offsetInBytes); Record.SetSizeInBytes(sizeInBytes); TReadResult(AddPayload(std::move(data))).Serialize(Record.MutableReadResult()); + // Opt-in, mirrors TEvWritePersistentBuffer.Checksums: only attached when the persisted + // record actually carries sender-supplied payload checksums (see + // TPersistentBuffer::TRecord::PayloadChecksums). + for (ui64 checksum : checksums) { + Record.AddChecksums(checksum); + } } } }; diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor.h b/ydb/core/blobstorage/ddisk/ddisk_actor.h index e780c0f362b..f3d0606c8b6 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor.h +++ b/ydb/core/blobstorage/ddisk/ddisk_actor.h @@ -753,6 +753,9 @@ namespace NKikimr::NDDisk { std::map<ui64, TRope> DataParts; ui32 PartsCount; std::vector<TPersistentBufferSectorInfo> Sectors; + // Sender-supplied per-MinSectorSize-block payload checksums for this record, in order. + // Empty when the write carried no checksums. See TPersistentBuffer::TRecord::PayloadChecksums. + std::vector<ui64> PayloadChecksums; TRope JoinData(ui32 sectorSize); }; @@ -813,7 +816,7 @@ namespace NKikimr::NDDisk { void IssuePersistentBufferChunkAllocation(); void ProcessDeallocatePersistentBufferChunk(bool forceToNextChunk = false); void ProcessPersistentBufferQueue(); - std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors); + std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, const std::vector<ui64>& payloadChecksums); std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBufferData(TRope& data, std::vector<TPersistentBufferSectorInfo>& sectors); void StartRestorePersistentBuffer(); void RestorePersistentBufferChunk(TEvPrivate::TEvReadPersistentBufferPart::TPtr ev); @@ -822,7 +825,12 @@ namespace NKikimr::NDDisk { bool PreprocessPersistentBufferWrite(NActors::TEventHandle<TEvWritePersistentBuffer>& ev); void ProcessPersistentBufferWrite(TEvWritePersistentBuffer::TPtr ev); - bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev); + // ev is taken by reference (not TPtr by value, unlike its sibling above): TPtr is a TAutoPtr + // with ownership-transferring copy semantics, so a by-value parameter here would null out the + // caller's ev as soon as this is invoked -- including on the "doesn't fit, fall back" (false) + // return path, where Handle(TEvWritePersistentBuffer) still needs a valid ev afterwards to retry + // via ProcessPersistentBufferWrite. + bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev); void ProcessPersistentBufferBatchWrite(); double GetPersistentBufferFreeSpace(); void ErasePersistentBuffer(IEventHandle& queryEv, const TQueryCredentials& creds, const std::vector<TEraseLsnId>& erases); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp index 611a01228a1..3660a33cf16 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp @@ -15,6 +15,20 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxLsnsPerPack = (DataAlignment - sizeof(TPersistentBufferHeader)) / (sizeof(TPersistentBufferLsnRecordHeader) + TPersistentBufferLsnRecordHeader::MaxSectorsPerPackBufferRecord * sizeof(TPersistentBufferSectorInfo)); + // Size, in bytes, of one LSN record's metadata block within the shared 4 KiB batch header sector: + // TPersistentBufferLsnRecordHeader + its TPersistentBufferSectorInfo[] location array, plus an + // optional trailing ui64 PayloadChecksums[] (one entry per data sector) when the record carries + // sender-supplied checksums. Used both to lay out the header sector (SlicePersistentBuffer / + // ProcessPersistentBufferBatchWrite) and to decide whether one more record still fits in an + // already-open batch (ProcessPersistentBufferBatchWriteData) - MaxLsnsPerPack above only bounds + // the pre-allocated data-sector budget and is deliberately not tightened for checksums, since + // checksums add header-sector bytes, not data sectors. + static ui32 LsnRecordMetadataSize(ui32 dataSectorsCnt, bool hasPayloadChecksums) { + return sizeof(TPersistentBufferLsnRecordHeader) + + dataSectorsCnt * sizeof(TPersistentBufferSectorInfo) + + (hasPayloadChecksums ? dataSectorsCnt * sizeof(ui64) : 0); + } + void TDDiskActor::IssuePersistentBufferChunkAllocation() { Y_ABORT_UNLESS(IsPersistentBufferActor); if (!IssuePersistentBufferChunkAllocationInflight) { @@ -229,7 +243,8 @@ namespace NKikimr::NDDisk { std::vector<std::tuple<ui32, ui32, TRope>> TDDiskActor::SlicePersistentBuffer( ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 sizeInBytes, - TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors) + TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, + const std::vector<ui64>& payloadChecksums) { TRope fullData(std::move(payloadWithHeader)); @@ -263,7 +278,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -281,6 +295,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = offsetInBytes; lsnRecordHeader->Size = sizeInBytes; lsnRecordHeader->Lsn = lsn; + const bool hasPayloadChecksums = !payloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + header->Version = hasPayloadChecksums ? 1 : 0; auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(fullData.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader) @@ -295,6 +312,16 @@ namespace NKikimr::NDDisk { sectors[i].Checksum = loc.Checksum; } + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(payloadChecksums.size() == sectors.size() - 1); + auto* checksums = reinterpret_cast<ui64*>(fullData.Begin().UnsafeContiguousDataMut() + + sizeof(TPersistentBufferHeader) + + sizeof(TPersistentBufferLsnRecordHeader) + + (sectors.size() - 1) * sizeof(TPersistentBufferSectorInfo) + ); + memcpy(checksums, payloadChecksums.data(), payloadChecksums.size() * sizeof(ui64)); + } + header->Checksum = CalculateChecksum(fullData.Begin(), SectorSize); // Slice fullData ([header | payload], a single contiguous, SectorSize-aligned buffer) into one part @@ -587,6 +614,12 @@ namespace NKikimr::NDDisk { pr.Sectors.push_back(locations[i]); } pos += sectorsCnt * sizeof(TPersistentBufferSectorInfo); + + if (recordHeader->Flags & TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + const ui64* checksums = reinterpret_cast<const ui64*>(pos); + pr.PayloadChecksums.assign(checksums, checksums + sectorsCnt); + pos += sectorsCnt * sizeof(ui64); + } } } else { PersistentBufferSectorsChecksum[chunkIdx][sectorIdx] = CalculateChecksum(dataPos); @@ -721,7 +754,8 @@ namespace NKikimr::NDDisk { .Size = (ui32)record.Size, .Sectors = std::move(record.Sectors), .VChunkIndex = record.VChunkIndex, - .Timestamp = TInstant::Now() + .Timestamp = TInstant::Now(), + .PayloadChecksums = std::move(record.PayloadChecksums), }; auto& pbh = PersistentBufferHeaders[{pr.Sectors[0].ChunkIdx, pr.Sectors[0].SectorIdx}]; @@ -898,6 +932,7 @@ namespace NKikimr::NDDisk { if (instr.PayloadId) { payload = ev->Get()->GetPayload(*instr.PayloadId); } + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferWrite", {"marker", "BSPB"}, {"PBufferId", SelfId()}, @@ -959,7 +994,8 @@ namespace NKikimr::NDDisk { TRcBuf payloadWithHeader = ev->Get()->GetPayloadWithHeader(*instr.PayloadId); auto parts = SlicePersistentBuffer(creds.TabletId, creds.Generation, - selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors); + selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors, + payloadChecksums); auto opCookie = NextCookie++; auto& inflightRecord = PersistentBufferDiskOperationInflight[opCookie]; @@ -982,6 +1018,9 @@ namespace NKikimr::NDDisk { .DataParts = {{0, std::move(payload)}}, .PartsCount = 1, .Sectors = std::move(sectors), + // payloadChecksums was only read (never mutated) above by SlicePersistentBuffer; safe to + // move into the inflight record now that this is its last use. + .PayloadChecksums = std::move(payloadChecksums), }); PersistentBufferWriteInflightsByRecord[TPersistentBufferRecordId{creds.TabletId, creds.Generation, lsn}].emplace_back( opCookie, @@ -1008,7 +1047,7 @@ namespace NKikimr::NDDisk { } } - bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev) { + bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev) { const auto& record = ev->Get()->Record; const TQueryCredentials creds(record.GetCredentials()); const TBlockSelector selector(record.GetSelector()); @@ -1020,6 +1059,9 @@ namespace NKikimr::NDDisk { const TWriteInstruction instr(record.GetInstruction()); Y_ABORT_UNLESS(instr.PayloadId, "WritePersistentBuffer without a payload"); TRope payload = ev->Get()->GetPayload(*instr.PayloadId); + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); + const ui32 sectorsCnt = selector.Size / SectorSize; + Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferBatchWriteData", {"marker", "BSPB"}, @@ -1043,6 +1085,22 @@ namespace NKikimr::NDDisk { }); Y_ABORT_UNLESS(inserted); Schedule(TDuration::MicroSeconds(PersistentBufferFormat.WritesBatchingPeriodMicroseconds), new TEvents::TEvWakeup(EWakeupTag::WakeupProcessPersistentBufferBatchWrite)); + } else { + // Reusing an already-open batch: this record's metadata (TPersistentBufferLsnRecordHeader + + // sector locations + optional payload checksums) must still fit in the shared 4 KiB header + // sector alongside every record already queued. A brand-new batch (handled above) always + // fits a single record trivially, so this check only matters here. Bail out (fall back to + // the direct, non-batched write path in the caller) rather than overflow the header sector. + const auto it = PersistentBufferDiskOperationInflight.find(PersistentBufferBatchWriteCookie); + Y_ABORT_UNLESS(it != PersistentBufferDiskOperationInflight.end()); + const auto& existingRecords = it->second.Records; + ui32 usedBytes = sizeof(TPersistentBufferHeader); + for (auto& r : existingRecords) { + usedBytes += LsnRecordMetadataSize(r.Sectors.size() - 1, !r.PayloadChecksums.empty()); + } + if (usedBytes + LsnRecordMetadataSize(sectorsCnt, !payloadChecksums.empty()) > SectorSize) { + return false; + } } auto& inflight = PersistentBufferDiskOperationInflight[PersistentBufferBatchWriteCookie]; @@ -1071,11 +1129,10 @@ namespace NKikimr::NDDisk { .Size = selector.Size, .DataParts = {{0, payload}}, .PartsCount = 1, + .PayloadChecksums = std::move(payloadChecksums), }); auto& r = inflight.Records.back(); - ui32 sectorsCnt = selector.Size / SectorSize; - Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); r.Sectors.push_back(inflight.OccupiedSectors[0]); auto sectorsIt = inflight.OccupiedSectors.begin() + inflight.DataToWrite.size() / SectorSize; @@ -1119,7 +1176,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -1129,6 +1185,7 @@ namespace NKikimr::NDDisk { header->Flags = 0; header->BatchSize = inflight.Records.size(); + bool anyPayloadChecksums = false; auto* pos = inflight.DataToWrite.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader); for (auto& record : inflight.Records) { auto* lsnRecordHeader = reinterpret_cast<TPersistentBufferLsnRecordHeader*>(pos); @@ -1139,6 +1196,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = record.OffsetInBytes; lsnRecordHeader->Size = record.Size; lsnRecordHeader->Lsn = record.Lsn; + const bool hasPayloadChecksums = !record.PayloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + anyPayloadChecksums = anyPayloadChecksums || hasPayloadChecksums; pos += sizeof(TPersistentBufferLsnRecordHeader); auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(pos); @@ -1148,7 +1208,16 @@ namespace NKikimr::NDDisk { loc = record.Sectors[i]; } pos += sizeof(TPersistentBufferSectorInfo) * (record.Sectors.size() - 1); + + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(record.PayloadChecksums.size() == record.Sectors.size() - 1); + memcpy(pos, record.PayloadChecksums.data(), record.PayloadChecksums.size() * sizeof(ui64)); + pos += record.PayloadChecksums.size() * sizeof(ui64); + } } + Y_ABORT_UNLESS(static_cast<size_t>(pos - inflight.DataToWrite.Begin().UnsafeContiguousDataMut()) <= SectorSize, + "persistent buffer batch header overflow"); + header->Version = anyPayloadChecksums ? 1 : 0; header->Checksum = CalculateChecksum(inflight.DataToWrite.Begin(), SectorSize); auto parts = SlicePersistentBufferData(inflight.DataToWrite, inflight.OccupiedSectors); @@ -1407,11 +1476,23 @@ namespace NKikimr::NDDisk { Y_ABORT_UNLESS(inflight.Records.size() == 1); auto& inflightRecord = inflight.Records[0]; TRope data; + std::vector<ui64> checksums; if (pr.Data) { data = std::move(TrimData(pr.Data, pr.OffsetInBytes, pr.Size, inflightRecord.OffsetInBytes, inflightRecord.Size)); + if (!pr.PayloadChecksums.empty()) { + // Persisted checksums cover [pr.OffsetInBytes, pr.OffsetInBytes + pr.Size) one entry + // per MinSectorSize block, same order as the trimmed data above - slice out the + // sub-range the selector actually asked for. Returned as-is (never recomputed from + // on-disk bytes), matching the write path's opt-in, pre-signature-correction values. + Y_ABORT_UNLESS(pr.PayloadChecksums.size() == pr.Size / SectorSize); + const ui32 firstBlock = (inflightRecord.OffsetInBytes - pr.OffsetInBytes) / SectorSize; + const ui32 blockCount = inflightRecord.Size / SectorSize; + Y_ABORT_UNLESS(firstBlock + blockCount <= pr.PayloadChecksums.size()); + checksums.assign(pr.PayloadChecksums.begin() + firstBlock, pr.PayloadChecksums.begin() + firstBlock + blockCount); + } } auto replyEv = std::make_unique<TEvReadPersistentBufferResult>(status, errorMessage, - pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data)); + pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data), checksums); auto h = std::make_unique<IEventHandle>(inflightRecord.Sender, SelfId(), replyEv.release(), 0, inflightRecord.Cookie); if (inflightRecord.Session) { h->Rewrite(TEvInterconnect::EvForward, inflightRecord.Session); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp index a1db18c0ae6..0c4c1071914 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp @@ -89,13 +89,14 @@ namespace NKikimr::NDDisk { } } + // Unlike TEvWritePersistentBuffer, TEvWrite's checksums (if any) are validated here but never + // persisted: TEvWrite goes straight to the final DDisk chunk, which has no per-record metadata + // slot to carry them (see RFC 006). Note this validation intentionally runs again for events + // that get parked in PendingEventsForChunk below and re-dispatched to this same handler once + // their chunk allocation completes -- harmless (checksums don't change), just redundant. if (record.ChecksumsSize() > 0) { - // Checksum validation is opt-in (see TEvWritePersistentBuffer for details); checked here, - // before any chunk allocation queueing or disk I/O, and the checksums are not persisted. - // Note: an event parked in PendingEventsForChunk below re-enters this handler in full once - // the chunk is allocated, so its checksums get validated twice. This is harmless (validation - // stays right next to the actual write) and intentional, not an oversight. Y_ABORT_UNLESS(instr.PayloadId, "TEvWrite without a payload, but with checksums"); + const TRope& payload = ev->Get()->GetPayload(*instr.PayloadId); if (const auto result = ValidatePayloadChecksums(record, payload)) { const bool isCorrupted = result->Status == NKikimrBlobStorage::NDDisk::TReplyStatus::CORRUPTED; diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp index b3194ec154f..ec1ac876e97 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp @@ -199,7 +199,8 @@ namespace NKikimr::NDDisk { } } - Y_DEBUG_ABORT_UNLESS(SyncReadCookiesInFlight.emplace(requestId).second); + const bool inserted = SyncReadCookiesInFlight.emplace(requestId).second; + Y_DEBUG_ABORT_UNLESS(inserted); const TActorId& sourceActorId = fromPersistentBuffer ? sourceInfo.PersistentBufferActorId : sourceInfo.DDiskActorId; diff --git a/ydb/core/blobstorage/ddisk/ddisk_checksums.cpp b/ydb/core/blobstorage/ddisk/ddisk_checksums.cpp new file mode 100644 index 00000000000..673bda5e419 --- /dev/null +++ b/ydb/core/blobstorage/ddisk/ddisk_checksums.cpp @@ -0,0 +1,59 @@ +#include "ddisk_checksums.h" + +#include <util/generic/utility.h> + +#define XXH_INLINE_ALL +#include <contrib/libs/xxhash/xxhash.h> + +namespace NKikimr::NDDisk { + +ui64 CalculateBlockChecksum(TRope::TConstIterator it, size_t numBytes) { + Y_ABORT_UNLESS(numBytes > 0); + Y_ABORT_UNLESS((numBytes & (IntegrityUnitSize - 1)) == 0); + + // Fast path: the block is fully contiguous (always true for TEvWrite payloads, and common for PB + // ones), so a single one-shot hash call avoids the XXH3 streaming state overhead. + if (it.Valid() && it.ContiguousSize() >= numBytes) { + return XXH3_64bits(it.ContiguousData(), numBytes); + } + + XXH3_state_t state; + XXH3_64bits_reset(&state); + for (; numBytes && it.Valid(); it.AdvanceToNextContiguousBlock()) { + const size_t n = Min(numBytes, it.ContiguousSize()); + XXH3_64bits_update(&state, it.ContiguousData(), n); + numBytes -= n; + } + // The iterator must not run out before numBytes bytes are consumed: otherwise the digest would + // silently cover fewer bytes than the caller asked for, which is worse than crashing. + Y_ABORT_UNLESS(numBytes == 0); + return XXH3_64bits_digest(&state); +} + +std::vector<ui64> CalculatePayloadChecksums(const TRope& payload) { + Y_ABORT_UNLESS(payload.size() > 0); + Y_ABORT_UNLESS((payload.size() & (IntegrityUnitSize - 1)) == 0); + + std::vector<ui64> checksums; + checksums.reserve(payload.size() / IntegrityUnitSize); + + auto it = payload.Begin(); + for (size_t offset = 0; offset < payload.size(); offset += IntegrityUnitSize) { + checksums.push_back(CalculateBlockChecksum(it, IntegrityUnitSize)); + it += IntegrityUnitSize; + } + + return checksums; +} + +ui64 Contribution(ui64 vchunkGeneration, ui64 blockIdx, ui64 blockChecksum) { + ui64 parts[3] = {vchunkGeneration, blockIdx, blockChecksum}; + return XXH3_64bits(parts, sizeof(parts)); +} + +void UpdateRoot(ui64& integrityBlockDigest, ui64 vchunkGeneration, ui64 idx, ui64 oldCsum, ui64 newCsum) { + integrityBlockDigest ^= Contribution(vchunkGeneration, idx, oldCsum); + integrityBlockDigest ^= Contribution(vchunkGeneration, idx, newCsum); +} + +} // namespace NKikimr::NDDisk diff --git a/ydb/core/blobstorage/ddisk/ddisk_checksums.h b/ydb/core/blobstorage/ddisk/ddisk_checksums.h new file mode 100644 index 00000000000..a5033549cc0 --- /dev/null +++ b/ydb/core/blobstorage/ddisk/ddisk_checksums.h @@ -0,0 +1,166 @@ +#pragma once + +#include "defs.h" + +#include <ydb/library/actors/util/rope.h> + +#include <vector> + +namespace NKikimr::NDDisk { + +// Checksum unit size (4 KiB), independent of LBA format. Matches MinSectorSize / DataAlignment. +constexpr size_t IntegrityUnitSize = 4096; +constexpr size_t IntegritySubBlockSize = 512; + +// DataChunk / IntegrityChunk are PDisk chunks (default 128 MiB). IntegrityChunk reserves +// 128 KiB at the front for its own metadata (TIntegrityChunkHeader replicas, etc.). +constexpr size_t IntegrityChunkSize = 128_MB; +constexpr size_t IntegrityChunkHeaderRegionSize = 128_KB; +constexpr ui32 DataBlocksPerChunk = IntegrityChunkSize / IntegrityUnitSize; + +// Placeholder magics (PDisk-style unique values); not yet finalized for production format. +constexpr ui64 MagicIntegrityBlock = 0x1B71E6A7B10C4E55ull; +constexpr ui64 MagicIntegritySubBlock = 0x51B71E6A75B10C4Eull; +constexpr ui64 MagicIntegrityChunkHeader = 0xC8A7C8A71B71E6A7ull; + +enum class EIntegrityFormatVersion : ui16 { + BaseAwupf4KiB = 1, + Awupf512B = 2, +}; + +// Computes an XXH3-64 checksum over exactly numBytes bytes starting at it. The iterator is passed by +// value, so the caller's own iterator is not advanced. This is a raw data checksum with no identity +// or salt mixed in. +ui64 CalculateBlockChecksum(TRope::TConstIterator it, size_t numBytes); + +// Splits payload into IntegrityUnitSize (4 KiB) blocks and computes a checksum for each block, in order. +// payload.size() must be a non-zero multiple of IntegrityUnitSize (same as MinSectorSize). +std::vector<ui64> CalculatePayloadChecksums(const TRope& payload); + +// Incremental IntegrityBlockDigest helpers (RFC): Contribution is XXH3_64 over +// (vchunkGeneration || blockIdx || blockChecksum); UpdateRoot XORs out the old contribution and +// XORs in the new one. +ui64 Contribution(ui64 vchunkGeneration, ui64 blockIdx, ui64 blockChecksum); +void UpdateRoot(ui64& integrityBlockDigest, ui64 vchunkGeneration, ui64 idx, ui64 oldCsum, ui64 newCsum); + +#pragma pack(push, 1) + +struct TIntegrityBlockHeader { + ui64 Magic; + ui64 BlockChecksum; // checksum over the whole 4 KiB block with this field zeroed + + ui16 Flags; + ui16 FormatVersion; + + ui32 ChecksumBlockIdx; // index inside IntegrityExtent + + ui64 OwnerId; // tablet/user disk owner + ui64 VChunkId; + ui64 VChunkGeneration; // changes on allocate/free/allocate + + ui64 IntegrityChunkId; + ui64 IntegrityExtentId; // optional, or service chunk + extent slot + ui64 IntegrityChunkGeneration; // changes when this chunk is reinitialized/reused + + ui64 IntegrityBlockDigest; // Incremental checksum of data block checksums + + ui8 UsedBlocksBitmap[2]; // which of 500 blocks are used + ui8 Reserved[14]; // pad so packed sizeof == 96 +}; + +static_assert(sizeof(TIntegrityBlockHeader) == 96); + +// Base format (AWUPF >= 4 KiB): how many data-block checksums fit after the header in one unit. +constexpr ui32 ChecksumsPerIntegrityBlock = + (IntegrityUnitSize - sizeof(TIntegrityBlockHeader)) / sizeof(ui64); +static_assert(ChecksumsPerIntegrityBlock == 500); + +struct TIntegrityBlock { + TIntegrityBlockHeader Header; + ui64 Checksums[ChecksumsPerIntegrityBlock]; +}; + +static_assert(sizeof(TIntegrityBlock) == IntegrityUnitSize); + +struct TIntegritySubBlockHeader { + ui64 Magic; + ui64 SubBlockChecksum; // XXH3_64 over this 512 B sub-block with this field zeroed + ui32 ChecksumBlockIdx; // parent TIntegrityBlock index inside IntegrityExtent + ui16 SubBlockIdx; // 1..7 + ui16 FormatVersion; + ui64 VChunkGeneration; +}; + +static_assert(sizeof(TIntegritySubBlockHeader) == 32); + +// AWUPF = 512 B: TIntegrityBlock is still one IntegrityUnitSize I/O, split into self-contained +// sub-blocks. Sub-block 0 carries TIntegrityBlockHeader; sub-blocks 1..N-1 carry mini-headers. +constexpr ui32 IntegritySubBlocksPerBlock = IntegrityUnitSize / IntegritySubBlockSize; +static_assert(IntegritySubBlocksPerBlock == 8); + +constexpr ui32 ChecksumsPerIntegritySubBlock = + (IntegritySubBlockSize - sizeof(TIntegritySubBlockHeader)) / sizeof(ui64); +static_assert(ChecksumsPerIntegritySubBlock == 60); + +constexpr ui32 ChecksumsInIntegrityBlockHeaderSubBlock = + (IntegritySubBlockSize - sizeof(TIntegrityBlockHeader)) / sizeof(ui64); +static_assert(ChecksumsInIntegrityBlockHeaderSubBlock == 52); + +constexpr ui32 ChecksumsPerIntegrityBlockAwupf512 = + ChecksumsInIntegrityBlockHeaderSubBlock + + (IntegritySubBlocksPerBlock - 1) * ChecksumsPerIntegritySubBlock; +static_assert(ChecksumsPerIntegrityBlockAwupf512 == 472); + +struct TIntegritySubBlock { + TIntegritySubBlockHeader Header; + ui64 Checksums[ChecksumsPerIntegritySubBlock]; +}; + +static_assert(sizeof(TIntegritySubBlock) == IntegritySubBlockSize); + +// IntegrityExtent covers one DataChunk: ceil(data blocks / checksums per integrity block). +constexpr ui32 IntegrityBlocksPerExtent = + (DataBlocksPerChunk + ChecksumsPerIntegrityBlock - 1) / ChecksumsPerIntegrityBlock; +static_assert(IntegrityBlocksPerExtent == 66); + +constexpr ui32 IntegrityBlocksPerExtentAwupf512 = + (DataBlocksPerChunk + ChecksumsPerIntegrityBlockAwupf512 - 1) / ChecksumsPerIntegrityBlockAwupf512; +static_assert(IntegrityBlocksPerExtentAwupf512 == 70); + +constexpr size_t IntegrityExtentSize = IntegrityBlocksPerExtent * IntegrityUnitSize; +constexpr size_t IntegrityExtentSizeAwupf512 = IntegrityBlocksPerExtentAwupf512 * IntegrityUnitSize; + +constexpr ui32 IntegrityExtentsPerChunk = + (IntegrityChunkSize - IntegrityChunkHeaderRegionSize) / IntegrityExtentSize; +static_assert(IntegrityExtentsPerChunk == 496); + +constexpr ui32 IntegrityExtentsPerChunkAwupf512 = + (IntegrityChunkSize - IntegrityChunkHeaderRegionSize) / IntegrityExtentSizeAwupf512; +static_assert(IntegrityExtentsPerChunkAwupf512 == 467); + +struct TIntegrityChunkHeader { + ui64 Magic; + ui64 HeaderChecksum; // XXH3_64 over this 4 KiB block with HeaderChecksum = 0 + ui32 FormatVersion; + ui32 HeaderSize; + + ui64 DDiskId; + ui64 PDiskGuid; + ui64 IntegrityChunkId; // PDisk chunk id + ui64 IntegrityChunkGeneration; // changes when this chunk is reinitialized/reused + + ui64 Flags; + + ui8 Reserved[IntegrityUnitSize - ( + sizeof(ui64) * 7 + // Magic, HeaderChecksum, DDiskId, PDiskGuid, IntegrityChunkId, IntegrityChunkGeneration, Flags + sizeof(ui32) * 2 // FormatVersion, HeaderSize + )]; +}; + +static_assert(sizeof(TIntegrityChunkHeader) == IntegrityUnitSize); + +#pragma pack(pop) + +static_assert(DataBlocksPerChunk == 32768); + +} // namespace NKikimr::NDDisk diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer.h b/ydb/core/blobstorage/ddisk/persistent_buffer.h index 9c94108d446..1710bb0554c 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer.h @@ -43,6 +43,10 @@ namespace NKikimr::NDDisk { ui64 VChunkIndex; TInstant Timestamp; std::unordered_set<ui64> ReadInflight; + // Sender-supplied per-MinSectorSize-block payload checksums, in order, covering + // [OffsetInBytes, OffsetInBytes + Size). Empty when the record was written without + // checksums (legacy / internal writes) - opt-in, mirrors the wire-level semantics. + std::vector<ui64> PayloadChecksums; }; std::map<ui64, TRecord> Records; diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h index 8f83c4d1142..7949fdfbbfd 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h @@ -73,9 +73,21 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxSectorsPerBufferRecord = 128; static constexpr ui32 MaxSectorsPerPackBufferRecord = 8; + enum EFlags : ui32 { + NONE = 0, + // When set, a ui64 payload checksum (sender-computed, unsalted XXH3-64 per MinSectorSize + // block - see CalculatePayloadChecksums) immediately follows this record's + // TPersistentBufferSectorInfo[] location array, one entry per data sector, same order. + // Opt-in per record: only present when the write that produced it carried sender-supplied + // checksums (TEvWritePersistentBuffer.Checksums). Legacy/internal records without checksums + // have this flag clear (the field was always zero-initialized via the full-sector memset + // that precedes header assembly, so old on-disk records are unaffected by this rename). + HAS_PAYLOAD_CHECKSUMS = 1, + }; + ui64 TabletId; ui32 Generation; - ui32 Reserved1; + ui32 Flags; ui64 VChunkIndex; ui32 OffsetInBytes; ui32 Size; @@ -83,8 +95,10 @@ namespace NKikimr::NDDisk { }; static_assert(sizeof(TPersistentBufferLsnRecordHeader) <= DataAlignment); + // Worst case: a single record spanning MaxSectorsPerBufferRecord sectors, all carrying a + // persisted payload checksum. static_assert(DataAlignment >= sizeof(TPersistentBufferHeader) + sizeof(TPersistentBufferLsnRecordHeader) - + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * sizeof(TPersistentBufferSectorInfo) + + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * (sizeof(TPersistentBufferSectorInfo) + sizeof(ui64)) ); #pragma pack(pop) } diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp index bea5e6eb01d..d7c92eb19ae 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp @@ -1129,12 +1129,16 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { // TPersistentBufferHeader // [record 0] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // [record 1] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // ... // Each record here has exactly 1 data sector, so Sectors.size() == 2 // (header sector + 1 data sector), meaning (Sectors.size()-1) == 1 - // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. + // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. Both A and B call + // ChecksumPayload(), so recB also carries 1 persisted payload checksum (ui64) after its + // TPersistentBufferSectorInfo entry. const ui32 headerChunkIdx = rawBC_combined->Get()->ChunkIdx; const ui32 headerOffset = rawBC_combined->Get()->Offset; const ui64 actualUniqueId = [&]() -> ui64 { @@ -1152,10 +1156,14 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { UNIT_ASSERT_VALUES_EQUAL_C(recB->Generation, credsB.Generation, "First batched record generation must match tablet B"); - // Advance past recB's LsnRecordHeader and its 1 TPersistentBufferSectorInfo - // (Sectors.size()-1 == 1 for a single-sector data write). + // Advance past recB's LsnRecordHeader, its 1 TPersistentBufferSectorInfo + // (Sectors.size()-1 == 1 for a single-sector data write), and (since B carries a + // sender checksum) its 1 persisted payload checksum. pos += sizeof(NDDisk::TPersistentBufferLsnRecordHeader); pos += sizeof(NDDisk::TPersistentBufferSectorInfo); // (Sectors.size()-1) entries + if (recB->Flags & NDDisk::TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + pos += sizeof(ui64); // (Sectors.size()-1) checksums + } // Record C: immediately follows. const auto* recC = reinterpret_cast<const NDDisk::TPersistentBufferLsnRecordHeader*>(pos); diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp index b36d7ebc065..19758b86f74 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp @@ -11,6 +11,7 @@ #include <ydb/core/blobstorage/ddisk/ddisk.h> #include <ydb/core/blobstorage/ddisk/ddisk_actor.h> +#include <ydb/core/blobstorage/ddisk/persistent_buffer_header.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h> @@ -22,6 +23,8 @@ #include <util/string/printf.h> +#include <cstring> + namespace NKikimr { namespace { @@ -184,6 +187,113 @@ public: auto res = new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0); SendPDiskResponse(disk, *checkSpace, res); } + + // Create a DDisk instance that simulates a restart where the PB chunks from a + // previous instance are passed via StartingPoints. Mirrors the helper of the same + // name in ddisk_actor_batch_write_ut.cpp. + TDiskHandle CreateDDiskWithRestoredChunkData(ui32 pdiskId, ui32 slotId, + const std::vector<ui32>& preExistingChunkIds, ui64 oldUniqueId, + const std::unordered_map<ui32, TString>& chunkData) { + const TActorId pdiskEdge = Runtime.AllocateEdgeActor(NodeId, __FILE__, __LINE__); + const TActorId pdiskServiceId = MakeBlobStoragePDiskID(NodeId, pdiskId); + Runtime.RegisterService(pdiskServiceId, pdiskEdge); + + TVector<TActorId> actorIds = { + MakeBlobStorageDDiskId(NodeId, pdiskId, slotId), + }; + auto groupInfo = MakeIntrusive<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureNone, ui32(1), ui32(1), + ui32(1), &actorIds); + + TVDiskConfig::TBaseInfo baseInfo( + TVDiskIdShort(groupInfo->GetVDiskId(0)), + pdiskServiceId, + 0x100000 + pdiskId, + pdiskId, + NPDisk::DEVICE_TYPE_NVME, + slotId, + NKikimrBlobStorage::TVDiskKind::Default, + 1, + "ddisk_pool"); + NDDisk::TPersistentBufferFormat pbFormat{256, 4, BlockSize * 128, 8, 5000, 512 * 1024}; + const TActorId ddiskActor = Runtime.Register(NDDisk::CreateDDiskActor(std::move(baseInfo), groupInfo, + std::move(pbFormat), NDDisk::TDDiskConfig{}, Counters), + NodeId); + const TActorId ddiskServiceId = MakeBlobStorageDDiskId(NodeId, pdiskId, slotId); + const TActorId pbServiceId = MakeBlobStoragePersistentBufferId(NodeId, pdiskId, slotId); + Runtime.RegisterService(ddiskServiceId, ddiskActor); + + TDiskHandle disk{ddiskServiceId, pbServiceId, pdiskEdge, pdiskId, slotId, 100000 + pdiskId * 1000}; + + const NPDisk::TOwner Owner = 1; + const NPDisk::TOwnerRound OwnerRound = 1; + + auto init = WaitPDiskRequest<NPDisk::TEvYardInit>(disk); + TVector<ui32> ownedChunks; + auto initReply = std::make_unique<NPDisk::TEvYardInitResult>( + NKikimrProto::OK, + 0, 0, 0, + BlockSize, BlockSize, BlockSize, + ChunkSize, + BlockSize, + Owner, + OwnerRound, + 1, + 0, + std::move(ownedChunks), + NPDisk::DEVICE_TYPE_NVME, + false, + BlockSize, + ""); + + NPDisk::TDiskFormat format = {}; + format.Clear(false); + initReply->DiskFormat = NPDisk::TDiskFormatPtr(new NPDisk::TDiskFormat(format), +[](NPDisk::TDiskFormat* ptr) { + delete ptr; + }); + + { + NKikimrBlobStorage::NDDisk::NInternal::TPersistentBufferChunkMapLogRecord pbChunkMap; + for (ui32 chunkIdx : preExistingChunkIds) { + pbChunkMap.AddChunkIdxs(chunkIdx); + } + pbChunkMap.SetUniqueId(oldUniqueId + 1); + + TString pbChunkMapData; + const bool serializeOk = pbChunkMap.SerializeToString(&pbChunkMapData); + Y_ABORT_UNLESS(serializeOk); + initReply->StartingPoints[TLogSignature::SignaturePersistentBufferChunkMap] = + NPDisk::TLogRecord(TLogSignature::SignaturePersistentBufferChunkMap, + TRcBuf(pbChunkMapData), 1 /*lsn*/); + } + + SendPDiskResponse(disk, *init, initReply.release()); + + auto readLog = WaitPDiskRequest<NPDisk::TEvReadLog>(disk); + auto readLogReply = std::make_unique<NPDisk::TEvReadLogResult>( + NKikimrProto::OK, + readLog->Get()->Position, + readLog->Get()->Position, + true, + 0, + "", + Owner); + SendPDiskResponse(disk, *readLog, readLogReply.release()); + + for (ui32 i = 0; i < preExistingChunkIds.size(); ++i) { + auto readRaw = WaitPDiskRequest<NPDisk::TEvChunkReadRaw>(disk); + const ui32 chunkIdx = readRaw->Get()->ChunkIdx; + auto it = chunkData.find(chunkIdx); + TString data; + if (it != chunkData.end()) { + data = it->second; + } else { + data = TString(TTestContext::ChunkSize, '\0'); + } + SendPDiskResponse(disk, *readRaw, new NPDisk::TEvChunkReadRawResult(TRope(data))); + } + + return disk; + } }; void SendToDDisk(TTestContext& ctx, const TActorId& serviceId, IEventBase* event, ui64 cookie = 0) { @@ -560,6 +670,765 @@ Y_UNIT_TEST_SUITE(TDDiskChecksumTests) { UNIT_ASSERT_VALUES_EQUAL(GetChecksumCounter(ctx, disk, "WritesWithoutChecksums")->Val(), 1); } + + // 9. Persistence: a PB write carrying sender checksums stores them, and an in-memory read + // (record still cached, no restart) returns exactly those checksums, in order, alongside the + // original payload. + Y_UNIT_TEST(SinglePBWriteWithChecksumsReadBackInMemory) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(60, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 110, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('Z', 2 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->GetPayload(0).ConvertToString(), payload); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i]); + } + } + + // 10. Persistence across restart, both the direct (non-batch) write path and the batched write + // path: record A is written directly, B and C are accumulated into a single batch header + // sector while A's disk write is in-flight. All three carry sender checksums. After a + // simulated restart (StartingPoints replay from a fresh DDisk instance), all three records + // must be listed and readable with the same checksums that were originally sent. + Y_UNIT_TEST(WritesWithChecksumsAndBatchSurviveRestart) { + TTestContext ctx; + + const TDiskHandle disk1 = ctx.CreateDDisk(61, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 111, 1); + + const ui64 lsnA = 1; + const ui64 lsnB = 77; + const ui64 lsnC = 88; + const TString payloadA = MakeData('A', BlockSize); + const TString payloadB = MakeData('B', BlockSize); + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + + // Accumulate all raw PDisk writes into per-chunk buffers so we can feed them back to + // disk2 during restore. + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + std::vector<ui64> checksumsA, checksumsB, checksumsC; + + // Step 1: send A, keep its PDisk write pending so B and C enter the batch path. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorA, lsnA, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadA)); + checksumsA.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1ActorId = rawA->Sender; + captureWrite(rawA); + + // Step 2/3: B and C accumulate into the same batch while A is in-flight. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorB, lsnB, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadB)); + checksumsB.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorC, lsnC, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Step 4: complete A. + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + // Step 5: wakeup fires -> combined write for B and C (header + B data + C data). + auto rawBC = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBC->Get()->Data.size(), 3 * BlockSize, + "Combined write must contain header + B data + C data (3 sectors)"); + captureWrite(rawBC); + + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawBC->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBC->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Header must record BatchSize=2 for B and C"); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 1u, + "Header Version must be 1 when any record in the batch carries payload checksums"); + return hdr->PersistentBufferUniqueId; + }(); + UNIT_ASSERT_C(actualUniqueId != 0, "PersistentBufferUniqueId must not be zero"); + + // Step 6: complete the combined write -> B and C both reply. + ctx.SendPDiskResponse(disk1, *rawBC, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + // Phase 2: restart with instance 2 (same PDiskId, same UniqueId). + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1ActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 61, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 111, 1); + + auto listResult = SendToDDiskAndWait<NDDisk::TEvListPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvListPersistentBuffer(creds2)); + AssertStatus(listResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(listResult->Get()->Record.RecordsSize(), 3, + "Records A, B and C must all be restored after restart"); + + auto checkRead = [&](const NDDisk::TBlockSelector& selector, ui64 lsn, const TString& payload, + const std::vector<ui64>& expectedChecksums, const TString& label) { + auto read = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(read, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->GetPayload(0).ConvertToString(), payload, + TStringBuilder() << label << ": restored payload must match"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(read->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + TStringBuilder() << label << ": checksum count mismatch after restore"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->Record.GetChecksums(i), expectedChecksums[i], + TStringBuilder() << label << ": checksum " << i << " mismatch after restore"); + } + }; + + checkRead(selectorA, lsnA, payloadA, checksumsA, "record A (non-batch)"); + checkRead(selectorB, lsnB, payloadB, checksumsB, "record B (batched)"); + checkRead(selectorC, lsnC, payloadC, checksumsC, "record C (batched)"); + } + + // 11. Signature-correction sector: the sender's checksum is validated over, and must be + // persisted as, the ORIGINAL payload bytes -- even though PB zeroes the colliding byte on disk + // and restores it on read. If the checksum were ever recomputed from the on-disk (corrected) + // bytes it would not match what the sender sent. + Y_UNIT_TEST(SignatureCorrectionChecksumPersistedAsOriginal) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(62, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 112, 1); + + const ui64 lsn = 1; + // The second data sector's first byte deliberately equals the PB header signature byte, + // forcing TDDiskActor to zero it on disk (HasSignatureCorrection) and restore it on read. + TString payload = MakeData('S', 2 * BlockSize); + payload.Detach()[BlockSize] = static_cast<char>(NDDisk::TPersistentBufferHeader::PersistentBufferHeaderSignature[0]); + const NDDisk::TBlockSelector selector{7, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + // Sanity: the on-disk sector's colliding byte was actually zeroed (header sector + 2 data + // sectors are written contiguously, so the second data sector starts at offset 2*BlockSize). + const TString writtenData = pbWriteRaw->Get()->Data.ConvertToString(); + UNIT_ASSERT_VALUES_EQUAL_C(writtenData[2 * BlockSize], '\0', + "Signature-correction must zero the colliding byte on disk"); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Original signature byte must be restored on read"); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Persisted checksum must be the pre-correction (original) value, never recomputed from on-disk bytes"); + } + } + + // 12. Legacy / no-checksum compatibility across restart: a write without sender checksums + // (opt-in, same as SinglePBWriteWithoutChecksums) must restore successfully and its read must + // return an empty Checksums list, both immediately and after a restart. + Y_UNIT_TEST(LegacyWriteWithoutChecksumsSurvivesRestartWithEmptyChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(63, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 113, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('L', BlockSize); + const NDDisk::TBlockSelector selector{4, 0, BlockSize}; + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payload)); + // Deliberately not calling AddPayloadThenChecksum(): checksum persistence is opt-in. + UNIT_ASSERT_VALUES_EQUAL(write->Record.ChecksumsSize(), 0u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + + auto rawWrite = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1SenderActorId = rawWrite->Sender; + captureWrite(rawWrite); + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawWrite->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawWrite->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 0u, "Header Version must stay 0 for a checksum-less write"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawWrite, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1SenderActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 63, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 113, 1); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Legacy record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 0u, + "A checksum-less record must return an empty Checksums list after restore"); + } + + // 13. Re-replication (TEvReadThenWritePersistentBuffers) must forward the checksums that were + // persisted on the source record, not silently drop them: the coordinator PB reads its own + // record (which carries sender-supplied checksums) and re-writes it to the target PBs via + // TEvWritePersistentBuffers -- the checksums must reach and be validated/persisted on every + // target, exactly as if the original sender had written them directly. Modeled on + // PersistentBufferReadThenWriteTunnel (ddisk_actor_ut.cpp). + Y_UNIT_TEST(ReadThenWriteForwardsPersistedChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(64, 1); + const TDiskHandle disk2 = ctx.CreateDDisk(65, 1); + const TDiskHandle disk3 = ctx.CreateDDisk(66, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk1.PBServiceId, 120, 1); + + const ui64 lsn = 10; + const TString payload = MakeData('P', 2 * BlockSize); + const NDDisk::TBlockSelector selector{3, 0, 2 * BlockSize}; + + auto pbs = std::vector<std::tuple<ui32, ui32, ui32>>{ + {NodeId, disk2.PDiskId, disk2.SlotId}, + {NodeId, disk3.PDiskId, disk3.SlotId}}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + ctx.SendPDiskResponse(disk1, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readThenWrite = std::make_unique<NDDisk::TEvReadThenWritePersistentBuffers>(creds, lsn, 1, pbs, 1000); + SendToDDisk(ctx, disk1.PBServiceId, readThenWrite.release()); + for (int i = 0; i < 2; ++i) { + auto peerWriteRaw = ctx.WaitPDiskRequests<NPDisk::TEvChunkWriteRaw>({disk2.PDiskEdge, disk3.PDiskEdge}); + const TDiskHandle& respondingDisk = (peerWriteRaw->Recipient == disk2.PDiskEdge) ? disk2 : disk3; + ctx.SendPDiskResponse(respondingDisk, *peerWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + } + + auto tunnelResult = ctx.Runtime.WaitForEdgeActorEvent<NDDisk::TEvWritePersistentBuffersResult>(ctx.Edge, false); + UNIT_ASSERT_VALUES_EQUAL(tunnelResult->Get()->Record.ResultSize(), 2u); + for (ui32 i = 0; i < tunnelResult->Get()->Record.ResultSize(); ++i) { + const auto& wr = tunnelResult->Get()->Record.GetResult(i); + UNIT_ASSERT_C(wr.GetResult().GetStatus() == TReplyStatus::OK, TStringBuilder() + << "Re-replicated write " << i << " expected OK, got " + << TReplyStatus::E_Name(static_cast<TReplyStatus::E>(wr.GetResult().GetStatus()))); + } + + for (auto disk : {disk2, disk3}) { + NDDisk::TQueryCredentials peerCreds = Connect(ctx, disk.PBServiceId, 120, 1); + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(peerCreds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Re-replicated payload must match the original"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + "Re-replicated record must carry the same checksums as the original write, not an empty list"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Checksum " << i << " must survive TEvReadThenWritePersistentBuffers re-replication"); + } + } + } + + // 14. Batch header overflow: checksummed 8-sector records (the largest size still eligible for + // batching) accumulate in an open batch until one more record's metadata (TPersistentBufferLsnRecordHeader + // + its sector-location array + one checksum per data sector, see LsnRecordMetadataSize) no longer + // fits the shared 4 KiB header sector -- even though the batch's outer record-count limit + // (MaxLsnsPerPack, sized for checksum-less records) is not yet reached. That overflowing record + // must fall back to a direct (own-header) write instead of corrupting the header sector, and every + // record -- batched and direct alike -- must restore with correct payload and checksums after a + // restart. + Y_UNIT_TEST(BatchHeaderOverflowFallsBackToDirectWrite) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(67, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 121, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path instead of a direct write. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Compute, the same way LsnRecordMetadataSize does, exactly how many checksummed 8-sector + // records fit in the shared 4 KiB header sector alongside TPersistentBufferHeader: that many + // are sent first (all must batch, i.e. produce no PDisk write of their own), then one more is + // sent (must overflow and fall back to a direct write). This is deterministic -- no need to + // race for "did a write appear immediately", since A's own raw write was already dequeued above + // and batched records never produce a raw write, so the next TEvChunkWriteRaw on disk1 can only + // be the overflowing record's direct write. + constexpr ui32 kSectorsPerRecord = 8; + const ui32 recordMetadataSize = sizeof(NDDisk::TPersistentBufferLsnRecordHeader) + + kSectorsPerRecord * sizeof(NDDisk::TPersistentBufferSectorInfo) + + kSectorsPerRecord * sizeof(ui64); + const ui32 headerBudget = BlockSize - sizeof(NDDisk::TPersistentBufferHeader); + const ui32 batchedCount = headerBudget / recordMetadataSize; + UNIT_ASSERT_C(batchedCount >= 1, "Sanity: at least one checksummed 8-sector record must fit the header sector"); + + struct TRecordInfo { + TString Payload; + std::vector<ui64> Checksums; + NDDisk::TBlockSelector Selector; + ui64 Lsn; + }; + auto makeRecord = [&](ui32 i) { + TString payload = MakeData(static_cast<char>('a' + (i % 26)), kSectorsPerRecord * BlockSize); + const NDDisk::TBlockSelector selector{ui64(10 + i), 0, kSectorsPerRecord * BlockSize}; + const ui64 lsn = 100 + i; + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + TRecordInfo info{payload, + std::vector<ui64>(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()), + selector, lsn}; + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + return info; + }; + + std::vector<TRecordInfo> batched; + for (ui32 i = 0; i < batchedCount; ++i) { + batched.push_back(makeRecord(i)); + } + const TRecordInfo overflow = makeRecord(batchedCount); + + auto rawOverflow = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawOverflow->Get()->Data.size(), (kSectorsPerRecord + 1) * BlockSize, + "Overflowing record must fall back to its own direct (header + data) write"); + captureWrite(rawOverflow); + ctx.SendPDiskResponse(disk1, *rawOverflow, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto overflowResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(overflowResult, TReplyStatus::OK); + + // Complete A's write, then the batch's combined write (header + all batched records' data). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), (1 + batched.size() * kSectorsPerRecord) * BlockSize, + "Batch combined write must contain the header sector plus every batched record's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, static_cast<ui32>(batched.size()), + "BatchSize must equal the number of successfully batched records"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + for (ui32 i = 0; i < batched.size(); ++i) { + auto wr = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wr, TReplyStatus::OK); + } + + // Restart: restore from the captured chunk data and verify every record -- the overflowing + // direct write and every batched record -- comes back with the correct payload and checksums. + // Reading a restored record fetches its data sectors from PDisk on demand (restore only + // reconstructs in-memory metadata, not payload bytes), so intercept TEvChunkReadRaw to disk2 + // and serve it from the captured chunkBufs; also auto-answer TEvCheckSpace, which the actor + // may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 67, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 121, 1); + + auto checkRecord = [&](const TRecordInfo& info, const TString& label) { + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, info.Selector, info.Lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), info.Payload, + label << " payload mismatch"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), info.Checksums.size(), + label << " checksum count mismatch"); + for (ui32 i = 0; i < info.Checksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), info.Checksums[i], + label << " checksum " << i << " mismatch"); + } + }; + + checkRecord(overflow, "overflow (direct) record"); + for (ui32 i = 0; i < batched.size(); ++i) { + checkRecord(batched[i], TStringBuilder() << "batched record " << i); + } + } + + // 15. Partial-range read: a read selector narrower than the written record (and starting at a + // nonzero offset within it) must return both the correct payload sub-range and the matching + // checksum sub-range -- exercising the firstBlock/blockCount slicing in ReplyReadPersistentBuffer. + Y_UNIT_TEST(PartialRangeReadSlicesChecksums) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(68, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 122, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('R', 4 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 4 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> allChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(allChecksums.size(), 4u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + // Read blocks [1, 3) out of the 4-block record, i.e. a 2-block selector starting one block in. + const NDDisk::TBlockSelector partialSelector{5, BlockSize, 2 * BlockSize}; + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, partialSelector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload.substr(BlockSize, 2 * BlockSize), + "Partial read must return only the requested payload sub-range"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 2u, + "Partial read must return only the checksums for the requested sub-range, not all 4"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(0), allChecksums[1], + "First returned checksum must be the record's block 1 (not block 0)"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(1), allChecksums[2], + "Second returned checksum must be the record's block 2"); + } + + // 16. Mixed batch: a checksummed record and a checksum-less record share the same batch header. + // After a restart, the checksummed record must restore with its checksums and the checksum-less + // one must restore with an empty Checksums list -- exercising the per-record HAS_PAYLOAD_CHECKSUMS + // flag check during batch restore parsing, as opposed to a single Version-wide switch. + Y_UNIT_TEST(MixedChecksummedAndLegacyBatchRestoresCorrectly) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(69, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 123, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Write B: batched, WITHOUT checksums (legacy/opt-out). + const TString payloadB = MakeData('B', BlockSize); + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorB, 2, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadB)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Write C: batched, WITH checksums -- lands in the same batch header as B. + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + std::vector<ui64> checksumsC; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorC, 3, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + UNIT_ASSERT_VALUES_EQUAL(checksumsC.size(), 1u); + + // Complete A's write, then the batch's combined write (header + B's + C's data sectors). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), 3 * BlockSize, + "Batch combined write must contain the header sector plus B's and C's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Both B and C must land in the same batch"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + // Restart. Reading a restored record fetches its data sectors from PDisk on demand, so + // intercept TEvChunkReadRaw to disk2 and serve it from the captured chunkBufs; also + // auto-answer TEvCheckSpace, which the actor may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 69, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 123, 1); + + auto readB = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorB, 2, 1, {true})); + AssertStatus(readB, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readB->Get()->GetPayload(0).ConvertToString(), payloadB, + "Checksum-less batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readB->Get()->Record.ChecksumsSize()), 0u, + "Checksum-less batched record must restore with an empty Checksums list"); + + auto readC = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorC, 3, 1, {true})); + AssertStatus(readC, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->GetPayload(0).ConvertToString(), payloadC, + "Checksummed batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readC->Get()->Record.ChecksumsSize()), checksumsC.size(), + "Checksummed batched record must restore with its checksums, not an empty list"); + for (ui32 i = 0; i < checksumsC.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->Record.GetChecksums(i), checksumsC[i], + "Checksum " << i << " must survive restart for the checksummed batched record"); + } + } } } // anonymous namespace diff --git a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp index cec89be4f79..6b1e3b13457 100644 --- a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp +++ b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp @@ -147,6 +147,11 @@ namespace NKikimr::NDDisk { auto msg = std::make_unique<TEvWritePersistentBuffers>(creds, selector, inflight.Lsn, NDDisk::TWriteInstruction(0), inflight.PersistentBufferIds, inflight.Timeout); msg->AddPayload(TRope(payload)); + // Forward the checksums persisted on the source record, if any (TEvReadPersistentBufferResult + // carries them opt-in, see TPersistentBuffer::TRecord::PayloadChecksums). Without this, records + // re-replicated via TEvReadThenWritePersistentBuffers would silently lose their checksums on + // every target PB, exactly the recovery scenario where corruption detection matters most. + msg->Record.MutableChecksums()->CopyFrom(record.GetChecksums()); auto h = std::make_unique<IEventHandle>(SelfId(), inflight.Sender, msg.release(), 0, inflight.Cookie); TActivationContext::Send(h.release()); diff --git a/ydb/core/blobstorage/ddisk/ya.make b/ydb/core/blobstorage/ddisk/ya.make index 177832a44c1..9839fc99d89 100644 --- a/ydb/core/blobstorage/ddisk/ya.make +++ b/ydb/core/blobstorage/ddisk/ya.make @@ -3,6 +3,8 @@ LIBRARY() SRCS( ddisk.cpp ddisk.h + ddisk_checksums.cpp + ddisk_checksums.h ddisk_actor.cpp ddisk_actor.h ddisk_actor_boot.cpp diff --git a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp index 601645f3ef9..534ec86a629 100644 --- a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp +++ b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp @@ -1,8 +1,10 @@ #include "dsproxy_mock.h" #include "model.h" #include <ydb/core/base/blobstorage.h> +#include <ydb/core/blobstorage/dsproxy/dsproxy.h> #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> #include <ydb/core/util/stlog.h> +#include <util/random/fast.h> #define YDB_LOG_THIS_FILE_COMPONENT BS_PROXY @@ -148,6 +150,105 @@ namespace NKikimr { , Model(MakeIntrusive<NFake::TProxyDS>(groupId)) {} }; + + //////////////////////////////////////////////////////////////////////////////// + + class TBlobStorageFailureInjectingActor final + : public TActor<TBlobStorageFailureInjectingActor> + { + private: + const TActorId RealProxy; + const TGroupId GroupId; + const double FailureProbability; + const ui64 RandomFailureSeed; + const NKikimrProto::EReplyStatus ErrorReplyStatus; + TFastRng64 Rng; + const TString FailureErrorReason; + + public: + TBlobStorageFailureInjectingActor( + TActorId realProxy, + TGroupId groupId, + TBSFailureInjectionConfig config) + : TActor(&TThis::StateWork) + , RealProxy(realProxy) + , GroupId(groupId) + , FailureProbability(std::clamp(config.GetFailureProbability(), 0.0, 1.0)) + , RandomFailureSeed(config.HasRandomSeed() ? config.GetRandomSeed() : RandomNumber<ui64>()) + , ErrorReplyStatus(config.GetErrorReplyStatus()) + , Rng(RandomFailureSeed) + , FailureErrorReason(TStringBuilder() + << "injected by BSProxyInterceptor" + << " group " << GroupId + << " seed " << RandomFailureSeed) + { + } + + private: + bool ShouldInjectFailure() + { + return Rng.GenRandReal4() < FailureProbability; + } + + template <typename TRequest> + bool MaybeInjectFailure( + TAutoPtr<IEventHandle>& ev, + TRequest& request, + const TString& eventName) + { + if (!ShouldInjectFailure()) { + return false; + } + + auto response = request.MakeErrorResponse(ErrorReplyStatus, FailureErrorReason, GroupId); + response->ExecutionRelay = std::move(request.ExecutionRelay); + + LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId + << " injecting " << eventName + << " failure; not forwarding to " << RealProxy.ToString() + << " sender " << ev->Sender.ToString() + << " cookie " << ev->Cookie + << " probability " << FailureProbability + << " seed " << RandomFailureSeed); + + Send(ev->Sender, response.release(), 0, ev->Cookie); + return true; + } + + STFUNC(StateWork) + { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId << " " << ev->GetTypeName() << " " << ev->ToString()); + + #define HANDLE_EVENT(evType) \ + case evType::EventType: { \ + auto* msg = ev->Get<evType>(); \ + if (MaybeInjectFailure(ev, *msg, ev->GetTypeName())) { \ + return; \ + } \ + break; \ + } + + switch (ev->GetTypeRewrite()) { + DSPROXY_ENUM_EVENTS(HANDLE_EVENT) + case TEvents::TEvPoison::EventType: { + TActor::PassAway(); + [[fallthrough]]; + } + default: { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId + << " skipping event type " << ev->GetTypeName()); + break; + } + } + + TActivationContext::Forward(ev, RealProxy); + #undef HANDLE_EVENT + } + }; + } // anon IActor *CreateBlobStorageGroupProxyMockActor(TIntrusivePtr<NFake::TProxyDS> model) { @@ -158,4 +259,8 @@ namespace NKikimr { return new TBlobStorageGroupProxyMockActor(groupId); } + IActor *CreateBlobStorageGroupFailureInjectingActor(TActorId actorId, TGroupId groupId, const TBSFailureInjectionConfig& config) { + return new TBlobStorageFailureInjectingActor(actorId, groupId, config); + } + } // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h index eb0aa2e0e51..ea57c8dd1cc 100644 --- a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h +++ b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h @@ -2,6 +2,8 @@ #include "defs.h" #include <ydb/core/base/blobstorage_common.h> +#include <ydb/core/protos/blobstorage.pb.h> + namespace NKikimr { namespace NFake { @@ -11,4 +13,7 @@ namespace NKikimr { IActor *CreateBlobStorageGroupProxyMockActor(TIntrusivePtr<NFake::TProxyDS> model); IActor *CreateBlobStorageGroupProxyMockActor(TGroupId groupId); + using TBSFailureInjectionConfig = NKikimrBlobStorage::TNodeWardenServiceSet::TFailureInjectionConfig; + IActor *CreateBlobStorageGroupFailureInjectingActor(TActorId actorId, TGroupId groupId, const TBSFailureInjectionConfig& config); + } // NKikimr diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index bcdda630bf5..7f18616ae00 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -2,6 +2,7 @@ #include <ydb/core/base/tablet_resolver.h> #include <ydb/core/base/statestorage_impl.h> +#include <ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h> #include <ydb/core/blobstorage/nodewarden/node_warden.h> #include <ydb/core/blobstorage/nodewarden/node_warden_impl.h> #include <ydb/core/blobstorage/base/blobstorage_events.h> @@ -22,6 +23,7 @@ #include <library/cpp/testing/unittest/registar.h> #include <functional> +#include <optional> const bool STRAND_PDISK = true; #ifndef NDEBUG @@ -1136,8 +1138,9 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { TestInferPDiskSlotCount(17999'117'418'496, c_2000GB, 24, 9, 1u, 0.001); } - void CheckInferredPDiskSettings(TTestBasicRuntime& runtime, TActorId fakeWhiteboard, TActorId fakeNodeWarden, - ui32 pdiskId, ui32 expectedSlotCount, ui32 expectedSlotSizeInUnits, + void CheckInferredPDiskSettings(TTestBasicRuntime& runtime, TActorId fakeWhiteboard, + TActorId fakeNodeWarden, ui32 pdiskId, ui32 expectedSlotCount, ui32 expectedSlotSizeInUnits, + std::optional<ui64> expectedSlotSize = std::nullopt, TDuration simTimeout = TDuration::Seconds(10)) { const int maxAttempts = 10; for (int attempt = 1; attempt <= maxAttempts; ++attempt) { @@ -1157,6 +1160,9 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT(pdiskInfo.HasTotalSize()); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetExpectedSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetSlotSizeInUnits(), expectedSlotSizeInUnits); + // the field is always present in whiteboard updates; 0 means 'not set' + UNIT_ASSERT(pdiskInfo.HasExpectedSlotSize()); + UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetExpectedSlotSize(), expectedSlotSize.value_or(0)); UNIT_ASSERT(pdiskInfo.HasPDiskUsage()); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetPDiskUsage(), 0.0); UNIT_ASSERT(pdiskInfo.HasPDiskCapacityAlert()); @@ -1178,10 +1184,18 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT_LT_C(attempt, maxAttempts, "last attempt failed"); continue; } - UNIT_ASSERT(metrics.HasSlotCount()); + // metrics are replaced as a whole on the receiving side, so zero values are + // reported by omitting the field + UNIT_ASSERT_VALUES_EQUAL(metrics.HasSlotCount(), expectedSlotCount != 0); UNIT_ASSERT(metrics.HasSlotSizeInUnits()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotSizeInUnits(), expectedSlotSizeInUnits); + if (expectedSlotSize) { + UNIT_ASSERT(metrics.HasExpectedSlotSize()); + UNIT_ASSERT_VALUES_EQUAL(metrics.GetExpectedSlotSize(), *expectedSlotSize); + } else { + UNIT_ASSERT(!metrics.HasExpectedSlotSize()); + } UNIT_ASSERT(metrics.HasPDiskUsage()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskUsage(), 0.0); UNIT_ASSERT(metrics.HasPDiskCapacityAlert()); @@ -1217,6 +1231,334 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { return realNodeWarden; } + class TNoReplyPDiskActor : public TActorBootstrapped<TNoReplyPDiskActor> { + const TActorId Observer; + + public: + TNoReplyPDiskActor(TActorId observer) + : Observer(observer) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + } + + void Handle(NPDisk::TEvSlay::TPtr ev) { + const auto *msg = ev->Get(); + Send(Observer, new NPDisk::TEvSlay(msg->VDiskId, msg->SlayOwnerRound, msg->PDiskId, msg->VSlotId)); + } + + STRICT_STFUNC(StateFunc, + hFunc(NPDisk::TEvSlay, Handle); + cFunc(TEvents::TSystem::Poison, PassAway); + ) + }; + + class TNoReplyPDiskServiceFactory : public IPDiskServiceFactory { + TActorId Observer; + + public: + void SetObserver(TActorId observer) { + Observer = observer; + } + + void Create(const TActorContext& ctx, ui32 pdiskId, const TIntrusivePtr<TPDiskConfig>&, + const NPDisk::TMainKey&, ui32 poolId, ui32 nodeId) override { + Y_ABORT_UNLESS(Observer); + const TActorId actorId = ctx.Register(new TNoReplyPDiskActor(Observer), TMailboxType::HTSwap, poolId); + ctx.ActorSystem()->RegisterLocalService(MakeBlobStoragePDiskID(nodeId, pdiskId), actorId); + } + }; + + TActorId SetupNodeWardenForSlayTest(TTestActorSystem& runtime, + TIntrusivePtr<IPDiskServiceFactory> pdiskServiceFactory = {}) { + runtime.Start(); + + auto& appData = *runtime.GetNode(1)->AppData; + appData.DomainsInfo->AddDomain(TDomainsInfo::TDomain::ConstructEmptyDomain("dom", 1).Release()); + appData.DynamicNameserviceConfig = new TDynamicNameserviceConfig(); + + if (!pdiskServiceFactory) { + pdiskServiceFactory.Reset(new TRealPDiskServiceFactory()); + } + TIntrusivePtr<TNodeWardenConfig> config(new TNodeWardenConfig(pdiskServiceFactory)); + ObtainStaticKey(&config->StaticKey); + const TActorId nodeWardenId = runtime.Register(CreateBSNodeWarden(config.Release()), 1); + runtime.RegisterService(MakeBlobStorageNodeWardenID(1), nodeWardenId); + runtime.WrapInActorContext(nodeWardenId, [](IActor *actor) { + dynamic_cast<NStorage::TNodeWarden*>(actor)->Bootstrap(); + }); + return nodeWardenId; + } + + void SetSlayTestVDisk(NStorage::TNodeWarden::TVDiskRecord& record, ui32 nodeId, ui32 pdiskId, + ui32 vdiskSlotId, const TVDiskID& vdiskId) { + auto *location = record.Config.MutableVDiskLocation(); + location->SetNodeID(nodeId); + location->SetPDiskID(pdiskId); + location->SetVDiskSlotID(vdiskSlotId); + VDiskIDFromVDiskID(vdiskId, record.Config.MutableVDiskID()); + } + + CUSTOM_UNIT_TEST(TestSlayCompletesWhenPDiskIsDestroyedBeforeReply) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + TIntrusivePtr<TNoReplyPDiskServiceFactory> pdiskServiceFactory = new TNoReplyPDiskServiceFactory; + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime, pdiskServiceFactory); + const ui32 nodeId = 1; + const ui32 pdiskId = 2007; + const ui32 vdiskSlotId = 10; + const NStorage::TNodeWarden::TVSlotId vslotId(nodeId, pdiskId, vdiskSlotId); + const TVDiskID vdiskId(100507, 15, 0, 0, 0); + const TActorId slayObserver = runtime.AllocateEdgeActor(nodeId); + pdiskServiceFactory->SetObserver(slayObserver); + + NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk pdisk; + pdisk.SetNodeID(nodeId); + pdisk.SetPDiskID(pdiskId); + pdisk.SetPath("slay-test-pdisk"); + pdisk.SetPDiskGuid(1); + pdisk.SetPDiskCategory(0); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + nodeWarden.StartLocalPDisk(pdisk, false); + + NStorage::TNodeWarden::TVDiskRecord record; + SetSlayTestVDisk(record, nodeId, pdiskId, vdiskSlotId, vdiskId); + nodeWarden.Slay(record, NStorage::TNodeWarden::ESlayAction::DESTROY); + UNIT_ASSERT(nodeWarden.SlayInFlight.contains(vslotId)); + }); + + auto slay = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(slayObserver, false); + UNIT_ASSERT_VALUES_EQUAL(slay->Get()->VDiskId, vdiskId); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + const auto it = nodeWarden.SlayInFlight.find(vslotId); + UNIT_ASSERT(it != nodeWarden.SlayInFlight.end()); + UNIT_ASSERT_VALUES_EQUAL(it->second.Round, slay->Get()->SlayOwnerRound); + + nodeWarden.DestroyLocalPDisk(pdiskId); + + UNIT_ASSERT(!nodeWarden.SlayInFlight.contains(vslotId)); + }); + + const TActorId probeActor = runtime.AllocateEdgeActor(nodeId); + runtime.Send(new IEventHandle(nodeWardenId, MakeBlobStoragePDiskID(nodeId, pdiskId), + new NPDisk::TEvSlayResult(NKikimrProto::OK, 0, vdiskId, slay->Get()->SlayOwnerRound, + pdiskId, vdiskSlotId, {})), nodeId); + runtime.Send(new IEventHandle(probeActor, nodeWardenId, new TEvents::TEvWakeup), nodeId); + runtime.WaitForEdgeActorEvent<TEvents::TEvWakeup>(probeActor); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + UNIT_ASSERT(!nodeWarden.SlayInFlight.contains(vslotId)); + }); + } + + CUSTOM_UNIT_TEST(TestUnconfirmedSlayIsRetriedByTimer) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2004; + const ui32 vdiskSlotId = 7; + const NStorage::TNodeWarden::TVSlotId vslotId(nodeId, pdiskId, vdiskSlotId); + const TVDiskID vdiskId(100504, 12, 0, 0, 0); + const TActorId pdiskActor = runtime.AllocateEdgeActor(nodeId); + runtime.RegisterService(MakeBlobStoragePDiskID(nodeId, pdiskId), pdiskActor); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + auto [it, inserted] = nodeWarden.SlayInFlight.emplace(vslotId, + NStorage::TNodeWarden::TSlayInFlight{vdiskId, NStorage::TNodeWarden::ESlayAction::WIPE}); + UNIT_ASSERT(inserted); + nodeWarden.IssueSlay(vslotId, it->second); + }); + + auto first = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + auto retry = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + UNIT_ASSERT_VALUES_EQUAL(retry->Get()->VDiskId, vdiskId); + UNIT_ASSERT_UNEQUAL(retry->Get()->SlayOwnerRound, first->Get()->SlayOwnerRound); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + nodeWarden.SlayInFlight.erase(vslotId); + }); + } + + CUSTOM_UNIT_TEST(TestNotReadySlayRetryMakesInsuranceTimerStale) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2005; + const ui32 vdiskSlotId = 8; + const NStorage::TNodeWarden::TVSlotId vslotId(nodeId, pdiskId, vdiskSlotId); + const TVDiskID vdiskId(100505, 13, 0, 0, 0); + const TActorId pdiskActor = runtime.AllocateEdgeActor(nodeId); + const TActorId probeActor = runtime.AllocateEdgeActor(nodeId); + runtime.RegisterService(MakeBlobStoragePDiskID(nodeId, pdiskId), pdiskActor); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + auto [it, inserted] = nodeWarden.SlayInFlight.emplace(vslotId, + NStorage::TNodeWarden::TSlayInFlight{vdiskId, NStorage::TNodeWarden::ESlayAction::WIPE}); + UNIT_ASSERT(inserted); + nodeWarden.IssueSlay(vslotId, it->second); + }); + + auto first = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + runtime.Send(new IEventHandle(nodeWardenId, pdiskActor, + new NPDisk::TEvSlayResult(NKikimrProto::NOTREADY, 0, vdiskId, + first->Get()->SlayOwnerRound, pdiskId, vdiskSlotId, "PDisk is initializing")), nodeId); + + auto retry = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + UNIT_ASSERT_UNEQUAL(retry->Get()->SlayOwnerRound, first->Get()->SlayOwnerRound); + + runtime.Schedule(TDuration::Seconds(5), + new IEventHandle(probeActor, nodeWardenId, new TEvents::TEvWakeup), nullptr, nodeId); + auto next = runtime.WaitForEdgeActorEvent({pdiskActor, probeActor}); + UNIT_ASSERT_VALUES_EQUAL(next->GetTypeRewrite(), TEvents::TEvWakeup::EventType); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + const auto it = nodeWarden.SlayInFlight.find(vslotId); + UNIT_ASSERT(it != nodeWarden.SlayInFlight.end()); + UNIT_ASSERT_VALUES_EQUAL(it->second.Round, retry->Get()->SlayOwnerRound); + nodeWarden.SlayInFlight.erase(it); + }); + } + + CUSTOM_UNIT_TEST(TestSlayRetryKeepsOperationIdentity) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2001; + const ui32 vdiskSlotId = 3; + const TActorId pdiskActor = runtime.AllocateEdgeActor(nodeId); + runtime.RegisterService(MakeBlobStoragePDiskID(nodeId, pdiskId), pdiskActor); + + const TVDiskID initialVDiskId(100500, 7, 0, 0, 0); + const TVDiskID destroyVDiskId(100500, 8, 0, 0, 0); + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + NStorage::TNodeWarden::TVDiskRecord record; + SetSlayTestVDisk(record, nodeId, pdiskId, vdiskSlotId, initialVDiskId); + nodeWarden.Slay(record, NStorage::TNodeWarden::ESlayAction::WIPE); + + SetSlayTestVDisk(record, nodeId, pdiskId, vdiskSlotId, destroyVDiskId); + nodeWarden.Slay(record, NStorage::TNodeWarden::ESlayAction::DESTROY); + + const auto it = nodeWarden.SlayInFlight.find({nodeId, pdiskId, vdiskSlotId}); + UNIT_ASSERT(it != nodeWarden.SlayInFlight.end()); + UNIT_ASSERT(it->second.Action == NStorage::TNodeWarden::ESlayAction::DESTROY); + UNIT_ASSERT_VALUES_EQUAL(it->second.VDiskId, destroyVDiskId); + }); + + auto first = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + UNIT_ASSERT_VALUES_EQUAL(first->Get()->VDiskId, initialVDiskId); + + auto retry = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + UNIT_ASSERT_VALUES_EQUAL(retry->Get()->VDiskId, destroyVDiskId); + UNIT_ASSERT_UNEQUAL(retry->Get()->SlayOwnerRound, first->Get()->SlayOwnerRound); + } + + CUSTOM_UNIT_TEST(TestSlayIsReplayedAfterPDiskRestartWithoutLocalVDisk) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2002; + const ui32 vdiskSlotId = 4; + const TActorId pdiskActor = runtime.AllocateEdgeActor(nodeId); + runtime.RegisterService(MakeBlobStoragePDiskID(nodeId, pdiskId), pdiskActor); + + const TVDiskID vdiskId(100501, 9, 0, 0, 0); + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + NStorage::TNodeWarden::TVDiskRecord record; + SetSlayTestVDisk(record, nodeId, pdiskId, vdiskSlotId, vdiskId); + nodeWarden.Slay(record, NStorage::TNodeWarden::ESlayAction::DESTROY); + }); + + auto first = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + nodeWarden.PDiskRestartInFlight[pdiskId] = false; + nodeWarden.OnPDiskRestartFinished(pdiskId, NKikimrProto::OK); + }); + + auto replay = runtime.WaitForEdgeActorEvent<NPDisk::TEvSlay>(pdiskActor, false); + UNIT_ASSERT_VALUES_EQUAL(replay->Get()->VDiskId, vdiskId); + UNIT_ASSERT_UNEQUAL(replay->Get()->SlayOwnerRound, first->Get()->SlayOwnerRound); + } + + CUSTOM_UNIT_TEST(TestVDiskWithSlayInFlightIsNotStartedAfterFailedPDiskRestart) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2006; + const ui32 vdiskSlotId = 9; + const NStorage::TNodeWarden::TVSlotId vslotId(nodeId, pdiskId, vdiskSlotId); + const TVDiskID vdiskId(100506, 14, 0, 0, 0); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + auto& record = nodeWarden.LocalVDisks[vslotId]; + SetSlayTestVDisk(record, nodeId, pdiskId, vdiskSlotId, vdiskId); + nodeWarden.SlayInFlight.emplace(vslotId, + NStorage::TNodeWarden::TSlayInFlight{vdiskId, NStorage::TNodeWarden::ESlayAction::WIPE}); + nodeWarden.PDiskRestartInFlight[pdiskId] = false; + + nodeWarden.OnPDiskRestartFinished(pdiskId, NKikimrProto::ERROR); + + UNIT_ASSERT(!record.RuntimeData); + UNIT_ASSERT(nodeWarden.SlayInFlight.contains(vslotId)); + UNIT_ASSERT(!nodeWarden.PDiskRestartInFlight.contains(pdiskId)); + }); + } + + CUSTOM_UNIT_TEST(TestSlayCompletesWhenPDiskIsAlreadyMissing) { + TTestActorSystem runtime(1, NLog::PRI_ERROR, MakeIntrusive<TDomainsInfo>()); + const TActorId nodeWardenId = SetupNodeWardenForSlayTest(runtime); + const ui32 nodeId = 1; + const ui32 pdiskId = 2003; + const ui32 destroyVDiskSlotId = 5; + const ui32 wipeVDiskSlotId = 6; + const TVDiskID destroyVDiskId(100502, 10, 0, 0, 0); + const TVDiskID wipeVDiskId(100503, 11, 0, 0, 0); + const NStorage::TNodeWarden::TVSlotId destroyVSlotId(nodeId, pdiskId, destroyVDiskSlotId); + const NStorage::TNodeWarden::TVSlotId wipeVSlotId(nodeId, pdiskId, wipeVDiskSlotId); + + NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk destroyVDisk; + auto *location = destroyVDisk.MutableVDiskLocation(); + location->SetNodeID(nodeId); + location->SetPDiskID(pdiskId); + location->SetVDiskSlotID(destroyVDiskSlotId); + VDiskIDFromVDiskID(destroyVDiskId, destroyVDisk.MutableVDiskID()); + destroyVDisk.SetEntityStatus(NKikimrBlobStorage::EEntityStatus::DESTROY); + + NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk wipeVDisk; + location = wipeVDisk.MutableVDiskLocation(); + location->SetNodeID(nodeId); + location->SetPDiskID(pdiskId); + location->SetVDiskSlotID(wipeVDiskSlotId); + VDiskIDFromVDiskID(wipeVDiskId, wipeVDisk.MutableVDiskID()); + wipeVDisk.SetDoWipe(true); + + runtime.WrapInActorContext(nodeWardenId, [&](IActor *actor) { + auto& nodeWarden = *dynamic_cast<NStorage::TNodeWarden*>(actor); + nodeWarden.ApplyLocalVDiskInfo(destroyVDisk); + UNIT_ASSERT(!nodeWarden.LocalVDisks.contains(destroyVSlotId)); + UNIT_ASSERT(!nodeWarden.SlayInFlight.contains(destroyVSlotId)); + + nodeWarden.LocalVDisks[wipeVSlotId].UnderlyingPDiskDestroyed = true; + nodeWarden.ApplyLocalVDiskInfo(wipeVDisk); + UNIT_ASSERT(nodeWarden.LocalVDisks.contains(wipeVSlotId)); + UNIT_ASSERT(!nodeWarden.SlayInFlight.contains(wipeVSlotId)); + UNIT_ASSERT(!nodeWarden.LocalVDisks.at(wipeVSlotId).UnderlyingPDiskDestroyed); + }); + } + void UpdateInferPDiskSlotCountSettings(TTestBasicRuntime& runtime, TActorId realNodeWarden, ui64 unitSize, ui32 maxSlots, bool preferInferredSettings) { auto request = std::make_unique<NConsole::TEvConsole::TEvConfigNotificationRequest>(); @@ -1236,6 +1578,44 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { Y_UNUSED(response); } + void UpdateInferPDiskSlotCountFromSlotSizeSettings(TTestBasicRuntime& runtime, TActorId realNodeWarden, + ui64 slotSize, ui32 maxSlots, bool preferInferredSettings = false) { + auto request = std::make_unique<NConsole::TEvConsole::TEvConfigNotificationRequest>(); + auto& record = request->Record; + auto* blobStorageConfig = record.MutableConfig()->MutableBlobStorageConfig(); + auto* inferSettings = blobStorageConfig->MutableInferPDiskSlotCountSettings(); + auto* inferRotSettings = inferSettings->MutableRot(); + + inferRotSettings->SetSlotSize(slotSize); + inferRotSettings->SetMaxSlots(maxSlots); + inferRotSettings->SetPreferInferredSettingsOverExplicit(preferInferredSettings); + + TActorId sender = runtime.AllocateEdgeActor(); + runtime.Send(new IEventHandle(realNodeWarden, sender, request.release())); + + auto response = runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(sender); + Y_UNUSED(response); + } + + CUSTOM_UNIT_TEST(TestInferPDiskSlotCountSettingsSlotSizeValidation) { + NKikimrBlobStorage::TInferPDiskSlotCountSettings settings; + settings.MutableRot()->SetSlotSize(600ull << 30); + auto error = ValidateInferPDiskSlotCountSettings( + settings, "BlobStorageConfig.InferPDiskSlotCountSettings"); + UNIT_ASSERT(error); + UNIT_ASSERT_C(error->Contains("MaxSlots is mandatory with SlotSize or UnitSize"), *error); + + settings.MutableRot()->SetMaxSlots(16); + UNIT_ASSERT(!ValidateInferPDiskSlotCountSettings( + settings, "BlobStorageConfig.InferPDiskSlotCountSettings")); + + settings.MutableRot()->SetUnitSize(100_GB); + error = ValidateInferPDiskSlotCountSettings( + settings, "BlobStorageConfig.InferPDiskSlotCountSettings"); + UNIT_ASSERT(error); + UNIT_ASSERT_C(error->Contains("SlotSize is mutually exclusive with UnitSize"), *error); + } + CUSTOM_UNIT_TEST(TestInferPDiskSlotCountExplicitConfig) { TTestBasicRuntime runtime(1, false); TActorId realNodeWarden = SetupNodeWardenOnly(runtime); @@ -1284,6 +1664,208 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { pdiskId, 13, 0u); } + CUSTOM_UNIT_TEST(TestInferPDiskSlotCountFromSlotSizeWithRealNodeWarden) { + TTestBasicRuntime runtime(1, false); + TActorId realNodeWarden = SetupNodeWardenOnly(runtime); + const ui64 expectedSlotSize = 600ull << 30; + UpdateInferPDiskSlotCountFromSlotSizeSettings(runtime, realNodeWarden, expectedSlotSize, 64); + + const ui32 nodeId = runtime.GetNodeId(0); + const ui32 pdiskId = 1006; + const TString pdiskPath = "SectorMap:TestInferPDiskSlotCountFromSlotSizeWithRealNodeWarden:2400"; + + TActorId fakeNodeWarden = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden); + TActorId fakeWhiteboard = runtime.AllocateEdgeActor(); + runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard); + + NKikimrBlobStorage::TPDiskConfig pdiskConfig; + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 4, 0u, expectedSlotSize); + + const ui64 updatedExpectedSlotSize = 800ull << 30; + UpdateInferPDiskSlotCountFromSlotSizeSettings(runtime, realNodeWarden, updatedExpectedSlotSize, 64); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 3, 0u, updatedExpectedSlotSize); + + UpdateInferPDiskSlotCountFromSlotSizeSettings(runtime, realNodeWarden, updatedExpectedSlotSize, 2); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 2, 0u, updatedExpectedSlotSize); + + pdiskConfig.SetExpectedSlotCount(17); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 17, 0u); + + UpdateInferPDiskSlotCountFromSlotSizeSettings(runtime, realNodeWarden, updatedExpectedSlotSize, 64, true); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 3, 0u, updatedExpectedSlotSize); + } + + CUSTOM_UNIT_TEST(TestExpectedSlotSizeCalculatesSlotCount) { + TTestBasicRuntime runtime(1, false); + TActorId realNodeWarden = SetupNodeWardenOnly(runtime); + + const ui32 nodeId = runtime.GetNodeId(0); + const ui32 pdiskId = 1003; + const TString pdiskPath = "SectorMap:TestExpectedSlotSizeCalculatesSlotCount:2400"; + const ui64 expectedSlotSize = 600ull << 30; + + TActorId fakeNodeWarden = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden); + TActorId fakeWhiteboard = runtime.AllocateEdgeActor(); + runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard); + + NKikimrBlobStorage::TPDiskConfig pdiskConfig; + pdiskConfig.SetExpectedSlotSize(expectedSlotSize); + pdiskConfig.SetMaxSlots(8); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 4, 0u, expectedSlotSize); + + const ui64 updatedExpectedSlotSize = 800ull << 30; + pdiskConfig.SetExpectedSlotSize(updatedExpectedSlotSize); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 3, 0u, updatedExpectedSlotSize); + + const ui64 smallExpectedSlotSize = 1ull << 30; + pdiskConfig.SetExpectedSlotSize(smallExpectedSlotSize); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 8u, 0u, smallExpectedSlotSize); + } + + CUSTOM_UNIT_TEST(TestExpectedSlotSettingsTransitions) { + TTestBasicRuntime runtime(1, false); + TActorId realNodeWarden = SetupNodeWardenOnly(runtime); + + const ui32 nodeId = runtime.GetNodeId(0); + const ui32 pdiskId = 1004; + const TString pdiskPath = "SectorMap:TestExpectedSlotSettingsTransitions:2400"; + const ui64 expectedSlotSize = 600ull << 30; + + TActorId fakeNodeWarden = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden); + TActorId fakeWhiteboard = runtime.AllocateEdgeActor(); + runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard); + + NKikimrBlobStorage::TPDiskConfig pdiskConfig; + pdiskConfig.SetExpectedSlotCount(7); + pdiskConfig.SetSlotSizeInUnits(2); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 7, 2u); + + pdiskConfig.ClearExpectedSlotCount(); + pdiskConfig.ClearSlotSizeInUnits(); + pdiskConfig.SetExpectedSlotSize(expectedSlotSize); + pdiskConfig.SetMaxSlots(8); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 4, 0u, expectedSlotSize); + + pdiskConfig.ClearExpectedSlotSize(); + pdiskConfig.ClearMaxSlots(); + pdiskConfig.SetExpectedSlotCount(9); + pdiskConfig.SetSlotSizeInUnits(3); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 9, 3u); + } + + CUSTOM_UNIT_TEST(TestInvalidExpectedSlotSettingsDoNotCrashNodeWarden) { + TTestBasicRuntime runtime(1, false); + TActorId realNodeWarden = SetupNodeWardenOnly(runtime); + + const ui32 nodeId = runtime.GetNodeId(0); + const ui32 pdiskId = 1005; + const TString pdiskPath = "SectorMap:TestInvalidExpectedSlotSettingsDoNotCrashNodeWarden:2400"; + const ui64 expectedSlotSize = 600ull << 30; + + TActorId fakeNodeWarden = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden); + TActorId fakeWhiteboard = runtime.AllocateEdgeActor(); + runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard); + + NKikimrBlobStorage::TPDiskConfig pdiskConfig; + pdiskConfig.SetExpectedSlotCount(17); + pdiskConfig.SetSlotSizeInUnits(7); + pdiskConfig.SetExpectedSlotSize(expectedSlotSize); + pdiskConfig.SetMaxSlots(8); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 4, 0u, expectedSlotSize); + + auto edge = runtime.AllocateEdgeActor(); + THttpRequestMock httpRequest; + NMonitoring::TMonService2HttpRequest monService2HttpRequest(nullptr, &httpRequest, nullptr, nullptr, "", + nullptr); + runtime.Send(new IEventHandle(realNodeWarden, edge, new NMon::TEvHttpInfo(monService2HttpRequest)), 0); + auto httpInfoRes = runtime.GrabEdgeEventRethrow<NMon::TEvHttpInfoRes>(edge, TDuration::Seconds(1)); + UNIT_ASSERT(httpInfoRes && httpInfoRes->Get()); + + TStringStream out; + httpInfoRes->Get()->Output(out); + UNIT_ASSERT_C(out.Str().Contains("PDiskConfig has ExpectedSlotSize"), out.Str()); + UNIT_ASSERT_C(out.Str().Contains("ExpectedSlotCount# 17"), out.Str()); + UNIT_ASSERT_C(out.Str().Contains("SlotSizeInUnits# 7"), out.Str()); + UNIT_ASSERT_C(out.Str().Contains("ExpectedSlotSize and MaxSlots take precedence"), out.Str()); + + // without MaxSlots the slot count cannot be derived from ExpectedSlotSize, so the + // invalid setting must be ignored and the explicit slot settings kept in effect + pdiskConfig.ClearMaxSlots(); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 17, 7u); + } + + CUSTOM_UNIT_TEST(TestExpectedSlotSizeLargerThanDriveKeepsZeroSlotCount) { + TTestBasicRuntime runtime(1, false); + TActorId realNodeWarden = SetupNodeWardenOnly(runtime); + + const ui32 nodeId = runtime.GetNodeId(0); + const ui32 pdiskId = 1007; + const TString pdiskPath = "SectorMap:TestExpectedSlotSizeLargerThanDriveKeepsZeroSlotCount:2400"; + const ui64 expectedSlotSize = 3000ull << 30; // larger than the 2400 GB drive + + TActorId fakeNodeWarden = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden); + TActorId fakeWhiteboard = runtime.AllocateEdgeActor(); + runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard); + + NKikimrBlobStorage::TPDiskConfig pdiskConfig; + pdiskConfig.SetExpectedSlotSize(expectedSlotSize); + pdiskConfig.SetMaxSlots(8); + CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0, + &pdiskConfig, realNodeWarden); + CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden, + pdiskId, 0, 0u, expectedSlotSize); + + auto edge = runtime.AllocateEdgeActor(); + THttpRequestMock httpRequest; + NMonitoring::TMonService2HttpRequest monService2HttpRequest(nullptr, &httpRequest, nullptr, nullptr, "", + nullptr); + runtime.Send(new IEventHandle(realNodeWarden, edge, new NMon::TEvHttpInfo(monService2HttpRequest)), 0); + auto httpInfoRes = runtime.GrabEdgeEventRethrow<NMon::TEvHttpInfoRes>(edge, TDuration::Seconds(1)); + UNIT_ASSERT(httpInfoRes && httpInfoRes->Get()); + + TStringStream out; + httpInfoRes->Get()->Output(out); + UNIT_ASSERT_C(out.Str().Contains("Drive is smaller than ExpectedSlotSize"), out.Str()); + } + CUSTOM_UNIT_TEST(TestInferPDiskSlotCountWithRealNodeWarden) { TTestBasicRuntime runtime(1, false); TActorId realNodeWarden = SetupNodeWardenOnly(runtime); diff --git a/ydb/core/blobstorage/nodewarden/distconf.h b/ydb/core/blobstorage/nodewarden/distconf.h index f22c277079f..a194af1a701 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.h +++ b/ydb/core/blobstorage/nodewarden/distconf.h @@ -476,17 +476,25 @@ namespace NKikimr::NStorage { std::optional<TString> GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID); - void AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, ui32 groupGeneration, - TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, - i64 requiredSpace, NKikimrBlobStorage::TBaseConfig *baseConfig, - bool convertToDonor, bool ignoreVSlotQuotaCheck, bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, - std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes = {}, - bool applyNodeAllowList = false); + struct TAllocateStaticGroupParams { + NKikimrBlobStorage::TStorageConfig *Config = nullptr; + TGroupId GroupId; + ui32 GroupGeneration = 0; + TBlobStorageGroupType GroupType; + THashMap<TVDiskIdShort, NBsController::TPDiskId> ReplacedDisks; + NBsController::TGroupMapper::TForbiddenPDisks ForbiddenPDisks; + i64 RequiredSpace = 0; + const NKikimrBlobStorage::TBaseConfig *BaseConfig = nullptr; + bool ConvertToDonor = false; + bool IgnoreVSlotQuotaCheck = false; + bool AllowUnusableDisks = false; + bool IsSelfHealReasonDecommit = false; + TBridgePileId BridgePileId; + std::optional<TGroupId> BridgeProxyGroupId; + bool ApplySelfHealNodeAllowList = false; + }; + + void AllocateStaticGroup(TAllocateStaticGroupParams params); bool UpdateConfig(NKikimrBlobStorage::TStorageConfig *config); @@ -518,7 +526,9 @@ namespace NKikimr::NStorage { bool GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig , std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse = {} , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {} + , bool automaticManagement = true , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 200 diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp index 4dec85b0917..9e99daffe52 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp @@ -37,10 +37,15 @@ namespace NKikimr::NStorage { TGroupId groupId = TGroupId::Zero(); auto allocateGroup = [&](TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId) { - AllocateStaticGroup(config, groupId, /*groupGeneration=*/ 1, TBlobStorageGroupType(species), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, {}, {}, 0, - nullptr, false, true, false, bridgePileId, bridgeProxyGroupId); + AllocateStaticGroup({ + .Config = config, + .GroupId = groupId, + .GroupGeneration = 1, + .GroupType = TBlobStorageGroupType(species), + .IgnoreVSlotQuotaCheck = true, + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + }); const auto& groups = config->GetBlobStorageConfig().GetServiceSet().GetGroups(); const auto& allocatedGroup = groups.at(groups.size() - 1); @@ -127,7 +132,12 @@ namespace NKikimr::NStorage { config->Mutable##NAME##Config()->CopyFrom(Cfg->DomainsConfig->GetExplicit##NAME##Config()); \ } \ if (!config->Has##NAME##Config()) { \ - GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes); \ + std::unordered_set<ui32> nodesToUse; \ + if (Cfg->SelfManagementConfig && Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes, nodesToUse); \ } UPDATE_EXPLICIT_CONFIG(StateStorage) @@ -139,32 +149,28 @@ namespace NKikimr::NStorage { return std::nullopt; } - void TDistributedConfigKeeper::AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, - ui32 groupGeneration, TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, i64 requiredSpace, - NKikimrBlobStorage::TBaseConfig *baseConfig, bool convertToDonor, bool ignoreVSlotQuotaCheck, - bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes, bool applyNodeAllowList) { + void TDistributedConfigKeeper::AllocateStaticGroup(TAllocateStaticGroupParams params) { using TPDiskId = NBsController::TPDiskId; - NKikimrConfig::TBlobStorageConfig *bsConfig = config->MutableBlobStorageConfig(); + auto& config = *params.Config; + auto& replacedDisks = params.ReplacedDisks; + const auto& selfManagementConfig = config.GetSelfManagementConfig(); + const auto& selfHealAllowedNodes = selfManagementConfig.GetStaticGroupSelfHealAllowedNodes(); + NKikimrConfig::TBlobStorageConfig *bsConfig = config.MutableBlobStorageConfig(); // build node location map THashMap<ui32, TNodeLocation> nodeLocations; THashSet<ui32> allowedNodeIds; - for (const auto& node : config->GetAllNodes()) { + for (const auto& node : config.GetAllNodes()) { TNodeLocation location(node.GetLocation()); nodeLocations.try_emplace(node.GetNodeId(), location); - if (bridgePileId == ResolveNodePileId(location)) { + if (params.BridgePileId == ResolveNodePileId(location)) { allowedNodeIds.insert(node.GetNodeId()); } } // when restricting self-heal targets, only these node ids may host relocated vdisks - const bool restrictSelfHealNodes = applyNodeAllowList && !selfHealAllowedNodes.empty(); + const bool restrictSelfHealNodes = params.ApplySelfHealNodeAllowList && !selfHealAllowedNodes.empty(); const THashSet<ui32> selfHealAllowedNodeSet = restrictSelfHealNodes ? THashSet<ui32>(selfHealAllowedNodes.begin(), selfHealAllowedNodes.end()) : THashSet<ui32>{}; @@ -178,13 +184,36 @@ namespace NKikimr::NStorage { bool AdjustSpaceAvailable = false; }; + struct TPDiskMapperSettings { + ui32 SlotCount = 0; + ui32 SlotSizeInUnits = 0; + ui64 SlotSizeInBytes = 0; + ui32 MaxSlots = 0; + }; + THashMap<TPDiskId, TPDiskInfo> pdisks; + THashMap<TPDiskId, TPDiskMapperSettings> pdiskMapperSettings; + + auto applyPDiskConfig = [](TPDiskMapperSettings& settings, const NKikimrBlobStorage::TPDiskConfig& pdiskConfig) { + if (pdiskConfig.HasExpectedSlotCount()) { + settings.SlotCount = pdiskConfig.GetExpectedSlotCount(); + } + if (pdiskConfig.HasSlotSizeInUnits()) { + settings.SlotSizeInUnits = pdiskConfig.GetSlotSizeInUnits(); + } + if (pdiskConfig.HasExpectedSlotSize()) { + settings.SlotSizeInBytes = pdiskConfig.GetExpectedSlotSize(); + } + if (pdiskConfig.HasMaxSlots()) { + settings.MaxSlots = pdiskConfig.GetMaxSlots(); + } + }; auto checkMatch = [&](NKikimrBlobStorage::EPDiskType type, bool sharedWithOs, bool readCentric, ui64 kind) { - if (type == pdiskType) { + if (selfManagementConfig.HasPDiskType() && type == selfManagementConfig.GetPDiskType()) { return true; } - for (const auto& pdiskFilter : pdiskFilters) { + for (const auto& pdiskFilter : selfManagementConfig.GetPDiskFilter()) { bool m = true; for (const auto& p : pdiskFilter.GetProperty()) { bool pMatch = false; @@ -218,12 +247,13 @@ namespace NKikimr::NStorage { ui32 defaultMaxSlots = 16; - if (baseConfig) { + if (params.BaseConfig) { + const auto& baseConfig = *params.BaseConfig; std::optional<NKikimrBlobStorage::TPDiskSpaceColor::E> pdiskSpaceColorBorder; ui32 pdiskSpaceMarginPromille = 150; - if (baseConfig->HasSettings()) { - const auto& settings = baseConfig->GetSettings(); + if (baseConfig.HasSettings()) { + const auto& settings = baseConfig.GetSettings(); if (settings.DefaultMaxSlotsSize()) { defaultMaxSlots = settings.GetDefaultMaxSlots(0); } @@ -235,7 +265,7 @@ namespace NKikimr::NStorage { } } - for (const auto& pdisk : baseConfig->GetPDisk()) { + for (const auto& pdisk : baseConfig.GetPDisk()) { if (!checkMatch(pdisk.GetType(), pdisk.GetSharedWithOs(), pdisk.GetReadCentric(), pdisk.GetKind())) { continue; } @@ -252,6 +282,13 @@ namespace NKikimr::NStorage { pdisk.GetKind()).GetRaw()); if (pdisk.HasPDiskConfig()) { r.MutablePDiskConfig()->CopyFrom(pdisk.GetPDiskConfig()); + applyPDiskConfig(pdiskMapperSettings[pdiskId], pdisk.GetPDiskConfig()); + } + if (pdisk.GetExpectedSlotCount()) { + pdiskMapperSettings[pdiskId].SlotCount = pdisk.GetExpectedSlotCount(); + } + if (pdisk.GetExpectedSlotSize()) { + pdiskMapperSettings[pdiskId].SlotSizeInBytes = pdisk.GetExpectedSlotSize(); } // this 'usable' logic repeats the one in BS_CONTROLLER @@ -261,14 +298,21 @@ namespace NKikimr::NStorage { } const bool usableInTermsOfDecommission = pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE || - pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !isSelfHealReasonDecommit; + pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !params.IsSelfHealReasonDecommit; if (!usableInTermsOfDecommission) { pdiskInfo.Usable = false; pdiskInfo.WhyUnusable += 'D'; } - if (!ignoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig->HasSettings()) { + if (!params.IgnoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig.HasSettings()) { const auto& m = pdisk.GetPDiskMetrics(); + auto& settings = pdiskMapperSettings[pdiskId]; + if (settings.SlotSizeInBytes && m.HasSlotCount()) { + settings.SlotCount = m.GetSlotCount(); + } + if (m.HasSlotSizeInUnits()) { + settings.SlotSizeInUnits = m.GetSlotSizeInUnits(); + } if (m.HasEnforcedDynamicSlotSize() && pdiskSpaceColorBorder >= NKikimrBlobStorage::TPDiskSpaceColor::YELLOW) { pdiskInfo.SpaceAvailable = m.GetEnforcedDynamicSlotSize() * (1000 - pdiskSpaceMarginPromille) / 1000; } else { @@ -282,7 +326,7 @@ namespace NKikimr::NStorage { } THashMap<ui32, ui64> maxGroupSlotSize; - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { if (vslot.HasVDiskMetrics()) { if (const auto& m = vslot.GetVDiskMetrics(); m.HasAllocatedSize()) { ui64& size = maxGroupSlotSize[vslot.GetGroupId()]; @@ -291,7 +335,7 @@ namespace NKikimr::NStorage { } } - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { const auto& vslotId = vslot.GetVSlotId(); const TPDiskId pdiskId(vslotId.GetNodeId(), vslotId.GetPDiskId()); if (const auto it = pdisks.find(pdiskId); it != pdisks.end()) { @@ -321,7 +365,7 @@ namespace NKikimr::NStorage { std::optional<ui32> generation; for (const auto& group : ss.GetGroups()) { - if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { generation.emplace(group.GetGroupGeneration()); ui32 failRealmIdx = 0; @@ -362,7 +406,7 @@ namespace NKikimr::NStorage { for (const auto& vdisk : ss.GetVDisks()) { const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID == groupId) { + if (vdiskId.GroupID == params.GroupId) { if (!generation) { throw TExConfigError() << "missing record for group being reconfigured"; } else if (vdiskId.GroupGeneration == *generation && !replacedDisks.contains(vdiskId)) { @@ -375,10 +419,15 @@ namespace NKikimr::NStorage { for (const auto& pdisk : ss.GetPDisks()) { const TPDiskId pdiskId(pdisk.GetNodeID(), pdisk.GetPDiskID()); - if (requiredPDiskIds.contains(pdiskId)) { + if (params.AllowUnusableDisks && requiredPDiskIds.contains(pdiskId)) { if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) { - auto& r = it->second.Record; - r.CopyFrom(pdisk); + TPDiskInfo& pdiskInfo = it->second; + pdiskInfo.Record.CopyFrom(pdisk); + pdiskInfo.Usable = false; + pdiskInfo.WhyUnusable += 'X'; + if (pdisk.HasPDiskConfig()) { + applyPDiskConfig(pdiskMapperSettings[pdiskId], pdisk.GetPDiskConfig()); + } } } @@ -423,7 +472,8 @@ namespace NKikimr::NStorage { if (checkMatch(drive.GetType(), drive.GetSharedWithOs(), drive.GetReadCentric(), drive.GetKind())) { const TPDiskId pdiskId(nodeId, ++maxPDiskId[nodeId]); if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) { - auto& r = it->second.Record; + TPDiskInfo& pdiskInfo = it->second; + auto& r = pdiskInfo.Record; r.SetNodeID(pdiskId.NodeId); r.SetPDiskID(pdiskId.PDiskId); r.SetPath(drive.GetPath()); @@ -432,16 +482,17 @@ namespace NKikimr::NStorage { drive.GetKind())); if (drive.HasPDiskConfig()) { r.MutablePDiskConfig()->CopyFrom(drive.GetPDiskConfig()); + applyPDiskConfig(pdiskMapperSettings[pdiskId], drive.GetPDiskConfig()); } } else { throw TExConfigError() << "duplicate PDiskId"; } } }; - EnumerateConfigDrives(*config, 0, processDrive, nullptr, true); + EnumerateConfigDrives(config, 0, processDrive, nullptr, true); // group mapper - NBsController::TGroupGeometryInfo geom(gtype.GetErasure(), geometry); + NBsController::TGroupGeometryInfo geom(params.GroupType.GetErasure(), selfManagementConfig.GetGeometry()); NBsController::TGroupMapper mapper(geom); for (const auto& [pdiskId, item] : pdisks) { @@ -450,17 +501,22 @@ namespace NKikimr::NStorage { throw TExConfigError() << "no location for node"; } + const auto settingsIt = pdiskMapperSettings.find(pdiskId); + const TPDiskMapperSettings settings = settingsIt != pdiskMapperSettings.end() + ? settingsIt->second + : TPDiskMapperSettings(); + ui32 maxSlots = defaultMaxSlots; - ui32 slotSizeInUnits = 0; - if (item.Record.HasPDiskConfig()) { - const auto& pdiskConfig = item.Record.GetPDiskConfig(); - if (pdiskConfig.HasExpectedSlotCount()) { - maxSlots = pdiskConfig.GetExpectedSlotCount(); - } - slotSizeInUnits = pdiskConfig.GetSlotSizeInUnits(); + if (settings.SlotCount) { + maxSlots = settings.SlotCount; + } else if (settings.SlotSizeInBytes) { + // Slot count for byte-sized slots is calculated by NodeWarden and arrives via PDisk config or + // metrics. Until then MaxSlots is its upper bound (ExpectedSlotCount = min(size/slot, MaxSlots)), + // which keeps fresh drives usable for static group allocation during bootstrap. + maxSlots = settings.MaxSlots; } - const bool pileFilter = !bridgePileId || allowedNodeIds.contains(pdiskId.NodeId); + const bool pileFilter = !params.BridgePileId || allowedNodeIds.contains(pdiskId.NodeId); const bool nodeAllowFilter = !restrictSelfHealNodes || selfHealAllowedNodeSet.contains(pdiskId.NodeId); TString whyUnusable = item.WhyUnusable; @@ -474,7 +530,8 @@ namespace NKikimr::NStorage { .Usable = item.Usable && pileFilter && nodeAllowFilter, .NumSlots = item.UsedSlots, .MaxSlots = maxSlots, - .SlotSizeInUnits = slotSizeInUnits, + .SlotSizeInUnits = settings.SlotSizeInUnits, + .SlotSizeInBytes = settings.SlotSizeInBytes, .Groups{}, .SpaceAvailable = item.SpaceAvailable, .Operational = true, @@ -502,8 +559,8 @@ namespace NKikimr::NStorage { NBsController::TGroupMapperError error; const ui32 groupSizeInUnits = 1; // static groups are always single-unit - if (!mapper.AllocateGroup(groupId.GetRawId(), groupDefinition, replacedDisks, forbid, - groupSizeInUnits, requiredSpace, false, {}, error)) { + if (!mapper.AllocateGroup(params.GroupId.GetRawId(), groupDefinition, replacedDisks, + params.ForbiddenPDisks, groupSizeInUnits, params.RequiredSpace, false, {}, error)) { throw TExConfigError() << "group allocation failed Error# " << error.ErrorMessage << " groupDefinition# " << dumpGroupDefinition(); } @@ -513,38 +570,38 @@ namespace NKikimr::NStorage { NKikimrBlobStorage::TGroupInfo *sGroup = nullptr; for (size_t i = 0; i < sSet->GroupsSize(); ++i) { if (const auto& group = sSet->GetGroups(i); TGroupId::FromProto(&group, - &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { sGroup = sSet->MutableGroups(i); break; } } if (!sGroup) { sGroup = sSet->AddGroups(); - groupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); - sGroup->SetErasureSpecies(gtype.GetErasure()); + params.GroupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); + sGroup->SetErasureSpecies(params.GroupType.GetErasure()); } else { sGroup->ClearRings(); } - sGroup->SetGroupGeneration(groupGeneration); + sGroup->SetGroupGeneration(params.GroupGeneration); - if (bridgeProxyGroupId) { - bridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); + if (params.BridgeProxyGroupId) { + params.BridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); } - bridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); + params.BridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); THashMap<TVDiskIdShort, NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk::TDonor>> donors; for (size_t i = 0; i < sSet->VDisksSize(); ++i) { const auto& vdisk = sSet->GetVDisks(i); const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID != groupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { + if (vdiskId.GroupID != params.GroupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { continue; } auto *m = sSet->MutableVDisks(i); if (replacedDisks.contains(vdiskId)) { if (m->HasDonorMode()) { // this disk is already a donor, nothing to do about it - } else if (convertToDonor) { + } else if (params.ConvertToDonor) { // make this disk a donor auto *donorMode = m->MutableDonorMode(); donorMode->SetNumFailRealms(groupDefinition.size()); @@ -560,7 +617,7 @@ namespace NKikimr::NStorage { donor->MutableVDiskId()->CopyFrom(m->GetVDiskID()); donor->MutableVDiskLocation()->CopyFrom(m->GetVDiskLocation()); } else { - m->MutableVDiskID()->SetGroupGeneration(groupGeneration); + m->MutableVDiskID()->SetGroupGeneration(params.GroupGeneration); } } @@ -596,7 +653,7 @@ namespace NKikimr::NStorage { sLoc->SetPDiskGuid(pdisk.GetPDiskGuid()); auto *sDisk = sSet->AddVDisks(); - VDiskIDFromVDiskID(TVDiskID(groupId, groupGeneration, vdiskId), sDisk->MutableVDiskID()); + VDiskIDFromVDiskID(TVDiskID(params.GroupId, params.GroupGeneration, vdiskId), sDisk->MutableVDiskID()); sDisk->SetVDiskKind(NKikimrBlobStorage::TVDiskKind::Default); sDisk->MutableVDiskLocation()->CopyFrom(*sLoc); if (const auto it = donors.find(vdiskId); it != donors.end()) { @@ -608,14 +665,40 @@ namespace NKikimr::NStorage { bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig + , bool automaticManagement , ui32 overrideReplicasInRingCount , ui32 overrideRingsCount , ui32 replicasSpecificVolume ) { + if (!automaticManagement) { + ss->CopyFrom(oldConfig); + + const auto collectNodes = [&](const auto& self, const auto& ring) -> void { + for (ui32 nodeId : ring.GetNode()) { + usedNodes.insert(nodeId); + } + for (const auto& subRing : ring.GetRing()) { + self(self, subRing); + } + }; + + if (oldConfig.HasRing()) { + collectNodes(collectNodes, oldConfig.GetRing()); + } + for (const auto& rg : oldConfig.GetRingGroups()) { + collectNodes(collectNodes, rg); + } + + return true; + } std::map<TBridgePileId, THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>> nodes; bool goodConfig = true; for (const auto& node : baseConfig.GetAllNodes()) { + if (!nodesToUse.empty() && !nodesToUse.contains(node.GetNodeId())) { + continue; + } TNodeLocation location(node.GetLocation()); TBridgePileId pileId = ResolveNodePileId(location); nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location); @@ -625,6 +708,13 @@ namespace NKikimr::NStorage { generator.AddRingGroup(ss); goodConfig &= generator.IsGoodConfig(); } + if (ss->RingGroupsSize() == 0) { + // nodesToUse was non-empty, but none of the specified node IDs exist in baseConfig + // (e.g. StateStorageSelfHealAllowedNodes referencing decommissioned nodes); avoid + // returning a bogus empty state storage config that would trigger a destructive + // full reconfiguration. + return false; + } return goodConfig; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke.h b/ydb/core/blobstorage/nodewarden/distconf_invoke.h index 173710ba3c9..f6c887a0ea1 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke.h +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke.h @@ -81,6 +81,7 @@ namespace NKikimr::NStorage { THashMultiMap<ui32, TVDiskID> NodeToVDisk; THashMap<TActorId, TVDiskID> ActorToVDisk; + THashMap<ui32, TVector<std::pair<TActorId, TVDiskID>>> VStatusQueriesAwaitingConnection; std::optional<NKikimrBlobStorage::TBaseConfig> BaseConfig; THashSet<TVDiskID> PendingVDiskIds; TIntrusivePtr<TBlobStorageGroupInfo> GroupInfo; @@ -88,6 +89,8 @@ namespace NKikimr::NStorage { void ReassignGroupDisk(const TQuery::TReassignGroupDisk& cmd); void IssueVStatusQueries(const NKikimrBlobStorage::TGroupInfo& group); + void SendVStatusQuery(TActorId actorId, TVDiskID vdiskId, TActorId sessionId = {}); + void SendPendingVStatusQueries(ui32 nodeId, TActorId sessionId); void Handle(TEvBlobStorage::TEvVStatusResult::TPtr ev); void Handle(TEvents::TEvUndelivered::TPtr ev); void OnVStatusError(TVDiskID vdiskId); diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_common.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_common.cpp index 79f5515c49c..1687ceeb25d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_common.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_common.cpp @@ -94,10 +94,12 @@ namespace NKikimr::NStorage { } void TInvokeRequestHandlerActor::Handle(TEvInterconnect::TEvNodeConnected::TPtr ev) { - const auto it = Subscriptions.find(ev->Get()->NodeId); + const ui32 nodeId = ev->Get()->NodeId; + const auto it = Subscriptions.find(nodeId); Y_ABORT_UNLESS(it != Subscriptions.end()); Y_ABORT_UNLESS(!it->second || it->second == ev->Sender); it->second = ev->Sender; + SendPendingVStatusQueries(nodeId, ev->Sender); } void TInvokeRequestHandlerActor::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr ev) { @@ -106,6 +108,7 @@ namespace NKikimr::NStorage { Y_ABORT_UNLESS(it != Subscriptions.end()); Y_ABORT_UNLESS(!it->second || it->second == ev->Sender); Subscriptions.erase(it); + VStatusQueriesAwaitingConnection.erase(nodeId); for (auto [begin, end] = NodeToVDisk.equal_range(nodeId); begin != end; ++begin) { OnVStatusError(begin->second); } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp index 2dc6f9d839d..9b3d68d6c2f 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp @@ -15,15 +15,27 @@ namespace NKikimr::NStorage { const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig; bool result = true; std::unordered_set<ui32> usedNodes; - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + +#define GENERATE_CONFIG(NAME) \ + { \ + std::unordered_set<ui32> nodesToUse; \ + if (Self->Cfg->SelfManagementConfig && Self->Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Self->Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + const bool automaticManagement = !Self->Cfg->SelfManagementConfig || Self->Cfg->SelfManagementConfig->GetAutomatic##NAME##Management(); \ + result &= Self->GenerateStateStorageConfig(currentConfig->Mutable##NAME##Config(), config, usedNodes, nodesToUse, config.Get##NAME##Config(), automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); \ + if (pileupReplicas) { \ + usedNodes.clear(); \ + } \ + } + + GENERATE_CONFIG(StateStorage) + GENERATE_CONFIG(StateStorageBoard) + GENERATE_CONFIG(SchemeBoard) + +#undef GENERATE_CONFIG + return result; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp index 30884b3f0cf..8710dc9eb78 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp @@ -45,24 +45,56 @@ namespace NKikimr::NStorage { } SuccessfulVDisks.emplace(&GroupInfo->GetTopology()); + TVector<ui32> nodesToConnect; for (ui32 i = 0, num = GroupInfo->GetTotalVDisksNum(); i < num; ++i) { const TVDiskID vdiskId = GroupInfo->GetVDiskId(i); const TActorId actorId = GroupInfo->GetActorId(i); - const ui32 flags = IEventHandle::FlagTrackDelivery | - (actorId.NodeId() == SelfId().NodeId() ? 0 : IEventHandle::FlagSubscribeOnSession); - YDB_LOG_DEBUG("Sending TEvVStatus", - {"marker", "NWDC73"}, - {"selfId", SelfId()}, - {"VDiskId", vdiskId}, - {"actorId", actorId}); - Send(actorId, new TEvBlobStorage::TEvVStatus(vdiskId), flags); - if (actorId.NodeId() != SelfId().NodeId()) { - NodeToVDisk.emplace(actorId.NodeId(), vdiskId); - Subscriptions.try_emplace(actorId.NodeId()); + const ui32 nodeId = actorId.NodeId(); + if (nodeId == SelfId().NodeId()) { + SendVStatusQuery(actorId, vdiskId); + } else { + NodeToVDisk.emplace(nodeId, vdiskId); + const auto [it, inserted] = Subscriptions.try_emplace(nodeId); + if (it->second) { + SendVStatusQuery(actorId, vdiskId, it->second); + } else { + VStatusQueriesAwaitingConnection[nodeId].emplace_back(actorId, vdiskId); + } + if (inserted) { + nodesToConnect.push_back(nodeId); + } } ActorToVDisk.emplace(actorId, vdiskId); PendingVDiskIds.emplace(vdiskId); } + + for (const ui32 nodeId : nodesToConnect) { + Send(TActivationContext::InterconnectProxy(nodeId), new TEvInterconnect::TEvConnectNode); + } + } + + void TInvokeRequestHandlerActor::SendVStatusQuery(TActorId actorId, TVDiskID vdiskId, TActorId sessionId) { + YDB_LOG_DEBUG("Sending TEvVStatus", + {"marker", "NWDC73"}, + {"selfId", SelfId()}, + {"VDiskId", vdiskId}, + {"actorId", actorId}); + + auto ev = std::make_unique<IEventHandle>(actorId, SelfId(), new TEvBlobStorage::TEvVStatus(vdiskId), IEventHandle::FlagTrackDelivery); + if (sessionId) { + ev->Rewrite(TEvInterconnect::EvForward, sessionId); + } + TActivationContext::Send(ev.release()); + } + + void TInvokeRequestHandlerActor::SendPendingVStatusQueries(ui32 nodeId, TActorId sessionId) { + if (const auto it = VStatusQueriesAwaitingConnection.find(nodeId); it != VStatusQueriesAwaitingConnection.end()) { + auto queries = std::move(it->second); + VStatusQueriesAwaitingConnection.erase(it); + for (const auto& [actorId, vdiskId] : queries) { + SendVStatusQuery(actorId, vdiskId, sessionId); + } + } } void TInvokeRequestHandlerActor::Handle(TEvBlobStorage::TEvVStatusResult::TPtr ev) { @@ -90,7 +122,9 @@ namespace NKikimr::NStorage { } void TInvokeRequestHandlerActor::OnVStatusError(TVDiskID vdiskId) { - PendingVDiskIds.erase(vdiskId); + if (!PendingVDiskIds.erase(vdiskId)) { + return; + } CheckReassignGroupDisk(); } @@ -176,8 +210,6 @@ namespace NKikimr::NStorage { } const auto& ss = bsConfig.GetServiceSet(); - const auto& smConfig = config.GetSelfManagementConfig(); - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; NBsController::TGroupMapper::TForbiddenPDisks forbid; for (const auto& vdisk : ss.GetVDisks()) { @@ -207,14 +239,23 @@ namespace NKikimr::NStorage { std::optional<TGroupId> bridgeProxyGroupId = group.HasBridgeProxyGroupId() ? std::make_optional(TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetBridgeProxyGroupId)) : std::nullopt; - Self->AllocateStaticGroup(&config, vdiskId.GroupID, vdiskId.GroupGeneration + 1, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, - replacedDisks, forbid, maxSlotSize, - &BaseConfig.value(), cmd.GetConvertToDonor(), cmd.GetIgnoreVSlotQuotaCheck(), - cmd.GetIsSelfHealReasonDecommit(), bridgePileId, bridgeProxyGroupId, - smConfig.GetStaticGroupSelfHealAllowedNodes(), cmd.GetFromSelfHeal()); + Self->AllocateStaticGroup({ + .Config = &config, + .GroupId = vdiskId.GroupID, + .GroupGeneration = vdiskId.GroupGeneration + 1, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .RequiredSpace = static_cast<i64>(maxSlotSize), + .BaseConfig = &BaseConfig.value(), + .ConvertToDonor = cmd.GetConvertToDonor(), + .IgnoreVSlotQuotaCheck = cmd.GetIgnoreVSlotQuotaCheck(), + .AllowUnusableDisks = cmd.GetAllowUnusableDisks(), + .IsSelfHealReasonDecommit = cmd.GetIsSelfHealReasonDecommit(), + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + .ApplySelfHealNodeAllowList = cmd.GetFromSelfHeal(), + }); } catch (const TExConfigError& ex) { YDB_LOG_NOTICE("ReassignGroupDisk failed to allocate group", {"marker", "NWDC76"}, diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp index aa8b5bc41fb..052c77e5e6d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp @@ -70,13 +70,23 @@ namespace NKikimr::NStorage { RingsInGroupCount = minNodesInGroup < 8 ? minNodesInGroup : 8; if (RingsInGroupCount > minNodesInGroup) { RingsInGroupCount = minNodesInGroup; + if (OverrideRingsCount > 0) { + EnoughNodesForOverride = false; + } } NToSelect = RingsInGroupCount < 3 ? 1 : (RingsInGroupCount < 5 ? 3 : 5); ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / ReplicasSpecificVolume); } else { if (OverrideRingsCount == 3 || OverrideRingsCount == 9) { RingsInGroupCount = OverrideRingsCount / 3; + if (RingsInGroupCount > minNodesInGroup) { + RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; + EnoughNodesForOverride = false; + } } else { + // any other value (including 0, meaning "no override") is not applicable to the + // multi-group (3-DC) topology and is silently ignored - this is not an insufficient + // nodes condition, just a non-applicable override RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; } NToSelect = RingsInGroupCount < 3 ? 3 : 9; @@ -87,11 +97,17 @@ namespace NKikimr::NStorage { ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / ReplicasSpecificVolume); } if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) { + if (OverrideReplicasInRingCount > 1) { + EnoughNodesForOverride = false; + } ReplicasInRingCount = 1; } } bool TStateStoragePerPileGenerator::IsGoodConfig() const { + if (!EnoughNodesForOverride) { + return false; + } for (auto &nodes : Rings) { for (auto nodeId : nodes) { if (CalcNodeState(nodeId, false) > 1) { @@ -102,6 +118,10 @@ namespace NKikimr::NStorage { return true; } + bool TStateStoragePerPileGenerator::IsEnoughNodesForOverride() const { + return EnoughNodesForOverride; + } + void TStateStoragePerPileGenerator::AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss) { auto *rg = ss->AddRingGroups(); PileId.CopyToProto(rg, &NKikimrConfig::TDomainsConfig::TStateStorage::TRing::SetBridgePileId); diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h index 1a55b2c7bd0..4fb5702b0f9 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h @@ -20,6 +20,8 @@ namespace NKikimr::NStorage { bool IsGoodConfig() const; void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss); + bool IsEnoughNodesForOverride() const; + private: static constexpr ui32 NodeStatesSize = (ui32)ENodeState::NODE_STATE_MAX; @@ -48,5 +50,6 @@ namespace NKikimr::NStorage { ui32 OverrideReplicasInRingCount = 0; ui32 OverrideRingsCount = 0; ui32 ReplicasSpecificVolume = 200; + bool EnoughNodesForOverride = true; }; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp index 28d707927f0..044f7264bef 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp @@ -23,7 +23,112 @@ namespace NBlobStorageNodeWardenTest{ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { - NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200) { + Y_UNIT_TEST(AllocateStaticGroupRespectsExpectedSlotSizeFromBaseConfig) { + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); + + NKikimrBlobStorage::TStorageConfig config; + auto *node = config.AddAllNodes(); + node->SetNodeId(1); + node->MutableLocation()->SetDataCenter("dc-1"); + node->MutableLocation()->SetRack("rack-1"); + node->MutableLocation()->SetUnit("unit-1"); + + NKikimrBlobStorage::TBaseConfig baseConfig; + baseConfig.MutableSettings()->AddDefaultMaxSlots(16); + + auto *pdisk = baseConfig.AddPDisk(); + pdisk->SetNodeId(1); + pdisk->SetPDiskId(1); + pdisk->SetPath("/dev/disk1"); + pdisk->SetType(NKikimrBlobStorage::SSD); + pdisk->SetKind(0); + pdisk->SetGuid(1); + pdisk->SetDriveStatus(NKikimrBlobStorage::ACTIVE); + pdisk->SetDecommitStatus(NKikimrBlobStorage::DECOMMIT_NONE); + pdisk->SetExpectedSlotCount(4); + pdisk->SetExpectedSlotSize(100); + pdisk->MutablePDiskConfig()->SetExpectedSlotSize(100); + pdisk->MutablePDiskConfig()->SetMaxSlots(4); + pdisk->MutablePDiskMetrics()->SetTotalSize(1000); + pdisk->MutablePDiskMetrics()->SetAvailableSize(1000); + + NKikimrBlobStorage::TGroupGeometry geometry; + geometry.SetNumFailRealms(1); + geometry.SetNumFailDomainsPerFailRealm(1); + geometry.SetNumVDisksPerFailDomain(1); + auto *selfManagementConfig = config.MutableSelfManagementConfig(); + selfManagementConfig->MutableGeometry()->CopyFrom(geometry); + selfManagementConfig->SetPDiskType(NKikimrBlobStorage::SSD); + + try { + keeper.AllocateStaticGroup({ + .Config = &config, + .GroupId = TGroupId::Zero(), + .GroupGeneration = 1, + .GroupType = TBlobStorageGroupType(TBlobStorageGroupType::ErasureNone), + .RequiredSpace = 200, + .BaseConfig = &baseConfig, + }); + UNIT_FAIL("Expected group allocation to fail"); + } catch (const NStorage::TDistributedConfigKeeper::TExConfigError& ex) { + const TString error = ex.what(); + UNIT_ASSERT_C(error.Contains("group allocation failed"), error); + UNIT_ASSERT_C(error.Contains("-v"), error); + } + } + + Y_UNIT_TEST(AllocateStaticGroupOnFreshDrivesWithExpectedSlotSize) { + // Bootstrap self-assembly: the static group must be allocatable on drives that come + // straight from the config with expected_slot_size + max_slots, when neither the + // materialized ExpectedSlotCount nor PDisk metrics exist yet. MaxSlots serves as the + // slot count upper bound until NodeWarden computes the real value from the drive size. + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); + + NKikimrBlobStorage::TStorageConfig config; + auto *node = config.AddAllNodes(); + node->SetNodeId(1); + node->MutableLocation()->SetDataCenter("dc-1"); + node->MutableLocation()->SetRack("rack-1"); + node->MutableLocation()->SetUnit("unit-1"); + + auto *bsConfig = config.MutableBlobStorageConfig(); + auto *hostConfig = bsConfig->AddDefineHostConfig(); + hostConfig->SetHostConfigId(1); + auto *drive = hostConfig->AddDrive(); + drive->SetPath("/dev/disk1"); + drive->SetType(NKikimrBlobStorage::SSD); + drive->MutablePDiskConfig()->SetExpectedSlotSize(100ull << 30); + drive->MutablePDiskConfig()->SetMaxSlots(4); + + auto *host = bsConfig->MutableDefineBox()->AddHost(); + host->SetHostConfigId(1); + host->SetEnforcedNodeId(1); + + NKikimrBlobStorage::TGroupGeometry geometry; + geometry.SetNumFailRealms(1); + geometry.SetNumFailDomainsPerFailRealm(1); + geometry.SetNumVDisksPerFailDomain(1); + auto *selfManagementConfig = config.MutableSelfManagementConfig(); + selfManagementConfig->MutableGeometry()->CopyFrom(geometry); + selfManagementConfig->SetPDiskType(NKikimrBlobStorage::SSD); + + keeper.AllocateStaticGroup({ + .Config = &config, + .GroupId = TGroupId::Zero(), + .GroupGeneration = 1, + .GroupType = TBlobStorageGroupType(TBlobStorageGroupType::ErasureNone), + }); + + const auto& serviceSet = config.GetBlobStorageConfig().GetServiceSet(); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.PDisksSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.GetPDisks(0).GetPath(), "/dev/disk1"); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.GetPDisks(0).GetPDiskConfig().GetExpectedSlotSize(), 100ull << 30); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.GetPDisks(0).GetPDiskConfig().GetMaxSlots(), 4); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.VDisksSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(serviceSet.GroupsSize(), 1); + } + + NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200, std::unordered_set<ui32> nodesToUse = {}, bool *goodConfigOut = nullptr, const NKikimrConfig::TDomainsConfig::TStateStorage& oldSS = {}, bool automaticManagement = true) { NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); NKikimrConfig::TDomainsConfig::TStateStorage ss; NKikimrBlobStorage::TStorageConfig config; @@ -31,7 +136,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { auto *node = config.AddAllNodes(); node->SetNodeId(i + 1); } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -46,6 +154,9 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 1000 + , std::unordered_set<ui32> nodesToUse = {} + , bool *goodConfigOut = nullptr + , bool automaticManagement = true ) { NKikimrBlobStorage::TStorageConfig config; ui32 nodeId = 1; @@ -74,7 +185,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { for (auto [nodeId, state] : nodesState) { keeper.SelfHealNodesState[nodeId] = state; } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -209,6 +323,279 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { " Ring { Node: 4 Node: 5 Node: 6 } Ring { Node: 7 Node: 8 Node: 9 } } }"); CheckStateStorage2(GenerateSimpleStateStorage(16, {}, 0, 2, 5), "{ RingGroups { NToSelect: 1 Ring { Node: 1 Node: 2 Node: 3 Node: 4 } Ring { Node: 5 Node: 6 Node: 7 Node: 8 } } }"); } + + Y_UNIT_TEST(NodesToUseEmptyMeansNoRestriction) { + // Explicit regression check: passing an empty nodesToUse set behaves exactly + // like not passing the parameter at all (default argument in other tests). + CheckStateStorage(GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}), 5, {1, 2, 3, 4, 5, 6, 7, 8}); + CheckStateStorage(GenerateDCStateStorage(3, 1, 1, {}, {}, {}, 9, 0, 0, 1000, {}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseRestrictsSimplePool) { + // Out of 10 available nodes, only {5, 6, 7} are allowed to be used. + CheckStateStorage(GenerateSimpleStateStorage(10, {}, 0, 0, 200, {5, 6, 7}), 3, {5, 6, 7}); + } + + Y_UNIT_TEST(NodesToUseKeepsOriginalNodeIds) { + // Restrict a big pool of 100 nodes down to 8 specific nodes; the algorithm + // should behave exactly as if only those 8 nodes existed, while preserving + // their original (non-contiguous) node ids. + CheckStateStorage( + GenerateSimpleStateStorage(100, {}, 0, 0, 200, {10, 20, 30, 40, 50, 60, 70, 80}), + 5, {10, 20, 30, 40, 50, 60, 70, 80}); + } + + Y_UNIT_TEST(NodesToUseIgnoresUnknownNodeIds) { + // Node ids in nodesToUse that are not present in baseConfig's AllNodes + // must simply be ignored, not cause a crash or a bad config. + CheckStateStorage(GenerateSimpleStateStorage(3, {}, 0, 0, 200, {1, 2, 3, 9999}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseAllUnknownYieldsBadConfigNotEmptySS) { + // Regression test: if nodesToUse is non-empty, but *none* of the specified node ids + // exist in baseConfig's AllNodes (e.g. StateStorageSelfHealAllowedNodes referencing + // decommissioned nodes), the generator must not silently produce an empty (but "good") + // state storage config, since that would trigger a destructive full reconfiguration + // to an empty state storage in the self-heal path. Instead, GenerateStateStorageConfig + // must report the config as bad. + bool goodConfig = true; + auto ss = GenerateSimpleStateStorage(3, {}, 0, 0, 200, {9998, 9999}, &goodConfig); + UNIT_ASSERT(!goodConfig); + UNIT_ASSERT_EQUAL(ss.RingGroupsSize(), 0); + } + + Y_UNIT_TEST(NodesToUseAllUnknownInDCTopologyYieldsBadConfig) { + // Same regression, but exercised through the multi-DC/rack topology generator, since + // that is the code path used in practice by the self-heal state storage logic. + bool goodConfig = true; + auto ss = GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, 0, 1000, /*nodesToUse=*/{9998, 9999}, &goodConfig); + UNIT_ASSERT(!goodConfig); + UNIT_ASSERT_EQUAL(ss.RingGroupsSize(), 0); + } + + Y_UNIT_TEST(NodesToUseForcesAlternateNodeSelectionInDC) { + // Without restriction, the default pick within each 3-node rack is its + // first node: {1, 4, 7, 10, 13, 16, 19, 22, 25}. + CheckStateStorage(GenerateDCStateStorage(3, 3, 3), 9, {1, 4, 7, 10, 13, 16, 19, 22, 25}); + + // Excluding those default nodes from nodesToUse forces the generator to + // fall back to the next available node in each rack. + std::unordered_set<ui32> nodesToUse = { + 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, 24, 26, 27 + }; + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, 0, 1000, nodesToUse), + 9, {2, 5, 8, 11, 14, 17, 20, 23, 26}); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Cases where the requested configuration (via overrideRingsCount/overrideReplicasInRingCount) + // can not be honored because nodesToUse restricts the available nodes too much. + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountSingleGroup) { + // Only 3 nodes are usable (all with a known GOOD self-heal state, via GenerateDCStateStorage), + // but the caller explicitly asks for 8 rings: impossible to satisfy. + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig); + UNIT_ASSERT(!goodConfig); + + // Sanity check: with exactly as many nodes as requested rings, the override is satisfiable. + bool goodConfig2 = false; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, 0, /*overrideRingsCount=*/3, 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig2); + UNIT_ASSERT(goodConfig2); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideReplicasInRingCountSingleGroup) { + // Only 2 nodes are usable, but the caller explicitly asks for 3 replicas per ring: + // impossible to satisfy (would need at least 3 nodes for a single ring). + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/3, /*overrideRingsCount=*/1, + 1000, /*nodesToUse=*/{1, 2}, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountMultiGroup) { + // 3-DC topology, but nodesToUse leaves only 2 nodes in one of the DCs while the + // caller explicitly requests 9 rings (3 per group) - the smallest group can't supply 3 rings. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, // dc-0: full rack representation (3 nodes) + 10, 13, // dc-1: only 2 nodes available + 19, 22, 25, // dc-2: full rack representation (3 nodes) + }; + bool goodConfig = true; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseSufficientForOverrideKeepsGoodConfig) { + // Same restriction style as above, but this time nodesToUse leaves exactly enough + // nodes in every group to satisfy the override - the config must be reported as good. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, + 10, 13, 16, + 19, 22, 25, + }; + bool goodConfig = false; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(goodConfig); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // AutomaticManagement (AutomaticStateStorageManagement / AutomaticStateStorageBoardManagement / + // AutomaticSchemeBoardManagement) semantics at the GenerateStateStorageConfig level: when the + // subsystem's automatic management is disabled, self-heal must keep the current (old) config + // untouched and report the result as "good", regardless of node states or overrides. + + Y_UNIT_TEST(AutomaticManagementDisabledKeepsOldConfigUnchanged) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/false); + + // The generator must not have touched node selection at all: result equals oldSS verbatim. + UNIT_ASSERT_EQUAL(TStringBuilder() << ss, TStringBuilder() << oldSS); + UNIT_ASSERT(goodConfig); + } + + Y_UNIT_TEST(AutomaticManagementEnabledGeneratesNewConfig) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/true); + + // With automatic management enabled (default), a fresh configuration must be generated + // from the current node pool, ignoring the stale oldSS content. + UNIT_ASSERT(TStringBuilder() << ss != TStringBuilder() << oldSS); + CheckStateStorage(ss, 5, {1, 2, 3, 4, 5, 6, 7, 8}); + } + + Y_UNIT_TEST(AutomaticManagementDisabledIgnoresBadNodeStates) { + // Even if nodesToUse/overrides would normally produce a bad config, disabling automatic + // management must short-circuit generation entirely and still report "good", since the + // old config is kept as-is and self-heal is not supposed to touch it. + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(7); + } + bool goodConfig = false; + GenerateSimpleStateStorage(100, {}, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 200, /*nodesToUse=*/{1, 2, 3}, &goodConfig, oldSS, /*automaticManagement=*/false); + UNIT_ASSERT(goodConfig); + } + + Y_UNIT_TEST(AutomaticManagementDisabledPopulatesUsedNodes) { + // Regression test: when automaticManagement is false, the generator must still populate + // usedNodes with the node IDs taken from oldConfig, because usedNodes is shared across + // the StateStorage / StateStorageBoard / SchemeBoard generator invocations, and subsequent + // subsystem generators rely on it to avoid co-locating replicas on the same nodes. + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(2); + auto* ring1 = rg->AddRing(); + ring1->AddNode(3); + auto* ring2 = rg->AddRing(); + ring2->AddNode(5); + } + + std::unordered_set<ui32> usedNodes; + bool goodConfig = false; + GenerateSimpleStateStorage(8, usedNodes, 0, 0, 200, {}, &goodConfig, oldSS, + /*automaticManagement=*/false); + UNIT_ASSERT(goodConfig); + + // usedNodes is passed by value into GenerateSimpleStateStorage's helper, so re-derive it + // directly via the keeper API to check that the reference-passed set was actually filled. + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); + NKikimrConfig::TDomainsConfig::TStateStorage ssOut; + NKikimrBlobStorage::TStorageConfig config; + for (ui32 i : xrange(8)) { + auto *node = config.AddAllNodes(); + node->SetNodeId(i + 1); + } + std::unordered_set<ui32> refUsedNodes; + bool refGoodConfig = keeper.GenerateStateStorageConfig(&ssOut, config, refUsedNodes, {}, oldSS, + /*automaticManagement=*/false, 0, 0, 200); + UNIT_ASSERT(refGoodConfig); + UNIT_ASSERT(refUsedNodes.contains(3)); + UNIT_ASSERT(refUsedNodes.contains(5)); + UNIT_ASSERT_EQUAL(refUsedNodes.size(), 2u); + } + + Y_UNIT_TEST(AutomaticManagementDisabledUsedNodesAvoidedBySubsequentGenerator) { + // Simulate the real-world scenario from the bug report: StateStorage has automatic + // management disabled (its nodes are kept as-is), while StateStorageBoard has it enabled. + // The board generator must avoid nodes already occupied by StateStorage replicas, which + // are threaded through the shared usedNodes set. + NKikimrConfig::TDomainsConfig::TStateStorage oldStateStorage; + { + auto* rg = oldStateStorage.AddRingGroups(); + rg->SetNToSelect(3); + for (ui32 node : {1, 2, 3}) { + auto* ring = rg->AddRing(); + ring->AddNode(node); + } + } + + // Use a node pool large enough that plenty of spare nodes remain even after 3 are marked + // as used by StateStorage, so the board generator has enough room to build its own config. + NKikimrBlobStorage::TStorageConfig config; + for (ui32 i : xrange(20)) { + auto *node = config.AddAllNodes(); + node->SetNodeId(i + 1); + } + + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); + // Explicitly mark all nodes as GOOD; otherwise nodes absent from SelfHealNodesState are + // treated as UNKNOWN, which would make IsGoodConfig() always return false. + for (ui32 i : xrange(20)) { + keeper.SelfHealNodesState[i + 1] = 0; + } + std::unordered_set<ui32> usedNodes; + + // Step 1: StateStorage generation with automatic management disabled. + NKikimrConfig::TDomainsConfig::TStateStorage stateStorageOut; + bool goodConfig1 = keeper.GenerateStateStorageConfig(&stateStorageOut, config, usedNodes, {}, + oldStateStorage, /*automaticManagement=*/false, 0, 0, 200); + UNIT_ASSERT(goodConfig1); + UNIT_ASSERT(usedNodes.contains(1)); + UNIT_ASSERT(usedNodes.contains(2)); + UNIT_ASSERT(usedNodes.contains(3)); + + // Step 2: StateStorageBoard generation with automatic management enabled, sharing usedNodes. + NKikimrConfig::TDomainsConfig::TStateStorage boardOut; + NKikimrConfig::TDomainsConfig::TStateStorage emptyOldBoard; + bool goodConfig2 = keeper.GenerateStateStorageConfig(&boardOut, config, usedNodes, {}, + emptyOldBoard, /*automaticManagement=*/true, 0, 0, 200); + UNIT_ASSERT(goodConfig2); + + // None of the nodes selected for the board must overlap with the nodes already used by + // StateStorage, since those were marked as used via usedNodes. + for (const auto& rg : boardOut.GetRingGroups()) { + for (const auto& ring : rg.GetRing()) { + for (ui32 node : ring.GetNode()) { + UNIT_ASSERT_C(!(node == 1 || node == 2 || node == 3), + "StateStorageBoard replica placed on node " << node + << " which is already occupied by StateStorage"); + } + } + } + } } Y_UNIT_TEST_SUITE(TDeriveStorageConfigCleanupTest) { @@ -438,7 +825,7 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { NKikimrBlobStorage::TStorageConfig Config; NKikimrBlobStorage::TBaseConfig BaseConfig; - void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, + void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, const TString& dataCenter = "", const TString& rack = "") { auto *node = Config.AddAllNodes(); node->SetNodeId(nodeId); @@ -552,6 +939,16 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } return result; } + + void SetPDiskType(ui32 nodeId, NKikimrBlobStorage::EPDiskType type) { + for (auto& pdisk : *BaseConfig.MutablePDisk()) { + if (pdisk.GetNodeId() == nodeId) { + pdisk.SetType(type); + return; + } + } + UNIT_FAIL("PDisk not found"); + } }; NKikimrBlobStorage::TGroupGeometry Geometry(ui32 numFailDomains) { @@ -563,16 +960,28 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } void Reallocate(TSetup& s, const NProtoBuf::RepeatedField<ui32>& allowedNodeIds, bool applyNodeAllowList, - i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1) { + i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1, bool allowUnusableDisks = false) { + auto *selfManagementConfig = s.Config.MutableSelfManagementConfig(); + selfManagementConfig->MutableGeometry()->CopyFrom(Geometry(numFailDomains)); + selfManagementConfig->SetPDiskType(NKikimrBlobStorage::EPDiskType::ROT); + selfManagementConfig->MutableStaticGroupSelfHealAllowedNodes()->CopyFrom(allowedNodeIds); + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; replacedDisks.emplace(TVDiskIdShort(0, 0, 0), NBsController::TPDiskId()); NBsController::TGroupMapper::TForbiddenPDisks forbid; - keeper.AllocateStaticGroup(&s.Config, TGroupId::FromValue(0), /*groupGeneration=*/ 2, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), Geometry(numFailDomains), - /*pdiskFilters=*/ {}, NKikimrBlobStorage::EPDiskType::ROT, replacedDisks, forbid, - /*requiredSpace=*/ 0, &s.BaseConfig, /*convertToDonor=*/ false, /*ignoreVSlotQuotaCheck=*/ true, - /*isSelfHealReasonDecommit=*/ false, TBridgePileId(), std::nullopt, allowedNodeIds, applyNodeAllowList); + keeper.AllocateStaticGroup({ + .Config = &s.Config, + .GroupId = TGroupId::FromValue(0), + .GroupGeneration = 2, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .BaseConfig = &s.BaseConfig, + .IgnoreVSlotQuotaCheck = true, + .AllowUnusableDisks = allowUnusableDisks, + .ApplySelfHealNodeAllowList = applyNodeAllowList, + }); } NProtoBuf::RepeatedField<ui32> NodeIds(const std::vector<ui32>& nodeIds) { @@ -651,6 +1060,23 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { Reallocate(s, NodeIds({2}), /*applyNodeAllowList=*/ true, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); } + + Y_UNIT_TEST(Block42RejectsUnusablePreservedPDiskByDefault) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), + NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); + } + + Y_UNIT_TEST(Block42AllowsUnusablePreservedPDiskWhenRequested) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_NO_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8, + /*allowUnusableDisks=*/ true)); + UNIT_ASSERT_VALUES_EQUAL(s.GetGroupDomainNodes()[1], 2u); + } } } diff --git a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp index 53a002459ec..cbe48741ed4 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_validate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_validate.cpp @@ -1,5 +1,8 @@ #include "distconf.h" +#include <ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h> +#include <ydb/core/blobstorage/base/pdisk_config_validation.h> + namespace NKikimr::NStorage { //////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -192,6 +195,13 @@ namespace NKikimr::NStorage { } const ui32 pdiskId = pdisk.GetPDiskID(); + if (pdisk.HasPDiskConfig()) { + const TString context = TStringBuilder() << "pdisk NodeID# " << nodeId << " PDiskID# " << pdiskId; + if (auto error = ::NKikimr::ValidatePDiskConfig(pdisk.GetPDiskConfig(), context)) { + return error; + } + } + if (const auto [_, inserted] = pdisksById.emplace(std::make_tuple(nodeId, pdiskId), pdisk.GetPDiskGuid()); !inserted) { return "duplicate NodeID:PDiskID for pdisk"; } @@ -377,6 +387,13 @@ namespace NKikimr::NStorage { } } + if (config.HasInferPDiskSlotCountSettings()) { + if (auto error = ValidateInferPDiskSlotCountSettings( + config.GetInferPDiskSlotCountSettings(), "BlobStorageConfig.InferPDiskSlotCountSettings")) { + return error; + } + } + return {}; } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index 50b095134e0..2487b48f921 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -166,6 +166,7 @@ STATEFN(TNodeWarden::StateOnline) { hFunc(TEvPrivate::TEvUpdateStats, Handle); hFunc(TEvPrivate::TEvUpdateNodeDrives, Handle); hFunc(TEvPrivate::TEvRetrySaveConfig, Handle); + hFunc(TEvPrivate::TEvRetrySlay, Handle); hFunc(NMon::TEvHttpInfo, Handle); cFunc(NActors::TEvents::TSystem::Poison, PassAway); @@ -698,43 +699,49 @@ void TNodeWarden::Handle(NPDisk::TEvSlayResult::TPtr ev) { const NPDisk::TEvSlayResult &msg = *ev->Get(); const TVSlotId vslotId(LocalNodeId, msg.PDiskId, msg.VSlotId); const auto it = SlayInFlight.find(vslotId); - Y_DEBUG_ABORT_UNLESS(it != SlayInFlight.end()); YDB_LOG_INFO_COMP(BS_NODE, "Handle(NPDisk::TEvSlayResult)", {"marker", "NW28"}, {"msg", msg}, - {"expectedRound", it != SlayInFlight.end() ? std::make_optional(it->second) : std::nullopt}); - if (it == SlayInFlight.end() || it->second != msg.SlayOwnerRound) { + {"expectedRound", it != SlayInFlight.end() ? std::make_optional(it->second.Round) : std::nullopt}); + if (it == SlayInFlight.end() || it->second.Round != msg.SlayOwnerRound) { return; // outdated response } switch (msg.Status) { case NKikimrProto::NOTREADY: { - const ui64 round = NextLocalPDiskInitOwnerRound(); - TActivationContext::Schedule(TDuration::Seconds(1), new IEventHandle(MakeBlobStoragePDiskID(LocalNodeId, - msg.PDiskId), SelfId(), new NPDisk::TEvSlay(msg.VDiskId, round, msg.PDiskId, msg.VSlotId))); - it->second = round; + // Keep the short NOTREADY retry separate from the slower insurance retry scheduled by + // IssueSlay. Both carry the current round, so whichever retry issues a new request first + // makes the other timer stale. + Schedule(TDuration::Seconds(1), + new TEvPrivate::TEvRetrySlay(vslotId.NodeId, vslotId.PDiskId, vslotId.VDiskSlotId, it->second.Round, + TEvPrivate::TEvRetrySlay::EReason::NOTREADY)); break; } case NKikimrProto::OK: - case NKikimrProto::ALREADY: + case NKikimrProto::ALREADY: { + const TSlayInFlight slay = it->second; SlayInFlight.erase(it); - if (const auto vdiskIt = LocalVDisks.find(vslotId); vdiskIt == LocalVDisks.end()) { - SendVDiskReport(vslotId, msg.VDiskId, NKikimrBlobStorage::TEvControllerNodeReport::DESTROYED); - } else { - SendVDiskReport(vslotId, msg.VDiskId, NKikimrBlobStorage::TEvControllerNodeReport::WIPED); + SendVDiskReport(vslotId, slay.VDiskId, + slay.Action == ESlayAction::DESTROY + ? NKikimrBlobStorage::TEvControllerNodeReport::DESTROYED + : NKikimrBlobStorage::TEvControllerNodeReport::WIPED); + if (const auto vdiskIt = LocalVDisks.find(vslotId); vdiskIt != LocalVDisks.end()) { TVDiskRecord& vdisk = vdiskIt->second; - StartLocalVDiskActor(vdisk); // restart actor after successful wiping + StartLocalVDiskActor(vdisk); // start the current VDisk after the previous slot contents are gone } break; + } case NKikimrProto::CORRUPTED: // this branch doesn't really work - case NKikimrProto::ERROR: + case NKikimrProto::ERROR: { + const TVDiskID vdiskId = it->second.VDiskId; SlayInFlight.erase(it); YDB_LOG_ERROR_COMP(BS_NODE, "Handle(NPDisk::TEvSlayResult) error", {"marker", "NW29"}, {"msg", msg}); - SendVDiskReport(vslotId, msg.VDiskId, NKikimrBlobStorage::TEvControllerNodeReport::OPERATION_ERROR); + SendVDiskReport(vslotId, vdiskId, NKikimrBlobStorage::TEvControllerNodeReport::OPERATION_ERROR); break; + } case NKikimrProto::RACE: Y_ABORT("Unexpected# %s", msg.ToString().data()); @@ -767,6 +774,32 @@ void TNodeWarden::Handle(NPDisk::TEvChangeExpectedSlotCountResult::TPtr ev) { } } +void TNodeWarden::Handle(TEvPrivate::TEvRetrySlay::TPtr& ev) { + const auto *msg = ev->Get(); + const TVSlotId vslotId(msg->NodeId, msg->PDiskId, msg->VDiskSlotId); + if (const auto it = SlayInFlight.find(vslotId); + it != SlayInFlight.end() && it->second.Round == msg->Round) { + switch (msg->Reason) { + case TEvPrivate::TEvRetrySlay::EReason::NOTREADY: + YDB_LOG_INFO_COMP(BS_NODE, "Retrying PDisk slay after NOTREADY", + {"marker", "NW111"}, + {"VDiskId", it->second.VDiskId}, + {"VSlotId", vslotId}, + {"round", msg->Round}); + break; + + case TEvPrivate::TEvRetrySlay::EReason::UNCONFIRMED: + YDB_LOG_WARN_COMP(BS_NODE, "Retrying unconfirmed PDisk slay", + {"marker", "NW111"}, + {"VDiskId", it->second.VDiskId}, + {"VSlotId", vslotId}, + {"round", msg->Round}); + break; + } + IssueSlay(vslotId, it->second); + } +} + void TNodeWarden::Handle(NPDisk::TEvShredPDisk::TPtr ev) { // the message has returned to sender -- PDisk was terminated before processing it; normally it must never happen, // because NodeWarden issues PoisonPill synchronously with removing PDisk from the LocalPDisks set @@ -1421,23 +1454,29 @@ void TNodeWarden::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPt if (!equals(InferPDiskSlotCountSettings, inferSettings)) { InferPDiskSlotCountSettings.CopyFrom(inferSettings); for (auto& [key, localPDisk] : LocalPDisks) { - TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig(localPDisk.Record); - ui64 newExpectedSlotCount = newPDiskConfig->ExpectedSlotCount; + TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig( + localPDisk.Record, &localPDisk.PDiskConfigWarning); + ui32 newExpectedSlotCount = newPDiskConfig->ExpectedSlotCount; ui32 newSlotSizeInUnits = newPDiskConfig->SlotSizeInUnits; + ui64 newExpectedSlotSize = newPDiskConfig->ExpectedSlotSize; if (newExpectedSlotCount != localPDisk.ExpectedSlotCount || - newSlotSizeInUnits != localPDisk.SlotSizeInUnits) { + newSlotSizeInUnits != localPDisk.SlotSizeInUnits || + newExpectedSlotSize != localPDisk.ExpectedSlotSize) { YDB_LOG_DEBUG_COMP(BS_NODE, "SendChangeExpectedSlotCount from config notification", {"marker", "NW112"}, {"PDiskId", key.PDiskId}, {"expectedSlotCount", newExpectedSlotCount}, - {"slotSizeInUnits", newSlotSizeInUnits}); + {"slotSizeInUnits", newSlotSizeInUnits}, + {"expectedSlotSize", newExpectedSlotSize}); const TActorId pdiskActorId = MakeBlobStoragePDiskID(LocalNodeId, key.PDiskId); - Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount(newExpectedSlotCount, newSlotSizeInUnits)); + Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount( + newExpectedSlotCount, newSlotSizeInUnits, newExpectedSlotSize)); localPDisk.ExpectedSlotCount = newExpectedSlotCount; localPDisk.SlotSizeInUnits = newSlotSizeInUnits; + localPDisk.ExpectedSlotSize = newExpectedSlotSize; } } } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 1416551804a..a9ab131b101 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -6,6 +6,7 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/blobstorage/dsproxy/group_sessions.h> #include <ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h> +#include <ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h> #include <ydb/core/blobstorage/incrhuge/incrhuge.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> @@ -37,6 +38,7 @@ namespace NKikimr::NStorage { constexpr ui32 ProxyConfigurationTimeoutMilliseconds = 200; constexpr TDuration BackoffMin = TDuration::MilliSeconds(20); constexpr TDuration BackoffMax = TDuration::Seconds(5); + constexpr TDuration SlayRetryInitialDelay = TDuration::Seconds(5); constexpr const char *MockDevicesPath = "/Berkanavt/kikimr/testing/mock_devices.txt"; constexpr const char *YamlConfigFileName = "config.yaml"; constexpr const char *StorageConfigFileName = "storage.yaml"; @@ -98,6 +100,8 @@ namespace NKikimr::NStorage { bool Temporary = false; ui32 ExpectedSlotCount = 0; ui32 SlotSizeInUnits = 0; + ui64 ExpectedSlotSize = 0; + TString PDiskConfigWarning; std::optional<ui64> ShredGenerationIssued; std::variant<std::monostate, ui64, TString> ShredState; // not issued, finished with generation, aborted @@ -186,6 +190,7 @@ namespace NKikimr::NStorage { EvDereferencePDisk, EvSaveConfigResult, EvRetrySaveConfig, + EvRetrySlay, }; struct TEvSendDiskMetrics : TEventLocal<TEvSendDiskMetrics, EvSendDiskMetrics> {}; @@ -211,6 +216,27 @@ namespace NKikimr::NStorage { , StorageYamlVersion(storageYamlVersion) {} }; + + struct TEvRetrySlay : TEventLocal<TEvRetrySlay, EvRetrySlay> { + enum class EReason { + UNCONFIRMED, + NOTREADY, + }; + + ui32 NodeId; + ui32 PDiskId; + ui32 VDiskSlotId; + ui64 Round; + EReason Reason; + + TEvRetrySlay(ui32 nodeId, ui32 pdiskId, ui32 vdiskSlotId, ui64 round, EReason reason) + : NodeId(nodeId) + , PDiskId(pdiskId) + , VDiskSlotId(vdiskSlotId) + , Round(round) + , Reason(reason) + {} + }; }; TControlWrapper EnablePutBatching; @@ -313,8 +339,10 @@ namespace NKikimr::NStorage { return LocalPDiskInitOwnerRound; } - TIntrusivePtr<TPDiskConfig> CreatePDiskConfig(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk); - static void InferPDiskSlotCount(TIntrusivePtr<TPDiskConfig> pdiskConfig, ui64 driveSize, ui64 unitSizeInBytes, ui32 maxSlots); + TIntrusivePtr<TPDiskConfig> CreatePDiskConfig(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk, + TString *configWarning = nullptr); + static void InferPDiskSlotCount(TIntrusivePtr<TPDiskConfig> pdiskConfig, ui64 driveSize, + ui64 unitSizeInBytes, ui32 maxSlots); void StartLocalPDisk(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk, bool temporary); void AskBSCToRestartPDisk(ui32 pdiskId, bool ignoreDegradedGroups, ui64 requestCookie); void OnPDiskRestartFinished(ui32 pdiskId, NKikimrProto::EReplyStatus status); @@ -521,7 +549,19 @@ namespace NKikimr::NStorage { std::map<TVSlotId, TVDiskRecord> LocalVDisks; THashMap<TActorId, TVSlotId> VDiskIdByActor; - std::map<TVSlotId, ui64> SlayInFlight; + enum class ESlayAction { + WIPE, + DESTROY, + }; + + struct TSlayInFlight { + TVDiskID VDiskId; + ESlayAction Action; + ui64 Round = 0; + TDuration RetryDelay = SlayRetryInitialDelay; + }; + + std::map<TVSlotId, TSlayInFlight> SlayInFlight; // PDiskId -> is another restart required after the current restart. std::unordered_map<ui32, bool> PDiskRestartInFlight; TIntrusiveList<TVDiskRecord, TUnreportedMetricTag> VDisksWithUnreportedMetrics; @@ -535,7 +575,8 @@ namespace NKikimr::NStorage { // process VDisk configuration void ApplyLocalVDiskInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& vdisk); - void Slay(TVDiskRecord& vdisk); + void Slay(TVDiskRecord& vdisk, ESlayAction action); + void IssueSlay(const TVSlotId& vslotId, TSlayInFlight& slay); void UpdateGroupInfoForDisk(TVDiskRecord& vdisk, const TIntrusivePtr<TBlobStorageGroupInfo>& newInfo); @@ -677,6 +718,7 @@ namespace NKikimr::NStorage { void Handle(TEvPrivate::TEvSendDiskMetrics::TPtr&); void Handle(TEvPrivate::TEvUpdateNodeDrives ::TPtr&); void Handle(TEvPrivate::TEvRetrySaveConfig::TPtr&); + void Handle(TEvPrivate::TEvRetrySlay::TPtr&); void Handle(TEvPrivate::TEvUpdateStats::TPtr&); void Handle(NMon::TEvHttpInfo::TPtr&); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp index 4eb88066394..e855444b1ed 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp @@ -153,6 +153,7 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) { TABLEH() { out << "Category"; } TABLEH() { out << "Temporary"; } TABLEH() { out << "Pending"; } + TABLEH() { out << "PDiskConfig warning"; } } } TABLEBODY() { @@ -175,6 +176,7 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) { TABLED() { out << value.Record.GetPDiskCategory(); } TABLED() { out << value.Temporary; } TABLED() { out << pending; } + TABLED() { out << value.PDiskConfigWarning; } } } } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp index 07d621e1869..961966e1e27 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp @@ -2,6 +2,7 @@ #include "node_warden_events.h" #include "node_warden_impl.h" +#include <ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h> #include <ydb/core/blobstorage/crypto/default.h> #include <ydb/library/pdisk_io/file_params.h> #include <ydb/library/pdisk_io/wcache.h> @@ -18,6 +19,19 @@ namespace NKikimr::NStorage { {NPDisk::DEVICE_TYPE_NVME, 300000000}, }; + static ui32 CalculateExpectedSlotCountFromSlotSize(ui64 driveSize, ui64 slotSizeInBytes, ui32 maxSlots) { + Y_ABORT_UNLESS(driveSize); + Y_ABORT_UNLESS(slotSizeInBytes); + + ui64 slotCount = driveSize / slotSizeInBytes; + if (maxSlots) { + slotCount = Min(slotCount, ui64(maxSlots)); + } + return slotCount > ui64(Max<ui32>()) + ? Max<ui32>() + : static_cast<ui32>(slotCount); + } + void TNodeWarden::InferPDiskSlotCount(TIntrusivePtr<TPDiskConfig> pdiskConfig, ui64 driveSize, ui64 unitSizeInBytes, ui32 maxSlots) { Y_ABORT_UNLESS(driveSize); Y_ABORT_UNLESS(unitSizeInBytes); @@ -34,7 +48,12 @@ namespace NKikimr::NStorage { pdiskConfig->SlotSizeInUnits = slotSizeInUnits; } - TIntrusivePtr<TPDiskConfig> TNodeWarden::CreatePDiskConfig(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk) { + TIntrusivePtr<TPDiskConfig> TNodeWarden::CreatePDiskConfig( + const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk, TString *configWarning) { + if (configWarning) { + configWarning->clear(); + } + const TString& path = pdisk.GetPath(); const ui64 pdiskGuid = pdisk.GetPDiskGuid(); const ui32 pdiskID = pdisk.GetPDiskID(); @@ -144,8 +163,151 @@ namespace NKikimr::NStorage { pdiskConfig->FeatureFlags.SetEnablePDiskDataEncryption(!pdiskConfig->SectorMap); } + const bool hasExpectedSlotCount = pdiskConfig->ExpectedSlotCount != 0; + const bool hasSlotSizeInUnits = pdiskConfig->SlotSizeInUnits != 0; + const bool hasExpectedSlotSize = pdiskConfig->ExpectedSlotSize != 0; + const bool hasMaxSlots = pdiskConfig->MaxSlots != 0; + if (hasExpectedSlotSize && (hasExpectedSlotCount || hasSlotSizeInUnits)) { + const TString warning = TStringBuilder() + << "PDiskConfig has ExpectedSlotSize with ExpectedSlotCount or SlotSizeInUnits; " + << "ExpectedSlotSize and MaxSlots take precedence for slot count calculation " + << "when MaxSlots is set and drive size is available" + << " ExpectedSlotCount# " << pdiskConfig->ExpectedSlotCount + << " SlotSizeInUnits# " << pdiskConfig->SlotSizeInUnits + << " ExpectedSlotSize# " << pdiskConfig->ExpectedSlotSize + << " MaxSlots# " << pdiskConfig->MaxSlots; + YDB_LOG_ERROR("PDiskConfig has ExpectedSlotSize with ExpectedSlotCount or SlotSizeInUnits", + {"marker", "NW113"}, + {"PDiskId", pdiskID}, + {"path", path}, + {"expectedSlotCount", pdiskConfig->ExpectedSlotCount}, + {"slotSizeInUnits", pdiskConfig->SlotSizeInUnits}, + {"expectedSlotSize", pdiskConfig->ExpectedSlotSize}, + {"maxSlots", pdiskConfig->MaxSlots}); + if (configWarning) { + *configWarning = warning; + } + } + if (hasExpectedSlotSize && !hasMaxSlots) { + const TString warning = TStringBuilder() + << "PDiskConfig has ExpectedSlotSize without MaxSlots; " + << "ignoring ExpectedSlotSize, explicit slot settings remain in effect" + << " ExpectedSlotSize# " << pdiskConfig->ExpectedSlotSize; + YDB_LOG_ERROR("PDiskConfig has ExpectedSlotSize without MaxSlots", + {"marker", "NW117"}, + {"PDiskId", pdiskID}, + {"path", path}, + {"expectedSlotSize", pdiskConfig->ExpectedSlotSize}); + if (configWarning && configWarning->empty()) { + *configWarning = warning; + } + // slot count cannot be derived without MaxSlots, so do not switch the PDisk to + // fixed-size slot quotas either: behave as if ExpectedSlotSize was not set + pdiskConfig->ExpectedSlotSize = 0; + } + + std::optional<ui64> driveSize; + TString driveSizeDetails; + auto getDriveSize = [&] { + if (driveSize) { + return *driveSize; + } + + ui64 size = 0; + TStringStream outDetails; + if (pdiskConfig->SectorMap) { + size = pdiskConfig->SectorMap->DeviceSize; + outDetails << "drive size obtained from SectorMap"; + } else if (std::optional<NPDisk::TDriveData> data = NPDisk::GetDriveData(path, &outDetails)) { + size = data->Size; + } + driveSizeDetails = outDetails.Str(); + driveSize = size; + return size; + }; + + bool validInferSettings = true; + if (auto error = ValidateInferPDiskSlotCountSettings( + InferPDiskSlotCountSettings, "BlobStorageConfig.InferPDiskSlotCountSettings")) { + validInferSettings = false; + YDB_LOG_ERROR("Invalid InferPDiskSlotCountSettings", + {"marker", "NW114"}, + {"PDiskId", pdiskID}, + {"path", path}, + {"error", *error}); + if (configWarning && configWarning->empty()) { + *configWarning = *error; + } + } + auto inferSettings = TInferPDiskSlotCountSettingsForDriveType(InferPDiskSlotCountSettings, deviceType); - if (!inferSettings) { + if (pdiskConfig->ExpectedSlotSize) { + pdiskConfig->ExpectedSlotCount = 0; + pdiskConfig->SlotSizeInUnits = 0; + if (const ui64 size = getDriveSize(); !size) { + YDB_LOG_ERROR("Unable to determine drive size for calculating PDisk slot count", + {"marker", "NW115"}, + {"path", path}, + {"expectedSlotSize", pdiskConfig->ExpectedSlotSize}, + {"details", driveSizeDetails}); + } else { + pdiskConfig->ExpectedSlotCount = CalculateExpectedSlotCountFromSlotSize( + size, pdiskConfig->ExpectedSlotSize, pdiskConfig->MaxSlots); + if (!pdiskConfig->ExpectedSlotCount) { + const TString warning = TStringBuilder() + << "Drive is smaller than ExpectedSlotSize, slot count is not materialized" + << " ExpectedSlotSize# " << pdiskConfig->ExpectedSlotSize + << " DriveSize# " << size; + YDB_LOG_ERROR("Drive is smaller than ExpectedSlotSize", + {"marker", "NW118"}, + {"PDiskId", pdiskID}, + {"path", path}, + {"expectedSlotSize", pdiskConfig->ExpectedSlotSize}, + {"driveSize", size}); + if (configWarning && configWarning->empty()) { + *configWarning = warning; + } + } + YDB_LOG_DEBUG("Calculated PDisk slot count from expected slot size", + {"marker", "NW102"}, + {"path", path}, + {"slotCount", pdiskConfig->ExpectedSlotCount}, + {"expectedSlotSize", pdiskConfig->ExpectedSlotSize}, + {"fromDriveSize", size}, + {"fromMaxSlots", pdiskConfig->MaxSlots}); + } + } else if (validInferSettings && inferSettings.SlotSize) { + if ((pdiskConfig->ExpectedSlotCount || pdiskConfig->SlotSizeInUnits) + && !inferSettings.PreferInferredSettingsOverExplicit) { + YDB_LOG_DEBUG("Skipped inferring PDisk slot count from slot size, using explicit settings", + {"marker", "NW102"}, + {"path", path}, + {"slotCount", pdiskConfig->ExpectedSlotCount}, + {"slotSizeInUnits", pdiskConfig->SlotSizeInUnits}, + {"fromSlotSize", inferSettings.SlotSize}); + } else { + const ui64 size = getDriveSize(); + if (!size) { + YDB_LOG_ERROR("Unable to determine drive size for calculating PDisk slot count", + {"marker", "NW116"}, + {"path", path}, + {"slotSize", inferSettings.SlotSize}, + {"details", driveSizeDetails}); + } else { + pdiskConfig->ExpectedSlotSize = inferSettings.SlotSize; + pdiskConfig->ExpectedSlotCount = CalculateExpectedSlotCountFromSlotSize( + size, inferSettings.SlotSize, inferSettings.MaxSlots); + pdiskConfig->SlotSizeInUnits = 0; + YDB_LOG_DEBUG("Calculated PDisk slot count from inferred slot size", + {"marker", "NW102"}, + {"path", path}, + {"slotCount", pdiskConfig->ExpectedSlotCount}, + {"slotSize", inferSettings.SlotSize}, + {"fromDriveSize", size}, + {"fromMaxSlots", inferSettings.MaxSlots}); + } + } + } else if (!validInferSettings || !inferSettings) { YDB_LOG_DEBUG("Inferring PDisk slot count not configured", {"marker", "NW102"}, {"path", path}, @@ -158,28 +320,20 @@ namespace NKikimr::NStorage { {"slotCount", pdiskConfig->ExpectedSlotCount}, {"slotSizeInUnits", pdiskConfig->SlotSizeInUnits}); } else { - ui64 driveSize = 0; - TStringStream outDetails; - if (pdiskConfig->SectorMap) { - driveSize = pdiskConfig->SectorMap->DeviceSize; - outDetails << "drive size obtained from SectorMap"; - } else if (std::optional<NPDisk::TDriveData> data = NPDisk::GetDriveData(path, &outDetails)) { - driveSize = data->Size; - } - - if (!driveSize) { + const ui64 size = getDriveSize(); + if (!size) { YDB_LOG_ERROR("Unable to determine drive size for inferring PDisk slot count", {"marker", "NW96"}, {"path", path}, - {"details", outDetails.Str()}); + {"details", driveSizeDetails}); } else { - InferPDiskSlotCount(pdiskConfig, driveSize, inferSettings.UnitSize, inferSettings.MaxSlots); + InferPDiskSlotCount(pdiskConfig, size, inferSettings.UnitSize, inferSettings.MaxSlots); YDB_LOG_DEBUG("Inferred PDisk slot count", {"marker", "NW102"}, {"path", path}, {"slotCount", pdiskConfig->ExpectedSlotCount}, {"slotSizeInUnits", pdiskConfig->SlotSizeInUnits}, - {"fromDriveSize", driveSize}, + {"fromDriveSize", size}, {"fromUnitSize", inferSettings.UnitSize}, {"fromMaxSlots", inferSettings.MaxSlots}); } @@ -257,12 +411,13 @@ namespace NKikimr::NStorage { record.ReplPDiskWriteQuoter = std::make_shared<TReplQuoter>(*writeBytesPerSecond); } - auto pdiskConfig = CreatePDiskConfig(pdisk); + auto pdiskConfig = CreatePDiskConfig(pdisk, &record.PDiskConfigWarning); if (temporary) { pdiskConfig->MetadataOnly = true; } record.ExpectedSlotCount = pdiskConfig->ExpectedSlotCount; record.SlotSizeInUnits = pdiskConfig->SlotSizeInUnits; + record.ExpectedSlotSize = pdiskConfig->ExpectedSlotSize; YDB_LOG_DEBUG("StartLocalPDisk", {"marker", "NW04"}, @@ -272,6 +427,7 @@ namespace NKikimr::NStorage { {"PDiskCategory", TPDiskCategory(record.Record.GetPDiskCategory())}, {"expectedSlotCount", record.ExpectedSlotCount}, {"slotSizeInUnits", record.SlotSizeInUnits}, + {"expectedSlotSize", record.ExpectedSlotSize}, {"temporary", temporary}); const ui32 pdiskID = pdisk.GetPDiskID(); @@ -313,6 +469,20 @@ namespace NKikimr::NStorage { it->first.NodeId == LocalNodeId && it->first.PDiskId == pdiskId; ++it) { it->second.UnderlyingPDiskDestroyed = true; } + + // A pending slay no longer needs an acknowledgement from a PDisk which has itself been removed. + // Complete it here so that a lost TEvSlayResult can't keep the VSlot state forever. + for (auto it = SlayInFlight.lower_bound({LocalNodeId, pdiskId, 0}); + it != SlayInFlight.end() && it->first.NodeId == LocalNodeId && it->first.PDiskId == pdiskId; ) { + const auto current = it++; + const TVSlotId vslotId = current->first; + const TSlayInFlight slay = current->second; + SlayInFlight.erase(current); + SendVDiskReport(vslotId, slay.VDiskId, + slay.Action == ESlayAction::DESTROY + ? NKikimrBlobStorage::TEvControllerNodeReport::DESTROYED + : NKikimrBlobStorage::TEvControllerNodeReport::WIPED); + } } if (pending) { @@ -414,16 +584,22 @@ namespace NKikimr::NStorage { bool first = true; vdisks << "{"; for (auto it = LocalVDisks.lower_bound(from); it != LocalVDisks.end() && it->first <= to; ++it) { - auto& [key, value] = *it; + auto& value = it->second; PoisonLocalVDisk(value); vdisks << (std::exchange(first, false) ? "" : ", ") << value.GetVDiskId().ToString(); - if (const auto it = SlayInFlight.find(key); it != SlayInFlight.end()) { - const ui64 round = NextLocalPDiskInitOwnerRound(); - Send(MakeBlobStoragePDiskID(key.NodeId, key.PDiskId), new NPDisk::TEvSlay(value.GetVDiskId(), round, - key.PDiskId, key.VDiskSlotId)); - it->second = round; - } else { + } + + // Slay state owns the original VDisk identity, so it can be replayed even when a deleted VDisk + // has already been removed from LocalVDisks. + for (auto it = SlayInFlight.lower_bound(from); it != SlayInFlight.end() && it->first <= to; ++it) { + it->second.RetryDelay = SlayRetryInitialDelay; + IssueSlay(it->first, it->second); + } + + for (auto it = LocalVDisks.lower_bound(from); it != LocalVDisks.end() && it->first <= to; ++it) { + auto& [key, value] = *it; + if (!SlayInFlight.contains(key)) { StartLocalVDiskActor(value); } } @@ -436,7 +612,7 @@ namespace NKikimr::NStorage { } else { for (auto it = LocalVDisks.lower_bound(from); it != LocalVDisks.end() && it->first <= to; ++it) { auto& [key, value] = *it; - if (!value.RuntimeData) { + if (!value.RuntimeData && !SlayInFlight.contains(key)) { StartLocalVDiskActor(value); } } @@ -480,7 +656,8 @@ namespace NKikimr::NStorage { const TActorId actorId = MakeBlobStoragePDiskID(LocalNodeId, pdiskId); - TIntrusivePtr<TPDiskConfig> pdiskConfig = CreatePDiskConfig(it->second.Record); + TIntrusivePtr<TPDiskConfig> pdiskConfig = CreatePDiskConfig( + it->second.Record, &it->second.PDiskConfigWarning); Cfg->PDiskKey.Initialize(); Send(actorId, new TEvBlobStorage::TEvAskWardenRestartPDiskResult(pdiskId, Cfg->PDiskKey, true, pdiskConfig)); @@ -574,30 +751,39 @@ namespace NKikimr::NStorage { const TPDiskKey key(pdisk); if (auto it = LocalPDisks.find(key); it != LocalPDisks.end()) { TPDiskRecord& localPDisk = it->second; - TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig(pdisk); + TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig( + pdisk, &localPDisk.PDiskConfigWarning); ui32 newExpectedSlotCount = newPDiskConfig->ExpectedSlotCount; ui32 newSlotSizeInUnits = newPDiskConfig->SlotSizeInUnits; + ui64 newExpectedSlotSize = newPDiskConfig->ExpectedSlotSize; YDB_LOG_DEBUG("ApplyServiceSetPDisks", {"marker", "NW110"}, {"PDiskId", key.PDiskId}, {"newExpectedSlotCount", newExpectedSlotCount}, {"oldExpectedSlotCount", localPDisk.ExpectedSlotCount}, {"newSlotSizeInUnits", newSlotSizeInUnits}, - {"oldSlotSizeInUnits", localPDisk.SlotSizeInUnits}); + {"oldSlotSizeInUnits", localPDisk.SlotSizeInUnits}, + {"newExpectedSlotSize", newExpectedSlotSize}, + {"oldExpectedSlotSize", localPDisk.ExpectedSlotSize}); if (newExpectedSlotCount != localPDisk.ExpectedSlotCount || - newSlotSizeInUnits != localPDisk.SlotSizeInUnits) { + newSlotSizeInUnits != localPDisk.SlotSizeInUnits || + newExpectedSlotSize != localPDisk.ExpectedSlotSize) { YDB_LOG_DEBUG("SendChangeExpectedSlotCount", {"marker", "NW107"}, {"PDiskId", key.PDiskId}, {"expectedSlotCount", newExpectedSlotCount}, - {"slotSizeInUnits", newSlotSizeInUnits}); + {"slotSizeInUnits", newSlotSizeInUnits}, + {"expectedSlotSize", newExpectedSlotSize}); const TActorId pdiskActorId = MakeBlobStoragePDiskID(LocalNodeId, key.PDiskId); - Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount(newExpectedSlotCount, newSlotSizeInUnits)); + Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount( + newExpectedSlotCount, newSlotSizeInUnits, newExpectedSlotSize)); localPDisk.ExpectedSlotCount = newExpectedSlotCount; localPDisk.SlotSizeInUnits = newSlotSizeInUnits; + localPDisk.ExpectedSlotSize = newExpectedSlotSize; } + localPDisk.Record = pdisk; } else { StartLocalPDisk(pdisk, false); } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp index 21056c02f9a..688b5824fb0 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp @@ -126,8 +126,23 @@ void TNodeWarden::StartLocalProxy(ui32 groupId) { // subscribe for group information changes through distconf cache Send(SelfId(), new TEvNodeWardenQueryCache(Sprintf("G%08" PRIx32, groupId), true)); - group.ProxyId = as->Register(proxy.release(), TMailboxType::ReadAsFilled, AppData()->SystemPoolId); - as->RegisterLocalService(MakeBlobStorageProxyID(groupId), group.ProxyId); + auto id = as->Register(proxy.release(), TMailboxType::ReadAsFilled, AppData()->SystemPoolId); + + // determine if we want to inject BS errors + if (Cfg->BlobStorageConfig.GetServiceSet().HasFailureInjectionConfig()) { + auto const& fiConfig = Cfg->BlobStorageConfig.GetServiceSet().GetFailureInjectionConfig(); + if (fiConfig.GetFailureProbability() > 0) { + const auto gid = TGroupId::FromValue(groupId); + if (IsDynamicGroup(gid) || fiConfig.GetIncludeStaticGroups()) { + id = as->Register( + CreateBlobStorageGroupFailureInjectingActor(id, gid, fiConfig), + TMailboxType::ReadAsFilled, AppData()->SystemPoolId); + } + } + } + + group.ProxyId = id; + as->RegisterLocalService(MakeBlobStorageProxyID(groupId), id); } void TNodeWarden::StartVirtualGroupAgent(ui32 groupId) { diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp index fb7090e9d85..d6010da0a7d 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp @@ -518,19 +518,29 @@ namespace NKikimr::NStorage { } } + const TPDiskKey pdiskKey(vslotId.NodeId, vslotId.PDiskId); + const bool pdiskMissing = !LocalPDisks.contains(pdiskKey); if (vdisk.GetDoDestroy() || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { - if (record.UnderlyingPDiskDestroyed) { + if (record.UnderlyingPDiskDestroyed || pdiskMissing) { PoisonLocalVDisk(record); + SlayInFlight.erase(vslotId); SendVDiskReport(vslotId, record.GetVDiskId(), NKikimrBlobStorage::TEvControllerNodeReport::DESTROYED); record.UnderlyingPDiskDestroyed = false; } else { - Slay(record); + Slay(record, ESlayAction::DESTROY); } DestroyLocalVDisk(record); LocalVDisks.erase(it); ApplyServiceSetPDisks(); // destroy unneeded PDisk actors } else if (vdisk.GetDoWipe()) { - Slay(record); + if (pdiskMissing) { + PoisonLocalVDisk(record); + SlayInFlight.erase(vslotId); + SendVDiskReport(vslotId, record.GetVDiskId(), NKikimrBlobStorage::TEvControllerNodeReport::WIPED); + record.UnderlyingPDiskDestroyed = false; + } else { + Slay(record, ESlayAction::WIPE); + } } else if (!record.RuntimeData) { StartLocalVDiskActor(record); } else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode() || record.RuntimeData->ReadOnly != record.Config.GetReadOnly()) { @@ -539,23 +549,59 @@ namespace NKikimr::NStorage { } } - void TNodeWarden::Slay(TVDiskRecord& vdisk) { + void TNodeWarden::Slay(TVDiskRecord& vdisk, ESlayAction action) { const TVSlotId vslotId = vdisk.GetVSlotId(); + const TVDiskID vdiskId = vdisk.GetVDiskId(); YDB_LOG_INFO("Slay", {"marker", "NW33"}, - {"VDiskId", vdisk.GetVDiskId()}, - {"VSlotId", vdisk.GetVSlotId()}, + {"VDiskId", vdiskId}, + {"VSlotId", vslotId}, + {"action", action == ESlayAction::DESTROY ? "destroy" : "wipe"}, {"slayInFlight", SlayInFlight.contains(vslotId)}); - if (!SlayInFlight.contains(vslotId)) { + + if (auto it = SlayInFlight.find(vslotId); it != SlayInFlight.end()) { + if (!it->second.VDiskId.SameExceptGeneration(vdiskId)) { + YDB_LOG_ERROR("Slay VDiskId mismatch", + {"marker", "NW113"}, + {"existingVDiskId", it->second.VDiskId}, + {"incomingVDiskId", vdiskId}, + {"VSlotId", vslotId}); + return; + } + + const bool upgradeToDestroy = action == ESlayAction::DESTROY && + it->second.Action != ESlayAction::DESTROY; + it->second.VDiskId = vdiskId; + if (action == ESlayAction::DESTROY) { + it->second.Action = ESlayAction::DESTROY; + } + if (upgradeToDestroy) { + it->second.RetryDelay = SlayRetryInitialDelay; + IssueSlay(vslotId, it->second); + } + } else { PoisonLocalVDisk(vdisk); - const TVSlotId vslotId = vdisk.GetVSlotId(); - const TActorId pdiskServiceId = MakeBlobStoragePDiskID(vslotId.NodeId, vslotId.PDiskId); - const ui64 round = NextLocalPDiskInitOwnerRound(); - Send(pdiskServiceId, new NPDisk::TEvSlay(vdisk.GetVDiskId(), round, vslotId.PDiskId, vslotId.VDiskSlotId)); - SlayInFlight.emplace(vslotId, round); + auto [insertedIt, inserted] = SlayInFlight.emplace(vslotId, TSlayInFlight{ + .VDiskId = vdiskId, + .Action = action, + }); + Y_ABORT_UNLESS(inserted); + IssueSlay(vslotId, insertedIt->second); } } + void TNodeWarden::IssueSlay(const TVSlotId& vslotId, TSlayInFlight& slay) { + const ui64 round = NextLocalPDiskInitOwnerRound(); + slay.Round = round; + + Send(MakeBlobStoragePDiskID(vslotId.NodeId, vslotId.PDiskId), + new NPDisk::TEvSlay(slay.VDiskId, round, vslotId.PDiskId, vslotId.VDiskSlotId)); + Schedule(slay.RetryDelay, + new TEvPrivate::TEvRetrySlay(vslotId.NodeId, vslotId.PDiskId, vslotId.VDiskSlotId, round, + TEvPrivate::TEvRetrySlay::EReason::UNCONFIRMED)); + slay.RetryDelay = Min(slay.RetryDelay * 2, TDuration::Minutes(1)); + } + void TNodeWarden::Handle(TEvBlobStorage::TEvAskRestartVDisk::TPtr ev) { const auto& [pDiskId, vDiskId] = *ev->Get(); const auto nodeId = SelfId().NodeId(); // Skeleton and NodeWarden are on the same node diff --git a/ydb/core/blobstorage/nodewarden/ya.make b/ydb/core/blobstorage/nodewarden/ya.make index e76218c3263..a3e2d5268ea 100644 --- a/ydb/core/blobstorage/nodewarden/ya.make +++ b/ydb/core/blobstorage/nodewarden/ya.make @@ -54,6 +54,7 @@ PEERDIR( ydb/core/base ydb/core/blob_depot ydb/core/blob_depot/agent + ydb/core/blobstorage/base ydb/core/blobstorage/bridge/proxy ydb/core/blobstorage/bridge/syncer ydb/core/blobstorage/common diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 2c7295b1f3e..4824d2b89d6 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -328,10 +328,12 @@ struct TEvYardResizeResult : TEventLocal<TEvYardResizeResult, TEvBlobStorage::Ev struct TEvChangeExpectedSlotCount : TEventLocal<TEvChangeExpectedSlotCount, TEvBlobStorage::EvChangeExpectedSlotCount> { ui32 ExpectedSlotCount; ui32 SlotSizeInUnits; + ui64 ExpectedSlotSize; - TEvChangeExpectedSlotCount(ui32 expectedSlotCount, ui32 slotSizeInUnits) + TEvChangeExpectedSlotCount(ui32 expectedSlotCount, ui32 slotSizeInUnits, ui64 expectedSlotSize = 0) : ExpectedSlotCount(expectedSlotCount) , SlotSizeInUnits(slotSizeInUnits) + , ExpectedSlotSize(expectedSlotSize) {} TString ToString() const { @@ -344,6 +346,7 @@ struct TEvChangeExpectedSlotCount : TEventLocal<TEvChangeExpectedSlotCount, TEvB str << "EvChangeExpectedSlotCount "; str << " ExpectedSlotCount# " << record.ExpectedSlotCount; str << " SlotSizeInUnits# " << record.SlotSizeInUnits; + str << " ExpectedSlotSize# " << record.ExpectedSlotSize; str << "}"; return str.Str(); } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h index 06876c590c3..133fe8a6b39 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h @@ -24,12 +24,17 @@ class TPerOwnerQuotaTracker { TColorLimits ColorLimits; i64 Total; size_t ExpectedOwnerCount; // 0 means 'add and remove owners as you go' + i64 ExpectedOwnerSize; // 0 means 'derive owner quota from expected/active owner count' TStackVec<TOwner, 256> ActiveOwnerIds; // Can be accessed only from the main thread (changes only when owner is // added or removed). std::array<TQuotaRecord, 256> QuotaForOwner; // Always allocated, can be read from anywhere static_assert(sizeof(TOwner) == 1, "Make sure to use large enough QuotaForOwner buffer"); + ui32 NormalizeOwnerWeight(ui32 weight) const { + return ExpectedOwnerSize ? 1 : weight; + } + public: TPerOwnerQuotaTracker() { TColorLimits limits; @@ -40,6 +45,7 @@ public: ColorLimits = limits; Total = total; ExpectedOwnerCount = 0; + ExpectedOwnerSize = 0; ActiveOwnerIds.clear(); QuotaForOwner.fill(TQuotaRecord{}); } @@ -48,7 +54,22 @@ public: // Increasing expected owner count is fundamentally unfair and may cause instant jumps right into 0 free, // overusers will keep their unfair share as a result. void SetExpectedOwnerCount(size_t newOwnerCount) { + SetExpectedOwnerSettings(newOwnerCount, ExpectedOwnerSize); + } + + void SetExpectedOwnerSize(i64 newOwnerSize) { + SetExpectedOwnerSettings(ExpectedOwnerCount, newOwnerSize); + } + + void SetExpectedOwnerSettings(size_t newOwnerCount, i64 newOwnerSize) { + Y_VERIFY(newOwnerSize >= 0); ExpectedOwnerCount = newOwnerCount; + ExpectedOwnerSize = newOwnerSize; + if (ExpectedOwnerSize) { + for (TOwner id : ActiveOwnerIds) { + QuotaForOwner[id].SetWeight(1); + } + } RedistributeQuotas(); } @@ -66,14 +87,20 @@ public: } void RedistributeQuotas() { - size_t parts = Max(ExpectedOwnerCount, GetNumActiveSlots()); - if (parts) { - i64 limit = Total / parts; - - // Divide into equal parts and that's it. + if (ExpectedOwnerSize) { for (TOwner id : ActiveOwnerIds) { - auto weight = QuotaForOwner[id].GetWeight(); - ForceHardLimit(id, limit * weight); + ForceHardLimit(id, ExpectedOwnerSize); + } + } else { + size_t parts = Max(ExpectedOwnerCount, GetNumActiveSlots()); + if (parts) { + i64 limit = Total / parts; + + // Divide into equal parts and that's it. + for (TOwner id : ActiveOwnerIds) { + auto weight = QuotaForOwner[id].GetWeight(); + ForceHardLimit(id, limit * weight); + } } } } @@ -84,7 +111,7 @@ public: Y_VERIFY(record.GetFree() == 0); record.SetName(TStringBuilder() << "Owner# " << id); record.SetVDiskId(vdiskId); - record.SetWeight(weight); + record.SetWeight(NormalizeOwnerWeight(weight)); ActiveOwnerIds.push_back(id); RedistributeQuotas(); @@ -95,7 +122,7 @@ public: Y_VERIFY(it != ActiveOwnerIds.end()); TQuotaRecord &record = QuotaForOwner[id]; - record.SetWeight(weight); + record.SetWeight(NormalizeOwnerWeight(weight)); RedistributeQuotas(); } @@ -190,6 +217,7 @@ public: ColorLimits.Print(str); str << "\nTotal# " << Total; str << "\nExpectedOwnerCount# " << ExpectedOwnerCount; + str << "\nExpectedOwnerSize# " << ExpectedOwnerSize; str << "\nActiveOwners# " << ActiveOwnerIds.size(); str << "\nNumActiveSlots# " << GetNumActiveSlots(); if (colorBorder) { @@ -338,7 +366,7 @@ public: TColorLimits chunkLimits = TColorLimits::MakeChunkLimits(params.ChunkBaseLimit); SharedQuota->ForceHardLimit(GlobalQuota->GetHardLimit(OwnerBeginUser), chunkLimits); OwnerQuota->Reset(GlobalQuota->GetHardLimit(OwnerBeginUser), chunkLimits); - OwnerQuota->SetExpectedOwnerCount(params.ExpectedOwnerCount); + OwnerQuota->SetExpectedOwnerSettings(params.ExpectedOwnerCount, params.ExpectedOwnerSize); for (auto& [ownerId, ownerInfo] : params.OwnersInfo) { i64 chunks = ownerInfo.ChunksOwned; @@ -530,8 +558,18 @@ public: bool TryAllocate(TOwner owner, i64 count, TString &outErrorReason) { if (IsOwnerUser(owner)) { - OwnerQuota->ForceAllocate(owner, count); - return SharedQuota->TryAllocate(count, outErrorReason); + if (Params.ExpectedOwnerSize) { + if (!OwnerQuota->TryAllocate(owner, count, outErrorReason)) { + return false; + } + } else { + OwnerQuota->ForceAllocate(owner, count); + } + if (SharedQuota->TryAllocate(count, outErrorReason)) { + return true; + } + OwnerQuota->Release(owner, count); + return false; } else { switch (owner) { case OwnerCommonStaticLog: @@ -622,9 +660,21 @@ public: } void SetExpectedOwnerCount(size_t newOwnerCount) { + Params.ExpectedOwnerCount = newOwnerCount; OwnerQuota->SetExpectedOwnerCount(newOwnerCount); } + void SetExpectedOwnerSize(i64 newOwnerSize) { + Params.ExpectedOwnerSize = newOwnerSize; + OwnerQuota->SetExpectedOwnerSize(newOwnerSize); + } + + void SetExpectedOwnerSettings(size_t newOwnerCount, i64 newOwnerSize) { + Params.ExpectedOwnerCount = newOwnerCount; + Params.ExpectedOwnerSize = newOwnerSize; + OwnerQuota->SetExpectedOwnerSettings(newOwnerCount, newOwnerSize); + } + void SetColorBorder(NKikimrBlobStorage::TPDiskSpaceColor::E colorBorder) { ColorBorder = colorBorder; ColorBorderOccupancy = OwnerQuota->GetOccupancyForColor(ColorBorder); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h index a86e117e11c..9a1678af28b 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h @@ -142,9 +142,11 @@ struct TPDiskConfig : public TThrRefBase { ui32 IoPieceSizeBytes = 0; bool UseSpdkNvmeDriver; - // Next 2 are either user-defined or inferred from drive size + // Slot sizing settings are either user-defined or inferred from drive size ui32 ExpectedSlotCount = 0; ui32 SlotSizeInUnits = 0; + ui64 ExpectedSlotSize = 0; + ui32 MaxSlots = 0; // Free chunk permille that triggers Cyan color (e.g. 100 is 10%). Between 130 (default) and 13. ui32 ChunkBaseLimit = 130; @@ -341,6 +343,8 @@ struct TPDiskConfig : public TThrRefBase { str << " MaxQueuedCompletionActions# " << MaxQueuedCompletionActions << x; str << " IoPieceSizeBytes# " << IoPieceSizeBytes << x; str << " ExpectedSlotCount# " << ExpectedSlotCount << x; + str << " ExpectedSlotSize# " << ExpectedSlotSize << x; + str << " MaxSlots# " << MaxSlots << x; str << " SlotSizeInUnits# " << SlotSizeInUnits << x; str << " ReserveLogChunksMultiplier# " << ReserveLogChunksMultiplier << x; @@ -437,6 +441,14 @@ struct TPDiskConfig : public TThrRefBase { ExpectedSlotCount = cfg->GetExpectedSlotCount(); } + if (cfg->HasExpectedSlotSize()) { + ExpectedSlotSize = cfg->GetExpectedSlotSize(); + } + + if (cfg->HasMaxSlots()) { + MaxSlots = cfg->GetMaxSlots(); + } + if (cfg->HasChunkBaseLimit()) { ui32 limit = cfg->GetChunkBaseLimit(); limit = Min<ui32>(130, limit); @@ -472,7 +484,11 @@ struct TPDiskConfig : public TThrRefBase { } ui32 GetOwnerWeight(ui32 groupSizeInUnits) { - return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, SlotSizeInUnits); + return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, SlotSizeInUnits, ExpectedSlotSize); + } + + static ui32 GetOwnerWeight(ui32 groupSizeInUnits, ui32 slotSizeInUnits, ui64 expectedSlotSize) { + return expectedSlotSize ? 1 : GetOwnerWeight(groupSizeInUnits, slotSizeInUnits); } static ui32 GetOwnerWeight(ui32 groupSizeInUnits, ui32 slotSizeInUnits) { @@ -483,6 +499,7 @@ struct TPDiskConfig : public TThrRefBase { }; struct TInferPDiskSlotCountSettingsForDriveType { + ui64 SlotSize = 0; ui64 UnitSize = 0; ui32 MaxSlots = 0; bool PreferInferredSettingsOverExplicit = false; @@ -490,12 +507,14 @@ struct TInferPDiskSlotCountSettingsForDriveType { TInferPDiskSlotCountSettingsForDriveType(const NKikimrBlobStorage::TInferPDiskSlotCountSettings& settings, NPDisk::EDeviceType type) { switch (type) { case NPDisk::DEVICE_TYPE_ROT: + SlotSize = settings.GetRot().GetSlotSize(); UnitSize = settings.GetRot().GetUnitSize(); MaxSlots = settings.GetRot().GetMaxSlots(); PreferInferredSettingsOverExplicit = settings.GetRot().GetPreferInferredSettingsOverExplicit(); break; case NPDisk::DEVICE_TYPE_SSD: case NPDisk::DEVICE_TYPE_NVME: + SlotSize = settings.GetSsd().GetSlotSize(); UnitSize = settings.GetSsd().GetUnitSize(); MaxSlots = settings.GetSsd().GetMaxSlots(); PreferInferredSettingsOverExplicit = settings.GetSsd().GetPreferInferredSettingsOverExplicit(); @@ -506,7 +525,7 @@ struct TInferPDiskSlotCountSettingsForDriveType { } explicit operator bool() const { - return UnitSize && MaxSlots; + return (SlotSize || UnitSize) && MaxSlots; } }; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index 1d597380fc0..1c7cc705129 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -56,6 +56,7 @@ TPDisk::TPDisk(std::shared_ptr<TPDiskCtx> pCtx, const TIntrusivePtr<TPDiskConfig , Cfg(cfg) , CreationTime(TInstant::Now()) , ExpectedSlotCount(cfg->ExpectedSlotCount) + , ExpectedSlotSize(cfg->ExpectedSlotSize) , UseHugePages(cfg->UseSpdkNvmeDriver) { SlowdownAddLatencyNs = TControlWrapper(0, 0, 100'000'000'000ll); @@ -124,6 +125,25 @@ TPDisk::TPDisk(std::shared_ptr<TPDiskCtx> pCtx, const TIntrusivePtr<TPDiskConfig }; } +void TPDisk::NormalizeExpectedSlotSettings() { + Cfg->ExpectedSlotCount = ExpectedSlotCount; + Cfg->ExpectedSlotSize = ExpectedSlotSize; +} + +i64 TPDisk::GetExpectedOwnerSizeInChunks() const { + if (!ExpectedSlotSize || !Format.ChunkSize) { + return 0; + } + // Quota accounting is chunk-based; round down to keep ExpectedSlotSize an upper bound. + ui64 chunks = ExpectedSlotSize / Format.ChunkSize; + Y_VERIFY(chunks <= ui64(Max<i64>())); + return chunks; +} + +ui32 TPDisk::GetOwnerWeight(ui32 groupSizeInUnits) const { + return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, Cfg->SlotSizeInUnits, ExpectedSlotSize); +} + TString TPDisk::DynamicStateToString(bool isMultiline) { TStringStream str; const char *x = isMultiline ? "\n" : ""; @@ -1692,9 +1712,10 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { } pdiskState.SetNumActiveSlots(numActiveSlots); pdiskState.SetSlotSizeInUnits(Cfg->SlotSizeInUnits); - if (ExpectedSlotCount) { - pdiskState.SetExpectedSlotCount(ExpectedSlotCount); - } + // set unconditionally: whiteboard merges updates field by field, so a field skipped + // when its value returns to 0 would keep the stale nonzero value until node restart + pdiskState.SetExpectedSlotCount(ExpectedSlotCount); + pdiskState.SetExpectedSlotSize(ExpectedSlotSize); *Mon.NumActiveSlots = numActiveSlots; *Mon.SlotSizeInUnits = Cfg->SlotSizeInUnits; @@ -1767,6 +1788,9 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { if (ExpectedSlotCount) { pDiskMetrics.SetSlotCount(ExpectedSlotCount); } + if (ExpectedSlotSize) { + pDiskMetrics.SetExpectedSlotSize(ExpectedSlotSize); + } double pdiskUsage = Keeper.GetPDiskUsage(); pDiskMetrics.SetPDiskUsage(pdiskUsage); @@ -2186,7 +2210,7 @@ void TPDisk::YardInitFinish(TYardInit &evYardInit) { } // Allocate quota for the owner - Keeper.AddOwner(owner, vDiskId, Cfg->GetOwnerWeight(evYardInit.GroupSizeInUnits)); + Keeper.AddOwner(owner, vDiskId, GetOwnerWeight(evYardInit.GroupSizeInUnits)); TOwnerData& ownerData = OwnerData[owner]; ownerData.Reset(false); @@ -2263,7 +2287,7 @@ void TPDisk::YardResize(TYardResize &ev) { { TGuard<TMutex> guard(StateMutex); OwnerData[ev.Owner].GroupSizeInUnits = ev.GroupSizeInUnits; - Keeper.SetOwnerWeight(ev.Owner, Cfg->GetOwnerWeight(ev.GroupSizeInUnits)); + Keeper.SetOwnerWeight(ev.Owner, GetOwnerWeight(ev.GroupSizeInUnits)); } auto result = std::make_unique<NPDisk::TEvYardResizeResult>(NKikimrProto::OK, GetStatusFlags(ev.Owner, ev.OwnerGroupType), TString()); @@ -2279,12 +2303,13 @@ void TPDisk::YardResize(TYardResize &ev) { void TPDisk::ProcessChangeExpectedSlotCount(TChangeExpectedSlotCount& request) { TGuard<TMutex> guard(StateMutex); ExpectedSlotCount = request.ExpectedSlotCount; - Cfg->ExpectedSlotCount = request.ExpectedSlotCount; + ExpectedSlotSize = request.ExpectedSlotSize; + NormalizeExpectedSlotSettings(); Cfg->SlotSizeInUnits = request.SlotSizeInUnits; - Keeper.SetExpectedOwnerCount(ExpectedSlotCount); + Keeper.SetExpectedOwnerSettings(ExpectedSlotCount, GetExpectedOwnerSizeInChunks()); for (TOwner owner = OwnerBeginUser; owner < OwnerEndUser; ++owner) { if (OwnerData[owner].VDiskId != TVDiskID::InvalidId) { - Keeper.SetOwnerWeight(owner, Cfg->GetOwnerWeight(OwnerData[owner].GroupSizeInUnits)); + Keeper.SetOwnerWeight(owner, GetOwnerWeight(OwnerData[owner].GroupSizeInUnits)); } } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h index a34a501c523..ee6a3569a0b 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h @@ -230,6 +230,7 @@ public: ui64 LastInitialSectorIdx; ui32 ExpectedSlotCount = 0; // Number of slots to use for space limit calculation. + ui64 ExpectedSlotSize = 0; // Slot size to use for space limit calculation, 0 if not set. TAtomic TotalOwners = 0; // number of registered owners @@ -384,6 +385,9 @@ public: bool YardInitForKnownVDisk(TYardInit &evYardInit, TOwner owner); void YardResize(TYardResize &evYardResize); void ProcessChangeExpectedSlotCount(TChangeExpectedSlotCount& request); + void NormalizeExpectedSlotSettings(); + i64 GetExpectedOwnerSizeInChunks() const; + ui32 GetOwnerWeight(ui32 groupSizeInUnits) const; // Scheduler weight configuration void ConfigureCbs(ui32 ownerId, EGate gate, ui64 weight); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp index 210216f1e8b..32856307800 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp @@ -1732,8 +1732,10 @@ void TPDisk::ProcessReadLogResult(const NPDisk::TEvReadLogResult &evReadLogResul // Reset chunk trackers TKeeperParams params; + NormalizeExpectedSlotSettings(); params.TotalChunks = Format.DiskSizeChunks(); params.ExpectedOwnerCount = Cfg->ExpectedSlotCount; + params.ExpectedOwnerSize = GetExpectedOwnerSizeInChunks(); params.SysLogSize = Format.SystemChunkCount; // sysLogSize = chunk 0 + additional SysLog chunks params.CommonLogSize = LogChunks.size(); @@ -1752,7 +1754,7 @@ void TPDisk::ProcessReadLogResult(const NPDisk::TEvReadLogResult &evReadLogResul params.OwnersInfo[ownerId] = { .ChunksOwned = usedForOwner[ownerId], .VDiskId = OwnerData[ownerId].VDiskId, - .Weight = Cfg->GetOwnerWeight(OwnerData[ownerId].GroupSizeInUnits), + .Weight = GetOwnerWeight(OwnerData[ownerId].GroupSizeInUnits), }; if (OwnerData[ownerId].IsStaticGroupOwner()) { params.HasStaticGroups = true; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h index d181672aa76..ed6d9802547 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h @@ -215,6 +215,14 @@ public: ChunkTracker.SetExpectedOwnerCount(newOwnerCount); } + void SetExpectedOwnerSize(i64 newOwnerSize) { + ChunkTracker.SetExpectedOwnerSize(newOwnerSize); + } + + void SetExpectedOwnerSettings(size_t newOwnerCount, i64 newOwnerSize) { + ChunkTracker.SetExpectedOwnerSettings(newOwnerCount, newOwnerSize); + } + void SetColorBorder(NKikimrBlobStorage::TPDiskSpaceColor::E colorBorder) { ChunkTracker.SetColorBorder(colorBorder); } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper_params.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper_params.h index 74fd5149d01..b126083a20b 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper_params.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper_params.h @@ -27,6 +27,9 @@ struct TKeeperParams { // Split main chunk pool between this many owners (or 0 for 'split as you go' mode) i64 ExpectedOwnerCount = 0; + // Exact chunk quota for one owner weight unit; 0 means derive it from ExpectedOwnerCount + i64 ExpectedOwnerSize = 0; + // Number of chunks used for format record and system log i64 SysLogSize = 0; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_requestimpl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_requestimpl.h index c9577feab50..7bae1fe2576 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_requestimpl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_requestimpl.h @@ -1286,11 +1286,13 @@ class TChangeExpectedSlotCount : public TRequestBase { public: ui32 ExpectedSlotCount; ui32 SlotSizeInUnits; + ui64 ExpectedSlotSize; TChangeExpectedSlotCount(const NPDisk::TEvChangeExpectedSlotCount &ev, const TActorId &sender, TAtomicBase reqIdx) : TRequestBase(sender, TReqId(TReqId::ChangeExpectedSlotCount, reqIdx), OwnerSystem, 0, NPriInternal::Other) , ExpectedSlotCount(ev.ExpectedSlotCount) , SlotSizeInUnits(ev.SlotSizeInUnits) + , ExpectedSlotSize(ev.ExpectedSlotSize) {} ERequestType GetType() const override { @@ -1302,6 +1304,7 @@ public: str << "TChangeExpectedSlotCount {" << " ExpectedSlotCount# " << ExpectedSlotCount << " SlotSizeInUnits# " << SlotSizeInUnits + << " ExpectedSlotSize# " << ExpectedSlotSize << " }"; return str.Str(); } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index 69ea6929d2c..c5335681c5c 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -2047,6 +2047,35 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 0.99, 0.0, 99.6, 3, 4, TColor::RED); } + Y_UNIT_TEST(ExpectedSlotSizeHardLimitRoundsDownToChunkSize) { + TActorTestContext testCtx({ + .DiskSize = 1_GB, + .ChunkSize = 1_MB, + }); + + const ui32 formatChunkSize = testCtx.SafeRunOnPDisk([](const NPDisk::TPDisk* pdisk) { + return pdisk->Format.ChunkSize; + }); + const ui64 expectedSlotSize = 3ull * formatChunkSize + 1; + + auto pdiskConfig = testCtx.GetPDiskConfig(); + pdiskConfig->ExpectedSlotSize = expectedSlotSize; + testCtx.UpdateConfigRecreatePDisk(pdiskConfig); + + TVDiskMock vdisk(&testCtx); + vdisk.InitFull(); + vdisk.SendEvLogSync(); + + const auto evCheckSpaceResult = testCtx.TestResponse<NPDisk::TEvCheckSpaceResult>( + new NPDisk::TEvCheckSpace(vdisk.PDiskParams->Owner, vdisk.PDiskParams->OwnerRound), + NKikimrProto::OK); + + const ui32 expectedHardLimitChunks = expectedSlotSize / formatChunkSize; + UNIT_ASSERT_VALUES_EQUAL(expectedHardLimitChunks, 3u); + UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->TotalChunks, expectedHardLimitChunks); + UNIT_ASSERT_LE(ui64(evCheckSpaceResult->TotalChunks) * formatChunkSize, expectedSlotSize); + } + Y_UNIT_TEST(PDiskCapacityAlertWithFullCommonLog) { using TColor = NKikimrBlobStorage::TPDiskSpaceColor; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_chunk_tracker.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_chunk_tracker.cpp index a1aea722e94..44bcc14e8ab 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_chunk_tracker.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_chunk_tracker.cpp @@ -136,6 +136,101 @@ Y_UNIT_TEST_SUITE(TChunkTrackerTest) { UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(103), 50); } + Y_UNIT_TEST(SharedQuotaFailureReleasesOwnerQuota) { + using namespace NPDisk; + + TChunkTracker chunkTracker; + TKeeperParams params { + .TotalChunks = 205 /*system*/ + 10, + .ExpectedOwnerCount = 1, + }; + + TString errorReason; + UNIT_ASSERT_C(chunkTracker.Reset(params, TColorLimits::MakeLogLimits(), errorReason), errorReason); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetTotalHardLimit(), 10); + chunkTracker.AddOwner(101, TVDiskID(), 1); + + // in the count-based mode the owner quota is force-allocated before the shared quota + // check; when the shared quota rejects the request, the owner quota accounting must + // be rolled back instead of keeping the never-used allocation + UNIT_ASSERT(!chunkTracker.TryAllocate(101, 11, errorReason)); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerUsed(101), 0); + + // the rolled-back quota must be allocatable again + UNIT_ASSERT_C(chunkTracker.TryAllocate(101, 5, errorReason), errorReason); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerUsed(101), 5); + } + + Y_UNIT_TEST(ExpectedOwnerSizeForcesOwnerWeightToOne) { + using namespace NPDisk; + + TChunkTracker chunkTracker; + TKeeperParams params { + .TotalChunks = 205 /*system*/ + 100, + .ExpectedOwnerCount = 4, + .ExpectedOwnerSize = 30, + }; + + TString errorReason; + bool ok; + + ok = chunkTracker.Reset(params, TColorLimits::MakeLogLimits(), errorReason); + UNIT_ASSERT_C(ok, errorReason); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetTotalHardLimit(), 100); + + chunkTracker.AddOwner(101, TVDiskID(), 1); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(101), 30); + + chunkTracker.AddOwner(102, TVDiskID(), 2); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(101), 30); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(102), 30); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerWeight(102), 1); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetNumActiveSlots(), 2); + + chunkTracker.SetExpectedOwnerSize(0); + chunkTracker.SetOwnerWeight(102, 2); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(101), 25); + UNIT_ASSERT_EQUAL_X(chunkTracker.GetOwnerHardLimit(102), 50); + } + + Y_UNIT_TEST(ExpectedOwnerSizeLimitsOwnerAllocation) { + using namespace NPDisk; + + TChunkTracker chunkTracker; + TKeeperParams params { + .TotalChunks = 205 /*system*/ + 100, + .ExpectedOwnerCount = 4, + .ExpectedOwnerSize = 30, + }; + + TString errorReason; + UNIT_ASSERT_C(chunkTracker.Reset(params, TColorLimits::MakeLogLimits(), errorReason), errorReason); + chunkTracker.AddOwner(101, TVDiskID(), 1); + + UNIT_ASSERT_C(chunkTracker.TryAllocate(101, 27, errorReason), errorReason); + UNIT_ASSERT(!chunkTracker.TryAllocate(101, 1, errorReason)); + UNIT_ASSERT_C(errorReason.Contains("Owner# 101"), errorReason); + } + + Y_UNIT_TEST(ExpectedOwnerSizeRuntimeUpdateLimitsOwnerAllocation) { + using namespace NPDisk; + + TChunkTracker chunkTracker; + TKeeperParams params { + .TotalChunks = 205 /*system*/ + 100, + .ExpectedOwnerCount = 4, + }; + + TString errorReason; + UNIT_ASSERT_C(chunkTracker.Reset(params, TColorLimits::MakeLogLimits(), errorReason), errorReason); + chunkTracker.AddOwner(101, TVDiskID(), 1); + chunkTracker.SetExpectedOwnerSettings(4, 30); + + UNIT_ASSERT_C(chunkTracker.TryAllocate(101, 27, errorReason), errorReason); + UNIT_ASSERT(!chunkTracker.TryAllocate(101, 1, errorReason)); + UNIT_ASSERT_C(errorReason.Contains("Owner# 101"), errorReason); + } + Y_UNIT_TEST(ZeroWeight) { using namespace NPDisk; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_pdisk_config.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_pdisk_config.cpp index 4cc3a158116..960ca23a337 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_pdisk_config.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_pdisk_config.cpp @@ -45,10 +45,18 @@ Y_UNIT_TEST_SUITE(TPDiskConfig) { UNIT_ASSERT_VALUES_EQUAL(TPDiskConfig::GetOwnerWeight(99, 100), 1); UNIT_ASSERT_VALUES_EQUAL(TPDiskConfig::GetOwnerWeight(101, 100), 2); + UNIT_ASSERT_VALUES_EQUAL(TPDiskConfig::GetOwnerWeight(4, 1, 100ull << 30), 1); + UNIT_ASSERT_VALUES_EQUAL(TPDiskConfig::GetOwnerWeight(101, 100, 100ull << 30), 1); + TPDiskConfig pdiskConfig3u(0, 0, 0); pdiskConfig3u.SlotSizeInUnits = 3; UNIT_ASSERT_VALUES_EQUAL(pdiskConfig3u.GetOwnerWeight(10), 4); + TPDiskConfig pdiskConfigFixedSize(0, 0, 0); + pdiskConfigFixedSize.SlotSizeInUnits = 3; + pdiskConfigFixedSize.ExpectedSlotSize = 100ull << 30; + UNIT_ASSERT_VALUES_EQUAL(pdiskConfigFixedSize.GetOwnerWeight(10), 1); + // TODO(ydynnikov): test the case of groupSizeInUnits > UI8_MAX (255) } diff --git a/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp b/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp index 7f2abb69aa5..febc2353cba 100644 --- a/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp @@ -648,4 +648,286 @@ Y_UNIT_TEST_SUITE(GroupAllocation) { checkGroupStatus(); } } + + // DomainLevelEnd values corresponding to fail_domain_type setting (see ydb/tools/cfg/types.py, DistinctionLevels) + constexpr ui32 DomainLevelEndRack = 40; // fail_domain_type: rack + constexpr ui32 DomainLevelEndUnit = 50; // fail_domain_type: body (unit) + constexpr ui32 DomainLevelEndDiskScope = 60; // fail_domain_type: disk_scope + constexpr ui32 DomainLevelEndDisk = 256; // fail_domain_type: disk + + std::function<TNodeLocation(ui32)> MakeLocationGenerator() { + return [](ui32 nodeId) { + NActorsInterconnect::TNodeLocation proto; + proto.SetDataCenter("A"); + proto.SetRack(std::to_string(nodeId)); + proto.SetUnit("1"); + return TNodeLocation(proto); + }; + } + + NKikimrBlobStorage::TGroupGeometry MakeGeometry(ui32 domainLevelEnd) { + NKikimrBlobStorage::TGroupGeometry geometry; + geometry.SetNumFailRealms(1); + geometry.SetNumFailDomainsPerFailRealm(8); + geometry.SetNumVDisksPerFailDomain(1); + geometry.SetRealmLevelBegin(10); + geometry.SetRealmLevelEnd(20); + geometry.SetDomainLevelBegin(10); + geometry.SetDomainLevelEnd(domainLevelEnd); + return geometry; + } + + // scopes: list of {diskScope, numDrives}; empty diskScope means drives without DiskScope set + void DefineHostConfig(NKikimrBlobStorage::TConfigRequest& request, ui32 hostConfigId, + const std::vector<std::pair<TString, ui32>>& scopes) { + auto *cmd = request.AddCommand()->MutableDefineHostConfig(); + cmd->SetHostConfigId(hostConfigId); + for (const auto& [diskScope, numDrives] : scopes) { + for (ui32 pdiskIdx = 0; pdiskIdx < numDrives; ++pdiskIdx) { + auto *drive = cmd->AddDrive(); + const TString pdiskName = (diskScope ? diskScope : "noscope") + ToString(pdiskIdx); + drive->SetPath("SectorMap:" + pdiskName + ":1000"); + drive->SetType(NKikimrBlobStorage::EPDiskType::ROT); + if (diskScope) { + drive->SetDiskScope(diskScope); + } + } + } + } + + void DefineBox(NKikimrBlobStorage::TConfigRequest& request, TEnvironmentSetup& env, + const std::function<ui32(ui32)>& hostConfigIdForNode) { + auto *cmd = request.AddCommand()->MutableDefineBox(); + cmd->SetBoxId(1); + for (ui32 nodeId : env.Runtime->GetNodes()) { + auto *host = cmd->AddHost(); + host->MutableKey()->SetNodeId(nodeId); + host->SetHostConfigId(hostConfigIdForNode(nodeId)); + } + } + + void DefineStoragePool(NKikimrBlobStorage::TConfigRequest& request, TEnvironmentSetup& env, + ui32 storagePoolId, const TString& name, ui32 numGroups, ui32 domainLevelEnd) { + auto *cmd = request.AddCommand()->MutableDefineStoragePool(); + cmd->SetBoxId(1); + cmd->SetStoragePoolId(storagePoolId); + cmd->SetName(name); + cmd->SetKind(name); + cmd->SetErasureSpecies(TBlobStorageGroupType::ErasureSpeciesName(env.Settings.Erasure.GetErasure())); + cmd->SetVDiskKind("Default"); + cmd->SetNumGroups(numGroups); + cmd->AddPDiskFilter()->AddProperty()->SetType(NKikimrBlobStorage::EPDiskType::ROT); + cmd->MutableGeometry()->CopyFrom(MakeGeometry(domainLevelEnd)); + } + + std::map<std::pair<ui32, ui32>, TString> GetPDiskScopes(const NKikimrBlobStorage::TBaseConfig& cfg) { + std::map<std::pair<ui32, ui32>, TString> scopes; + for (const auto& pdisk : cfg.GetPDisk()) { + if (pdisk.HasDiskScope()) { + scopes.emplace(std::make_pair(pdisk.GetNodeId(), pdisk.GetPDiskId()), pdisk.GetDiskScope()); + } + } + return scopes; + } + + Y_UNIT_TEST(DiskScopeUniformDistribution) { + const ui32 numNodes = 4; + const ui32 disksPerNode = 4; + const ui32 numGroups = 16; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); + DefineBox(request, env, [](ui32) { return 1; }); + DefineStoragePool(request, env, 1, env.StoragePoolName, numGroups, DomainLevelEndDiskScope); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), numGroups); + UNIT_ASSERT_VALUES_EQUAL(base.PDiskSize(), numNodes * disksPerNode); + + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, DomainLevelEndDiskScope); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, base, true, error), error); + + const auto scopes = GetPDiskScopes(base); + + std::map<ui32, std::set<std::pair<ui32, TString>>> groupNodeScopePairs; + std::map<std::pair<ui32, ui32>, ui32> slotsPerPDisk; + for (const auto& vslot : base.GetVSlot()) { + const auto key = std::make_pair(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()); + ++slotsPerPDisk[key]; + const auto it = scopes.find(key); + UNIT_ASSERT_C(it != scopes.end(), "group VDisk resides on PDisk without DiskScope, nodeId# " + << key.first << " pdiskId# " << key.second); + const bool inserted = groupNodeScopePairs[vslot.GetGroupId()].emplace(key.first, it->second).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group share node and DiskScope, groupId# " + << vslot.GetGroupId() << " nodeId# " << key.first << " scope# " << it->second); + } + + // every group occupies 8 distinct (node, scope) fail domains + UNIT_ASSERT_VALUES_EQUAL(groupNodeScopePairs.size(), numGroups); + for (const auto& [groupId, nodeScopePairs] : groupNodeScopePairs) { + UNIT_ASSERT_VALUES_EQUAL_C(nodeScopePairs.size(), 8, "groupId# " << groupId); + } + + // VDisks are distributed uniformly among all PDisks + const ui32 expectedSlotsPerPDisk = numGroups * 8 / (numNodes * disksPerNode); + UNIT_ASSERT_VALUES_EQUAL(slotsPerPDisk.size(), numNodes * disksPerNode); + for (const auto& [pdisk, slots] : slotsPerPDisk) { + UNIT_ASSERT_VALUES_EQUAL_C(slots, expectedSlotsPerPDisk, "uneven VDisk distribution, nodeId# " + << pdisk.first << " pdiskId# " << pdisk.second); + } + } + + Y_UNIT_TEST(AllFailDomainTypes) { + const ui32 numNodes = 8; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + const std::vector<std::pair<TString, ui32>> pools = { + {"disk_scope", DomainLevelEndDiskScope}, + {"unit", DomainLevelEndUnit}, + {"disk", DomainLevelEndDisk}, + {"rack", DomainLevelEndRack}, + }; + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); + DefineBox(request, env, [](ui32) { return 1; }); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + for (size_t i = 0; i < pools.size(); ++i) { + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, i + 1, "pool-" + pools[i].first, 1, pools[i].second); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), "failed to allocate group with fail_domain_type# " + << pools[i].first << " error# " << response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), pools.size()); + + std::map<ui32, ui32> groupToPool; + for (const auto& group : base.GetGroup()) { + groupToPool[group.GetGroupId()] = group.GetStoragePoolId(); + } + + const auto scopes = GetPDiskScopes(base); + + std::map<ui32, std::vector<std::pair<ui32, ui32>>> pdisksPerPool; // storagePoolId -> pdisks with group VDisks + for (const auto& vslot : base.GetVSlot()) { + const auto it = groupToPool.find(vslot.GetGroupId()); + UNIT_ASSERT(it != groupToPool.end()); + pdisksPerPool[it->second].emplace_back(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()); + } + + for (size_t i = 0; i < pools.size(); ++i) { + const auto& [name, domainLevelEnd] = pools[i]; + const auto& pdisks = pdisksPerPool[i + 1]; + UNIT_ASSERT_VALUES_EQUAL_C(pdisks.size(), 8, "fail_domain_type# " << name); + + // verify layout of this pool's group against pool geometry + NKikimrBlobStorage::TBaseConfig poolConfig = base; + poolConfig.ClearVSlot(); + for (const auto& vslot : base.GetVSlot()) { + if (groupToPool[vslot.GetGroupId()] == i + 1) { + poolConfig.AddVSlot()->CopyFrom(vslot); + } + } + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, domainLevelEnd); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, poolConfig, true, error), "fail_domain_type# " << name + << " error# " << error); + + if (domainLevelEnd == DomainLevelEndDiskScope) { + // disk_scope fail domain type must not allow two VDisks to share the same node and DiskScope + std::set<std::pair<ui32, TString>> nodeScopePairs; + for (const auto& pdisk : pdisks) { + const auto it = scopes.find(pdisk); + UNIT_ASSERT_C(it != scopes.end(), "group VDisk resides on PDisk without DiskScope, nodeId# " + << pdisk.first << " pdiskId# " << pdisk.second); + const bool inserted = nodeScopePairs.emplace(pdisk.first, it->second).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group share node and DiskScope, nodeId# " + << pdisk.first << " scope# " << it->second); + } + } else if (domainLevelEnd == DomainLevelEndRack) { + // rack fail domain type must not allow two VDisks on the same node + std::set<ui32> distinctNodes; + for (const auto& pdisk : pdisks) { + distinctNodes.insert(pdisk.first); + } + UNIT_ASSERT_VALUES_EQUAL_C(distinctNodes.size(), pdisks.size(), + "two VDisks of the same group occupy the same node, fail_domain_type# " << name); + } + } + } + + Y_UNIT_TEST(PartialDiskScopeMarkdown) { + const ui32 numNodes = 4; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); // with disk scopes + DefineHostConfig(request, 2, {{"", 4}}); // without disk scopes + DefineBox(request, env, [&](ui32 nodeId) { return nodeId <= numNodes / 2 ? 1 : 2; }); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + { + // allocation with fail_domain_type: disk_scope is impossible: nodes without + // DiskScope markdown provide only one fail domain each, 6 fail domains total + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, 1, "pool-disk-scope", 1, DomainLevelEndDiskScope); + auto response = env.Invoke(request); + UNIT_ASSERT_C(!response.GetSuccess(), "group allocation with fail_domain_type# disk_scope must fail"); + } + + { + // allocation with fail_domain_type: disk is possible: every PDisk is a separate fail domain + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, 2, "pool-disk", 1, DomainLevelEndDisk); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), 1); + + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, DomainLevelEndDisk); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, base, true, error), error); + + std::set<std::pair<ui32, ui32>> usedPDisks; + for (const auto& vslot : base.GetVSlot()) { + const bool inserted = usedPDisks.emplace(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group occupy the same PDisk"); + } + UNIT_ASSERT_VALUES_EQUAL(usedPDisks.size(), 8); + } } diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index 4db609adf59..eff49dd92eb 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -389,6 +389,7 @@ struct TCommonAppOptions { ui32 GRpcPublicPort = 0; ui32 GRpcsPublicPort = 0; ui32 KafkaPort = 0; + TString KafkaListenAddress = ""; TVector<TString> GRpcPublicAddressesV4; TVector<TString> GRpcPublicAddressesV6; TString GRpcPublicTargetNameOverride = ""; @@ -470,6 +471,7 @@ struct TCommonAppOptions { opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT").StoreResult(&GRpcPublicPort); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT").StoreResult(&GRpcsPublicPort); opts.AddLongOption("kafka-port", "enable kafka proxy to listen on port").OptionalArgument("PORT").StoreResult(&KafkaPort); + opts.AddLongOption("kafka-address", "set kafka proxy listen address").RequiredArgument("ADDR").StoreResult(&KafkaListenAddress); // Should be provided in yaml config: TGRpcConfig.PublicAddressesV4 opts.AddLongOption("grpc-public-address-v4", "set public ipv4 address for discovery").RequiredArgument("ADDR").Hidden().EmplaceTo(&GRpcPublicAddressesV4); // Should be provided in yaml config: TGRpcConfig.PublicAddressesV6 @@ -732,6 +734,11 @@ struct TCommonAppOptions { conf.SetListeningPort(KafkaPort); ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::KafkaProxyConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); } + if (KafkaListenAddress) { + auto& conf = *appConfig.MutableKafkaProxyConfig(); + conf.SetListeningAddress(KafkaListenAddress); + ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::KafkaProxyConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + } if (HttpProxyPort) { auto* httpProxyConfig = appConfig.MutableHttpProxyConfig(); httpProxyConfig->SetEnabled(true); diff --git a/ydb/core/config/validation/client_certificate_authorization_validator.cpp b/ydb/core/config/validation/client_certificate_authorization_validator.cpp new file mode 100644 index 00000000000..18ccf27eec8 --- /dev/null +++ b/ydb/core/config/validation/client_certificate_authorization_validator.cpp @@ -0,0 +1,48 @@ +#include "validators.h" + +#include <ydb/core/protos/config.pb.h> + +#include <util/generic/string.h> + +#include <vector> + +namespace NKikimr::NConfig { + +namespace { + +bool HasGrpcCa(const NKikimrConfig::TAppConfig& config) { + if (!config.HasGRpcConfig()) { + return false; + } + + const auto& grpcConfig = config.GetGRpcConfig(); + return !grpcConfig.GetCA().empty() || !grpcConfig.GetPathToCaFile().empty(); +} + +} // namespace + +EValidationResult ValidateClientCertificateAuthorization( + const NKikimrConfig::TAppConfig& config, + std::vector<TString>& msg +) { + if (!config.HasClientCertificateAuthorization()) { + return EValidationResult::Ok; + } + + const auto& clientCertificateAuthorization = config.GetClientCertificateAuthorization(); + if (clientCertificateAuthorization.GetClientCertificateRequired() && + !clientCertificateAuthorization.GetRequestClientCertificate() + ) { + msg.push_back("RequestClientCertificate is disabled, but ClientCertificateRequired is enabled"); + return EValidationResult::Error; + } + + if (clientCertificateAuthorization.GetClientCertificateRequired() && !HasGrpcCa(config)) { + msg.push_back("gRPC CA is not set, but ClientCertificateRequired is enabled"); + return EValidationResult::Error; + } + + return EValidationResult::Ok; +} + +} // NKikimr::NConfig diff --git a/ydb/core/config/validation/monitoring_config_validator.cpp b/ydb/core/config/validation/monitoring_config_validator.cpp index 03061465018..648b43ba46c 100644 --- a/ydb/core/config/validation/monitoring_config_validator.cpp +++ b/ydb/core/config/validation/monitoring_config_validator.cpp @@ -21,6 +21,11 @@ bool GetEnforceUserToken(const NKikimrConfig::TAppConfig& config) { return securityConfig.GetEnforceUserTokenRequirement(); } +bool HasMonitoringTlsCertificate(const NKikimrConfig::TMonitoringConfig& monitoringConfig) { + return !monitoringConfig.GetMonitoringCertificateFile().empty() || + !monitoringConfig.GetMonitoringCertificate().empty(); +} + } // namespace EValidationResult ValidateMonitoringConfig(const NKikimrConfig::TAppConfig& config, std::vector<TString>& msg) { @@ -40,6 +45,18 @@ EValidationResult ValidateMonitoringConfig(const NKikimrConfig::TAppConfig& conf } } + if (monitoringConfig.GetClientCertificateRequired()) { + if (!HasMonitoringTlsCertificate(monitoringConfig)) { + msg.push_back("Monitoring server certificate is not set, but ClientCertificateRequired is enabled"); + return EValidationResult::Error; + } + + if (monitoringConfig.GetMonitoringCaFile().empty()) { + msg.push_back("MonitoringCaFile is not set, but ClientCertificateRequired is enabled"); + return EValidationResult::Error; + } + } + return EValidationResult::Ok; } diff --git a/ydb/core/config/validation/validators.cpp b/ydb/core/config/validation/validators.cpp index 1218120688d..3a62794ae39 100644 --- a/ydb/core/config/validation/validators.cpp +++ b/ydb/core/config/validation/validators.cpp @@ -1,6 +1,8 @@ #include "validators.h" #include <ydb/core/base/statestorage.h> +#include <ydb/core/blobstorage/base/infer_pdisk_slot_count_settings.h> +#include <ydb/core/blobstorage/base/pdisk_config_validation.h> #include <ydb/core/config/protos/marker.pb.h> #include <ydb/core/protos/blobstorage.pb.h> #include <ydb/core/protos/blobstorage_base.pb.h> @@ -57,6 +59,16 @@ bool IsSame(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& lhs, const IsSame(lhs.GetVDiskID(), rhs.GetVDiskID()) && IsSame(lhs.GetVDiskLocation(), rhs.GetVDiskLocation()); } +bool ValidatePDiskConfig(const NKikimrBlobStorage::TPDiskConfig& config, const TString& context, + std::vector<TString>& msg) { + if (auto error = ::NKikimr::ValidatePDiskConfig(config, context)) { + msg.push_back(*error); + return false; + } + + return true; +} + EValidationResult ValidateStaticGroup(const NKikimrConfig::TAppConfig& current, const NKikimrConfig::TAppConfig& proposed, std::vector<TString>& msg) { const auto& currentBsConfig = current.GetBlobStorageConfig(); const auto& proposedBsConfig = proposed.GetBlobStorageConfig(); @@ -268,6 +280,55 @@ EValidationResult ValidateConfig(const NKikimrConfig::TAppConfig& config, std::v return EValidationResult::Error; } } + if (config.HasBlobStorageConfig()) { + const auto& blobStorageConfig = config.GetBlobStorageConfig(); + if (blobStorageConfig.HasInferPDiskSlotCountSettings()) { + if (auto error = ValidateInferPDiskSlotCountSettings( + blobStorageConfig.GetInferPDiskSlotCountSettings(), + "BlobStorageConfig.InferPDiskSlotCountSettings")) { + msg.push_back(*error); + return EValidationResult::Error; + } + } + for (ui32 hostConfigIndex = 0; hostConfigIndex < blobStorageConfig.DefineHostConfigSize(); ++hostConfigIndex) { + const auto& hostConfig = blobStorageConfig.GetDefineHostConfig(hostConfigIndex); + if (hostConfig.HasDefaultHostPDiskConfig() && !ValidatePDiskConfig( + hostConfig.GetDefaultHostPDiskConfig(), + TStringBuilder() << "BlobStorageConfig.DefineHostConfig[" << hostConfigIndex + << "].DefaultHostPDiskConfig", + msg)) { + return EValidationResult::Error; + } + for (ui32 driveIndex = 0; driveIndex < hostConfig.DriveSize(); ++driveIndex) { + const auto& drive = hostConfig.GetDrive(driveIndex); + if (drive.HasPDiskConfig() && !ValidatePDiskConfig( + drive.GetPDiskConfig(), + TStringBuilder() << "BlobStorageConfig.DefineHostConfig[" << hostConfigIndex + << "].Drive[" << driveIndex << "].PDiskConfig", + msg)) { + return EValidationResult::Error; + } + } + } + if (blobStorageConfig.HasServiceSet()) { + const auto& serviceSet = blobStorageConfig.GetServiceSet(); + for (ui32 pdiskIndex = 0; pdiskIndex < serviceSet.PDisksSize(); ++pdiskIndex) { + const auto& pdisk = serviceSet.GetPDisks(pdiskIndex); + if (pdisk.HasPDiskConfig() && !ValidatePDiskConfig( + pdisk.GetPDiskConfig(), + TStringBuilder() << "BlobStorageConfig.ServiceSet.PDisks[" << pdiskIndex << "].PDiskConfig", + msg)) { + return EValidationResult::Error; + } + } + } + } + { + NKikimr::NConfig::EValidationResult result = NKikimr::NConfig::ValidateClientCertificateAuthorization(config, msg); + if (result == NKikimr::NConfig::EValidationResult::Error) { + return EValidationResult::Error; + } + } NKikimr::NConfig::EValidationResult result = NKikimr::NConfig::ValidateStateStorageConfig(config, msg); if (result == NKikimr::NConfig::EValidationResult::Error) { return EValidationResult::Error; diff --git a/ydb/core/config/validation/validators.h b/ydb/core/config/validation/validators.h index c73be3755df..895e8844a8d 100644 --- a/ydb/core/config/validation/validators.h +++ b/ydb/core/config/validation/validators.h @@ -54,6 +54,10 @@ EValidationResult ValidateMonitoringConfig( const NKikimrConfig::TAppConfig& config, std::vector<TString>& msg); +EValidationResult ValidateClientCertificateAuthorization( + const NKikimrConfig::TAppConfig& config, + std::vector<TString>& msg); + EValidationResult ValidateDatabaseConfig( const NKikimrConfig::TAppConfig& config, std::vector<TString>& msg); diff --git a/ydb/core/config/validation/validators_ut.cpp b/ydb/core/config/validation/validators_ut.cpp index 0028e292c40..9330a8654e1 100644 --- a/ydb/core/config/validation/validators_ut.cpp +++ b/ydb/core/config/validation/validators_ut.cpp @@ -2,7 +2,9 @@ #include <ydb/core/protos/blobstorage.pb.h> #include <ydb/core/protos/blobstorage_base.pb.h> +#include <ydb/core/protos/blobstorage_config.pb.h> #include <ydb/core/protos/blobstorage_disk.pb.h> +#include <ydb/core/protos/blobstorage_pdisk_config.pb.h> #include <ydb/core/protos/feature_flags.pb.h> #include <ydb/core/protos/table_service_config.pb.h> @@ -536,6 +538,79 @@ Y_UNIT_TEST_SUITE(StateStorageConfigValidation) { UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); } + Y_UNIT_TEST(ValidateConfigInferPDiskSlotSizeSettings) { + NKikimrConfig::TAppConfig proposed; + auto* inferSettings = proposed.MutableBlobStorageConfig()->MutableInferPDiskSlotCountSettings(); + inferSettings->MutableRot()->SetSlotSize(600ull << 30); + std::vector<TString> err; + auto res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 1); + UNIT_ASSERT_C(err[0].Contains("MaxSlots is mandatory with SlotSize or UnitSize"), err[0]); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + + inferSettings->MutableRot()->SetMaxSlots(16); + err.clear(); + res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 0); + UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); + + inferSettings->MutableRot()->SetUnitSize(100ull << 30); + err.clear(); + res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 1); + UNIT_ASSERT_C(err[0].Contains("SlotSize is mutually exclusive with UnitSize"), err[0]); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + + Y_UNIT_TEST(ValidateConfigExpectedSlotSizeRequiresMaxSlots) { + NKikimrConfig::TAppConfig proposed; + auto* pdiskConfig = proposed.MutableBlobStorageConfig() + ->AddDefineHostConfig() + ->AddDrive() + ->MutablePDiskConfig(); + pdiskConfig->SetExpectedSlotSize(600ull << 30); + + std::vector<TString> err; + auto res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 1); + UNIT_ASSERT_C(err[0].Contains("ExpectedSlotSize requires MaxSlots"), err[0]); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + + pdiskConfig->SetMaxSlots(16); + err.clear(); + res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 0); + UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); + + pdiskConfig->SetExpectedSlotCount(4); + err.clear(); + res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 1); + UNIT_ASSERT_C(err[0].Contains("ExpectedSlotSize is mutually exclusive with ExpectedSlotCount"), err[0]); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + + Y_UNIT_TEST(ValidateConfigMaxSlotsRequiresExpectedSlotSize) { + NKikimrConfig::TAppConfig proposed; + auto* pdiskConfig = proposed.MutableBlobStorageConfig() + ->AddDefineHostConfig() + ->AddDrive() + ->MutablePDiskConfig(); + pdiskConfig->SetMaxSlots(16); + + std::vector<TString> err; + auto res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 1); + UNIT_ASSERT_C(err[0].Contains("MaxSlots requires ExpectedSlotSize"), err[0]); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + + pdiskConfig->SetExpectedSlotSize(600ull << 30); + err.clear(); + res = ValidateConfig(proposed, err); + UNIT_ASSERT_VALUES_EQUAL(err.size(), 0); + UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); + } + Y_UNIT_TEST(ValidateConfigDomainEmpty) { NKikimrConfig::TAppConfig proposed; auto* domains = proposed.MutableDomainsConfig(); @@ -688,4 +763,76 @@ Y_UNIT_TEST_SUITE(MonitoringConfigValidation) { UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); } } + + Y_UNIT_TEST(ClientCertificateRequired) { + { // Without monitoring TLS certificate data + NKikimrConfig::TAppConfig config; + config.MutableMonitoringConfig()->SetClientCertificateRequired(true); + std::vector<TString> msg; + auto res = ValidateMonitoringConfig(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 1); + UNIT_ASSERT_EQUAL(msg[0], "Monitoring server certificate is not set, but ClientCertificateRequired is enabled"); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + { // With server certificate, but without CA + NKikimrConfig::TAppConfig config; + auto* monitoringConfig = config.MutableMonitoringConfig(); + monitoringConfig->SetMonitoringCertificateFile("/path/to/cert.pem"); + monitoringConfig->SetMonitoringPrivateKeyFile("/path/to/key.pem"); + monitoringConfig->SetClientCertificateRequired(true); + std::vector<TString> msg; + auto res = ValidateMonitoringConfig(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 1); + UNIT_ASSERT_EQUAL(msg[0], "MonitoringCaFile is not set, but ClientCertificateRequired is enabled"); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + { // With server certificate and CA + NKikimrConfig::TAppConfig config; + auto* monitoringConfig = config.MutableMonitoringConfig(); + monitoringConfig->SetMonitoringCertificateFile("/path/to/cert.pem"); + monitoringConfig->SetMonitoringPrivateKeyFile("/path/to/key.pem"); + monitoringConfig->SetMonitoringCaFile("/path/to/ca.pem"); + monitoringConfig->SetClientCertificateRequired(true); + std::vector<TString> msg; + auto res = ValidateMonitoringConfig(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 0); + UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); + } + } +} + +Y_UNIT_TEST_SUITE(ClientCertificateAuthorizationValidation) { + Y_UNIT_TEST(ClientCertificateRequired) { + { // Without RequestClientCertificate + NKikimrConfig::TAppConfig config; + config.MutableClientCertificateAuthorization()->SetClientCertificateRequired(true); + std::vector<TString> msg; + auto res = ValidateClientCertificateAuthorization(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 1); + UNIT_ASSERT_EQUAL(msg[0], "RequestClientCertificate is disabled, but ClientCertificateRequired is enabled"); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + { // With RequestClientCertificate, with CA file + NKikimrConfig::TAppConfig config; + auto* clientCertificateAuthorization = config.MutableClientCertificateAuthorization(); + clientCertificateAuthorization->SetRequestClientCertificate(true); + clientCertificateAuthorization->SetClientCertificateRequired(true); + config.MutableGRpcConfig()->SetPathToCaFile("/path/to/ca.pem"); + std::vector<TString> msg; + auto res = ValidateClientCertificateAuthorization(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 0); + UNIT_ASSERT_EQUAL(res, EValidationResult::Ok); + } + { // With RequestClientCertificate, but without CA file + NKikimrConfig::TAppConfig config; + auto* clientCertificateAuthorization = config.MutableClientCertificateAuthorization(); + clientCertificateAuthorization->SetRequestClientCertificate(true); + clientCertificateAuthorization->SetClientCertificateRequired(true); + std::vector<TString> msg; + auto res = ValidateClientCertificateAuthorization(config, msg); + UNIT_ASSERT_VALUES_EQUAL(msg.size(), 1); + UNIT_ASSERT_EQUAL(msg[0], "gRPC CA is not set, but ClientCertificateRequired is enabled"); + UNIT_ASSERT_EQUAL(res, EValidationResult::Error); + } + } } diff --git a/ydb/core/config/validation/ya.make b/ydb/core/config/validation/ya.make index 3da692bcca6..fc944afe3f1 100644 --- a/ydb/core/config/validation/ya.make +++ b/ydb/core/config/validation/ya.make @@ -4,11 +4,13 @@ SRCS( validators.h validators.cpp auth_config_validator.cpp + client_certificate_authorization_validator.cpp column_shard_config_validator.cpp monitoring_config_validator.cpp ) PEERDIR( + ydb/core/blobstorage/base ydb/core/protos ydb/core/formats/arrow/serializer library/cpp/protobuf/json diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp index d5aab8185e2..80a3dce28c0 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_genconfig.cpp @@ -229,6 +229,7 @@ public: .Usable = true, .NumSlots = 0, .MaxSlots = 1, + .SlotSizeInBytes = 0, .SpaceAvailable = 0, .Operational = true, .Decommitted = false, diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index db06805e410..82980028679 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -69,6 +69,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("grpc-port", "enable gRPC server on port").RequiredArgument("PORT"); opts.AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT"); opts.AddLongOption("kafka-port", "enable kafka proxy server on port").OptionalArgument("PORT"); + opts.AddLongOption("kafka-address", "set kafka proxy listen address").RequiredArgument("ADDR"); opts.AddLongOption("grpc-public-host", "set public gRPC host for discovery").RequiredArgument("HOST"); opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT"); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT"); @@ -169,6 +170,11 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu conf.SetListeningPort(FromString<ui16>(res.Get("kafka-port"))); } + if (res.Has("kafka-address")) { + auto& conf = *Config.AppConfig.MutableKafkaProxyConfig(); + conf.SetListeningAddress(res.Get("kafka-address")); + } + if (res.Has("grpc-public-host")) { auto& conf = *Config.AppConfig.MutableGRpcConfig(); conf.SetPublicHost(res.Get("grpc-public-host")); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index fe1e6570650..64c129f2d5f 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -84,6 +84,7 @@ #include <ydb/core/kafka_proxy/kafka_proxy.h> #include <ydb/core/kafka_proxy/kafka_transactions_coordinator.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/core/kqp/common/kqp.h> #include <ydb/core/kqp/proxy_service/kqp_proxy_service.h> #include <ydb/core/kqp/rm_service/kqp_rm_service.h> @@ -595,6 +596,14 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter result.EnableUringSQPOLL = config.GetEnableUringSQPOLL(); } + if (config.HasEnableInterconnectSessionV2()) { + result.EnableInterconnectSessionV2 = config.GetEnableInterconnectSessionV2(); + } + + if (config.HasChecksumInterconnectSessionV2()) { + result.ChecksumInterconnectSessionV2 = config.GetChecksumInterconnectSessionV2(); + } + return result; } @@ -2420,6 +2429,17 @@ void TQuoterServiceInitializer::InitializeServices(NActors::TActorSystemSetup* s ); } +TWorkloadManagerServiceInitializer::TWorkloadManagerServiceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + +void TWorkloadManagerServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + auto workloadManager = NWorkloadManager::CreateService(NWorkloadManager::GetWorkloadManagerCounters(appData->Counters)); + setup->LocalServices.push_back(std::make_pair( + NWorkloadManager::MakeServiceId(NodeId), + TActorSetupCmd(workloadManager, TMailboxType::HTSwap, appData->UserPoolId))); +} + TKqpServiceInitializer::TKqpServiceInitializer( const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories, @@ -3227,6 +3247,7 @@ void TKafkaProxyServiceInitializer::InitializeServices(NActors::TActorSystemSetu if (Config.GetKafkaProxyConfig().GetEnableKafkaProxy()) { NKafka::TListenerSettings settings; settings.Port = Config.GetKafkaProxyConfig().GetListeningPort(); + settings.Address = Config.GetKafkaProxyConfig().GetListeningAddress(); settings.SslCertificatePem = Config.GetKafkaProxyConfig().GetSslCertificate(); settings.CertificateFile = Config.GetKafkaProxyConfig().GetCert(); settings.PrivateKeyFile = Config.GetKafkaProxyConfig().GetKey(); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 284a53b8108..b67e8da3bca 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -408,6 +408,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TWorkloadManagerServiceInitializer : public IKikimrServicesInitializer { +public: + TWorkloadManagerServiceInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TKqpServiceInitializer : public IKikimrServicesInitializer { public: TKqpServiceInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories, diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 7718289fd78..9b490bb0366 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -202,6 +202,16 @@ namespace NKikimr { namespace { +// client_certificate_authorization (request/required) applies to grpc_config.ssl_port only. +void FillGrpcSslDataFromClientCertificateAuthorization( + NYdbGrpc::TSslData& sslData, + const NKikimrConfig::TAppConfig& appConfig +) { + const auto& clientCertificateAuthorization = appConfig.GetClientCertificateAuthorization(); + sslData.DoRequestClientCertificate = clientCertificateAuthorization.GetRequestClientCertificate(); + sslData.ClientCertificateRequired = clientCertificateAuthorization.GetClientCertificateRequired(); +} + void StopGRpcServers(std::weak_ptr<TGRpcServersWrapper> grpcServersWrapper, bool isDisabled = false) { auto wrapper = grpcServersWrapper.lock(); if (!wrapper) { @@ -669,6 +679,7 @@ void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool monConfig.CertificateFile = appConfig.GetMonitoringConfig().GetMonitoringCertificateFile(); monConfig.PrivateKeyFile = appConfig.GetMonitoringConfig().GetMonitoringPrivateKeyFile(); monConfig.CaFile = appConfig.GetMonitoringConfig().GetMonitoringCaFile(); + monConfig.ClientCertificateRequired = appConfig.GetMonitoringConfig().GetClientCertificateRequired(); monConfig.RedirectMainPageTo = appConfig.GetMonitoringConfig().GetRedirectMainPageTo(); monConfig.RequireCountersAuthentication = appConfig.GetMonitoringConfig().GetRequireCountersAuthentication(); if (appConfig.GetMonitoringConfig().CompressContentTypesSize() > 0) { @@ -1295,7 +1306,7 @@ TGRpcServers TKikimrRunner::CreateGRpcServers(const TKikimrRunConfig& runConfig) sslData.Root = ReadFile(pathToCaFile); sslData.Cert = ReadFile(pathToCertificateFile); sslData.Key = ReadFile(pathToPrivateKeyFile); - sslData.DoRequestClientCertificate = appConfig.GetClientCertificateAuthorization().GetRequestClientCertificate(); + FillGrpcSslDataFromClientCertificateAuthorization(sslData, appConfig); sslOpts.SetSslData(sslData); grpcServers.push_back({ "grpcs", new NYdbGrpc::TGRpcServer(sslOpts, Counters) }); @@ -1352,6 +1363,8 @@ TGRpcServers TKikimrRunner::CreateGRpcServers(const TKikimrRunConfig& runConfig) pathToPrivateKeyFile = GET_PATH_TO_FILE(grpcConfig, PathToPrivateKeyFile, Key); } sslData.Key = ReadFile(pathToPrivateKeyFile); + // client_certificate_authorization applies to the main grpcs port only. + // ext_endpoints need their own mTLS settings (not implemented yet). #undef GET_PATH_TO_FILE xopts.SetSslData(sslData); @@ -2127,6 +2140,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TMemoryControllerInitializer(runConfig, ProcessMemoryInfoProvider)); + if (serviceMask.EnableWorkloadManagerService) { + sil->AddServiceInitializer(new TWorkloadManagerServiceInitializer(runConfig)); + } + if (serviceMask.EnableKqp) { sil->AddServiceInitializer(new TKqpServiceInitializer(runConfig, ModuleFactories, *this)); } diff --git a/ydb/core/driver_lib/run/service_mask.h b/ydb/core/driver_lib/run/service_mask.h index 0db11ca100a..812834bdbd1 100644 --- a/ydb/core/driver_lib/run/service_mask.h +++ b/ydb/core/driver_lib/run/service_mask.h @@ -86,6 +86,7 @@ union TBasicKikimrServicesMask { bool EnableCountersInfoProvider : 1; bool EnableNBSService : 1; bool EnableUdfStore : 1; + bool EnableWorkloadManagerService : 1; }; struct { diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index ad8c280ef8f..3b751848e0b 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -182,6 +182,7 @@ PEERDIR( ydb/services/tablet ydb/services/test_shard ydb/services/view + ydb/services/workload_manager/service ydb/services/ydb yql/essentials/minikql/comp_nodes/llvm16 yql/essentials/public/udf/service/exception_policy diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp index 73d3b0907d1..8e20e6b7ca0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp @@ -1,17 +1,36 @@ #include "json_value_view.h" +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> + #include <ydb/library/actors/core/log.h> #include <yql/essentials/types/binary_json/read.h> +#include <yql/essentials/types/binary_json/write.h> #include <util/generic/ylimits.h> +#include <util/stream/str.h> #include <util/string/cast.h> +#include <util/string/escape.h> #include <cmath> #include <limits> namespace NKikimr::NArrow::NAccessor { +namespace { + +NBinaryJson::TBinaryJson JsonValueToBinaryJsonVerified(const NJson::TJsonValue& json) { + const auto dumpedJson = WriteJsonRoundTripSafe(json); + auto result = NBinaryJson::SerializeToBinaryJson(dumpedJson); + if (std::holds_alternative<TString>(result)) { + AFL_VERIFY(false)("error", std::get<TString>(result))("type", json.GetType())("input_dump", EscapeC(dumpedJson)); + } + return std::get<NBinaryJson::TBinaryJson>(std::move(result)); +} + +} // namespace + std::optional<TString> TJsonValueView::JsonNumberToString(double jsonNumber) { if (std::isnan(jsonNumber)) { return std::nullopt; @@ -90,6 +109,34 @@ std::optional<TStringBuf> TJsonValueView::GetBinaryJsonBlobOptional() const { return Bytes; } +NJson::TJsonValue TJsonValueView::ToJsonValue() const { + switch (Kind) { + case EKind::BinaryJson: { + const auto text = NBinaryJson::SerializeToJson(Bytes); + NJson::TJsonValue result; + AFL_VERIFY(NJson::ReadJsonTree(text, &result)); + return result; + } + case EKind::String: + return NJson::TJsonValue(Bytes); + case EKind::Number: + return NJson::TJsonValue(Number); + case EKind::Bool: + return NJson::TJsonValue(Bool); + } +} + +NBinaryJson::TBinaryJson TJsonValueView::ToBinaryJson() const { + switch (Kind) { + case EKind::BinaryJson: + return NBinaryJson::TBinaryJson(Bytes.data(), Bytes.size()); + case EKind::String: + case EKind::Number: + case EKind::Bool: + return JsonValueToBinaryJsonVerified(ToJsonValue()); + } +} + std::optional<TStringBuf> TJsonValueView::ScalarFromBinaryJson() const { if (Bytes.empty()) { return std::nullopt; @@ -117,4 +164,13 @@ std::optional<TStringBuf> TJsonValueView::ScalarFromBinaryJson() const { } } +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json) { + NJson::TJsonWriterConfig config; + // avoid precision loss on double conversion + config.FloatToStringMode = PREC_AUTO; + TStringStream dumpedJson; + NJson::WriteJson(&dumpedJson, &json, config); + return dumpedJson.Str(); +} + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.h b/ydb/core/formats/arrow/accessor/common/json_value_view.h index 4a680f68cb9..3d6bbc39fc0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.h +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.h @@ -1,5 +1,8 @@ #pragma once +#include <library/cpp/json/writer/json_value.h> +#include <yql/essentials/types/binary_json/format.h> + #include <util/generic/string.h> #include <optional> @@ -22,6 +25,9 @@ public: // So result will be different for a native scalar and the same scalar wrapped in BinaryJson. // Expected to be used only for containers, but no checks enforce that. std::optional<TStringBuf> GetBinaryJsonBlobOptional() const; + + NJson::TJsonValue ToJsonValue() const; + NBinaryJson::TBinaryJson ToBinaryJson() const; private: static std::optional<TString> JsonNumberToString(double jsonNumber); @@ -51,4 +57,7 @@ private: mutable TString ScalarHolder; }; +// Serialize a JSON value to text with round-trip-safe number formatting. +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json); + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp b/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp index fc3773bd177..b9c83bbeb0a 100644 --- a/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp +++ b/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp @@ -73,4 +73,13 @@ Y_UNIT_TEST_SUITE(JsonValueView) { UNIT_ASSERT_VALUES_EQUAL(ScalarStr(TJsonValueView::OfNumber(2.5)), BlobScalar("2.5")); UNIT_ASSERT_VALUES_EQUAL(ScalarStr(TJsonValueView::OfBool(true)), BlobScalar("true")); } + + Y_UNIT_TEST(ToJsonValue) { + UNIT_ASSERT_VALUES_EQUAL(TJsonValueView::OfString("hi").ToJsonValue().GetString(), "hi"); + UNIT_ASSERT_VALUES_EQUAL(TJsonValueView::OfNumber(2.5).ToJsonValue().GetDouble(), 2.5); + UNIT_ASSERT(TJsonValueView::OfBool(true).ToJsonValue().GetBoolean()); + const auto blob = ToBinaryJson(R"({"a":1})"); + const auto json = TJsonValueView::OfBinaryJson(TStringBuf(blob.data(), blob.size())).ToJsonValue(); + UNIT_ASSERT_VALUES_EQUAL(json["a"].GetInteger(), 1); + } } diff --git a/ydb/core/formats/arrow/accessor/common/ya.make b/ydb/core/formats/arrow/accessor/common/ya.make index 06cb1580445..9d9cd062ace 100644 --- a/ydb/core/formats/arrow/accessor/common/ya.make +++ b/ydb/core/formats/arrow/accessor/common/ya.make @@ -2,6 +2,7 @@ LIBRARY(library-formats-arrow-accessor-common) PEERDIR( contrib/libs/apache/arrow + library/cpp/json library/cpp/json/writer ydb/library/actors/core ydb/library/formats/arrow/protos diff --git a/ydb/core/formats/arrow/accessor/plain/accessor.h b/ydb/core/formats/arrow/accessor/plain/accessor.h index 5a3e7d36a77..246217052c2 100644 --- a/ydb/core/formats/arrow/accessor/plain/accessor.h +++ b/ydb/core/formats/arrow/accessor/plain/accessor.h @@ -70,41 +70,36 @@ public: { } - template <class TArrowDataType = arrow::StringType> - class TPlainBuilder { + class TPlainBuilderBase { private: std::unique_ptr<arrow::ArrayBuilder> Builder; std::optional<ui32> LastRecordIndex; - public: - TPlainBuilder(const ui32 reserveItems = 0, const ui32 reserveSize = 0) { - Builder = NArrow::MakeBuilder(arrow::TypeTraits<TArrowDataType>::type_singleton(), reserveItems, reserveSize); - } - - void AddRecord(const ui32 recordIndex, const std::string_view value) { - AddValue(recordIndex, arrow::util::string_view(value.data(), value.size())); + protected: + arrow::ArrayBuilder& GetBuilder() { + return *Builder; } - template <class TValue> - void AddValue(const ui32 recordIndex, const TValue& value) { + template <class TAppender> + void AddAt(const ui32 recordIndex, const TAppender& append) { if (LastRecordIndex) { AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex - 1)); } else { TStatusValidator::Validate(Builder->AppendNulls(recordIndex)); } + append(*Builder); LastRecordIndex = recordIndex; - AFL_VERIFY(NArrow::Append<TArrowDataType>(*Builder, value)); + } + + public: + TPlainBuilderBase(std::unique_ptr<arrow::ArrayBuilder> builder) + : Builder(std::move(builder)) { + AFL_VERIFY(!!Builder); } void AddNull(const ui32 recordIndex) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); - TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex)); - } else { - TStatusValidator::Validate(Builder->AppendNulls(recordIndex + 1)); - } - LastRecordIndex = recordIndex; + AddAt(recordIndex, [](arrow::ArrayBuilder& builder) { TStatusValidator::Validate(builder.AppendNull()); }); } std::shared_ptr<IChunkedArray> Finish(const ui32 recordsCount) { @@ -118,6 +113,23 @@ public: } }; + template <class TArrowDataType = arrow::StringType> + class TPlainBuilder: public TPlainBuilderBase { + public: + TPlainBuilder(const ui32 reserveItems = 0, const ui32 reserveSize = 0) + : TPlainBuilderBase(NArrow::MakeBuilder(arrow::TypeTraits<TArrowDataType>::type_singleton(), reserveItems, reserveSize)) { + } + + void AddRecord(const ui32 recordIndex, const std::string_view value) { + AddValue(recordIndex, arrow::util::string_view(value.data(), value.size())); + } + + template <class TValue> + void AddValue(const ui32 recordIndex, const TValue& value) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append<TArrowDataType>(builder, value)); }); + } + }; + static TPlainBuilder<arrow::BinaryType> MakeBuilderBinary(const ui32 reserveItems = 0, const ui32 reserveData = 0) { return TPlainBuilder<arrow::BinaryType>(reserveItems, reserveData); } diff --git a/ydb/core/formats/arrow/accessor/sparsed/accessor.h b/ydb/core/formats/arrow/accessor/sparsed/accessor.h index 63758d3969a..c72a9e0031b 100644 --- a/ydb/core/formats/arrow/accessor/sparsed/accessor.h +++ b/ydb/core/formats/arrow/accessor/sparsed/accessor.h @@ -253,38 +253,46 @@ public: return Record.GetRecords(); } - template <class TDataType> - class TSparsedBuilder { + class TSparsedBuilderBase { private: std::unique_ptr<arrow::ArrayBuilder> IndexBuilder; std::unique_ptr<arrow::ArrayBuilder> ValueBuilder; - ui32 RecordsCount = 0; + const std::shared_ptr<arrow::DataType> ValueType; const std::shared_ptr<arrow::Scalar> DefaultValue; std::optional<ui32> LastRecordIndex; - public: - TSparsedBuilder(const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems, const ui32 reserveData) - : DefaultValue(defaultValue) - { - IndexBuilder = NArrow::MakeBuilder(arrow::uint32(), reserveItems, 0); - ValueBuilder = NArrow::MakeBuilder(arrow::TypeTraits<TDataType>::type_singleton(), reserveItems, reserveData); - } - - void AddRecord(const ui32 recordIndex, const std::string_view value) { - if (!!LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex); + void AdvanceRecordIndex(const ui32 recordIndex) { + if (LastRecordIndex) { + AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); } LastRecordIndex = recordIndex; + } + + protected: + arrow::ArrayBuilder& GetValueBuilder() { + return *ValueBuilder; + } + + template <class TAppender> + void AddAt(const ui32 recordIndex, const TAppender& append) { + AdvanceRecordIndex(recordIndex); AFL_VERIFY(NArrow::Append<arrow::UInt32Type>(*IndexBuilder, recordIndex)); - AFL_VERIFY(NArrow::Append<TDataType>(*ValueBuilder, arrow::util::string_view(value.data(), value.size()))); - ++RecordsCount; + append(*ValueBuilder); + } + + public: + TSparsedBuilderBase(std::unique_ptr<arrow::ArrayBuilder> valueBuilder, const std::shared_ptr<arrow::DataType>& valueType, + const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems) + : IndexBuilder(NArrow::MakeBuilder(arrow::uint32(), reserveItems, 0)) + , ValueBuilder(std::move(valueBuilder)) + , ValueType(valueType) + , DefaultValue(defaultValue) + { + AFL_VERIFY(!!ValueBuilder); } void AddNull(const ui32 recordIndex) { - if (!!LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex); - } - LastRecordIndex = recordIndex; + AdvanceRecordIndex(recordIndex); if (!!DefaultValue && DefaultValue->type->id() != arrow::null()->id()) { AFL_VERIFY(NArrow::Append<arrow::UInt32Type>(*IndexBuilder, recordIndex)); TStatusValidator::Validate(ValueBuilder->AppendNull()); @@ -292,16 +300,27 @@ public: } std::shared_ptr<IChunkedArray> Finish(const ui32 recordsCount) { - TSparsedArray::TBuilder builder(DefaultValue, arrow::TypeTraits<TDataType>::type_singleton()); - std::vector<std::unique_ptr<arrow::ArrayBuilder>> builders; - builders.emplace_back(std::move(IndexBuilder)); - builders.emplace_back(std::move(ValueBuilder)); - builder.AddChunk(recordsCount, arrow::RecordBatch::Make(TSparsedArray::BuildSchema(arrow::TypeTraits<TDataType>::type_singleton()), - RecordsCount, NArrow::Finish(std::move(builders)))); + TBuilder builder(DefaultValue, ValueType); + builder.AddChunk(recordsCount, NArrow::FinishBuilder(std::move(IndexBuilder)), NArrow::FinishBuilder(std::move(ValueBuilder))); return builder.Finish(); } }; + template <class TDataType> + class TSparsedBuilder: public TSparsedBuilderBase { + public: + TSparsedBuilder(const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems, const ui32 reserveData) + : TSparsedBuilderBase(NArrow::MakeBuilder(arrow::TypeTraits<TDataType>::type_singleton(), reserveItems, reserveData), + arrow::TypeTraits<TDataType>::type_singleton(), defaultValue, reserveItems) { + } + + void AddRecord(const ui32 recordIndex, const std::string_view value) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { + AFL_VERIFY(NArrow::Append<TDataType>(builder, arrow::util::string_view(value.data(), value.size()))); + }); + } + }; + static TSparsedBuilder<arrow::BinaryType> MakeBuilderBinary(const ui32 reserveItems = 0, const ui32 reserveData = 0) { return TSparsedBuilder<arrow::BinaryType>(nullptr, reserveItems, reserveData); } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp index 58894ffd10f..2e69ef7d567 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp @@ -1,4 +1,5 @@ #include "accessor.h" +#include "types.h" #include "direct_builder.h" #include "signals.h" @@ -137,7 +138,7 @@ TConclusion<NBinaryJson::TBinaryJson> ToBinaryJson(const TJsonRestorer& restorer [](NBinaryJson::TBinaryJson&& val) -> TConclusion<NBinaryJson::TBinaryJson> { return std::move(val); }}, - NBinaryJson::SerializeToBinaryJson(restorer.GetResult().GetStringRobust())); + NBinaryJson::SerializeToBinaryJson(WriteJsonRoundTripSafe(restorer.GetResult()))); } std::shared_ptr<arrow::Array> TSubColumnsArray::BuildBJsonArray(const TColumnConstructionContext& context) const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h b/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h index 840694c9f04..3ea238c2f79 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h @@ -80,12 +80,12 @@ public: const arrow::Array& GetArray() const { return *CurrentArrayData; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { return ChunkAddress->GetAddress().GetLocalIndex(CurrentIndex); } NArrow::NAccessor::TJsonValueView GetValue() const { - return ArrayElementToJsonValueView(*CurrentArrayData, GetLocalIndex(), ValueType); + return GetCodecForValueType(ValueType)->ReadValueView(*CurrentArrayData, GetLocalIndex()); } bool HasValue() const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp index a67b4ef3867..eea3998e997 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp @@ -1,78 +1,40 @@ #include "accessor.h" #include "columns_storage.h" #include "direct_builder.h" +#include "encoding_builders.h" #include <util/string/escape.h> #include <ydb/core/formats/arrow/accessor/common/chunk_data.h> #include <ydb/core/formats/arrow/accessor/dictionary/constructor.h> #include <ydb/core/formats/arrow/accessor/plain/accessor.h> -#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> #include <ydb/core/formats/arrow/serializer/abstract.h> #include <contrib/libs/simdjson/include/simdjson.h> namespace NKikimr::NArrow::NAccessor::NSubColumns { -namespace { - -std::string_view MakeStoredBytesView(const NBinaryJson::TBinaryJson& rec, const EValueType valueType) { - if (valueType == EValueType::BinaryJson) { - return std::string_view(rec.Data(), rec.Size()); - } - AFL_VERIFY(valueType == EValueType::String)("value_type", (ui32)valueType); - const auto scalar = ExtractStringScalar(rec); - return std::string_view(scalar.data(), scalar.size()); -} - -template <class TArrow, class TExtractor> -std::shared_ptr<IChunkedArray> BuildTypedPlain(const std::deque<NBinaryJson::TBinaryJson>& values, - const std::vector<ui32>& recordIndexes, const ui32 recordsCount, const ui32 reserveData, const TExtractor& extract) { - TTrivialArray::TPlainBuilder<TArrow> builder(recordsCount, reserveData); - for (ui32 i = 0; i < recordIndexes.size(); ++i) { - builder.AddValue(recordIndexes[i], extract(values[i])); - } - return builder.Finish(recordsCount); -} -} // namespace - void TColumnElements::BuildSparsedAccessor(const ui32 recordsCount, const EValueType valueType) { AFL_VERIFY(!Accessor); - // Columns with non-binary encoding are not supported. - AFL_VERIFY(valueType == EValueType::BinaryJson || valueType == EValueType::String)("value_type", (ui32)valueType); - auto recordsBuilder = TSparsedArray::MakeBuilderBinary(RecordIndexes.size(), DataSize); - for (ui32 idx = 0; idx < RecordIndexes.size(); ++idx) { - recordsBuilder.AddRecord(RecordIndexes[idx], MakeStoredBytesView(Values[idx], valueType)); + TEncodingSparsedBuilder builder(GetCodecForValueType(valueType), RecordIndexes.size(), DataSize); + for (ui32 i = 0; i < RecordIndexes.size(); ++i) { + builder.AddFromBinaryJson(RecordIndexes[i], Values[i]); } - Accessor = recordsBuilder.Finish(recordsCount); + Accessor = builder.Finish(recordsCount); } void TColumnElements::BuildPlainAccessor(const ui32 recordsCount, const EValueType valueType) { AFL_VERIFY(!Accessor); - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - Accessor = BuildTypedPlain<arrow::BinaryType>(Values, RecordIndexes, recordsCount, DataSize, - [valueType](const NBinaryJson::TBinaryJson& rec) { - const auto sv = MakeStoredBytesView(rec, valueType); - return arrow::util::string_view(sv.data(), sv.size()); - }); - break; - case EValueType::Double: - // No need to reserve by size for fixed-size arrays - Accessor = BuildTypedPlain<arrow::DoubleType>(Values, RecordIndexes, recordsCount, 0, - &ExtractDoubleScalar); - break; - case EValueType::Bool: - Accessor = BuildTypedPlain<arrow::BooleanType>(Values, RecordIndexes, recordsCount, 0, - &ExtractBoolScalar); - break; + TEncodingPlainBuilder builder(GetCodecForValueType(valueType), recordsCount, DataSize); + for (ui32 i = 0; i < RecordIndexes.size(); ++i) { + builder.AddFromBinaryJson(RecordIndexes[i], Values[i]); } + Accessor = builder.Finish(recordsCount); } void TColumnElements::BuildDictionaryAccessor(const ui32 recordsCount, const EValueType valueType) { BuildPlainAccessor(recordsCount, valueType); - const TChunkConstructionData cData( - recordsCount, nullptr, GetArrowTypeForValueType(valueType), NSerialization::TSerializerContainer::GetDefaultSerializer()); + const TChunkConstructionData cData(recordsCount, nullptr, GetCodecForValueType(valueType)->GetArrowType(), + NSerialization::TSerializerContainer::GetDefaultSerializer()); Accessor = NDictionary::TConstructor().Construct(Accessor, cData).DetachResult(); } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h index 5c2148566b0..59e979499c1 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h @@ -192,10 +192,8 @@ public: if (separateColumns && settings.GetEnableNativeColumnsResolved()) { valueType = DetectValueTypeForArray(i->GetValues()); } - IChunkedArray::EType accessorType = IChunkedArray::EType::Array; - if (settings.IsSparsed(presentCount, recordsCount)) { - accessorType = IChunkedArray::EType::SparsedArray; - } else if (separateColumns && DictionaryApplicableForValueType(valueType) && + auto accessorType = settings.IsSparsed(presentCount, recordsCount) ? IChunkedArray::EType::SparsedArray : IChunkedArray::EType::Array; + if (separateColumns && CanBeDictionaryEncoded(valueType) && settings.IsDictionary(presentCount, enumerateValues)) { accessorType = IChunkedArray::EType::Dictionary; } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h b/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h new file mode 100644 index 00000000000..e1441e91421 --- /dev/null +++ b/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h @@ -0,0 +1,57 @@ +#pragma once + +#include "types.h" + +#include <ydb/core/formats/arrow/accessor/plain/accessor.h> +#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> + +#include <ydb/library/actors/core/log.h> + +#include <yql/essentials/types/binary_json/format.h> + +// Subcolumn builders that either copy an arrow-native value or decode one from BinaryJson. +namespace NKikimr::NArrow::NAccessor::NSubColumns { + +class TEncodingPlainBuilder: public TTrivialArray::TPlainBuilderBase { +private: + std::shared_ptr<const IValueArrowCodec> Codec; + +public: + TEncodingPlainBuilder(const std::shared_ptr<const IValueArrowCodec>& codec, const ui32 reserveItems, const ui32 reserveData) + : TPlainBuilderBase(codec->MakeBuilder(reserveItems, reserveData)) + , Codec(codec) + { + } + + void AddFromBinaryJson(const ui32 recordIndex, const NBinaryJson::TBinaryJson& blob) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { Codec->AppendFromBinaryJson(builder, blob); }); + } + + void AddArrayElement(const ui32 recordIndex, const arrow::Array& array, const ui32 position) { + AFL_VERIFY(GetBuilder().type()->id() == array.type_id())("builder", GetBuilder().type()->ToString())("array", array.type()->ToString()); + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append(builder, array, position)); }); + } +}; + +class TEncodingSparsedBuilder: public TSparsedArray::TSparsedBuilderBase { +private: + std::shared_ptr<const IValueArrowCodec> Codec; + +public: + TEncodingSparsedBuilder(const std::shared_ptr<const IValueArrowCodec>& codec, const ui32 reserveItems, const ui32 reserveData) + : TSparsedBuilderBase(codec->MakeBuilder(reserveItems, reserveData), codec->GetArrowType(), nullptr, reserveItems) + , Codec(codec) + { + } + + void AddFromBinaryJson(const ui32 recordIndex, const NBinaryJson::TBinaryJson& blob) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { Codec->AppendFromBinaryJson(builder, blob); }); + } + + void AddArrayElement(const ui32 recordIndex, const arrow::Array& array, const ui32 position) { + AFL_VERIFY(GetValueBuilder().type()->id() == array.type_id())("builder", GetValueBuilder().type()->ToString())("array", array.type()->ToString()); + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append(builder, array, position)); }); + } +}; + +} // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp b/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp index 27b59bb364c..6d981ce4dc9 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { NJson::TJsonValue TGeneralIterator::GetValue() const { AFL_VERIFY(IsValidFlag); - return ArrayElementToJsonValue(*CurrentArray, LocalIndex, ValueType); + return Codec->ReadValueView(*CurrentArray, LocalIndex).ToJsonValue(); } } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/iterators.h b/ydb/core/formats/arrow/accessor/sub_columns/iterators.h index de5b129de4e..f9f4d516051 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/iterators.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/iterators.h @@ -15,10 +15,10 @@ private: bool IsValidFlag = false; bool HasValueFlag = false; bool IsColumnKeyFlag = false; - EValueType ValueType = EValueType::BinaryJson; - // Current value as (array, local index); the reader interprets it per ValueType. + std::shared_ptr<const IValueArrowCodec> Codec; + // Current physical position as (array, local index) const arrow::Array* CurrentArray = nullptr; - i64 LocalIndex = 0; + ui32 LocalIndex = 0; void InitFromIterator(const TColumnsData::TIterator& iterator) { RecordIndex = iterator.GetCurrentRecordIndex(); @@ -72,12 +72,14 @@ public: TGeneralIterator(TColumnsData::TIterator&& iterator, const EValueType valueType, const std::optional<ui32> remappedKey = {}) : Iterator(iterator) , RemappedKey(remappedKey) - , ValueType(valueType) { + , Codec(GetCodecForValueType(valueType)) { Initialize(); } + TGeneralIterator(TOthersData::TIterator&& iterator, const std::vector<ui32>& remapKeys = {}) : Iterator(iterator) - , RemapKeys(remapKeys) { + , RemapKeys(remapKeys) + , Codec(GetCodecForValueType(EValueType::BinaryJson)) { Initialize(); } bool IsColumnKey() const { @@ -159,28 +161,23 @@ public: // Re-encode the current value to BinaryJson. NBinaryJson::TBinaryJson GetValueAsBinaryJson() const { AFL_VERIFY(IsValidFlag); - return ArrayElementToBinaryJson(*CurrentArray, LocalIndex, ValueType); + return Codec->ReadValueView(*CurrentArray, LocalIndex).ToBinaryJson(); } EValueType GetValueType() const { - return ValueType; + return Codec->GetValueType(); } const arrow::Array& GetArray() const { AFL_VERIFY(IsValidFlag); return *CurrentArray; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { + AFL_VERIFY(IsValidFlag); return LocalIndex; } ui32 GetValueSize() const { AFL_VERIFY(IsValidFlag); - return ArrayElementSize(*CurrentArray, LocalIndex, ValueType); - } - TStringBuf GetStorageView() const { - AFL_VERIFY(IsValidFlag); - AFL_VERIFY(ValueType == EValueType::String || ValueType == EValueType::BinaryJson)("value_type", (ui32)ValueType); - const auto view = static_cast<const arrow::BinaryArray&>(*CurrentArray).GetView(LocalIndex); - return TStringBuf(view.data(), view.size()); + return Codec->GetElementSize(*CurrentArray, LocalIndex); } NJson::TJsonValue GetValue() const; diff --git a/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp b/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp index 166f9b8773f..067f60d14e6 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp @@ -139,6 +139,7 @@ void TJsonPathAccessor::VisitValues(const TValuesVisitor& visitor) const { return; } + const auto codec = GetCodecForValueType(ValueType); ChunkedArrayAccessor->VisitValues([&](std::shared_ptr<arrow::Array> arr) { AFL_VERIFY(arr); for (int64_t i = 0; i < arr->length(); ++i) { @@ -147,7 +148,7 @@ void TJsonPathAccessor::VisitValues(const TValuesVisitor& visitor) const { continue; } - const auto value = ArrayElementToJsonValueView(*arr, i, ValueType); + const auto value = codec->ReadValueView(*arr, i); if (auto scalar = value.GetScalarOptional()) { // A scalar has no sub-structure, so a remaining path cannot resolve against it. visitor(RemainingPathPtr ? std::nullopt : scalar); diff --git a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp index 7c01560f1fc..f6f8fd975e0 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp @@ -121,7 +121,7 @@ public: TDictStats::TBuilder statBuilder; for (auto&& i : UsedKeys) { statBuilder.Add(i.second.GetKeyName(), i.second.GetRecordsCount(), i.second.GetDataSize(), - i.second.GetAccessorType(settings, recordsCount), i.second.GetValueType()); + i.second.GetAccessorType(settings, recordsCount), OthersExplicitBinaryJson); } return statBuilder.Finish(); } @@ -250,7 +250,7 @@ TOthersData TOthersData::BuildEmpty() { TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(const std::string_view path, const ui32 recordsCount) const { auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < Stats.GetColumnsCount(); ++i) { - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("error", insertResult.GetErrorMessage()); } auto accessorResult = jsonPathAccessorTrie->GetAccessor(path); @@ -266,7 +266,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con auto idx = accessor->GetCookie(); if (!idx) { return std::make_shared<TJsonPathAccessor>( - std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, EValueType::BinaryJson); + std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, OthersExplicitBinaryJson); } TColumnFilter filter = TColumnFilter::BuildAllowFilter(); for (TIterator it(Records); it.IsValid(); it.Next()) { @@ -278,7 +278,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con TSparsedArray::TBuilder builder(nullptr, arrow::binary()); auto batch = ToBatch(table); builder.AddChunk(recordsCount, batch->GetColumnByName("record_idx"), batch->GetColumnByName("value")); - return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), EValueType::BinaryJson); + return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), OthersExplicitBinaryJson); } NArrow::NAccessor::TJsonValueView TOthersData::TIterator::GetValue() const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h index 00233ff7555..799ef5f2396 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h @@ -123,7 +123,7 @@ public: AFL_VERIFY(IsValid()); return *Values; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { AFL_VERIFY(IsValid()); return CurrentIndex; } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp index b08040273ae..02e05754011 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp @@ -12,7 +12,7 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 recordsCount) const { +TDictStats TDictStats::SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const { std::map<ui64, std::vector<TRTStats>> bySize; ui64 sumSize = 0; for (ui32 i = 0; i < GetColumnsCount(); ++i) { @@ -20,31 +20,21 @@ TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 sumSize += GetColumnSize(i); } std::vector<TRTStats> columnStats; - std::vector<TRTStats> otherStats; TSettings::TColumnsDistributor distributor = settings.BuildDistributor(sumSize, recordsCount); for (auto it = bySize.rbegin(); it != bySize.rend(); ++it) { for (auto&& i : it->second) { - switch (distributor.TakeAndDetect(it->first, i.GetRecordsCount())) { - case TSettings::TColumnsDistributor::EColumnType::Separated: - columnStats.emplace_back(std::move(i)); - break; - case TSettings::TColumnsDistributor::EColumnType::Other: - otherStats.emplace_back(std::move(i)); - break; + // Keys not taken as separated fall into the Others store, whose stats are built elsewhere. + if (distributor.TakeAndDetect(it->first, i.GetRecordsCount()) == TSettings::TColumnsDistributor::EColumnType::Separated) { + columnStats.emplace_back(std::move(i)); } } } std::sort(columnStats.begin(), columnStats.end()); - std::sort(otherStats.begin(), otherStats.end()); auto columnsBuilder = MakeBuilder(); - auto othersBuilder = MakeBuilder(); for (auto&& i : columnStats) { columnsBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); } - for (auto&& i : otherStats) { - othersBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); - } - return TSplittedColumns(columnsBuilder.Finish(), othersBuilder.Finish()); + return columnsBuilder.Finish(); } TDictStats TDictStats::Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount) { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.h b/ydb/core/formats/arrow/accessor/sub_columns/stats.h index 8f6c7388495..b065ce7290f 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.h @@ -15,8 +15,6 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -class TSplittedColumns; - class TDictStats { private: using TJsonPathAccessorTriePtr = std::shared_ptr<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>; @@ -33,7 +31,7 @@ private: auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < DataNames->length(); ++i) { const auto arrView = DataNames->GetView(i); - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("key_name", TStringBuf(arrView.data(), arrView.size())) ("json_path", ToJsonPath(TStringBuf(arrView.data(), arrView.size()))) ("error", insertResult.GetErrorMessage()); @@ -156,7 +154,9 @@ public: static TDictStats Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount); - TSplittedColumns SplitByVolume(const TSettings& settings, const ui32 recordsCount) const; + // Selects which keys become separated columns; + // the rest fall into the Others store, whose stats are built by the caller. + TDictStats SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const; class TBuilder: TNonCopyable { private: @@ -194,7 +194,7 @@ public: std::shared_ptr<arrow::Field> GetField(const ui32 index) const { AFL_VERIFY(index < DataNames->length()); auto name = DataNames->GetView(index); - return std::make_shared<arrow::Field>(std::string(name.data(), name.size()), GetArrowTypeForValueType(GetValueType(index))); + return std::make_shared<arrow::Field>(std::string(name.data(), name.size()), GetCodecForValueType(GetValueType(index))->GetArrowType()); } TRTStats GetRTStats(const ui32 index) const { @@ -240,25 +240,6 @@ public: return result; } - bool IsSparsed(const ui32 columnIndex, const ui32 recordsCount, const TSettings& settings) const; TDictStats(const std::shared_ptr<arrow::RecordBatch>& original); }; - -class TSplittedColumns { -private: - TDictStats Columns; - TDictStats Others; - -public: - TDictStats ExtractColumns() { - return std::move(Columns); - } - TDictStats ExtractOthers() { - return std::move(Others); - } - TSplittedColumns(TDictStats&& columns, TDictStats&& others) - : Columns(std::move(columns)) - , Others(std::move(others)) { - } -}; } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp index 26f517749ab..6a2e18197b8 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp @@ -3,22 +3,29 @@ #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_binary.h> #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_primitive.h> -#include <library/cpp/json/json_reader.h> -#include <library/cpp/json/json_writer.h> - #include <ydb/library/actors/core/log.h> +#include <ydb/library/formats/arrow/arrow_helpers.h> +#include <ydb/library/formats/arrow/switch/switch_type.h> #include <yql/essentials/types/binary_json/read.h> -#include <yql/essentials/types/binary_json/write.h> namespace NKikimr::NArrow::NAccessor::NSubColumns { namespace { -NBinaryJson::TBinaryJson ToBinaryJson(const NJson::TJsonValue& json) { - auto result = NBinaryJson::SerializeToBinaryJson(NJson::WriteJson(&json, false)); - AFL_VERIFY(std::holds_alternative<NBinaryJson::TBinaryJson>(result)); - return std::get<NBinaryJson::TBinaryJson>(std::move(result)); +TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetString(); +} + +double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetNumber(); +} + +bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetType() == NBinaryJson::EEntryType::BoolTrue; } EValueType ValueTypeForItem(const NBinaryJson::TBinaryJson& blob) { @@ -41,27 +48,104 @@ EValueType ValueTypeForItem(const NBinaryJson::TBinaryJson& blob) { } } -} // namespace +TStringBuf BinaryView(const arrow::Array& array, const i64 index) { + AFL_VERIFY(array.type()->id() == arrow::Type::BINARY); + const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); + return TStringBuf(view.data(), view.size()); +} -std::shared_ptr<arrow::DataType> GetArrowTypeForValueType(const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - return arrow::binary(); - case EValueType::Double: - return arrow::float64(); - case EValueType::Bool: - return arrow::boolean(); +// BinaryJson and String share arrow::binary() storage; they differ only in what the bytes mean. +class TBinaryBackedCodec: public IValueArrowCodec { +public: + std::shared_ptr<arrow::DataType> GetArrowType() const override { + return arrow::binary(); + } + + ui32 GetElementSize(const arrow::Array& array, const i64 index) const override { + return BinaryView(array, index).size(); + } + std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 reserveData) const override { + return NArrow::MakeBuilder(arrow::binary(), reserveItems, reserveData); } +}; + +class TBinaryJsonCodec: public TBinaryBackedCodec { +public: + EValueType GetValueType() const override { + return EValueType::BinaryJson; + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + return TJsonValueView::OfBinaryJson(BinaryView(array, index)); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + AFL_VERIFY(NArrow::Append<arrow::BinaryType>(builder, arrow::util::string_view(blob.data(), blob.size()))); + } +}; + +class TStringCodec: public TBinaryBackedCodec { +public: + EValueType GetValueType() const override { + return EValueType::String; + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + return TJsonValueView::OfString(BinaryView(array, index)); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + const auto scalar = ExtractStringScalar(blob); + AFL_VERIFY(NArrow::Append<arrow::BinaryType>(builder, arrow::util::string_view(scalar.data(), scalar.size()))); + } +}; + +template <class TArrow, EValueType ValueType, auto ExtractScalar, auto MakeView> +class TNativeScalarCodec: public IValueArrowCodec { +private: + using TArray = typename arrow::TypeTraits<TArrow>::ArrayType; + +public: + EValueType GetValueType() const override { + return ValueType; + } + std::shared_ptr<arrow::DataType> GetArrowType() const override { + return arrow::TypeTraits<TArrow>::type_singleton(); + } + ui32 GetElementSize(const arrow::Array& /*array*/, const i64 /*index*/) const override { + return sizeof(typename TArrow::c_type); + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + AFL_VERIFY(array.type()->id() == TArrow::type_id); + return MakeView(static_cast<const TArray&>(array).Value(index)); + } + std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 /*reserveData*/) const override { + return NArrow::MakeBuilder(arrow::TypeTraits<TArrow>::type_singleton(), reserveItems, 0); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + AFL_VERIFY(NArrow::Append<TArrow>(builder, ExtractScalar(blob))); + } +}; + +using TDoubleCodec = TNativeScalarCodec<arrow::DoubleType, EValueType::Double, ExtractDoubleScalar, TJsonValueView::OfNumber>; +using TBoolCodec = TNativeScalarCodec<arrow::BooleanType, EValueType::Bool, ExtractBoolScalar, TJsonValueView::OfBool>; + +} // namespace + +bool CanBeDictionaryEncoded(EValueType valueType) { + return valueType == EValueType::BinaryJson || valueType == EValueType::String; } -bool DictionaryApplicableForValueType(const EValueType valueType) { +std::shared_ptr<const IValueArrowCodec> GetCodecForValueType(const EValueType valueType) { + static const std::shared_ptr<const IValueArrowCodec> binaryJson = std::make_shared<TBinaryJsonCodec>(); + static const std::shared_ptr<const IValueArrowCodec> string = std::make_shared<TStringCodec>(); + static const std::shared_ptr<const IValueArrowCodec> doubleValue = std::make_shared<TDoubleCodec>(); + static const std::shared_ptr<const IValueArrowCodec> boolValue = std::make_shared<TBoolCodec>(); switch (valueType) { case EValueType::BinaryJson: + return binaryJson; case EValueType::String: - return true; - default: - return false; + return string; + case EValueType::Double: + return doubleValue; + case EValueType::Bool: + return boolValue; } } @@ -83,82 +167,4 @@ EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& v return common.value_or(EValueType::BinaryJson); } -TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetString(); -} - -double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetNumber(); -} - -bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetType() == NBinaryJson::EEntryType::BoolTrue; -} - -NJson::TJsonValue ArrayElementToJsonValue(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::String: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return NJson::TJsonValue(TStringBuf(view.data(), view.size())); - } - case EValueType::Double: - return NJson::TJsonValue(static_cast<const arrow::DoubleArray&>(array).Value(index)); - case EValueType::Bool: - return NJson::TJsonValue(static_cast<const arrow::BooleanArray&>(array).Value(index)); - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - const auto text = NBinaryJson::SerializeToJson(TStringBuf(view.data(), view.size())); - NJson::TJsonValue result; - AFL_VERIFY(NJson::ReadJsonTree(text, &result)); - return result; - } - } -} - -NBinaryJson::TBinaryJson ArrayElementToBinaryJson(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return NBinaryJson::TBinaryJson(view.data(), view.size()); - } - case EValueType::String: - case EValueType::Double: - case EValueType::Bool: - return ToBinaryJson(ArrayElementToJsonValue(array, index, valueType)); - } -} - -TJsonValueView ArrayElementToJsonValueView(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::String: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return TJsonValueView::OfString(TStringBuf(view.data(), view.size())); - } - case EValueType::Double: - return TJsonValueView::OfNumber(static_cast<const arrow::DoubleArray&>(array).Value(index)); - case EValueType::Bool: - return TJsonValueView::OfBool(static_cast<const arrow::BooleanArray&>(array).Value(index)); - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return TJsonValueView::OfBinaryJson(TStringBuf(view.data(), view.size())); - } - } -} - -ui32 ArrayElementSize(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - return static_cast<const arrow::BinaryArray&>(array).GetView(index).size(); - case EValueType::Double: - return sizeof(double); - case EValueType::Bool: - // actually only 1 bit in arrow representation, not 1 byte, but let's not overcomplicate things - return 1; - } -} - } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/types.h b/ydb/core/formats/arrow/accessor/sub_columns/types.h index 90be2c2a8c8..2ed2ab6d78b 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/types.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/types.h @@ -5,43 +5,43 @@ #include <deque> #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_base.h> +#include <contrib/libs/apache/arrow/cpp/src/arrow/array/builder_base.h> #include <ydb/core/formats/arrow/accessor/common/json_value_view.h> #include <library/cpp/json/writer/json_value.h> #include <yql/essentials/types/binary_json/format.h> -// Type conversions between BinaryJson, dedicated scalar types and arrow storage types. namespace NKikimr::NArrow::NAccessor::NSubColumns { -std::shared_ptr<arrow::DataType> GetArrowTypeForValueType(const EValueType valueType); +// Conversion between physical arrow storage and logical JsonValue/BinaryJson in-program representation +class IValueArrowCodec { +public: + virtual ~IValueArrowCodec() = default; + virtual EValueType GetValueType() const = 0; + virtual std::shared_ptr<arrow::DataType> GetArrowType() const = 0; -// Dictionary encoding only enabled for the binary-backed types (BinaryJson blobs and raw strings). -// Integral types would trade fixed-size position in array for fixed-size dictionary ref. -// May still be good for compression, but requires further experiments. -bool DictionaryApplicableForValueType(const EValueType valueType); + // Read path - wrap the physical element `index` as a logical value view (a native scalar for + // Double/Bool/String, or a BinaryJson blob for BinaryJson). + // The view aliases `array`, which must outlive it. + virtual TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const = 0; + // Approximate size of element `index` for accounting: variable for binary values, fixed for scalar types. + virtual ui32 GetElementSize(const arrow::Array& array, const i64 index) const = 0; -// Element type to represent result of merging arrays with arg types -EValueType MergeValueTypes(const std::optional<EValueType>& acc, const EValueType next); - -EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& values); + // Write path - initialize builder and write values to it. + // reserveData makes sense only for variable-length types (BinaryJson and string). + virtual std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 reserveData) const = 0; + virtual void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const = 0; +}; -// Convert json blob to its contained scalar. -// Fail on verify if blob is not of specified type. -TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob); -double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob); -bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob); +bool CanBeDictionaryEncoded(EValueType valueType); -// Read element `index` of a materialized native column array (interpreted per valueType) as a JSON -// value (document reconstruction) or a BinaryJson blob. The array's physical arrow type must match valueType. -NJson::TJsonValue ArrayElementToJsonValue(const arrow::Array& array, const i64 index, const EValueType valueType); -NBinaryJson::TBinaryJson ArrayElementToBinaryJson(const arrow::Array& array, const i64 index, const EValueType valueType); +std::shared_ptr<const IValueArrowCodec> GetCodecForValueType(const EValueType valueType); -// Wrap element `index` as a logical value view: a native scalar for Double/Bool/String, or a BinaryJson -// blob for BinaryJson. The returned view aliases `array`, which must outlive it. -TJsonValueView ArrayElementToJsonValueView(const arrow::Array& array, const i64 index, const EValueType valueType); +// Element type to represent result of merging arrays with arg types +EValueType MergeValueTypes(const std::optional<EValueType>& acc, const EValueType next); -ui32 ArrayElementSize(const arrow::Array& array, const i64 index, const EValueType valueType); +EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& values); } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp index 8335760024a..5f83768df24 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp @@ -11,6 +11,7 @@ #include <yql/essentials/types/binary_json/write.h> #include <algorithm> +#include <limits> using NKikimr::NArrow::NAccessor::NSubColumns::NTesting::PrintBinaryJsons; @@ -220,4 +221,25 @@ Y_UNIT_TEST_SUITE(SubColumnsNativeScalars) { UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.n"), "1;2.5;<null>;-3;"); UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.b"), "true;false;<null>;true;"); } + + Y_UNIT_TEST(ReencodeNearMaxDouble) { + const std::vector<std::pair<TString, double>> cases = { + {"1.7976931348623157e308", 1.7976931348623157e308}, + {"-1.7976931348623157e308", -1.7976931348623157e308}, + }; + for (const auto& [lit, expected] : cases) { + const TString doc = TString(TStringBuilder() << "{\"n\":" << lit << "}"); + auto native = BuildSubColumns({doc}, NativeSettings(0)); + UNIT_ASSERT_VALUES_EQUAL_C(CountValueType(native, EValueType::Double), 1, native->DebugJson().GetStringRobust()); + UNIT_ASSERT(native->GetChunkedArray()); // read-back document reconstruction must not abort + const auto assertExact = [&](const std::shared_ptr<TSubColumnsArray>& arr) { + auto it = arr->GetColumnsData().BuildIterator(0); + auto bj = GetCodecForValueType(EValueType::Double)->ReadValueView(it.GetArray(), it.GetLocalIndex()).ToBinaryJson(); + auto reader = NBinaryJson::TBinaryJsonReader::Make(bj); + UNIT_ASSERT_VALUES_EQUAL(reader->GetRootCursor().GetElement(0).GetNumber(), expected); + }; + assertExact(native); + assertExact(SerializeRoundTrip(native, NativeSettings(0))); + } + } }; diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp new file mode 100644 index 00000000000..eb49570fc68 --- /dev/null +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp @@ -0,0 +1,64 @@ +#include <ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/types.h> + +#include <contrib/libs/apache/arrow/cpp/src/arrow/chunked_array.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/types/binary_json/write.h> + +Y_UNIT_TEST_SUITE(SubColumnsSparsedBuilder) { + using namespace NKikimr; + using namespace NKikimr::NArrow; + using namespace NKikimr::NArrow::NAccessor; + using namespace NKikimr::NArrow::NAccessor::NSubColumns; + + NBinaryJson::TBinaryJson ToBinaryJson(const TString& json) { + auto v = NBinaryJson::SerializeToBinaryJson(json); + auto* bj = std::get_if<NBinaryJson::TBinaryJson>(&v); + UNIT_ASSERT(bj); + return *bj; + } + + std::shared_ptr<IChunkedArray> BuildColumn( + const std::vector<std::pair<ui32, TString>>& data, const ui32 recordsCount, const EValueType vt, const bool sparsed) { + TColumnElements col("k"); + for (auto&& [idx, json] : data) { + col.AddData(ToBinaryJson(json), idx); + } + if (sparsed) { + col.BuildSparsedAccessor(recordsCount, vt); + } else { + col.BuildPlainAccessor(recordsCount, vt); + } + return col.GetAccessorVerified(); + } + + void CheckSparsedMatchesPlain( + const std::vector<std::pair<ui32, TString>>& data, const ui32 recordsCount, const EValueType vt) { + auto sparsed = BuildColumn(data, recordsCount, vt, true); + auto plain = BuildColumn(data, recordsCount, vt, false); + UNIT_ASSERT_VALUES_EQUAL((ui32)sparsed->GetType(), (ui32)IChunkedArray::EType::SparsedArray); + UNIT_ASSERT_VALUES_EQUAL((ui32)plain->GetType(), (ui32)IChunkedArray::EType::Array); + UNIT_ASSERT_VALUES_EQUAL(sparsed->GetRecordsCount(), recordsCount); + UNIT_ASSERT_VALUES_EQUAL(plain->GetRecordsCount(), recordsCount); + auto sc = sparsed->GetChunkedArray(); + auto pc = plain->GetChunkedArray(); + UNIT_ASSERT_C(sc->Equals(pc), TStringBuilder() << "sparsed=" << sc->ToString() << " plain=" << pc->ToString()); + } + + Y_UNIT_TEST(StringMatchesPlain) { + CheckSparsedMatchesPlain({ { 1, R"("x")" }, { 4, R"("yy")" }, { 7, R"("zzz")" } }, 10, EValueType::String); + } + + Y_UNIT_TEST(DoubleMatchesPlain) { + CheckSparsedMatchesPlain({ { 0, "3.5" }, { 2, "-2" }, { 9, "1000000" } }, 10, EValueType::Double); + } + + Y_UNIT_TEST(BoolMatchesPlain) { + CheckSparsedMatchesPlain({ { 3, "true" }, { 5, "false" } }, 8, EValueType::Bool); + } + + Y_UNIT_TEST(BinaryJsonMatchesPlain) { + CheckSparsedMatchesPlain({ { 1, R"([1,2])" }, { 2, R"({"a":1})" } }, 5, EValueType::BinaryJson); + } +} diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make b/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make index 45aa769c406..e32403a1a1b 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make @@ -13,6 +13,7 @@ SRCS( ut_sub_columns.cpp ut_native_scalars.cpp ut_dictionary.cpp + ut_sparsed.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h index a6174a04bf2..a42012d4807 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h @@ -13,4 +13,7 @@ enum class EValueType : ui8 { String = 3, }; +// The Others store holds and reads every value as BinaryJson, whatever its logical type. +inline constexpr EValueType OthersExplicitBinaryJson = EValueType::BinaryJson; + } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/fq/libs/actors/logging/log.h b/ydb/core/fq/libs/actors/logging/log.h deleted file mode 100644 index a5e79b85706..00000000000 --- a/ydb/core/fq/libs/actors/logging/log.h +++ /dev/null @@ -1,197 +0,0 @@ -#pragma once -#include <ydb/library/services/services.pb.h> - -#include <ydb/library/actors/core/log.h> - -#define LOG_STREAMS_IMPL(level, component, logRecordStream) \ - LOG_LOG_S(::NActors::TActivationContext::AsActorContext(), ::NActors::NLog:: Y_CAT(PRI_, level), ::NKikimrServices::component, logRecordStream); - -#define LOG_STREAMS_IMPL_AS(actorSystem, level, component, logRecordStream) \ - LOG_LOG_S(actorSystem, ::NActors::NLog:: Y_CAT(PRI_, level), ::NKikimrServices::component, logRecordStream); - -// Component: STREAMS. -#define LOG_STREAMS_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS, logRecordStream) -#define LOG_STREAMS_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS, logRecordStream) -#define LOG_STREAMS_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS, logRecordStream) -#define LOG_STREAMS_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS, logRecordStream) -#define LOG_STREAMS_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS, logRecordStream) -#define LOG_STREAMS_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS, logRecordStream) -#define LOG_STREAMS_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS, logRecordStream) -#define LOG_STREAMS_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS, logRecordStream) -#define LOG_STREAMS_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS, logRecordStream) - -// Component: STREAMS_SERVICE. -#define LOG_STREAMS_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_SERVICE, logRecordStream) - -// Component: STREAMS_STORAGE_SERVICE. -#define LOG_STREAMS_STORAGE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_STORAGE_SERVICE, logRecordStream) - -#define LOG_STREAMS_STORAGE_SERVICE_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, STREAMS_STORAGE_SERVICE, logRecordStream) - -// Component: ROW_DISPATCHER. -#define LOG_ROW_DISPATCHER_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) - -// Component: STREAMS_SCHEDULER_SERVICE. -#define LOG_STREAMS_SCHEDULER_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_SCHEDULER_SERVICE, logRecordStream) - -// Component: STREAMS_RESOURCE_SERVICE. -#define LOG_STREAMS_RESOURCE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_RESOURCE_SERVICE, logRecordStream) - -// Component: STREAMS_CHECKPOINT_COORDINATOR. -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) - -// Component: STREAMS_CONTROL_PLANE_SERVICE. -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) - -// Component: STREAMS_GRAPH_LEADER. -#define LOG_STREAMS_GRAPH_LEADER_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_GRAPH_LEADER, logRecordStream) - -// Component: YQ_CONTROL_PLANE_STORAGE. -#define LOG_YQ_CONTROL_PLANE_STORAGE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) - - -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_EMERG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, EMERG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_ALERT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ALERT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_CRIT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, CRIT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_NOTICE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, NOTICE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_TRACE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, TRACE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) - -// Component: YQ_CONTROL_PLANE_PROXY. -#define LOG_YQ_CONTROL_PLANE_PROXY_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_CONTROL_PLANE_PROXY, logRecordStream) - -// Component: YQ_TEST_CONNECTION. -#define LOG_YQ_TEST_CONNECTION_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_TEST_CONNECTION, logRecordStream) - -#define LOG_YQ_TEST_CONNECTION_AS_EMERG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, EMERG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_ALERT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ALERT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_CRIT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, CRIT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_NOTICE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, NOTICE, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_TRACE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, TRACE, YQ_TEST_CONNECTION, logRecordStream) - -// Component: YQ_AUDIT. -#define LOG_YQ_AUDIT_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_AUDIT, logRecordStream) - -// Component: YQ_HEALTH. -#define LOG_YQ_HEALTH_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_HEALTH, logRecordStream) - -// Component: FQ_CONTROL_PLANE_CONFIG. -#define LOG_FQ_CONTROL_PLANE_CONFIG_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, FQ_CONTROL_PLANE_CONFIG, logRecordStream) diff --git a/ydb/core/fq/libs/actors/logging/ya.make b/ydb/core/fq/libs/actors/logging/ya.make deleted file mode 100644 index 92b7640dc0a..00000000000 --- a/ydb/core/fq/libs/actors/logging/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -LIBRARY() - -SRCS( - log.h -) - -PEERDIR( - ydb/library/actors/core - ydb/core/protos -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/core/fq/libs/actors/nodes_health_check.cpp b/ydb/core/fq/libs/actors/nodes_health_check.cpp index 850ec710369..4f2f0c78734 100644 --- a/ydb/core/fq/libs/actors/nodes_health_check.cpp +++ b/ydb/core/fq/libs/actors/nodes_health_check.cpp @@ -13,10 +13,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <google/protobuf/util/time_util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_NODES_MANAGER namespace NFq { @@ -44,7 +41,7 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TNodesHealthCheckActor::OnUndelivered"); + YDB_LOG_ERROR("TNodesHealthCheckActor::OnUndelivered"); auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); res->Status = Ydb::StatusIds::GENERIC_ERROR; res->Issues.AddIssue("UNDELIVERED"); @@ -60,9 +57,9 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("Failed with", + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); diff --git a/ydb/core/fq/libs/actors/nodes_manager.cpp b/ydb/core/fq/libs/actors/nodes_manager.cpp index 60dc041f93f..87968e38686 100644 --- a/ydb/core/fq/libs/actors/nodes_manager.cpp +++ b/ydb/core/fq/libs/actors/nodes_manager.cpp @@ -22,15 +22,8 @@ #include <random> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_NODES_MANAGER -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_I(stream) \ - LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_T(stream) \ - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) namespace NFq { @@ -78,14 +71,16 @@ public: static constexpr char ActorName[] = "YQ_NODES_MANAGER"; void PassAway() final { - LOG_I("PassAway, InstanceId: " << InstanceId); + YDB_LOG_INFO("PassAway", + {"instanceId", InstanceId}); NActors::IActor::PassAway(); } void Bootstrap() { Become(&TNodesManagerActor::StateFunc); ServiceCounters.Counters->GetCounter("EvBootstrap", true)->Inc(); - LOG_I("Bootstrap, InstanceId: " << InstanceId); + YDB_LOG_INFO("Bootstrap", + {"instanceId", InstanceId}); ResolveSelfAddress(); } @@ -122,7 +117,8 @@ private: error.SetMessage(TStringBuilder{} << "Error choosing scheduler. Invalid settings: " << scheduler << ", error: " << CurrentExceptionMessage()); } } - LOG_D("TEvAllocateWorkersResponse " << response->Record.DebugString()); + YDB_LOG_DEBUG("TEvAllocateWorkersResponse", + {"response", response->Record.DebugString()}); Send(ev->Sender, response.Release()); } @@ -229,7 +225,7 @@ private: nodeFilter.insert(nodeIdProto.begin(), nodeIdProto.end()); } TPeer node = {SelfId().NodeId(), InstanceId + "," + HostName(), 0, 0, 0, DataCenter}; - + if (!Peers.empty()) { for (ui32 i = 0; i < Peers.size(); ++i) { auto nextNode = Peers[SingleNodeScheduler.NodeOrder[NextPeer]]; @@ -290,18 +286,20 @@ private: Address = NAddr::PrintHost(*ev->Get()->Address); NodesHealthCheck(); } else { - LOG_E("TNodesManagerActor::TEvAddressInfo: empty Address"); + YDB_LOG_ERROR("TNodesManagerActor::TEvAddressInfo: empty Address"); ResolveSelfAddress(); } } void Handle(NActors::TEvResolveError::TPtr& ev) { - LOG_E("TNodesManagerActor::TEvResolveError: " << ev->Get()->Explain << ", Host: " << ev->Get()->Host); + YDB_LOG_ERROR("TNodesManagerActor::TEvResolveError", + {"ev", ev->Get()->Explain}, + {"host", ev->Get()->Host}); ResolveSelfAddress(); } void Handle(NFq::TEvNodesManager::TEvGetNodesRequest::TPtr& ev) { - LOG_T("Received TNodesManagerActor::TEvGetNodesRequest"); + YDB_LOG_TRACE("Received TNodesManagerActor::TEvGetNodesRequest"); auto response = MakeHolder<NFq::TEvNodesManager::TEvGetNodesResponse>(); response->NodeIds.reserve(Peers.size()); for (const auto& info : Peers) { @@ -311,7 +309,7 @@ private: } void ResolveSelfAddress() { - LOG_D("TNodesManagerActor::ResolveSelfAddress"); + YDB_LOG_DEBUG("TNodesManagerActor::ResolveSelfAddress"); auto resolve = MakeHolder<NActors::TEvResolveAddress>(); resolve->Address = HostName(); resolve->Port = IcPort; @@ -319,7 +317,7 @@ private: } void NodesHealthCheck() { - LOG_T("TNodesManagerActor::NodesHealthCheck"); + YDB_LOG_TRACE("TNodesManagerActor::NodesHealthCheck"); const TDuration ttl = TDuration::Seconds(5); Schedule(ttl, new NActors::TEvents::TEvWakeup(WU_NodesHealthCheck)); @@ -346,7 +344,7 @@ private: } void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TNodesManagerActor::OnUndelivered"); + YDB_LOG_ERROR("TNodesManagerActor::OnUndelivered"); ServiceCounters.Counters->GetCounter("OnUndelivered", true)->Inc(); } @@ -390,13 +388,15 @@ private: ServiceCounters.Counters->GetCounter("PeerCount", false)->Set(Peers.size()); ServiceCounters.Counters->GetCounter("NodesHealthCheckOk", true)->Inc(); - LOG_T("Send NodeInfo with size: " << nodesInfo->size() << " to DynamicNameserver"); + YDB_LOG_TRACE("Send NodeInfo with to DynamicNameserver", + {"size", nodesInfo->size()}); if (!nodesInfo->empty()) { THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo(nodesInfo)); Send(GetNameserviceActorId(), nameServiceUpdateReq.Release()); } } catch (yexception &e) { - LOG_E(e.what()); + YDB_LOG_ERROR("Exception", + {"exception", e.what()}); ServiceCounters.Counters->GetCounter("NodesHealthCheckFail", true)->Inc(); } } diff --git a/ydb/core/fq/libs/actors/pending_fetcher.cpp b/ydb/core/fq/libs/actors/pending_fetcher.cpp index c665e61b52c..f2a7960ae60 100644 --- a/ydb/core/fq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/fq/libs/actors/pending_fetcher.cpp @@ -64,11 +64,7 @@ #include <util/generic/guid.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_PENDING_FETCHER namespace NFq { @@ -197,7 +193,7 @@ public: static constexpr char ActorName[] = "YQ_PENDING_FETCHER"; void PassAway() final { - LOG_D("Stop Fetcher"); + YDB_LOG_DEBUG("Stop Fetcher"); Send(DatabaseResolver, new NActors::TEvents::TEvPoison()); NActors::IActor::PassAway(); } @@ -219,7 +215,7 @@ public: private: void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TYqlPendingFetcher::OnUndelivered"); + YDB_LOG_ERROR("TYqlPendingFetcher::OnUndelivered"); HasRunningRequest = false; } @@ -251,19 +247,21 @@ private: void Handle(TEvInternalService::TEvGetTaskResponse::TPtr& ev) { HasRunningRequest = false; - LOG_T("Got GetTask response from PrivateApi"); + YDB_LOG_TRACE("Got GetTask response from PrivateApi"); GetTaskCounters.LatencyMs->Collect((TInstant::Now() - StartGetTaskTime).MilliSeconds()); GetTaskCounters.InFly->Dec(); if (!ev->Get()->Status.IsSuccess()) { GetTaskCounters.Error->Inc(); - LOG_E("Error with GetTask: "<< ev->Get()->Status.GetIssues().ToString()); + YDB_LOG_ERROR("Error with", + {"getTask", ev->Get()->Status.GetIssues()}); return; } GetTaskCounters.Ok->Inc(); const auto& res = ev->Get()->Result; - LOG_T("Tasks count: " << res.tasks().size()); + YDB_LOG_TRACE("Tasks", + {"count", res.tasks().size()}); if (!res.tasks().empty()) { ProcessTask(res); HasRunningRequest = true; @@ -309,7 +307,8 @@ private: auto itA = RunActorMap.find(runActorId); if (itA == RunActorMap.end()) { - LOG_W("Unknown RunActor " << runActorId << " destroyed"); + YDB_LOG_WARN("Unknown RunActor destroyed", + {"runActorId", runActorId}); return; } auto queryId = itA->second.QueryId; @@ -326,7 +325,10 @@ private: void GetPendingTask() { FetcherGeneration++; - LOG_T("Request Private::GetTask" << ", Owner: " << GetOwnerId() << ", Host: " << HostName() << ", Tenant: " << TenantName); + YDB_LOG_TRACE("Request Private::GetTask", + {"owner", GetOwnerId()}, + {"host", HostName()}, + {"tenant", TenantName}); Fq::Private::GetTaskRequest request; request.set_owner_id(GetOwnerId()); request.set_host(HostName()); @@ -354,11 +356,11 @@ private: } void RunTask(const Fq::Private::GetTaskResult::Task& task) { - LOG_D("NewTask:" - << " Scope: " << task.scope() - << " Id: " << task.query_id().value() - << " UserId: " << task.user_id() - << " AuthToken: " << NKikimr::MaskTicket(task.user_token())); + YDB_LOG_DEBUG("NewTask", + {"scope", task.scope()}, + {"id", task.query_id().value()}, + {"userId", task.user_id()}, + {"authToken", NKikimr::MaskTicket(task.user_token())}); THashMap<TString, TString> serviceAccounts; for (const auto& identity : task.service_accounts()) { diff --git a/ydb/core/fq/libs/actors/proxy_private.cpp b/ydb/core/fq/libs/actors/proxy_private.cpp index e335244ead9..114c04c6892 100644 --- a/ydb/core/fq/libs/actors/proxy_private.cpp +++ b/ydb/core/fq/libs/actors/proxy_private.cpp @@ -15,9 +15,7 @@ #include <util/generic/guid.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -182,7 +180,7 @@ private: ) void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered"); + YDB_LOG_ERROR("TYqlAnalyticsPrivateProxy::OnUndelivered"); Counters->GetCounter("OnUndelivered", true)->Inc(); } diff --git a/ydb/core/fq/libs/actors/rate_limiter_resources.cpp b/ydb/core/fq/libs/actors/rate_limiter_resources.cpp index a02a4773146..54b4d127111 100644 --- a/ydb/core/fq/libs/actors/rate_limiter_resources.cpp +++ b/ydb/core/fq/libs/actors/rate_limiter_resources.cpp @@ -15,15 +15,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <google/protobuf/util/time_util.h> -#define C_LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateCreateRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define C_LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateCreateRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) - -#define D_LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateDeleteRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define D_LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateDeleteRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -60,9 +52,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - C_LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateCreateRateLimiterResource - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvCreateRateLimiterResourceResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -75,7 +69,10 @@ public: const auto& req = Ev->Record; OperationId = req.query_id().value(); OwnerId = req.owner_id(); - C_LOG_D("Request CP::CreateRateLimiterResource with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateCreateRateLimiterResource - Request CP::CreateRateLimiterResource with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); try { auto event = CreateControlPlaneEvent(); @@ -99,7 +96,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - C_LOG_D("Got CP::CreateRateLimiterResourceResponse"); + YDB_LOG_DEBUG("PrivateCreateRateLimiterResource - Got CP::CreateRateLimiterResourceResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); @@ -157,9 +156,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - D_LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateDeleteRateLimiterResource - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvDeleteRateLimiterResourceResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -172,7 +173,10 @@ public: const auto& req = Ev->Record; OperationId = req.query_id().value(); OwnerId = req.owner_id(); - D_LOG_D("Request CP::DeleteRateLimiterResource with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateDeleteRateLimiterResource - Request CP::DeleteRateLimiterResource with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); try { auto event = CreateControlPlaneEvent(); @@ -196,7 +200,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvDeleteRateLimiterResourceResponse::TPtr& ev) { - D_LOG_D("Got CP::DeleteRateLimiterResourceResponse"); + YDB_LOG_DEBUG("PrivateDeleteRateLimiterResource - Got CP::DeleteRateLimiterResourceResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/result_writer.cpp b/ydb/core/fq/libs/actors/result_writer.cpp index 5b239463742..a253b211301 100644 --- a/ydb/core/fq/libs/actors/result_writer.cpp +++ b/ydb/core/fq/libs/actors/result_writer.cpp @@ -22,10 +22,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/private_client/internal_service.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_RESULT_WRITER namespace NFq { @@ -59,7 +56,8 @@ public: static constexpr char ActorName[] = "YQ_RESULT_WRITER"; void Bootstrap() { - LOG_I("Bootstrap"); + YDB_LOG_INFO("Bootstrap", + {"traceId", TraceId}); Become(&TResultWriter::StateFunc); } @@ -77,14 +75,16 @@ private: void PassAway() { auto duration = (TInstant::Now()-StartTime); - LOG_I("FinishWrite, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024 - << " ResultChunks.size(): " << ResultChunks.size()); + YDB_LOG_INFO("FinishWrite", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}, + {"resultChunks", ResultChunks.size()}); NActors::IActor::PassAway(); } @@ -95,7 +95,9 @@ private: } void SendIssuesAndSetErrorFlag(const TIssues& issues, NYql::NDqProto::StatusIds::StatusCode statusCode = NYql::NDqProto::StatusIds::INTERNAL_ERROR) { - LOG_E("ControlPlane WriteResult Issues: " << issues.ToString()); + YDB_LOG_ERROR("ControlPlane WriteResult", + {"traceId", TraceId}, + {"issues", issues}); Issues.AddIssues(issues); HasError = true; auto req = MakeHolder<TEvDqFailure>(statusCode, Issues); @@ -123,7 +125,8 @@ private: queryResult.SetTruncated(Truncated); queryResult.SetRowsCount(Rows); - LOG_D("Send response to executer"); + YDB_LOG_DEBUG("Send response to executer", + {"traceId", TraceId}); Send(ExecuterId, new TEvQueryResponse(std::move(queryResult))); } @@ -166,13 +169,15 @@ private: auto duration = (TInstant::Now()-StartTime); - LOG_T("ChannelData, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + YDB_LOG_TRACE("ChannelData", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}); } Requests.erase(it); @@ -222,13 +227,15 @@ private: auto duration = (TInstant::Now()-StartTime); - LOG_T("ChannelData Shift, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + YDB_LOG_TRACE("ChannelData Shift", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}); ResultChunks.pop_front(); } @@ -328,7 +335,9 @@ private: try { ProcessData(ev); } catch (...) { - LOG_E(CurrentExceptionMessage()); + YDB_LOG_ERROR("Exception", + {"traceId", TraceId}, + {"currentExceptionMessage", CurrentExceptionMessage()}); auto req = MakeHolder<TEvDqFailure>(NYql::NDqProto::StatusIds::INTERNAL_ERROR, TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR)); Send(ExecuterId, req.Release()); HasError = true; diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp index db33230a4f2..52c67f50fc1 100644 --- a/ydb/core/fq/libs/actors/run_actor.cpp +++ b/ydb/core/fq/libs/actors/run_actor.cpp @@ -82,12 +82,7 @@ #include <util/string/split.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_QE(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << QueryId << " " << stream) -#define LOG_QW(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << QueryId << " " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_RUN_ACTOR namespace NFq { @@ -290,13 +285,16 @@ public: TString Plan2Json(const TString& ysonPlan) { if (!ysonPlan) { - LOG_QW("Can't convert plan from yson to json: plan is empty"); + YDB_LOG_WARN("Can't convert plan from yson to json: plan is empty", + {"queryId", QueryId}); return {}; } try { return NJson2Yson::ConvertYson2Json(ysonPlan); } catch (...) { - LOG_QE("Can't convert plan from yson to json: " << CurrentExceptionMessage()); + YDB_LOG_ERROR("Can't convert plan from yson", + {"queryId", QueryId}, + {"json", CurrentExceptionMessage()}); } return {}; } @@ -373,7 +371,9 @@ public: static constexpr char ActorName[] = "YQ_RUN_ACTOR"; void Bootstrap() { - LOG_D("Start run actor. Compute state: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)); + YDB_LOG_DEBUG("Start run actor. Compute", + {"queryId", Params.QueryId}, + {"state", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}); FillConnections(); @@ -467,11 +467,14 @@ private: ) void FillConnections() { - LOG_D("FillConnections"); + YDB_LOG_DEBUG("FillConnections", + {"queryId", Params.QueryId}); for (const auto& connection : Params.Connections) { if (!connection.content().name()) { - LOG_D("Connection with empty name " << connection.meta().id()); + YDB_LOG_DEBUG("Connection with empty name", + {"queryId", Params.QueryId}, + {"connectionId", connection.meta().id()}); continue; } YqConnections.emplace(connection.meta().id(), connection); @@ -510,7 +513,8 @@ private: void CancelRunningQuery() { if (ReadRulesCreatorId) { - LOG_D("Cancel read rules creation"); + YDB_LOG_DEBUG("Cancel read rules creation", + {"queryId", Params.QueryId}); Send(ReadRulesCreatorId, new NActors::TEvents::TEvPoison()); } @@ -526,7 +530,8 @@ private: } if (ControlId) { - LOG_D("Cancel running query"); + YDB_LOG_DEBUG("Cancel running query", + {"queryId", Params.QueryId}); Send(ControlId, new NDq::TEvDq::TEvAbortExecution(NYql::NDqProto::StatusIds::ABORTED, FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus))); } else { QueryResponseArrived = true; @@ -534,7 +539,8 @@ private: } void PassAway() override { - LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"queryId", Params.QueryId}); Send(FetcherId, new NActors::TEvents::TEvPoisonTaken()); KillChildrenActors(); NActors::TActorBootstrapped<TRunActor>::PassAway(); @@ -569,7 +575,8 @@ private: if (QueryStateUpdateRequest.resources().rate_limiter() == Fq::Private::TaskResources::PREPARE) { if (!RateLimiterResourceCreatorId) { - LOG_D("Start rate limiter resource creator"); + YDB_LOG_DEBUG("Start rate limiter resource creator", + {"queryId", Params.QueryId}); RateLimiterResourceCreatorId = Register(CreateRateLimiterResourceCreator(SelfId(), Params.Owner, Params.QueryId, Params.Scope, Params.TenantName)); } return; @@ -656,7 +663,10 @@ private: } void Fail(const TString& errorMessage) { - LOG_E("Fail for query " << Params.QueryId << ", finishing: " << Finishing << ", details: " << errorMessage); + YDB_LOG_ERROR("Fail for query", + {"queryId", Params.QueryId}, + {"finishing", Finishing}, + {"details", errorMessage}); if (YqConnections.empty()) { Issues.AddIssue("YqConnections array is empty"); @@ -688,7 +698,9 @@ private: void Handle(TEvents::TEvQueryActionResult::TPtr& ev) { Action = ev->Get()->Action; - LOG_D("New query action received: " << FederatedQuery::QueryAction_Name(Action)); + YDB_LOG_DEBUG("New query action", + {"queryId", Params.QueryId}, + {"received", FederatedQuery::QueryAction_Name(Action)}); switch (Action) { case FederatedQuery::ABORT: case FederatedQuery::ABORT_GRACEFULLY: // not fully implemented @@ -756,7 +768,9 @@ private: THashMap<TString, TTopicIndependentConsumers> topicToIndependentConsumers; ui32 graphIndex = 0; for (auto& graphParams : DqGraphParams) { - LOG_D("Graph " << graphIndex); + YDB_LOG_DEBUG("Graph", + {"queryId", Params.QueryId}, + {"graphIndex", graphIndex}); graphIndex++; const TString consumerNamePrefix = graphIndex == 1 ? Params.QueryId : TStringBuilder() << Params.QueryId << '-' << graphIndex; // Simple name in simple case for (NYql::NDqProto::TDqTask& task : *graphParams.MutableTasks()) { @@ -774,7 +788,10 @@ private: srcDesc.SetConsumerName(consumerName); settingsAny.PackFrom(srcDesc); if (isNewConsumer) { - LOG_D("Create consumer \"" << srcDesc.GetConsumerName() << "\" for topic \"" << srcDesc.GetTopicPath() << "\""); + YDB_LOG_DEBUG("Create consumer for topic", + {"queryId", Params.QueryId}, + {"consumerName", srcDesc.GetConsumerName()}, + {"topicPath", srcDesc.GetTopicPath()}); auto& consumer = *QueryStateUpdateRequest.mutable_resources()->add_topic_consumers(); consumer.set_database_id(srcDesc.GetDatabaseId()); consumer.set_database(srcDesc.GetDatabase()); @@ -859,7 +876,10 @@ private: } void Handle(TEvents::TEvForwardPingResponse::TPtr& ev) { - LOG_T("Forward ping response. Success: " << ev->Get()->Success << ". Cookie: " << ev->Cookie); + YDB_LOG_TRACE("Forward ping response", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Success}, + {"cookie", ev->Cookie}); if (!ev->Get()->Success) { // Failed setting new status or lease was lost ResignQuery(NYql::NDqProto::StatusIds::UNAVAILABLE); return; @@ -878,14 +898,18 @@ private: } void HandleFinish(TEvents::TEvForwardPingResponse::TPtr& ev) { - LOG_T("Forward ping response. Success: " << ev->Get()->Success << ". Cookie: " << ev->Cookie); + YDB_LOG_TRACE("Forward ping response", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Success}, + {"cookie", ev->Cookie}); if (!ev->Get()->Success) { // Failed setting new status or lease was lost Fail("Failed to write finalizing status"); return; } if (ev->Cookie == SaveFinalizingStatusCookie) { - LOG_D("Finalizing status is saved"); + YDB_LOG_DEBUG("Finalizing status is saved", + {"queryId", Params.QueryId}); FinalizingStatusIsWritten = true; ContinueFinish(); } @@ -897,7 +921,9 @@ private: return; } if (ev->Get()->Issues) { - LOG_W("Effect Issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN("Effect", + {"queryId", Params.QueryId}, + {"issues", ev->Get()->Issues.ToOneLineString()}); Issues.AddIssues(ev->Get()->Issues); } ++EffectApplicatorFinished; @@ -909,7 +935,9 @@ private: for (const auto& dqGraph : DqGraphParams) { dqTasks += dqGraph.TasksSize(); } - LOG_D("Overall dq tasks: " << dqTasks); + YDB_LOG_DEBUG("Overall dq", + {"queryId", Params.QueryId}, + {"tasks", dqTasks}); if (dqTasks > MaxTasksPerOperation) { return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation; } @@ -926,7 +954,10 @@ private: } void Handle(TEvents::TEvGraphParams::TPtr& ev) { - LOG_D("Graph (" << (ev->Get()->IsEvaluation ? "evaluation" : "execution") << ") with tasks: " << ev->Get()->GraphParams.TasksSize()); + YDB_LOG_DEBUG("Graph with", + {"queryId", Params.QueryId}, + {"kind", (ev->Get()->IsEvaluation ? "evaluation" : "execution")}, + {"tasks", ev->Get()->GraphParams.TasksSize()}); if (Params.Resources.rate_limiter_path()) { const TString rateLimiterResource = GetRateLimiterResourcePath(Params.CloudId, Params.Scope.ParseFolder(), Params.QueryId); @@ -947,7 +978,8 @@ private: } void Handle(TEvCheckpointCoordinator::TEvZeroCheckpointDone::TPtr&) { - LOG_D("Coordinator saved zero checkpoint"); + YDB_LOG_DEBUG("Coordinator saved zero checkpoint", + {"queryId", Params.QueryId}); Y_ABORT_UNLESS(ControlId); SetLoadFromCheckpointMode(); } @@ -970,7 +1002,9 @@ private: GraphKey = "Precompute=" + ToString(it->second.Index); } else { if (ev->Sender != ExecuterId) { - LOG_E("TEvDqStats received from UNKNOWN Actor (TDqExecuter?) " << ev->Sender); + YDB_LOG_ERROR("TEvDqStats received from UNKNOWN Actor (TDqExecuter?)", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } GraphKey = "Graph=" + ToString(DqGraphIndex); @@ -1209,7 +1243,9 @@ private: statistics = NJson2Yson::ConvertYson2Json(out.Str()); return true; } catch (NYson::TYsonException& ex) { - LOG_E(ex.what()); + YDB_LOG_ERROR("Exception", + {"queryId", Params.QueryId}, + {"exception", ex.what()}); return false; } } @@ -1223,16 +1259,21 @@ private: TIssue WrapInternalIssues(const TIssues& issues) { NYql::IssuesToMessage(TruncateIssues(issues), QueryStateUpdateRequest.mutable_internal_issues()); TString referenceId = GetEntityIdAsString(Params.Config.GetCommon().GetIdsPrefix(), EEntityType::UNDEFINED); - LOG_E(referenceId << ": " << issues.ToOneLineString()); + YDB_LOG_ERROR("", + {"queryId", Params.QueryId}, + {"referenceId", referenceId}, + {"issues", issues.ToOneLineString()}); return TIssue("Contact technical support and provide query information and this id: " + referenceId + "_" + Now().ToStringUpToSeconds()); } void SaveQueryResponse(NYql::NDqs::TEvQueryResponse::TPtr& ev) { auto& result = ev->Get()->Record; - LOG_D("Query response " << NYql::NDqProto::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode()) - << ". Result set index: " << DqGraphIndex - << ". Issues count: " << result.IssuesSize() - << ". Rows count: " << result.GetRowsCount()); + YDB_LOG_DEBUG("Query response result set", + {"queryId", Params.QueryId}, + {"statusCode", NYql::NDqProto::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode())}, + {"index", DqGraphIndex}, + {"issuesCount", result.IssuesSize()}, + {"rowsCount", result.GetRowsCount()}); if (ev->Get()->Record.GetStatusCode() == NYql::NDqProto::StatusIds::INTERNAL_ERROR && !Params.Config.GetCommon().GetKeepInternalErrors()) { TIssues issues; @@ -1287,9 +1328,14 @@ private: QueryEvalStatusCode = result.GetStatusCode(); - LOG_D("Query evaluation " << NYql::NDqProto::StatusIds_StatusCode_Name(QueryEvalStatusCode) - << ". " << it->second.Index << " response. Issues count: " << result.IssuesSize() - << ". Rows count: " << result.GetRowsCount() << ", Sample count: " << result.SampleSize() << ", Truncated: " << result.GetTruncated()); + YDB_LOG_DEBUG("Query evaluation response", + {"queryId", Params.QueryId}, + {"queryEvalStatusCode", NYql::NDqProto::StatusIds_StatusCode_Name(QueryEvalStatusCode)}, + {"index", it->second.Index}, + {"issuesCount", result.IssuesSize()}, + {"rowsCount", result.GetRowsCount()}, + {"sampleCount", result.SampleSize()}, + {"truncated", result.GetTruncated()}); TVector<NDq::TDqSerializedBatch> rows; for (const auto& s : result.GetSample()) { @@ -1348,7 +1394,9 @@ private: } if (ev->Sender != ExecuterId) { - LOG_E("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) " << ev->Sender); + YDB_LOG_ERROR("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?)", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } @@ -1356,7 +1404,8 @@ private: auto statusCode = ev->Get()->Record.GetStatusCode(); if (statusCode == NYql::NDqProto::StatusIds::UNSPECIFIED) { - LOG_E("StatusCode == NYql::NDqProto::StatusIds::UNSPECIFIED, it is not expected, the query will be failed."); + YDB_LOG_ERROR("StatusCode == NYql::NDqProto::StatusIds::UNSPECIFIED, it is not expected, the query will be failed", + {"queryId", Params.QueryId}); } if (statusCode != NYql::NDqProto::StatusIds::SUCCESS) { @@ -1375,7 +1424,8 @@ private: // Continue with the next graph QueryStateUpdateRequest.set_dq_graph_index(++DqGraphIndex); RunNextDqGraph(); - LOG_D("Send save query response request to pinger"); + YDB_LOG_DEBUG("Send save query response request to pinger", + {"queryId", Params.QueryId}); Send(Pinger, new TEvents::TEvForwardPingRequest(QueryStateUpdateRequest)); } @@ -1392,11 +1442,14 @@ private: } if (ev->Sender != ExecuterId) { - LOG_E("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) when FINISHED " << ev->Sender); + YDB_LOG_ERROR("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) when FINISHED", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } - LOG_D("TEvQueryResponse received on finish"); + YDB_LOG_DEBUG("TEvQueryResponse received on finish", + {"queryId", Params.QueryId}); QueryResponseArrived = true; SaveQueryResponse(ev); @@ -1404,11 +1457,15 @@ private: } void Handle(TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { - LOG_D("Read rules creation finished. Issues: " << ev->Get()->Issues.Size()); + YDB_LOG_DEBUG("Read rules creation finished", + {"queryId", Params.QueryId}, + {"issues", ev->Get()->Issues.Size()}); ReadRulesCreatorId = {}; if (ev->Get()->Issues) { AddIssueWithSubIssues("Problems with read rules creation", ev->Get()->Issues); - LOG_D(Issues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_Issues.ToOneLineString", + {"queryId", Params.QueryId}, + {"issues", Issues.ToOneLineString()}); Finish(FederatedQuery::QueryMeta::FAILED); } else { QueryStateUpdateRequest.mutable_resources()->set_topic_consumers_state(Fq::Private::TaskResources::READY); @@ -1420,7 +1477,9 @@ private: ReadRulesCreatorId = {}; if (ev->Get()->Issues) { TransientIssues.AddIssues(ev->Get()->Issues); - LOG_D(TransientIssues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_TransientIssues.ToOneLineString", + {"queryId", Params.QueryId}, + {"transientIssues", TransientIssues.ToOneLineString()}); } if (CanRunReadRulesDeletionActor()) { RunReadRulesDeletionActor(); @@ -1476,11 +1535,15 @@ private: } void Handle(NFq::TEvInternalService::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource creation finished. Success: " << ev->Get()->Status.IsSuccess()); + YDB_LOG_DEBUG("Rate limiter resource creation finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}); RateLimiterResourceCreatorId = {}; if (!ev->Get()->Status.IsSuccess()) { AddIssueWithSubIssues("Problems with rate limiter resource creation", NYdb::NAdapters::ToYqlIssues(ev->Get()->Status.GetIssues())); - LOG_D(Issues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_Issues.ToOneLineString", + {"queryId", Params.QueryId}, + {"issues", Issues.ToOneLineString()}); Finish(FederatedQuery::QueryMeta::FAILED); } else { Params.Resources.set_rate_limiter_path(ev->Get()->Result.rate_limiter()); @@ -1491,14 +1554,20 @@ private: } void HandleFinish(NFq::TEvInternalService::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource creation finished. Success: " << ev->Get()->Status.IsSuccess() << ". Issues: " << ev->Get()->Status.GetIssues().ToOneLineString()); + YDB_LOG_DEBUG("Rate limiter resource creation finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}, + {"issues", ev->Get()->Status.GetIssues().ToOneLineString()}); RateLimiterResourceCreatorId = {}; StartRateLimiterResourceDeleterIfCan(); } void HandleFinish(NFq::TEvInternalService::TEvDeleteRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource deletion finished. Success: " << ev->Get()->Status.IsSuccess() << ". Issues: " << ev->Get()->Status.GetIssues().ToOneLineString()); + YDB_LOG_DEBUG("Rate limiter resource deletion finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}, + {"issues", ev->Get()->Status.GetIssues().ToOneLineString()}); RateLimiterResourceDeleterId = {}; RateLimiterResourceWasDeleted = true; @@ -1507,7 +1576,8 @@ private: bool StartRateLimiterResourceDeleterIfCan() { if (!RateLimiterResourceDeleterId && !RateLimiterResourceCreatorId && FinalizingStatusIsWritten && QueryResponseArrived && Params.Config.GetRateLimiter().GetEnabled()) { - LOG_D("Start rate limiter resource deleter"); + YDB_LOG_DEBUG("Start rate limiter resource deleter", + {"queryId", Params.QueryId}); RateLimiterResourceDeleterId = Register(CreateRateLimiterResourceDeleter(SelfId(), Params.Owner, Params.QueryId, Params.Scope, Params.TenantName)); return true; } @@ -1516,7 +1586,8 @@ private: TEvaluationGraphInfo RunEvalDqGraph(NFq::NProto::TGraphParams& dqGraphParams) { - LOG_D("RunEvalDqGraph"); + YDB_LOG_DEBUG("RunEvalDqGraph", + {"queryId", Params.QueryId}); FillGraphMemoryInfo(dqGraphParams); @@ -1544,7 +1615,8 @@ private: dqGraphParams.GetResultType(), empty, false).Release()); } else { - LOG_D("ResultReceiver was NOT CREATED since ResultType is empty"); + YDB_LOG_DEBUG("ResultReceiver was NOT CREATED since ResultType is empty", + {"queryId", Params.QueryId}); info.ResultId = info.ExecuterId; } @@ -1560,7 +1632,11 @@ private: PrepareResultFormatSettings(info.ResultFormatSettings, dqGraphParams, *dqConfiguration); NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); Send(info.ExecuterId, new NYql::NDqs::TEvGraphRequest(request, info.ControlId, info.ResultId)); - LOG_D("Evaluation Executer: " << info.ExecuterId << ", Controller: " << info.ControlId << ", ResultActor: " << info.ResultId); + YDB_LOG_DEBUG("Evaluation", + {"queryId", Params.QueryId}, + {"executer", info.ExecuterId}, + {"controller", info.ControlId}, + {"resultActor", info.ResultId}); return info; } @@ -1598,7 +1674,8 @@ private: PrepareResultFormatSettings(ResultFormatSettings, dqGraphParams, *dqConfiguration); } else { - LOG_D("ResultWriter was NOT CREATED since ResultType is empty"); + YDB_LOG_DEBUG("ResultWriter was NOT CREATED since ResultType is empty", + {"queryId", Params.QueryId}); resultId = ExecuterId; ClearResultFormatSettings(); } @@ -1652,7 +1729,11 @@ private: NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); Send(ExecuterId, new NYql::NDqs::TEvGraphRequest(request, ControlId, resultId)); - LOG_D("Executer: " << ExecuterId << ", Controller: " << ControlId << ", ResultIdActor: " << resultId); + YDB_LOG_DEBUG("Dump queryId, executer, controller, resultIdActor", + {"queryId", Params.QueryId}, + {"executer", ExecuterId}, + {"controller", ControlId}, + {"resultIdActor", resultId}); } void PrepareResultFormatSettings(TMaybe<NCommon::TResultFormatSettings>& resultFormatSettings, NFq::NProto::TGraphParams& dqGraphParams, const TDqConfiguration& dqConfiguration) { @@ -1819,14 +1900,18 @@ private: void WriteFinalizingStatus() { const FederatedQuery::QueryMeta::ComputeStatus finalizingStatus = GetFinalizingStatus(); Params.Status = finalizingStatus; - LOG_D("Write finalizing status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(finalizingStatus)); + YDB_LOG_DEBUG("Write finalizing", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(finalizingStatus)}); Fq::Private::PingTaskRequest request; request.set_status(finalizingStatus); Send(Pinger, new TEvents::TEvForwardPingRequest(request), 0, SaveFinalizingStatusCookie); } void Finish(FederatedQuery::QueryMeta::ComputeStatus status) { - LOG_D("Is about to finish query with status " << FederatedQuery::QueryMeta::ComputeStatus_Name(status)); + YDB_LOG_DEBUG("Is about to finish query with status", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(status)}); Finishing = true; FinalQueryStatus = status; @@ -1846,7 +1931,9 @@ private: } void StartEffectApplicators() { - LOG_D("Apply effects with status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)); + YDB_LOG_DEBUG("Apply effects with", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}); for (const auto& externalEffect : QueryStateUpdateRequest.resources().external_effects()) { auto providerName = externalEffect.GetProviderName(); if (providerName == "S3Sink") { @@ -1872,12 +1959,16 @@ private: if (Params.Status == FederatedQuery::QueryMeta::ABORTING_BY_USER) { for (auto& prefix : S3Prefixes) { - LOG_W("Partial results are possible with prefix: " << prefix); + YDB_LOG_WARN("Partial results are possible with", + {"queryId", Params.QueryId}, + {"prefix", prefix}); Issues.AddIssue(TIssue(TStringBuilder() << "Partial results are possible with prefix: " << prefix)); } } } else { - LOG_E("Unknown effect applicator: " << providerName); + YDB_LOG_ERROR("Unknown effect", + {"queryId", Params.QueryId}, + {"applicator", providerName}); } } } @@ -1913,12 +2004,16 @@ private: void ResignQuery(NYql::NDqProto::StatusIds::StatusCode statusCode) { QueryStateUpdateRequest.set_resign_query(true); QueryStateUpdateRequest.set_status_code(statusCode); - LOG_W("ResignQuery, status " << NYql::NDqProto::StatusIds::StatusCode_Name(statusCode)); + YDB_LOG_WARN("ResignQuery, status", + {"queryId", Params.QueryId}, + {"statusCode", NYql::NDqProto::StatusIds::StatusCode_Name(statusCode)}); SendPingAndPassAway(); } void SendPingAndPassAway() { - LOG_D("SendPingAndPassAway, FinalQueryStatus " << FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus)); + YDB_LOG_DEBUG("SendPingAndPassAway, FinalQueryStatus", + {"queryId", Params.QueryId}, + {"finalQueryStatus", FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus)}); // Run ping. if (QueryStateUpdateRequest.resign_query()) { // Retry state => all issues are not fatal. TransientIssues.AddIssues(Issues); @@ -1965,7 +2060,8 @@ private: //NYql::NLog::YqlLogger().SetComponentLevel(NYql::NLog::EComponent::CoreEval, NYql::NLog::ELevel::TRACE); //NYql::NLog::YqlLogger().SetComponentLevel(NYql::NLog::EComponent::CorePeepHole, NYql::NLog::ELevel::TRACE); - LOG_D("Compiling query ..."); + YDB_LOG_DEBUG("Compiling query", + {"queryId", Params.QueryId}); NYql::TGatewaysConfig gatewaysConfig; SetupYqlCore(*gatewaysConfig.MutableYqlCore()); @@ -2119,7 +2215,8 @@ private: } void Handle(TEvents::TEvAsyncContinue::TPtr& ev) { - LOG_D("Compiling finished"); + YDB_LOG_DEBUG("Compiling finished", + {"queryId", Params.QueryId}); NYql::TProgram::TStatus status = TProgram::TStatus::Error; const auto& f = ev->Get()->Future; @@ -2320,24 +2417,24 @@ private: } void LogReceivedParams() { - LOG_D("Run actors params: { QueryId: " << Params.QueryId - << " CloudId: " << Params.CloudId - << " UserId: " << Params.UserId - << " Owner: " << Params.Owner - << " PreviousQueryRevision: " << Params.PreviousQueryRevision - << " Connections: " << Params.Connections.size() - << " Bindings: " << Params.Bindings.size() - << " AccountIdSignatures: " << Params.AccountIdSignatures.size() - << " QueryType: " << FederatedQuery::QueryContent::QueryType_Name(Params.QueryType) - << " ExecuteMode: " << FederatedQuery::ExecuteMode_Name(Params.ExecuteMode) - << " ResultId: " << Params.ResultId - << " StateLoadMode: " << FederatedQuery::StateLoadMode_Name(Params.StateLoadMode) - << " StreamingDisposition: " << Params.StreamingDisposition - << " Status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status) - << " DqGraphs: " << Params.DqGraphs.size() - << " DqGraphIndex: " << Params.DqGraphIndex - << " Resource.TopicConsumers: " << Params.Resources.topic_consumers().size() - << " }"); + YDB_LOG_DEBUG("Run actors params", + {"queryId", Params.QueryId}, + {"cloudId", Params.CloudId}, + {"userId", Params.UserId}, + {"owner", Params.Owner}, + {"previousQueryRevision", Params.PreviousQueryRevision}, + {"connections", Params.Connections.size()}, + {"bindings", Params.Bindings.size()}, + {"accountIdSignatures", Params.AccountIdSignatures.size()}, + {"queryType", FederatedQuery::QueryContent::QueryType_Name(Params.QueryType)}, + {"executeMode", FederatedQuery::ExecuteMode_Name(Params.ExecuteMode)}, + {"resultId", Params.ResultId}, + {"stateLoadMode", FederatedQuery::StateLoadMode_Name(Params.StateLoadMode)}, + {"streamingDisposition", Params.StreamingDisposition}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}, + {"dqGraphs", Params.DqGraphs.size()}, + {"dqGraphIndex", Params.DqGraphIndex}, + {"topicConsumers", Params.Resources.topic_consumers().size()}); } bool CollectBasic() { diff --git a/ydb/core/fq/libs/actors/task_get.cpp b/ydb/core/fq/libs/actors/task_get.cpp index 6ffbd5066fd..d5393e70778 100644 --- a/ydb/core/fq/libs/actors/task_get.cpp +++ b/ydb/core/fq/libs/actors/task_get.cpp @@ -18,10 +18,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/library/security/util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", Tenant: " << Tenant << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", Tenant: " << Tenant << ", " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -51,7 +48,10 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TGetTaskRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivateGetTask - TGetTaskRequestActor::OnUndelivered", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}); auto response = MakeHolder<TEvents::TEvGetTaskResponse>(); response->Status = Ydb::StatusIds::GENERIC_ERROR; response->Issues.AddIssue("UNDELIVERED"); @@ -67,9 +67,12 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateGetTask - Failed with", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}, + {"code", codeStr}, + {"details", Issues}); auto response = MakeHolder<TEvents::TEvGetTaskResponse>(); response->Status = reqStatus; response->Issues.AddIssues(Issues); @@ -83,14 +86,21 @@ public: OwnerId = request.owner_id(); Host = request.host(); Tenant = request.tenant(); - LOG_D("Request CP::GetTask with size: " << request.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateGetTask - Request CP::GetTask with bytes", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}, + {"size", request.ByteSize()}); RequestedMBytes->Collect(request.ByteSize() / 1024 / 1024); Send(ControlPlaneConfigActorId(), new TEvControlPlaneConfig::TEvGetTenantInfoRequest()); } private: void HandleResponse(NFq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev) { // YQ - LOG_D("Got CP::GetTask Response"); + YDB_LOG_DEBUG("PrivateGetTask - Got CP::GetTask Response", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}); const auto& issues = ev->Get()->Issues; if (issues) { diff --git a/ydb/core/fq/libs/actors/task_ping.cpp b/ydb/core/fq/libs/actors/task_ping.cpp index 0e187a072ab..ea9bd6a433c 100644 --- a/ydb/core/fq/libs/actors/task_ping.cpp +++ b/ydb/core/fq/libs/actors/task_ping.cpp @@ -17,10 +17,7 @@ #include <google/protobuf/util/time_util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -48,7 +45,9 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TTaskPingRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivatePingTask - TTaskPingRequestActor::OnUndelivered", + {"queryId", OperationId}, + {"owner", OwnerId}); auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); res->Status = Ydb::StatusIds::GENERIC_ERROR; res->Issues.AddIssue("UNDELIVERED"); @@ -64,9 +63,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivatePingTask - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -82,7 +83,10 @@ public: TenantName = req.tenant(); Scope = req.scope(); Deadline = NProtoInterop::CastFromProto(req.deadline()); - LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivatePingTask - Request CP::PingTask with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); Send(ControlPlaneConfigActorId(), new TEvControlPlaneConfig::TEvGetTenantInfoRequest()); } @@ -102,7 +106,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev) { - LOG_D("Got CP::PingTaskResponse"); + YDB_LOG_DEBUG("PrivatePingTask - Got CP::PingTaskResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/task_result_write.cpp b/ydb/core/fq/libs/actors/task_result_write.cpp index cf2a2efb70e..78a538048a3 100644 --- a/ydb/core/fq/libs/actors/task_result_write.cpp +++ b/ydb/core/fq/libs/actors/task_result_write.cpp @@ -14,10 +14,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/control_plane_storage/control_plane_storage.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -47,7 +44,9 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TWriteTaskRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivateWriteTask - TWriteTaskRequestActor::OnUndelivered", + {"resultId", ResultId}, + {"requestId", RequestId}); Res->Status = Ydb::StatusIds::GENERIC_ERROR; Res->Issues.AddIssue("UNDELIVERED"); Send(ev->Sender, Res.Release()); @@ -62,9 +61,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateWriteTask - Failed with", + {"resultId", ResultId}, + {"requestId", RequestId}, + {"code", codeStr}, + {"details", Issues}); Res->Status = reqStatus; Res->Issues.AddIssues(Issues); Send(Sender, Res.Release()); @@ -76,7 +77,10 @@ public: auto request = Ev->Record; ResultId = request.result_id().value(); RequestId = request.request_id(); - LOG_D("Request CP::WriteTaskResult with size: " << request.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateWriteTask - Request CP::WriteTaskResult with bytes", + {"resultId", ResultId}, + {"requestId", RequestId}, + {"size", request.ByteSize()}); RequestedMBytes->Collect(request.ByteSize() / 1024 / 1024); Send(NFq::ControlPlaneStorageServiceActorId(), new NFq::TEvControlPlaneStorage::TEvWriteResultDataRequest(std::move(request)), 0, RequestId); @@ -91,7 +95,9 @@ private: ) void HandleResponse(NFq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev) { - LOG_D("Got CP::WriteTaskResult Response"); + YDB_LOG_DEBUG("PrivateWriteTask - Got CP::WriteTaskResult Response", + {"resultId", ResultId}, + {"requestId", RequestId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/ya.make b/ydb/core/fq/libs/actors/ya.make index 644b6907375..a94d516b906 100644 --- a/ydb/core/fq/libs/actors/ya.make +++ b/ydb/core/fq/libs/actors/ya.make @@ -27,7 +27,6 @@ PEERDIR( library/cpp/yson library/cpp/yson/node ydb/core/base - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/checkpointing ydb/core/fq/libs/checkpointing_common ydb/core/fq/libs/common @@ -96,10 +95,6 @@ YQL_LAST_ABI_VERSION() END() -RECURSE( - logging -) - RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make index 6f2c93b83f7..45c744c0035 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make @@ -12,7 +12,6 @@ FORK_SUBTESTS() PEERDIR( library/cpp/retry library/cpp/testing/unittest - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/checkpoint_storage/events ydb/core/testlib ydb/core/testlib/default diff --git a/ydb/core/fq/libs/checkpoint_storage/ya.make b/ydb/core/fq/libs/checkpoint_storage/ya.make index c3357f3237f..e086b27b01d 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ya.make @@ -12,7 +12,6 @@ SRCS( PEERDIR( contrib/libs/fmt ydb/library/actors/core - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/ydb diff --git a/ydb/core/fq/libs/checkpointing/ya.make b/ydb/core/fq/libs/checkpointing/ya.make index e1553e35bab..cb436b3244b 100644 --- a/ydb/core/fq/libs/checkpointing/ya.make +++ b/ydb/core/fq/libs/checkpointing/ya.make @@ -11,7 +11,7 @@ SRCS( PEERDIR( ydb/library/actors/core - ydb/core/fq/libs/actors/logging + ydb/core/fq/libs/config/protos ydb/core/fq/libs/checkpointing_common ydb/core/fq/libs/checkpoint_storage/events ydb/library/yql/dq/actors/compute diff --git a/ydb/core/fq/libs/compute/ydb/control_plane/database_monitoring.cpp b/ydb/core/fq/libs/compute/ydb/control_plane/database_monitoring.cpp index f0077feedee..b128d269e44 100644 --- a/ydb/core/fq/libs/compute/ydb/control_plane/database_monitoring.cpp +++ b/ydb/core/fq/libs/compute/ydb/control_plane/database_monitoring.cpp @@ -1,7 +1,7 @@ #include <ydb/core/fq/libs/compute/ydb/events/events.h> #include <ydb/core/fq/libs/control_plane_storage/util.h> -#include <ydb/core/kqp/workload_service/common/cpu_quota_manager.h> +#include <ydb/services/workload_manager/common/cpu_quota_manager.h> #include <ydb/library/services/services.pb.h> @@ -193,7 +193,7 @@ private: const ui32 PendingQueueSize; const bool Strict; - NKikimr::NKqp::NWorkload::TCpuQuotaManager CpuQuotaManager; + NKikimr::NWorkloadManager::TCpuQuotaManager CpuQuotaManager; TQueue<TEvYdbCompute::TEvCpuQuotaRequest::TPtr> PendingQueue; TInstant StartCpuLoad; diff --git a/ydb/core/fq/libs/compute/ydb/control_plane/ya.make b/ydb/core/fq/libs/compute/ydb/control_plane/ya.make index 1c1019d3247..feb602740df 100644 --- a/ydb/core/fq/libs/compute/ydb/control_plane/ya.make +++ b/ydb/core/fq/libs/compute/ydb/control_plane/ya.make @@ -15,7 +15,7 @@ PEERDIR( ydb/core/fq/libs/compute/ydb/synchronization_service ydb/core/fq/libs/control_plane_storage/proto ydb/core/fq/libs/quota_manager/proto - ydb/core/kqp/workload_service/common + ydb/services/workload_manager/common ydb/core/protos ydb/library/actors/core ydb/library/actors/protos diff --git a/ydb/core/fq/libs/control_plane_proxy/ut/ya.make b/ydb/core/fq/libs/control_plane_proxy/ut/ya.make index 11f42c49525..6c1870d6486 100644 --- a/ydb/core/fq/libs/control_plane_proxy/ut/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/ut/ya.make @@ -5,7 +5,6 @@ SIZE(MEDIUM) PEERDIR( library/cpp/testing/unittest ydb/core/base - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/test_connection ydb/core/fq/libs/quota_manager/ut_helpers diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/logs.h b/ydb/core/fq/libs/control_plane_proxy/utils/logs.h deleted file mode 100644 index e2d2f8432a6..00000000000 --- a/ydb/core/fq/libs/control_plane_proxy/utils/logs.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include <ydb/core/fq/libs/actors/logging/log.h> - -#define CPP_LOG_D(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_DEBUG(s) -#define CPP_LOG_I(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_INFO(s) -#define CPP_LOG_W(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_WARN(s) -#define CPP_LOG_E(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_ERROR(s) -#define CPP_LOG_T(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_TRACE(s) diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/ya.make b/ydb/core/fq/libs/control_plane_proxy/utils/ya.make index c6adf22c468..e95cdc4e053 100644 --- a/ydb/core/fq/libs/control_plane_proxy/utils/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/utils/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/compute/common ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage diff --git a/ydb/core/fq/libs/control_plane_proxy/ya.make b/ydb/core/fq/libs/control_plane_proxy/ya.make index 47713ff69eb..45e1ffb27be 100644 --- a/ydb/core/fq/libs/control_plane_proxy/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/ya.make @@ -9,7 +9,6 @@ PEERDIR( ydb/library/actors/core ydb/core/base ydb/core/fq/libs/actors - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/compute/ydb ydb/core/fq/libs/compute/ydb/control_plane ydb/core/fq/libs/control_plane_config diff --git a/ydb/core/fq/libs/control_plane_storage/ya.make b/ydb/core/fq/libs/control_plane_storage/ya.make index a2af1b7af86..c74ea6448b0 100644 --- a/ydb/core/fq/libs/control_plane_storage/ya.make +++ b/ydb/core/fq/libs/control_plane_storage/ya.make @@ -22,7 +22,6 @@ PEERDIR( library/cpp/protobuf/interop ydb/core/base ydb/core/external_sources - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/common ydb/core/fq/libs/config ydb/core/fq/libs/config/protos diff --git a/ydb/core/fq/libs/row_dispatcher/coordinator.cpp b/ydb/core/fq/libs/row_dispatcher/coordinator.cpp index 42037c096cf..20414235b63 100644 --- a/ydb/core/fq/libs/row_dispatcher/coordinator.cpp +++ b/ydb/core/fq/libs/row_dispatcher/coordinator.cpp @@ -1,6 +1,5 @@ #include "coordinator.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/actors/nodes_manager_events.h> #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/core/fq/libs/ydb/schema.h> @@ -12,11 +11,14 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/interconnect.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/public/sdk/cpp/adapters/issue/issue.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -25,7 +27,7 @@ using namespace NThreading; using NYql::TIssues; namespace { - + //////////////////////////////////////////////////////////////////////////////// struct TCoordinatorMetrics { @@ -116,12 +118,12 @@ class TActorCoordinator : public TActorBootstrapped<TActorCoordinator> { }; struct TRowDispatcherInfo { - TRowDispatcherInfo(bool connected, ENodeState state, bool isLocal) + TRowDispatcherInfo(bool connected, ENodeState state, bool isLocal) : Connected(connected) , State(state) , IsLocal(isLocal) {} bool Connected = false; - ENodeState State; + ENodeState State; bool IsLocal = false; THashSet<TPartitionKey, TPartitionKeyHash> Locations; }; @@ -289,15 +291,19 @@ void TActorCoordinator::Bootstrap() { if (PrintStateEnabled) { Schedule(TDuration::Seconds(PrintStatePeriodSec), new TEvPrivate::TEvPrintState()); } - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped coordinator, id " << SelfId() - << ", NodesManagerId " << NodesManagerId - << ", rebalancing timeout " << RebalancingTimeout); + YDB_LOG_DEBUG("Successfully bootstrapped coordinator, id NodesManagerId rebalancing timeout", + {"logPrefix", LogPrefix}, + {"selfId", SelfId()}, + {"nodesManagerId", NodesManagerId}, + {"rebalancingTimeout", RebalancingTimeout}); auto nodeGroup = Metrics.Counters->GetSubgroup("node", ToString(SelfId().NodeId())); Metrics.IsActive = nodeGroup->GetCounter("IsActive"); } void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, bool isLocal) { - LOG_ROW_DISPATCHER_TRACE("UpdateKnownRowDispatchers " << actorId.ToString()); + YDB_LOG_TRACE("UpdateKnownRowDispatchers", + {"logPrefix", LogPrefix}, + {"actorId", actorId}); auto it = RowDispatchers.find(actorId); if (it != RowDispatchers.end()) { @@ -311,7 +317,10 @@ void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, boo continue; } - LOG_ROW_DISPATCHER_TRACE("Move all Locations from old actor " << oldActorId.ToString() << " to new " << actorId.ToString()); + YDB_LOG_TRACE("Move all Locations from old actor to new", + {"logPrefix", LogPrefix}, + {"oldActorId", oldActorId}, + {"actorId", actorId}); for (auto& key : info.Locations) { PartitionLocations[key] = actorId; } @@ -327,12 +336,15 @@ void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, boo nodeState = ENodeState::Started; } - LOG_ROW_DISPATCHER_TRACE("Add new row dispatcher to map (state " << static_cast<int>(nodeState) << ")"); + YDB_LOG_TRACE("Add new row dispatcher to map (state", + {"logPrefix", LogPrefix}, + {"nodeState", static_cast<int>(nodeState)}); RowDispatchers.emplace(actorId, TRowDispatcherInfo{true, nodeState, isLocal}); UpdateGlobalState(); if (nodeState == ENodeState::Initializing && !RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -354,10 +366,14 @@ void TActorCoordinator::UpdateInterconnectSessions(const NActors::TActorId& inte } void TActorCoordinator::Handle(NActors::TEvents::TEvPing::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvPing received, " << ev->Sender); + YDB_LOG_TRACE("TEvPing received", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); UpdateInterconnectSessions(ev->InterconnectSession); UpdateKnownRowDispatchers(ev->Sender, false); - LOG_ROW_DISPATCHER_TRACE("Send TEvPong to " << ev->Sender); + YDB_LOG_TRACE("Send TEvPong", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Send(ev->Sender, new NActors::TEvents::TEvPong(), IEventHandle::FlagTrackDelivery); } @@ -388,19 +404,25 @@ void TActorCoordinator::PrintInternalState() { auto str = GetInternalState(); auto buf = TStringBuf(str); for (ui64 offset = 0; offset < buf.size(); offset += PrintStateToLogSplitSize) { - LOG_ROW_DISPATCHER_DEBUG(buf.SubString(offset, PrintStateToLogSplitSize)); + YDB_LOG_DEBUG("Dump logPrefix, state", + {"logPrefix", LogPrefix}, + {"state", buf.SubString(offset, PrintStateToLogSplitSize)}); } } void TActorCoordinator::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("EvNodeConnected " << ev->Get()->NodeId); + YDB_LOG_DEBUG("EvNodeConnected", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); // Dont set Connected = true. // Wait TEvPing from row dispatchers. } void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvNodeDisconnected, node id " << ev->Get()->NodeId); - + YDB_LOG_DEBUG("TEvNodeDisconnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); + for (auto& [actorId, info] : RowDispatchers) { if (ev->Get()->NodeId != actorId.NodeId()) { continue; @@ -409,7 +431,8 @@ void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected: info.Connected = false; if (!RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -417,10 +440,14 @@ void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected: } void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvUndelivered, ev: " << ev->Get()->ToString()); + YDB_LOG_DEBUG("TEvUndelivered", + {"logPrefix", LogPrefix}, + {"ev", ev->Get()->ToString()}); if (ev->Sender == NodesManagerId) { - LOG_ROW_DISPATCHER_INFO("TEvUndelivered, from nodes manager, reason: " << ev->Get()->Reason); + YDB_LOG_INFO("TEvUndelivered, from nodes manager", + {"logPrefix", LogPrefix}, + {"reason", ev->Get()->Reason}); NActors::TActivationContext::Schedule(NodesManagerRetryPeriod, new IEventHandle(NodesManagerId, SelfId(), new NFq::TEvNodesManager::TEvGetNodesRequest(), IEventHandle::FlagTrackDelivery)); return; } @@ -432,7 +459,8 @@ void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { info.Connected = false; if (!RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -441,7 +469,10 @@ void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { } void TActorCoordinator::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("New leader " << ev->Get()->CoordinatorActorId << ", SelfId " << SelfId()); + YDB_LOG_DEBUG("New leader SelfId", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", ev->Get()->CoordinatorActorId}, + {"selfId", SelfId()}); Metrics.LeaderChanged->Inc(); bool isActive = (ev->Get()->CoordinatorActorId == SelfId()); @@ -508,14 +539,21 @@ void TActorCoordinator::Handle(NFq::TEvRowDispatcher::TEvCoordinatorRequest::TPt UpdateInterconnectSessions(ev->InterconnectSession); TStringStream str; - LOG_ROW_DISPATCHER_INFO("TEvCoordinatorRequest from " << ev->Sender.ToString() << ", " << source.GetTopicPath() << ", partIds: " << JoinSeq(", ", ev->Get()->Record.GetPartitionIds())); + YDB_LOG_INFO("TEvCoordinatorRequest", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", source.GetTopicPath()}, + {"partIds", JoinSeq(", ", ev->Get()->Record.GetPartitionIds())}); Metrics.IncomingRequests->Inc(); TCoordinatorRequest request = {.Cookie = ev->Cookie, .Record = ev->Get()->Record}; if (ComputeCoordinatorRequest(ev->Sender, request)) { PendingReadActors.erase(ev->Sender); } else { - LOG_ROW_DISPATCHER_INFO("Not all nodes connected, nodes count: " << NodesCount << ", known rd count: " << RowDispatchers.size() << ", add request into pending queue"); + YDB_LOG_INFO("Not all nodes connected, known add request into pending queue", + {"logPrefix", LogPrefix}, + {"nodesCount", NodesCount}, + {"rdCount", RowDispatchers.size()}); // We save only last request from each read actor PendingReadActors[ev->Sender] = request; } @@ -563,7 +601,9 @@ bool TActorCoordinator::ComputeCoordinatorRequest(TActorId readActorId, const TC } } - LOG_ROW_DISPATCHER_DEBUG("Send TEvCoordinatorResult to " << readActorId); + YDB_LOG_DEBUG("Send TEvCoordinatorResult", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); Send(readActorId, response.release(), IEventHandle::FlagTrackDelivery, request.Cookie); return true; } @@ -588,11 +628,15 @@ void TActorCoordinator::Handle(TEvPrivate::TEvPrintState::TPtr&) { void TActorCoordinator::Handle(NKikimr::TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) { if (!ev->Get()->Success) { - LOG_ROW_DISPATCHER_ERROR("Failed to get TEvLookupResult, try later..."); + YDB_LOG_ERROR("Failed to get TEvLookupResult, try later", + {"logPrefix", LogPrefix}); ScheduleNodeInfoRequest(); return; } - LOG_ROW_DISPATCHER_INFO("Updated node info, node count: " << ev->Get()->AssignedNodes.size() << ", AssignedNodes: " << JoinSeq(", ", ev->Get()->AssignedNodes)); + YDB_LOG_INFO("Updated node info, node", + {"logPrefix", LogPrefix}, + {"count", ev->Get()->AssignedNodes.size()}, + {"assignedNodes", JoinSeq(", ", ev->Get()->AssignedNodes)}); NodesCount = ev->Get()->AssignedNodes.size(); UpdateGlobalState(); UpdatePendingReadActors(); @@ -600,30 +644,41 @@ void TActorCoordinator::Handle(NKikimr::TEvTenantNodeEnumerator::TEvLookupResult void TActorCoordinator::Handle(TEvPrivate::TEvListNodes::TPtr&) { if (NodesManagerId) { - LOG_ROW_DISPATCHER_DEBUG("Send TEvGetNodesRequest to NodesManager"); + YDB_LOG_DEBUG("Send TEvGetNodesRequest to NodesManager", + {"logPrefix", LogPrefix}); Send(NodesManagerId, new NFq::TEvNodesManager::TEvGetNodesRequest(), IEventHandle::FlagTrackDelivery); } else { - LOG_ROW_DISPATCHER_DEBUG("Send NodeEnumerationLookup request"); + YDB_LOG_DEBUG("Send NodeEnumerationLookup request", + {"logPrefix", LogPrefix}); Register(NKikimr::CreateTenantNodeEnumerationLookup(SelfId(), Tenant)); } } void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("Rebalancing..."); + YDB_LOG_DEBUG("Rebalancing", + {"logPrefix", LogPrefix}); RebalancingScheduled = false; bool needRebalance = false; TSet<TActorId> toDelete; auto printState = [&](const TString& str){ - LOG_ROW_DISPATCHER_DEBUG(str); + YDB_LOG_DEBUG("Dump logPrefix, str", + {"logPrefix", LogPrefix}, + {"str", str}); for (auto& [actorId, info] : RowDispatchers) { - LOG_ROW_DISPATCHER_DEBUG(" node " << actorId.NodeId() << " (" << actorId << ") state " << (info.State == ENodeState::Initializing ? "Initializing" : "Started") << " connected " << info.Connected << " partitions count " << info.Locations.size()); + YDB_LOG_DEBUG("Node state connected partitions count", + {"logPrefix", LogPrefix}, + {"nodeId", actorId.NodeId()}, + {"actorId", actorId}, + {"state", (info.State == ENodeState::Initializing ? "Initializing" : "Started")}, + {"connected", info.Connected}, + {"locations", info.Locations.size()}); } }; printState("Current state (rebalancing):"); - + for (auto& [actorId, info] : RowDispatchers) { if (info.State == ENodeState::Initializing) { if (info.Connected) { @@ -649,7 +704,9 @@ void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { } for (const auto& readActorId : KnownReadActors) { - LOG_ROW_DISPATCHER_TRACE("Send TEvCoordinatorDistributionReset to " << readActorId); + YDB_LOG_TRACE("Send TEvCoordinatorDistributionReset", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); Send(readActorId, new TEvRowDispatcher::TEvCoordinatorDistributionReset(), IEventHandle::FlagTrackDelivery); } @@ -666,7 +723,8 @@ void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { void TActorCoordinator::Handle(TEvPrivate::TEvStartingTimeout::TPtr&) { if (State != ENodeState::Started) { - LOG_ROW_DISPATCHER_TRACE("Change global state to Started (by timeout)"); + YDB_LOG_TRACE("Change global state to Started (by timeout)", + {"logPrefix", LogPrefix}); State = ENodeState::Started; PrintInternalState(); } @@ -680,7 +738,10 @@ bool TActorCoordinator::IsReadyPartitionDistribution() const { } void TActorCoordinator::SendError(TActorId readActorId, const TCoordinatorRequest& request, const TString& message) { - LOG_ROW_DISPATCHER_WARN("Send TEvCoordinatorResult to " << readActorId << ", issues: " << message); + YDB_LOG_WARN("Send TEvCoordinatorResult", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"issues", message}); auto response = std::make_unique<TEvRowDispatcher::TEvCoordinatorResult>(); NYql::IssuesToMessage(NYql::TIssues{NYql::TIssue{message}}, response->Record.MutableIssues()); Send(readActorId, response.release(), IEventHandle::FlagTrackDelivery, request.Cookie); @@ -692,7 +753,9 @@ void TActorCoordinator::ScheduleNodeInfoRequest() const { void TActorCoordinator::Handle(NFq::TEvNodesManager::TEvGetNodesResponse::TPtr& ev) { NodesCount = ev->Get()->NodeIds.size(); - LOG_ROW_DISPATCHER_INFO("Updated node info, node count: " << NodesCount); + YDB_LOG_INFO("Updated node info, node", + {"logPrefix", LogPrefix}, + {"count", NodesCount}); UpdateGlobalState(); if (!NodesCount) { ScheduleNodeInfoRequest(); @@ -702,7 +765,8 @@ void TActorCoordinator::Handle(NFq::TEvNodesManager::TEvGetNodesResponse::TPtr& void TActorCoordinator::UpdateGlobalState() { if (State != ENodeState::Started && NodesCount && RowDispatchers.size() >= NodesCount) { - LOG_ROW_DISPATCHER_TRACE("Change global state to Started (by nodes count)"); + YDB_LOG_TRACE("Change global state to Started (by nodes count)", + {"logPrefix", LogPrefix}); State = ENodeState::Started; PrintInternalState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp index 17dba9be709..2cc58e6e887 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp @@ -2,7 +2,9 @@ #include "purecalc_filter.h" -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER namespace NFq::NRowDispatcher { @@ -31,7 +33,10 @@ public: {} void ProcessData(const TVector<ui64>& columnIndex, const TVector<ui64>& offsets, const TVector<std::span<NYql::NUdf::TUnboxedValue>>& values, ui64 numberRows) override { - LOG_ROW_DISPATCHER_TRACE("ProcessData for " << RunHandlers_.size() << " clients, number rows: " << numberRows); + YDB_LOG_TRACE("ProcessData for clients, number", + {"logPrefix", LogPrefix}, + {"runHandlers", RunHandlers_.size()}, + {"rows", numberRows}); if (!numberRows) { return; @@ -44,7 +49,9 @@ public: continue; } if (const auto nextOffset = consumer->GetNextMessageOffset(); nextOffset && offsets[numberRows - 1] < *nextOffset) { - LOG_ROW_DISPATCHER_TRACE("Ignore processing for " << consumer->GetClientId() << ", historical offset"); + YDB_LOG_TRACE("Ignore processing for historical offset", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); continue; } @@ -54,11 +61,15 @@ public: } void OnCompileResponse(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { - LOG_ROW_DISPATCHER_TRACE("Got compile response for request with id " << ev->Cookie); + YDB_LOG_TRACE("Got compile response for request with id", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}); auto compileHandlerStatus = RemoveCompileProgram(ev->Cookie); if (compileHandlerStatus.IsFail()) { - LOG_ROW_DISPATCHER_ERROR(compileHandlerStatus.GetError().GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"error", compileHandlerStatus.GetError().GetErrorMessage()}); return; } const auto compileHandler = compileHandlerStatus.DetachResult(); @@ -71,7 +82,9 @@ public: auto runHandlerStatus = AddRunProgram(compileHandler->GetConsumer(), compileHandler->GetProgram()); if (runHandlerStatus.IsFail()) { - LOG_ROW_DISPATCHER_ERROR(runHandlerStatus.GetError().GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"error", runHandlerStatus.GetError().GetErrorMessage()}); return; } @@ -91,7 +104,9 @@ public: } void RemoveProgram(NActors::TActorId clientId) override { - LOG_ROW_DISPATCHER_TRACE("Remove program with client id " << clientId); + YDB_LOG_TRACE("Remove program with client id", + {"logPrefix", LogPrefix}, + {"clientId", clientId}); AbortCompileProgram(clientId); RemoveRunProgram(clientId); @@ -107,20 +122,27 @@ private: return; } - LOG_ROW_DISPATCHER_TRACE("Start program with client id " << consumer->GetClientId()); + YDB_LOG_TRACE("Start program with client id", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); consumer->OnStart(); } TStatus AddProgram(IProcessedDataConsumer::TPtr consumer, IProgramHolder::TPtr programHolder) { - LOG_ROW_DISPATCHER_TRACE("Create program with client id " << consumer->GetClientId()); + YDB_LOG_TRACE("Create program with client id", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); if (!programHolder) { auto runHandlerStatus = AddRunProgram(std::move(consumer), std::move(programHolder)); return runHandlerStatus.IsSuccess() ? TStatus::Success() : runHandlerStatus.GetError(); } - LOG_ROW_DISPATCHER_TRACE("Create purecalc program for query '" << programHolder->GetQuery() << "' (client id: " << consumer->GetClientId() << ")"); + YDB_LOG_TRACE("Create purecalc program for query (client", + {"logPrefix", LogPrefix}, + {"query", programHolder->GetQuery()}, + {"id", consumer->GetClientId()}); const auto cookie = NextCookie_++; auto compileHandlerStatus = AddCompileProgram(std::move(consumer), std::move(programHolder), cookie); @@ -215,12 +237,18 @@ private: if (const auto value = values[index]; !value.empty()) { result.emplace_back(value); } else { - LOG_ROW_DISPATCHER_TRACE("Ignore processing for " << consumer->GetClientId() << ", client got parsing error for column " << columnId); + YDB_LOG_TRACE("Ignore processing for client got parsing error for column", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}, + {"columnId", columnId}); return; } } - LOG_ROW_DISPATCHER_TRACE("Pass " << numberRows << " rows to purecalc filter (client id: " << consumer->GetClientId() << ")"); + YDB_LOG_TRACE("Pass rows to purecalc filter (client", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"id", consumer->GetClientId()}); programRunHandler->ProcessData(result, numberRows); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp index d6567f06cca..9f908654100 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp @@ -2,12 +2,14 @@ #include <fmt/format.h> -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/providers/common/schema/parser/yql_type_parser.h> #include <yql/essentials/public/purecalc/common/interface.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -338,7 +340,9 @@ public: } void Compile() override { - LOG_ROW_DISPATCHER_TRACE("Send compile request with id " << Cookie_); + YDB_LOG_TRACE("Send compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", Cookie_}); auto compileRequest = std::make_unique<TEvRowDispatcher::TEvPurecalcCompileRequest>(std::exchange(ProgramHolder_, nullptr), Consumer_->GetPurecalcSettings()); NActors::TActivationContext::ActorSystem()->Send( @@ -353,7 +357,9 @@ public: } void AbortCompilation() override { - LOG_ROW_DISPATCHER_TRACE("Send abort compile request with id " << Cookie_); + YDB_LOG_TRACE("Send abort compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", Cookie_}); NActors::TActivationContext::ActorSystem()->Send( new NActors::IEventHandle( CompileServiceId_, @@ -367,12 +373,15 @@ public: void OnCompileResponse(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { ProgramHolder_ = ev->Get()->ProgramHolder.Release(); - LOG_ROW_DISPATCHER_TRACE("Program compilation finished"); + YDB_LOG_TRACE("Program compilation finished", + {"logPrefix", LogPrefix}); } void OnCompileError(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { auto status = TStatus::Fail(ev->Get()->Status, std::move(ev->Get()->Issues)); - LOG_ROW_DISPATCHER_ERROR("Program compilation error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Program compilation", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); CompileErrors_->Inc(); Consumer_->OnError(status.AddParentIssue("Failed to compile client program")); } @@ -405,10 +414,15 @@ public: } void ProcessData(const TVector<std::span<NYql::NUdf::TUnboxedValue>>& values, ui64 numberRows) const override { - LOG_ROW_DISPATCHER_TRACE("ProcessData for " << numberRows << " rows"); + YDB_LOG_TRACE("ProcessData for rows", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}); if (!ProgramHolder_) { - LOG_ROW_DISPATCHER_TRACE("Add " << numberRows << " rows to client " << Consumer_->GetClientId() << " without processing"); + YDB_LOG_TRACE("Add rows to client without processing", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"clientId", Consumer_->GetClientId()}); for (ui64 rowId = 0; rowId < numberRows; ++rowId) { NYql::NUdf::TUnboxedValue value = NYql::NUdf::TUnboxedValuePod{rowId}; Consumer_->OnData(&value); @@ -436,7 +450,8 @@ private: const auto& watermarkExpr = consumer->GetWatermarkExpr(); if (!filterExpr && !watermarkExpr) { - LOG_ROW_DISPATCHER_TRACE("No sql was generated"); + YDB_LOG_TRACE("No sql was generated", + {"logPrefix", LogPrefix}); return {}; } @@ -458,7 +473,9 @@ private: "watermark_expr"_a = watermarkExpr ? static_cast<TString>(TStringBuilder() << ", (" << watermarkExpr << ") AS " << WATERMARK_FIELD_NAME) : "" ); - LOG_ROW_DISPATCHER_DEBUG("Generated sql:\n" << result); + YDB_LOG_DEBUG("Generated sql:\n", + {"logPrefix", LogPrefix}, + {"result", result}); return result; } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make index ac9610d29a5..4e4f8687753 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make @@ -8,7 +8,6 @@ SRCS( PEERDIR( contrib/libs/fmt - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common ydb/core/fq/libs/row_dispatcher/purecalc_no_pg_wrapper diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp index c43453b08e2..9a92ca79244 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp @@ -5,14 +5,16 @@ #include <util/generic/queue.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_base.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/common/rope_over_buffer.h> #include <yql/essentials/minikql/computation/mkql_computation_node_pack.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -69,12 +71,16 @@ private: } void OnParsingError(TStatus status) override { - LOG_ROW_DISPATCHER_ERROR("Got parsing error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Got parsing", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); Self.FatalError(status); } void OnParsedData(ui64 numberRows) override { - LOG_ROW_DISPATCHER_TRACE("Got parsed data, number rows: " << numberRows); + YDB_LOG_TRACE("Got parsed data, number", + {"logPrefix", LogPrefix}, + {"rows", numberRows}); Self.ParsedData.assign(ParerSchema.size(), std::span<NYql::NUdf::TUnboxedValue>()); for (size_t i = 0; i < ParerSchema.size(); ++i) { @@ -93,7 +99,10 @@ private: private: void OnColumnError(ui64 columnIndex, TStatus status) { const auto& column = ParerSchema[columnIndex]; - LOG_ROW_DISPATCHER_WARN("Failed to parse column " << column.ToString() << ", " << status.GetErrorMessage()); + YDB_LOG_WARN("Failed to parse column", + {"logPrefix", LogPrefix}, + {"column", column}, + {"error", status.GetErrorMessage()}); const auto columnIt = Self.ColumnsDesc.find(column.Name); if (columnIt == Self.ColumnsDesc.end()) { @@ -171,12 +180,16 @@ private: FinishPacking(); TQueue<TDataBatch> result; result.swap(ClientData); - LOG_ROW_DISPATCHER_TRACE("ExtractClientData, number batches: " << result.size()); + YDB_LOG_TRACE("ExtractClientData, number", + {"logPrefix", LogPrefix}, + {"batches", result.size()}); return result; } void OnClientError(TStatus status) { - LOG_ROW_DISPATCHER_WARN("OnClientError, " << status.GetErrorMessage()); + YDB_LOG_WARN("OnClientError", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); Client->OnClientError(std::move(status)); } @@ -227,7 +240,10 @@ private: if (Watermark < watermark) { Watermark = watermark; } - LOG_ROW_DISPATCHER_TRACE("OnWatermark, offset: " << offset << ", watermark: " << watermark); + YDB_LOG_TRACE("OnWatermark", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"watermark", watermark}); } public: @@ -253,7 +269,10 @@ private: Offset = Self.Offsets->at(rowId); if (const auto nextOffset = Client->GetNextMessageOffset(); nextOffset && Offset < *nextOffset) { - LOG_ROW_DISPATCHER_TRACE("OnData, skip historical offset: " << Offset << ", next message offset: " << *nextOffset); + YDB_LOG_TRACE("OnData, skip next", + {"logPrefix", LogPrefix}, + {"historicalOffset", Offset}, + {"messageOffset", *nextOffset}); return; } @@ -289,7 +308,12 @@ private: return; } - LOG_ROW_DISPATCHER_TRACE("OnBatchFinish, offset: " << Offset << ", number rows: " << numberRows << ", row size: " << rowSize << ", watermark: " << Watermark); + YDB_LOG_TRACE("OnBatchFinish, number row", + {"logPrefix", LogPrefix}, + {"offset", Offset}, + {"rows", numberRows}, + {"size", rowSize}, + {"watermark", Watermark}); Client->AddDataToClient(Offset, numberRows, rowSize, Watermark); @@ -321,7 +345,10 @@ private: void FinishPacking() { if (!DataPacker->IsEmpty() || !Watermark.Empty()) { - LOG_ROW_DISPATCHER_TRACE("FinishPacking, batch size: " << DataPackerSize << ", number rows: " << FilteredOffsets.size()); + YDB_LOG_TRACE("FinishPacking, batch number", + {"logPrefix", LogPrefix}, + {"size", DataPackerSize}, + {"rows", FilteredOffsets.size()}); if (FilteredOffsets.empty()) { FilteredOffsets.push_back(Offset); } @@ -393,7 +420,8 @@ public: RefreshScheduled = false; if (Parser) { - LOG_ROW_DISPATCHER_TRACE("Refresh parser"); + YDB_LOG_TRACE("Refresh parser", + {"logPrefix", LogPrefix}); Parser->Refresh(); ScheduleRefresh(); } @@ -410,13 +438,17 @@ public: } void HandleException(const std::exception& error) { - LOG_ROW_DISPATCHER_ERROR("Got unexpected exception: " << error.what()); + YDB_LOG_ERROR("Got unexpected", + {"logPrefix", LogPrefix}, + {"exception", error.what()}); FatalError(TStatus::Fail(EStatusId::INTERNAL_ERROR, TStringBuilder() << "Format handler error, got unexpected exception: " << error.what())); } public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Send " << messages.size() << " messages to parser"); + YDB_LOG_TRACE("Send messages to parser", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); if (!messages.empty()) { CurrentOffset = messages.back().GetOffset(); @@ -439,11 +471,16 @@ public: } TStatus AddClient(IClientDataConsumer::TPtr client) override { - LOG_ROW_DISPATCHER_DEBUG("Add client with id " << client->GetClientId()); + YDB_LOG_DEBUG("Add client with id", + {"logPrefix", LogPrefix}, + {"clientId", client->GetClientId()}); if (const auto clientOffset = client->GetNextMessageOffset()) { if (Parser && CurrentOffset && *CurrentOffset > *clientOffset) { - LOG_ROW_DISPATCHER_DEBUG("Parser was flushed due to new historical offset " << *clientOffset << " (previous parser offset: " << *CurrentOffset << ")"); + YDB_LOG_DEBUG("Parser was flushed due to new historical offset (previous parser", + {"logPrefix", LogPrefix}, + {"clientOffset", *clientOffset}, + {"offset", *CurrentOffset}); Parser->Refresh(true); } } @@ -476,7 +513,9 @@ public: } void RemoveClient(NActors::TActorId clientId) override { - LOG_ROW_DISPATCHER_DEBUG("Remove client with id " << clientId); + YDB_LOG_DEBUG("Remove client with id", + {"logPrefix", LogPrefix}, + {"clientId", clientId}); if (Filters) { Filters->RemoveProgram(clientId); @@ -558,7 +597,9 @@ private: Parser->Refresh(true); } - LOG_ROW_DISPATCHER_DEBUG("UpdateParser to new schema with size " << parerSchema.size()); + YDB_LOG_DEBUG("UpdateParser to new schema with size", + {"logPrefix", LogPrefix}, + {"schemaSize", parerSchema.size()}); ParserHandler = MakeIntrusive<TParserHandler>(*this, std::move(parerSchema)); if (const ui64 schemaSize = ParserHandler->GetColumns().size()) { @@ -569,13 +610,17 @@ private: } Parser = newParser.DetachResult(); - LOG_ROW_DISPATCHER_DEBUG("Parser was created on schema with " << schemaSize << " columns"); + YDB_LOG_DEBUG("Parser was created on schema with columns", + {"logPrefix", LogPrefix}, + {"schemaSize", schemaSize}); } else { if (auto status = Parser->ChangeConsumer(ParserHandler); status.IsFail()) { return status; } - LOG_ROW_DISPATCHER_DEBUG("Parser was updated on new schema with " << schemaSize << " columns"); + YDB_LOG_DEBUG("Parser was updated on new schema with columns", + {"logPrefix", LogPrefix}, + {"schemaSize", schemaSize}); } ParserSchemaIndex.resize(MaxColumnId, std::numeric_limits<ui64>::max()); @@ -583,7 +628,8 @@ private: ParserSchemaIndex[columnDesc.ColumnId] = i++; } } else { - LOG_ROW_DISPATCHER_INFO("No columns to parse, reset parser"); + YDB_LOG_INFO("No columns to parse, reset parser", + {"logPrefix", LogPrefix}); Parser.Reset(); } @@ -613,7 +659,11 @@ private: } const ui64 lastOffset = Offsets->at(numberRows - 1); - LOG_ROW_DISPATCHER_TRACE("Send " << numberRows << " messages to programs, first offset: " << Offsets->front() << ", last offset: " << lastOffset); + YDB_LOG_TRACE("Send messages to programs", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"firstOffset", Offsets->front()}, + {"lastOffset", lastOffset}); if (Filters) { Filters->ProcessData(ParserSchemaIndex, *Offsets, ParsedData, numberRows); @@ -621,14 +671,19 @@ private: for (const auto& [_, client] : Clients) { if (client->IsStarted()) { - LOG_ROW_DISPATCHER_TRACE("Commit client " << client->GetClient()->GetClientId() << " offset " << lastOffset); + YDB_LOG_TRACE("Commit client offset", + {"logPrefix", LogPrefix}, + {"clientId", client->GetClient()->GetClientId()}, + {"lastOffset", lastOffset}); client->GetClient()->UpdateClientOffset(lastOffset); } } } void FatalError(TStatus status) const { - LOG_ROW_DISPATCHER_ERROR("Got fatal error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Got fatal", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); for (const auto& [_, client] : Clients) { client->OnClientError(status); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp index 888bd4da50b..e6c050ae1ce 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp @@ -7,7 +7,7 @@ #include <library/cpp/containers/absl/flat_hash_map.h> #include <util/string/join.h> -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/dom/json.h> #include <yql/essentials/minikql/mkql_node_cast.h> @@ -17,6 +17,8 @@ #include <sstream> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -66,7 +68,10 @@ struct TJsonParserBuffer { const auto offset = message.GetOffset(); if (Y_UNLIKELY(Offsets && Offsets.back() > offset)) { - LOG_ROW_DISPATCHER_WARN("Got message with offset " << offset << " which is less than previous offset " << Offsets.back()); + YDB_LOG_WARN("Got message with offset which is less than previous offset", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"offsetsBack", Offsets.back()}); } NumberValues++; @@ -827,11 +832,15 @@ public: FillColumnsBuffers(); Buffer.Reserve(Config.BatchSize, MaxNumberRows); - LOG_ROW_DISPATCHER_INFO("JsonParser was created, simdjson active implementation " << simdjson::get_active_implementation()->name() - << " (" << simdjson::get_active_implementation()->description() << ")" - << ", config: error skip mode: " << Config.SkipErrors << ", batch size: " << Config.BatchSize << ", latency limit " << Config.LatencyLimit - << ", buffer cell count: " << Config.BufferCellCount - << ", max number rows: " << MaxNumberRows ); + YDB_LOG_INFO("JsonParser was created, simdjson active implementation config: error skip batch latency limit buffer cell max number", + {"logPrefix", LogPrefix}, + {"name", simdjson::get_active_implementation()->name()}, + {"description", simdjson::get_active_implementation()->description()}, + {"mode", Config.SkipErrors}, + {"size", Config.BatchSize}, + {"latencyLimit", Config.LatencyLimit}, + {"count", Config.BufferCellCount}, + {"rows", MaxNumberRows}); Parser.threaded = false; } @@ -864,7 +873,9 @@ public: public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); + YDB_LOG_TRACE("Add messages to parse", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); Y_ENSURE(!Buffer.Finished, "Cannot parse messages with finished buffer"); for (const auto& message : messages) { @@ -878,7 +889,9 @@ public: if (!Config.LatencyLimit) { ParseBuffer(); } else { - LOG_ROW_DISPATCHER_TRACE("Collecting data to parse, skip parsing, current buffer size: " << Buffer.GetSize()); + YDB_LOG_TRACE("Collecting data to parse, skip parsing, current buffer", + {"logPrefix", LogPrefix}, + {"size", Buffer.GetSize()}); } } } @@ -894,7 +907,9 @@ public: if (force || creationDuration > Config.LatencyLimit) { ParseBuffer(); } else { - LOG_ROW_DISPATCHER_TRACE("Refresh, skip parsing, buffer creation duration: " << creationDuration); + YDB_LOG_TRACE("Refresh, skip parsing, buffer creation", + {"logPrefix", LogPrefix}, + {"duration", creationDuration}); } } @@ -907,7 +922,10 @@ public: MaxNumberRows = CalculateMaxNumberRows(); FillColumnsBuffers(); - LOG_ROW_DISPATCHER_DEBUG("Parser columns count changed from " << Columns.size() << " to " << Consumer->GetColumns().size()); + YDB_LOG_DEBUG("Parser columns count changed", + {"logPrefix", LogPrefix}, + {"columns", Columns.size()}, + {"consumerColumns", Consumer->GetColumns().size()}); return InitColumnsParsers(); } @@ -930,7 +948,10 @@ protected: auto [values, size] = Buffer.Finish(); OutputOffsets.resize(Buffer.Offsets.size()); - LOG_ROW_DISPATCHER_TRACE("Do parsing, first offset: " << Buffer.Offsets.front() << ", values:\n" << values); + YDB_LOG_TRACE("Do parsing, first values:\n", + {"logPrefix", LogPrefix}, + {"offset", Buffer.Offsets.front()}, + {"values", values}); if (Config.SkipErrors) { OutputOffsets = Buffer.Offsets; @@ -942,7 +963,7 @@ protected: if (status == EParsingStatus::Finish) { break; } - size_t inputRowId = state.OutputRowId + state.ErrorsCount; + size_t inputRowId = state.OutputRowId + state.ErrorsCount; if (inputRowId < Buffer.MessageOffsets.size()) { auto nextJsonOffset = Buffer.MessageOffsets[inputRowId]; state.CurrentBufferPtr = values + nextJsonOffset; @@ -990,7 +1011,7 @@ protected: ParsingFailedRowCount = 0; } -private: +private: struct TParsingState { const char* InitialBufferPtr; const char* CurrentBufferPtr; @@ -1016,7 +1037,7 @@ private: } auto currentJsonOffset = Buffer.MessageOffsets[inputRowId]; - auto nextJsonOffset = Buffer.MessageOffsets[inputRowId + 1]; + auto nextJsonOffset = Buffer.MessageOffsets[inputRowId + 1]; ui16 parsedNonOptional = 0; auto len = nextJsonOffset - currentJsonOffset; @@ -1059,7 +1080,11 @@ private: state.Status = status; return EParsingStatus::Finish; } - LOG_ROW_DISPATCHER_DEBUG("Unbatched parser, skipped " << state.ErrorsCount << ", outputRowId " << state.OutputRowId << ", recovering from " << status.GetErrorMessage()); + YDB_LOG_DEBUG("Unbatched parser, skipped outputRowId recovering", + {"logPrefix", LogPrefix}, + {"errorsCount", state.ErrorsCount}, + {"outputRowId", state.OutputRowId}, + {"error", status.GetErrorMessage()}); ClearRowBuffer(state.OutputRowId); if (TryParseOneJson(state)) { state.OutputRowId++; @@ -1071,7 +1096,11 @@ private: }; EParsingStatus ParseRows(TParsingState& state) { - LOG_ROW_DISPATCHER_TRACE("Init parser, skipped " << state.ErrorsCount << ", outputRowId " << state.OutputRowId << " size " << state.Size); + YDB_LOG_TRACE("Init parser, skipped outputRowId size", + {"logPrefix", LogPrefix}, + {"errorsCount", state.ErrorsCount}, + {"outputRowId", state.OutputRowId}, + {"stateSize", state.Size}); /* Batch size must be at least maximum of document size. diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp index 076f947df22..1d1ff36092e 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp @@ -2,11 +2,13 @@ #include "parser_base.h" -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/mkql_node_cast.h> #include <yql/essentials/minikql/mkql_type_ops.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -53,7 +55,9 @@ public: public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); + YDB_LOG_TRACE("Add messages to parse", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); for (const auto& message : messages) { CurrentMessage = message.GetData(); @@ -90,7 +94,10 @@ public: protected: TStatus DoParsing() override { - LOG_ROW_DISPATCHER_TRACE("Do parsing, first offset: " << Offsets.front() << ", value: " << CurrentMessage); + YDB_LOG_TRACE("Do parsing, first", + {"logPrefix", LogPrefix}, + {"offset", Offsets.front()}, + {"value", CurrentMessage}); NYql::NUdf::TUnboxedValue value = LockObject(NKikimr::NMiniKQL::ValueFromString(DataSlot, CurrentMessage)); if (value) { diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make index 0c36d850dfa..0a479fba44b 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make @@ -12,7 +12,6 @@ PEERDIR( library/cpp/containers/absl - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make index c2647897cde..4066bf2c0ec 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common diff --git a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp index dea204c31d8..a0d87d1e369 100644 --- a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp +++ b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp @@ -1,6 +1,5 @@ #include "leader_election.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/events/events.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/ydb/schema.h> @@ -8,6 +7,7 @@ #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/library/logger/actor.h> @@ -15,6 +15,8 @@ #include <memory> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -245,9 +247,13 @@ void TLeaderElection::Bootstrap() { ActorSystemPtr->store(TActivationContext::ActorSystem(), std::memory_order_release); LogPrefix = "TLeaderElection " + SelfId().ToString() + " "; - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped, local coordinator id " << CoordinatorId.ToString() - << ", tenant id " << TenantId << ", local mode " << Config.GetLocalMode() << ", coordination node path " << CoordinationNodePath - << ", endpoint " << Config.GetDatabase().GetEndpoint()); + YDB_LOG_DEBUG("Successfully bootstrapped, local coordinator id tenant id local mode coordination node path endpoint", + {"logPrefix", LogPrefix}, + {"coordinatorId", CoordinatorId}, + {"tenantId", TenantId}, + {"localMode", Config.GetLocalMode()}, + {"coordinationNodePath", CoordinationNodePath}, + {"endpoint", Config.GetDatabase().GetEndpoint()}); if (Config.GetLocalMode()) { TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(CoordinatorId, 0)); return; @@ -310,16 +316,18 @@ void TLeaderElection::ResetState() { void TLeaderElection::CreateSemaphore() { Session->CreateSemaphore(SemaphoreName, 1 /* limit */) .Subscribe( + [actorId = this->SelfId(), actorSystemPtr = ActorSystemPtr](const NYdb::NCoordination::TAsyncResult<void>& future) { SendToActorSystem(actorSystemPtr, actorId, new TEvPrivate::TEvCreateSemaphoreResult(future)); - }); + }); } void TLeaderElection::AcquireSemaphore() { if (PendingAcquire) { return; } - LOG_ROW_DISPATCHER_DEBUG("Try to acquire semaphore"); + YDB_LOG_DEBUG("Try to acquire semaphore", + {"logPrefix", LogPrefix}); NActorsProto::TActorId protoId; ActorIdToProto(CoordinatorId, &protoId); @@ -338,11 +346,12 @@ void TLeaderElection::AcquireSemaphore() { } void TLeaderElection::StartSession() { - LOG_ROW_DISPATCHER_DEBUG("Start session"); + YDB_LOG_DEBUG("Start session", + {"logPrefix", LogPrefix}); YdbConnection->CoordinationClient .StartSession( - CoordinationNodePath, + CoordinationNodePath, NYdb::NCoordination::TSessionSettings() .Timeout(CoordinationSessionTimeout) .OnStopped([actorId = this->SelfId(), actorSystemPtr = ActorSystemPtr]() { @@ -355,12 +364,15 @@ void TLeaderElection::StartSession() { void TLeaderElection::Handle(NFq::TEvents::TEvSchemaCreated::TPtr& ev) { if (!IsTableCreated(ev->Get()->Result)) { - LOG_ROW_DISPATCHER_ERROR("Schema creation error " << ev->Get()->Result.GetIssues()); + YDB_LOG_ERROR("Schema creation error", + {"logPrefix", LogPrefix}, + {"issues", ev->Get()->Result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Coordination node successfully created"); + YDB_LOG_DEBUG("Coordination node successfully created", + {"logPrefix", LogPrefix}); CoordinationNodeCreated = true; ProcessState(); } @@ -368,26 +380,32 @@ void TLeaderElection::Handle(NFq::TEvents::TEvSchemaCreated::TPtr& ev) { void TLeaderElection::Handle(TEvPrivate::TEvCreateSessionResult::TPtr& ev) { auto result = ev->Get()->Result.GetValue(); if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("CreateSession failed, " << result.GetIssues()); + YDB_LOG_ERROR("CreateSession failed", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } Session = result.GetResult(); - LOG_ROW_DISPATCHER_DEBUG("Session successfully created"); + YDB_LOG_DEBUG("Session successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } void TLeaderElection::Handle(TEvPrivate::TEvCreateSemaphoreResult::TPtr& ev) { auto result = ev->Get()->Result.GetValue(); if (!IsTableCreated(result)) { - LOG_ROW_DISPATCHER_ERROR("Semaphore creating error " << result.GetIssues()); + YDB_LOG_ERROR("Semaphore creating error", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } SemaphoreCreated = true; - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully created"); + YDB_LOG_DEBUG("Semaphore successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } @@ -396,16 +414,20 @@ void TLeaderElection::Handle(TEvPrivate::TEvAcquireSemaphoreResult::TPtr& ev) { PendingAcquire = false; if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Failed to acquire semaphore, " << result.GetIssues()); + YDB_LOG_ERROR("Failed to acquire semaphore", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully acquired"); + YDB_LOG_DEBUG("Semaphore successfully acquired", + {"logPrefix", LogPrefix}); } void TLeaderElection::PassAway() { - LOG_ROW_DISPATCHER_DEBUG("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"logPrefix", LogPrefix}); StopYdb(); TActorBootstrapped::PassAway(); } @@ -423,7 +445,8 @@ void TLeaderElection::StopYdb() { } void TLeaderElection::Handle(TEvPrivate::TEvSessionStopped::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("TEvSessionStopped"); + YDB_LOG_DEBUG("TEvSessionStopped", + {"logPrefix", LogPrefix}); Session.Clear(); PendingAcquire = false; PendingDescribe = false; @@ -440,15 +463,17 @@ void TLeaderElection::SetTimeout() { void TLeaderElection::Handle(TEvPrivate::TEvRestart::TPtr&) { RestartScheduled = false; - LOG_ROW_DISPATCHER_DEBUG("TEvRestart"); - ProcessState(); + YDB_LOG_DEBUG("TEvRestart", + {"logPrefix", LogPrefix}); + ProcessState(); } void TLeaderElection::DescribeSemaphore() { if (PendingDescribe) { return; } - LOG_ROW_DISPATCHER_DEBUG("Describe semaphore"); + YDB_LOG_DEBUG("Describe semaphore", + {"logPrefix", LogPrefix}); PendingDescribe = true; Session->DescribeSemaphore( SemaphoreName, @@ -466,7 +491,8 @@ void TLeaderElection::DescribeSemaphore() { } void TLeaderElection::Handle(TEvPrivate::TEvOnChangedResult::TPtr& /*ev*/) { - LOG_ROW_DISPATCHER_DEBUG("Semaphore changed"); + YDB_LOG_DEBUG("Semaphore changed", + {"logPrefix", LogPrefix}); PendingDescribe = false; ProcessState(); } @@ -475,7 +501,9 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { PendingDescribe = false; auto result = ev->Get()->Result.GetValue(); if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Semaphore describe fail, " << result.GetIssues()); + YDB_LOG_ERROR("Semaphore describe fail", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; @@ -484,7 +512,8 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { const NYdb::NCoordination::TSemaphoreDescription& description = result.GetResult(); Y_ABORT_UNLESS(description.GetOwners().size() <= 1, "To many owners"); if (description.GetOwners().empty()) { - LOG_ROW_DISPATCHER_DEBUG("Empty owners"); + YDB_LOG_DEBUG("Empty owners", + {"logPrefix", LogPrefix}); // Wait OnChanged. return; } @@ -497,9 +526,16 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { } NActors::TActorId id = ActorIdFromProto(protoId); - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully described: coordinator id " << id << " generation " << generation); + YDB_LOG_DEBUG("Semaphore successfully described: coordinator id generation", + {"logPrefix", LogPrefix}, + {"id", id}, + {"generation", generation}); if (!LeaderActorId || (*LeaderActorId != id)) { - LOG_ROW_DISPATCHER_INFO("Send TEvCoordinatorChanged to " << ParentId << ", new coordinator id " << id << ", previous coordinator id " << LeaderActorId.GetOrElse(TActorId())); + YDB_LOG_INFO("Send TEvCoordinatorChanged to new coordinator id previous coordinator id", + {"logPrefix", LogPrefix}, + {"parentId", ParentId}, + {"id", id}, + {"leaderActorId", LeaderActorId.GetOrElse(TActorId())}); TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(id, generation)); Metrics.LeaderChanged->Inc(); } @@ -507,7 +543,9 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { } void TLeaderElection::HandleException(const std::exception& e) { - LOG_ROW_DISPATCHER_ERROR("Internal error: exception:" << e.what()); + YDB_LOG_ERROR("Internal error", + {"logPrefix", LogPrefix}, + {"exception", e.what()}); Metrics.Errors->Inc(); ResetState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp b/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp index 432b2b5508c..08ce60a9473 100644 --- a/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp +++ b/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp @@ -1,7 +1,6 @@ #include "local_leader_election.h" #include <memory> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/ydb/util.h> @@ -11,9 +10,12 @@ #include <ydb/core/grpc_services/service_coordination.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/library/logger/actor.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -225,8 +227,10 @@ void TLocalLeaderElection::Bootstrap() { CoordinationNodePath = JoinPath(NKikimr::AppData()->TenantName, DefaultCoordinationNodePath); LogPrefix = "TLeaderElection " + SelfId().ToString() + " "; - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped, local coordinator id " << CoordinatorId.ToString() - << ", coordination node path " << CoordinationNodePath); + YDB_LOG_DEBUG("Successfully bootstrapped, local coordinator id coordination node path", + {"logPrefix", LogPrefix}, + {"coordinatorId", CoordinatorId}, + {"coordinationNodePath", CoordinationNodePath}); ProcessState(); } @@ -274,7 +278,7 @@ void TLocalLeaderElection::ResetState() { State = EState::Init; SetTimeout(); SessionId = 0; - while (PendingRpcResponses) { + while (PendingRpcResponses) { SendSessionEventFail(); } PendingRpcResponses = 0; @@ -288,7 +292,8 @@ void TLocalLeaderElection::ResetState() { } void TLocalLeaderElection::CreateSemaphore() { - LOG_ROW_DISPATCHER_DEBUG("Try to create semaphore"); + YDB_LOG_DEBUG("Try to create semaphore", + {"logPrefix", LogPrefix}); TRpcIn message; auto& inner = *message.mutable_create_semaphore(); uint64_t reqId = NextReqId++; @@ -308,7 +313,8 @@ void TLocalLeaderElection::AcquireSemaphore() { return; } LastAcquireSemaphore = now; - LOG_ROW_DISPATCHER_DEBUG("Try to acquire semaphore"); + YDB_LOG_DEBUG("Try to acquire semaphore", + {"logPrefix", LogPrefix}); NActorsProto::TActorId protoId; ActorIdToProto(CoordinatorId, &protoId); @@ -330,7 +336,8 @@ void TLocalLeaderElection::AcquireSemaphore() { } void TLocalLeaderElection::StartSession() { - LOG_ROW_DISPATCHER_DEBUG("Start session"); + YDB_LOG_DEBUG("Start session", + {"logPrefix", LogPrefix}); auto token = NACLib::TUserToken(BUILTIN_ACL_METADATA, {}).SerializeAsString(); auto ctx = MakeIntrusive<TLocalRpcCtx>(ActorContext().ActorSystem(), SelfId(), TLocalRpcCtx::TSettings{ .Database = NKikimr::AppData()->TenantName, @@ -339,7 +346,7 @@ void TLocalLeaderElection::StartSession() { .RequestType = std::nullopt, .RpcMethodName = "CoordinationService.Session", }); - + auto ev = std::make_unique<NKikimr::NGRpcService::TEvCoordinationSessionRequest>(std::move(ctx), NKikimr::NGRpcService::TRequestAuxSettings{}); if (token) { ev->SetInternalToken(MakeIntrusive<NACLib::TUserToken>(token)); @@ -348,7 +355,8 @@ void TLocalLeaderElection::StartSession() { } void TLocalLeaderElection::PassAway() { - LOG_ROW_DISPATCHER_DEBUG("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"logPrefix", LogPrefix}); if (RpcActor) { CloseSession(NYdb::EStatus::ABORTED, {NYql::TIssue("Actor is shutting down")}); } @@ -356,7 +364,8 @@ void TLocalLeaderElection::PassAway() { } void TLocalLeaderElection::Handle(TEvPrivate::TEvSelfPing::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("TEvSelfPing"); + YDB_LOG_DEBUG("TEvSelfPing", + {"logPrefix", LogPrefix}); if (SessionClosed) { return; } @@ -391,15 +400,17 @@ void TLocalLeaderElection::SetTimeout() { void TLocalLeaderElection::Handle(TEvPrivate::TEvRestart::TPtr&) { RestartScheduled = false; - LOG_ROW_DISPATCHER_DEBUG("TEvRestart"); - ProcessState(); + YDB_LOG_DEBUG("TEvRestart", + {"logPrefix", LogPrefix}); + ProcessState(); } void TLocalLeaderElection::DescribeSemaphore() { if (PendingDescribe || PendingDescribeChanged) { return; } - LOG_ROW_DISPATCHER_DEBUG("Describe semaphore"); + YDB_LOG_DEBUG("Describe semaphore", + {"logPrefix", LogPrefix}); PendingDescribe = true; PendingDescribeChanged = true; @@ -416,7 +427,9 @@ void TLocalLeaderElection::DescribeSemaphore() { } bool TLocalLeaderElection::OnUnhandledException(const std::exception& e) { - LOG_ROW_DISPATCHER_ERROR("Internal error: exception:" << e.what()); + YDB_LOG_ERROR("Internal error", + {"logPrefix", LogPrefix}, + {"exception", e.what()}); Metrics.Errors->Inc(); ResetState(); return true; @@ -440,12 +453,15 @@ void TLocalLeaderElection::CreateNode(const std::string& path, const NYdb::NCoo void TLocalLeaderElection::Handle(TEvPrivate::TEvCreateNodeResult::TPtr& ev) { const auto& status = ev->Get()->Status; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Coordination node creation error: " << status.GetIssues()); + YDB_LOG_ERROR("Coordination node creation", + {"logPrefix", LogPrefix}, + {"error", status.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Coordination node successfully created"); + YDB_LOG_DEBUG("Coordination node successfully created", + {"logPrefix", LogPrefix}); CoordinationNodeCreated = true; ProcessState(); } @@ -455,7 +471,9 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvActorAttached::TP RpcActor = ev->Get()->RpcActor; SessionClosed = false; - LOG_ROW_DISPATCHER_DEBUG("RpcActor attached: " << RpcActor); + YDB_LOG_DEBUG("RpcActor", + {"logPrefix", LogPrefix}, + {"attached", RpcActor}); RpcResponses = {}; SendStartSession(); } @@ -464,12 +482,14 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvReadRequest::TPtr PendingRpcResponses++; if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Rpc read request skipped, session is closed"); + YDB_LOG_DEBUG("Rpc read request skipped, session is closed", + {"logPrefix", LogPrefix}); SendSessionEventFail(); return; } - LOG_ROW_DISPATCHER_DEBUG("Rpc read request"); + YDB_LOG_DEBUG("Rpc read request", + {"logPrefix", LogPrefix}); SendSessionEvents(); } @@ -478,7 +498,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt auto response = std::make_unique<TLocalRpcCtx::TEvWriteFinished>(); if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Rpc write request skipped, session is closed"); + YDB_LOG_DEBUG("Rpc write request skipped, session is closed", + {"logPrefix", LogPrefix}); response->Success = false; Send(RpcActor, response.release()); return; @@ -489,7 +510,9 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt auto& message = ev->Get()->Message; const auto messageCase = message.response_case(); - LOG_ROW_DISPATCHER_DEBUG("Rpc write request: " << static_cast<i64>(messageCase)); + YDB_LOG_DEBUG("Rpc write", + {"logPrefix", LogPrefix}, + {"request", static_cast<i64>(messageCase)}); switch (messageCase) { case TRpcOut::kPing: @@ -532,7 +555,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt ProcessDeleteSemaphoreResult(message); break; default: - LOG_ROW_DISPATCHER_DEBUG("Unknown message case"); + YDB_LOG_DEBUG("Unknown message case", + {"logPrefix", LogPrefix}); } ProcessState(); } @@ -540,9 +564,13 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvFinishRequest::TPtr& ev) { const auto& status = ev->Get()->Status; if (!status.ok()) { - LOG_ROW_DISPATCHER_ERROR("Rpc session finished with error status code: " << static_cast<ui64>(status.error_code()) << ", message: " << status.error_message()); + YDB_LOG_ERROR("Rpc session finished with error status", + {"logPrefix", LogPrefix}, + {"code", static_cast<ui64>(status.error_code())}, + {"message", status.error_message()}); } else { - LOG_ROW_DISPATCHER_INFO("Rpc session successfully finished"); + YDB_LOG_INFO("Rpc session successfully finished", + {"logPrefix", LogPrefix}); } CloseSession(status, "Read session closed"); @@ -551,7 +579,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvFinishRequest::TP void TLocalLeaderElection::SendStartSession() { Y_VALIDATE(RpcActor, "RpcActor is not set before read request"); - LOG_ROW_DISPATCHER_DEBUG("Sending start message"); + YDB_LOG_DEBUG("Sending start message", + {"logPrefix", LogPrefix}); TRpcIn message; auto& start = *message.mutable_session_start(); @@ -563,12 +592,15 @@ void TLocalLeaderElection::SendStartSession() { void TLocalLeaderElection::AddSessionEvent(TRpcIn&& message) { if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Session already closed, skip session event"); + YDB_LOG_DEBUG("Session already closed, skip session event", + {"logPrefix", LogPrefix}); return; } RpcResponses.push(message); - LOG_ROW_DISPATCHER_DEBUG("Added session event: " << static_cast<i64>(message.request_case())); + YDB_LOG_DEBUG("Added session", + {"logPrefix", LogPrefix}, + {"event", static_cast<i64>(message.request_case())}); if (RpcActor) { SendSessionEvents(); @@ -577,7 +609,10 @@ void TLocalLeaderElection::AddSessionEvent(TRpcIn&& message) { void TLocalLeaderElection::SendSessionEvents() { Y_VALIDATE(RpcActor, "RpcActor is not set before read request"); - LOG_ROW_DISPATCHER_DEBUG("Going to send session events, PendingRpcResponses: " << PendingRpcResponses << ", RpcResponses #" << RpcResponses.size()); + YDB_LOG_DEBUG("Going to send session events, RpcResponses", + {"logPrefix", LogPrefix}, + {"pendingRpcResponses", PendingRpcResponses}, + {"rpcResponses", RpcResponses.size()}); while (PendingRpcResponses > 0 && !RpcResponses.empty()) { SendSessionEvent(std::move(RpcResponses.front())); @@ -586,7 +621,10 @@ void TLocalLeaderElection::SendSessionEvents() { } void TLocalLeaderElection::SendSessionEvent(TRpcIn&& message, bool success) { - LOG_ROW_DISPATCHER_DEBUG("Sending session event: " << static_cast<i64>(message.request_case()) << ", success: " << success); + YDB_LOG_DEBUG("Sending session", + {"logPrefix", LogPrefix}, + {"event", static_cast<i64>(message.request_case())}, + {"success", success}); Y_VALIDATE(PendingRpcResponses > 0, "Rpc read is not expected"); PendingRpcResponses--; @@ -606,13 +644,20 @@ void TLocalLeaderElection::CloseSession(NYdb::EStatus status, const NYql::TIssue const bool success = status == NYdb::EStatus::SUCCESS; if (!success) { Metrics.Errors->Inc(); - LOG_ROW_DISPATCHER_ERROR("Closing session with status " << status << " and issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR("Closing session with status and", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else { - LOG_ROW_DISPATCHER_INFO("Closing session with success status"); + YDB_LOG_INFO("Closing session with success status", + {"logPrefix", LogPrefix}); } if (SessionClosed) { - LOG_ROW_DISPATCHER_WARN("Session already closed, but got status " << status << " and issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("Session already closed, but got status and", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); return; } SessionClosed = true; @@ -652,7 +697,8 @@ void TLocalLeaderElection::CloseSession(const grpc::Status& status, const TStrin } void TLocalLeaderElection::ProcessPing(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received Ping, send Pong"); + YDB_LOG_DEBUG("Received Ping, send Pong", + {"logPrefix", LogPrefix}); const auto& source = message.ping(); TRpcIn in; in.mutable_pong()->set_opaque(source.opaque()); @@ -660,7 +706,8 @@ void TLocalLeaderElection::ProcessPing(const TRpcOut& message) { } void TLocalLeaderElection::ProcessPong(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received Pong"); + YDB_LOG_DEBUG("Received Pong", + {"logPrefix", LogPrefix}); const auto& source = message.pong(); const uint64_t reqId = source.opaque(); auto* op = FindSentRequest(reqId); @@ -676,14 +723,18 @@ void TLocalLeaderElection::ProcessFailure(const TRpcOut& message) { NYql::TIssues issues; IssuesFromMessage(failure.issues(), issues); - LOG_ROW_DISPATCHER_DEBUG("Received Failure, status: " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("Received Failure", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); CloseSession(static_cast<NYdb::EStatus>(status), issues); ResetState(); } void TLocalLeaderElection::ProcessSessionStarted(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received SessionStarted, session successfully created"); + YDB_LOG_DEBUG("Received SessionStarted, session successfully created", + {"logPrefix", LogPrefix}); const auto& source = message.session_started(); SessionId = source.session_id(); @@ -692,20 +743,26 @@ void TLocalLeaderElection::ProcessSessionStarted(const TRpcOut& message) { } void TLocalLeaderElection::ProcessSessionStopped(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received SessionStopped"); + YDB_LOG_DEBUG("Received SessionStopped", + {"logPrefix", LogPrefix}); ResetState(); } void TLocalLeaderElection::ProcessCreateSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received CreateSemaphoreResult"); + YDB_LOG_DEBUG("Received CreateSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.create_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); - + auto status = NYdb::TStatus(static_cast<NYdb::EStatus>(source.status()), std::move(issues)); - LOG_ROW_DISPATCHER_INFO("Semaphore creating status: " << status); + YDB_LOG_INFO("Semaphore creating", + {"logPrefix", LogPrefix}, + {"status", status}); if (!IsTableCreated(status)) { - LOG_ROW_DISPATCHER_ERROR("Semaphore creating error " << status.GetIssues().ToOneLineString()); + YDB_LOG_ERROR("Semaphore creating error", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues().ToOneLineString()}); Metrics.Errors->Inc(); ResetState(); return; @@ -717,12 +774,14 @@ void TLocalLeaderElection::ProcessCreateSemaphoreResult(const TRpcOut& message) } SentRequests.erase(reqId); SemaphoreCreated = true; - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully created"); + YDB_LOG_DEBUG("Semaphore successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received AcquireSemaphoreResult"); + YDB_LOG_DEBUG("Received AcquireSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.acquire_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); @@ -731,7 +790,9 @@ void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) PendingAcquire = false; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Failed to acquire semaphore, " << status.GetIssues().ToOneLineString()); + YDB_LOG_ERROR("Failed to acquire semaphore", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues().ToOneLineString()}); Metrics.Errors->Inc(); ResetState(); return; @@ -743,15 +804,18 @@ void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) } SentRequests.erase(reqId); if (source.acquired()) { - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully acquired"); + YDB_LOG_DEBUG("Semaphore successfully acquired", + {"logPrefix", LogPrefix}); LastAcquireSemaphore = TInstant::Now(); // delay } else { - LOG_ROW_DISPATCHER_DEBUG("Semaphore acquire timed out"); + YDB_LOG_DEBUG("Semaphore acquire timed out", + {"logPrefix", LogPrefix}); } } void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DescribeSemaphoreResult"); + YDB_LOG_DEBUG("Received DescribeSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.describe_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); @@ -759,7 +823,9 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message PendingDescribe = false; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Semaphore describe fail, " << status.GetIssues()); + YDB_LOG_ERROR("Semaphore describe fail", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; @@ -776,7 +842,8 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message const NYdb::NCoordination::TSemaphoreDescription& description = source.semaphore_description(); Y_ABORT_UNLESS(description.GetOwners().size() <= 1, "To many owners"); if (description.GetOwners().empty()) { - LOG_ROW_DISPATCHER_DEBUG("Empty owners"); + YDB_LOG_DEBUG("Empty owners", + {"logPrefix", LogPrefix}); // Wait OnChanged. return; } @@ -789,9 +856,16 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message } NActors::TActorId id = ActorIdFromProto(protoId); - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully described: coordinator id " << id << " generation " << generation); + YDB_LOG_DEBUG("Semaphore successfully described: coordinator id generation", + {"logPrefix", LogPrefix}, + {"id", id}, + {"generation", generation}); if (!LeaderActorId || (*LeaderActorId != id)) { - LOG_ROW_DISPATCHER_INFO("Send TEvCoordinatorChanged to " << ParentId << ", new coordinator id " << id << ", previous coordinator id " << LeaderActorId.GetOrElse(TActorId())); + YDB_LOG_INFO("Send TEvCoordinatorChanged to new coordinator id previous coordinator id", + {"logPrefix", LogPrefix}, + {"parentId", ParentId}, + {"id", id}, + {"leaderActorId", LeaderActorId.GetOrElse(TActorId())}); TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(id, generation)); Metrics.LeaderChanged->Inc(); } @@ -799,7 +873,8 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message } void TLocalLeaderElection::ProcessDescribeSemaphoreChanged(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DescribeSemaphoreChanged"); + YDB_LOG_DEBUG("Received DescribeSemaphoreChanged", + {"logPrefix", LogPrefix}); PendingDescribeChanged = false; const auto& source = message.describe_semaphore_changed(); @@ -811,16 +886,19 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreChanged(const TRpcOut& messag SentRequests.erase(reqId); } -void TLocalLeaderElection::ProcessReleaseSemaphoreResult(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received ReleaseSemaphoreResult"); +void TLocalLeaderElection::ProcessReleaseSemaphoreResult(const TRpcOut& /*message*/) { + YDB_LOG_DEBUG("Received ReleaseSemaphoreResult", + {"logPrefix", LogPrefix}); } void TLocalLeaderElection::ProcessAcquireSemaphorePending(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received AcquireSemaphorePending"); + YDB_LOG_DEBUG("Received AcquireSemaphorePending", + {"logPrefix", LogPrefix}); } void TLocalLeaderElection::ProcessUpdateSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received UpdateSemaphoreResult"); + YDB_LOG_DEBUG("Received UpdateSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.update_semaphore_result(); const uint64_t reqId = source.req_id(); @@ -832,7 +910,8 @@ void TLocalLeaderElection::ProcessUpdateSemaphoreResult(const TRpcOut& message) } void TLocalLeaderElection::ProcessDeleteSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DeleteSemaphoreResult"); + YDB_LOG_DEBUG("Received DeleteSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.delete_semaphore_result(); const uint64_t reqId = source.req_id(); auto* op = FindSentRequest(reqId); @@ -844,7 +923,10 @@ void TLocalLeaderElection::ProcessDeleteSemaphoreResult(const TRpcOut& message) void TLocalLeaderElection::UpdateLastKnownGoodTimestampLocked(TInstant timestamp) { SessionLastKnownGoodTimestamp = Max(SessionLastKnownGoodTimestamp, timestamp); - LOG_ROW_DISPATCHER_TRACE("UpdateLastKnownGoodTimestampLocked timestamp " << timestamp << " new " << SessionLastKnownGoodTimestamp); + YDB_LOG_TRACE("UpdateLastKnownGoodTimestampLocked timestamp new", + {"logPrefix", LogPrefix}, + {"timestamp", timestamp}, + {"sessionLastKnownGoodTimestamp", SessionLastKnownGoodTimestamp}); } TLocalLeaderElection::TOperation* TLocalLeaderElection::FindSentRequest(uint64_t reqId) const { @@ -860,7 +942,9 @@ void TLocalLeaderElection::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { const auto reason = ev->Get()->Reason; Y_VALIDATE(sourceType == NKikimr::NGRpcService::TEvCoordinationSessionRequest::EventType, "Unexpected undelivered event: " << sourceType << ", reason: " << reason); - LOG_ROW_DISPATCHER_ERROR("Coordination service is unavailable, reason: " << reason); + YDB_LOG_ERROR("Coordination service is unavailable", + {"logPrefix", LogPrefix}, + {"reason", reason}); CloseSession(NYdb::EStatus::INTERNAL_ERROR, {NYql::TIssue("Coordination service is unavailable, please contact internal support")}); ResetState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp index a013676774a..ccda4d49c53 100644 --- a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp +++ b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp @@ -1,13 +1,15 @@ #include "compile_service.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/common/common.h> #include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/core/hfunc.h> #include <yql/essentials/public/purecalc/common/interface.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -49,7 +51,8 @@ public: Finish(); }; - LOG_ROW_DISPATCHER_TRACE("Started compile request"); + YDB_LOG_TRACE("Started compile request", + {"logPrefix", LogPrefix}); IProgramHolder::TPtr programHolder = std::move(Request->Get()->ProgramHolder); TStatus status = TStatus::Success(); @@ -64,10 +67,12 @@ public: } if (status.IsFail()) { - LOG_ROW_DISPATCHER_ERROR("Compilation failed for request"); + YDB_LOG_ERROR("Compilation failed for request", + {"logPrefix", LogPrefix}); Send(Request->Sender, new TEvRowDispatcher::TEvPurecalcCompileResponse(status.GetStatus(), status.GetErrorDescription()), 0, Request->Cookie); } else { - LOG_ROW_DISPATCHER_TRACE("Compilation completed for request"); + YDB_LOG_TRACE("Compilation completed for request", + {"logPrefix", LogPrefix}); Send(Request->Sender, new TEvRowDispatcher::TEvPurecalcCompileResponse(std::move(programHolder)), 0, Request->Cookie); } } @@ -128,7 +133,10 @@ public: void Handle(TEvRowDispatcher::TEvPurecalcCompileRequest::TPtr& ev) { const auto requestActor = ev->Sender; const ui64 requestId = ev->Cookie; - LOG_ROW_DISPATCHER_TRACE("Add to compile queue request with id " << requestId << " from " << requestActor); + YDB_LOG_TRACE("Add to compile queue request with id", + {"logPrefix", LogPrefix}, + {"requestId", requestId}, + {"requestActor", requestActor}); // Remove old compile request RemoveRequest(requestActor, requestId); @@ -142,13 +150,19 @@ public: } void Handle(TEvRowDispatcher::TEvPurecalcCompileAbort::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("Abort compile request with id " << ev->Cookie << " from " << ev->Sender); + YDB_LOG_TRACE("Abort compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}); RemoveRequest(ev->Sender, ev->Cookie); } void Handle(TEvPrivate::TEvCompileFinished::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("Compile finished for request with id " << ev->Get()->RequestId << " from " << ev->Get()->RequestActor); + YDB_LOG_TRACE("Compile finished for request with id", + {"logPrefix", LogPrefix}, + {"requestId", ev->Get()->RequestId}, + {"requestActor", ev->Get()->RequestActor}); InFlightCompilations.erase(ev->Sender); Counters.ActiveCompileActors->Dec(); diff --git a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make index 39657da6767..90611406e68 100644 --- a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events diff --git a/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp b/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp index 11fc56c6d26..0495a4acbd9 100644 --- a/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp +++ b/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp @@ -9,7 +9,6 @@ #include <ydb/core/base/appdata_fwd.h> #include <ydb/core/base/feature_flags.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/events/events.h> #include <ydb/core/fq/libs/metrics/sanitize_label.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> @@ -20,6 +19,7 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/interconnect.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/actors/common/retry_queue.h> #include <ydb/library/yql/providers/dq/counters/counters.h> @@ -30,6 +30,8 @@ #include <util/generic/queue.h> #include <util/stream/format.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -92,7 +94,7 @@ struct TEvPrivate { struct TEvUpdateMetrics : public NActors::TEventLocal<TEvUpdateMetrics, EvUpdateMetrics> {}; struct TEvPrintStateToLog : public NActors::TEventLocal<TEvPrintStateToLog, EvPrintStateToLog> {}; struct TEvTryConnect : public NActors::TEventLocal<TEvTryConnect, EvTryConnect> { - TEvTryConnect(ui32 nodeId = 0) + TEvTryConnect(ui32 nodeId = 0) : NodeId(nodeId) {} ui32 NodeId = 0; }; @@ -333,7 +335,7 @@ class TRowDispatcher : public TActorBootstrapped<TRowDispatcher> { } void PrintInternalState(TStringStream& stream) const { - stream << "Nodes states: \n"; + stream << "Nodes states: \n"; for (const auto& [nodeId, state] : Nodes) { stream << " id " << nodeId << " connected " << state.Connected << " retry scheduled " << state.RetryScheduled << " connected count " << state.Counters.Connected << " disconnected count " << state.Counters.Disconnected << "\n"; @@ -373,7 +375,7 @@ class TRowDispatcher : public TActorBootstrapped<TRowDispatcher> { NActors::TMon* Monitoring; TNodesTracker NodesTracker; NYql::TCounters::TEntry AllSessionsDateRate; - TAggregatedStats AggrStats; + TAggregatedStats AggrStats; ui64 LastCpuTime = 0; NActors::TActorId NodesManagerId; TInstant LastUpdateMetricsTime = TInstant::Now(); @@ -493,7 +495,7 @@ public: void Handle(NFq::TEvPrivate::TEvPrintStateToLog::TPtr&); void Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&); void Handle(const NMon::TEvHttpInfo::TPtr&); - + void DeleteConsumer(NActors::TActorId readActorId); void UpdateMetrics(); TString GetInternalState(); @@ -569,7 +571,10 @@ TRowDispatcher::TRowDispatcher( void TRowDispatcher::Bootstrap() { Become(&TRowDispatcher::StateFunc); - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped row dispatcher, id " << SelfId() << ", tenant " << Tenant); + YDB_LOG_DEBUG("Successfully bootstrapped row dispatcher, id tenant", + {"logPrefix", LogPrefix}, + {"selfId", SelfId()}, + {"tenant", Tenant}); const auto& config = Config.GetCoordinator(); auto coordinatorId = Register(NewCoordinator(SelfId(), config, Tenant, Counters, NodesManagerId).release()); @@ -597,9 +602,16 @@ void TRowDispatcher::Bootstrap() { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& ev) { LWPROBE(CoordinatorChanged, ev->Sender.ToString(), ev->Get()->Generation, ev->Get()->CoordinatorActorId.ToString(), CoordinatorGeneration, CoordinatorActorId->ToString()); - LOG_ROW_DISPATCHER_DEBUG("Coordinator changed, old leader " << CoordinatorActorId << ", new " << ev->Get()->CoordinatorActorId << " generation " << ev->Get()->Generation); + YDB_LOG_DEBUG("Coordinator changed, old leader new generation", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", CoordinatorActorId}, + {"evCoordinatorActorId", ev->Get()->CoordinatorActorId}, + {"generation", ev->Get()->Generation}); if (ev->Get()->Generation < CoordinatorGeneration) { - LOG_ROW_DISPATCHER_ERROR("New generation (" << ev->Get()->Generation << ") is less previous (" << CoordinatorGeneration << "), ignore updates"); + YDB_LOG_ERROR("New generation is less previous ignore updates", + {"logPrefix", LogPrefix}, + {"generation", ev->Get()->Generation}, + {"coordinatorGeneration", CoordinatorGeneration}); return; } CoordinatorActorId = ev->Get()->CoordinatorActorId; @@ -615,7 +627,9 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& void TRowDispatcher::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev) { LWPROBE(NodeConnected, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_DEBUG("EvNodeConnected, node id " << ev->Get()->NodeId); + YDB_LOG_DEBUG("EvNodeConnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); Metrics.NodesReconnect->Inc(); NodesTracker.HandleNodeConnected(ev->Get()->NodeId); for (auto& [actorId, consumer] : Consumers) { @@ -625,7 +639,9 @@ void TRowDispatcher::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { LWPROBE(NodeDisconnected, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_DEBUG("TEvNodeDisconnected, node id " << ev->Get()->NodeId); + YDB_LOG_DEBUG("TEvNodeDisconnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); Metrics.NodesReconnect->Inc(); bool isTimeout = NodesTracker.HandleNodeDisconnected(ev->Get()->NodeId); for (auto& [actorId, consumer] : Consumers) { @@ -633,13 +649,15 @@ void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TP } TVector<TActorId> toDelete; if (isTimeout) { - LOG_ROW_DISPATCHER_DEBUG("Node disconnected, node id " << ev->Get()->NodeId << " is timeout"); + YDB_LOG_DEBUG("Node disconnected, node id is timeout", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); for (auto& [actorId, consumer] : Consumers) { if (actorId.NodeId() != ev->Get()->NodeId) { continue; } toDelete.push_back(actorId); - } + } } for (auto& actorId : toDelete) { DeleteConsumer(actorId); @@ -648,7 +666,10 @@ void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TP void TRowDispatcher::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { LWPROBE(UndeliveredStart, ev->Sender.ToString(), ev->Get()->Reason, ev->Cookie); - LOG_ROW_DISPATCHER_TRACE("TEvUndelivered, from " << ev->Sender << ", reason " << ev->Get()->Reason); + YDB_LOG_TRACE("TEvUndelivered, from reason", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"reason", ev->Get()->Reason}); for (auto& [key, consumer] : Consumers) { if (ev->Cookie != consumer->Generation) { // Several partitions in one read_actor have different Generation. LWPROBE(UndeliveredSkipGeneration, ev->Sender.ToString(), ev->Get()->Reason, ev->Cookie, consumer->Generation); @@ -668,17 +689,22 @@ void TRowDispatcher::Handle(TEvPrivate::TEvCoordinatorPing::TPtr&) { return; } LWPROBE(CoordinatorPing, CoordinatorActorId->ToString()); - LOG_ROW_DISPATCHER_TRACE("Send ping to " << *CoordinatorActorId); + YDB_LOG_TRACE("Send ping", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", *CoordinatorActorId}); Send(*CoordinatorActorId, new NActors::TEvents::TEvPing()); } void TRowDispatcher::Handle(NActors::TEvents::TEvPong::TPtr&) { LWPROBE(Pong); - LOG_ROW_DISPATCHER_TRACE("NActors::TEvents::TEvPong"); + YDB_LOG_TRACE("NActors::TEvents::TEvPong", + {"logPrefix", LogPrefix}); } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChangesSubscribe::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvCoordinatorChangesSubscribe from " << ev->Sender); + YDB_LOG_DEBUG("TEvCoordinatorChangesSubscribe", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); NodesTracker.AddNode(ev->Sender.NodeId()); CoordinatorChangedSubscribers.insert(ev->Sender); if (!CoordinatorActorId) { @@ -819,7 +845,7 @@ TString TRowDispatcher::GetInternalState() { ui64 minInitialOffset = std::numeric_limits<ui64>::max(); for (const auto& [formatName, formatStats] : sessionInfo.Stat.FormatHandlers) { - str << " " << formatName + str << " " << formatName << " parse and filter lantecy " << formatStats.ParseAndFilterLatency << " (parse " << formatStats.ParserStats.ParserLatency << ", filter " << formatStats.FilterStats.FilterLatency << ")\n"; } @@ -834,9 +860,9 @@ TString TRowDispatcher::GetInternalState() { << toHuman(stat.QueuedBytes) << " (" << leftPad(stat.QueuedRows) << " rows) " << " offset " << leftPad(stat.Offset) << " init offset " << leftPad(stat.InitialOffset) << " get " << leftPad(consumer->Counters.GetNextBatch) - << " arr " << leftPad(consumer->Counters.NewDataArrived) << " btc " << leftPad(consumer->Counters.MessageBatch) + << " arr " << leftPad(consumer->Counters.NewDataArrived) << " btc " << leftPad(consumer->Counters.MessageBatch) << " pend get " << leftPad(partition.PendingGetNextBatch) << " pend new " << leftPad(partition.PendingNewDataArrived) - << " waiting " << stat.IsWaiting << " read lag " << leftPad(stat.ReadLagMessages) + << " waiting " << stat.IsWaiting << " read lag " << leftPad(stat.ReadLagMessages) << " conn id " << consumer->Generation << "\n"; maxInitialOffset = std::max(maxInitialOffset, stat.InitialOffset); minInitialOffset = std::min(minInitialOffset, stat.InitialOffset); @@ -848,9 +874,9 @@ TString TRowDispatcher::GetInternalState() { str << "Consumers:\n"; for (auto& [readActorId, consumer] : Consumers) { str << " " << consumer->QueryId << " " << LeftPad(readActorId, 32) << " Generation " << consumer->Generation << "\n"; - str << " partitions: "; + str << " partitions: "; for (const auto& [partitionId, info] : consumer->Partitions) { - str << partitionId << ","; + str << partitionId << ","; } str << "\n retry queue: "; consumer->EventsQueue.PrintInternalState(str); @@ -893,8 +919,14 @@ void TRowDispatcher::UpdateReadActorsInternalState() { } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("Received TEvStartSession from " << ev->Sender << ", read group " << ev->Get()->Record.GetSource().GetReadGroup() << ", topicPath " << ev->Get()->Record.GetSource().GetTopicPath() << - " part id " << JoinSeq(',', ev->Get()->Record.GetPartitionIds()) << " query id " << ev->Get()->Record.GetQueryId() << " cookie " << ev->Cookie); + YDB_LOG_DEBUG("Received TEvStartSession from read group topicPath part id query id cookie", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"Record.GetSource().GetReadGroup", ev->Get()->Record.GetSource().GetReadGroup()}, + {"Record.GetSource().GetTopicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"partitionIds", JoinSeq(',', ev->Get()->Record.GetPartitionIds())}, + {"Record.GetQueryId", ev->Get()->Record.GetQueryId()}, + {"cookie", ev->Cookie}); if (EnableStreamingQueriesCounters) { auto queryGroup = Metrics.Counters->GetSubgroup("query_id", ev->Get()->Record.GetQueryId()); auto topicGroup = queryGroup->GetSubgroup("read_group", SanitizeLabel(ev->Get()->Record.GetSource().GetReadGroup())); @@ -907,12 +939,16 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it != Consumers.end()) { if (ev->Cookie <= it->second->Generation) { - LOG_ROW_DISPATCHER_WARN("Consumer already exists, ignore StartSession"); + YDB_LOG_WARN("Consumer already exists, ignore StartSession", + {"logPrefix", LogPrefix}); return; } - LOG_ROW_DISPATCHER_WARN("Consumer already exists, new consumer with new generation (" << ev->Cookie << ", current " - << it->second->Generation << "), remove old consumer, sender " << ev->Sender << ", topicPath " - << ev->Get()->Record.GetSource().GetTopicPath() << " cookie " << ev->Cookie); + YDB_LOG_WARN("Consumer already exists, new consumer with new generation current remove old consumer, sender topicPath cookie", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}, + {"generation", it->second->Generation}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}); DeleteConsumer(ev->Sender); } const auto& source = ev->Get()->Record.GetSource(); @@ -933,8 +969,11 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { Y_ENSURE(topicSessionInfo.Sessions.size() <= 1); if (topicSessionInfo.Sessions.empty()) { - LOG_ROW_DISPATCHER_DEBUG("Create new session: read group " << source.GetReadGroup() << " topic " << source.GetTopicPath() - << " part id " << partitionId); + YDB_LOG_DEBUG("Create new session: read group topic part id", + {"logPrefix", LogPrefix}, + {"readGroup", source.GetReadGroup()}, + {"topicPath", source.GetTopicPath()}, + {"partitionId", partitionId}); sessionActorId = ActorFactory->RegisterTopicSession( source.GetReadGroup(), source.GetTopicPath(), @@ -974,18 +1013,28 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvGetNextBatch from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_WARN("Ignore (no consumer) TEvGetNextBatch from part id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } auto& session = it->second; LWPROBE(GetNextBatch, ev->Sender.ToString(), ev->Get()->Record.GetPartitionId(), session->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Received TEvGetNextBatch from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId() << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Received TEvGetNextBatch from part id query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}, + {"queryId", it->second->QueryId}); if (!CheckSession(session, ev)) { return; } auto partitionIt = session->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == session->Partitions.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore TEvGetNextBatch from " << ev->Sender << ", wrong partition id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_ERROR("Ignore TEvGetNextBatch from wrong partition id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } partitionIt->second.PendingNewDataArrived = false; @@ -997,17 +1046,27 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvHeartbeat::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Consumer not found, sender " << ev->Sender << ", part id " << ev->Get()->Record.GetPartitionId() << ", sending TEvNoSession"); + YDB_LOG_WARN("Consumer not found, sender part id sending TEvNoSession", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); Send(ev->Sender, new NFq::TEvRowDispatcher::TEvNoSession(), 0, ev->Cookie); return; } LWPROBE(Heartbeat, ev->Sender.ToString(), ev->Get()->Record.GetPartitionId(), it->second->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Received TEvHeartbeat from " << ev->Sender << ", part id " << ev->Get()->Record.GetPartitionId() << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Received TEvHeartbeat from part id query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}, + {"queryId", it->second->QueryId}); CheckSession(it->second, ev); } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNoSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("Received TEvNoSession from " << ev->Sender << ", generation " << ev->Cookie); + YDB_LOG_DEBUG("Received TEvNoSession from generation", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto consumerIt = Consumers.find(ev->Sender); if (consumerIt == Consumers.end()) { return; @@ -1022,12 +1081,23 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNoSession::TPtr& ev) { template <class TEventPtr> bool TRowDispatcher::CheckSession(TAtomicSharedPtr<TConsumerInfo>& consumer, const TEventPtr& ev) { if (ev->Cookie != consumer->Generation) { - LOG_ROW_DISPATCHER_WARN("Wrong message generation (" << typeid(TEventPtr).name() << "), sender " << ev->Sender << " cookie " << ev->Cookie << ", session generation " << consumer->Generation << ", query id " << consumer->QueryId); + YDB_LOG_WARN("Wrong message generation sender cookie session generation query id", + {"logPrefix", LogPrefix}, + {"typeName", typeid(TEventPtr).name()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"Generation", consumer->Generation}, + {"QueryId", consumer->QueryId}); return false; } if (!consumer->EventsQueue.OnEventReceived(ev)) { const NYql::NDqProto::TMessageTransportMeta& meta = ev->Get()->Record.GetTransportMeta(); - LOG_ROW_DISPATCHER_WARN("Wrong seq num, ignore message (" << typeid(TEventPtr).name() << ") seqNo " << meta.GetSeqNo() << " from " << ev->Sender.ToString() << ", query id " << consumer->QueryId); + YDB_LOG_WARN("Wrong seq num, ignore message seqNo from query id", + {"logPrefix", LogPrefix}, + {"typeName", typeid(TEventPtr).name()}, + {"seqNo", meta.GetSeqNo()}, + {"sender", ev->Sender}, + {"QueryId", consumer->QueryId}); return false; } return true; @@ -1036,12 +1106,18 @@ bool TRowDispatcher::CheckSession(TAtomicSharedPtr<TConsumerInfo>& consumer, con void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore TEvStopSession from " << ev->Sender); + YDB_LOG_WARN("Ignore TEvStopSession", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } LWPROBE(StopSession, ev->Sender.ToString(), it->second->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_DEBUG("Received TEvStopSession from " << ev->Sender << " topic " << ev->Get()->Record.GetSource().GetTopicPath() << " query id " << it->second->QueryId); + YDB_LOG_DEBUG("Received TEvStopSession from topic query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"queryId", it->second->QueryId}); if (!CheckSession(it->second, ev)) { return; } @@ -1051,12 +1127,18 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { auto consumerIt = Consumers.find(readActorId); if (consumerIt == Consumers.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore (no consumer) DeleteConsumer, " << " read actor id " << readActorId); + YDB_LOG_ERROR("Ignore (no consumer) DeleteConsumer, read actor id", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); return; } const auto& consumer = consumerIt->second; - LOG_ROW_DISPATCHER_DEBUG("DeleteConsumer, readActorId " << readActorId << " query id " << consumer->QueryId << ", partitions size " << consumer->Partitions.size()); + YDB_LOG_DEBUG("DeleteConsumer, readActorId query id partitions size", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"QueryId", consumer->QueryId}, + {"Partitions.size", consumer->Partitions.size()}); for (auto& [partitionId, partition] : consumer->Partitions) { auto event = std::make_unique<NFq::TEvRowDispatcher::TEvStopSession>(); *event->Record.MutableSource() = consumer->SourceParams; @@ -1073,10 +1155,14 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { TTopicSessionInfo& topicSessionInfo = sessionIt->second; TSessionInfo& sessionInfo = topicSessionInfo.Sessions[partition.TopicSessionId]; if (!sessionInfo.Consumers.erase(consumer->ReadActorId)) { - LOG_ROW_DISPATCHER_ERROR("Wrong readActorId " << consumer->ReadActorId << ", no such consumer"); + YDB_LOG_ERROR("Wrong readActorId no such consumer", + {"logPrefix", LogPrefix}, + {"ReadActorId", consumer->ReadActorId}); } if (sessionInfo.Consumers.empty()) { - LOG_ROW_DISPATCHER_DEBUG("Session is not used, sent TEvPoisonPill to " << partition.TopicSessionId); + YDB_LOG_DEBUG("Session is not used, sent TEvPoisonPill", + {"logPrefix", LogPrefix}, + {"topicSessionId", partition.TopicSessionId}); topicSessionInfo.Sessions.erase(partition.TopicSessionId); Send(partition.TopicSessionId, new NActors::TEvents::TEvPoisonPill()); if (topicSessionInfo.Sessions.empty()) { @@ -1088,7 +1174,7 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { const TString queryId = consumerIt->second->QueryId; ConsumersByEventQueueId.erase(consumerIt->second->EventQueueId); Consumers.erase(consumerIt); - + auto queryIt = ConsumersByQueryId.find(queryId); if (queryIt != ConsumersByQueryId.end()) { queryIt->second.erase(readActorId); @@ -1104,14 +1190,18 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { void TRowDispatcher::Handle(const TEvPrivate::TEvTryConnect::TPtr& ev) { LWPROBE(TryConnect, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_TRACE("TEvTryConnect to node id " << ev->Get()->NodeId); + YDB_LOG_TRACE("TEvTryConnect to node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); NodesTracker.TryConnect(ev->Get()->NodeId); } void TRowDispatcher::Handle(const NYql::NDq::TEvRetryQueuePrivate::TEvEvHeartbeat::TPtr& ev) { auto it = ConsumersByEventQueueId.find(ev->Get()->EventQueueId); if (it == ConsumersByEventQueueId.end()) { - LOG_ROW_DISPATCHER_TRACE("No consumer with EventQueueId = " << ev->Get()->EventQueueId); + YDB_LOG_TRACE("No consumer with EventQueueId", + {"logPrefix", LogPrefix}, + {"eventQueueId", ev->Get()->EventQueueId}); return; } auto& sessionInfo = it->second; @@ -1119,21 +1209,31 @@ void TRowDispatcher::Handle(const NYql::NDq::TEvRetryQueuePrivate::TEvEvHeartbea bool needSend = sessionInfo->EventsQueue.Heartbeat(); if (needSend) { - LOG_ROW_DISPATCHER_TRACE("Send TEvHeartbeat to " << sessionInfo->ReadActorId << " query id " << sessionInfo->QueryId); + YDB_LOG_TRACE("Send TEvHeartbeat to query id", + {"logPrefix", LogPrefix}, + {"readActorId", sessionInfo->ReadActorId}, + {"queryId", sessionInfo->QueryId}); auto event = std::make_unique<NFq::TEvRowDispatcher::TEvHeartbeat>(); sessionInfo->EventsQueue.Send(new NFq::TEvRowDispatcher::TEvHeartbeat(), sessionInfo->Generation); } } -void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) { +void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvNewDataArrived from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_WARN("Ignore (no consumer) TEvNewDataArrived from part id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; LWPROBE(NewDataArrived, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), consumerInfoPtr->QueryId, consumerInfoPtr->Generation, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Forward TEvNewDataArrived from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << consumerInfoPtr->QueryId); + YDB_LOG_TRACE("Forward TEvNewDataArrived from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", consumerInfoPtr->QueryId}); auto partitionIt = consumerInfoPtr->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == consumerInfoPtr->Partitions.end()) { // Ignore TEvNewDataArrived because read actor now read others partitions. @@ -1147,12 +1247,19 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvMessageBatch::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvMessageBatch from " << ev->Sender << " to " << ev->Get()->ReadActorId); + YDB_LOG_WARN("Ignore (no consumer) TEvMessageBatch", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}); return; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; LWPROBE(MessageBatch, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), consumerInfoPtr->QueryId, consumerInfoPtr->Generation, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Forward TEvMessageBatch from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << consumerInfoPtr->QueryId); + YDB_LOG_TRACE("Forward TEvMessageBatch from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", consumerInfoPtr->QueryId}); Metrics.RowsSent->Add(ev->Get()->Record.MessagesSize()); auto partitionIt = consumerInfoPtr->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == consumerInfoPtr->Partitions.end()) { @@ -1167,17 +1274,26 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvMessageBatch::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionError::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvSessionError from " << ev->Sender << " to " << ev->Get()->ReadActorId); + YDB_LOG_WARN("Ignore (no consumer) TEvSessionError", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}); return; } LWPROBE(SessionError, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), it->second->QueryId, it->second->Generation, ev->Get()->Record.ByteSizeLong()); ++*Metrics.ErrorsCount; - LOG_ROW_DISPATCHER_TRACE("Forward TEvSessionError from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Forward TEvSessionError from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", it->second->QueryId}); if (ev->Get()->IsFatalError) { auto consumerIt = Consumers.find(ev->Get()->ReadActorId); if (consumerIt == Consumers.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore (no consumer) DeleteConsumer, " << " read actor id " << ev->Get()->ReadActorId); + YDB_LOG_ERROR("Ignore (no consumer) DeleteConsumer, read actor id", + {"logPrefix", LogPrefix}, + {"readActorId", ev->Get()->ReadActorId}); return; } const auto& consumer = consumerIt->second; @@ -1192,7 +1308,9 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionError::TPtr& ev) { if (sessionIt != TopicSessions.end()) { TTopicSessionInfo& topicSessionInfo = sessionIt->second; if (topicSessionInfo.Sessions.erase(ev->Sender)) { - LOG_ROW_DISPATCHER_WARN("Fatal session error, remove session " << ev->Sender); + YDB_LOG_WARN("Fatal session error, remove session", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Send(ev->Sender, new NActors::TEvents::TEvPoisonPill()); if (topicSessionInfo.Sessions.empty()) { TopicSessions.erase(sessionIt); @@ -1221,12 +1339,15 @@ void TRowDispatcher::PrintStateToLog() { auto str = GetInternalState(); auto buf = TStringBuf(str); for (ui64 offset = 0; offset < buf.size(); offset += PrintStateToLogSplitSize) { - LOG_ROW_DISPATCHER_DEBUG(buf.SubString(offset, PrintStateToLogSplitSize)); + YDB_LOG_DEBUG("Dump logPrefix, #_buf.SubString(offset, PrintStateToLogSplitSize)", + {"logPrefix", LogPrefix}, + {"state", buf.SubString(offset, PrintStateToLogSplitSize)}); } } void TRowDispatcher::Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvPrivate::TEvSendStatistic"); + YDB_LOG_TRACE("TEvPrivate::TEvSendStatistic", + {"logPrefix", LogPrefix}); UpdateCpuTime(); Schedule(Config.GetSendStatusPeriod(), new NFq::TEvPrivate::TEvSendStatistic()); @@ -1286,11 +1407,13 @@ void TRowDispatcher::Handle(const NMon::TEvHttpInfo::TPtr& ev) { } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionStatistic::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvSessionStatistic from " << ev->Sender); + YDB_LOG_TRACE("TEvSessionStatistic", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); const auto& stat = ev->Get()->Stat; const auto& key = stat.SessionKey; - LWPROBE(SessionStatistic, + LWPROBE(SessionStatistic, ev->Sender.ToString(), key.ReadGroup, key.Endpoint, @@ -1321,7 +1444,7 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionStatistic::TPtr& ev if (it == sessionInfo.Consumers.end()) { continue; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; auto partitionIt = consumerInfoPtr->Partitions.find(key.PartitionId); if (partitionIt == consumerInfoPtr->Partitions.end()) { continue; @@ -1352,7 +1475,10 @@ void TRowDispatcher::UpdateCpuTime() { for (auto& [actorId, consumer] : Consumers) { consumer->CpuMicrosec += diff; } - LOG_ROW_DISPATCHER_TRACE("UpdateCpuTime, currentCpuTime " << currentCpuTime << ", diff " << diff); + YDB_LOG_TRACE("UpdateCpuTime, currentCpuTime diff", + {"logPrefix", LogPrefix}, + {"currentCpuTime", currentCpuTime}, + {"diff", diff}); LastCpuTime = currentCpuTime; } diff --git a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp index db1003f8362..03d587ce519 100644 --- a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp +++ b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp @@ -3,12 +3,12 @@ #include <ydb/core/base/appdata_fwd.h> #include <ydb/core/base/feature_flags.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/metrics/sanitize_label.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/actors/dq.h> #include <ydb/library/yql/providers/pq/common/pq_events_processor.h> #include <ydb/public/sdk/cpp/adapters/issue/issue.h> @@ -16,6 +16,8 @@ #include <util/generic/queue.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -35,7 +37,7 @@ struct TTopicSessionMetrics { const auto topicGroup = counters->GetSubgroup("topic", SanitizeLabel(topicPath)); ReadGroup = topicGroup->GetSubgroup("read_group", SanitizeLabel(readGroupName)); PartitionGroup = ReadGroup->GetSubgroup("partition", ToString(partitionId)); - PartitionGroup = PartitionGroup->GetSubgroup("host", ""); // This partition report metrics only on one host. + PartitionGroup = PartitionGroup->GetSubgroup("host", ""); // This partition report metrics only on one host. } AllSessionsDataRate = ReadGroup->GetCounter("AllSessionsDataRate", true); InFlyAsyncInputData = PartitionGroup->GetCounter("InFlyAsyncInputData"); @@ -208,7 +210,13 @@ private: void AddDataToClient(ui64 offset, ui64 numberRows, ui64 rowSize, TMaybe<TInstant> watermark) override { Y_ENSURE(!NextMessageOffset || offset >= *NextMessageOffset, "Unexpected historical offset"); - LOG_ROW_DISPATCHER_TRACE("AddDataToClient to " << ReadActorId << ", offset: " << offset << ", number rows: " << numberRows << ", row size: " << rowSize << ", watermark: " << watermark); + YDB_LOG_TRACE("AddDataToClient to number row", + {"logPrefix", LogPrefix}, + {"readActorId", ReadActorId}, + {"offset", offset}, + {"rows", numberRows}, + {"size", rowSize}, + {"watermark", watermark}); NextMessageOffset = offset + 1; QueuedRows += numberRows; @@ -220,7 +228,10 @@ private: } void UpdateClientOffset(ui64 offset) override { - LOG_ROW_DISPATCHER_TRACE("UpdateClientOffset for " << ReadActorId << ", new offset: " << offset); + YDB_LOG_TRACE("UpdateClientOffset for new", + {"logPrefix", LogPrefix}, + {"readActorId", ReadActorId}, + {"offset", offset}); if (!NextMessageOffset || *NextMessageOffset < offset + 1) { NextMessageOffset = offset + 1; } @@ -453,13 +464,16 @@ void TTopicSession::Bootstrap() { Become(&TTopicSession::StateFunc); Metrics.Init(Counters, TopicPath, ReadGroup, PartitionId, EnableStreamingQueriesCounters); LogPrefix = LogPrefix + " " + SelfId().ToString() + " [" + TopicPathPartition + "] "; - LOG_ROW_DISPATCHER_INFO("Bootstrap, Timeout " << Config.GetTimeoutBeforeStartSession() << " sec"); + YDB_LOG_INFO("Bootstrap, Timeout sec", + {"logPrefix", LogPrefix}, + {"timeout", Config.GetTimeoutBeforeStartSession()}); Schedule(TDuration::Seconds(SendStatisticPeriodSec), new NFq::TEvPrivate::TEvSendStatistic()); Schedule(TDuration::Seconds(GetEventByTimerPeriodSec), new NFq::TEvPrivate::TEvGetEventByTimerEvent()); } void TTopicSession::PassAway() { - LOG_ROW_DISPATCHER_INFO("PassAway"); + YDB_LOG_INFO("PassAway", + {"logPrefix", LogPrefix}); StopReadSession(); for (const auto& [actorId, clientInfo] : Clients) { if (const auto formatIt = FormatHandlers.find(clientInfo->HandlerSettings); formatIt != FormatHandlers.end()) { @@ -472,16 +486,22 @@ void TTopicSession::PassAway() { void TTopicSession::SubscribeOnNextEvent(bool checkIsWaitingEvents) { if (!ReadSession || (checkIsWaitingEvents && IsWaitingEvents)) { - LOG_ROW_DISPATCHER_TRACE("Skip SubscribeOnNextEvent, has ReadSession: " << (ReadSession ? "true" : "false") << ", IsWaitingEvents: " << IsWaitingEvents); + YDB_LOG_TRACE("Skip SubscribeOnNextEvent, has", + {"logPrefix", LogPrefix}, + {"readSession", (ReadSession ? "true" : "false")}, + {"isWaitingEvents", IsWaitingEvents}); return; } if (Config.GetMaxSessionUsedMemory() && QueuedBytes > Config.GetMaxSessionUsedMemory()) { - LOG_ROW_DISPATCHER_TRACE("Too much used memory (" << QueuedBytes << " bytes), skip subscribing to WaitEvent()"); + YDB_LOG_TRACE("Too much used memory bytes), skip subscribing to WaitEvent()", + {"logPrefix", LogPrefix}, + {"queuedBytes", QueuedBytes}); return; } - LOG_ROW_DISPATCHER_TRACE("SubscribeOnNextEvent"); + YDB_LOG_TRACE("SubscribeOnNextEvent", + {"logPrefix", LogPrefix}); IsWaitingEvents = true; Metrics.InFlySubscribe->Set(1); TActorSystem* actorSystem = TActivationContext::ActorSystem(); @@ -529,9 +549,12 @@ NYdb::NTopic::TReadSessionSettings TTopicSession::GetReadSessionSettings(const T topicReadSettings.AppendPartitionIds(PartitionId); TInstant minTime = GetMinStartingMessageTimestamp(); - LOG_ROW_DISPATCHER_INFO("Create topic session, Path " << TopicPathPartition - << ", StartingMessageTimestamp " << minTime - << ", BufferSize " << BufferSize << ", GetConsumerMode " << Config.GetConsumerMode()); + YDB_LOG_INFO("Create topic session, Path StartingMessageTimestamp BufferSize GetConsumerMode", + {"logPrefix", LogPrefix}, + {"topicPathPartition", TopicPathPartition}, + {"minTime", minTime}, + {"bufferSize", BufferSize}, + {"consumerMode", Config.GetConsumerMode()}); auto settings = NYdb::NTopic::TReadSessionSettings() .TraceId(LogPrefix) @@ -567,7 +590,9 @@ void TTopicSession::CreateTopicSession() { // Use any sourceParams. ReconnectPeriod = Clients.begin()->second->ReconnectPeriod; if (ReconnectPeriod != TDuration::Zero()) { - LOG_ROW_DISPATCHER_INFO("ReconnectPeriod " << ReconnectPeriod.ToString()); + YDB_LOG_INFO("ReconnectPeriod", + {"logPrefix", LogPrefix}, + {"reconnectPeriod", ReconnectPeriod}); Metrics.ReconnectRate->Inc(); Schedule(ReconnectPeriod, new NFq::TEvPrivate::TEvReconnectSession()); InflightReconnect = true; @@ -576,7 +601,8 @@ void TTopicSession::CreateTopicSession() { } void TTopicSession::Handle(NFq::TEvPrivate::TEvPqEventsReady::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvPqEventsReady"); + YDB_LOG_TRACE("TEvPqEventsReady", + {"logPrefix", LogPrefix}); Metrics.InFlySubscribe->Set(0); IsWaitingEvents = false; auto waitEventDurationMs = (TInstant::Now() - WaitEventStartedAt).MilliSeconds(); @@ -597,20 +623,27 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvReconnectSession::TPtr&) { return; } StartingMessageTimestamp = GetMinStartingMessageTimestamp(); - LOG_ROW_DISPATCHER_DEBUG("Reconnect topic session, " << TopicPathPartition - << ", StartingMessageTimestamp " << StartingMessageTimestamp - << ", BufferSize " << BufferSize << ", ConsumerMode " << Config.GetConsumerMode()); + YDB_LOG_DEBUG("Reconnect topic session, StartingMessageTimestamp BufferSize ConsumerMode", + {"logPrefix", LogPrefix}, + {"topicPathPartition", TopicPathPartition}, + {"startingMessageTimestamp", StartingMessageTimestamp}, + {"bufferSize", BufferSize}, + {"consumerMode", Config.GetConsumerMode()}); RefreshParsers(); StopReadSession(); CreateTopicSession(); } void TTopicSession::Handle(TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvGetNextBatch from " << ev->Sender.ToString()); + YDB_LOG_TRACE("TEvGetNextBatch", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Metrics.InFlyAsyncInputData->Set(0); auto it = Clients.find(ev->Sender); if (it == Clients.end()) { - LOG_ROW_DISPATCHER_ERROR("Wrong client, sender " << ev->Sender); + YDB_LOG_ERROR("Wrong client, sender", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } SendData(*it->second); @@ -626,7 +659,9 @@ bool TTopicSession::HandleNewEvents() { return false; } if (Config.GetMaxSessionUsedMemory() && QueuedBytes > Config.GetMaxSessionUsedMemory()) { - LOG_ROW_DISPATCHER_TRACE("Too much used memory (" << QueuedBytes << " bytes), stop reading from yds"); + YDB_LOG_TRACE("Too much used memory bytes), stop reading from yds", + {"logPrefix", LogPrefix}, + {"queuedBytes", QueuedBytes}); break; } std::optional<NYdb::NTopic::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false); @@ -651,7 +686,8 @@ void TTopicSession::CloseTopicSession() { if (!ReadSession) { return; } - LOG_ROW_DISPATCHER_DEBUG("Close session"); + YDB_LOG_DEBUG("Close session", + {"logPrefix", LogPrefix}); ReadSession->Close(TDuration::Zero()); ReadSession.reset(); } @@ -670,13 +706,22 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } if (hasOldMessages) { - LOG_ROW_DISPATCHER_TRACE("Skip data. StartingMessageTimestamp: " << Self.StartingMessageTimestamp << ". Write time: " << messages.begin()->GetWriteTime()); + YDB_LOG_TRACE("Skip data. Write", + {"logPrefix", LogPrefix}, + {"startingMessageTimestamp", Self.StartingMessageTimestamp}, + {"time", messages.begin()->GetWriteTime()}); Self.LastMessageOffset = std::prev(it)->GetOffset(); messages.erase(messages.begin(), it); } for (const auto& message : messages) { - LOG_ROW_DISPATCHER_TRACE("Data received: " << message.DebugString(true)); + YDB_LOG_TRACE("Data received", + {"logPrefix", LogPrefix}, + {"data", message.DebugString(true)}); + YDB_LOG_TRACE("Data received", + {"logPrefix", LogPrefix}, + {"data", message.DebugString(true)}); + try { dataSize += message.GetData().size(); } catch (...) { @@ -694,7 +739,10 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TSessionClosedEvent& ev) { const TString message = TStringBuilder() << "Read session to topic \"" << Self.TopicPathPartition << "\" was closed"; - LOG_ROW_DISPATCHER_DEBUG(message << ": " << ev.DebugString()); + YDB_LOG_DEBUG("Dump logPrefix, message, #_ev.DebugString", + {"logPrefix", LogPrefix}, + {"message", message}, + {"ev", ev.DebugString()}); Self.ThrowFatalError(TStatus::Fail( NYql::NDq::YdbStatusToDqStatus(static_cast<Ydb::StatusIds::StatusCode>(ev.GetStatus())), @@ -703,7 +751,8 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TSessionClose } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TStartPartitionSessionEvent& event) { - LOG_ROW_DISPATCHER_DEBUG("StartPartitionSessionEvent received"); + YDB_LOG_DEBUG("StartPartitionSessionEvent received", + {"logPrefix", LogPrefix}); std::optional<ui64> minOffset; for (const auto& [actorId, info] : Self.Clients) { @@ -711,7 +760,9 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE minOffset = info->NextMessageOffset; } } - LOG_ROW_DISPATCHER_DEBUG("Confirm StartPartitionSession with offset " << minOffset); + YDB_LOG_DEBUG("Confirm StartPartitionSession with offset", + {"logPrefix", LogPrefix}, + {"minOffset", minOffset}); event.Confirm(minOffset); if (minOffset) { // ensure we restart session if new client wants earlier offset @@ -722,12 +773,15 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TStopPartitionSessionEvent& event) { - LOG_ROW_DISPATCHER_DEBUG("SessionId: " << Self.GetSessionId() << " StopPartitionSessionEvent received"); + YDB_LOG_DEBUG("StopPartitionSessionEvent received", + {"logPrefix", LogPrefix}, + {"sessionId", Self.GetSessionId()}); event.Confirm(); } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TEndPartitionSessionEvent& /*event*/) { - LOG_ROW_DISPATCHER_WARN("TEndPartitionSessionEvent"); + YDB_LOG_WARN("TEndPartitionSessionEvent", + {"logPrefix", LogPrefix}); Self.ThrowFatalError(TStatus::Fail( EStatusId::SCHEME_ERROR, @@ -735,7 +789,8 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TPartitionSessionClosedEvent& /*event*/) { - LOG_ROW_DISPATCHER_WARN("TPartitionSessionClosedEvent"); + YDB_LOG_WARN("TPartitionSessionClosedEvent", + {"logPrefix", LogPrefix}); } TString TTopicSession::GetSessionId() const { @@ -743,7 +798,9 @@ TString TTopicSession::GetSessionId() const { } void TTopicSession::SendToParsing(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) { - LOG_ROW_DISPATCHER_TRACE("SendToParsing, messages: " << messages.size()); + YDB_LOG_TRACE("SendToParsing", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); for (const auto& [_, formatHandler] : FormatHandlers) { if (formatHandler->HasClients()) { formatHandler->ParseMessages(messages); @@ -759,14 +816,18 @@ void TTopicSession::SendData(TClientsInfo& info) { info.DataArrivedSent = false; if (buffer.empty()) { - LOG_ROW_DISPATCHER_TRACE("Buffer empty"); + YDB_LOG_TRACE("Buffer empty", + {"logPrefix", LogPrefix}); } ui64 dataSize = 0; ui64 eventsSize = info.QueuedRows; if (!info.NextMessageOffset) { - LOG_ROW_DISPATCHER_ERROR("Try SendData() without NextMessageOffset, " << info.ReadActorId - << " unread " << info.QueuedBytes << " DataArrivedSent " << info.DataArrivedSent); + YDB_LOG_ERROR("Try SendData() without NextMessageOffset, unread DataArrivedSent", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"queuedBytes", info.QueuedBytes}, + {"dataArrivedSent", info.DataArrivedSent}); return; } @@ -800,7 +861,10 @@ void TTopicSession::SendData(TClientsInfo& info) { if (buffer.empty()) { event->Record.SetNextMessageOffset(*info.NextMessageOffset); } - LOG_ROW_DISPATCHER_TRACE("SendData to " << info.ReadActorId << ", batch size " << event->Record.MessagesSize()); + YDB_LOG_TRACE("SendData to batch size", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"messagesSize", event->Record.MessagesSize()}); Send(RowDispatcherActorId, event.release()); } while(!buffer.empty()); @@ -816,11 +880,17 @@ void TTopicSession::SendData(TClientsInfo& info) { } void TTopicSession::StartClientSession(TClientsInfo& info) { - LOG_ROW_DISPATCHER_TRACE("StartClientSession, read actor id " << info.ReadActorId << ", offset " << info.GetNextMessageOffset()); + YDB_LOG_TRACE("StartClientSession, read actor id offset", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"getNextMessageOffset", info.GetNextMessageOffset()}); if (ReadSession) { auto offset = info.GetNextMessageOffset(); if (offset && offset <= LastMessageOffset) { - LOG_ROW_DISPATCHER_INFO("New client has less offset (" << offset << ") than the last message (" << LastMessageOffset << "), stop (restart) topic session"); + YDB_LOG_INFO("New client has less offset than the last message stop (restart) topic session", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"lastMessageOffset", LastMessageOffset}); Metrics.RestartSessionByOffsets->Inc(); ++RestartSessionByOffsets; info.RestartSessionByOffsetsByQuery->Inc(); @@ -840,7 +910,12 @@ void TTopicSession::StartClientSession(TClientsInfo& info) { void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto offset = GetOffset(ev->Get()->Record); const auto& source = ev->Get()->Record.GetSource(); - LOG_ROW_DISPATCHER_INFO("New client: read actor id " << ev->Sender.ToString() << ", predicate: " << source.GetPredicate() << ", watermark expr: " << source.GetWatermarkExpr() << ", offset: " << offset); + YDB_LOG_INFO("New client: read actor id watermark", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"predicate", source.GetPredicate()}, + {"expr", source.GetWatermarkExpr()}, + {"offset", offset}); if (!CheckNewClient(ev)) { return; @@ -880,11 +955,17 @@ void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { } void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvStopSession from " << ev->Sender << " topicPath " << ev->Get()->Record.GetSource().GetTopicPath() << " clients count " << Clients.size()); + YDB_LOG_DEBUG("TEvStopSession from topicPath clients count", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"clientsSize", Clients.size()}); auto it = Clients.find(ev->Sender); if (it == Clients.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore TEvStopSession from " << ev->Sender << ", no client"); + YDB_LOG_WARN("Ignore TEvStopSession from no client", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } auto& info = *it->second; @@ -929,7 +1010,10 @@ void TTopicSession::RestartSessionIfOldestClient(const TClientsInfo& info) { if (info.NextMessageOffset >= minMessageOffset) { return; } - LOG_ROW_DISPATCHER_INFO("Client (on StopSession) has less offset (" << info.NextMessageOffset << ") than others clients (" << minMessageOffset << "), stop (restart) topic session"); + YDB_LOG_INFO("Client (on StopSession) has less offset than others clients stop (restart) topic session", + {"logPrefix", LogPrefix}, + {"nextMessageOffset", info.NextMessageOffset}, + {"minMessageOffset", minMessageOffset}); Metrics.RestartSessionByOffsets->Inc(); ++RestartSessionByOffsets; info.RestartSessionByOffsetsByQuery->Inc(); @@ -942,10 +1026,14 @@ void TTopicSession::RestartSessionIfOldestClient(const TClientsInfo& info) { } void TTopicSession::FatalError(const TStatus& status) { - LOG_ROW_DISPATCHER_ERROR("FatalError: " << status.GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"fatalError", status.GetErrorMessage()}); for (auto& [readActorId, info] : Clients) { - LOG_ROW_DISPATCHER_DEBUG("Send TEvSessionError to " << readActorId); + YDB_LOG_DEBUG("Send TEvSessionError", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); SendSessionError(readActorId, status, true); } StopReadSession(); @@ -958,7 +1046,10 @@ void TTopicSession::ThrowFatalError(const TStatus& status) { } void TTopicSession::SendSessionError(TActorId readActorId, TStatus status, bool isFatalError) { - LOG_ROW_DISPATCHER_WARN("SendSessionError to " << readActorId << ", status: " << status.GetErrorMessage()); + YDB_LOG_WARN("SendSessionError", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"status", status.GetErrorMessage()}); auto event = std::make_unique<TEvRowDispatcher::TEvSessionError>(); event->Record.SetStatusCode(status.GetStatus()); event->Record.SetPartitionId(PartitionId); @@ -973,7 +1064,8 @@ void TTopicSession::SendSessionError(TActorId readActorId, TStatus status, bool void TTopicSession::StopReadSession() { if (ReadSession) { - LOG_ROW_DISPATCHER_DEBUG("Close read session"); + YDB_LOG_DEBUG("Close read session", + {"logPrefix", LogPrefix}); ReadSession->Close(TDuration::Zero()); ReadSession.reset(); } @@ -985,7 +1077,9 @@ void TTopicSession::SendDataArrived(TClientsInfo& info) { return; } info.DataArrivedSent = true; - LOG_ROW_DISPATCHER_TRACE("Send TEvNewDataArrived to " << info.ReadActorId); + YDB_LOG_TRACE("Send TEvNewDataArrived", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}); Metrics.InFlyAsyncInputData->Set(1); auto event = std::make_unique<TEvRowDispatcher::TEvNewDataArrived>(); event->Record.SetPartitionId(PartitionId); @@ -1001,7 +1095,8 @@ void TTopicSession::HandleException(const std::exception& e) { } void TTopicSession::SendStatistics() { - LOG_ROW_DISPATCHER_TRACE("SendStatistics"); + YDB_LOG_TRACE("SendStatistics", + {"logPrefix", LogPrefix}); TTopicSessionStatistic sessionStatistic; auto& commonStatistic = sessionStatistic.Common; commonStatistic.QueuedBytes = QueuedBytes; @@ -1046,7 +1141,8 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&) { } void TTopicSession::Handle(NFq::TEvPrivate::TEvGetEventByTimerEvent::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvGetEventByTimerEvent"); + YDB_LOG_TRACE("TEvGetEventByTimerEvent", + {"logPrefix", LogPrefix}); // Workaround for a partition reading bug: // In some cases, the partition may stop delivering new events due to missed notifications or lost subscriptions, // causing the session to stall and not receive further data. To address this, we periodically schedule a timer event @@ -1063,20 +1159,27 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvGetEventByTimerEvent::TPtr&) { bool TTopicSession::CheckNewClient(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto it = Clients.find(ev->Sender); if (it != Clients.end()) { - LOG_ROW_DISPATCHER_ERROR("Such a client already exists"); + YDB_LOG_ERROR("Such a client already exists", + {"logPrefix", LogPrefix}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::INTERNAL_ERROR, TStringBuilder() << "Client with id " << ev->Sender << " already exists"), false); return false; } const auto& source = ev->Get()->Record.GetSource(); if (Config.GetConsumerMode() != TRowDispatcherSettings::EConsumerMode::Without && ConsumerName && ConsumerName != source.GetConsumerName()) { - LOG_ROW_DISPATCHER_INFO("Different consumer, expected " << ConsumerName << ", actual " << source.GetConsumerName() << ", send error"); + YDB_LOG_INFO("Different consumer, expected actual send error", + {"logPrefix", LogPrefix}, + {"consumerName", ConsumerName}, + {"sourceConsumerName", source.GetConsumerName()}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::PRECONDITION_FAILED, TStringBuilder() << "Use the same consumer in all queries via RD (current consumer " << ConsumerName << ")"), false); return false; } if (SkipJsonErrors && SkipJsonErrors != source.GetSkipJsonErrors()) { - LOG_ROW_DISPATCHER_INFO("Different skip json errors mode, expected " << SkipJsonErrors << ", actual " << source.GetSkipJsonErrors() << ", send error"); + YDB_LOG_INFO("Different skip json errors mode, expected actual send error", + {"logPrefix", LogPrefix}, + {"skipJsonErrors", SkipJsonErrors}, + {"sourceSkipJsonErrors", source.GetSkipJsonErrors()}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::PRECONDITION_FAILED, TStringBuilder() << "Use the same skip json errors settings in all queries via RD (current mode " << SkipJsonErrors << ")"), false); return false; } diff --git a/ydb/core/fq/libs/row_dispatcher/ya.make b/ydb/core/fq/libs/row_dispatcher/ya.make index 6e36f7d67aa..5a4171635f9 100644 --- a/ydb/core/fq/libs/row_dispatcher/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/ya.make @@ -12,7 +12,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/metrics ydb/core/fq/libs/row_dispatcher/common diff --git a/ydb/core/fq/libs/test_connection/ya.make b/ydb/core/fq/libs/test_connection/ya.make index ccc86c1ebbb..cfa347a59bd 100644 --- a/ydb/core/fq/libs/test_connection/ya.make +++ b/ydb/core/fq/libs/test_connection/ya.make @@ -13,7 +13,6 @@ PEERDIR( library/cpp/lwtrace library/cpp/xml/document ydb/core/fq/libs/actors - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/test_connection/events diff --git a/ydb/core/grpc_services/query/rpc_execute_query.cpp b/ydb/core/grpc_services/query/rpc_execute_query.cpp index 406b2a6154c..ac448ec5135 100644 --- a/ydb/core/grpc_services/query/rpc_execute_query.cpp +++ b/ydb/core/grpc_services/query/rpc_execute_query.cpp @@ -473,6 +473,12 @@ private: hasTrailingMessage = true; response.mutable_tx_meta()->set_id(kqpResponse.GetTxMeta().id()); } + + if (kqpResponse.HasCommitTimestamp()) { + hasTrailingMessage = true; + response.mutable_commit_timestamp()->set_plan_step(kqpResponse.GetCommitTimestamp().plan_step()); + response.mutable_commit_timestamp()->set_tx_id(kqpResponse.GetCommitTimestamp().tx_id()); + } } if (hasTrailingMessage) { diff --git a/ydb/core/grpc_services/query/rpc_kqp_tx.cpp b/ydb/core/grpc_services/query/rpc_kqp_tx.cpp index 2914aa83aaa..24f30b1154b 100644 --- a/ydb/core/grpc_services/query/rpc_kqp_tx.cpp +++ b/ydb/core/grpc_services/query/rpc_kqp_tx.cpp @@ -181,7 +181,7 @@ public: private: virtual std::pair<TString, TString> GetReqData() const = 0; virtual void Fill(NKikimrKqp::TQueryRequest* req) const = 0; - virtual NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) const = 0; + virtual NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const NKikimrKqp::TQueryResponse& kqpResponse) const = 0; void StateWork(TAutoPtr<IEventHandle>& ev) { try { @@ -233,14 +233,15 @@ private: FillCommonKqpRespFields(record, Request.get()); NYql::TIssues issues; + const NKikimrKqp::TQueryResponse* kqpResponse = nullptr; if (record.HasResponse()) { - const auto& kqpResponse = record.GetResponse(); - const auto& issueMessage = kqpResponse.GetQueryIssues(); + kqpResponse = &record.GetResponse(); + const auto& issueMessage = kqpResponse->GetQueryIssues(); NYql::IssuesFromMessage(issueMessage, issues); Request->RaiseIssues(issues); } - Reply(record.GetYdbStatus(), CreateResult(record.GetYdbStatus(), issues)); + Reply(record.GetYdbStatus(), CreateResult(record.GetYdbStatus(), issues, kqpResponse ? *kqpResponse : NKikimrKqp::TQueryResponse::default_instance())); } void InternalError(const TString& message) { @@ -285,10 +286,14 @@ private: req->MutableTxControl()->set_commit_tx(true); } - NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) const override { + NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const NKikimrKqp::TQueryResponse& kqpResponse) const override { auto result = TEvCommitTransactionRequest::AllocateResult<Ydb::Query::CommitTransactionResponse>(Request); result->set_status(status); NYql::IssuesToMessage(issues, result->mutable_issues()); + if (kqpResponse.HasCommitTimestamp()) { + result->mutable_commit_timestamp()->set_plan_step(kqpResponse.GetCommitTimestamp().plan_step()); + result->mutable_commit_timestamp()->set_tx_id(kqpResponse.GetCommitTimestamp().tx_id()); + } return result; } }; @@ -308,7 +313,8 @@ private: req->SetAction(NKikimrKqp::QUERY_ACTION_ROLLBACK_TX); } - NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) const override { + NProtoBuf::Message* CreateResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const NKikimrKqp::TQueryResponse& kqpResponse) const override { + Y_UNUSED(kqpResponse); auto result = TEvRollbackTransactionRequest::AllocateResult<Ydb::Query::RollbackTransactionResponse>(Request); result->set_status(status); NYql::IssuesToMessage(issues, result->mutable_issues()); diff --git a/ydb/core/grpc_services/rpc_cluster_state.cpp b/ydb/core/grpc_services/rpc_cluster_state.cpp index a95f711e82e..2e571029292 100644 --- a/ydb/core/grpc_services/rpc_cluster_state.cpp +++ b/ydb/core/grpc_services/rpc_cluster_state.cpp @@ -110,7 +110,7 @@ public: {{ "Sid", "IsEnabled", "IsLockedOut", "CreatedAt", "LastSuccessfulAttemptAt", "LastFailedAttemptAt", "FailedAttemptCount", "PasswordHash" }, ".sys/auth_users", true }, {{ "NodeId", "QueryId", {"Query", true}, "AccessCount", "CompiledAt", "UserSID", "LastAccessedAt", "CompilationDurationMs", "Warnings", "Metadata", "QueryType", "Syntax" }, ".sys/compile_cache_queries" }, {{ "BridgeSyncRunning", "GroupId", "Generation", "ErasureSpecies", "BoxId", "StoragePoolId", "EncryptionMode", "LifeCyclePhase", "AllocatedSize", "AvailableSize", "SeenOperational", "PutTabletLogLatency", "PutUserDataLatency", "GetFastLatency", "LayoutCorrect", "OperatingStatus", "ExpectedStatus", "ProxyGroupId", "BridgePileId", "GroupSizeInUnits", "BridgeSyncStage", "BridgeDataSyncProgress", "BridgeDataSyncErrors", "BridgeSyncLastError", "BridgeSyncLastErrorTimestamp", "BridgeSyncFirstErrorTimestamp", "BridgeSyncErrorCount" }, ".sys/ds_groups" }, - {{ "NodeId", "PDiskId", "Type", "Kind", "Path", "Guid", "BoxId", "SharedWithOS", "ReadCentric", "AvailableSize", "TotalSize", "Status", "StatusChangeTimestamp", "ExpectedSlotCount", "NumActiveSlots", "DecommitStatus", "State", "SlotSizeInUnits", "MaintenanceStatus" }, ".sys/ds_pdisks" }, + {{ "NodeId", "PDiskId", "Type", "Kind", "Path", "Guid", "BoxId", "SharedWithOS", "ReadCentric", "AvailableSize", "TotalSize", "Status", "StatusChangeTimestamp", "ExpectedSlotCount", "ExpectedSlotSize", "NumActiveSlots", "DecommitStatus", "State", "SlotSizeInUnits", "MaintenanceStatus" }, ".sys/ds_pdisks" }, {{ "BoxId", "StoragePoolId", "Name", "Generation", "ErasureSpecies", "VDiskKind", "Kind", "NumGroups", "EncryptionMode", "SchemeshardId", "PathId", "DefaultGroupSizeInUnits" }, ".sys/ds_storage_pools" }, {{ "AvailableGroupsToCreate", "AvailableSizeToCreate", "PDiskFilter", "ErasureSpecies", "CurrentGroupsCreated", "CurrentAllocatedSize", "CurrentAvailableSize" }, ".sys/ds_storage_stats" }, {{ "DiskSpace", "State", "NodeId", "PDiskId", "VSlotId", "GroupId", "GroupGeneration", "FailDomain", "VDisk", "AllocatedSize", "AvailableSize", "Status", "Kind", "FailRealm", "Replicated", "PhantomOnly" }, ".sys/ds_vslots" }, diff --git a/ydb/core/grpc_services/rpc_config.cpp b/ydb/core/grpc_services/rpc_config.cpp index bda5c4513d0..8bd0804fc98 100644 --- a/ydb/core/grpc_services/rpc_config.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -227,7 +227,15 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb newDrive->set_shared_with_os(drive.GetSharedWithOs()); newDrive->set_read_centric(drive.GetReadCentric()); newDrive->set_kind(drive.GetKind()); - newDrive->set_expected_slot_count(hostConfig.GetDefaultHostPDiskConfig().GetExpectedSlotCount()); + const auto& pdiskConfig = drive.HasPDiskConfig() + ? drive.GetPDiskConfig() + : hostConfig.GetDefaultHostPDiskConfig(); + if (pdiskConfig.GetExpectedSlotSize()) { + newDrive->set_expected_slot_size(pdiskConfig.GetExpectedSlotSize()); + newDrive->set_max_slots(pdiskConfig.GetMaxSlots()); + } else { + newDrive->set_expected_slot_count(pdiskConfig.GetExpectedSlotCount()); + } } } auto boxes = boxStatus.GetBox(); diff --git a/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp b/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp index 9f7af45356a..7af2113867d 100644 --- a/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp +++ b/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp @@ -23,13 +23,6 @@ constexpr TStringBuf NotImplementedMessage = "Topic deferred publish is not impl constexpr TStringBuf DisabledMessage = "Topic deferred publish is not enabled"; constexpr TStringBuf AuthenticationRequiredMessage = "Authentication is required"; -TString GetSerializedUserToken(const IRequestOpCtx* request) { - if (request == nullptr) { - return {}; - } - return request->GetSerializedToken(); -} - TString GetUserSID(const IRequestOpCtx* request) { if (request == nullptr) { return BUILTIN_ACL_NO_USER_SID; @@ -423,7 +416,6 @@ public: *database, protoRequest->int_publication_id(), NPQ::NDeferredPublish::EFinalizePublicationOp::Publish, - GetSerializedUserToken(Request.get()), callerSid)); Become(&TPublishRequestActor::StateFunc); } @@ -492,7 +484,6 @@ public: *database, protoRequest->int_publication_id(), NPQ::NDeferredPublish::EFinalizePublicationOp::Cancel, - GetSerializedUserToken(Request.get()), callerSid)); Become(&TCancelPublicationRequestActor::StateFunc); } diff --git a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp index 8b2dcd59d27..696cd100a5f 100644 --- a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/services/lib/actors/pq_schema_actor.h> #include <ydb/core/persqueue/public/schema/common.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -53,7 +55,10 @@ public: , CleanupPolicy(cleanupPolicy) , TimestampType(timestampType) { - KAFKA_LOG_D("Alter configs actor. DatabaseName: " << databaseName << ". TopicPath: " << TopicPath); + YDB_LOG_DEBUG("Alter configs actor", + {LogPrefix()}, + {"databaseName", databaseName}, + {"topicPath", TopicPath}); }; ~TAlterConfigsActor() = default; @@ -117,7 +122,9 @@ NActors::IActor* CreateKafkaAlterConfigsActor( void TKafkaAlterConfigsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); @@ -267,7 +274,9 @@ void TKafkaAlterConfigsActor::ProcessValidateOnly(const NActors::TActorContext& response->Responses.push_back(responseResource); } - KAFKA_LOG_D("KLACK TKafkaAlterConfigsActor::ProcessValidateOnly: CorrelationId == " << CorrelationId); + YDB_LOG_DEBUG("KLACK TKafkaAlterConfigsActor::ProcessValidateOnly: CorrelationId", + {LogPrefix()}, + {"correlationId", CorrelationId}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, NONE_ERROR)); Die(ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp index d4b62e0a92b..1ebf566e588 100644 --- a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp @@ -1,5 +1,7 @@ #include "kafka_balancer_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NKikimr; @@ -38,7 +40,8 @@ void TKafkaBalancerActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr& } void TKafkaBalancerActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("Pass away."); + YDB_LOG_DEBUG("Pass away", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -96,7 +99,8 @@ void TKafkaBalancerActor::RequestFullRetry() { void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Cookie != KqpReqCookie) { - KAFKA_LOG_CRIT("Unexpected cookie in TEvQueryResponse."); + YDB_LOG_CRIT("Unexpected cookie in TEvQueryResponse", + {LogPrefix()}); return; } @@ -111,7 +115,9 @@ void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const } if (CurrentTxAbortRetryNumber < TX_ABORT_RETRY_MAX_COUNT) { - KAFKA_LOG_ERROR(TStringBuilder() << "Retry after tx aborted. CurrentTxAbortRetryNumber# " << static_cast<int>(CurrentTxAbortRetryNumber)); + YDB_LOG_ERROR("Retry after tx aborted", + {LogPrefix()}, + {"currentTxAbortRetryNumber", static_cast<int>(CurrentTxAbortRetryNumber)}); RequestFullRetry(); Die(ctx); return; @@ -135,7 +141,8 @@ void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const } void TKafkaBalancerActor::HandleResponse(NKqp::TEvKqp::TEvQueryResponse::TPtr ev, const TActorContext& ctx) { - KAFKA_LOG_I(TStringBuilder() << "Handle kqp response"); + YDB_LOG_INFO("Handle kqp response", + {LogPrefix()}); switch (RequestType) { case JOIN_GROUP: @@ -151,7 +158,8 @@ void TKafkaBalancerActor::HandleResponse(NKqp::TEvKqp::TEvQueryResponse::TPtr ev HeartbeatNextStep(ev, ctx); break; default: - KAFKA_LOG_ERROR("Unknown RequestType in TEvCreateSessionResponse"); + YDB_LOG_ERROR("Unknown RequestType in TEvCreateSessionResponse", + {LogPrefix()}); Die(ctx); break; } @@ -221,7 +229,8 @@ void TKafkaBalancerActor::JoinGroupNextStep( break; } default: - KAFKA_LOG_CRIT("JOIN_GROUP: Unexpected step" ); + YDB_LOG_CRIT("JOIN_GROUP: Unexpected step", + {LogPrefix()}); SendJoinGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -269,7 +278,8 @@ void TKafkaBalancerActor::SyncGroupNextStep( } default: { - KAFKA_LOG_CRIT("SYNC_GROUP: Unexpected step"); + YDB_LOG_CRIT("SYNC_GROUP: Unexpected step", + {LogPrefix()}); SendSyncGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Failed to get assignments from master"); break; } @@ -312,7 +322,8 @@ void TKafkaBalancerActor::HeartbeatNextStep( } default: { - KAFKA_LOG_CRIT("HEARTBEAT: Unexpected step"); + YDB_LOG_CRIT("HEARTBEAT: Unexpected step", + {LogPrefix()}); SendHeartbeatResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -340,7 +351,8 @@ void TKafkaBalancerActor::LeaveGroupStep( } default: { - KAFKA_LOG_CRIT("LEAVE_GROUP: Unexpected step"); + YDB_LOG_CRIT("LEAVE_GROUP: Unexpected step", + {LogPrefix()}); SendLeaveGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -382,17 +394,18 @@ void TKafkaBalancerActor::JoinStepCreateNewOrJoinGroup(NKqp::TEvKqp::TEvQueryRes } if (groupStatus->Exists) { - KAFKA_LOG_I(TStringBuilder() << "Check group before join status." - "\n memberId: " << MemberId << - "\n instanceId: " << InstanceId << - "\n group: " << GroupId << - "\n exists: " << groupStatus->Exists << - "\n protocolType: " << groupStatus->ProtocolType << - "\n protocolName: " << groupStatus->ProtocolName << - "\n master: " << groupStatus->MasterId << - "\n generation: " << groupStatus->Generation << - "\n lastSuccessGeneration: " << groupStatus->LastSuccessGeneration << - "\n state: " << groupStatus->State); + YDB_LOG_INFO("Check group before join status", + {LogPrefix()}, + {"memberId", MemberId}, + {"instanceId", InstanceId}, + {"group", GroupId}, + {"exists", groupStatus->Exists}, + {"protocolType", groupStatus->ProtocolType}, + {"protocolName", groupStatus->ProtocolName}, + {"master", groupStatus->MasterId}, + {"generation", groupStatus->Generation}, + {"lastSuccessGeneration", groupStatus->LastSuccessGeneration}, + {"state", groupStatus->State}); } if (!groupStatus->Exists) { @@ -631,13 +644,19 @@ void TKafkaBalancerActor::JoinStepWaitMembersAndChooseProtocol(NKqp::TEvKqp::TEv ui32 memberRebalanceTimeoutMs = prevGenerationMembersAndTimeoutsIt->second.RebalanceTimeoutMs; const TInstant& memberHeartbeatDeadline = prevGenerationMembersAndTimeoutsIt->second.HeartbeatDeadline; if (AllWorkerStates.count(prevGenerationMembersAndTimeoutsIt->first) == 1) { - KAFKA_LOG_D(TStringBuilder() << "Waited member connected: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Waited member", + {LogPrefix()}, + {"connected", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else if ((RebalanceStartTime + TDuration::MilliSeconds(memberRebalanceTimeoutMs)) < now) { - KAFKA_LOG_D(TStringBuilder() << "Rebalance deadline: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Rebalance", + {LogPrefix()}, + {"deadline", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else if (memberHeartbeatDeadline < now) { - KAFKA_LOG_D(TStringBuilder() << "Waited member connect session deadline: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Waited member connect session", + {LogPrefix()}, + {"deadline", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else { ++prevGenerationMembersAndTimeoutsIt; @@ -645,7 +664,9 @@ void TKafkaBalancerActor::JoinStepWaitMembersAndChooseProtocol(NKqp::TEvKqp::TEv } if (WaitedMemberIdsAndTimeouts.size() != 0) { - KAFKA_LOG_D(TStringBuilder() << "Members waited count# : " << WaitedMemberIdsAndTimeouts.size()); + YDB_LOG_DEBUG("Members waited", + {LogPrefix()}, + {"count", WaitedMemberIdsAndTimeouts.size()}); WaitedMemberIdsAndTimeouts.clear(); AllWorkerStates.clear(); WorkerStatesPaginationMemberId = ""; @@ -1321,7 +1342,9 @@ NYdb::TParamsBuilder TKafkaBalancerActor::BuildAssignmentsParams() { auto& assignmentList = params.AddParam("$Assignments").BeginList(); - KAFKA_LOG_D(TStringBuilder() << "Assignments count: " << SyncGroupRequestData->Assignments.size()); + YDB_LOG_DEBUG("Assignments", + {LogPrefix()}, + {"count", SyncGroupRequestData->Assignments.size()}); for (auto& assignment: SyncGroupRequestData->Assignments) { assignmentList.AddListItem() @@ -1446,7 +1469,8 @@ void TKafkaBalancerActor::SendResponseFail(const TActorContext& ctx, EKafkaError } void TKafkaBalancerActor::SendJoinGroupResponseOk(const TActorContext& ctx, ui64 correlationId) { - KAFKA_LOG_I(TStringBuilder() << "JOIN_GROUP success."); + YDB_LOG_INFO("JOIN_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TJoinGroupResponseData>(); response->ProtocolType = ProtocolType; @@ -1474,7 +1498,8 @@ void TKafkaBalancerActor::SendJoinGroupResponseOk(const TActorContext& ctx, ui64 } void TKafkaBalancerActor::SendSyncGroupResponseOk(const TActorContext& ctx, ui64 correlationId) { - KAFKA_LOG_I(TStringBuilder() << "SYNC_GROUP success."); + YDB_LOG_INFO("SYNC_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TSyncGroupResponseData>(); response->ProtocolType = ProtocolType; response->ProtocolName = Protocol; @@ -1486,7 +1511,8 @@ void TKafkaBalancerActor::SendSyncGroupResponseOk(const TActorContext& ctx, ui64 } void TKafkaBalancerActor::SendLeaveGroupResponseOk(const TActorContext& ctx, ui64 corellationId) { - KAFKA_LOG_I(TStringBuilder() << "LEAVE_GROUP success."); + YDB_LOG_INFO("LEAVE_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = EKafkaErrors::NONE_ERROR; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, EKafkaErrors::NONE_ERROR)); @@ -1496,7 +1522,8 @@ void TKafkaBalancerActor::SendLeaveGroupResponseOk(const TActorContext& ctx, ui6 void TKafkaBalancerActor::SendHeartbeatResponseOk(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error) { - KAFKA_LOG_I(TStringBuilder() << "HEARTBEAT success."); + YDB_LOG_INFO("HEARTBEAT success", + {LogPrefix()}); auto response = std::make_shared<THeartbeatResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1508,7 +1535,9 @@ void TKafkaBalancerActor::SendJoinGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "JOIN_GROUP failed. reason# " << message); + YDB_LOG_ERROR("JOIN_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TJoinGroupResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1519,7 +1548,9 @@ void TKafkaBalancerActor::SendSyncGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "SYNC_GROUP failed. reason# " << message); + YDB_LOG_ERROR("SYNC_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TSyncGroupResponseData>(); response->ErrorCode = error; @@ -1534,7 +1565,9 @@ void TKafkaBalancerActor::SendLeaveGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "LEAVE_GROUP failed. reason# " << message); + YDB_LOG_ERROR("LEAVE_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1545,17 +1578,22 @@ void TKafkaBalancerActor::SendHeartbeatResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "HEARTBEAT failed. reason# " << message); + YDB_LOG_ERROR("HEARTBEAT failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<THeartbeatResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); Die(ctx); } -TString TKafkaBalancerActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaBalancerActor: GroupId# " << GroupId << ", MemberId# " << MemberId << ", CurrentStep# " << static_cast<int>(CurrentStep) << ". "; - return sb; +NActors::NStructuredLog::TStructuredMessage TKafkaBalancerActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaBalancerActor"}, + {"selfId", SelfId()}, + {"groupId", GroupId}, + {"memberId", MemberId}, + {"currentStep", static_cast<int>(CurrentStep)}); } } // namespace NKafka diff --git a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h index 6e8a4b6c11b..f4a43e9e688 100644 --- a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h @@ -143,7 +143,9 @@ public: InstanceId = JoinGroupRequestData->GroupInstanceId.value_or(""); MemberId = JoinGroupRequestData->MemberId.value_or(""); - KAFKA_LOG_D(TStringBuilder() << "JOIN_GROUP request. MemberId# " << MemberId); + YDB_LOG_DEBUG_COMP(NKikimrServices::KAFKA_PROXY, "JOIN_GROUP request", + {LogPrefix()}, + {"memberId", MemberId}); if (JoinGroupRequestData->SessionTimeoutMs) { SessionTimeoutMs = JoinGroupRequestData->SessionTimeoutMs; @@ -308,7 +310,7 @@ private: void SendLeaveGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message = ""); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); void SendResponseFail(const TActorContext& ctx, EKafkaErrors error, const TString& message); std::optional<TGroupStatus> ParseGroupState(NKqp::TEvKqp::TEvQueryResponse::TPtr ev); diff --git a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp index cfe1360802e..ec90091f854 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/kafka_proxy/kafka_constants.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -223,10 +225,11 @@ public: databaseName) , PartionsNumber(partitionsNumber) { - KAFKA_LOG_D( - "Create partitions actor. DatabaseName: " << databaseName << - ". TopicPath: " << TopicPath << - ". PartitionsNumber: " << PartionsNumber); + YDB_LOG_DEBUG("Create partitions actor", + {LogPrefix()}, + {"databaseName", databaseName}, + {"topicPath", TopicPath}, + {"partitionsNumber", PartionsNumber}); }; void ModifyPersqueueConfig( @@ -257,7 +260,9 @@ NActors::IActor* CreateKafkaCreatePartitionsActor( } void TKafkaCreatePartitionsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp index 62d858ef481..b33af01aa80 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/kafka_proxy/kafka_constants.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -58,7 +60,9 @@ NActors::IActor* CreateKafkaCreateTopicsActor( } void TKafkaCreateTopicsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); @@ -168,7 +172,10 @@ void TKafkaCreateTopicsActor::Bootstrap(const NActors::TActorContext& ctx) { void TKafkaCreateTopicsActor::Handle(const NKikimr::NPQ::NSchema::TEvSchemaResponse::TPtr& ev) { auto eventPtr = ev->Release(); - KAFKA_LOG_D(TStringBuilder() << "Create topics actor. Topic's " << eventPtr->Path << " response received." << std::to_string(eventPtr->Status)); + YDB_LOG_DEBUG("Create topics actor. Topic's response received", + {LogPrefix()}, + {"path", eventPtr->Path}, + {"status", std::to_string(eventPtr->Status)}); EKafkaErrors status; switch(eventPtr->Status) { diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp index 82263c37abf..73456c8455b 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/core/ydb_convert/topic_description.h> #include <ydb/core/ydb_convert/ydb_convert.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -105,7 +107,9 @@ void TKafkaDescribeTopicActor::HandleCacheNavigateResponse(NKikimr::TEvTxProxySc void TKafkaDescribeConfigsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); THashSet<TString> requestedTopics{}; diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp index 51a98a62a81..16c879adcc2 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp @@ -17,6 +17,8 @@ #include "kafka_describe_groups_actor.h" #include "kafka_state_name_to_int.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -40,7 +42,8 @@ void TKafkaDescribeGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { new IEventHandle(SelfId(), SelfId(), wakeup.release()) ); } else { - KAFKA_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set."); + YDB_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set", + {LogPrefix()}); TDescribeGroupsResponseData groupsDescriptionResponseWithError; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, @@ -50,7 +53,8 @@ void TKafkaDescribeGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } void TKafkaDescribeGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); InitedTablesCount++; if (InitedTablesCount == TABLES_TO_INIT_COUNT) { StartKqpSession(ctx); @@ -59,7 +63,8 @@ void TKafkaDescribeGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared: void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -69,9 +74,11 @@ void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::T } void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP DescribeGroups request"); + YDB_LOG_DEBUG("Received query response from KQP DescribeGroups request", + {LogPrefix()}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, *error); Die(ctx); return; @@ -80,14 +87,17 @@ void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, } void TKafkaDescribeGroupsActor::Handle(TEvents::TEvWakeup::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_W("Sending fail response because of request timeout " << WAIT_REQUESTS_SECONDS << " sec"); + YDB_LOG_WARN("Sending fail response because of request timeout sec", + {LogPrefix()}, + {"timeout", WAIT_REQUESTS_SECONDS}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, BuildResponse(), EKafkaErrors::REQUEST_TIMED_OUT)); Die(ctx); } void TKafkaDescribeGroupsActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -95,15 +105,20 @@ void TKafkaDescribeGroupsActor::Die(const TActorContext &ctx) { } void TKafkaDescribeGroupsActor::StartKqpSession(const TActorContext& ctx) { - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } void TKafkaDescribeGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryResponse& response, const TActorContext& ctx) { - KAFKA_LOG_D("Handling Select Response for DescribeGroups. SELECT result size: " << response.Record.GetResponse().GetYdbResults().size()); + YDB_LOG_DEBUG("Handling Select Response for DescribeGroups. SELECT result", + {LogPrefix()}, + {"size", response.Record.GetResponse().GetYdbResults().size()}); if (response.Record.GetResponse().GetYdbResults().size() != 2) { - TString errorMessage = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected 2, got " << response.Record.GetResponse().GetYdbResults().size() << "."; - KAFKA_LOG_W(errorMessage); + YDB_LOG_WARN("KQP returned wrong number of result sets size!=2 on SELECT query", + {LogPrefix()}, + {"size", response.Record.GetResponse().GetYdbResults().size()}); return; } ParseGroupDescriptionMetadata(response); @@ -115,9 +130,11 @@ void TKafkaDescribeGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQuer } void TKafkaDescribeGroupsActor::ParseGroupDescriptionMetadata(const NKqp::TEvKqp::TEvQueryResponse& response) { - KAFKA_LOG_D("Parsing Groups metadata"); + YDB_LOG_DEBUG("Parsing Groups metadata", + {LogPrefix()}); ParseGroupMetadata(response); - KAFKA_LOG_D("Parsing Members metadata"); + YDB_LOG_DEBUG("Parsing Members metadata", + {LogPrefix()}); ParseMembersMetadata(response); } @@ -181,7 +198,9 @@ NYdb::TParams TKafkaDescribeGroupsActor::BuildSelectParams() { params.AddParam("$Database").Utf8(DatabasePath).Build(); auto& groupIds = params.AddParam("$GroupIds").BeginList(); - KAFKA_LOG_D(TStringBuilder() << "Groups count: " << DescribeGroupsRequestData->Groups.size()); + YDB_LOG_DEBUG("Groups", + {LogPrefix()}, + {"count", DescribeGroupsRequestData->Groups.size()}); for (auto& groupId: DescribeGroupsRequestData->Groups) { groupIds.AddListItem().Utf8(*groupId); @@ -245,9 +264,14 @@ void TKafkaDescribeGroupsActor::SendFailResponse(EKafkaErrors errorCode, const s GroupIdToDescription[*groupId].ErrorCode = errorCode; } if (errorMessage.has_value()) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } Send(Context->ConnectionId, diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h index 08b93e21223..1211fdb18fe 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h @@ -54,8 +54,11 @@ public: void Bootstrap(const NActors::TActorContext& ctx); -TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaDescribeGroupsActor{DatabasePath=" << DatabasePath << "}: "; +NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaDescribeGroupsActor"}, + {"selfId", SelfId()}, + {"databasePath", DatabasePath}); } struct TDescribeGroupsKqpQuery { diff --git a/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp index 14192f8392f..d7dc773c313 100644 --- a/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp @@ -17,6 +17,8 @@ #include "actors.h" #include "kafka_fetch_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -53,8 +55,11 @@ void TKafkaFetchActor::Bootstrap(const NActors::TActorContext& ctx) { void TKafkaFetchActor::SendFetchRequests(const TActorContext& ctx) { Response->Responses.resize(FetchRequestData->Topics.size()); - KAFKA_LOG_D(TStringBuilder() << "Fetch actor: New request. DatabasePath: " << Context->DatabasePath - << " MaxWaitMs: " << FetchRequestData->MaxWaitMs << " MaxBytes: " << FetchRequestData->MaxBytes); + YDB_LOG_DEBUG("Fetch actor: New request", + {LogPrefix()}, + {"databasePath", Context->DatabasePath}, + {"maxWaitMs", FetchRequestData->MaxWaitMs}, + {"maxBytes", FetchRequestData->MaxBytes}); for (size_t topicIndex = 0; topicIndex < Response->Responses.size(); topicIndex++) { auto partPQRequests = PrepareFetchRequestData(topicIndex); auto ruPerRequest = topicIndex == 0 && Context->Config.GetMeteringV2Enabled(); @@ -90,10 +95,12 @@ TVector<NKikimr::NPQ::TPartitionFetchRequest> TKafkaFetchActor::PrepareFetchRequ topicKafkaResponse.Partitions.resize(topicKafkaRequest.Partitions.size()); for (size_t partIndex = 0; partIndex < topicKafkaRequest.Partitions.size(); partIndex++) { auto& partKafkaRequest = topicKafkaRequest.Partitions[partIndex]; - KAFKA_LOG_D(TStringBuilder() << "Fetch actor: New request. Topic: " << topicKafkaRequest.Topic.value() - << " Partition: " << partKafkaRequest.Partition - << " FetchOffset: " << partKafkaRequest.FetchOffset - << " PartitionMaxBytes: " << partKafkaRequest.PartitionMaxBytes); + YDB_LOG_DEBUG("Fetch actor: New request", + {LogPrefix()}, + {"topic", topicKafkaRequest.Topic.value()}, + {"partition", partKafkaRequest.Partition}, + {"fetchOffset", partKafkaRequest.FetchOffset}, + {"partitionMaxBytes", partKafkaRequest.PartitionMaxBytes}); auto& partPQRequest = partPQRequests[partIndex]; partPQRequest.Topic = NormalizePath(Context->DatabasePath, topicKafkaRequest.Topic.value()); // FIXME(savnik): handle empty topic partPQRequest.Partition = partKafkaRequest.Partition; @@ -126,7 +133,8 @@ size_t TKafkaFetchActor::CheckTopicIndex(const NKikimr::TEvPQ::TEvFetchResponse: Y_DEBUG_ABORT_UNLESS(topicIt != TopicIndexes.end()); if (topicIt == TopicIndexes.end()) { - KAFKA_LOG_ERROR("Fetch actor: Received unexpected TEvFetchResponse. Ignoring. Expect malformed/incompled fetch reply."); + YDB_LOG_ERROR("Fetch actor: Received unexpected TEvFetchResponse. Ignoring. Expect malformed/incompled fetch reply", + {LogPrefix()}); return std::numeric_limits<size_t>::max(); } @@ -139,7 +147,11 @@ void TKafkaFetchActor::HandleErrorResponse(const NKikimr::TEvPQ::TEvFetchRespons for (auto& partitionResponse : topicResponse.Partitions) { partitionResponse.ErrorCode = code; } - KAFKA_LOG_ERROR("Fetch actor: Failed to get responses for topic: " << topicResponse.Topic << ". Code: " << ev->Get()->Status << ". Reason: " << ev->Get()->Message); + YDB_LOG_ERROR("Fetch actor: Failed to get responses", + {LogPrefix()}, + {"topic", topicResponse.Topic}, + {"code", ev->Get()->Status}, + {"reason", ev->Get()->Message}); } void TKafkaFetchActor::HandleSuccessResponse(const NKikimr::TEvPQ::TEvFetchResponse::TPtr& ev, TFetchResponseData::TFetchableTopicResponse& topicResponse, const TActorContext& ctx) { @@ -155,10 +167,12 @@ void TKafkaFetchActor::HandleSuccessResponse(const NKikimr::TEvPQ::TEvFetchRespo partKafkaResponse.ErrorCode = ConvertErrorCode(partPQResponse.GetReadResult().GetErrorCode()); if (partPQResponse.GetReadResult().GetErrorCode() != NPersQueue::NErrorCode::EErrorCode::OK) { - KAFKA_LOG_D("Fetch actor: Failed to get responses for topic: " << topicResponse.Topic << - ", partition: " << partPQResponse.GetPartition() << - ". Code: " << static_cast<size_t>(partPQResponse.GetReadResult().GetErrorCode()) << - ". Reason: " + partPQResponse.GetReadResult().GetErrorReason()); + YDB_LOG_DEBUG("Fetch actor: Failed to get responses", + {LogPrefix()}, + {"topic", topicResponse.Topic}, + {"partition", partPQResponse.GetPartition()}, + {"code", static_cast<size_t>(partPQResponse.GetReadResult().GetErrorCode())}, + {"reason", partPQResponse.GetReadResult().GetErrorReason()}); } partKafkaResponse.HighWatermark = partPQResponse.GetReadResult().GetMaxOffset(); @@ -206,9 +220,14 @@ void TKafkaFetchActor::FillRecordsBatch(const NKikimrClient::TPersQueueFetchResp //recordsBatch.Attributes https://kafka.apache.org/documentation/#recordbatch recordsBatch.BatchLength = recordsBatch.Size(TKafkaRecordBatch::MessageMeta::PresentVersions.Max) - BatchFirstTwoFieldsSize; - KAFKA_LOG_D("Fetch actor: RecordBatch info. BaseOffset: " << recordsBatch.BaseOffset << ", LastOffsetDelta: " << recordsBatch.LastOffsetDelta << - ", BaseTimestamp: " << recordsBatch.BaseTimestamp << ", MaxTimestamp: " << recordsBatch.MaxTimestamp << - ", BaseSequence: " << recordsBatch.BaseSequence << ", BatchLength: " << recordsBatch.BatchLength); + YDB_LOG_DEBUG("Fetch actor: RecordBatch info", + {LogPrefix()}, + {"baseOffset", recordsBatch.BaseOffset}, + {"lastOffsetDelta", recordsBatch.LastOffsetDelta}, + {"baseTimestamp", recordsBatch.BaseTimestamp}, + {"maxTimestamp", recordsBatch.MaxTimestamp}, + {"baseSequence", recordsBatch.BaseSequence}, + {"batchLength", recordsBatch.BatchLength}); kafkaBatchRecords += WriteKafkaRecordBatch(recordsBatch); recordsBatch = TKafkaRecordBatch(); @@ -246,15 +265,17 @@ void TKafkaFetchActor::FillRecordsBatch(const NKikimrClient::TPersQueueFetchResp const bool isKafkaBatch = dataChunk.HasCodec() && dataChunk.GetCodec() == KafkaBatchCodec(); - KAFKA_LOG_D("Fetch actor: Kafka record candidate. ResultsCount=" << partPQResponse.GetReadResult().GetResult().size() - << ", Offset=" << result.GetOffset() - << ", IsBatch=" << result.GetIsBatch() - << ", MessageCount=" << result.GetLogicalMessageCount() - << ", ChunkType=" << static_cast<int>(dataChunk.GetChunkType()) - << ", Codec=" << (dataChunk.HasCodec() ? static_cast<int>(dataChunk.GetCodec()) : -1) - << ", KafkaBatchCodec=" << static_cast<int>(KafkaBatchCodec()) - << ", DataSize=" << dataChunk.GetData().size() - << ", IsKafkaBatch=" << isKafkaBatch); + YDB_LOG_DEBUG("Fetch actor: Kafka record candidate", + {LogPrefix()}, + {"resultsCount", partPQResponse.GetReadResult().GetResult().size()}, + {"offset", result.GetOffset()}, + {"isBatch", result.GetIsBatch()}, + {"messageCount", result.GetLogicalMessageCount()}, + {"chunkType", static_cast<int>(dataChunk.GetChunkType())}, + {"codec", (dataChunk.HasCodec() ? static_cast<int>(dataChunk.GetCodec()) : -1)}, + {"kafkaBatchCodec", static_cast<int>(KafkaBatchCodec())}, + {"dataSize", dataChunk.GetData().size()}, + {"isKafkaBatch", isKafkaBatch}); if (isKafkaBatch) { flushRecordsBatch(); diff --git a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp index d9b37601029..ef280c01e53 100644 --- a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp @@ -2,6 +2,8 @@ #include <ydb/core/kafka_proxy/kafka_events.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -9,10 +11,10 @@ NActors::IActor* CreateKafkaFindCoordinatorActor(const TContext::TPtr context, c return new TKafkaFindCoordinatorActor(context, correlationId, message); } -TString TKafkaFindCoordinatorActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaFindCoordinatorActor " << SelfId().ToString() << ": "; - return sb; +NActors::NStructuredLog::TStructuredMessage TKafkaFindCoordinatorActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaFindCoordinatorActor"}, + {"selfId", SelfId()}); } void TKafkaFindCoordinatorActor::Bootstrap(const NActors::TActorContext& ctx) { @@ -34,7 +36,9 @@ void TKafkaFindCoordinatorActor::SendResponseOkAndDie(const TString& host, i32 p TFindCoordinatorResponseData::TPtr response = std::make_shared<TFindCoordinatorResponseData>(); for (auto coordinatorKey: Message->CoordinatorKeys) { - KAFKA_LOG_I("FIND_COORDINATOR incoming request for group# " << coordinatorKey); + YDB_LOG_INFO("FIND_COORDINATOR incoming request", + {LogPrefix()}, + {"group", coordinatorKey}); TFindCoordinatorResponseData::TCoordinator coordinator; coordinator.ErrorCode = NONE_ERROR; @@ -51,7 +55,11 @@ void TKafkaFindCoordinatorActor::SendResponseOkAndDie(const TString& host, i32 p response->Port = port; response->NodeId = nodeId; - KAFKA_LOG_D("FIND_COORDINATOR response. Host#: " << host << ", Port#: " << port << ", NodeId# " << nodeId); + YDB_LOG_DEBUG("FIND_COORDINATOR response", + {LogPrefix()}, + {"host", host}, + {"port", port}, + {"nodeId", nodeId}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, static_cast<EKafkaErrors>(response->ErrorCode))); Die(ctx); @@ -61,7 +69,9 @@ void TKafkaFindCoordinatorActor::SendResponseFailAndDie(EKafkaErrors error, cons TFindCoordinatorResponseData::TPtr response = std::make_shared<TFindCoordinatorResponseData>(); for (auto coordinatorKey: Message->CoordinatorKeys) { - KAFKA_LOG_CRIT("FIND_COORDINATOR request failed. Reason# " << message); + YDB_LOG_CRIT("FIND_COORDINATOR request failed", + {LogPrefix()}, + {"reason", message}); TFindCoordinatorResponseData::TCoordinator coordinator; coordinator.ErrorCode = error; diff --git a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h index 81c3c588fb8..77e5c39135c 100644 --- a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h @@ -22,7 +22,7 @@ private: void SendResponseOkAndDie(const TString& host, i32 port, ui64 nodeId, const NActors::TActorContext& ctx); void SendResponseFailAndDie(EKafkaErrors error, const TString& message, const NActors::TActorContext& ctx); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); private: const TContext::TPtr Context; diff --git a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp index 9da00537985..a64ca707f26 100644 --- a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp @@ -21,6 +21,8 @@ #include <util/datetime/base.h> #include <regex> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { NActors::IActor* CreateKafkaInitProducerIdActor(const TContext::TPtr context, const ui64 correlationId, const TMessagePtr<TInitProducerIdRequestData>& message) { @@ -60,7 +62,8 @@ namespace NKafka { return; } Kqp = std::make_unique<TKqpTxHelper>(Context->ResourceDatabasePath); - KAFKA_LOG_D("Bootstrapping actor for transactional producer. Sending init table request to KQP."); + YDB_LOG_DEBUG("Bootstrapping actor for transactional producer. Sending init table request to KQP", + {LogPrefix()}); if (Context->ResourceDatabasePath == AppData(ctx)->TenantName) { Kqp->SendInitTableRequest(ctx, NKikimr::NGRpcProxy::V1::TTransactionalProducersInitManager::GetInstant()); } else { @@ -86,7 +89,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); Kqp->SendCreateSessionRequest(ctx); } @@ -102,7 +106,10 @@ namespace NKafka { void TKafkaInitProducerIdActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Cookie != KqpReqCookie) { - KAFKA_LOG_CRIT(TStringBuilder() << "Unexpected cookie in TEvQueryResponse. Expected: " << KqpReqCookie << ", Actual: " << ev->Cookie << "."); + YDB_LOG_CRIT("Unexpected cookie in TEvQueryResponse", + {LogPrefix()}, + {"expected", KqpReqCookie}, + {"actual", ev->Cookie}); SendResponseFail(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to send request to producer_state table"); Die(ctx); return; @@ -112,7 +119,9 @@ namespace NKafka { auto status = record.GetYdbStatus(); if (status == Ydb::StatusIds::ABORTED) { if (CurrentTxAbortRetryNumber < TX_ABORT_RETRY_MAX_COUNT) { - KAFKA_LOG_ERROR(TStringBuilder() << "Retry after tx aborted. CurrentTxAbortRetryNumber# " << static_cast<int>(CurrentTxAbortRetryNumber)); + YDB_LOG_ERROR("Retry after tx aborted", + {LogPrefix()}, + {"currentTxAbortRetryNumber", static_cast<int>(CurrentTxAbortRetryNumber)}); RequestFullRetry(ctx); return; } @@ -150,7 +159,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("Pass away."); + YDB_LOG_DEBUG("Pass away", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -158,13 +168,16 @@ namespace NKafka { } void TKafkaInitProducerIdActor::StartTxProducerInitCycle(const TActorContext& ctx) { - KAFKA_LOG_D("Beginning transaction"); + YDB_LOG_DEBUG("Beginning transaction", + {LogPrefix()}); Kqp->BeginTransaction(++KqpReqCookie, ctx); LastSentToKqpRequest = EInitProducerIdKqpRequests::BEGIN_TRANSACTION; } void TKafkaInitProducerIdActor::HandleQueryResponseFromKqp(NKqp::TEvKqp::TEvQueryResponse::TPtr ev, const TActorContext& ctx) { - KAFKA_LOG_D("Handle kqp response for " << GetAsStr(LastSentToKqpRequest) << " request"); + YDB_LOG_DEBUG("Handle kqp response for request", + {LogPrefix()}, + {"lastSentToKqpRequest", GetAsStr(LastSentToKqpRequest)}); try { switch (LastSentToKqpRequest) { @@ -184,7 +197,8 @@ namespace NKafka { SendInsertRequest(ctx); break; default: - KAFKA_LOG_ERROR("Unknown EInitProducerIdKqpRequests"); + YDB_LOG_ERROR("Unknown EInitProducerIdKqpRequests", + {LogPrefix()}); Die(ctx); break; } @@ -275,14 +289,17 @@ namespace NKafka { // send responses methods void TKafkaInitProducerIdActor::SendResponseFail(EKafkaErrors error, const TString& message) { - KAFKA_LOG_ERROR(TStringBuilder() << "request failed. reason# " << message); + YDB_LOG_ERROR("Request failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TInitProducerIdResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, error)); } void TKafkaInitProducerIdActor::SendSuccessfullResponseForTxProducer(const TProducerState& producerState, const TActorContext& ctx) { - KAFKA_LOG_D("Sending succesfull response for transactional producer"); + YDB_LOG_DEBUG("Sending succesfull response for transactional producer", + {LogPrefix()}); auto response = std::make_shared<TInitProducerIdResponseData>(); response->ErrorCode = EKafkaErrors::NONE_ERROR; response->ProducerId = producerState.ProducerId; @@ -293,7 +310,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::SendSaveTxnProducerStateRequest(const TProducerState& producerState) { - KAFKA_LOG_D("Sending save txn producer state request"); + YDB_LOG_DEBUG("Sending save txn producer state request", + {LogPrefix()}); Send(NKafka::MakeTransactionsServiceID(SelfId().NodeId()), new TEvKafka::TEvSaveTxnProducerRequest( producerState.TransactionalId, @@ -370,8 +388,10 @@ namespace NKafka { ); } - TString TKafkaInitProducerIdActor::LogPrefix() { - return "InitProducerId actor: "; + NActors::NStructuredLog::TStructuredMessage TKafkaInitProducerIdActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaInitProducerIdActor"}, + {"selfId", SelfId()}); } TString TKafkaInitProducerIdActor::GetAsStr(EInitProducerIdKqpRequests request) { diff --git a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h index 0b6971797ad..f9c80cecd54 100644 --- a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h @@ -21,7 +21,7 @@ namespace NKafka { enum EInitProducerIdKqpRequests : ui8 { NO_REQUEST = 0, - + SELECT, INSERT, UPDATE, @@ -41,7 +41,7 @@ namespace NKafka { const ui64 CorrelationId; const std::optional<TString> TransactionalId; const std::optional<i32> TransactionTimeoutMs; - + // kqp related staff std::unique_ptr<NKafka::TKqpTxHelper> Kqp; ui64 KqpReqCookie = 0; @@ -70,7 +70,7 @@ namespace NKafka { void RequestFullRetry(const TActorContext& ctx); // event from KafkaTransactionCoordinator actor about successful or unsuccessful save of producer new state void Handle(NKafka::TEvKafka::TEvSaveTxnProducerResponse::TPtr& ev, const TActorContext& ctx); - + void Die(const TActorContext& ctx); @@ -103,7 +103,7 @@ namespace NKafka { EKafkaErrors KqpStatusToKafkaError(Ydb::StatusIds::StatusCode status); std::optional<TProducerState> ParseProducerState(NKqp::TEvKqp::TEvQueryResponse::TPtr ev); TString GetYqlWithTableName(const TString& templateStr); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); TString GetAsStr(EInitProducerIdKqpRequests request); }; diff --git a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp index 0c9caf508ab..73e40bf4423 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp @@ -16,6 +16,8 @@ #include "kafka_list_groups_actor.h" #include "kafka_state_name_to_int.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -33,7 +35,8 @@ void TKafkaListGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } Become(&TKafkaListGroupsActor::StateWork); } else { - KAFKA_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set."); + YDB_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set", + {LogPrefix()}); TListGroupsResponseData consumerGroupsResponseWithError; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, @@ -43,12 +46,14 @@ void TKafkaListGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } void TKafkaListGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); StartKqpSession(ctx); } void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -59,9 +64,11 @@ void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& } void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP ListGroups request"); + YDB_LOG_DEBUG("Received query response from KQP ListGroups request", + {LogPrefix()}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, error->data()); Die(ctx); return; @@ -71,10 +78,14 @@ void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, con } void TKafkaListGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryResponse& response, const TActorContext& ctx) { - KAFKA_LOG_D("Handling Select Response " << response.Record.GetResponse().GetYdbResults().size()); + YDB_LOG_DEBUG("Handling Select Response", + {LogPrefix()}, + {"responseResults", response.Record.GetResponse().GetYdbResults().size()}); if (response.Record.GetResponse().GetYdbResults().size() != 1) { TString errorMessage = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected 1, got " << response.Record.GetResponse().GetYdbResults().size() << "."; - KAFKA_LOG_W(errorMessage); + YDB_LOG_WARN("KQP returned wrong number of result sets size!=1 on SELECT query", + {LogPrefix()}, + {"responseResults", response.Record.GetResponse().GetYdbResults().size()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, errorMessage); Die(ctx); return; @@ -88,7 +99,8 @@ void TKafkaListGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryRes } void TKafkaListGroupsActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -96,12 +108,15 @@ void TKafkaListGroupsActor::Die(const TActorContext &ctx) { } void TKafkaListGroupsActor::StartKqpSession(const TActorContext& ctx) { - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } TListGroupsResponseData TKafkaListGroupsActor::ParseGroupsMetadata(const NKqp::TEvKqp::TEvQueryResponse& response) { - KAFKA_LOG_D("Parsing KQP response"); + YDB_LOG_DEBUG("Parsing KQP response", + {LogPrefix()}); TListGroupsResponseData listGroupsResponse; NYdb::TResultSetParser parser(response.Record.GetResponse().GetYdbResults(0)); @@ -159,7 +174,9 @@ std::shared_ptr<TListGroupsResponseData> TKafkaListGroupsActor::BuildResponse(TL }; void TKafkaListGroupsActor::SendToKqpConsumerGroupsRequest(const TActorContext& ctx) { - KAFKA_LOG_W("Sending select request to KQP for database " << DatabasePath); + YDB_LOG_WARN("Sending select request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendYqlRequest( GetYqlWithTableNames(ListGroupsRequestData->StatesFilter.size() > 0 ? SELECT_GROUPS_WITH_FILTER : @@ -173,9 +190,14 @@ void TKafkaListGroupsActor::SendToKqpConsumerGroupsRequest(const TActorContext& void TKafkaListGroupsActor::SendFailResponse(EKafkaErrors errorCode, const std::optional<TString>& errorMessage = std::nullopt) { if (errorMessage.has_value()) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } TListGroupsResponseData consumerGroupsResponseWithError; diff --git a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h index db6b2dce4e4..44714737b19 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h @@ -53,9 +53,11 @@ public: void Bootstrap(const NActors::TActorContext& ctx); - -TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaListGroupsActor{DatabasePath=" << DatabasePath << "}: "; +NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaListGroupsActor"}, + {"selfId", SelfId()}, + {"databasePath", DatabasePath}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp index da9019c8c8c..a9c45e08e5e 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp @@ -9,6 +9,8 @@ #include "kafka_list_offsets_actor.h" #include "kafka_topic_offsets_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -64,7 +66,10 @@ void TKafkaListOffsetsActor::HandleMissingTopicName(const TListOffsetsRequestDat } TActorId TKafkaListOffsetsActor::SendOffsetsRequest(const TListOffsetsRequestData::TListOffsetsTopic& topic, const NActors::TActorContext&) { - KAFKA_LOG_D("ListOffsets actor: Get offsets for topic '" << topic.Name << "' for user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("ListOffsets actor: Get offsets for topic for user", + {LogPrefix()}, + {"topicName", topic.Name}, + {"userName", GetUsernameOrAnonymous(Context)}); TEvKafka::TGetOffsetsRequest offsetsRequest; offsetsRequest.Topic = NormalizePath(Context->DatabasePath, topic.Name.value()); @@ -85,7 +90,8 @@ void TKafkaListOffsetsActor::Handle(TEvKafka::TEvTopicOffsetsResponse::TPtr& ev, Y_DEBUG_ABORT_UNLESS(it != TopicsRequestsInfo.end()); if (it == TopicsRequestsInfo.end()) { - KAFKA_LOG_CRIT("ListOffsets actor: received unexpected TEvTopicOffsetsResponse. Ignoring."); + YDB_LOG_CRIT("ListOffsets actor: received unexpected TEvTopicOffsetsResponse. Ignoring", + {LogPrefix()}); return RespondIfRequired(ctx); } @@ -104,7 +110,8 @@ void TKafkaListOffsetsActor::Handle(TEvKafka::TEvTopicOffsetsResponse::TPtr& ev, if (ev->Get()->Status == Ydb::StatusIds::SUCCESS) { auto it = responseFromPQPartitionsMap.find(partitionRequestInfo.PartitionId); if (it == responseFromPQPartitionsMap.end()) { - KAFKA_LOG_CRIT("ListOffsets actor: partition not found. Expect malformed/incompled reply"); + YDB_LOG_CRIT("ListOffsets actor: partition not found. Expect malformed/incompled reply", + {LogPrefix()}); continue; } auto& responseFromPQPartition = it->second; diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp index a6a8843c0e2..14f27c53907 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/persqueue/public/list_topics/list_all_topics_actor.h> #include <ydb/services/persqueue_v1/actors/schema_actors.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NKikimr; using namespace NKikimr::NGRpcProxy::V1; @@ -61,8 +63,11 @@ void TKafkaMetadataActor::SendDiscoveryRequest() { void TKafkaMetadataActor::HandleDiscoveryError(TEvDiscovery::TEvError::TPtr& ev) { PendingResponses--; HaveError = true; - KAFKA_LOG_ERROR("Port discovery failed for database '" << Context->DatabasePath << "' with error '" << ev->Get()->Error - << ", request " << CorrelationId); + YDB_LOG_ERROR("Port discovery failed for database with error request", + {LogPrefix()}, + {"databasePath", Context->DatabasePath}, + {"error", ev->Get()->Error}, + {"correlationId", CorrelationId}); RespondIfRequired(ActorContext()); } @@ -90,7 +95,9 @@ void TKafkaMetadataActor::ProcessDiscoveryData(TEvDiscovery::TEvDiscoveryData::T ok = leResponse.operation().result().UnpackTo(&leResult); } if (!ok) { - KAFKA_LOG_ERROR("Port discovery failed, unable to parse discovery response for request " << CorrelationId); + YDB_LOG_ERROR("Port discovery failed, unable to parse discovery response for request", + {LogPrefix()}, + {"correlationId", CorrelationId}); HaveError = true; return; } @@ -145,7 +152,10 @@ void TKafkaMetadataActor::AddTopic(const TString& topic, ui64 index) { } TActorId TKafkaMetadataActor::SendTopicRequest(const TString& topic) { - KAFKA_LOG_D("Describe partitions locations for topic '" << topic << "' for user '" << GetUsernameOrAnonymous(Context) << "'"); + YDB_LOG_DEBUG("Describe partitions locations for topic for user", + {LogPrefix()}, + {"topic", topic}, + {"userName", GetUsernameOrAnonymous(Context)}); TGetPartitionsLocationRequest locationRequest{}; locationRequest.Topic = NormalizePath(Context->DatabasePath, topic); @@ -237,12 +247,14 @@ void TKafkaMetadataActor::HandleLocationResponse(TEvLocationResponse::TPtr ev, c Y_DEBUG_ABORT_UNLESS(!actorIter->second.empty()); if (actorIter.IsEnd()) { - KAFKA_LOG_CRIT("Got unexpected location response, ignoring. Expect malformed/incompled reply"); + YDB_LOG_CRIT("Got unexpected location response, ignoring. Expect malformed/incompled reply", + {LogPrefix()}); return RespondIfRequired(ctx); } if (actorIter->second.empty()) { - KAFKA_LOG_CRIT("Corrupted state (empty actorId in mapping). Ignored location response, expect incomplete reply"); + YDB_LOG_CRIT("Corrupted state (empty actorId in mapping). Ignored location response, expect incomplete reply", + {LogPrefix()}); return RespondIfRequired(ctx); } @@ -250,20 +262,27 @@ void TKafkaMetadataActor::HandleLocationResponse(TEvLocationResponse::TPtr ev, c auto& topic = Response->Topics[index]; Ydb::StatusIds::StatusCode status = locationResponse->Status; if (status == Ydb::StatusIds::SUCCESS) { - KAFKA_LOG_D("Describe topic '" << topic.Name << "' location finishied successful"); + YDB_LOG_DEBUG("Describe topic location finishied successful", + {LogPrefix()}, + {"topicName", topic.Name}); PendingTopicResponses.emplace(index, locationResponse); } else if (status == Ydb::StatusIds::SCHEME_ERROR && Message->AllowAutoTopicCreation && Context->Config.GetAutoCreateTopicsEnable() && TopicСreationAttempts.find(*topic.Name) == TopicСreationAttempts.end() ) { - KAFKA_LOG_D("Sending create topic'" << topic.Name << "' request"); + YDB_LOG_DEBUG("Sending create topic' request", + {LogPrefix()}, + {"topicName", topic.Name}); TopicСreationAttempts.insert(*topic.Name); PendingResponses++; SendCreateTopicsRequest(*topic.Name, index, ctx); } else { - KAFKA_LOG_ERROR("Describe topic '" << topic.Name << "' location finishied with error: Code=" - << locationResponse->Status << ", Issues=" << locationResponse->Issues.ToOneLineString()); + YDB_LOG_ERROR("Describe topic location finishied with error", + {LogPrefix()}, + {"topicName", topic.Name}, + {"code", locationResponse->Status}, + {"issues", locationResponse->Issues.ToOneLineString()}); AddTopicError(topic, ConvertErrorCode(locationResponse->Status)); } } @@ -352,7 +371,9 @@ void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { auto topicNodes = CheckTopicNodes(ev.Get()); if (topicNodes.empty()) { // Already tried YDB discovery. Throw error - KAFKA_LOG_ERROR("Could not discovery kafka port for topic '" << topic.Name); + YDB_LOG_ERROR("Could not discovery kafka port for topic", + {LogPrefix()}, + {"topicName", topic.Name}); AddTopicError(topic, EKafkaErrors::LISTENER_NOT_FOUND); } else { AddTopicResponse(topic, ev.Get(), topicNodes); @@ -371,8 +392,10 @@ void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { Respond(); } -TString TKafkaMetadataActor::LogPrefix() const { - return TStringBuilder() << "TKafkaMetadataActor " << SelfId() << " "; +NStructuredLog::TStructuredMessage TKafkaMetadataActor::LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaMetadataActor"}, + {"selfId", SelfId()}); } } // namespace NKafka diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h index 176881707e1..d7ac9c8f928 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h @@ -73,7 +73,8 @@ private: } } - TString LogPrefix() const; + + NStructuredLog::TStructuredMessage LogPrefix() const; private: const TContext::TPtr Context; diff --git a/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp index 91ca26c7e3d..f8b830cf8e8 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp @@ -24,8 +24,10 @@ namespace NKafka { TBase::Become(&TKafkaMetricsActor::StateWork); } - TStringBuilder LogPrefix() const { - return {}; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaMetricsActor"}, + {"selfId", SelfId()}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp index 658098285f1..6ff8facfb1c 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp @@ -1,5 +1,7 @@ #include "kafka_offset_commit_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -7,12 +9,15 @@ NActors::IActor* CreateKafkaOffsetCommitActor(const TContext::TPtr context, cons return new TKafkaOffsetCommitActor(context, correlationId, message); } -TString TKafkaOffsetCommitActor::LogPrefix() { - return "TKafkaOffsetCommitActor"; +NActors::NStructuredLog::TStructuredMessage TKafkaOffsetCommitActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaOffsetCommitActor"}, + {"selfId", SelfId()}); } void TKafkaOffsetCommitActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); ctx.Send(AuthInitActor, new TEvents::TEvPoisonPill()); for (const auto& tabletToPipePair: TabletIdToPipe) { NTabletPipe::CloseClient(ctx, tabletToPipePair.second); @@ -24,7 +29,9 @@ void TKafkaOffsetCommitActor::Die(const TActorContext& ctx) { } void TKafkaOffsetCommitActor::Handle(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvCloseSession::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_CRIT("Auth failed. reason# " << ev->Get()->Reason); + YDB_LOG_CRIT("Auth failed", + {LogPrefix()}, + {"reason", ev->Get()->Reason}); Error = ConvertErrorCode(ev->Get()->ErrorCode); if (Error == GROUP_ID_NOT_FOUND && Context->Config.GetAutoCreateConsumersEnable()) { for (auto topicReq: Message->Topics) { @@ -91,9 +98,13 @@ void TKafkaOffsetCommitActor::SendFailedForAllPartitions(EKafkaErrors error, con void TKafkaOffsetCommitActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlterTopicResponse::TPtr& ev, const TActorContext& ctx) { NYdb::TStatus& result = ev->Get()->Result; if (result.GetStatus() == NYdb::EStatus::SUCCESS) { - KAFKA_LOG_D("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_DEBUG("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } else { - KAFKA_LOG_I("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_INFO("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } PendingResponses--; if (result.GetStatus() != NYdb::EStatus::ALREADY_EXISTS && result.GetStatus() != NYdb::EStatus::SUCCESS) { @@ -108,13 +119,16 @@ void TKafkaOffsetCommitActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev TEvTabletPipe::TEvClientConnected *msg = ev->Get(); if (msg->Status != NKikimrProto::OK) { - KAFKA_LOG_CRIT("Pipe to tablet is dead. status# " << ev->Get()->Status); + YDB_LOG_CRIT("Pipe to tablet is dead", + {LogPrefix()}, + {"status", ev->Get()->Status}); ProcessPipeProblem(msg->TabletId, ctx); } } void TKafkaOffsetCommitActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_CRIT("Pipe to tablet is destroyed"); + YDB_LOG_CRIT("Pipe to tablet is destroyed", + {LogPrefix()}); ProcessPipeProblem(ev->Get()->TabletId, ctx); } @@ -134,8 +148,10 @@ void TKafkaOffsetCommitActor::ProcessPipeProblem(ui64 tabletId, const TActorCont } void TKafkaOffsetCommitActor::SendGenerationCheckRequest(const TActorContext& ctx) { - KAFKA_LOG_D("Sending generation check KQP request for group# " << Message->GroupId.value() - << ", generationId# " << Message->GenerationId); + YDB_LOG_DEBUG("Sending generation check KQP request", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"generationId", Message->GenerationId}); NYdb::TParamsBuilder params; params.AddParam("$ConsumerGroup").Utf8(*Message->GroupId).Build(); @@ -150,9 +166,10 @@ void TKafkaOffsetCommitActor::SendGenerationCheckRequest(const TActorContext& ct void TKafkaOffsetCommitActor::Handle(NKikimr::NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - KAFKA_LOG_CRIT("Generation check KQP query failed." - << " group# " << Message->GroupId.value() - << " status# " << record.GetYdbStatus()); + YDB_LOG_CRIT("Generation check KQP query failed", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"status", record.GetYdbStatus()}); Error = UNKNOWN_SERVER_ERROR; SendFailedForAllPartitions(Error, ctx); return; @@ -174,22 +191,28 @@ void TKafkaOffsetCommitActor::Handle(NKikimr::NKqp::TEvKqp::TEvQueryResponse::TP auto tableGeneration = parser.ColumnParser("generation").GetUint64(); if (tableGeneration != static_cast<ui64>(Message->GenerationId)) { - KAFKA_LOG_I("Generation mismatch for group# " << Message->GroupId.value() - << ". Expected# " << Message->GenerationId - << ", got# " << tableGeneration); + YDB_LOG_INFO("Generation mismatch", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"expected", Message->GenerationId}, + {"got", tableGeneration}); Error = ILLEGAL_GENERATION; SendFailedForAllPartitions(Error, ctx); return; } - KAFKA_LOG_D("Generation check passed for group# " << Message->GroupId.value() - << ", generation# " << tableGeneration); + YDB_LOG_DEBUG("Generation check passed", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"generation", tableGeneration}); SendCommits(ctx); } void TKafkaOffsetCommitActor::Handle(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Auth success. Topics count: " << ev->Get()->TopicAndTablets.size()); + YDB_LOG_DEBUG("Auth success. Topics", + {LogPrefix()}, + {"count", ev->Get()->TopicAndTablets.size()}); TopicAndTablets = std::move(ev->Get()->TopicAndTablets); if (Message->GenerationId == -1) { @@ -203,7 +226,8 @@ void TKafkaOffsetCommitActor::Handle(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk void TKafkaOffsetCommitActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { - KAFKA_LOG_ERROR("Failed to create KQP session"); + YDB_LOG_ERROR("Failed to create KQP session", + {LogPrefix()}); Error = EKafkaErrors::UNKNOWN_SERVER_ERROR; SendFailedForAllPartitions(Error, ctx); return; @@ -256,10 +280,12 @@ void TKafkaOffsetCommitActor::SendCommits(const TActorContext& ctx) { } PendingResponses++; - KAFKA_LOG_D("Send commit request for group# " << Message->GroupId.value() << - ", topic# " << topicIt->second.TopicNameConverter->GetPrimaryPath() << - ", partition# " << partitionRequest.PartitionIndex << - ", offset# " << partitionRequest.CommittedOffset); + YDB_LOG_DEBUG("Send commit request", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"topic", topicIt->second.TopicNameConverter->GetPrimaryPath()}, + {"partition", partitionRequest.PartitionIndex}, + {"offset", partitionRequest.CommittedOffset}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); @@ -278,7 +304,10 @@ void TKafkaOffsetCommitActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const requestInfo->second.Done = true; if (ev->Get()->Record.GetErrorCode() != NPersQueue::NErrorCode::OK) { - KAFKA_LOG_CRIT("Commit offset error. status# " << EErrorCode_Name(ev->Get()->Record.GetErrorCode()) << ", reason# " << ev->Get()->Record.GetErrorReason()); + YDB_LOG_CRIT("Commit offset error", + {LogPrefix()}, + {"status", EErrorCode_Name(ev->Get()->Record.GetErrorCode())}, + {"reason", ev->Get()->Record.GetErrorReason()}); } AddPartitionResponse(ConvertErrorCode(NGRpcProxy::V1::ConvertOldCode(ev->Get()->Record.GetErrorCode())), requestInfo->second.TopicName, requestInfo->second.PartitionId, ctx); @@ -330,7 +359,8 @@ void TKafkaOffsetCommitActor::SendAuthRequest(const NActors::TActorContext& ctx) auto topicsToConverter = topicHandler->GetReadTopicsList(topicsToResolve, false, Context->DatabasePath); if (!topicsToConverter.IsValid) { - KAFKA_LOG_CRIT("Commit offsets failed. reason# topicsToConverter is not valid"); + YDB_LOG_CRIT("Commit offsets failed. topicsToConverter is not valid", + {LogPrefix()}); Error = INVALID_REQUEST; SendFailedForAllPartitions(Error, ctx); return; diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h index 1f9992243f4..7178d4716ff 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h @@ -53,11 +53,13 @@ public: void Bootstrap(const NActors::TActorContext& ctx); private: - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); void Die(const TActorContext& ctx) override; STATEFN(StateWork) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp index 4ffbde3fc8f..518cb6d783a 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp @@ -10,6 +10,8 @@ #include "ydb/services/persqueue_v1/actors/schema_actors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { NKikimr::NGRpcProxy::V1::TDescribeTopicActorSettings ConsumerOffsetSettings(std::shared_ptr<TSet<TString>> consumers, std::shared_ptr<TSet<ui32>>& partitions) { NKikimr::NGRpcProxy::V1::TDescribeTopicActorSettings settings { @@ -58,14 +60,18 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< void Bootstrap(const NActors::TActorContext& ctx) override { Y_UNUSED(ctx); - KAFKA_LOG_D("Get commited offsets for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'"); + YDB_LOG_DEBUG("Get commited offsets for topic for user", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}); SendDescribeProposeRequest(); Become(&TTopicOffsetActor::StateWork); }; - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTopicOffsetActor: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTopicOffsetActor"}, + {"selfId", SelfId()}); } void StateWork(TAutoPtr<IEventHandle>& ev) { @@ -84,11 +90,13 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< const TActorContext& ctx) override { Y_UNUSED(ctx); - KAFKA_LOG_D("Error raised for '" << OriginalTopicName << "'" - << " for user '" << UserSID << "'." - << " Error: '" << error << "'," - << " ErrorCode: '" << static_cast<int>(errorCode) << "'," - << " StatusCode: '" << status<< "'."); + YDB_LOG_DEBUG("Error raised for for user Error: ErrorCode: StatusCode", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}, + {"error", error}, + {"errorCode", static_cast<int>(errorCode)}, + {"status", status}); THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; @@ -142,8 +150,11 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< void HandleCacheNavigateResponse(NKikimr::TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) override { const auto& response = ev->Get()->Request.Get()->ResultSet.front(); - KAFKA_LOG_D("TEvNavigateKeySetResult received for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'. PQGroupInfo is present: " << (response.PQGroupInfo.Get() != nullptr)); + YDB_LOG_DEBUG("TEvNavigateKeySetResult received for topic for user PQGroupInfo is", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}, + {"present", (response.PQGroupInfo.Get() != nullptr)}); if (!response.PQGroupInfo) { THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; @@ -173,8 +184,10 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< } void Reply(const TActorContext& ctx) override { - KAFKA_LOG_D("Replying for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'" << " with status NONE_ERROR"); + YDB_LOG_DEBUG("Replying for topic for user with status NONE_ERROR", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}); THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; response->Status = NONE_ERROR; @@ -199,7 +212,9 @@ NActors::IActor* CreateKafkaOffsetFetchActor(const TContext::TPtr context, const void TKafkaOffsetFetchActor::Bootstrap(const NActors::TActorContext& ctx) { // If API level <= 7, Groups would be empty. In this case we convert message to level 8 and process it uniformely later - KAFKA_LOG_D("New request for user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("New request for user", + {LogPrefix()}, + {"userName", GetUsernameOrAnonymous(Context)}); if (Message->Groups.empty()) { TOffsetFetchRequestData::TOffsetFetchRequestGroup group; @@ -230,7 +245,8 @@ void TKafkaOffsetFetchActor::Bootstrap(const NActors::TActorContext& ctx) { // topics for such groups will be retrieved from the table Kqp = std::make_unique<TKqpTxHelper>(Context->ResourceDatabasePath); Kqp->SendCreateSessionRequest(ctx); - KAFKA_LOG_D("Creating KQP Session"); + YDB_LOG_DEBUG("Creating KQP Session", + {LogPrefix()}); } else { FillMapWithGroupRequests(); for (const auto& topicToEntities : TopicToEntities) { @@ -334,9 +350,13 @@ void TKafkaOffsetFetchActor::Handle(const TEvKafka::TEvResponse::TPtr& ev, const void TKafkaOffsetFetchActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlterTopicResponse::TPtr& ev, const TActorContext& ctx) { NYdb::TStatus& result = ev->Get()->Result; if (result.GetStatus() == NYdb::EStatus::SUCCESS) { - KAFKA_LOG_D("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_DEBUG("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } else { - KAFKA_LOG_I("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_INFO("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } if (result.GetStatus() != NYdb::EStatus::SUCCESS) { InflyTopics--; @@ -369,10 +389,12 @@ void TKafkaOffsetFetchActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlter } void TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Got KQP CreateSession response"); + YDB_LOG_DEBUG("Got KQP CreateSession response", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, std::make_shared<TOffsetFetchResponseData>(), EKafkaErrors::UNKNOWN_SERVER_ERROR)); - KAFKA_LOG_D("KQP Session Error"); + YDB_LOG_DEBUG("KQP Session Error", + {LogPrefix()}); return; } NYdb::TParamsBuilder params = BuildFetchAssignmentsParams(GroupsToFetch); @@ -381,12 +403,15 @@ void TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr void NKafka::TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { std::vector<std::pair<std::optional<TString>, TConsumerProtocolAssignment>> assignments; - KAFKA_LOG_D("Received KQP response"); + YDB_LOG_DEBUG("Received KQP response", + {LogPrefix()}); ParseGroupsAssignments(ev, assignments); if (assignments.empty()) { auto response = GetOffsetFetchResponse(); - KAFKA_LOG_D("Sending response to user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("Sending response to user", + {LogPrefix()}, + {"userName", GetUsernameOrAnonymous(Context)}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, static_cast<EKafkaErrors>(response->ErrorCode))); Die(ctx); return; @@ -586,11 +611,17 @@ TOffsetFetchResponseData::TOffsetFetchResponseGroup::TOffsetFetchResponseTopics partition.ErrorCode = NONE_ERROR; } else { partition.ErrorCode = RESOURCE_NOT_FOUND; - KAFKA_LOG_ERROR("Group " << groupId << " not found for topic " << topicName); + YDB_LOG_ERROR("Group not found for topic", + {LogPrefix()}, + {"groupId", groupId}, + {"topicName", topicName}); } } else { partition.ErrorCode = RESOURCE_NOT_FOUND; - KAFKA_LOG_ERROR("Partition " << requestPartition << " not found for topic " << topicName); + YDB_LOG_ERROR("Partition not found for topic", + {LogPrefix()}, + {"requestPartition", requestPartition}, + {"topicName", topicName}); } topic.Partitions.push_back(partition); } @@ -627,7 +658,8 @@ void TKafkaOffsetFetchActor::FillMapWithGroupRequests() { } void NKafka::TKafkaOffsetFetchActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); for (const TActorId& actorId : DependantActors) { Send(actorId, new TEvents::TEvPoisonPill()); } diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h index b7eeb7dbd14..d76036ae710 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h @@ -99,8 +99,12 @@ public: void ReplyError(const TActorContext& ctx); void Die(const TActorContext &ctx); - TStringBuilder LogPrefix() const { - return TStringBuilder() << "TKafkaOffsetFetchActor{GroupId=" << Message->GroupId.value() << ",DatabasePath=" << DatabasePath << "}: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaOffsetFetchActor"}, + {"selfId", SelfId()}, + {"groupId", Message->GroupId.value()}, + {"databasePath", DatabasePath}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp index 416f0576899..697bb562952 100644 --- a/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp @@ -13,6 +13,8 @@ #include <limits> #include <util/string/join.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { namespace { @@ -46,14 +48,18 @@ bool CanStoreRawRecords(const TKafkaBatchHeader& header, bool batchingEnabled) { EKafkaErrors ValidateBatchCompression(const TKafkaBatchHeader& header, bool batchingEnabled) { if (!IsRecordBatchV2OrNewer(header) && !IsRawRecordBatch(header)) { - KAFKA_LOG_ERROR("Compressed legacy Kafka record batches are not supported. Magic=" - << header.Magic << ", compressionType=" << static_cast<int>(CompressionType(header))); + YDB_LOG_ERROR("Compressed legacy Kafka record batches are not supported", + {LogPrefix()}, + {"magic", header.Magic}, + {"compressionType", static_cast<int>(CompressionType(header))}); return EKafkaErrors::UNSUPPORTED_COMPRESSION_TYPE; } if (!batchingEnabled && !IsRawRecordBatch(header)) { - KAFKA_LOG_ERROR("Compressed Kafka record batches are not supported when topic messages batching is disabled. Magic=" - << header.Magic << ", compressionType=" << static_cast<int>(CompressionType(header))); + YDB_LOG_ERROR("Compressed Kafka record batches are not supported when topic messages batching is disabled", + {LogPrefix()}, + {"magic", header.Magic}, + {"compressionType", static_cast<int>(CompressionType(header))}); return EKafkaErrors::UNSUPPORTED_COMPRESSION_TYPE; } @@ -110,22 +116,28 @@ NKikimrPQClient::TDataChunk MakeDataChunk(const TKafkaRecord& record) { return proto; } -TString TKafkaProduceActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaProduceActor " << SelfId() << " State: "; +NStructuredLog::TStructuredMessage TKafkaProduceActor::LogPrefix() { + TString state; + auto stateFunc = CurrentStateFunc(); if (stateFunc == &TKafkaProduceActor::StateInit) { - sb << "Init "; + state = "Init"; } else if (stateFunc == &TKafkaProduceActor::StateWork) { - sb << "Work "; + state = "Work"; } else { - sb << "Unknown "; + state = "Unknown"; } - return sb; + + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaProduceActor"}, + {"selfId", SelfId()}, + {"state", state}); } void TKafkaProduceActor::LogEvent(IEventHandle& ev) { - KAFKA_LOG_T("Produce actor: Received event: " << ev.GetTypeName()); + YDB_LOG_TRACE("Produce actor: Received", + {LogPrefix()}, + {"event", ev.GetTypeName()}); } void TKafkaProduceActor::SendMetrics(const TString& topicName, size_t delta, const TString& name, const TActorContext& ctx) { @@ -147,7 +159,8 @@ void TKafkaProduceActor::Bootstrap(const NActors::TActorContext& /*ctx*/) { } void TKafkaProduceActor::Handle(TEvKafka::TEvWakeup::TPtr /*request*/, const TActorContext& ctx) { - KAFKA_LOG_T("Produce actor: Wakeup"); + YDB_LOG_TRACE("Produce actor: Wakeup", + {LogPrefix()}); SendResults(ctx); CleanTopics(ctx); @@ -155,11 +168,13 @@ void TKafkaProduceActor::Handle(TEvKafka::TEvWakeup::TPtr /*request*/, const TAc Schedule(WAKEUP_INTERVAL, new TEvKafka::TEvWakeup()); - KAFKA_LOG_T("Produce actor: Wakeup was completed successfully"); + YDB_LOG_TRACE("Produce actor: Wakeup was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::PassAway() { - KAFKA_LOG_D("Produce actor: PassAway"); + YDB_LOG_DEBUG("Produce actor: PassAway", + {LogPrefix()}); for(const auto& [_, partitionWriters] : NonTransactionalWriters) { for(const auto& [_, w] : partitionWriters) { @@ -174,7 +189,8 @@ void TKafkaProduceActor::PassAway() { TActorBootstrapped::PassAway(); - KAFKA_LOG_T("Produce actor: PassAway was completed successfully"); + YDB_LOG_TRACE("Produce actor: PassAway was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::CleanTopics(const TActorContext& ctx) { @@ -190,7 +206,8 @@ void TKafkaProduceActor::CleanTopics(const TActorContext& ctx) { } void TKafkaProduceActor::CleanWriters(const TActorContext& ctx) { - KAFKA_LOG_T("Produce actor: CleanWriters"); + YDB_LOG_TRACE("Produce actor: CleanWriters", + {LogPrefix()}); const auto earliestAllowedTs = ctx.Now() - WRITER_EXPIRATION_INTERVAL; for (auto& [topicPath, partitionWriters] : NonTransactionalWriters) { @@ -213,11 +230,15 @@ void TKafkaProduceActor::CleanWriters(const TActorContext& ctx) { } } - KAFKA_LOG_T("Produce actor: CleanWriters was completed successfully"); + YDB_LOG_TRACE("Produce actor: CleanWriters was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::CleanWriter(const TTopicPartition& topicPartition, const TActorId& writerId) { - KAFKA_LOG_D("Produce actor: Destroing inactive PartitionWriter. Topic='" << topicPartition.TopicPath << "', Partition=" << topicPartition.PartitionId); + YDB_LOG_DEBUG("Produce actor: Destroing inactive PartitionWriter. Topic='", + {LogPrefix()}, + {"topicPath", topicPartition.TopicPath}, + {"partition", topicPartition.PartitionId}); Send(writerId, new TEvents::TEvPoison()); } @@ -231,7 +252,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu for (auto& info : navigate->ResultSet) { if (NSchemeCache::TSchemeCacheNavigate::EStatus::Ok == info.Status) { auto topicPath = CanonizePath(NKikimr::JoinPath(info.Path)); - KAFKA_LOG_D("Produce actor: Received topic '" << topicPath << "' description"); + YDB_LOG_DEBUG("Produce actor: Received topic description", + {LogPrefix()}, + {"topicPath", topicPath}); TopicsForInitialization.erase(topicPath); auto& topic = Topics[topicPath]; @@ -242,7 +265,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu topic.ExpirationTime = now + TOPIC_OK_EXPIRATION_INTERVAL; topic.PartitionChooser = CreatePartitionChooser(info.PQGroupInfo->Description); } else { - KAFKA_LOG_W("Produce actor: Unauthorized PRODUCE to topic '" << topicPath << "'"); + YDB_LOG_WARN("Produce actor: Unauthorized PRODUCE to topic", + {LogPrefix()}, + {"topicPath", topicPath}); topic.Status = UNAUTHORIZED; topic.ExpirationTime = now + TOPIC_UNATHORIZED_EXPIRATION_INTERVAL; } @@ -254,7 +279,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu } for(auto& topicPath : TopicsForInitialization) { - KAFKA_LOG_D("Produce actor: Topic '" << topicPath << "' not found"); + YDB_LOG_DEBUG("Produce actor: Topic not found", + {LogPrefix()}, + {"topicPath", topicPath}); auto& topicInfo = Topics[topicPath]; topicInfo.Status = NOT_FOUND; topicInfo.ExpirationTime = now + TOPIC_NOT_FOUND_EXPIRATION_INTERVAL; @@ -264,14 +291,17 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu Become(&TKafkaProduceActor::StateWork); - KAFKA_LOG_T("Produce actor: HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr) was completed successfully"); + YDB_LOG_TRACE("Produce actor: HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr) was completed successfully", + {LogPrefix()}); ProcessRequests(ctx); } void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev, const TActorContext& ctx) { auto& path = ev->Get()->Path; - KAFKA_LOG_I("Produce actor: Topic '" << path << "' was deleted"); + YDB_LOG_INFO("Produce actor: Topic was deleted", + {LogPrefix()}, + {"path", path}); auto it = NonTransactionalWriters.find(path); if (it != NonTransactionalWriters.end()) { @@ -297,7 +327,9 @@ void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TP void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev, const TActorContext& ctx) { auto* e = ev->Get(); auto& path = e->Path; - KAFKA_LOG_I("Produce actor: Topic '" << path << "' was updated"); + YDB_LOG_INFO("Produce actor: Topic was updated", + {LogPrefix()}, + {"path", path}); auto& topic = Topics[path]; if (topic.Status == UNAUTHORIZED) { @@ -459,7 +491,9 @@ std::pair<EKafkaErrors, THolder<TEvPartitionWriter::TEvWriteRequest>> Convert( TString str; const bool res = proto.SerializeToString(&str); if (!res) { - KAFKA_LOG_ERROR("Produce actor: Failed to serialize TDataChunk to string: " << proto.DebugString()); + YDB_LOG_ERROR("Produce actor: Failed to serialize TDataChunk to string", + {LogPrefix()}, + {"message", proto.DebugString()}); return {EKafkaErrors::INVALID_RECORD, nullptr}; } @@ -514,7 +548,9 @@ std::pair<EKafkaErrors, THolder<TEvPartitionWriter::TEvWriteRequest>> Convert( // set seqno if (enableKafkaDeduplication && batch.BaseSequence < 0) { - KAFKA_LOG_ERROR("Idempotent producer enabled and batch base sequence is less then zero: " << batch.BaseSequence); + YDB_LOG_ERROR("Idempotent producer enabled and batch base sequence is less then zero", + {LogPrefix()}, + {"baseSequence", batch.BaseSequence}); return {EKafkaErrors::INVALID_RECORD, nullptr}; } w->SetSeqNo(GetRecordSeqNo(batch, batchIndex, record)); @@ -547,7 +583,8 @@ size_t PartsCount(const TMessagePtr<TProduceRequestData>& r) { void TKafkaProduceActor::ProcessRequest(TPendingRequest::TPtr pendingRequest, const TActorContext& ctx) { auto r = pendingRequest->Request->Get()->Request; - KAFKA_LOG_D("Processing request"); + YDB_LOG_DEBUG("Processing request", + {LogPrefix()}); pendingRequest->Results.resize(PartsCount(r)); pendingRequest->StartTime = ctx.Now(); @@ -574,7 +611,9 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteAccepted::TPtr reque auto it = Cookies.find(cookie); if (it == Cookies.end()) { - KAFKA_LOG_W("Produce actor: Received TEvWriteAccepted with unexpected cookie " << cookie); + YDB_LOG_WARN("Produce actor: Received TEvWriteAccepted with unexpected cookie", + {LogPrefix()}, + {"cookie", cookie}); return; } @@ -585,22 +624,31 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteAccepted::TPtr reque if (expectedCookies.empty()) { ProcessRequests(ctx); } else { - KAFKA_LOG_W("Still in accepting after receive TEvPartitionWriter::TEvWriteAccepted cause cookies are expected: " << JoinSeq(", ", expectedCookies)); + YDB_LOG_WARN("Still in accepting after receive TEvPartitionWriter::TEvWriteAccepted cause cookies are expected", + {LogPrefix()}, + {"expected", JoinSeq(", ", expectedCookies)}); } } void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvInitResult::TPtr request, const TActorContext& /*ctx*/) { - KAFKA_LOG_D("Produce actor: Init " << request->Get()->ToString()); + YDB_LOG_DEBUG("Produce actor: Init", + {LogPrefix()}, + {"request", request->Get()->ToString()}); if (!request->Get()->IsSuccess()) { auto sender = request->Sender; if (WriterDied(sender, EKafkaErrors::UNKNOWN_SERVER_ERROR, request->Get()->GetError().Reason)) { - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvInitResult for " << sender << " with error: " << request->Get()->GetError().Reason); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvInitResult for with error", + {LogPrefix()}, + {"sender", sender}, + {"error", request->Get()->GetError().Reason}); return; } - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvInitResult with unexpected writer " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvInitResult with unexpected writer", + {LogPrefix()}, + {"sender", sender}); } } @@ -608,11 +656,15 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvDisconnected::TPtr reques auto sender = request->Sender; if (WriterDied(sender, EKafkaErrors::NOT_LEADER_OR_FOLLOWER, TStringBuilder() << "Partition writer " << sender << " disconnected")) { - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvDisconnected for " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvDisconnected", + {LogPrefix()}, + {"sender", sender}); return; } - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvDisconnected with unexpected writer " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvDisconnected with unexpected writer", + {LogPrefix()}, + {"sender", sender}); } bool TKafkaProduceActor::WriterDied(const TActorId& writerId, EKafkaErrors errorCode, TStringBuf errorMessage) { @@ -671,11 +723,15 @@ bool TKafkaProduceActor::WriterDied(const TActorId& writerId, EKafkaErrors error void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteResponse::TPtr request, const TActorContext& ctx) { auto r = request->Get(); auto cookie = r->Record.GetPartitionResponse().GetCookie(); - KAFKA_LOG_T("Handling TEvPartitionWriter::TEvWriteResponse with cookie " << cookie); + YDB_LOG_TRACE("Handling TEvPartitionWriter::TEvWriteResponse with cookie", + {LogPrefix()}, + {"cookie", cookie}); auto it = Cookies.find(cookie); if (it == Cookies.end()) { - KAFKA_LOG_W("Produce actor: Received TEvWriteResponse with unexpected cookie " << cookie); + YDB_LOG_WARN("Produce actor: Received TEvWriteResponse with unexpected cookie", + {LogPrefix()}, + {"cookie", cookie}); return; } auto& cookieInfo = it->second; @@ -710,7 +766,9 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteResponse::TPtr reque if (cookieInfo.Request->WaitResultCookies.empty()) { SendResults(ctx); } else { - KAFKA_LOG_T("Skipping sending results in Handle TEvPartitionWriter::TEvWriteResponse. WaitResultCookies=" << JoinSeq(", ", cookieInfo.Request->WaitResultCookies)); + YDB_LOG_TRACE("Skipping sending results in Handle TEvPartitionWriter::TEvWriteResponse", + {LogPrefix()}, + {"waitResultCookies", JoinSeq(", ", cookieInfo.Request->WaitResultCookies)}); } Cookies.erase(cookie); @@ -730,7 +788,10 @@ EKafkaErrors Convert(TEvPartitionWriter::TEvWriteResponse::EErrorCode value) { void TKafkaProduceActor::SendResults(const TActorContext& ctx) { auto expireTime = ctx.Now() - REQUEST_EXPIRATION_INTERVAL; - KAFKA_LOG_T("Produce actor: Sending results. QueueSize= " << PendingRequests.size() << ", ExpirationTime=" << expireTime); + YDB_LOG_TRACE("Produce actor: Sending results", + {LogPrefix()}, + {"queueSize", PendingRequests.size()}, + {"expirationTime", expireTime}); // We send the results in the order of receipt of the request while (!PendingRequests.empty()) { @@ -740,7 +801,10 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { bool expired = expireTime > pendingRequest->StartTime; if (!expired && !pendingRequest->WaitResultCookies.empty()) { - KAFKA_LOG_T("Skipping sending results. Expired=" << expired << " WaitResultCookies=" << JoinSeq(", ", pendingRequest->WaitResultCookies)); + YDB_LOG_TRACE("Skipping sending results", + {LogPrefix()}, + {"expired", expired}, + {"waitResultCookies", JoinSeq(", ", pendingRequest->WaitResultCookies)}); return; } @@ -748,7 +812,10 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { auto correlationId = pendingRequest->Request->Get()->CorrelationId; EKafkaErrors metricsErrorCode = EKafkaErrors::NONE_ERROR; - KAFKA_LOG_D("Produce actor: Send result for correlation=" << correlationId << ". Expired=" << expired); + YDB_LOG_DEBUG("Produce actor: Send result", + {LogPrefix()}, + {"correlation", correlationId}, + {"expired", expired}); const auto topicsCount = request->TopicData.size(); auto response = std::make_shared<TProduceResponseData>(); @@ -770,15 +837,18 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { size_t recordsCount = result.RecordsCount; partitionResponse.Index = partitionData.Index; if (EKafkaErrors::NONE_ERROR != result.ErrorCode) { - KAFKA_LOG_ERROR("Produce actor: Partition result with error: ErrorCode=" << static_cast<int>(result.ErrorCode) - << ", ErrorMessage=" << result.ErrorMessage << ", #01"); + YDB_LOG_ERROR("Produce actor: Partition result with error: #01", + {LogPrefix()}, + {"errorCode", static_cast<int>(result.ErrorCode)}, + {"errorMessage", result.ErrorMessage}); partitionResponse.ErrorCode = result.ErrorCode; metricsErrorCode = result.ErrorCode; partitionResponse.ErrorMessage = result.ErrorMessage; SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); } else if (expired) { - KAFKA_LOG_ERROR("Partition write expired."); + YDB_LOG_ERROR("Partition write expired", + {LogPrefix()}); SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); partitionResponse.ErrorCode = EKafkaErrors::REQUEST_TIMED_OUT; metricsErrorCode = EKafkaErrors::REQUEST_TIMED_OUT; @@ -786,7 +856,8 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { } else { auto* msg = result.Value->Get(); if (msg->IsSuccess()) { - KAFKA_LOG_T("Produce actor: Partition result success."); + YDB_LOG_TRACE("Produce actor: Partition result success", + {LogPrefix()}); partitionResponse.ErrorCode = EKafkaErrors::NONE_ERROR; auto& writeResults = msg->Record.GetPartitionResponse().GetCmdWriteResult(); if (!writeResults.empty()) { @@ -796,11 +867,11 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { partitionResponse.BaseOffset = writeResults.at(0).GetOffset(); } } else { - KAFKA_LOG_ERROR("Produce actor: Partition result with error: ErrorCode=" - << static_cast<int>(Convert(msg->GetError().Code)) - << ", ErrorMessage=" << msg->GetError().Reason - << ", Error from writer=" << static_cast<int>(msg->Record.GetErrorCode()) - << ", #02"); + YDB_LOG_ERROR("Produce actor: Partition result with error: Error #02", + {LogPrefix()}, + {"errorCode", static_cast<int>(Convert(msg->GetError().Code))}, + {"errorMessage", msg->GetError().Reason}, + {"fromWriter", static_cast<int>(msg->Record.GetErrorCode())}); SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); if (msg->Record.GetErrorCode() == NPersQueue::NErrorCode::KAFKA_INVALID_PRODUCER_EPOCH) { @@ -846,7 +917,9 @@ void TKafkaProduceActor::ProcessInitializationRequests(const TActorContext& ctx) auto request = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); for(auto& topicPath : TopicsForInitialization) { - KAFKA_LOG_D("Produce actor: Describe topic '" << topicPath << "'"); + YDB_LOG_DEBUG("Produce actor: Describe topic", + {LogPrefix()}, + {"topicPath", topicPath}); NSchemeCache::TSchemeCacheNavigate::TEntry entry; entry.Path = NKikimr::SplitPath(topicPath); entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; @@ -864,7 +937,8 @@ void TKafkaProduceActor::RecreatePartitionWriterAndRetry(ui64 cookie, const TAct auto it = Cookies.find(cookie); if (it != Cookies.end()) { auto& cookieInfo = it->second; - KAFKA_LOG_D("Transaction was committed. Retrying produce request as a part of next transaction"); + YDB_LOG_DEBUG("Transaction was committed. Retrying produce request as a part of next transaction", + {LogPrefix()}); auto txnIt = TransactionalWriters.find({cookieInfo.TopicPath, cookieInfo.PartitionId}); if (txnIt != TransactionalWriters.end()) { Send(txnIt->second.ActorId, new TEvents::TEvPoison()); @@ -932,7 +1006,9 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu parsedRecords = std::move(parsed); result.RecordsCount = parsedRecords.RecordsCount; } catch (const yexception& e) { - KAFKA_LOG_ERROR("Failed to read Kafka records metadata: " << e.what()); + YDB_LOG_ERROR("Failed to read Kafka records", + {LogPrefix()}, + {"metadata", e.what()}); result.ErrorCode = EKafkaErrors::INVALID_RECORD; return; } @@ -960,7 +1036,10 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu pendingRequest->WaitResultCookies.insert(ownCookie); ruPerRequest = false; - KAFKA_LOG_T("Sending TEvPartitionWriter::TEvWriteRequest to " << writer.second << " with cookie " << ownCookie); + YDB_LOG_TRACE("Sending TEvPartitionWriter::TEvWriteRequest to with cookie", + {LogPrefix()}, + {"writerSecond", writer.second}, + {"ownCookie", ownCookie}); Send(writer.second, std::move(ev)); result.ErrorCode = NONE_ERROR; } else { @@ -983,14 +1062,19 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu } if (result.ErrorCode != EKafkaErrors::NONE_ERROR) { - KAFKA_LOG_ERROR("Write request failed with error " << result.ErrorCode << " and message " << result.ErrorMessage); + YDB_LOG_ERROR("Write request failed with error and message", + {LogPrefix()}, + {"errorCode", result.ErrorCode}, + {"errorMessage", result.ErrorMessage}); } } std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::PartitionWriter(const TTopicPartition& topicPartition, const TProducerInstanceId& producerInstanceId, const TMaybe<TString>& transactionalId, const TActorContext& ctx) { auto it = Topics.find(topicPartition.TopicPath); if (it == Topics.end()) { - KAFKA_LOG_ERROR("Produce actor: Internal error: topic '" << topicPartition.TopicPath << "' isn`t initialized"); + YDB_LOG_ERROR("Produce actor: Internal error: topic isn`t initialized", + {LogPrefix()}, + {"topicPath", topicPartition.TopicPath}); return { NOT_FOUND, TActorId{} }; } @@ -1055,7 +1139,12 @@ std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::GetOrC } std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::CreateTransactionalWriter(const TTopicPartition& topicPartition, const TTopicInfo& topicInfo, const TProducerInstanceId& producerInstanceId, const TString& transactionalId, const TActorContext& ctx) { - KAFKA_LOG_D("Created transactional writer for producerId=" << producerInstanceId.Id << " and producerEpoch=" << producerInstanceId.Epoch << " for topic-partition " << topicPartition.TopicPath << ":" << topicPartition.PartitionId); + YDB_LOG_DEBUG("Created transactional writer for and for topic-partition", + {LogPrefix()}, + {"producerId", producerInstanceId.Id}, + {"producerEpoch", producerInstanceId.Epoch}, + {"topicPath", topicPartition.TopicPath}, + {"partitionId", topicPartition.PartitionId}); auto* partition = topicInfo.PartitionChooser->GetPartition(topicPartition.PartitionId); if (!partition) { return { NOT_FOUND, TActorId{} }; diff --git a/ydb/core/kafka_proxy/actors/kafka_produce_actor.h b/ydb/core/kafka_proxy/actors/kafka_produce_actor.h index 963aa52bf90..753f5c23089 100644 --- a/ydb/core/kafka_proxy/actors/kafka_produce_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_produce_actor.h @@ -20,7 +20,7 @@ using namespace NKikimrClient; // Each request can contain data for writing to several topics, and in each topic to several partitions. // When a request to write to an unknown topic arrives, the actor changes the state to Init until it receives // information about all the topics needed to process the request. -// +// // Requests are processed in parallel, but it is guaranteed that the recording order will be preserved. // The order of responses to requests is also guaranteed. // @@ -116,7 +116,7 @@ private: std::pair<ETopicStatus, TActorId> PartitionWriter(const TTopicPartition& topicPartition, const TProducerInstanceId& producerInstanceId, const TMaybe<TString>& transactionalId, const TActorContext& ctx); bool WriterDied(const TActorId& writerId, EKafkaErrors errorCode, TStringBuf errorMessage); - TString LogPrefix(); + NStructuredLog::TStructuredMessage LogPrefix(); void LogEvent(IEventHandle& ev); void SendMetrics(const TString& topicName, size_t delta, const TString& name, const TActorContext& ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp index b56b9f2d1e4..a1dea83bff4 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp @@ -1,6 +1,8 @@ #include "kafka_read_session_actor.h" #include "kafka_read_session_utils.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { static constexpr TDuration WAKEUP_INTERVAL = TDuration::Seconds(1); static constexpr TDuration LOCK_PARTITION_DELAY = TDuration::Seconds(3); @@ -15,14 +17,16 @@ void TKafkaReadSessionActor::Bootstrap(const NActors::TActorContext&) { Become(&TKafkaReadSessionActor::StateWork); } -TString TKafkaReadSessionActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaReadSessionActor " << (Session == "" ? SelfId().ToString() : Session) << ": "; - return sb; +NStructuredLog::TStructuredMessage TKafkaReadSessionActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaReadSessionActor"}, + {"selfId", SelfId()}, + {"session", Session}); } void TKafkaReadSessionActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); for (auto& [topicName, topicInfo] : TopicsInfo) { NTabletPipe::CloseClient(ctx, topicInfo.PipeClient); } @@ -31,7 +35,8 @@ void TKafkaReadSessionActor::Die(const TActorContext& ctx) { void TKafkaReadSessionActor::HandleWakeup(TEvKafka::TEvWakeup::TPtr, const TActorContext& ctx) { if (CheckHeartbeatIsExpired()) { - KAFKA_LOG_D("Heartbeat expired"); + YDB_LOG_DEBUG("Heartbeat expired", + {LogPrefix()}); CloseReadSession(ctx); return; } @@ -241,7 +246,9 @@ void TKafkaReadSessionActor::SendJoinGroupResponseOk(const TActorContext&, ui64 } void TKafkaReadSessionActor::SendJoinGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("JOIN_GROUP failed. reason# " << message); + YDB_LOG_CRIT("JOIN_GROUP failed", + {LogPrefix()}, + {"reason", message}); TJoinGroupResponseData::TPtr response = std::make_shared<TJoinGroupResponseData>(); response->ErrorCode = error; @@ -269,7 +276,9 @@ void TKafkaReadSessionActor::SendSyncGroupResponseOk(const TActorContext& ctx, u } void TKafkaReadSessionActor::SendSyncGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("SYNC_GROUP failed. reason# " << message); + YDB_LOG_CRIT("SYNC_GROUP failed", + {LogPrefix()}, + {"reason", message}); TSyncGroupResponseData::TPtr response = std::make_shared<TSyncGroupResponseData>(); response->ErrorCode = error; @@ -287,7 +296,9 @@ void TKafkaReadSessionActor::SendLeaveGroupResponseOk(const TActorContext&, ui64 } void TKafkaReadSessionActor::SendLeaveGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("LEAVE_GROUP failed. reason# " << message); + YDB_LOG_CRIT("LEAVE_GROUP failed", + {LogPrefix()}, + {"reason", message}); TLeaveGroupResponseData::TPtr response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = error; @@ -303,7 +314,9 @@ void TKafkaReadSessionActor::SendHeartbeatResponseOk(const TActorContext&, ui64 void TKafkaReadSessionActor::SendHeartbeatResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { THeartbeatResponseData::TPtr response = std::make_shared<THeartbeatResponseData>(); - KAFKA_LOG_CRIT("HEARTBEAT failed. reason# " << message); + YDB_LOG_CRIT("HEARTBEAT failed", + {LogPrefix()}, + {"reason", message}); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); } @@ -337,7 +350,9 @@ bool TKafkaReadSessionActor::TryFillTopicsToRead(const TMessagePtr<TJoinGroupReq if (result.has_value()) { for (auto topic: result->Topics) { if (topic.has_value()) { - KAFKA_LOG_D("JOIN_GROUP requested topic to read: " << topic); + YDB_LOG_DEBUG("JOIN_GROUP requested topic", + {LogPrefix()}, + {"toRead", topic}); auto normalizedTopicName = NormalizePath(Context->DatabasePath, topic.value()); OriginalTopicNames[normalizedTopicName] = topic.value(); @@ -352,7 +367,9 @@ bool TKafkaReadSessionActor::TryFillTopicsToRead(const TMessagePtr<TJoinGroupReq TConsumerProtocolAssignment TKafkaReadSessionActor::BuildAssignmentAndInformBalancerIfRelease(const TActorContext& ctx) { TConsumerProtocolAssignment assignment; - KAFKA_LOG_D("SYNC_GROUP topics to assign count: " << TopicPartitions.size()); + YDB_LOG_DEBUG("SYNC_GROUP topics to assign", + {LogPrefix()}, + {"count", TopicPartitions.size()}); for (auto& [topicName, partitions] : TopicPartitions) { auto topicInfoIt = TopicsInfo.find(topicName); @@ -378,9 +395,13 @@ TConsumerProtocolAssignment TKafkaReadSessionActor::BuildAssignmentAndInformBala } partitions.ToRelease.clear(); - KAFKA_LOG_D("SYNC_GROUP partitions assigned: " << finalPartitionsToRead.size()); + YDB_LOG_DEBUG("SYNC_GROUP partitions", + {LogPrefix()}, + {"assigned", finalPartitionsToRead.size()}); for (auto part: finalPartitionsToRead) { - KAFKA_LOG_D("SYNC_GROUP assigned partition number: " << part); + YDB_LOG_DEBUG("SYNC_GROUP assigned partition", + {LogPrefix()}, + {"number", part}); topicPartition.Partitions.push_back(part); partitions.ReadingNow.emplace(part); } @@ -466,7 +487,9 @@ void TKafkaReadSessionActor::HandleBalancerError(TEvPersQueue::TEvError::TPtr& e } void TKafkaReadSessionActor::HandleAuthOk(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("JOIN_GROUP auth success. Topics count: " << ev->Get()->TopicAndTablets.size()); + YDB_LOG_DEBUG("JOIN_GROUP auth success. Topics", + {LogPrefix()}, + {"count", ev->Get()->TopicAndTablets.size()}); for (const auto& [name, t] : ev->Get()->TopicAndTablets) { auto internalName = t.TopicNameConverter->GetInternalName(); @@ -506,7 +529,9 @@ TActorId TKafkaReadSessionActor::CreatePipeClient(ui64 tabletId, const TActorCon } void TKafkaReadSessionActor::RegisterBalancerSession(const TString& topic, const TActorId& pipe, const TVector<ui32>& groups, const TActorContext& ctx) { - KAFKA_LOG_I("register session: topic# " << topic ); + YDB_LOG_INFO("Register session", + {LogPrefix()}, + {"topic", topic}); auto request = MakeHolder<TEvPersQueue::TEvRegisterReadSession>(); auto& req = request->Record; @@ -524,7 +549,10 @@ void TKafkaReadSessionActor::RegisterBalancerSession(const TString& topic, const void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; - KAFKA_LOG_D("partition lock is coming from PQRB topic# " << record.GetTopic() << ", partition# " << record.GetPartition()); + YDB_LOG_DEBUG("Partition lock is coming from PQRB", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); Y_ABORT_UNLESS(record.GetSession() == Session); Y_ABORT_UNLESS(record.GetClientId() == GroupId); @@ -537,9 +565,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path)); if (converterIter == FullPathToConverter.end()) { - KAFKA_LOG_I("ignored ev lock topic# " << record.GetTopic() - << ", partition# " << record.GetPartition() - << ", reason# path not recognized"); + YDB_LOG_INFO("Ignored ev lock path not recognized", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); return; } @@ -547,9 +576,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: auto topicInfoIt = TopicsInfo.find(topicName); if (topicInfoIt == TopicsInfo.end() || (topicInfoIt->second.PipeClient != ActorIdFromProto(record.GetPipeClient()))) { - KAFKA_LOG_I("ignored ev lock topic# " << record.GetTopic() - << ", partition# " << record.GetPartition() - << ", reason# topic is unknown"); + YDB_LOG_INFO("Ignored ev lock topic is unknown", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); return; } @@ -562,7 +592,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePartition::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; const ui32 group = record.HasGroup() ? record.GetGroup() : 0; - KAFKA_LOG_D("partition release is coming from PQRB topic# " << record.GetTopic() << ", group# " << group); + YDB_LOG_DEBUG("Partition release is coming from PQRB", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"group", group}); Y_ABORT_UNLESS(record.GetSession() == Session); Y_ABORT_UNLESS(record.GetClientId() == GroupId); @@ -574,8 +607,9 @@ void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePart Y_ABORT_UNLESS(topicInfoIt != TopicsInfo.end()); if (topicInfoIt->second.PipeClient != ActorIdFromProto(record.GetPipeClient())) { - KAFKA_LOG_I("ignored ev release topic# " << record.GetTopic() - << ", reason# topic is unknown"); + YDB_LOG_INFO("Ignored ev release topic is unknown", + {LogPrefix()}, + {"topic", record.GetTopic()}); return; } @@ -622,13 +656,17 @@ void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePart } } - KAFKA_LOG_I("ignored ev release topic# " << record.GetTopic() - << ", reason# partition " << partitionToRelease << " isn`t locked"); + YDB_LOG_INFO("Ignored ev release partition isn`t locked", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partitionToRelease", partitionToRelease}); } void TKafkaReadSessionActor::InformBalancerAboutPartitionRelease(const TString& topic, ui64 partition, const TActorContext& ctx) { - KAFKA_LOG_I("released topic# " << topic - << ", partition# " << partition); + YDB_LOG_INFO("Released", + {LogPrefix()}, + {"topic", topic}, + {"partition", partition}); auto request = MakeHolder<TEvPersQueue::TEvPartitionReleased>(); auto topicIt = TopicsInfo.find(topic); diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h index 3ca986a6bf7..aa2b89b32ab 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h @@ -92,7 +92,7 @@ public: private: using TActorContext = NActors::TActorContext; - TString LogPrefix(); + NStructuredLog::TStructuredMessage LogPrefix(); void Die(const TActorContext& ctx) override; diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp b/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp index 96ab74a505e..16ab825f3e2 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp @@ -2,6 +2,8 @@ #include "kafka_read_session_utils.h" #include "kafka_balancer_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { KafkaReadSessionProxyActor::KafkaReadSessionProxyActor(const TContext::TPtr context, ui64 cookie) @@ -19,14 +21,18 @@ template<bool handlePending, typename TRequest> void KafkaReadSessionProxyActor::DoHandle(TRequest& ev, const TString& event) { if constexpr (handlePending) { if (Context->ReadSession.PendingBalancingMode.has_value()) { - KAFKA_LOG_D("DoHandle " << event << " with pending balance mode"); + YDB_LOG_DEBUG("DoHandle with pending balance mode", + {LogPrefix()}, + {"event", event}); auto response = CreateChangeResponse(*ev->Get()->Request); Send(Context->ConnectionId, new TEvKafka::TEvResponse(ev->Get()->CorrelationId, response, EKafkaErrors::REBALANCE_IN_PROGRESS)); return; } } - KAFKA_LOG_D("DoHandle " << event); + YDB_LOG_DEBUG("DoHandle", + {LogPrefix()}, + {"event", event}); switch (Context->ReadSession.BalancingMode) { case EBalancingMode::Native: Register(new TKafkaBalancerActor(Context, 0, ev->Get()->CorrelationId, ev->Get()->Request)); @@ -40,10 +46,13 @@ void KafkaReadSessionProxyActor::DoHandle(TRequest& ev, const TString& event) { } void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvJoinGroupRequest::TPtr& ev) { - KAFKA_LOG_D("Handle TEvKafka::TEvJoinGroupRequest"); + YDB_LOG_DEBUG("Handle TEvKafka::TEvJoinGroupRequest", + {LogPrefix()}); Context->ReadSession.BalancingMode = Context->ReadSession.PendingBalancingMode.value_or(GetBalancingMode(*ev->Get()->Request)); Context->ReadSession.PendingBalancingMode.reset(); - KAFKA_LOG_D("Balancing mode: " << Context->ReadSession.BalancingMode); + YDB_LOG_DEBUG("Balancing", + {LogPrefix()}, + {"mode", Context->ReadSession.BalancingMode}); DoHandle<false>(ev, "TEvKafka::TEvJoinGroupRequest"); } @@ -61,7 +70,8 @@ void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvLeaveGroupRequest::TPtr& ev } void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvFetchRequest::TPtr& ev) { - KAFKA_LOG_D("Handle TEvKafka::TEvFetchRequest"); + YDB_LOG_DEBUG("Handle TEvKafka::TEvFetchRequest", + {LogPrefix()}); Register(CreateKafkaFetchActor(Context, ev->Get()->CorrelationId, ev->Get()->Request)); } diff --git a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp index a34d01f29e7..cb9a5d7042f 100644 --- a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp @@ -12,6 +12,8 @@ #include <ydb/library/login/sasl/scram.h> #include <ydb/services/persqueue_v1/actors/persqueue_utils.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -244,8 +246,9 @@ void TKafkaSaslAuthActor::SendResponse() { responseToClient->AuthBytes = ToRawBytes(responseToClient->AuthBytesStr); responseToClient->ErrorMessage = ""; - KAFKA_LOG_D("Authentication first step finished." - << " FirstServerMessage='" << AuthResponse << "'"); + YDB_LOG_DEBUG("Authentication first step finished. FirstServerMessage='", + {LogPrefix()}, + {"authResponse", AuthResponse}); auto evResponse = std::make_unique<TEvKafka::TEvResponse>(CorrelationId, responseToClient, EKafkaErrors::NONE_ERROR); Send(Context->ConnectionId, evResponse.release()); @@ -265,7 +268,9 @@ void TKafkaSaslAuthActor::SendResponseAndDie(EKafkaErrors errorCode, Ydb::Status TStringBuilder authenticationFailureReason; authenticationFailureReason << (errorMessage ? " " + errorMessage : "") << (details ? " " + details : ""); - KAFKA_LOG_ERROR("Authentication failure." << authenticationFailureReason); + YDB_LOG_ERROR("Authentication failure", + {LogPrefix()}, + {"authenticationFailureReason", authenticationFailureReason}); const auto& securityConfig = AppData()->DomainsConfig.GetSecurityConfig(); TStringBuilder responseErrorMessage; responseErrorMessage << "Authentication failure."; @@ -278,12 +283,14 @@ void TKafkaSaslAuthActor::SendResponseAndDie(EKafkaErrors errorCode, Ydb::Status auto authResult = new TEvKafka::TEvAuthResult(EAuthSteps::FAILED, evResponse, errorMessage); Send(Context->ConnectionId, authResult); } else { - KAFKA_LOG_D("Authentication success. Database='" << DatabasePath << "', " - << "FolderId='" << FolderId << "', " - << "ServiceAccountId='" << ServiceAccountId << "', " - << "DatabaseId='" << DatabaseId << "', " - << "Coordinator='" << Coordinator << "', " - << "ResourcePath='" << ResourcePath << "'"); + YDB_LOG_DEBUG("Authentication success. Database=' FolderId=' ServiceAccountId=' DatabaseId=' Coordinator=' ResourcePath='", + {LogPrefix()}, + {"databasePath", DatabasePath}, + {"folderId", FolderId}, + {"serviceAccountId", ServiceAccountId}, + {"databaseId", DatabaseId}, + {"coordinator", Coordinator}, + {"resourcePath", ResourcePath}); responseToClient->ErrorMessage = ""; auto evResponse = std::make_shared<TEvKafka::TEvResponse>(CorrelationId, responseToClient, errorCode); diff --git a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h index 9788d073e4a..6e67085e391 100644 --- a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h @@ -34,7 +34,9 @@ public: private: STATEFN(StateWork) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvKafka::TEvAuthRequest, HandleAuthRequest); HFunc(TEvKafka::TEvMtlsAuthRequest, HandleMtlsAuthRequest); @@ -43,7 +45,9 @@ private: } STATEFN(StateResolveDatabase) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); CFunc(TEvents::TEvPoison::EventType, Die); @@ -51,7 +55,9 @@ private: } STATEFN(StateResolveSharedDatabase) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); CFunc(TEvents::TEvPoison::EventType, Die); @@ -59,7 +65,9 @@ private: } STATEFN(StateSaslPlainLogin) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(NSasl::TEvSasl::TEvSaslPlainLoginResponse, HandleLoginResult); HFunc(NSasl::TEvSasl::TEvSaslPlainLdapLoginResponse, HandleLoginResult); @@ -69,7 +77,9 @@ private: } STATEFN(StateSaslScramLogin) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { hFunc(NSasl::TEvSasl::TEvSaslScramFirstServerResponse, HandleFirstLoginResponse); HFunc(TEvKafka::TEvAuthRequest, HandleAuthRequest); @@ -80,7 +90,9 @@ private: } STATEFN(StateTicketResolve) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTicketParser::TEvAuthorizeTicketResult, Handle); CFunc(TEvents::TEvPoison::EventType, Die); diff --git a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp index 66f26739d61..d69ce77c21d 100644 --- a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp @@ -8,6 +8,8 @@ #include <util/generic/cast.h> #include <regex> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + #define VALIDATE_PRODUCER_IN_REQUEST(ErrorResponseType) \ if (!ProducerInRequestIsValid(ev->Get()->Request)) { \ auto& kafkaRequest = ev->Get()->Request; \ @@ -22,7 +24,8 @@ if (!ProducerInRequestIsValid(ev->Get()->Request)) { \ namespace NKafka { void TTransactionActor::Handle(TEvKafka::TEvAddPartitionsToTxnRequest::TPtr& ev, const TActorContext&){ - KAFKA_LOG_D("Received ADD_PARTITIONS_TO_TXN request"); + YDB_LOG_DEBUG("Received ADD_PARTITIONS_TO_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TAddPartitionsToTxnResponseData); for (auto& topicInRequest : ev->Get()->Request->Topics) { @@ -37,13 +40,15 @@ namespace NKafka { // in YDB Topics we store offsets in table and do not need this extra action. // Thus we can just ignore this request. void TTransactionActor::Handle(TEvKafka::TEvAddOffsetsToTxnRequest::TPtr& ev, const TActorContext&) { - KAFKA_LOG_D("Received ADD_OFFSETS_TO_TXN request"); + YDB_LOG_DEBUG("Received ADD_OFFSETS_TO_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TAddOffsetsToTxnResponseData); SendOkResponse<TAddOffsetsToTxnResponseData>(ev); } void TTransactionActor::Handle(TEvKafka::TEvTxnOffsetCommitRequest::TPtr& ev, const TActorContext&) { - KAFKA_LOG_D("Received TXN_OFFSET_COMMIT request"); + YDB_LOG_DEBUG("Received TXN_OFFSET_COMMIT request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TTxnOffsetCommitResponseData); // save offsets for future use @@ -79,7 +84,8 @@ namespace NKafka { 7. Close KQP session, send to coordinator TEvTransactionActorDied, die. */ void TTransactionActor::Handle(TEvKafka::TEvEndTxnRequest::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received END_TXN request"); + YDB_LOG_DEBUG("Received END_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TEndTxnResponseData); bool txnAborted = !ev->Get()->Request->Committed; @@ -103,7 +109,8 @@ namespace NKafka { } void TTransactionActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -116,9 +123,13 @@ namespace NKafka { } void TTransactionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP for " << GetAsStr(LastSentToKqpRequest) << " request"); + YDB_LOG_DEBUG("Received query response from KQP for request", + {LogPrefix()}, + {"lastSentToKqpRequest", GetAsStr(LastSentToKqpRequest)}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}, + {"error", error}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error->data()); Die(ctx); return; @@ -141,12 +152,16 @@ namespace NKafka { void TTransactionActor::StartKqpSession(const TActorContext& ctx) { Kqp = std::make_unique<TKqpTxHelper>(AppData(ctx)->TenantName); - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } void TTransactionActor::SendToKqpValidationRequests(const TActorContext& ctx) { - KAFKA_LOG_D("Sending select request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending select request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendYqlRequest( GetYqlWithTablesNames(), BuildSelectParams(), @@ -170,9 +185,14 @@ namespace NKafka { template<class ErrorResponseType, class EventType> void TTransactionActor::SendFailResponse(TAutoPtr<TEventHandle<EventType>>& evHandle, EKafkaErrors errorCode, const TString& errorMessage) { if (errorMessage) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } auto response = NKafkaTransactions::BuildResponse<ErrorResponseType>(evHandle->Get()->Request, errorCode); @@ -182,14 +202,19 @@ namespace NKafka { template<class ResponseType, class EventType> void TTransactionActor::SendOkResponse(TAutoPtr<TEventHandle<EventType>>& evHandle) { auto& kafkaRequest = evHandle->Get()->Request; - KAFKA_LOG_D("Sending OK response to " << evHandle->Get()->ConnectionId << " with correlationId " << evHandle->Get()->CorrelationId << " and transactionalId " << TransactionalId); + YDB_LOG_DEBUG("Sending OK response to with correlationId and transactionalId", + {LogPrefix()}, + {"connectionId", evHandle->Get()->ConnectionId}, + {"correlationId", evHandle->Get()->CorrelationId}, + {"transactionalId", TransactionalId}); std::shared_ptr<ResponseType> response = NKafkaTransactions::BuildResponse<ResponseType>(kafkaRequest, EKafkaErrors::NONE_ERROR); Send(evHandle->Get()->ConnectionId, new TEvKafka::TEvResponse(evHandle->Get()->CorrelationId, response, EKafkaErrors::NONE_ERROR)); } // helper methods void TTransactionActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -294,7 +319,8 @@ namespace NKafka { int expectedResultsSize = OffsetsToCommit.empty() ? 1 : 2; // if there were no consumer in transactions we do not send request to consumer table if (expectedResultsSize != resultsSize) { TString error = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected " << expectedResultsSize << ", got " << resultsSize << "."; - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error); Die(ctx); return; @@ -306,13 +332,15 @@ namespace NKafka { producerState = ParseProducerState(response); } catch (const yexception& y) { TString error = TStringBuilder() << "Error parsing producer state response from KQP. Reason: " << y.what(); - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error); Die(ctx); return; } if (auto error = GetErrorInProducerState(producerState)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::PRODUCER_FENCED, error->data()); Die(ctx); return; @@ -322,28 +350,32 @@ namespace NKafka { // parse and validate consumers std::unordered_map<TString, i32> consumerGenerationsByName = ParseConsumersGenerations(response); if (auto error = GetErrorInConsumersStates(consumerGenerationsByName)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::PRODUCER_FENCED, error->data()); Die(ctx); return; } } - KAFKA_LOG_D("Validated producer and consumers states. Everything is alright, adding kafka operations to transaction."); + YDB_LOG_DEBUG("Validated producer and consumers states. Everything is alright, adding kafka operations to transaction", + {LogPrefix()}); auto kqpTxnId = response.Record.GetResponse().GetTxMeta().id(); // finally everything is valid and we can add kafka operations to transaction and attempt to commit SendAddKafkaOperationsToTxRequest(kqpTxnId); } void TTransactionActor::HandleAddKafkaOperationsResponse(const TString& kqpTransactionId, const TActorContext& ctx) { - KAFKA_LOG_D("Successfully added kafka operations to transaction. Committing transaction."); + YDB_LOG_DEBUG("Successfully added kafka operations to transaction. Committing transaction", + {LogPrefix()}); Kqp->SetTxId(kqpTransactionId); Kqp->CommitTx(++KqpCookie, ctx); LastSentToKqpRequest = EKafkaTxnKqpRequests::COMMIT; } void TTransactionActor::HandleCommitResponse(const TActorContext& ctx) { - KAFKA_LOG_D("Successfully committed transaction. Sending ok and dying."); + YDB_LOG_DEBUG("Successfully committed transaction. Sending ok and dying", + {LogPrefix()}); SendOkResponse<TEndTxnResponseData>(EndTxnRequestPtr); Die(ctx); } diff --git a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h index 2daa546b98c..ace54800325 100644 --- a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h @@ -48,8 +48,15 @@ namespace NKafka { TxnTimeoutMs(txnTimeoutMs), CreatedAt(TAppData::TimeProvider->Now()) {}; - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTransactionActor{TransactionalId=" << TransactionalId << "; ProducerId=" << ProducerInstanceId.Id << "; ProducerEpoch=" << ProducerInstanceId.Epoch << "}: "; + + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTransactionActor"}, + {"selfId", SelfId()}, + {"transactionalId", TransactionalId}, + {"producerId", ProducerInstanceId.Id}, + {"producerEpoch", ProducerInstanceId.Epoch}, + {"databasePath", DatabasePath}); } private: @@ -65,7 +72,9 @@ namespace NKafka { HFunc(TEvents::TEvPoison, Handle); } } catch (const yexception& y) { - KAFKA_LOG_CRIT("Critical error happened. Reason: " << y.what()); + YDB_LOG_CRIT_COMP(NKikimrServices::KAFKA_PROXY, "Critical error happened", + {LogPrefix()}, + {"reason", y.what()}); if (EndTxnRequestPtr) { SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::UNKNOWN_SERVER_ERROR, y.what()); } diff --git a/ydb/core/kafka_proxy/kafka_connection.cpp b/ydb/core/kafka_proxy/kafka_connection.cpp index 2685c3dd1df..fdf2f498e1c 100644 --- a/ydb/core/kafka_proxy/kafka_connection.cpp +++ b/ydb/core/kafka_proxy/kafka_connection.cpp @@ -11,9 +11,14 @@ #include "actors/actors.h" #include "kafka_connection.h" #include "kafka_events.h" + + #include <ydb/core/kafka_proxy/kafka_log_impl.h> + #include "kafka_metrics.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NActors; @@ -126,11 +131,14 @@ public: Become(&TKafkaConnection::StateAccepting); Schedule(InactivityTimeout, InactivityEvent = new TEvPollerReady(nullptr, false, false)); - KAFKA_LOG_I("incoming connection opened " << Address); + YDB_LOG_INFO("Incoming connection opened", + {LogPrefix()}, + {"address", Address}); } void PassAway() override { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); ConnectionEstablished = false; if (ProduceActorId) { @@ -149,7 +157,9 @@ public: protected: void LogEvent(IEventHandle& ev) { - KAFKA_LOG_T("Received event: " << ev.GetTypeName()); + YDB_LOG_TRACE("Received", + {LogPrefix()}, + {"event", ev.GetTypeName()}); } void SetNonBlock() noexcept { @@ -157,7 +167,8 @@ protected: } void Shutdown() { - KAFKA_LOG_D("Shutdown"); + YDB_LOG_DEBUG("Shutdown", + {LogPrefix()}); PendingRequests.clear(); PendingRequestsQueue.clear(); @@ -168,7 +179,9 @@ protected: } ssize_t SocketSend(const void* data, size_t size) { - KAFKA_LOG_T("SocketSend Size=" << size); + YDB_LOG_TRACE("SocketSend", + {LogPrefix()}, + {"size", size}); return Socket->Send(data, size); } @@ -184,18 +197,24 @@ protected: return Socket->GetRawSocket(); } - TString LogPrefix() const { - TStringBuilder sb; - sb << "TKafkaConnection " << SelfId() << "(#" << GetRawSocket() << "," << Address->ToString() << ") State: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + TString state; + auto stateFunc = CurrentStateFunc(); if (stateFunc == &TKafkaConnection::StateConnected) { - sb << "Connected "; + state = "Connected"; } else if (stateFunc == &TKafkaConnection::StateAccepting) { - sb << "Accepting "; + state = "Accepting"; } else { - sb << "Unknown "; + state = "Unknown"; } - return sb; + + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaConnection"}, + {"selfId", SelfId()}, + {"state", state}, + {"rawSocket", GetRawSocket()}, + {"address", Address->ToString()}); } void SendRequestMetrics(const TActorContext& ctx) { @@ -241,7 +260,9 @@ protected: HFunc(TEvKafka::TEvResponse, Handle); sFunc(NActors::TEvents::TEvPoison, PassAway); default: - KAFKA_LOG_ERROR("TKafkaConnection: Unexpected " << ev.Get()->GetTypeName()); + YDB_LOG_ERROR("TKafkaConnection: Unexpected", + {LogPrefix()}, + {"typeName", ev.Get()->GetTypeName()}); } } @@ -397,12 +418,17 @@ protected: } bool ProcessRequest(const TActorContext& ctx) { - KAFKA_LOG_D("process message: ApiKey=" << Request->Header.RequestApiKey << ", ExpectedSize=" << Request->ExpectedSize - << ", Size=" << Request->Size); + YDB_LOG_DEBUG("Process message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}, + {"expectedSize", Request->ExpectedSize}, + {"size", Request->Size}); auto apiKeyNameIt = EApiKeyNames.find(static_cast<EApiKey>(Request->Header.RequestApiKey)); if (apiKeyNameIt == EApiKeyNames.end()) { - KAFKA_LOG_ERROR("Unsupported message: ApiKey=" << Request->Header.RequestApiKey); + YDB_LOG_ERROR("Unsupported message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}); PassAway(); return false; } @@ -418,7 +444,8 @@ protected: } if (IsTransactionalApiKey(Request->Header.RequestApiKey) && !TransactionsEnabled()) { - KAFKA_LOG_ERROR("Transactional API keys are not enabled. To enable them set \"EnableKafkaTransactions\" feature flag to true in cluster configuration."); + YDB_LOG_ERROR("Transactional API keys are not enabled. To enable them set \"EnableKafkaTransactions\" feature flag to true in cluster configuration", + {LogPrefix()}); PassAway(); return false; } @@ -524,7 +551,9 @@ protected: break; default: - KAFKA_LOG_ERROR("Unsupported message: ApiKey=" << Request->Header.RequestApiKey); + YDB_LOG_ERROR("Unsupported message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}); PassAway(); return false; } @@ -545,7 +574,9 @@ protected: void Handle(TEvKafka::TEvReadSessionInfo::TPtr readInfo, const TActorContext& /*ctx*/) { auto r = readInfo->Get(); - KAFKA_LOG_D("Initializing GroupId=" << r->GroupId); + YDB_LOG_DEBUG("Initializing", + {LogPrefix()}, + {"groupId", r->GroupId}); Context->GroupId = r->GroupId; } @@ -557,7 +588,9 @@ protected: if (IsSslActive && NKikimr::AppData()->KafkaProxyConfig.GetMtlsEnable()) { MtlsAuthStage = AUTH_FAILED; } - KAFKA_LOG_ERROR(event->Error); + YDB_LOG_ERROR("", + {LogPrefix()}, + {"error", event->Error}); Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); CloseConnection = true; return; @@ -569,7 +602,9 @@ protected: responseToClient->ErrorCode = kafkaError; TString errorMessage = TStringBuilder() << Context->SaslMechanism << " authentication mechanism is disabled, because mTLS flag is on. Turn of mTLS in configuration to use this mechanism."; responseToClient->ErrorMessage = errorMessage; - KAFKA_LOG_D(errorMessage); + YDB_LOG_DEBUG("Dump logPrefix, errorMessage", + {LogPrefix()}, + {"errorMessage", errorMessage}); std::shared_ptr<TEvKafka::TEvResponse> errorResponse = std::make_shared<TEvKafka::TEvResponse>(event->ClientResponse->CorrelationId, responseToClient, kafkaError); Reply(event->ClientResponse->CorrelationId, errorResponse->Response, kafkaError, ctx); @@ -588,7 +623,9 @@ protected: Context->IsServerless = event->IsServerless; Context->ResourceDatabasePath = event->ResourceDatabasePath ? NKikimr::CanonizePath(event->ResourceDatabasePath) : Context->DatabasePath; - KAFKA_LOG_D("Authentication successful. SID=" << Context->UserToken->GetUserSID()); + YDB_LOG_DEBUG("Authentication successful", + {LogPrefix()}, + {"SID", Context->UserToken->GetUserSID()}); if (Context->SaslMechanism != "MTLS") { Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); } else { @@ -607,7 +644,8 @@ protected: auto errorResponse = std::make_shared<TEvKafka::TEvResponse>(event->ClientResponse->CorrelationId, responseToClient, kafkaError); TString errorMessage = TStringBuilder() << event->SaslMechanism << " authentication mechanism is disabled, because mTLS flag is on. Turn of mTLS in configuration to use this mechanism."; - KAFKA_LOG_D(errorMessage); + YDB_LOG_DEBUG(errorMessage, + {LogPrefix()}); Reply(event->ClientResponse->CorrelationId, errorResponse->Response, kafkaError, ctx); CloseConnection = true; return; @@ -616,7 +654,8 @@ protected: Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); auto authStep = event->AuthStep; if (authStep == EAuthSteps::FAILED) { - KAFKA_LOG_ERROR(event->Error); + YDB_LOG_ERROR(event->Error, + {LogPrefix()}); CloseConnection = true; return; } @@ -636,7 +675,9 @@ protected: void Reply(const ui64 correlationId, TApiMessage::TPtr response, EKafkaErrors errorCode, const TActorContext& ctx) { auto it = PendingRequests.find(correlationId); if (it == PendingRequests.end()) { - KAFKA_LOG_ERROR("Unexpected correlationId " << correlationId); + YDB_LOG_ERROR("Unexpected correlationId", + {LogPrefix()}, + {"correlationId", correlationId}); return; } @@ -651,7 +692,9 @@ protected: } void OnRequestProcessed(const Msg::TPtr& request) { - KAFKA_LOG_T("Request with correlationId " << request->Header.CorrelationId << " processed. Erasing it from PendingRequests and PendingRequestsQueue"); + YDB_LOG_TRACE("Request with correlationId processed. Erasing it from PendingRequests and PendingRequestsQueue", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); InflightSize -= request->ExpectedSize; PendingRequests.erase(request->Header.CorrelationId); PendingRequestsQueue.pop_front(); @@ -660,9 +703,13 @@ protected: bool ProcessReplyQueue(const TActorContext& ctx) { while(!PendingRequestsQueue.empty()) { auto& request = PendingRequestsQueue.front(); - KAFKA_LOG_T("Processing reply queue for request with correlationId " << request->Header.CorrelationId); + YDB_LOG_TRACE("Processing reply queue for request with correlationId", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); if (request->Response.get() == nullptr) { - KAFKA_LOG_T("Response for request with correlationId " << request->Header.CorrelationId << " is empty."); + YDB_LOG_TRACE("Response for request with correlationId is empty", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); break; } @@ -681,7 +728,11 @@ protected: } bool Reply(const TRequestHeaderData* header, const TApiMessage* reply, const TString method, const TInstant requestStartTime, EKafkaErrors errorCode, const TActorContext& ctx) { - KAFKA_LOG_T("Building reply for method " << method << " and correlationId " << header->CorrelationId << " with error code: " << errorCode); + YDB_LOG_TRACE("Building reply for method and correlationId with error", + {LogPrefix()}, + {"method", method}, + {"correlationId", header->CorrelationId}, + {"code", errorCode}); TKafkaVersion headerVersion = ResponseHeaderVersion(header->RequestApiKey, header->RequestApiVersion); TKafkaVersion version = header->RequestApiVersion; @@ -706,19 +757,28 @@ protected: // PollerReady means that poller polled socket ready status if (res == -EAGAIN || res == -EWOULDBLOCK) { RetryingWriteToSocket = true; - KAFKA_LOG_D("Socket is busy. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size() << ". Waiting for PollerReady event"); + YDB_LOG_DEBUG("Socket is busy. Buffer queue Waiting for PollerReady event", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); RequestPoller(); return false; } else if (res < 0) { ythrow yexception() << "Error during flush of the written to socket data. Error code: " << strerror(-res) << " (" << res << ")"; } else { - KAFKA_LOG_D("Sent reply: ApiKey=" << header->RequestApiKey << ", Version=" << version << ", Correlation=" << responseHeader.CorrelationId << ", Size=" << size); + YDB_LOG_DEBUG("Sent reply", + {LogPrefix()}, + {"apiKey", header->RequestApiKey}, + {"version", version}, + {"correlation", responseHeader.CorrelationId}, + {"size", size}); } } catch(const yexception& e) { - KAFKA_LOG_ERROR("error on processing response: ApiKey=" << reply->ApiKey() - << ", Version=" << version - << ", CorrelationId=" << header->CorrelationId - << ", Error=" << e.what()); + YDB_LOG_ERROR("Error on processing response", + {LogPrefix()}, + {"apiKey", reply->ApiKey()}, + {"version", version}, + {"correlationId", header->CorrelationId}, + {"error", e.what()}); PassAway(); return false; } @@ -730,7 +790,9 @@ protected: if (IsSslRequired && !IsSslActive) { int res = Socket->TryUpgradeToSecure(NKikimrServices::KAFKA_PROXY, ServerCreds); if (res < 0) { - KAFKA_LOG_ERROR("connection closed - error in UpgradeToSecure: " << strerror(-res)); + YDB_LOG_ERROR("Connection closed - error in UpgradeToSecure", + {LogPrefix()}, + {"error", strerror(-res)}); PassAway(); return false; } @@ -740,7 +802,10 @@ protected: } bool DoRead(const TActorContext& ctx) { - KAFKA_LOG_T("DoRead: Demand=" << Demand.Length << ", Step=" << static_cast<i32>(Step)); + YDB_LOG_TRACE("DoRead", + {LogPrefix()}, + {"demand", Demand.Length}, + {"step", static_cast<i32>(Step)}); for (;;) { while (Demand) { ssize_t received = 0; @@ -750,11 +815,14 @@ protected: } else if (-res == EINTR) { continue; } else if (!res) { - KAFKA_LOG_I("connection closed"); + YDB_LOG_INFO("Connection closed", + {LogPrefix()}); PassAway(); return false; } else if (res < 0) { - KAFKA_LOG_I("connection closed - error in recv: " << strerror(-res)); + YDB_LOG_INFO("Connection closed - error", + {LogPrefix()}, + {"error", strerror(-res)}); PassAway(); return false; } @@ -775,18 +843,24 @@ protected: case SIZE_PREPARE: NormalizeNumber(Request->ExpectedSize); if (Request->ExpectedSize < 0) { - KAFKA_LOG_ERROR("Wrong message size. Size: " << Request->ExpectedSize); + YDB_LOG_ERROR("Wrong message size", + {LogPrefix()}, + {"size", Request->ExpectedSize}); PassAway(); return false; } if ((ui64)Request->ExpectedSize > Context->Config.GetMaxMessageSize()) { - KAFKA_LOG_ERROR("message is big. Size: " << Request->ExpectedSize << ". MaxSize: " - << Context->Config.GetMaxMessageSize()); + YDB_LOG_ERROR("Message is big", + {LogPrefix()}, + {"size", Request->ExpectedSize}, + {"maxSize", Context->Config.GetMaxMessageSize()}); PassAway(); return false; } if (static_cast<size_t>(Request->ExpectedSize) < HeaderSize) { - KAFKA_LOG_ERROR("message is small. Size: " << Request->ExpectedSize); + YDB_LOG_ERROR("Message is small", + {LogPrefix()}, + {"size", Request->ExpectedSize}); PassAway(); return false; } @@ -798,12 +872,17 @@ protected: case INFLIGHT_CHECK: if (!Context->Authenticated() && !PendingRequestsQueue.empty()) { // Allow only one message to be processed at a time for non-authenticated users - KAFKA_LOG_ERROR("DoRead: failed inflight check: there are " << PendingRequestsQueue.size() << " pending requests and user is not authnicated. Only one paraller request is allowed for a non-authenticated user."); + YDB_LOG_ERROR("DoRead: failed inflight check: there are pending requests and user is not authnicated. Only one paraller request is allowed for a non-authenticated user", + {LogPrefix()}, + {"pendingRequestsQueue", PendingRequestsQueue.size()}); return true; } if (InflightSize + Request->ExpectedSize > Context->Config.GetMaxInflightSize()) { // We limit the size of processed messages so as not to exceed the size of available memory - KAFKA_LOG_ERROR("DoRead: failed inflight check: InflightSize + Request->ExpectedSize=" << InflightSize + Request->ExpectedSize << " > Context->Config.GetMaxInflightSize=" << Context->Config.GetMaxInflightSize()); + YDB_LOG_ERROR("DoRead: failed inflight check: InflightSize + >", + {LogPrefix()}, + {"expectedSize", InflightSize + Request->ExpectedSize}, + {"getMaxInflightSize", Context->Config.GetMaxInflightSize()}); return true; } InflightSize += Request->ExpectedSize; @@ -812,7 +891,9 @@ protected: [[fallthrough]]; case HEADER_READ: - KAFKA_LOG_T("start read header. ExpectedSize=" << Request->ExpectedSize); + YDB_LOG_TRACE("Start read header", + {LogPrefix()}, + {"expectedSize", Request->ExpectedSize}); Request->Buffer->Resize(HeaderSize); Demand = TReadDemand(Request->Buffer->Data(), HeaderSize); @@ -830,12 +911,16 @@ protected: NormalizeNumber(Request->CorrelationId); if (PendingRequests.contains(Request->CorrelationId)) { - KAFKA_LOG_ERROR("CorrelationId " << Request->CorrelationId << " already processing"); + YDB_LOG_ERROR("CorrelationId already processing", + {LogPrefix()}, + {"correlationId", Request->CorrelationId}); PassAway(); return false; } if (!Context->Authenticated() && RequireAuthentication(static_cast<EApiKey>(Request->ApiKey))) { - KAFKA_LOG_ERROR("unauthenticated request: ApiKey=" << Request->ApiKey); + YDB_LOG_ERROR("Unauthenticated request", + {LogPrefix()}, + {"apiKey", Request->ApiKey}); PassAway(); return false; } @@ -845,7 +930,9 @@ protected: [[fallthrough]]; case MESSAGE_READ: - KAFKA_LOG_T("start read new message. ExpectedSize=" << Request->ExpectedSize); + YDB_LOG_TRACE("Start read new message", + {LogPrefix()}, + {"expectedSize", Request->ExpectedSize}); Request->Buffer->Resize(Request->ExpectedSize); Demand = TReadDemand(Request->Buffer->Data() + HeaderSize, Request->ExpectedSize - HeaderSize); @@ -856,10 +943,18 @@ protected: case MESSAGE_PROCESS: Request->StartTime = TInstant::Now(); if constexpr (DEBUG_ENABLED) { - KAFKA_LOG_D("received message. ApiKey=" << Request->ApiKey << ", Version=" << Request->ApiVersion << ", CorrelationId=" << Request->CorrelationId - << ", Data=" << Hex(Request->Buffer->Begin(), Request->Buffer->End())); + YDB_LOG_DEBUG("Received message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}, + {"data", Hex(Request->Buffer->Begin(), Request->Buffer->End())}); } else { - KAFKA_LOG_D("received message. ApiKey=" << Request->ApiKey << ", Version=" << Request->ApiVersion << ", CorrelationId=" << Request->CorrelationId); + YDB_LOG_DEBUG("Received message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}); } TKafkaReadable readable(*Request->Buffer); @@ -871,10 +966,12 @@ protected: Request->Header.Read(readable, RequestHeaderVersion(Request->ApiKey, Request->ApiVersion)); Request->Message->Read(readable, Request->ApiVersion); } catch(const yexception& e) { - KAFKA_LOG_ERROR("error on processing message: ApiKey=" << Request->ApiKey - << ", Version=" << Request->ApiVersion - << ", CorrelationId=" << Request->CorrelationId - << ", Error=" << e.what()); + YDB_LOG_ERROR("Error on processing message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}, + {"error", e.what()}); PassAway(); return false; } @@ -882,7 +979,8 @@ protected: Step = SIZE_READ; if (IsSslActive && NKikimr::AppData()->KafkaProxyConfig.GetMtlsEnable() && MtlsAuthStage != MtlsAuthStages::AUTH_SUCCESSFUL) { - KAFKA_LOG_D("Mtls authentication was not successful."); + YDB_LOG_DEBUG("Mtls authentication was not successful", + {LogPrefix()}); return false; } @@ -910,14 +1008,17 @@ protected: if (sslHandshakeResult != SslHandshakeErrors::ERROR_WANT_READ && sslHandshakeResult != SslHandshakeErrors::ERROR_WANT_WRITE && sslHandshakeResult != SslHandshakeErrors::ERROR_NONE) { - KAFKA_LOG_D("Error in ssl handshake, ssl ErrorCode=" << sslHandshakeResult); + YDB_LOG_DEBUG("Error in ssl handshake, ssl", + {LogPrefix()}, + {"errorCode", sslHandshakeResult}); PassAway(); return; } if (sslHandshakeResult == SslHandshakeErrors::ERROR_NONE) { TSslHelpers::TSslHolder<X509> cert = Socket->GetSslClientCert(); if (!cert) { - KAFKA_LOG_ERROR("No cert was received from client during ssl handshake for mTLS authentication."); + YDB_LOG_ERROR("No cert was received from client during ssl handshake for mTLS authentication", + {LogPrefix()}); PassAway(); return; } @@ -943,7 +1044,8 @@ protected: if (event->Get() == InactivityEvent) { const TDuration passed = TDuration::Seconds(std::abs(InactivityTimer.Passed())); if (passed >= InactivityTimeout) { - KAFKA_LOG_D("connection closed by inactivity timeout"); + YDB_LOG_DEBUG("Connection closed by inactivity timeout", + {LogPrefix()}); return PassAway(); // timeout } else { Schedule(InactivityTimeout - passed, InactivityEvent = new TEvPollerReady(nullptr, false, false)); @@ -951,21 +1053,32 @@ protected: } } if (event->Get()->Write && !BufferedWriter.Empty()) { - KAFKA_LOG_D("Retrying flush. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size()); + YDB_LOG_DEBUG("Retrying flush. Buffer queue", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); ssize_t res = BufferedWriter.flush(); if (res == -EAGAIN || res == -EWOULDBLOCK) { - KAFKA_LOG_D("Socket is busy during retry. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size() << ". Waiting for PollerReady event"); + YDB_LOG_DEBUG("Socket is busy during retry. Buffer queue Waiting for PollerReady event", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); RequestPoller(); return; } else if (res < 0) { - KAFKA_LOG_ERROR("connection closed - error in FlushOutput: " << strerror(-res) << ". Buffer queue size: " << BufferedWriter.GetBuffersDeque().size()); + YDB_LOG_ERROR("Connection closed - error Buffer queue", + {LogPrefix()}, + {"flushOutput", strerror(-res)}, + {"size", BufferedWriter.GetBuffersDeque().size()}); PassAway(); return; } else if (res > 0 && BufferedWriter.Empty()) { // we successfuly retried sending the response RetryingWriteToSocket = false; auto& request = PendingRequestsQueue.front(); auto& header = request->Header; - KAFKA_LOG_D("Sent reply (after retry): ApiKey=" << header.RequestApiKey << ", Version=" << header.RequestApiVersion << ", Correlation=" << header.CorrelationId); + YDB_LOG_DEBUG("Sent reply (after retry)", + {LogPrefix()}, + {"apiKey", header.RequestApiKey}, + {"version", header.RequestApiVersion}, + {"correlation", header.CorrelationId}); OnRequestProcessed(request); ProcessReplyQueue(ctx); @@ -976,7 +1089,8 @@ protected: } if (CloseConnection && BufferedWriter.Empty()) { - KAFKA_LOG_D("connection closed"); + YDB_LOG_DEBUG("Connection closed", + {LogPrefix()}); return PassAway(); } @@ -1003,7 +1117,9 @@ protected: sFunc(TEvKafka::TEvKillReadSession, HandleKillReadSession); sFunc(NActors::TEvents::TEvPoison, PassAway); default: - KAFKA_LOG_ERROR("TKafkaConnection: Unexpected " << ev.Get()->GetTypeName()); + YDB_LOG_ERROR("TKafkaConnection: Unexpected", + {LogPrefix()}, + {"typeName", ev.Get()->GetTypeName()}); } } }; diff --git a/ydb/core/kafka_proxy/kafka_log_impl.h b/ydb/core/kafka_proxy/kafka_log_impl.h index c6ba1e8e5dc..df427e3d00a 100644 --- a/ydb/core/kafka_proxy/kafka_log_impl.h +++ b/ydb/core/kafka_proxy/kafka_log_impl.h @@ -6,14 +6,6 @@ namespace NKafka { -inline TString LogPrefix() { return {}; } +inline NActors::NStructuredLog::TStructuredMessage LogPrefix() { return {}; } } - -#define KAFKA_LOG_T(stream) LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_NOTICE(stream) LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) diff --git a/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp b/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp index d950b37cb80..dc7fb3e1339 100644 --- a/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp +++ b/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp @@ -3,6 +3,8 @@ #include "actors/txn_actor_response_builder.h" #include <ydb/core/kqp/common/simple/services.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { // Handles new transactional_id+producer_id+producer_epoch: // 1. validates that producer is not a zombie (in case of parallel init_producer_requests) @@ -49,7 +51,8 @@ namespace NKafka { void TTransactionsCoordinator::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext&) { TablesInited++; if (TablesInited == TABLES_COUNT) { - KAFKA_LOG_D("All tables are prepared"); + YDB_LOG_DEBUG("All tables are prepared", + {LogPrefix()}); } }; @@ -59,28 +62,35 @@ namespace NKafka { if (it == ProducersByTransactionalId.end() || it->second.Id != deadActorProducerState) { // new actor was already registered, we can just ignore this event - KAFKA_LOG_D("Received TEvTransactionActorDied for transactionalId " << ev->Get()->TransactionalId << " but producer has already been reinitialized with new epoch or deleted. Ignoring this event"); + YDB_LOG_DEBUG("Received TEvTransactionActorDied for transactionalId but producer has already been reinitialized with new epoch or deleted. Ignoring this event", + {LogPrefix()}, + {"transactionalId", ev->Get()->TransactionalId}); } else { - KAFKA_LOG_D("Received TEvTransactionActorDied for transactionalId " << ev->Get()->TransactionalId - << " and producerId " << ev->Get()->ProducerState.Id - << " and producerEpoch " << ev->Get()->ProducerState.Epoch - << ". Erasing info about this actor."); + YDB_LOG_DEBUG("Received TEvTransactionActorDied for transactionalId and producerId and producerEpoch Erasing info about this actor", + {LogPrefix()}, + {"transactionalId", ev->Get()->TransactionalId}, + {"producerStateId", ev->Get()->ProducerState.Id}, + {"producerStateEpoch", ev->Get()->ProducerState.Epoch}); // erase info about actor to prevent future zombie requests from this producer TxnActorByTransactionalId.erase(ev->Get()->TransactionalId); } }; void TTransactionsCoordinator::Handle(TEvents::TEvPoison::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Got poison pill, killing all transaction actors"); + YDB_LOG_DEBUG("Got poison pill, killing all transaction actors", + {LogPrefix()}); for (auto& [transactionalId, txnActorId]: TxnActorByTransactionalId) { ctx.Send(txnActorId, new TEvents::TEvPoison()); - KAFKA_LOG_D("Sent poison pill to transaction actor for transactionalId " << transactionalId); + YDB_LOG_DEBUG("Sent poison pill to transaction actor for transactionalId", + {LogPrefix()}, + {"transactionalId", transactionalId}); } PassAway(); }; void TTransactionsCoordinator::PassAway() { - KAFKA_LOG_D("Killing myself"); + YDB_LOG_DEBUG("Killing myself", + {LogPrefix()}); TBase::PassAway(); }; @@ -90,7 +100,10 @@ namespace NKafka { template<class ErrorResponseType, class EventType> void TTransactionsCoordinator::HandleTransactionalRequest(TAutoPtr<TEventHandle<EventType>>& evHandle, const TActorContext& ctx) { EventType* ev = evHandle->Get(); - KAFKA_LOG_D("Received message for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Received message for transactionalId and ApiKey", + {LogPrefix()}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); // create helper struct to simplify methods interaction auto txnRequest = TTransactionalRequest( @@ -108,7 +121,8 @@ namespace NKafka { template<class ErrorResponseType, class RequestType> void TTransactionsCoordinator::SendProducerFencedResponse(TMessagePtr<RequestType> kafkaRequest, const TString& error, const TTransactionalRequest& txnRequestDetails) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); std::shared_ptr<ErrorResponseType> response = NKafkaTransactions::BuildResponse<ErrorResponseType>(kafkaRequest, EKafkaErrors::PRODUCER_FENCED); Send(txnRequestDetails.ConnectionId, new TEvKafka::TEvResponse(txnRequestDetails.CorrelationId, response, EKafkaErrors::PRODUCER_FENCED)); }; @@ -125,9 +139,17 @@ namespace NKafka { txnActorId = ctx.Register(new TTransactionActor(*ev->Request->TransactionalId, {ev->Request->ProducerId, ev->Request->ProducerEpoch}, ev->DatabasePath, producerInstance.TxnTimeoutMs, ev->ResourceDatabasePath ? ev->ResourceDatabasePath : ev->DatabasePath)); TxnActorByTransactionalId[*ev->Request->TransactionalId] = txnActorId; - KAFKA_LOG_D("Registered TTransactionActor with id " << txnActorId << " for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Registered TTransactionActor with id for transactionalId and ApiKey", + {LogPrefix()}, + {"txnActorId", txnActorId}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); } - KAFKA_LOG_D("Forwarded message to TTransactionActor with id " << txnActorId << " for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Forwarded message to TTransactionActor with id for transactionalId and ApiKey", + {LogPrefix()}, + {"txnActorId", txnActorId}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); TAutoPtr<IEventHandle> tmpPtr = evHandle.Release(); ctx.Forward(tmpPtr, txnActorId); }; diff --git a/ydb/core/kafka_proxy/kafka_transactions_coordinator.h b/ydb/core/kafka_proxy/kafka_transactions_coordinator.h index ff982afbdb6..4dc147ba157 100644 --- a/ydb/core/kafka_proxy/kafka_transactions_coordinator.h +++ b/ydb/core/kafka_proxy/kafka_transactions_coordinator.h @@ -51,8 +51,10 @@ namespace NKafka { TBase::Become(&TTransactionsCoordinator::StateWork); } - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTransactionsCoordinator "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTransactionsCoordinator"}, + {"selfId", SelfId()}); } void PassAway() override; diff --git a/ydb/core/kqp/common/buffer/events.h b/ydb/core/kqp/common/buffer/events.h index 922f32ea981..a40580e8539 100644 --- a/ydb/core/kqp/common/buffer/events.h +++ b/ydb/core/kqp/common/buffer/events.h @@ -9,6 +9,11 @@ namespace NKikimr { namespace NKqp { +struct TCommitTimestamp { + ui64 PlanStep = 0; + ui64 TxId = 0; +}; + struct TEvKqpBuffer { // To BufferActor @@ -34,8 +39,12 @@ struct TEvTerminate : public TEventLocal<TEvTerminate, TKqpBufferWriterEvents::E struct TEvResult : public TEventLocal<TEvResult, TKqpBufferWriterEvents::EvResult> { TEvResult() = default; TEvResult(NYql::NDqProto::TDqTaskStats&& stats) : Stats(std::move(stats)) {} + TEvResult(NYql::NDqProto::TDqTaskStats&& stats, std::optional<TCommitTimestamp>&& commitTimestamp) + : Stats(std::move(stats)) + , CommitTimestamp(std::move(commitTimestamp)) {} std::optional<NYql::NDqProto::TDqTaskStats> Stats; + std::optional<TCommitTimestamp> CommitTimestamp; }; struct TEvError : public TEventLocal<TEvError, TKqpBufferWriterEvents::EvError> { diff --git a/ydb/core/kqp/common/events/query.h b/ydb/core/kqp/common/events/query.h index cb9fba143df..8c883c04074 100644 --- a/ydb/core/kqp/common/events/query.h +++ b/ydb/core/kqp/common/events/query.h @@ -18,7 +18,7 @@ #include <memory> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { class ISessionUpdater; class IQueryClassifier; } @@ -368,19 +368,19 @@ public: return UserRequestContext; } - void SetWmSessionUpdater(const std::shared_ptr<NWorkload::ISessionUpdater>& wmSessionUpdater) { + void SetWmSessionUpdater(const std::shared_ptr<NWorkloadManager::ISessionUpdater>& wmSessionUpdater) { WmSessionUpdater = wmSessionUpdater; } - std::shared_ptr<NWorkload::ISessionUpdater> GetWmSessionUpdater() const { + std::shared_ptr<NWorkloadManager::ISessionUpdater> GetWmSessionUpdater() const { return WmSessionUpdater; } - void SetWmQueryClassifier(std::shared_ptr<NWorkload::IQueryClassifier> classifier) { + void SetWmQueryClassifier(std::shared_ptr<NWorkloadManager::IQueryClassifier> classifier) { WmQueryClassifier = std::move(classifier); } - std::shared_ptr<NWorkload::IQueryClassifier> GetWmQueryClassifier() const { + std::shared_ptr<NWorkloadManager::IQueryClassifier> GetWmQueryClassifier() const { return WmQueryClassifier; } @@ -521,8 +521,8 @@ private: std::shared_ptr<const NKikimrKqp::TQueryPhysicalGraph> QueryPhysicalGraph; i64 Generation = 0; bool DisableDefaultTimeout = false; - std::shared_ptr<NWorkload::ISessionUpdater> WmSessionUpdater; - std::shared_ptr<NWorkload::IQueryClassifier> WmQueryClassifier; + std::shared_ptr<NWorkloadManager::ISessionUpdater> WmSessionUpdater; + std::shared_ptr<NWorkloadManager::IQueryClassifier> WmQueryClassifier; }; struct TEvDataQueryStreamPart: public TEventPB<TEvDataQueryStreamPart, diff --git a/ydb/core/kqp/common/events/workload_service.cpp b/ydb/core/kqp/common/events/workload_service.cpp deleted file mode 100644 index 690db6a4904..00000000000 --- a/ydb/core/kqp/common/events/workload_service.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "workload_service.h" diff --git a/ydb/core/kqp/common/events/ya.make b/ydb/core/kqp/common/events/ya.make index 260928b73a4..160a1ccefa0 100644 --- a/ydb/core/kqp/common/events/ya.make +++ b/ydb/core/kqp/common/events/ya.make @@ -3,7 +3,6 @@ LIBRARY() SRCS( events.cpp query.cpp - workload_service.cpp ) PEERDIR( diff --git a/ydb/core/kqp/common/simple/kqp_event_ids.h b/ydb/core/kqp/common/simple/kqp_event_ids.h index d269f585847..3dda9315aa6 100644 --- a/ydb/core/kqp/common/simple/kqp_event_ids.h +++ b/ydb/core/kqp/common/simple/kqp_event_ids.h @@ -189,17 +189,7 @@ struct TKqpResourceInfoExchangerEvents { }; }; -struct TKqpWorkloadServiceEvents { - enum EKqpWorkloadServiceEvents { - EvPlaceRequestIntoPool = EventSpaceBegin(TKikimrEvents::ES_KQP) + 700, - EvContinueRequest, - EvCleanupRequest, - EvCleanupResponse, - EvUpdatePoolInfo, - EvSubscribeOnPoolChanges, - EvFetchDatabaseResponse, - }; -}; + struct TKqpBufferWriterEvents { enum EKqpBufferWriterEvents { diff --git a/ydb/core/kqp/common/simple/services.h b/ydb/core/kqp/common/simple/services.h index 2a53d208071..b978b0ea593 100644 --- a/ydb/core/kqp/common/simple/services.h +++ b/ydb/core/kqp/common/simple/services.h @@ -41,10 +41,6 @@ inline NActors::TActorId MakeKqpFinalizeScriptServiceId(ui32 nodeId) { return NActors::TActorId(nodeId, TStringBuf(name, 12)); } -inline NActors::TActorId MakeKqpWorkloadServiceId(ui32 nodeId) { - const char name[12] = "kqp_workld"; - return NActors::TActorId(nodeId, TStringBuf(name, 12)); -} inline NActors::TActorId MakeKqpSchedulerServiceId(ui32 nodeId) { const char name[12] = "kqp_schdlr"; diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 9e81c83f4ac..bd2267a62a3 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -301,6 +301,7 @@ public: if (ev->Get()->Stats && Stats) { Stats->AddBufferStats(std::move(*ev->Get()->Stats)); } + ResponseEv->CommitTimestamp = std::move(ev->Get()->CommitTimestamp); MakeResponseAndPassAway(); } diff --git a/ydb/core/kqp/executer_actor/kqp_executer.h b/ydb/core/kqp/executer_actor/kqp_executer.h index 6450d9fb5cf..3ead2fe51ff 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer.h +++ b/ydb/core/kqp/executer_actor/kqp_executer.h @@ -4,6 +4,7 @@ #include <ydb/core/kqp/common/kqp_batch_operations.h> #include <ydb/core/kqp/common/kqp_tx.h> #include <ydb/core/kqp/common/kqp_event_ids.h> +#include <ydb/core/kqp/common/buffer/events.h> #include <ydb/core/kqp/common/kqp_user_request_context.h> #include <ydb/core/kqp/executer_actor/kqp_partition_helper.h> #include <ydb/core/kqp/executer_actor/shards_resolver/kqp_shards_resolver_events.h> @@ -32,6 +33,7 @@ struct TEvKqpExecuter { NLWTrace::TOrbit Orbit; IKqpGateway::TKqpSnapshot Snapshot; + std::optional<TCommitTimestamp> CommitTimestamp; std::optional<NYql::TKikimrPathId> BrokenLockPathId; std::optional<ui64> BrokenLockShardId; std::optional<ui64> BrokenLockQuerySpanId; diff --git a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp index 961a6803288..52555e07776 100644 --- a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp @@ -1233,7 +1233,8 @@ public: if (state == Ydb::Table::SetNotNullState::STATE_DONE) { return ReplyErrorAndDie(Ydb::StatusIds::SUCCESS, record.MutableIssues()); - } else if (state == Ydb::Table::SetNotNullState::STATE_CANCELLED) { + } else if (state == Ydb::Table::SetNotNullState::STATE_CANCELLED || + state == Ydb::Table::SetNotNullState::STATE_REJECTED) { return ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, record.MutableIssues()); } else { return ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, record.MutableIssues()); diff --git a/ydb/core/kqp/gateway/behaviour/ya.make b/ydb/core/kqp/gateway/behaviour/ya.make index cc2c79e609f..374a4c9eab2 100644 --- a/ydb/core/kqp/gateway/behaviour/ya.make +++ b/ydb/core/kqp/gateway/behaviour/ya.make @@ -1,7 +1,5 @@ RECURSE( external_data_source - resource_pool - resource_pool_classifier streaming_query table tablestore diff --git a/ydb/core/kqp/gateway/ya.make b/ydb/core/kqp/gateway/ya.make index 737a4db8a46..c536651ceb5 100644 --- a/ydb/core/kqp/gateway/ya.make +++ b/ydb/core/kqp/gateway/ya.make @@ -14,8 +14,9 @@ PEERDIR( ydb/core/kqp/federated_query/actors ydb/core/kqp/gateway/actors ydb/core/kqp/gateway/behaviour/external_data_source - ydb/core/kqp/gateway/behaviour/resource_pool - ydb/core/kqp/gateway/behaviour/resource_pool_classifier + ydb/services/workload_manager/metadata_subscription + ydb/services/workload_manager/metadata_subscription/resource_pool_classifier + ydb/services/workload_manager/service ydb/core/kqp/gateway/behaviour/streaming_query ydb/core/kqp/gateway/behaviour/table ydb/core/kqp/gateway/behaviour/tablestore diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp index cd7e2a8f400..df938397094 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp @@ -165,6 +165,26 @@ TVector<TExprNode::TPtr> TPhysicalQueryBuilder::BuildPhysicalStageGraph() { TExprNode::TPtr stage; if (Graph.IsSourceStageRowType(id)) { stage = Stages.at(id); + // Want to build materialize for ranges. + // TODO: Actually old optimizer has some machinery to compute ranges during compilation for some cases, for + // example when `LiteralRange` is defined, but currenlty we put any case in separate tx. Performance improvement is possible here. + auto rowSettingsPtr = FindNode(stage, [](const TExprNode::TPtr& node) { return !!TMaybeNode<TKqpReadRangesSourceSettings>(node); }); + if (rowSettingsPtr) { + auto rowSettings = TExprBase(rowSettingsPtr).Cast<TKqpReadRangesSourceSettings>(); + if (!rowSettings.RangesExpr().Maybe<TCoVoid>()) { + const auto materializeResult = BuildMaterialize(rowSettings.RangesExpr().Ptr()); + // clang-fomrat off + const auto newRowSettings = Build<TKqpReadRangesSourceSettings>(ctx, rowSettingsPtr->Pos()) + .Table(rowSettings.Table()) + .Columns(rowSettings.Columns()) + .Settings(rowSettings.Settings()) + .RangesExpr(materializeResult) + .ExplainPrompt(rowSettings.ExplainPrompt()) + .Done().Ptr(); + // clang-format on + stage = ctx.ReplaceNode(std::move(stage), rowSettings.Ref(), newRowSettings); + } + } } else { TVector<TExprNode::TPtr> stageInputConnections; TVector<TExprNode::TPtr> stageInputArgs; diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp index 53efaa30b8f..bcb8695fa2c 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp @@ -34,6 +34,8 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { for (const auto& column : Read->Columns) { columns.push_back(Ctx.NewAtom(Pos, column)); } + // Extract ranges. + TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); switch (Read->GetTableStorageType()) { case NYql::EStorageType::RowStorage: { @@ -48,7 +50,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { .Add(columns) .Build() .Settings<TCoNameValueTupleList>().Build() - .RangesExpr<TCoVoid>().Build() + .RangesExpr(ranges) .ExplainPrompt<TCoNameValueTupleList>().Build() .Build() .Done().Ptr(); @@ -108,7 +110,6 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { settings.SequentialInFlight = 1; } - TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); // clang-format off auto olapRead = Build<TKqpBlockReadOlapTableRanges>(Ctx, Pos) .Table(Read->TableCallable) @@ -139,7 +140,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { break; } default: - Y_ENSURE(false, "Unsupported table source type"); + Y_ENSURE(false, "Unsupported table source type."); } YQL_CLOG(TRACE, CoreDq) << "[NEW RBO Physical source] " << KqpExprToPrettyString(TExprBase(source), Ctx); diff --git a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp index 80a8978fc6b..3614c0aa3b0 100644 --- a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp +++ b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp @@ -123,8 +123,8 @@ bool IsSuitableToExtractAndPushRanges(const TIntrusivePtr<IOperator>& input) { } const auto read = CastOperator<TOpRead>(maybeRead); - // Currently supported only for cs. - return !read->GetRanges() && read->GetTableStorageType() == NYql::EStorageType::ColumnStorage; + const auto tableType = read->GetTableStorageType(); + return !read->GetRanges() && (tableType == NYql::EStorageType::ColumnStorage || tableType == NYql::EStorageType::RowStorage); } TPredicateExtractorSettings PrepareExtractorSettings(TKqpOptimizeContext& kqpCtx) { diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_databases_cache.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_databases_cache.cpp index 4b949031032..3253fb50a87 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_databases_cache.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_databases_cache.cpp @@ -1,6 +1,6 @@ #include "kqp_proxy_service_impl.h" -#include <ydb/core/kqp/workload_service/actors/actors.h> +#include <ydb/services/workload_manager/actors/actors.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> @@ -69,7 +69,7 @@ public: if (databaseStateIt == DatabaseStates.End()) { DatabaseStates.Insert({database, TDatabaseState{.Database = database}}); - Register(NWorkload::CreateDatabaseFetcherActor(SelfId(), database)); + Register(NWorkloadManager::CreateDatabaseFetcherActor(SelfId(), database)); StartIdleCheck(); return; } @@ -87,7 +87,7 @@ public: } } - void Handle(NWorkload::TEvFetchDatabaseResponse::TPtr& ev) { + void Handle(NWorkloadManager::TEvFetchDatabaseResponse::TPtr& ev) { auto databaseStateIt = DatabaseStates.Find(ev->Get()->Database); if (databaseStateIt == DatabaseStates.End()) { return; @@ -144,7 +144,7 @@ public: STRICT_STFUNC(StateFunc, hFunc(TEvPrivate::TEvSubscribeOnDatabase, Handle); hFunc(TEvPrivate::TEvPingDatabaseSubscription, Handle); - hFunc(NWorkload::TEvFetchDatabaseResponse, Handle); + hFunc(NWorkloadManager::TEvFetchDatabaseResponse, Handle); sFunc(TEvents::TEvPoison, HandlePoison); sFunc(TEvents::TEvWakeup, HandleWakeup); diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp index 2935c169ebe..ad76fb9cb92 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp @@ -15,7 +15,7 @@ #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/row_dispatcher/row_dispatcher_service.h> #include <ydb/core/kqp/common/events/script_executions.h> -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/kqp/common/kqp_lwtrace_probes.h> #include <ydb/core/kqp/common/kqp_timeouts.h> #include <ydb/core/kqp/compile_service/kqp_compile_service.h> @@ -27,11 +27,10 @@ #include <ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.h> #include <ydb/core/kqp/gateway/behaviour/streaming_query/behaviour.h> #include <ydb/core/kqp/node_service/kqp_node_service.h> -#include <ydb/core/kqp/workload_service/kqp_query_classifier.h> +#include <ydb/services/workload_manager/query_classifier.h> #include <ydb/core/kqp/proxy_service/kqp_query_text_cache_service.h> #include <ydb/core/kqp/rm_service/kqp_rm_service.h> #include <ydb/core/kqp/session_actor/kqp_worker_common.h> -#include <ydb/core/kqp/workload_service/kqp_workload_service.h> #include <ydb/core/mon/mon.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> #include <ydb/core/protos/workload_manager_config.pb.h> @@ -350,7 +349,7 @@ public: limits.RemoteChannelInflightBytes = config.GetRemoteChannelInflightBytes(); limits.RemoteSessionInflightBytes = config.GetRemoteSessionInflightBytes(); limits.ReconciliationCount = config.GetReconciliationCount(); - limits.CleanupPeriod = TDuration::MilliSeconds(std::max<ui64>(config.GetCleanupPeriodMs(), 200)); + limits.CleanupPeriod = TDuration::MilliSeconds(config.GetCleanupPeriodMs()); limits.IdlePingPeriod = TDuration::MilliSeconds(config.GetIdlePingPeriodMs()); limits.IdleDestroyPeriod = TDuration::MilliSeconds(config.GetIdleDestroyPeriodMs()); } else { // deprecated @@ -390,10 +389,6 @@ public: TActivationContext::ActorSystem()->RegisterLocalService( MakeKqpNodeServiceID(SelfId().NodeId()), KqpNodeService); - KqpWorkloadService = TActivationContext::Register(CreateKqpWorkloadService(Counters->GetWorkloadManagerCounters())); - TActivationContext::ActorSystem()->RegisterLocalService( - MakeKqpWorkloadServiceId(SelfId().NodeId()), KqpWorkloadService); - auto updateFairSharePeriod = TDuration::MilliSeconds(TableServiceConfig.GetComputeSchedulerSettings().GetUpdateFairShareMs()); KqpComputeSchedulerService = TActivationContext::Register(CreateKqpComputeSchedulerService(updateFairSharePeriod)); TActivationContext::ActorSystem()->RegisterLocalService( @@ -516,7 +511,6 @@ public: Send(SpillingService, new TEvents::TEvPoison); Send(KqpNodeService, new TEvents::TEvPoison); - Send(KqpWorkloadService, new TEvents::TEvPoison()); Send(KqpComputeSchedulerService, new TEvents::TEvPoison()); Send(KqpQueryTextCacheService, new TEvents::TEvPoison()); if (RowDispatcherService) { @@ -1445,7 +1439,7 @@ public: hFunc(TEvInterconnect::TEvNodeDisconnected, Handle); hFunc(TEvKqp::TEvListSessionsRequest, Handle); hFunc(TEvKqp::TEvListProxyNodesRequest, Handle); - hFunc(NWorkload::TEvUpdatePoolInfo, Handle); + hFunc(NWorkloadManager::TEvUpdatePoolInfo, Handle); hFunc(TEvKqp::TEvUpdateDatabaseInfo, Handle); hFunc(TEvKqp::TEvDelayedRequestError, Handle); hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); @@ -1663,14 +1657,14 @@ private: auto poolId = ev->Get()->GetPoolId(); ResourcePoolsCache.GetPoolInfo(databaseId, poolId ? poolId : NResourcePool::DEFAULT_POOL_ID, ActorContext()); - auto context = TClassifyContext{ + auto context = NWorkloadManager::TClassifyContext{ // This parameter is set when user creates a request with an explicit PoolId .PoolId = poolId, .AppName = sessionInfo ? sessionInfo->ClientApplicationName : "", .UserToken = ev->Get()->GetUserToken() }; - auto classifier = NWorkload::CreateQueryClassifier( + auto classifier = NWorkloadManager::CreateQueryClassifier( ResourcePoolsCache.GetLastResourcePoolMapSnapshot(), ResourcePoolsCache.GetClassifierViewFor(databaseId), databaseId, @@ -1941,7 +1935,7 @@ private: Send(ev->Sender, result.release(), 0, ev->Cookie); } - void Handle(NWorkload::TEvUpdatePoolInfo::TPtr& ev) { + void Handle(NWorkloadManager::TEvUpdatePoolInfo::TPtr& ev) { ResourcePoolsCache.UpdatePoolInfo(ev->Get()->DatabaseId, ev->Get()->PoolId, ev->Get()->Config, ev->Get()->SecurityObject, ActorContext()); } @@ -1958,7 +1952,7 @@ private: } void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - ResourcePoolsCache.UpdateResourcePoolClassifiersInfo(ev->Get()->GetValidatedSnapshotAs<TResourcePoolClassifierSnapshot>(), ActorContext()); + ResourcePoolsCache.UpdateResourcePoolClassifiersInfo(ev->Get()->GetValidatedSnapshotAs<NWorkloadManager::TResourcePoolClassifierSnapshot>(), ActorContext()); } void InitSharedReading() { @@ -2077,7 +2071,6 @@ private: TActorId KqpNodeService; TActorId SpillingService; TActorId WhiteBoardService; - TActorId KqpWorkloadService; TActorId KqpComputeSchedulerService; TActorId KqpQueryTextCacheService; TActorId RowDispatcherService; diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h b/ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h index d34a10f4929..d7e469534ef 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h @@ -3,14 +3,14 @@ #include <ydb/core/base/appdata.h> #include <ydb/core/base/path.h> #include <ydb/core/kqp/common/kqp.h> -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/kqp/counters/kqp_counters.h> -#include <ydb/core/kqp/workload_service/kqp_query_classifier.h> -#include <ydb/core/kqp/proxy_service/kqp_session_state.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h> +#include <ydb/services/workload_manager/query_classifier.h> +#include <ydb/services/workload_manager/session_updater.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h> #include <ydb/core/kqp/rm_service/kqp_rm_service.h> #include <ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.h> -#include <ydb/core/kqp/workload_service/kqp_workload_service.h> #include <ydb/core/protos/feature_flags.pb.h> #include <ydb/core/protos/kqp.pb.h> #include <ydb/core/protos/workload_manager_config.pb.h> @@ -79,9 +79,9 @@ public: } }; -class TWmSessionUpdater final : public NWorkload::ISessionUpdater { +class TWmSessionUpdater final : public NWorkloadManager::ISessionUpdater { public: - using EState = NWorkload::ISessionUpdater::EState; + using EState = NWorkloadManager::ISessionUpdater::EState; void SetRequestState(EState state, TInstant timestamp) override { const ui64 ts = timestamp.MicroSeconds(); @@ -511,12 +511,12 @@ public: } std::optional<TPoolInfo> GetPoolInfo(const TString& databaseId, const TString& poolId, TActorContext actorContext) const { - auto it = PoolsCache.find(GetPoolKey(databaseId, poolId)); + auto it = PoolsCache.find(NWorkloadManager::GetPoolKey(databaseId, poolId)); if (it == PoolsCache.end()) { Y_ASSERT(!poolId.empty()); actorContext.Send(MakeKqpSchedulerServiceId(actorContext.SelfID.NodeId()), new NScheduler::TEvAddPool(databaseId, poolId)); - actorContext.Send(MakeKqpWorkloadServiceId(actorContext.SelfID.NodeId()), new NWorkload::TEvSubscribeOnPoolChanges(databaseId, poolId)); + actorContext.Send(NWorkloadManager::MakeServiceId(actorContext.SelfID.NodeId()), new NWorkloadManager::TEvSubscribeOnPoolChanges(databaseId, poolId)); return std::nullopt; } return it->second; @@ -533,7 +533,7 @@ public: } void UpdatePoolInfo(const TString& databaseId, const TString& poolId, const std::optional<NResourcePool::TPoolSettings>& config, const std::optional<NACLib::TSecurityObject>& securityObject, TActorContext actorContext) { - const TString& poolKey = GetPoolKey(databaseId, poolId); + const TString& poolKey = NWorkloadManager::GetPoolKey(databaseId, poolId); if (!config) { auto it = PoolsCache.find(poolKey); if (it == PoolsCache.end()) { @@ -545,7 +545,7 @@ public: } else { // Refresh pool subscription it->second.Expired = true; - actorContext.Send(MakeKqpWorkloadServiceId(actorContext.SelfID.NodeId()), new NWorkload::TEvSubscribeOnPoolChanges(databaseId, poolId)); + actorContext.Send(NWorkloadManager::MakeServiceId(actorContext.SelfID.NodeId()), new NWorkloadManager::TEvSubscribeOnPoolChanges(databaseId, poolId)); } } else { auto& poolInfo = PoolsCache[poolKey]; @@ -557,7 +557,7 @@ public: BuildResourcePoolMapSnapshot(); } - void UpdateResourcePoolClassifiersInfo(std::shared_ptr<TResourcePoolClassifierSnapshot> snapshot, TActorContext actorContext) { + void UpdateResourcePoolClassifiersInfo(std::shared_ptr<NWorkloadManager::TResourcePoolClassifierSnapshot> snapshot, TActorContext actorContext) { LastClassifierSnapshot = snapshot; for (const auto& [databaseId, info] : snapshot->GetResourcePoolClassifierConfigs()) { for (const auto& [_, classifier] : info.ByName) { @@ -569,23 +569,23 @@ public: void UnsubscribeFromResourcePoolClassifiers(TActorContext actorContext) { if (SubscribedOnResourcePoolClassifiers) { SubscribedOnResourcePoolClassifiers = false; - actorContext.Send(NMetadata::NProvider::MakeServiceId(actorContext.SelfID.NodeId()), new NMetadata::NProvider::TEvUnsubscribeExternal(std::make_shared<TResourcePoolClassifierSnapshotsFetcher>())); + actorContext.Send(NMetadata::NProvider::MakeServiceId(actorContext.SelfID.NodeId()), new NMetadata::NProvider::TEvUnsubscribeExternal(std::make_shared<NWorkloadManager::TResourcePoolClassifierSnapshotsFetcher>())); } } - TClassifierConfigsView GetClassifierViewFor(const TString& databaseId) const { - return TClassifierConfigsView(LastClassifierSnapshot, databaseId); + NWorkloadManager::TClassifierConfigsView GetClassifierViewFor(const TString& databaseId) const { + return NWorkloadManager::TClassifierConfigsView(LastClassifierSnapshot, databaseId); } private: void BuildResourcePoolMapSnapshot() { - auto pools = std::make_shared<TResourcePoolMap>(); + auto pools = std::make_shared<NWorkloadManager::TResourcePoolMap>(); pools->reserve(PoolsCache.size()); for (const auto& [key, info] : PoolsCache) { if (!info.Expired) { - pools->emplace(key, TResourcePoolEntry{info.Config, info.SecurityObject}); + pools->emplace(key, NWorkloadManager::TResourcePoolEntry{info.Config, info.SecurityObject}); } } @@ -603,7 +603,7 @@ private: void SubscribeOnResourcePoolClassifiers(TActorContext actorContext) { if (!SubscribedOnResourcePoolClassifiers && NMetadata::NProvider::TServiceOperator::IsEnabled()) { SubscribedOnResourcePoolClassifiers = true; - actorContext.Send(NMetadata::NProvider::MakeServiceId(actorContext.SelfID.NodeId()), new NMetadata::NProvider::TEvSubscribeExternal(std::make_shared<TResourcePoolClassifierSnapshotsFetcher>())); + actorContext.Send(NMetadata::NProvider::MakeServiceId(actorContext.SelfID.NodeId()), new NMetadata::NProvider::TEvSubscribeExternal(std::make_shared<NWorkloadManager::TResourcePoolClassifierSnapshotsFetcher>())); } } @@ -620,17 +620,17 @@ private: } public: - const std::shared_ptr<const TResourcePoolClassifierSnapshot>& GetLastClassifierSnapshot() const { + const std::shared_ptr<const NWorkloadManager::TResourcePoolClassifierSnapshot>& GetLastClassifierSnapshot() const { return LastClassifierSnapshot; } - const std::shared_ptr<const TResourcePoolMap>& GetLastResourcePoolMapSnapshot() const { + const std::shared_ptr<const NWorkloadManager::TResourcePoolMap>& GetLastResourcePoolMapSnapshot() const { return LastResourcePoolMapSnapshot; } private: - std::shared_ptr<const TResourcePoolClassifierSnapshot> LastClassifierSnapshot; - std::shared_ptr<const TResourcePoolMap> LastResourcePoolMapSnapshot; + std::shared_ptr<const NWorkloadManager::TResourcePoolClassifierSnapshot> LastClassifierSnapshot; + std::shared_ptr<const NWorkloadManager::TResourcePoolMap> LastResourcePoolMapSnapshot; std::unordered_map<TString, TPoolInfo> PoolsCache; std::unordered_map<TString, TDatabaseInfo> DatabasesCache; diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp index 0d9be16a32f..dc5f1853399 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_ut.cpp @@ -4,7 +4,7 @@ #include <ydb/core/kqp/proxy_service/kqp_proxy_service.h> #include <ydb/core/kqp/proxy_service/kqp_proxy_service_impl.h> #include <ydb/core/kqp/common/kqp.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <ydb/core/protos/config.pb.h> #include <ydb/core/protos/kqp.pb.h> #include <ydb/core/testlib/test_client.h> @@ -644,7 +644,7 @@ Y_UNIT_TEST_SUITE(KqpProxy) { } Y_UNIT_TEST(DatabasesCacheForServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .Create(); diff --git a/ydb/core/kqp/proxy_service/kqp_session_info.cpp b/ydb/core/kqp/proxy_service/kqp_session_info.cpp index bcddfa27619..125b468a940 100644 --- a/ydb/core/kqp/proxy_service/kqp_session_info.cpp +++ b/ydb/core/kqp/proxy_service/kqp_session_info.cpp @@ -84,7 +84,7 @@ void TKqpSessionInfo::SerializeTo(::NKikimrKqp::TSessionInfo* proto, const TFiel if (fieldsMap.NeedField(VSessions::WmState::ColumnId)) { // 18 if (WmState) { - using EWmState = NWorkload::ISessionUpdater::EState; + using EWmState = NWorkloadManager::ISessionUpdater::EState; switch(WmState->GetState()) { case EWmState::NONE: { proto->SetWmState("NONE"); diff --git a/ydb/core/kqp/proxy_service/ut/ya.make b/ydb/core/kqp/proxy_service/ut/ya.make index eeca3a762a1..6bbc1d8fe4c 100644 --- a/ydb/core/kqp/proxy_service/ut/ya.make +++ b/ydb/core/kqp/proxy_service/ut/ya.make @@ -19,7 +19,7 @@ PEERDIR( ydb/core/kqp/run_script_actor ydb/core/kqp/proxy_service ydb/core/kqp/ut/common - ydb/core/kqp/workload_service/ut/common + ydb/services/workload_manager/ut/common ydb/public/lib/ut_helpers ydb/public/sdk/cpp/src/client/driver ydb/public/sdk/cpp/src/client/query diff --git a/ydb/core/kqp/proxy_service/ya.make b/ydb/core/kqp/proxy_service/ya.make index f2f29746d2a..221f57cbfab 100644 --- a/ydb/core/kqp/proxy_service/ya.make +++ b/ydb/core/kqp/proxy_service/ya.make @@ -20,12 +20,12 @@ PEERDIR( ydb/core/kqp/common/events ydb/core/kqp/compile_service ydb/core/kqp/counters - ydb/core/kqp/gateway/behaviour/resource_pool_classifier + ydb/services/workload_manager/metadata_subscription/resource_pool_classifier ydb/core/kqp/gateway/behaviour/streaming_query ydb/core/kqp/proxy_service/proto ydb/core/kqp/proxy_service/script_executions_utils ydb/core/kqp/run_script_actor - ydb/core/kqp/workload_service + ydb/services/workload_manager ydb/core/mind ydb/core/mon ydb/core/protos diff --git a/ydb/core/kqp/runtime/kqp_write_actor.cpp b/ydb/core/kqp/runtime/kqp_write_actor.cpp index 508bcc72d84..ae71723000d 100644 --- a/ydb/core/kqp/runtime/kqp_write_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_write_actor.cpp @@ -4792,6 +4792,12 @@ public: case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned: TxProxyMon->ClientTxStatusPlanned->Inc(); TxPlanned = true; + if (TxManager->GetIsolationLevel() == NKqpProto::ISOLATION_LEVEL_STRICT_SERIALIZABLE) { + AFL_ENSURE(res->Record.HasStepId()); + AFL_ENSURE(res->Record.HasTxId()); + AFL_ENSURE(TxId && *TxId == res->Record.GetTxId()); + CommitTimestamp = TCommitTimestamp{res->Record.GetStepId(), res->Record.GetTxId()}; + } break; case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusOutdated: @@ -5553,7 +5559,8 @@ public: CA_LOG_D("Committed TxId=" << TxId.value_or(0)); OnOperationFinished(Counters->BufferActorCommitLatencyHistogram); Send<ESendingType::Tail>(ExecuterActorId, new TEvKqpBuffer::TEvResult{ - BuildStats() + BuildStats(), + std::move(CommitTimestamp) }); ExecuterActorId = {}; AFL_ENSURE(GetTotalMemory() == 0); @@ -5945,6 +5952,7 @@ private: bool IsImmediateCommit = false; bool TxPlanned = false; std::optional<ui64> Coordinator; + std::optional<TCommitTimestamp> CommitTimestamp; ui64 LocksBrokenAsBreaker = 0; ui64 LocksBrokenAsVictim = 0; @@ -6142,30 +6150,61 @@ private: void Handle(TEvBufferWriteResult::TPtr& result) { CA_LOG_D("TKqpForwardWriteActor receive EvBufferWriteResult from " << BufferActorId); + AFL_ENSURE(!PendingResult); WriteToken = result->Get()->Token; if (TransformOutput) { - AFL_ENSURE(Alloc); - TGuard<NMiniKQL::TScopedAlloc> allocGuard(*Alloc); - for (const auto& batch : result->Get()->Data) { - for (const auto& row : GetRows(batch)) { - AFL_ENSURE(row.size() == ReturningColumnsTypes.size()); - NUdf::TUnboxedValue* outputRowItems = nullptr; - auto outputRow = HolderFactory.CreateDirectArrayHolder(ReturningColumnsTypes.size(), outputRowItems); + PendingResult.Reset(result->Release().Release()); + PendingBatchIndex = 0; + PendingRowIndex = 0; + ConsumePendingReturning(); + } else { + OnFlushed(); + } + } - for (size_t index = 0; index < ReturningColumnsTypes.size(); ++index) { - outputRowItems[index] = NMiniKQL::GetCellValue(row[index], ReturningColumnsTypes[index]); - } + void ConsumePendingReturning() { + AFL_ENSURE(TransformOutput); + AFL_ENSURE(PendingResult); + AFL_ENSURE(Alloc); + TGuard<NMiniKQL::TScopedAlloc> allocGuard(*Alloc); - AFL_ENSURE(TransformOutput->GetFillLevel() == NYql::NDq::EDqFillLevel::NoLimit); - TransformOutput->Consume(std::move(outputRow)); + const auto& data = PendingResult->Data; + while (PendingBatchIndex < data.size()) { + const auto rows = GetRows(data[PendingBatchIndex], PendingRowIndex); + for (size_t rowIndex = 0; rowIndex < rows.size(); ++rowIndex) { + if (TransformOutput->GetFillLevel() != NYql::NDq::EDqFillLevel::NoLimit) { + CA_LOG_D("TKqpForwardWriteActor TransformOutput is full, waiting for drain"); + Callbacks->ResumeExecution(); + return; + } + const auto& row = rows[rowIndex]; + AFL_ENSURE(row.size() == ReturningColumnsTypes.size()); + NUdf::TUnboxedValue* outputRowItems = nullptr; + auto outputRow = HolderFactory.CreateDirectArrayHolder(ReturningColumnsTypes.size(), outputRowItems); + for (size_t colIndex = 0; colIndex < ReturningColumnsTypes.size(); ++colIndex) { + outputRowItems[colIndex] = NMiniKQL::GetCellValue(row[colIndex], ReturningColumnsTypes[colIndex]); } + TransformOutput->Consume(std::move(outputRow)); + ++PendingRowIndex; } + PendingRowIndex = 0; + ++PendingBatchIndex; } + PendingResult.Reset(); + PendingRowIndex = 0; + PendingBatchIndex = 0; OnFlushed(); } + void OnOutputConsumerReady() final { + if (!PendingResult) { + return; + } + ConsumePendingReturning(); + } + void OnFlushed() { InFlight = false; @@ -6376,6 +6415,7 @@ private: if (TransformOutput) { AFL_ENSURE(Alloc); TGuard<NMiniKQL::TScopedAlloc> allocGuard(*Alloc); + PendingResult.Reset(); TransformOutput.Reset(); } @@ -6407,6 +6447,10 @@ private: NWilson::TSpan ForwardWriteActorSpan; NYql::NDq::IDqOutputConsumer::TPtr TransformOutput; + THolder<TEvBufferWriteResult> PendingResult; + size_t PendingBatchIndex = 0; + size_t PendingRowIndex = 0; + private: template<typename TArgs> NYql::NDq::IDqOutputConsumer::TPtr ExtractTransformOutput(TArgs&& args) { diff --git a/ydb/core/kqp/runtime/kqp_write_table.cpp b/ydb/core/kqp/runtime/kqp_write_table.cpp index 59f1de93a9c..36689723b3e 100644 --- a/ydb/core/kqp/runtime/kqp_write_table.cpp +++ b/ydb/core/kqp/runtime/kqp_write_table.cpp @@ -1347,11 +1347,11 @@ IDataBatchProjectionPtr CreateFulltextTokenizeProjection( AFL_ENSURE(false)("Unsupported primary key type", columnTypes[1].GetTypeId()); } -std::vector<TConstArrayRef<TCell>> GetRows(const NKikimr::NKqp::IDataBatchPtr& batch) { +std::vector<TConstArrayRef<TCell>> GetRows(const NKikimr::NKqp::IDataBatchPtr& batch, const size_t offset) { auto* data = dynamic_cast<TRowBatch*>(batch.Get()); AFL_ENSURE(data); const auto& batchRows = data->GetRows(); - return std::vector<TConstArrayRef<TCell>>(batchRows.begin(), batchRows.end()); + return std::vector<TConstArrayRef<TCell>>(batchRows.begin() + offset, batchRows.end()); } std::vector<TConstArrayRef<TCell>> CutColumns( diff --git a/ydb/core/kqp/runtime/kqp_write_table.h b/ydb/core/kqp/runtime/kqp_write_table.h index 6a7e7c8a72d..c6ff9a7193e 100644 --- a/ydb/core/kqp/runtime/kqp_write_table.h +++ b/ydb/core/kqp/runtime/kqp_write_table.h @@ -115,7 +115,8 @@ bool IsEqual( TConstArrayRef<NScheme::TTypeInfo> types); std::vector<TConstArrayRef<TCell>> GetRows( - const NKikimr::NKqp::IDataBatchPtr& batch); + const NKikimr::NKqp::IDataBatchPtr& batch, + const size_t offset = 0); std::vector<TConstArrayRef<TCell>> CutColumns( const std::vector<TConstArrayRef<TCell>>& rows, const ui32 columnsCount); diff --git a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp index ddd3f482dda..8288d5f2b5b 100644 --- a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp +++ b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp @@ -7,7 +7,8 @@ #include <ydb/core/base/feature_flags.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/core/kqp/common/simple/services.h> #include <ydb/core/protos/feature_flags.pb.h> #include <ydb/core/protos/table_service_config.pb.h> @@ -58,7 +59,7 @@ public: hFunc(TEvAddDatabase, Handle); hFunc(TEvRemoveDatabase, Handle); hFunc(TEvAddPool, Handle); - hFunc(NWorkload::TEvUpdatePoolInfo, Handle); + hFunc(NWorkloadManager::TEvUpdatePoolInfo, Handle); hFunc(TEvRemovePool, Handle); hFunc(TEvAddQuery, Handle); hFunc(TEvRemoveQuery, Handle); @@ -126,7 +127,7 @@ public: if (PoolSubscribtions.insert({std::make_pair(databaseId, poolId), {.IsFirstRemoval=false, .ExternalWeight=resourceWeight}}).second) { PoolExternalWeightSum += resourceWeight; Scheduler->AddOrUpdatePool(databaseId, poolId, attrs); - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new NWorkload::TEvSubscribeOnPoolChanges(databaseId, poolId)); + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new NWorkloadManager::TEvSubscribeOnPoolChanges(databaseId, poolId)); if (resourceWeight > Epsilon) { UpdatePoolsGuarantee(); } @@ -137,7 +138,7 @@ public: Y_ABORT("Unsupported yet"); } - void Handle(NWorkload::TEvUpdatePoolInfo::TPtr& ev) { + void Handle(NWorkloadManager::TEvUpdatePoolInfo::TPtr& ev) { const auto& databaseId = ev->Get()->DatabaseId; const auto& poolId = ev->Get()->PoolId; auto poolIt = PoolSubscribtions.find(std::make_pair(databaseId, poolId)); @@ -172,7 +173,7 @@ public: if (!poolIt->second.IsFirstRemoval) { // The first removal - try to re-subscribe in case it's just the pool removal from cache. poolIt->second.IsFirstRemoval = true; - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new NWorkload::TEvSubscribeOnPoolChanges(databaseId, poolId)); + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new NWorkloadManager::TEvSubscribeOnPoolChanges(databaseId, poolId)); } else { // The second removal - the pool was really removed. PoolSubscribtions.erase(poolIt); diff --git a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service_ut.cpp b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service_ut.cpp index a58729c92ff..6841a75a0d8 100644 --- a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service_ut.cpp +++ b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service_ut.cpp @@ -1,10 +1,10 @@ -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <ydb/library/testlib/helpers.h> namespace NKikimr::NKqp { -using namespace NWorkload; +using namespace NWorkloadManager; Y_UNIT_TEST_SUITE(KqpComputeSchedulerService) { diff --git a/ydb/core/kqp/runtime/ut/ya.make b/ydb/core/kqp/runtime/ut/ya.make index dd8197c8a6b..6259bfafde1 100644 --- a/ydb/core/kqp/runtime/ut/ya.make +++ b/ydb/core/kqp/runtime/ut/ya.make @@ -18,7 +18,7 @@ PEERDIR( library/cpp/testing/unittest ydb/core/kqp/common ydb/core/kqp/ut/common - ydb/core/kqp/workload_service/ut/common + ydb/services/workload_manager/ut/common ydb/core/testlib/basics/pg yql/essentials/minikql/comp_nodes/llvm16 yql/essentials/public/udf/service/exception_policy diff --git a/ydb/core/kqp/session_actor/kqp_query_state.h b/ydb/core/kqp/session_actor/kqp_query_state.h index 46ac709df1f..22091a6159b 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.h +++ b/ydb/core/kqp/session_actor/kqp_query_state.h @@ -12,6 +12,7 @@ #include <ydb/core/kqp/common/kqp_resolve.h> #include <ydb/core/kqp/common/kqp_timeouts.h> #include <ydb/core/kqp/common/kqp_tx.h> +#include <ydb/core/kqp/common/buffer/events.h> #include <ydb/core/kqp/common/kqp_user_request_context.h> #include <ydb/core/kqp/common/kqp.h> #include <ydb/core/kqp/common/simple/temp_tables.h> @@ -26,11 +27,13 @@ #include <map> #include <memory> -namespace NKikimr::NKqp { +namespace NKikimr { -namespace NWorkload { +namespace NWorkloadManager { class IQueryClassifier; -} // namespace NWorkload +} // namespace NWorkloadManager + +namespace NKqp { class TKqpQueryCache; @@ -149,7 +152,7 @@ public: TActorId Sender; ui64 ProxyRequestId = 0; std::unique_ptr<TEvKqp::TEvQueryRequest> RequestEv; - std::shared_ptr<NWorkload::IQueryClassifier> QueryClassifier; + std::shared_ptr<NWorkloadManager::IQueryClassifier> QueryClassifier; ui64 ParametersSize = 0; TPreparedQueryHolder::TConstPtr PreparedQuery; TKqpCompileResult::TConstPtr CompileResult; @@ -192,6 +195,8 @@ public: bool Commit = false; bool Commited = false; + std::optional<TCommitTimestamp> CommitTimestamp; + NTopic::TTopicOperations TopicOperations; TDuration CpuTime; std::optional<NCpuTime::TCpuTimer> CurrentTimer; @@ -712,5 +717,6 @@ public: }; +} //namespace NKqp -} +} //namespace NKikimr diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index ac0d8afcd2e..7f981c0289d 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -14,7 +14,7 @@ #include <ydb/core/kqp/common/kqp_timeouts.h> #include <ydb/core/kqp/common/kqp_tx.h> #include <ydb/core/kqp/common/kqp.h> -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/kqp/common/simple/query_ast.h> #include <ydb/core/kqp/compile_service/kqp_compile_service.h> #include <ydb/core/kqp/executer_actor/kqp_executer.h> @@ -26,7 +26,7 @@ #include <ydb/core/kqp/opt/kqp_query_plan.h> #include <ydb/core/kqp/provider/yql_kikimr_provider.h> #include <ydb/core/kqp/provider/yql_kikimr_results.h> -#include <ydb/core/kqp/workload_service/kqp_query_classifier.h> +#include <ydb/services/workload_manager/query_classifier.h> #include <ydb/core/kqp/rm_service/kqp_snapshot_manager.h> #include <ydb/core/ydb_convert/ydb_convert.h> #include <ydb/core/tx/schemeshard/schemeshard.h> @@ -46,6 +46,7 @@ #include <ydb/core/protos/kqp.pb.h> #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/tx/tx_proxy/proxy.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/library/actors/async/wait_for_event.h> #include <ydb/library/actors/core/actor_bootstrapped.h> @@ -347,7 +348,7 @@ public: Y_VALIDATE(!QueryState->UserRequestContext->PoolConfig, "Cannot send to workload manager: PoolConfig is already resolved"); - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new NWorkload::TEvPlaceRequestIntoPool( + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new NWorkloadManager::TEvPlaceRequestIntoPool( QueryState->UserRequestContext->DatabaseId, SessionId, QueryState->UserRequestContext->PoolId, @@ -356,7 +357,7 @@ public: QueryState->RequestEv->GetWmSessionUpdater() ), IEventHandle::FlagTrackDelivery); - QueryState->PoolHandlerActor = MakeKqpWorkloadServiceId(SelfId().NodeId()); + QueryState->PoolHandlerActor = NWorkloadManager::MakeServiceId(SelfId().NodeId()); Become(&TKqpSessionActor::ExecuteState); } @@ -598,7 +599,7 @@ public: using TError = std::optional<std::pair<Ydb::StatusIds::StatusCode, TString>>; auto error = std::visit(TOverloaded { - [this, &sent](const NWorkload::IQueryClassifier::TResolvedPoolId& s) -> TError { + [this, &sent](const NWorkloadManager::IQueryClassifier::TResolvedPoolId& s) -> TError { STLOG_D("PreCompile Classify resolved", (pool_id, s.PoolId), (skip_admission, s.SkipAdmission), @@ -612,18 +613,18 @@ public: PassRequestToResourcePool(); return std::nullopt; }, - [this](const NWorkload::IQueryClassifier::TReject& r) -> TError { + [this](const NWorkloadManager::IQueryClassifier::TReject& r) -> TError { STLOG_N("PreCompile Classify rejected", (trace_id, TraceId())); return std::make_pair(r.Code, r.Message); }, - [this](const NWorkload::IQueryClassifier::TBypass&) -> TError { + [this](const NWorkloadManager::IQueryClassifier::TBypass&) -> TError { STLOG_D("PreCompile Classify bypass, compiling", (trace_id, TraceId())); QueryState->UserRequestContext->PoolId = NResourcePool::DEFAULT_POOL_ID; return std::nullopt; }, - [this](const NWorkload::IQueryClassifier::TPendingCompilation&) -> TError { + [this](const NWorkloadManager::IQueryClassifier::TPendingCompilation&) -> TError { STLOG_D("PreCompile Classify pending, compiling", (trace_id, TraceId())); return std::nullopt; @@ -639,7 +640,7 @@ public: } void Handle(TEvents::TEvUndelivered::TPtr& ev) { - if (ev->Get()->SourceType == TKqpWorkloadServiceEvents::EvPlaceRequestIntoPool) { + if (ev->Get()->SourceType == NWorkloadManager::TWorkloadManagerEvents::EvPlaceRequestIntoPool) { STLOG_W("Failed to deliver request to workload service, bypassing WLM", (trace_id, TraceId())); ContinueAfterWmAdmission(); @@ -659,7 +660,7 @@ public: } } - void Handle(NWorkload::TEvContinueRequest::TPtr& ev) { + void Handle(NWorkloadManager::TEvContinueRequest::TPtr& ev) { YQL_ENSURE(QueryState); QueryState->ContinueTime = TInstant::Now(); @@ -705,10 +706,10 @@ public: auto classifier = QueryState->QueryClassifier; auto state = classifier->GetState(); - if (state == NWorkload::IQueryClassifier::EState::PreCompileDone) { + if (state == NWorkloadManager::IQueryClassifier::EState::PreCompileDone) { STLOG_D("Pre-compile admission completed, compiling", (trace_id, TraceId())); CompileQuery(); - } else if (state == NWorkload::IQueryClassifier::EState::PostCompileDone) { + } else if (state == NWorkloadManager::IQueryClassifier::EState::PostCompileDone) { STLOG_D("Post-compile admission completed, executing", (trace_id, TraceId())); OnSuccessCompileRequest(); } else { @@ -1002,16 +1003,16 @@ public: bool WmPostCompileClassify() { auto classifier = QueryState->QueryClassifier; - if (!classifier || classifier->GetState() != NWorkload::IQueryClassifier::EState::WaitCompile) { + if (!classifier || classifier->GetState() != NWorkloadManager::IQueryClassifier::EState::WaitCompile) { return false; } bool sent = false; - const auto result = classifier->PostCompileClassify(*QueryState->PreparedQuery); + const auto result = classifier->PostCompileClassify(*QueryState->PreparedQuery, *QueryState->UserRequestContext); using TError = std::optional<std::pair<Ydb::StatusIds::StatusCode, TString>>; auto error = std::visit(TOverloaded { - [this, &sent](const NWorkload::IQueryClassifier::TResolvedPoolId& r) -> TError { + [this, &sent](const NWorkloadManager::IQueryClassifier::TResolvedPoolId& r) -> TError { STLOG_D("PostCompile Classify resolved", (pool_id, r.PoolId), (skip_admission, r.SkipAdmission), @@ -1025,12 +1026,12 @@ public: PassRequestToResourcePool(); return std::nullopt; }, - [this](const NWorkload::IQueryClassifier::TBypass&) -> TError { + [this](const NWorkloadManager::IQueryClassifier::TBypass&) -> TError { STLOG_D("PostCompile Classify bypass", (trace_id, TraceId())); return std::nullopt; }, - [this](const NWorkload::IQueryClassifier::TReject& r) -> TError { + [this](const NWorkloadManager::IQueryClassifier::TReject& r) -> TError { STLOG_N("PostCompile Classify rejected", (trace_id, TraceId())); return std::make_pair(r.Code, r.Message); @@ -2730,6 +2731,10 @@ public: QueryState->TxCtx->AcceptIncomingSnapshot(ev->Snapshot); + if (ev->CommitTimestamp) { + QueryState->CommitTimestamp = std::move(ev->CommitTimestamp); + } + if (ev->LockHandle) { QueryState->TxCtx->LockHandle = std::move(ev->LockHandle); } @@ -3040,6 +3045,12 @@ public: FillTxInfo(response); FillPoolId(response); + if (QueryState->CommitTimestamp) { + auto* ts = response->MutableCommitTimestamp(); + ts->set_plan_step(QueryState->CommitTimestamp->PlanStep); + ts->set_tx_id(QueryState->CommitTimestamp->TxId); + } + UpdateQueryExecutionCounters(); bool replyQueryId = false; @@ -3510,12 +3521,12 @@ public: CleanupCtx->IsWaitingForWorkloadServiceCleanup = true; const auto& stats = QueryState->QueryStats; - auto event = std::make_unique<NWorkload::TEvCleanupRequest>( + auto event = std::make_unique<NWorkloadManager::TEvCleanupRequest>( QueryState->UserRequestContext->DatabaseId, SessionId, QueryState->UserRequestContext->PoolId, TDuration::MicroSeconds(stats.DurationUs), TDuration::MicroSeconds(stats.WorkerCpuTimeUs) ); - auto forwardId = MakeKqpWorkloadServiceId(SelfId().NodeId()); + auto forwardId = NWorkloadManager::MakeServiceId(SelfId().NodeId()); Send(new IEventHandle(*QueryState->PoolHandlerActor, SelfId(), event.release(), IEventHandle::FlagForwardOnNondelivery, 0, &forwardId)); QueryState->PoolHandlerActor = Nothing(); } @@ -3591,7 +3602,7 @@ public: } } - void HandleCleanup(NWorkload::TEvCleanupResponse::TPtr& ev) { + void HandleCleanup(NWorkloadManager::TEvCleanupResponse::TPtr& ev) { YQL_ENSURE(CleanupCtx); CleanupCtx->IsWaitingForWorkloadServiceCleanup = false; @@ -3748,7 +3759,7 @@ public: hFunc(TEvKqpExecuter::TEvExecuterProgress, HandleNoop) hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNoop); hFunc(TEvents::TEvUndelivered, HandleNoop); - hFunc(NWorkload::TEvContinueRequest, HandleNoop); + hFunc(NWorkloadManager::TEvContinueRequest, HandleNoop); // message from KQP proxy in case of our reply just after kqp proxy timer tick hFunc(NYql::NDq::TEvDq::TEvAbortExecution, HandleNoop); // A finished request's client may be lost after we already replied and @@ -3783,7 +3794,7 @@ public: hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, Handle); hFunc(TEvents::TEvUndelivered, Handle); - hFunc(NWorkload::TEvContinueRequest, Handle); + hFunc(NWorkloadManager::TEvContinueRequest, Handle); hFunc(TEvKqpExecuter::TEvTxResponse, HandleExecute); hFunc(TEvKqpExecuter::TEvExecuterProgress, HandleExecute) @@ -3829,7 +3840,7 @@ public: hFunc(TEvKqp::TEvQueryRequest, Handle); hFunc(TEvKqpExecuter::TEvTxResponse, HandleCleanup); - hFunc(NWorkload::TEvCleanupResponse, HandleCleanup); + hFunc(NWorkloadManager::TEvCleanupResponse, HandleCleanup); hFunc(TEvKqp::TEvCloseSessionRequest, HandleCleanup); hFunc(NGRpcService::TEvClientLost, HandleNoop); @@ -3844,7 +3855,7 @@ public: hFunc(TEvents::TEvUndelivered, HandleNoop); hFunc(TEvTxUserProxy::TEvAllocateTxIdResult, HandleNoop); hFunc(TEvKqpExecuter::TEvStreamData, HandleNoop); - hFunc(NWorkload::TEvContinueRequest, HandleNoop); + hFunc(NWorkloadManager::TEvContinueRequest, HandleNoop); // always come from WorkerActor hFunc(TEvKqp::TEvCloseSessionResponse, HandleCleanup); @@ -3867,7 +3878,7 @@ public: hFunc(TEvents::TEvGone, HandleFinalCleanup); hFunc(TEvents::TEvUndelivered, HandleNoop); hFunc(TEvKqpSnapshot::TEvCreateSnapshotResponse, Handle); - hFunc(NWorkload::TEvContinueRequest, HandleNoop); + hFunc(NWorkloadManager::TEvContinueRequest, HandleNoop); hFunc(TEvKqp::TEvQueryRequest, HandleFinalCleanup); } } catch (const yexception& ex) { diff --git a/ydb/core/kqp/session_actor/ya.make b/ydb/core/kqp/session_actor/ya.make index 0398b0248e1..64f45b6ae0c 100644 --- a/ydb/core/kqp/session_actor/ya.make +++ b/ydb/core/kqp/session_actor/ya.make @@ -18,6 +18,7 @@ PEERDIR( ydb/library/security ydb/public/sdk/cpp/src/library/operation_id ydb/core/tx/schemeshard + ydb/services/workload_manager/service ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp index d6935ee8eaa..1c272f4dbc6 100644 --- a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp +++ b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp @@ -58,6 +58,8 @@ struct TWorkerSettings { int MinMessageSize = 10; int MaxMessageSize = 10000; bool EarlyFinish = false; + int PauseMessageIndex = -1; + int PauseDelayMs = 0; }; struct TFailureSettings { @@ -67,12 +69,13 @@ struct TFailureSettings { int Discovery = 0; }; -class TWorkerActor : public NActors::TActor<TWorkerActor> { +template <typename TDerived> +class TWorkerActor : public NActors::TActor<TDerived> { public: - TWorkerActor(std::shared_ptr<IDqChannelService> service, TEvTestPrivate::ERole role, ui32 channelId, const TWorkerSettings& settings) - : TActor(&TThis::StateFunc) + TWorkerActor(const TString& logPrefix, std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : NActors::TActor<TDerived>(&TWorkerActor::StateFunc) + , LogPrefix(logPrefix) , Service(service) - , Role(role) , ChannelId(channelId) , Settings(settings) {} @@ -86,98 +89,38 @@ public: } } - void Run() { - if (!Started) { - switch (Role) { - case TEvTestPrivate::ERole::Producer: { - TChannelFullInfo info(ChannelId, SelfId(), PeerId, 0, 1, TCollectStatsLevel::None); - Buffer = Service->GetOutputBuffer(info, nullptr, nullptr); - break; - } - case TEvTestPrivate::ERole::Consumer: { - TChannelFullInfo info(ChannelId, PeerId, SelfId(), 0, 1, TCollectStatsLevel::None); - Buffer = Service->GetInputBuffer(info, nullptr); - break; - } - } - Started = true; - } - - switch (Role) { - case TEvTestPrivate::ERole::Producer: { - if (Buffer->IsFinished()) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, false)); - PassAway(); - return; - } - while (Buffer->GetFillLevel() == EDqFillLevel::NoLimit && MessageIndex <= Settings.MessageCount) { - if (MessageIndex == Settings.MessageCount) { - Buffer->SendFinish(); - } else { - auto bytes = Settings.MinMessageSize + RandomNumber<ui64>(Settings.MaxMessageSize - Settings.MinMessageSize); - Buffer->Push(TDataChunk(NYql::TChunkedBuffer(TString(bytes, 'a')), 1, false)); - } - MessageIndex++; - } - break; - } - case TEvTestPrivate::ERole::Consumer: { - TDataChunk data; - while (MessageIndex < Settings.MessageCount && Buffer->Pop(data)) { - MessageIndex++; - } - if (Settings.EarlyFinish && MessageIndex == Settings.MessageCount) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST EARLY FINISH SelfId=" << SelfId() << ", ChannelId=" << ChannelId); - Buffer->EarlyFinish(); - MessageIndex++; - } - if (MessageIndex <= Settings.MessageCount && Buffer->Pop(data)) { - MessageIndex++; - } - if (Buffer->IsFinished()) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId - << ", Role=" << (Role == TEvTestPrivate::ERole::Producer ? "Producer" : "Consumer")); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, false)); - PassAway(); - } - break; - } - } - } + virtual void Run() = 0; - void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&) { + virtual void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&) { Run(); } - void HandleResume(TEvDqCompute::TEvResumeExecution::TPtr&) { + virtual void HandleResume(TEvDqCompute::TEvResumeExecution::TPtr&) { Run(); } - void HandleStart(TEvTestPrivate::TEvStart::TPtr& ev) { + virtual void HandleStart(TEvTestPrivate::TEvStart::TPtr& ev) { RunnerId = ev->Sender; PeerId = ev->Get()->PeerId; - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, - "TEST WORKER START SelfId=" << SelfId() << ", ChannelId=" << ChannelId << ", PeerId=" << PeerId << ", Role=" << (Role == TEvTestPrivate::ERole::Producer ? "Producer" : "Consumer") - ); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST START SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", PeerId=" << PeerId); if (Settings.StartDelayMs) { - Schedule(TDuration::MilliSeconds(RandomNumber<ui64>(Settings.StartDelayMs) + 1), new NActors::TEvents::TEvWakeup()); + this->Schedule(TDuration::MilliSeconds(RandomNumber<ui64>(Settings.StartDelayMs) + 1), new NActors::TEvents::TEvWakeup()); } else { Run(); } } - void HandleAbort(NYql::NDq::TEvDq::TEvAbortExecution::TPtr& ev) { + virtual void HandleAbort(NYql::NDq::TEvDq::TEvAbortExecution::TPtr& ev) { LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, - "TEST WORKER ABORT SelfId=" << SelfId() << ", ChannelId=" << ChannelId << ", " << ev->Get()->GetIssues().ToOneLineString() + LogPrefix << "TEST ABORT SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", " << ev->Get()->GetIssues().ToOneLineString() ); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, true)); - PassAway(); + this->Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Producer, true)); + this->PassAway(); } + TString LogPrefix; std::shared_ptr<IDqChannelService> Service; std::shared_ptr<IChannelBuffer> Buffer; - TEvTestPrivate::ERole Role; ui32 ChannelId; NActors::TActorId PeerId; NActors::TActorId RunnerId; @@ -186,105 +129,246 @@ public: bool Started = false; }; -Y_UNIT_TEST_SUITE(Channels20) { - - void LoadTest(int count, bool local, const TWorkerSettings& producerSettings, const TWorkerSettings& consumerSettings, const TFailureSettings& failureSettings = TFailureSettings{}) { +class TProducerActor : public TWorkerActor<TProducerActor> { +public: + TProducerActor(std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : TWorkerActor("PROD ", service, channelId, settings) + {} - TKikimrSettings settings; - settings.NodeCount = local ? 1 : 2; - settings.LogSettings = TTestLogSettings().AddLogPriority(NKikimrServices::KQP_CHANNELS, NActors::NLog::EPriority::PRI_TRACE); - settings.LogSettings->DefaultLogPriority = NActors::NLog::EPriority::PRI_CRIT; - TKikimrRunner kikimr(settings); - auto& runtime = *kikimr.GetTestServer().GetRuntime(); - runtime.SetUseRealInterconnect(); + void Run() override { + if (!Started) { + TChannelFullInfo info(ChannelId, SelfId(), PeerId, 0, 1, TCollectStatsLevel::None); + Buffer = Service->GetOutputBuffer(info, nullptr, nullptr); + Started = true; + } + if (Buffer->IsFinished()) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Producer, false)); + PassAway(); + return; + } + while (Buffer->GetFillLevel() == EDqFillLevel::NoLimit && MessageIndex < Settings.MessageCount) { + if (Settings.PauseMessageIndex == MessageIndex) { + if (!ResumeTime) { + ResumeTime = TInstant::Now() + TDuration::MilliSeconds(Settings.PauseDelayMs); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST PAUSED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + } + if (TInstant::Now() < ResumeTime) { + Schedule(ResumeTime, new NActors::TEvents::TEvWakeup()); + return; + } else { + ResumeTime = TInstant::Zero(); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST RESUMED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + } + } + auto bytes = Settings.MinMessageSize + RandomNumber<ui64>(Settings.MaxMessageSize - Settings.MinMessageSize); + Buffer->Push(TDataChunk(NYql::TChunkedBuffer(TString(bytes, 'a')), 1, false)); + MessageIndex++; + } + if (MessageIndex == Settings.MessageCount) { + Buffer->SendFinish(); + } + } - auto control0 = runtime.AllocateEdgeActor(0); - auto control1 = local ? control0 : runtime.AllocateEdgeActor(1); + TInstant ResumeTime; +}; - std::shared_ptr<TDqChannelService> service0; - std::shared_ptr<TDqChannelService> service1; +class TConsumerActor : public TWorkerActor<TConsumerActor> { +public: + TConsumerActor(std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : TWorkerActor("CONS ", service, channelId, settings) + {} - ui32 nodeIndex0 = 0; - ui32 nodeIndex1 = local ? 0 : 1; + void Run() override { + if (!Started) { + TChannelFullInfo info(ChannelId, PeerId, SelfId(), 0, 1, TCollectStatsLevel::None); + Buffer = Service->GetInputBuffer(info, nullptr); + Started = true; + } + TDataChunk data; + while (MessageIndex < Settings.MessageCount && Buffer->Pop(data)) { + MessageIndex++; + } + if (Settings.EarlyFinish && MessageIndex == Settings.MessageCount) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST EARLY FINISH SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Buffer->EarlyFinish(); + MessageIndex++; + } + if (MessageIndex <= Settings.MessageCount && Buffer->Pop(data)) { + MessageIndex++; + } + if (Buffer->IsFinished()) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Consumer, false)); + PassAway(); + } + } +}; - runtime.Send(MakeChannelServiceActorID(runtime.GetNodeId(0)), control0, new TEvPrivate::TEvServiceLookup(), nodeIndex0); - auto serviceReply = runtime.GrabEdgeEvent<TEvPrivate::TEvServiceReply>(control0)->Release(); - service0 = serviceReply->Service; +struct TLoadTest { - if (local) { - service1 = service0; - } else { - runtime.Send(MakeChannelServiceActorID(runtime.GetNodeId(1)), control1, new TEvPrivate::TEvServiceLookup(), nodeIndex1); - auto serviceReply = runtime.GrabEdgeEvent<TEvPrivate::TEvServiceReply>(control1)->Release(); - service1 = serviceReply->Service; + virtual void Prepare() { + settings.NodeCount = Local ? 1 : 2; + settings.LogSettings = TTestLogSettings().AddLogPriority(NKikimrServices::KQP_CHANNELS, NActors::NLog::EPriority::PRI_TRACE); + settings.LogSettings->DefaultLogPriority = NActors::NLog::EPriority::PRI_CRIT; + if (Local) { + NodeIndex1 = NodeIndex0; } + } - THashSet<NActors::TActorId> actors; + virtual void Init() { + Runner = std::make_unique<TKikimrRunner>(settings); + Runtime = Runner->GetTestServer().GetRuntime(); + Runtime->SetUseRealInterconnect(); - std::atomic<int> dataCount = 0; - NActors::TTestActorRuntime::TEventObserverHolder dataHolder; - if (failureSettings.Data) { - dataHolder = runtime.AddObserver<TEvDqCompute::TEvChannelDataV2>([&](TEvDqCompute::TEvChannelDataV2::TPtr& event) { - if (dataCount.fetch_add(1) == failureSettings.Data) { - dataCount.fetch_sub(failureSettings.Data); - event.Reset(); - } - }); + Control0 = Runtime->AllocateEdgeActor(0); + Control1 = Local ? Control0 : Runtime->AllocateEdgeActor(1); + + Runtime->Send(MakeChannelServiceActorID(Runtime->GetNodeId(0)), Control0, new TEvPrivate::TEvServiceLookup(), NodeIndex0); + auto serviceReply = Runtime->GrabEdgeEvent<TEvPrivate::TEvServiceReply>(Control0)->Release(); + Service0 = serviceReply->Service; + + if (Local) { + Service1 = Service0; + } else { + Runtime->Send(MakeChannelServiceActorID(Runtime->GetNodeId(1)), Control1, new TEvPrivate::TEvServiceLookup(), NodeIndex1); + auto serviceReply = Runtime->GrabEdgeEvent<TEvPrivate::TEvServiceReply>(Control1)->Release(); + Service1 = serviceReply->Service; } + } - for (auto i = 0; i < count; i ++) { + virtual void Start() { + for (auto i = 0; i < Count; i ++) { auto channelId = i + 1; if ((i & 1) == 0) { - auto producer = runtime.Register(new TWorkerActor(service0, TEvTestPrivate::ERole::Producer, channelId, producerSettings), nodeIndex0); - auto consumer = runtime.Register(new TWorkerActor(service1, TEvTestPrivate::ERole::Consumer, channelId, consumerSettings), nodeIndex1); - runtime.Send(consumer, control1, new TEvTestPrivate::TEvStart(producer), nodeIndex1, true); - runtime.Send(producer, control0, new TEvTestPrivate::TEvStart(consumer), nodeIndex0, true); - actors.insert(producer); - actors.insert(consumer); + auto producer = Runtime->Register(new TProducerActor(Service0, channelId, ProducerSettings), NodeIndex0); + auto consumer = Runtime->Register(new TConsumerActor(Service1, channelId, ConsumerSettings), NodeIndex1); + Runtime->Send(consumer, Control1, new TEvTestPrivate::TEvStart(producer), NodeIndex1, true); + Runtime->Send(producer, Control0, new TEvTestPrivate::TEvStart(consumer), NodeIndex0, true); + Actors.insert(producer); + Actors.insert(consumer); } else { - auto producer = runtime.Register(new TWorkerActor(service1, TEvTestPrivate::ERole::Producer, channelId, producerSettings), nodeIndex1); - auto consumer = runtime.Register(new TWorkerActor(service0, TEvTestPrivate::ERole::Consumer, channelId, consumerSettings), nodeIndex0); - runtime.Send(consumer, control0, new TEvTestPrivate::TEvStart(producer), nodeIndex0, true); - runtime.Send(producer, control1, new TEvTestPrivate::TEvStart(consumer), nodeIndex1, true); - actors.insert(producer); - actors.insert(consumer); + auto producer = Runtime->Register(new TProducerActor(Service1, channelId, ProducerSettings), NodeIndex1); + auto consumer = Runtime->Register(new TConsumerActor(Service0, channelId, ConsumerSettings), NodeIndex0); + Runtime->Send(consumer, Control0, new TEvTestPrivate::TEvStart(producer), NodeIndex0, true); + Runtime->Send(producer, Control1, new TEvTestPrivate::TEvStart(consumer), NodeIndex1, true); + Actors.insert(producer); + Actors.insert(consumer); } } + } - int finishCount[2][2] = {{0, 0}, {0, 0}}; - int errorCount = 0; + virtual void Wait() { try { - for (auto i = 0; i < count; i++) { - auto msg0 = runtime.GrabEdgeEvent<TEvTestPrivate::TEvFinished>(control0, TDuration::Seconds(10)); - actors.erase(msg0->Sender); - finishCount[nodeIndex0][msg0->Get()->Role]++; - errorCount += msg0->Get()->Error; - auto msg1 = runtime.GrabEdgeEvent<TEvTestPrivate::TEvFinished>(control1, TDuration::Seconds(10)); - actors.erase(msg1->Sender); - finishCount[nodeIndex1][msg1->Get()->Role]++; - errorCount += msg1->Get()->Error; + for (auto i = 0; i < Count; i++) { + auto msg0 = Runtime->GrabEdgeEvent<TEvTestPrivate::TEvFinished>(Control0, TDuration::Seconds(10)); + Actors.erase(msg0->Sender); + FinishCount[NodeIndex0][msg0->Get()->Role]++; + ErrorCount += msg0->Get()->Error; + auto msg1 = Runtime->GrabEdgeEvent<TEvTestPrivate::TEvFinished>(Control1, TDuration::Seconds(10)); + Actors.erase(msg1->Sender); + FinishCount[NodeIndex1][msg1->Get()->Role]++; + ErrorCount += msg1->Get()->Error; } } catch (NActors::TEmptyEventQueueException&) { - if (!actors.empty()) { + if (!Actors.empty()) { TStringBuilder builder; builder << "NOT FINISHED ACTORS "; - for (auto actorId : actors) { + for (auto actorId : Actors) { builder << ' ' << actorId; } UNIT_ASSERT_C(false, builder); } } + } - if (local) { - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Producer], count); - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Consumer], count); + virtual void Check() { + if (Local) { + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Producer], Count); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Consumer], Count); } else { - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Producer], (count + 1) / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Consumer], count / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[1][TEvTestPrivate::ERole::Producer], count / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[1][TEvTestPrivate::ERole::Consumer], (count + 1) / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Producer], (Count + 1) / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Consumer], Count / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[1][TEvTestPrivate::ERole::Producer], Count / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[1][TEvTestPrivate::ERole::Consumer], (Count + 1) / 2); } - UNIT_ASSERT_VALUES_EQUAL(errorCount, 0); + UNIT_ASSERT_VALUES_EQUAL(ErrorCount, 0); + } + + virtual void Run() { + Prepare(); + Init(); + Start(); + Wait(); + Check(); + } + + int Count = 1; + bool Local = true; + ui32 NodeIndex0 = 0; + ui32 NodeIndex1 = 1; + TKikimrSettings settings; + std::unique_ptr<TKikimrRunner> Runner; + NActors::TTestActorRuntime* Runtime; + std::shared_ptr<TDqChannelService> Service0; + std::shared_ptr<TDqChannelService> Service1; + NActors::TActorId Control0; + NActors::TActorId Control1; + TWorkerSettings ProducerSettings; + TWorkerSettings ConsumerSettings; + THashSet<NActors::TActorId> Actors; + int ErrorCount = 0; + int FinishCount[2][2] = {{0, 0}, {0, 0}}; +}; + +struct TReconTest : public TLoadTest { + + void Prepare() override { + TLoadTest::Prepare(); + settings.AppConfig.MutableTableServiceConfig()->MutableDqChannelConfig()->SetCleanupPeriodMs(20); + settings.AppConfig.MutableTableServiceConfig()->MutableDqChannelConfig()->SetIdlePingPeriodMs(10); + } + + void Start() override { + for (auto i = 0; i < Count; i ++) { + auto channelId = i + 1; + if ((i & 1) == 0) { + auto producerSettings = ProducerSettings; + producerSettings.PauseMessageIndex = (channelId + producerSettings.MessageCount / 2) % producerSettings.MessageCount; + producerSettings.PauseDelayMs = 50; + auto producer = Runtime->Register(new TProducerActor(Service0, channelId, producerSettings), NodeIndex0); + auto consumer = Runtime->Register(new TConsumerActor(Service1, channelId, ConsumerSettings), NodeIndex1); + Runtime->Send(consumer, Control1, new TEvTestPrivate::TEvStart(producer), NodeIndex1, true); + Runtime->Send(producer, Control0, new TEvTestPrivate::TEvStart(consumer), NodeIndex0, true); + Actors.insert(producer); + Actors.insert(consumer); + } else { + auto producerSettings = ProducerSettings; + producerSettings.PauseMessageIndex = channelId; + producerSettings.PauseDelayMs = 50; + auto producer = Runtime->Register(new TProducerActor(Service1, channelId, producerSettings), NodeIndex1); + auto consumer = Runtime->Register(new TConsumerActor(Service0, channelId, ConsumerSettings), NodeIndex0); + Runtime->Send(consumer, Control0, new TEvTestPrivate::TEvStart(producer), NodeIndex0, true); + Runtime->Send(producer, Control1, new TEvTestPrivate::TEvStart(consumer), NodeIndex1, true); + Actors.insert(producer); + Actors.insert(consumer); + } + } + } + +}; + +Y_UNIT_TEST_SUITE(Channels20) { + + void LoadTest(int count, bool local, const TWorkerSettings& producerSettings, const TWorkerSettings& consumerSettings, const TFailureSettings& = TFailureSettings{}) { + TLoadTest test; + + test.Count = count; + test.Local = local; + test.ProducerSettings = producerSettings; + test.ConsumerSettings = consumerSettings; + + test.Run(); } void LoadTest(int count, bool local, const TWorkerSettings& settings = TWorkerSettings{}, const TFailureSettings& failureSettings = TFailureSettings{}) { @@ -326,4 +410,15 @@ Y_UNIT_TEST_SUITE(Channels20) { Y_UNIT_TEST(MissedData) { LoadTest(100, false, TWorkerSettings{ .MessageCount = 100 }, TWorkerSettings{ .MessageCount = 100 }, TFailureSettings{ .Data = 10 }); } + + Y_UNIT_TEST(Reconciliation) { + TReconTest test; + + test.Count = 1; + test.Local = false; + test.ProducerSettings.MessageCount = 100; + test.ConsumerSettings.MessageCount = 100; + + test.Run(); + } } diff --git a/ydb/core/kqp/ut/federated_query/common/common.cpp b/ydb/core/kqp/ut/federated_query/common/common.cpp index de1411b5a50..0903c656efb 100644 --- a/ydb/core/kqp/ut/federated_query/common/common.cpp +++ b/ydb/core/kqp/ut/federated_query/common/common.cpp @@ -99,6 +99,10 @@ std::shared_ptr<TKikimrRunner> MakeKikimrRunner( if (appFlags.GetEnableColumnStore()) { featureFlags.SetEnableColumnStore(true); } + + if (appFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); + } } if (!appConfig) { diff --git a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp index 7d5ae2fdc07..9d0e4f288b0 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp @@ -66,14 +66,16 @@ void WaitClassifierVisible(TStreamingTestFixture& fixture, // HAS_PATH end-to-end coverage for the object kinds whose fixture requirements // (real local PQ, mock connector, mock PQ gateway, http gateway) are only met // by TStreamingTestFixture. Cheaper kinds (regular tables, sysview, secondary -// index, view underlying) live in ydb/core/kqp/workload_service/ut. +// index, view underlying) live in ydb/services/workload_manager/ut. Y_UNIT_TEST_SUITE(HasPathDatastreams) { // KindTopic — direct read of a local topic (no EDS in the chain). // The topic path lands in tx.Tables (B walk). Y_UNIT_TEST_F(DirectTopicMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); constexpr TStringBuf topicName = "test_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -96,7 +98,9 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // direct topic; path is `<table>/<stream>`. Y_UNIT_TEST_F(CdcStreamMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); ExecSchemeQuery(RejectClassifierDdl( "hp_cdc", "/Root/t_cdc/cf")); @@ -129,6 +133,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // Uses a loopback Ydb-typed EDS pointing at our own runner + a local topic // as the target object (SetupMockPqGateway + real local PQ serves reads). Y_UNIT_TEST_F(FederatedEdsPathMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "eds_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -162,6 +167,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // appears in TDqPqTopicSource.TopicPath — the (D) walk's unique-coverage // case verified on real cluster). Y_UNIT_TEST_F(FederatedTopicRemoteNameMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "remote_by_name_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -196,6 +202,8 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // KindExternalTable — External Table on top of an ObjectStorage EDS. // Both the ET path and the underlying EDS path land in tx.Tables (B walk). Y_UNIT_TEST_F(ExternalTableMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); + ExecSchemeQuery(RejectClassifierDdl( "hp_et", "/Root/et_s3")); diff --git a/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp index fed9351e487..b512a370adb 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp @@ -1445,6 +1445,167 @@ Y_UNIT_TEST_SUITE(KqpStreamingQueriesDdl) { ReadTopicMessage(outputTopicName, "message-2-value-2", disposition); } + Y_UNIT_TEST_F(StreamingQueryPrecomputeRecalculationOnRetry, TStreamingTestFixture) { + const auto pqGateway = SetupMockPqGateway(); + + constexpr char inputTopicName[] = "streamingQueryPrecomputeRecalculationOnRetryInputTopic"; + constexpr char outputTopicName[] = "streamingQueryPrecomputeRecalculationOnRetryOutputTopic"; + constexpr char pqSourceName[] = "pqSourceName"; + CreateTopic(inputTopicName); + CreateTopic(outputTopicName); + CreatePqSource(pqSourceName); + + constexpr char tableName[] = "oltpTable"; + ExecQuery(fmt::format(R"( + CREATE TABLE `{table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + );)", + "table_name"_a = tableName + )); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{table_name}` + (Key, Value) + VALUES + (1, "value-1");)", + "table_name"_a = tableName + )); + + constexpr char queryName[] = "streamingQuery"; + ExecQuery(fmt::format(R"( + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + $r = SELECT Value FROM `{table_name}`; + + INSERT INTO `{pq_source}`.`{output_topic}` + SELECT + Unwrap(Data || "-" || $r) + FROM `{pq_source}`.`{input_topic}` + END DO;)", + "query_name"_a = queryName, + "pq_source"_a = pqSourceName, + "input_topic"_a = inputTopicName, + "output_topic"_a = outputTopicName, + "table_name"_a = tableName + )); + CheckScriptExecutionsCount(1, 1); + Sleep(TDuration::Seconds(1)); + + const auto readSession = pqGateway->WaitReadSession(inputTopicName); + readSession->AddDataReceivedEvent(0, "message-1"); + pqGateway->WaitWriteSession(outputTopicName)->ExpectMessage("message-1-value-1"); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{table_name}` + (Key, Value) + VALUES + (1, "value-2");)", + "table_name"_a = tableName + )); + + readSession->AddCloseSessionEvent(EStatus::UNAVAILABLE, {NIssue::TIssue("Test pq session failure")}); + pqGateway->WaitReadSession(inputTopicName)->AddDataReceivedEvent(1, "message-2"); + pqGateway->WaitWriteSession(outputTopicName)->ExpectMessage("message-2-value-2"); + } + + Y_UNIT_TEST_F(StreamingQueryWithDifferentPrecomputeTypes, TStreamingTestFixture) { + constexpr char oltpTableName[] = "oltpTable"; + constexpr char olapTableName[] = "olapTable"; + ExecQuery(fmt::format(R"( + CREATE TABLE `{oltp_table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + CREATE TABLE `{olap_table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ) WITH ( + STORE = COLUMN + );)", + "oltp_table_name"_a = oltpTableName, + "olap_table_name"_a = olapTableName + )); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{oltp_table_name}` + (Key, Value) + VALUES + (1, "value-1"), + (2, "value-1"); + UPSERT INTO `{olap_table_name}` + (Key, Value) + VALUES + (1, "value-1"), + (2, "value-1");)", + "oltp_table_name"_a = oltpTableName, + "olap_table_name"_a = olapTableName + )); + + constexpr char sourceBucket[] = "test_streaming_query_with_s3_join"; + constexpr char objectContent[] = R"( +{"Key": 1, "Value": "value-1"} +{"Key": 2, "Value": "value-1"})"; + CreateBucketWithObject(sourceBucket, "path/test_object.json", objectContent); + + constexpr char pqSourceName[] = "pqSourceName"; + constexpr char s3SourceName[] = "s3Source"; + CreatePqSource(pqSourceName); + CreateS3Source(sourceBucket, s3SourceName); + + constexpr char externalTableName[] = "externalTable"; + ExecQuery(fmt::format(R"( + CREATE EXTERNAL TABLE `{external_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL + ) WITH ( + DATA_SOURCE = "{external_source}", + LOCATION = "path/test_object.json", + FORMAT = "json_each_row" + );)", + "external_table"_a = externalTableName, + "external_source"_a = s3SourceName + )); + + constexpr char inputTopicName[] = "streamingQueryWithDifferentPrecomputeTypesInputTopicName"; + constexpr char outputTopicName[] = "streamingQueryWithDifferentPrecomputeTypesOutputTopicName"; + CreateTopic(inputTopicName); + CreateTopic(outputTopicName); + + for (const auto& sourceName : {oltpTableName, olapTableName, externalTableName}) { + constexpr char queryName[] = "streamingQuery"; + + ExecQuery(fmt::format(R"( + CREATE OR REPLACE STREAMING QUERY `{query_name}` AS + DO BEGIN + $precompute_agg = SELECT CAST(MAX(Key) AS String) FROM `{precompute_source}`; + $precompute_limit = SELECT Value FROM `{precompute_source}` LIMIT 1; + $empty_precompute = SELECT Value FROM `{precompute_source}` WHERE Key = 3; + + INSERT INTO `{pq_source}`.`{output_topic}` + SELECT + Unwrap(Data || $precompute_agg || $precompute_limit || ($empty_precompute ?? "<null>")) + FROM `{pq_source}`.`{input_topic}`; + END DO;)", + "query_name"_a = queryName, + "pq_source"_a = pqSourceName, + "precompute_source"_a = sourceName, + "input_topic"_a = inputTopicName, + "output_topic"_a = outputTopicName + )); + + Sleep(TDuration::Seconds(1)); + + const auto disposition = TInstant::Now(); + auto message = TStringBuilder() << "test_message" << sourceName; + WriteTopicMessage(inputTopicName, message); + ReadTopicMessage(outputTopicName, message << "2value-1<null>", disposition); + } + } + Y_UNIT_TEST_F(StreamingQueryUnderSecureScriptExecutions, TStreamingTestFixture) { auto& appConfig = SetupAppConfig(); appConfig.MutableFeatureFlags()->SetEnableSecureScriptExecutions(true); diff --git a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp index 383e457683b..7d367abc7d4 100644 --- a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp @@ -161,9 +161,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( @@ -219,8 +219,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); Variator::ToExecutor(Variator::SingleScript(scriptChunkDetailsMinMaxWithLocalDBStorage)).Execute(settings); - } - + } + TString scriptChunkDetailsMinMaxWithBSStorage = R"( STOP_COMPACTION ------ @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { READ: SELECT COALESCE(sum(CAST(ChunkDetails = "{\"min\":\"x\",\"max\":\"x\"}" as Uint32) ), 0) = count(ChunkDetails) FROM `/Root/ColumnTable/.sys/primary_index_stats` WHERE EntityName="field_mm"; EXPECTED: [[%true]] )"; - + Y_UNIT_TEST(ChunkDetailsMinMaxWithBSStorage, ELocalIndexAsSchemeObject) { const bool LocalIndexAsSchemeObject = (Arg<0>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); @@ -309,14 +309,14 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_mm2` LOCAL USING min_max ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom` LOCAL USING bloom_filter ON(`value`); )"); assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom2` LOCAL USING bloom_filter ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); @@ -326,7 +326,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram2` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); )"); - + assertDDLQueryNotOk(R"( CREATE TABLE `/Root/two_minmax_in_create` ( `key` Int32 NOT NULL, @@ -360,7 +360,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); } - + TString scriptMinMaxIndexOnInsertEnabled = R"( STOP_COMPACTION @@ -695,9 +695,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); NYdb::TStatus status = runDDLQuery(R"( - ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); + ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); )"); - + UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToString()); } @@ -917,7 +917,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item+1 AS String) AS `value_str`, CAST(item+1 AS Utf8) AS `value_utf`, @@ -928,7 +928,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item AS String) AS `value_str`, CAST(item AS Utf8) AS `value_utf`, @@ -939,7 +939,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { csController->WaitCompactions(TDuration::Seconds(5)); - using TQueryResult = TVector<TString>; + using TQueryResult = TVector<TString>; auto runDMLQueryTyped = [&](TString text) -> TQueryResult { auto columns = runDMLQuery(text); @@ -950,7 +950,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } return res; }; - + { TQueryResult tierNamesStr = runDMLQueryTyped(R"( SELECT TierName FROM `/Root/minmax_test_appropriate_storage_location/.sys/primary_index_stats` WHERE EntityName == "value_str_mm"; @@ -958,7 +958,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { UNIT_ASSERT_VALUES_UNEQUAL_C(tierNamesStr.size(), 0, "portions are not min_max indexed with String column type"); for (const auto& tierName: tierNamesStr) { UNIT_ASSERT_VALUES_EQUAL_C(tierName, "__DEFAULT", "min_max index must store its data is BS when building over String column"); - } + } } { TQueryResult tierNamesUtf8 = runDMLQueryTyped(R"( @@ -2038,9 +2038,9 @@ Y_UNIT_TEST(RenameLocalBloomIndex, EUseQueryService) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( diff --git a/ydb/core/kqp/ut/olap/storage/dictionary_ut.cpp b/ydb/core/kqp/ut/olap/storage/dictionary_ut.cpp index 4d64fba96a4..746d7f0058c 100644 --- a/ydb/core/kqp/ut/olap/storage/dictionary_ut.cpp +++ b/ydb/core/kqp/ut/olap/storage/dictionary_ut.cpp @@ -36,6 +36,9 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) { settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); settings.AppConfig.MutableFeatureFlags()->SetEnableCsDictionaryEncoding(true); settings.AppConfig.MutableFeatureFlags()->SetEnableOlapCompression(true); + settings.AppConfig.MutableFeatureFlags()->SetEnableColumnshardInterval(true); + settings.AppConfig.MutableFeatureFlags()->SetEnableColumnshardUuid(true); + settings.AppConfig.MutableFeatureFlags()->SetEnableColumnshardDyNumber(true); return settings; } @@ -1746,8 +1749,46 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) { Variator::ToExecutor(Variator::SingleScript(scriptAlterDictOnExistingPlainWithoutActualization)).Execute(GetDictionarySettings()); } + TString scriptDictionaryIntervalUuidDyNumber = R"( + STOP_COMPACTION + ------ + SCHEMA: + CREATE TABLE `/Root/ColumnTable` ( + pk Uint64 NOT NULL, + c_interval Interval, + c_uuid Uuid, + c_dynumber DyNumber, + PRIMARY KEY (pk) + ) + PARTITION BY HASH(pk) + WITH (STORE = COLUMN, PARTITION_COUNT = 1); + ------ + SCHEMA: + ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`) + ------ + SCHEMA: + ALTER TABLE `/Root/ColumnTable` ALTER COLUMN `c_interval` SET ENCODING(DICT) + ------ + SCHEMA: + ALTER TABLE `/Root/ColumnTable` ALTER COLUMN `c_uuid` SET ENCODING(DICT) + ------ + SCHEMA: + ALTER TABLE `/Root/ColumnTable` ALTER COLUMN `c_dynumber` SET ENCODING(DICT) + ------ + DATA: + REPLACE INTO `/Root/ColumnTable` (pk, c_interval, c_uuid, c_dynumber) VALUES + (1u, CAST(1000000 AS Interval), CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid), CAST("3.14" AS DyNumber)); + ------ + READ: SELECT pk, c_interval, c_uuid, c_dynumber FROM `/Root/ColumnTable` ORDER BY pk; + EXPECTED: [[1u;[1000000];["550e8400-e29b-41d4-a716-446655440000"];[".314e1"]]] + )"; + Y_UNIT_TEST(DictionarySupportedIntervalUuidDyNumber) { + Variator::ToExecutor(Variator::SingleScript(scriptDictionaryIntervalUuidDyNumber)).Execute(GetDictionarySettings()); + } + // One table with a column per supported (comparable) type; set DICTIONARY on each, insert one row, read back. - // Supported for dictionary: Bool, Int*, Uint*, Float, Double, String/Utf8, Date, Datetime, Timestamp. (Bytes omitted: X'...' literal not supported in script.) + // Supported for dictionary: Bool, Int*, Uint*, Float, Double, String/Utf8, Date, Datetime, Timestamp, + // Interval, Uuid, DyNumber. (Bytes omitted: X'...' literal not supported in script.) TString scriptDictionarySupportedTypes = R"( STOP_COMPACTION ------ diff --git a/ydb/core/kqp/ut/olap/types/bool_ut.cpp b/ydb/core/kqp/ut/olap/types/bool_ut.cpp index 411f6a0603c..9b2dda5f0ca 100644 --- a/ydb/core/kqp/ut/olap/types/bool_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/bool_ut.cpp @@ -1,26 +1,10 @@ -#include "bool_test_enums.h" +#include "column_type_scenarios.h" +#include "column_type_test_base.h" -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> #include <ydb/core/tx/columnshard/test_helper/test_combinator.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - #include <ydb/library/actors/core/log.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> #include <yql/essentials/types/binary_json/write.h> #include <yql/essentials/types/uuid/uuid.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> namespace NKikimr { namespace NKqp { @@ -30,114 +14,55 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<bool> B; - }; - TKikimrSettings CreateKikimrSettingsWithBoolSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardBool(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } + struct TBoolTraits { + using TValue = bool; + static constexpr const char* ColumnName = "b"; + static constexpr const char* SqlTypeName = "Bool"; - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - b Bool, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardBool(true); }); + } - void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - )" << secondName << R"( Int64, - b Bool, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static auto GetTypeId() { return NScheme::NTypeIds::Bool; } - void BulkUpsertRowTableYdbValueWithColumnName( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(columnName).Int64(r.IntVal).AddMember("b"); - if (r.B.has_value()) { - builder.BeginOptional().Bool(*r.B).EndOptional(); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<bool>& val) { + if (val.has_value()) { + builder.BeginOptional().Bool(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::Bool); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, "int"); - } - - void BulkUpsertRowTableYdbValueWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(secondName).Int64(r.IntVal).AddMember("b"); - if (r.B.has_value()) { - builder.BeginOptional().Bool(*r.B).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Bool); + static void AppendCsvValue(TStringBuilder& builder, const std::optional<bool>& val) { + if (val.has_value()) { + builder << (*val ? "true" : "false"); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.B.has_value()) { - builder << (*r.B ? "true" : "false"); + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<bool>>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<std::optional<bool>> bs; + bs.reserve(rows.size()); + for (auto&& r : rows) { + bs.push_back(r.TypedVal); } - builder << '\n'; + return MakeBoolArrayAsUInt8Nullable(bs); } - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::uint8(); } + }; + + COLUMN_TYPE_TEST_USING(TBoolTraits); void BulkUpsertRowTableCSVWithFormat( TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& trueValue, const TString& falseValue) { TStringBuilder builder; for (auto&& r : rows) { builder << r.Id << "," << r.IntVal << ","; - if (r.B.has_value()) { - builder << (*r.B ? trueValue : falseValue); + if (r.TypedVal.has_value()) { + builder << (*r.TypedVal ? trueValue : falseValue); } builder << '\n'; @@ -147,148 +72,6 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - std::vector<std::optional<bool>> bs; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - bs.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - bs.push_back(r.B); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); - auto boolArr = MakeBoolArrayAsUInt8Nullable(bs); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(columnName, arrow::int64()), - arrow::field("b", arrow::uint8()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, boolArr }); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - return MakeArrowBatchWithColumnName(rows, "int"); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> seconds; - std::vector<std::optional<bool>> bs; - ids.reserve(rows.size()); - seconds.reserve(rows.size()); - bs.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - seconds.push_back(r.IntVal); - bs.push_back(r.B); - } - - auto idArr = MakeInt32Array(ids); - auto secondArr = MakeInt64Array(seconds); - auto boolArr = MakeBoolArrayAsUInt8Nullable(bs); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(secondName, arrow::int64()), - arrow::field("b", arrow::uint8()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, secondArr, boolArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableArrowWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - auto batch = MakeArrowBatchWithColumnName(rows, columnName); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - } - } - - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } - } - - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } - bool TryPrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, TVector<TTestHelper::TColumnSchema>* schemaOut) { if (tableKind == ETableKind::COLUMNSHARD) { @@ -309,224 +92,118 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { return false; } } else { - CreateDataShardTable(helper, tableName); + Base::CreateDataShardTable(helper, tableName); return true; } } - } // namespace - - class TBoolTestCase { - public: - TBoolTestCase() - : TestHelper(CreateKikimrSettingsWithBoolSupport()) { - } - - TTestHelper::TUpdatesBuilder Inserter() { - return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema)); - } - - void Upsert(TTestHelper::TUpdatesBuilder& inserter) { - TestHelper.BulkUpsert(TestTable, inserter); - } - - void CheckQuery(const TString& query, const TString& expected, EQueryMode mode = EQueryMode::SCAN_QUERY) const { - switch (mode) { - case EQueryMode::SCAN_QUERY: - TestHelper.ReadData(query, expected); - break; - case EQueryMode::EXECUTE_QUERY: { - TestHelper.ExecuteQuery(query); - break; - } - } - } - - void ExecuteDataQuery(const TString& query) const { - TestHelper.ExecuteQuery(query); - } - void PrepareTable1() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - TestTable.SetName("/Root/Table1").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); - - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(4).Add(true); - inserter.AddRow().Add(2).Add(3).Add(false); - Upsert(inserter); - } + TScenario<bool> FilterEqualScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(4).Add(1).Add(true); - inserter.AddRow().Add(3).Add(2).Add(true); - - Upsert(inserter); - } - } - - void PrepareTable2() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - TestTable.SetName("/Root/Table2").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + { "SELECT * FROM `/Root/Table1` WHERE b == true", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b != true order by id", "[[[%false];2;[3]]]" }, + }, + }; + } + TScenario<bool> FilterNullsScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 5, std::nullopt }, { 6, 6, std::nullopt } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(1).Add(true); - inserter.AddRow().Add(2).Add(1).Add(false); - inserter.AddRow().Add(3).Add(2).Add(true); - inserter.AddRow().Add(4).Add(2).Add(false); - Upsert(inserter); - } - } - - void PrepareTable3() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool).SetNullable(false), - }; - - TestTable.SetName("/Root/Table3").SetPrimaryKey({ "b" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + { "SELECT * FROM `/Root/Table1` WHERE b is NULL order by id", "[[#;5;[5]];[#;6;[6]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b is not NULL order by id", + "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; + } + TScenario<bool> FilterCompareScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(true); - inserter.AddRow().Add(2).Add(false); - Upsert(inserter); - } - } - - private: - TTestHelper TestHelper; - TVector<TTestHelper::TColumnSchema> Schema; - TTestHelper::TColumnTable TestTable; - }; - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[[%true];1;[4]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` order by id", "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + { "SELECT * FROM `/Root/Table1` WHERE b < true order by id", "[[[%false];2;[3]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b > false order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b <= true order by id", + "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b >= true order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b == true", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b != true order by id", "[[[%false];2;[3]]]", Scan); + TScenario<bool> OrderByScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, + { + { "SELECT * FROM `/Root/Table1` order by b, id", + "[[[%false];2;[3]];[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 5, std::nullopt }, { 6, 6, std::nullopt } }, &col, &schema); - const TString expectedNulls = "[[#;5;[5]];[#;6;[6]]]"; - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b is NULL order by id", expectedNulls, Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b is not NULL order by id", - "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TScenario<bool> GroupByScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 12, true }, { 6, 30, false } }, + { + { "SELECT b, count(*) FROM `/Root/Table1` group by b order by b", "[[[%false];2u];[[%true];4u]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b < true order by id", "[[[%false];2;[3]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` WHERE b > false order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b <= true order by id", - "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` WHERE b >= true order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TScenario<bool> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, + { + { "SELECT min(b) FROM `/Root/Table1`", "[[[%false]]]" }, + { "SELECT max(b) FROM `/Root/Table1`", "[[[%true]]]" }, + }, + }; } - Y_UNIT_TEST(TestOrderByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` order by b, id", "[[[%false];2;[3]];[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TJoinScenario<bool> JoinByBoolScenario() { + return { + "/Root/Table1", + "/Root/Table2", + { { 2, 3, true }, { 4, 1, true } }, + { { 2, 2, false }, { 4, 4, false }, { 1, 1, true }, { 3, 3, true } }, + { + { "SELECT t1.id, t2.id, t1.b FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.b = t2.b order by t1.id, t2.id, t1.b", + R"([[2;1;[%true]];[2;3;[%true]];[4;1;[%true]];[4;3;[%true]]])" }, + }, + }; } - Y_UNIT_TEST(TestGroupByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 12, true }, { 6, 30, false } }, &col, &schema); - CheckOrExec(helper, "SELECT b, count(*) FROM `/Root/Table1` group by b order by b", "[[[%false];2u];[[%true];4u]]", Scan); - } + } // namespace - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec(helper, "SELECT min(b) FROM `/Root/Table1`", "[[[%false]]]", Scan); - CheckOrExec(helper, "SELECT max(b) FROM `/Root/Table1`", "[[[%true]]]", Scan); + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[[%true];1;[4]]]", Scan); + CheckOrExec( + helper, "SELECT * FROM `/Root/Table1` order by id", "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByBool, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByBool, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); @@ -534,7 +211,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -555,19 +232,19 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); - CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); } - LoadData(helper, Table, Load, t1, { { 1, 4, true }, { 2, 3, true } }, &col1, &s1); + Base::LoadData(helper, Table, Load, t1, { { 1, 4, true }, { 2, 3, true } }, &col1, &s1); if (Table == ETableKind::COLUMNSHARD) { - LoadData(helper, Table, Load, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, &col2, &s2); + Base::LoadData(helper, Table, Load, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, &col2, &s2); } else { if (Load == ELoadKind::ARROW) { - BulkUpsertRowTableArrowWithSecondColumn( + Base::BulkUpsertRowTableArrowWithSecondColumn( helper, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, "table1_id"); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValueWithSecondColumn( + Base::BulkUpsertRowTableYdbValueWithSecondColumn( helper, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, "table1_id"); } else { TStringBuilder csv; @@ -582,54 +259,17 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { R"([[1;[%true];[%false]];[1;[%true];[%true]];[2;[%true];[%false]];[2;[%true];[%true]]])", Scan); } - Y_UNIT_TEST(TestJoinByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { { 2, 3, true }, { 4, 1, true } }, &col1, &s1); - LoadData(helper, Table, Load, t2, { { 2, 2, false }, { 4, 4, false }, { 1, 1, true }, { 3, 3, true } }, &col2, &s2); - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.b FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.b = t2.b order by t1.id, t2.id, t1.b", - R"([[2;1;[%true]];[2;3;[%true]];[4;1;[%true]];[4;3;[%true]]])", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByBool, JoinByBoolScenario); Y_UNIT_TEST(TestCSVBoolFormats, EQueryMode, ETableKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); + Base::PrepareBase(helper, Table, tableName, &col, &schema); struct TCSVFormat { TString TrueValue; @@ -668,15 +308,15 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { TVector<TRow> rows = { { 1, 100, true }, { 2, 200, false } }; if (Table == ETableKind::DATASHARD) { - PrepareBase(helperDisabled, Table, tableName, &col, &schema); - LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); + Base::PrepareBase(helperDisabled, Table, tableName, &col, &schema); + Base::LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); CheckOrExec( helperDisabled, "SELECT id, int, b FROM `" + tableName + "` ORDER BY id", "[[1;[100];[%true]];[2;[200];[%false]]]", Scan); } else { bool tableCreated = TryPrepareBase(helperDisabled, Table, tableName, &col, &schema); if (tableCreated) { try { - LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); + Base::LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); UNIT_ASSERT_C(false, "Expected error for ColumnShard with disabled feature flag"); } catch (const std::exception& e) { TString errorMsg = e.what(); @@ -686,11 +326,11 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } } - TTestHelper helperEnabled(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helperEnabled(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col2; TVector<TTestHelper::TColumnSchema> schema2; - PrepareBase(helperEnabled, Table, tableName, &col2, &schema2); - LoadData(helperEnabled, Table, ELoadKind::ARROW, tableName, rows, &col2, &schema2); + Base::PrepareBase(helperEnabled, Table, tableName, &col2, &schema2); + Base::LoadData(helperEnabled, Table, ELoadKind::ARROW, tableName, rows, &col2, &schema2); CheckOrExec(helperEnabled, "SELECT id, int, b FROM `" + tableName + "` ORDER BY id", "[[1;[100];[%true]];[2;[200];[%false]]]", Scan); } @@ -704,7 +344,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const TVector<TRow>& rows, const TString& selectExpr, const TString& expectedScan) -> decltype(auto) { - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; col.SetName(name).SetPrimaryKey(pkColumns).SetSharding(pkColumns).SetSchema(schema); helper.CreateTable(col); @@ -713,7 +353,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { if (Load == ELoadKind::ARROW) { using namespace NKikimr::NKqp::NTestArrow; std::vector<bool> bs; bs.reserve(rows.size()); - for (auto&& r : rows) bs.push_back(r.B.value()); + for (auto&& r : rows) bs.push_back(r.TypedVal.value()); auto bArr = MakeBoolArrayAsUInt8(bs); auto batch = MakeBatch({ arrow::field("b", arrow::uint8(), /*nullable*/ false) }, { bArr }); helper.BulkUpsert(col, batch); @@ -721,7 +361,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { TValueBuilder builder; builder.BeginList(); for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("b").Bool(r.B.value()).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("b").Bool(r.TypedVal.value()).EndStruct(); } builder.EndList(); @@ -730,7 +370,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } else { TStringBuilder csv; for (auto&& r : rows) { - csv << (r.B.value() ? "true" : "false") << "\n"; + csv << (r.TypedVal.value() ? "true" : "false") << "\n"; } auto res = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, csv).GetValueSync(); @@ -744,7 +384,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { std::vector<int32_t> ids; ids.reserve(rows.size()); std::vector<uint8_t> bools; bools.reserve(rows.size()); std::vector<int64_t> ints; ints.reserve(rows.size()); - for (auto&& r : rows) { ids.push_back(r.Id); bools.push_back(r.B.value() ? 1u : 0u); ints.push_back(r.IntVal); } + for (auto&& r : rows) { ids.push_back(r.Id); bools.push_back(r.TypedVal.value() ? 1u : 0u); ints.push_back(r.IntVal); } auto idArr = MakeInt32Array(ids); auto bArr = MakeUInt8Array(bools); auto iArr = MakeInt64Array(ints); @@ -756,7 +396,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { for (auto&& r : rows) { builder.AddListItem().BeginStruct() .AddMember("id").Int32(r.Id) - .AddMember("b").Bool(r.B.value()) + .AddMember("b").Bool(r.TypedVal.value()) .AddMember("int").Int64(r.IntVal) .EndStruct(); } @@ -767,7 +407,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } else { TStringBuilder csv; for (auto&& r : rows) { - csv << r.Id << "," << (r.B.value() ? "true" : "false") << "," << r.IntVal << "\n"; + csv << r.Id << "," << (r.TypedVal.value() ? "true" : "false") << "," << r.IntVal << "\n"; } auto res = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, csv).GetValueSync(); @@ -806,14 +446,14 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto settings = CreateKikimrSettingsWithBoolSupport(); + auto settings = TBoolTraits::CreateSettings(); settings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(settings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -903,7 +543,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolOps"; @@ -961,7 +601,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolOrderLimit"; @@ -1059,7 +699,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { Y_UNIT_TEST(TestBoolCompare, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolWhereCmp"; @@ -1139,7 +779,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { Y_UNIT_TEST(TestBoolFilterWithColumns, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolFilterCols"; @@ -1222,7 +862,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolWriteCmp"; @@ -1277,7 +917,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolNot"; @@ -1323,7 +963,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolGroup"; diff --git a/ydb/core/kqp/ut/olap/types/column_type_scenarios.h b/ydb/core/kqp/ut/olap/types/column_type_scenarios.h new file mode 100644 index 00000000000..1ce51e8c6b7 --- /dev/null +++ b/ydb/core/kqp/ut/olap/types/column_type_scenarios.h @@ -0,0 +1,165 @@ +#pragma once + +#include "column_type_test_base.h" + +namespace NKikimr::NKqp { + +struct TQueryCheck { + TString Query; + TString Expected; +}; + +template <typename TValue> +struct TScenarioLoad { + TVector<TTypedRow<TValue>> Data; + TVector<TQueryCheck> ChecksAfter; +}; + +template <typename TValue> +struct TScenario { + TString TableName; + TVector<TTypedRow<TValue>> Data; + TVector<TQueryCheck> Checks; + TVector<TScenarioLoad<TValue>> ExtraLoads{}; +}; + +template <typename TValue> +struct TJoinScenario { + TString Table1Name; + TString Table2Name; + TVector<TTypedRow<TValue>> Table1Data; + TVector<TTypedRow<TValue>> Table2Data; + TVector<TQueryCheck> Checks; +}; + +struct TPkLookupScenario { + TString TableName; + TVector<TQueryCheck> Checks; +}; + +struct TCsvScenario { + TString TableName; + TString CsvData; + TVector<TQueryCheck> Checks; +}; + +template <typename TTraits> +void RunScenario(const TScenario<typename TTraits::TValue>& scenario, + EQueryMode scan, ETableKind table, ELoadKind load) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, table, scenario.TableName, &col, &schema); + Base::LoadData(helper, table, load, scenario.TableName, scenario.Data, &col, &schema); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } + + for (auto&& extra : scenario.ExtraLoads) { + Base::LoadData(helper, table, load, scenario.TableName, extra.Data, &col, &schema); + for (auto&& check : extra.ChecksAfter) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } + } +} + +template <typename TTraits> +void RunJoinScenario(const TJoinScenario<typename TTraits::TValue>& scenario, + EQueryMode scan, ETableKind table, ELoadKind load) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col1; + TTestHelper::TColumnTable col2; + TVector<TTestHelper::TColumnSchema> s1; + TVector<TTestHelper::TColumnSchema> s2; + + if (table == ETableKind::COLUMNSHARD) { + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()), + }; + + s1 = schema; + col1.SetName(scenario.Table1Name).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col1); + s2 = schema; + col2.SetName(scenario.Table2Name).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col2); + } else { + Base::CreateDataShardTable(helper, scenario.Table1Name); + Base::CreateDataShardTable(helper, scenario.Table2Name); + } + + Base::LoadData(helper, table, load, scenario.Table1Name, scenario.Table1Data, &col1, &s1); + Base::LoadData(helper, table, load, scenario.Table2Name, scenario.Table2Data, &col2, &s2); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +template <typename TTraits> +void RunPkLookupScenario(const TPkLookupScenario& scenario, EQueryMode scan, ELoadKind load) { + TTestHelper helper(TTraits::CreateSettings()); + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()).SetNullable(false), + TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64), + }; + + TTestHelper::TColumnTable testTable; + testTable.SetName(scenario.TableName) + .SetPrimaryKey({ TTraits::ColumnName }) + .SetSharding({ TTraits::ColumnName }) + .SetSchema(schema); + helper.CreateTable(testTable); + TTraits::LoadPkTable(helper, load, scenario.TableName, testTable); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +template <typename TTraits> +void RunCsvScenario(const TCsvScenario& scenario, EQueryMode scan, ETableKind table) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, table, scenario.TableName, &col, &schema); + + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + scenario.TableName, EDataFormat::CSV, scenario.CsvData).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +#define Y_UNIT_TEST_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind, ELoadKind) { \ + RunScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>(), Arg<2>()); \ + } + +#define Y_UNIT_TEST_JOIN_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind, ELoadKind) { \ + RunJoinScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>(), Arg<2>()); \ + } + +#define Y_UNIT_TEST_PK_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ELoadKind) { \ + RunPkLookupScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>()); \ + } + +#define Y_UNIT_TEST_CSV_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind) { \ + RunCsvScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>()); \ + } + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/types/column_type_test_base.h b/ydb/core/kqp/ut/olap/types/column_type_test_base.h new file mode 100644 index 00000000000..b20411caabb --- /dev/null +++ b/ydb/core/kqp/ut/olap/types/column_type_test_base.h @@ -0,0 +1,299 @@ +#pragma once + +#include "column_type_test_enums.h" + +#include <ydb/core/formats/arrow/arrow_helpers.h> +#include <ydb/core/kqp/ut/common/arrow_builders.h> +#include <ydb/core/kqp/ut/common/columnshard.h> +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/core/testlib/common_helper.h> +#include <ydb/core/testlib/cs_helper.h> +#include <ydb/core/tx/columnshard/hooks/testing/controller.h> +#include <ydb/core/tx/tx_proxy/proxy.h> + +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> + +#include <library/cpp/threading/local_executor/local_executor.h> +#include <util/generic/serialized_enum.h> +#include <util/string/printf.h> + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +template <typename TValue> +struct TTypedRow { + i32 Id; + i64 IntVal; + std::optional<TValue> TypedVal; +}; + +inline void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { + if (scanMode == EQueryMode::SCAN_QUERY) { + helper.ReadData(query, expected); + } else { + helper.ReadDataExecQuery(query, expected); + } +} + +template <typename TConfigure> +TKikimrSettings CreateColumnshardSettings(TConfigure&& configure) { + NKikimrConfig::TFeatureFlags featureFlags; + configure(featureFlags); + return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); +} + +// TTraits contract: +// using TValue = ...; +// static constexpr const char* ColumnName; +// static constexpr const char* SqlTypeName; +// static TKikimrSettings CreateSettings(); +// static NScheme::TTypeId GetTypeId(); +// static void AppendYdbValue(TValueBuilder& builder, const std::optional<TValue>& val); +// static void AppendCsvValue(TStringBuilder& builder, const std::optional<TValue>& val); +// static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TValue>>& rows); +// static std::shared_ptr<arrow::DataType> ArrowType(); +template <typename TTraits> +struct TColumnTypeTestBase { + using TValue = typename TTraits::TValue; + using TRow = TTypedRow<TValue>; + + static TString GetSqlTypeName() { + if constexpr (requires { { TTraits::BuildSqlTypeName() } -> std::convertible_to<TString>; }) { + return TTraits::BuildSqlTypeName(); + } else { + return TTraits::SqlTypeName; + } + } + + static void CreateDataShardTable(TTestHelper& helper, const TString& name) { + auto& session = helper.GetSession(); + auto res = session + .ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE TABLE `)" << name << R"(` ( + id Int32 NOT NULL, + int Int64, + )" << TTraits::ColumnName << " " << GetSqlTypeName() << R"(, + PRIMARY KEY (id) + ); + )") + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); + } + + static void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + TValueBuilder builder; + builder.BeginList(); + for (auto&& r : rows) { + builder.AddListItem().BeginStruct() + .AddMember("id").Int32(r.Id) + .AddMember("int").Int64(r.IntVal) + .AddMember(TTraits::ColumnName); + TTraits::AppendYdbValue(builder, r.TypedVal); + builder.EndStruct(); + } + + builder.EndList(); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + BulkUpsertRowTableCSVWithColumnName(helper, name, rows, "int"); + } + + static void BulkUpsertRowTableCSVWithColumnName( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { + Y_UNUSED(columnName); + TStringBuilder builder; + for (auto&& r : rows) { + builder << r.Id << "," << r.IntVal << ","; + TTraits::AppendCsvValue(builder, r.TypedVal); + builder << '\n'; + } + + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<int32_t> ids; + std::vector<int64_t> vals; + ids.reserve(rows.size()); + vals.reserve(rows.size()); + for (auto&& r : rows) { + ids.push_back(r.Id); + vals.push_back(r.IntVal); + } + + auto idArr = MakeInt32Array(ids); + auto intArr = MakeInt64Array(vals); + auto typedArr = TTraits::MakeArrowArray(rows); + return MakeBatch( + { arrow::field("id", arrow::int32(), false), + arrow::field("int", arrow::int64()), + arrow::field(TTraits::ColumnName, TTraits::ArrowType()) }, + { idArr, intArr, typedArr }); + } + + static void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + auto batch = MakeArrowBatch(rows); + TString strBatch = NArrow::SerializeBatchNoCompression(batch); + TString strSchema = NArrow::SerializeSchema(*batch->schema()); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, + const TString& name, const TVector<TRow>& rows, + TTestHelper::TColumnTable* col = nullptr, + const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { + switch (table) { + case ETableKind::COLUMNSHARD: { + Y_ABORT_UNLESS(col && schema); + TString columnName = "int"; + if (schema->size() >= 2) { + columnName = (*schema)[1].GetName(); + } + + if (load == ELoadKind::ARROW) { + auto batch = MakeArrowBatchWithColumnName(rows, columnName); + helper.BulkUpsert(*col, batch); + } else if (load == ELoadKind::YDB_VALUE) { + BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); + } else { + BulkUpsertRowTableCSVWithColumnName(helper, name, rows, columnName); + } + + break; + } + case ETableKind::DATASHARD: { + if (load == ELoadKind::ARROW) { + BulkUpsertRowTableArrow(helper, name, rows); + } else if (load == ELoadKind::YDB_VALUE) { + BulkUpsertRowTableYdbValue(helper, name, rows); + } else { + BulkUpsertRowTableCSV(helper, name, rows); + } + + break; + } + } + } + + static void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, + TTestHelper::TColumnTable* colTableOut, TVector<TTestHelper::TColumnSchema>* schemaOut) { + if (tableKind == ETableKind::COLUMNSHARD) { + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()), + }; + + *schemaOut = schema; + TTestHelper::TColumnTable col; + col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col); + *colTableOut = col; + } else { + CreateDataShardTable(helper, tableName); + } + } + + static void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { + auto& session = helper.GetSession(); + auto res = session + .ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE TABLE `)" << name << R"(` ( + id Int32 NOT NULL, + )" << secondName << R"( Int64, + )" << TTraits::ColumnName << " " << GetSqlTypeName() << R"(, + PRIMARY KEY (id) + ); + )") + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); + } + + static void BulkUpsertRowTableYdbValueWithColumnName( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { + TValueBuilder builder; + builder.BeginList(); + for (auto&& r : rows) { + builder.AddListItem().BeginStruct() + .AddMember("id").Int32(r.Id) + .AddMember(columnName).Int64(r.IntVal) + .AddMember(TTraits::ColumnName); + TTraits::AppendYdbValue(builder, r.TypedVal); + builder.EndStruct(); + } + + builder.EndList(); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void BulkUpsertRowTableYdbValueWithSecondColumn( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { + BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, secondName); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<int32_t> ids; + std::vector<int64_t> vals; + ids.reserve(rows.size()); + vals.reserve(rows.size()); + for (auto&& r : rows) { + ids.push_back(r.Id); + vals.push_back(r.IntVal); + } + + auto idArr = MakeInt32Array(ids); + auto intArr = MakeInt64Array(vals); + auto typedArr = TTraits::MakeArrowArray(rows); + return MakeBatch( + { arrow::field("id", arrow::int32(), false), + arrow::field(std::string(columnName), arrow::int64()), + arrow::field(TTraits::ColumnName, TTraits::ArrowType()) }, + { idArr, intArr, typedArr }); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { + return MakeArrowBatchWithColumnName(rows, secondName); + } + + static void BulkUpsertRowTableArrowWithSecondColumn( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { + auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); + TString strBatch = NArrow::SerializeBatchNoCompression(batch); + TString strSchema = NArrow::SerializeSchema(*batch->schema()); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } +}; + +template <typename TTraits> +struct TColumnTypeTestContext { + using Traits = TTraits; + using Base = TColumnTypeTestBase<TTraits>; + using TRow = TTypedRow<typename TTraits::TValue>; + using TValue = typename TTraits::TValue; + + static TKikimrSettings Settings() { return TTraits::CreateSettings(); } +}; + +#define COLUMN_TYPE_TEST_USING(Traits) \ + using TTraits = Traits; \ + using Ctx = TColumnTypeTestContext<TTraits>; \ + using Base = Ctx::Base; \ + using TRow = Ctx::TRow; + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/types/bool_test_enums.h b/ydb/core/kqp/ut/olap/types/column_type_test_enums.h index 03c5a2eb7cc..03c5a2eb7cc 100644 --- a/ydb/core/kqp/ut/olap/types/bool_test_enums.h +++ b/ydb/core/kqp/ut/olap/types/column_type_test_enums.h diff --git a/ydb/core/kqp/ut/olap/types/decimal_ut.cpp b/ydb/core/kqp/ut/olap/types/decimal_ut.cpp index d9d9ae6872d..e89c3df331f 100644 --- a/ydb/core/kqp/ut/olap/types/decimal_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/decimal_ut.cpp @@ -1,22 +1,5 @@ -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/columnshard/test_helper/test_combinator.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <yql/essentials/types/binary_json/write.h> -#include <yql/essentials/types/uuid/uuid.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" namespace NKikimr { namespace NKqp { @@ -24,384 +7,544 @@ namespace NKqp { using namespace NYdb; using namespace NYdb::NTable; -enum class EQueryMode { - SCAN_QUERY, - EXECUTE_QUERY -}; - Y_UNIT_TEST_SUITE(KqpDecimalColumnShard) { - class TDecimalTestCase { - public: - TDecimalTestCase(ui32 precision, ui32 scale) - : TestHelper(TKikimrSettings().SetWithSampleTables(false)) - , Precision(precision) - , Scale(scale) - {} + namespace { - TTestHelper::TUpdatesBuilder Inserter() { - return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema)); - } + template <ui32 Precision, ui32 Scale> + struct TDecimalTraits { + using TValue = TString; + static constexpr const char* ColumnName = "dec"; + static constexpr ui32 DecimalPrecision = Precision; + static constexpr ui32 DecimalScale = Scale; - void Upsert(TTestHelper::TUpdatesBuilder& inserter) { - TestHelper.BulkUpsert(TestTable, inserter); + static TKikimrSettings CreateSettings() { + return TKikimrSettings().SetWithSampleTables(false); } - void CheckQuery(const TString& query, const TString& expected, EQueryMode mode = EQueryMode::SCAN_QUERY) const { - switch (mode) { - case EQueryMode::SCAN_QUERY: - TestHelper.ReadData(query, expected); - break; - case EQueryMode::EXECUTE_QUERY: { - TestHelper.ExecuteQuery(query); - break; - } - } - } + static auto GetTypeId() { return NScheme::TDecimalType(Precision, Scale); } - void ExecuteDataQuery(const TString& query) const { - TestHelper.ExecuteQuery(query); + static TString BuildSqlTypeName() { + return TStringBuilder() << "Decimal(" << Precision << "," << Scale << ")"; } - void PrepareTable1() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(Precision, Scale)), - }; - TestTable.SetName("/Root/Table1").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + static TString DecCast(const TString& val) { + return TStringBuilder() << "CAST(\"" << val << "\" AS Decimal(" << Precision << "," << Scale << "))"; + } - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(4).Add(TDecimalValue("3.14", Precision, Scale)); - inserter.AddRow().Add(2).Add(3).Add(TDecimalValue("8.16", Precision, Scale)); - Upsert(inserter); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().Decimal(TDecimalValue(*val, Precision, Scale)).EndOptional(); + } else { + builder.EmptyOptional(NYdb::TTypeBuilder().Decimal(TDecimalType(Precision, Scale)).Build()); } - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(4).Add(1).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("8.492", Precision, Scale)); + } - Upsert(inserter); + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } } - void PrepareTable2() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(Precision, Scale)), - }; - TestTable.SetName("/Root/Table2").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); - - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(1).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(2).Add(1).Add(TDecimalValue("8.16", Precision, Scale)); - inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(4).Add(2).Add(TDecimalValue("8.16", Precision, Scale)); - Upsert(inserter); + static void AppendDecimalBytes(arrow::FixedSizeBinaryBuilder& builder, const TString& val) { + TDecimalValue decimal(val, Precision, Scale); + char bytes[NScheme::FSB_SIZE] = { 0 }; + for (i32 i = 0; i < 8; ++i) { + bytes[i] = (decimal.Low_ >> (i << 3)) & 0xFF; + bytes[i + 8] = (decimal.Hi_ >> (i << 3)) & 0xFF; } + Y_ABORT_UNLESS(builder.Append(bytes).ok()); } - private: - TTestHelper TestHelper; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + auto type = arrow::fixed_size_binary(NScheme::FSB_SIZE); + arrow::FixedSizeBinaryBuilder builder(type); + for (auto&& r : rows) { + if (r.TypedVal.has_value()) { + AppendDecimalBytes(builder, *r.TypedVal); + } else { + Y_ABORT_UNLESS(builder.AppendNull().ok()); + } + } + std::shared_ptr<arrow::Array> arr; + Y_ABORT_UNLESS(builder.Finish(&arr).ok()); + return arr; + } - TVector<TTestHelper::TColumnSchema> Schema; - TTestHelper::TColumnTable TestTable; + static std::shared_ptr<arrow::DataType> ArrowType() { + return arrow::fixed_size_binary(NScheme::FSB_SIZE); + } - YDB_READONLY_DEF(ui32, Precision); - YDB_READONLY_DEF(ui32, Scale); + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); }; - Y_UNIT_TEST_DUO(TestSimpleQueries, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + using Decimal229 = TDecimalTraits<22, 9>; + COLUMN_TYPE_TEST_USING(Decimal229); - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE id=1", "[[[\"3.14\"];1;[4]]]", mode); - - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } + template <> + void TDecimalTraits<22, 9>::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + auto type = arrow::fixed_size_binary(NScheme::FSB_SIZE); + arrow::FixedSizeBinaryBuilder decB(type); + Decimal229::AppendDecimalBytes(decB, "3.14"); + Decimal229::AppendDecimalBytes(decB, "8.16"); + Decimal229::AppendDecimalBytes(decB, "12.46"); + using namespace NKikimr::NKqp::NTestArrow; + auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); + std::shared_ptr<arrow::Array> decArr; + Y_ABORT_UNLESS(decB.Finish(&decArr).ok()); + auto batch = MakeBatch( + { arrow::field("dec", arrow::fixed_size_binary(NScheme::FSB_SIZE), false), + arrow::field("val", arrow::int64()) }, + { decArr, valArr }); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("3.14", 22, 9)).AddMember("val").Int64(1).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("8.16", 22, 9)).AddMember("val").Int64(2).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("12.46", 22, 9)).AddMember("val").Int64(3).EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "3.14,1\n8.16,2\n12.46,3\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } + } - tester.CheckQuery("SELECT * FROM `/Root/Table1` order by id", expected, mode); + TScenario<TString> FilterEqualScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("3.14"), "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec != " + Decimal229::DecCast("3.14") + " ORDER BY id", "[[2];[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("12.46"), "[[4]]" }, + }, }; - - check(tester22); - check(tester35); - check(tester12); } - Y_UNIT_TEST_DUO(TestFilterEqual, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - auto check = [mode](const TDecimalTestCase& tester, ui32 precision, ui32 scale) { - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec == cast(\"3.14\" as decimal(", ToString(precision), ",", ToString(scale), "))"), "[[[\"3.14\"];1;[4]]]", mode); - - TString expected; - if (precision == 12 && scale == 2) { - expected = "[[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec != cast(\"3.14\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected, mode); + TScenario<TString> FilterNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, std::nullopt }, + { 3, 30, "8.16" }, + { 4, 40, std::nullopt }, + { 5, 50, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec IS NOT NULL ORDER BY id", "[[1];[3];[5]]" }, + }, }; - - check(tester22, 22, 9); - check(tester35, 35, 10); - check(tester12, 12, 2); } - Y_UNIT_TEST_DUO(TestFilterNulls, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - auto insert = [](TDecimalTestCase& tester) { - TTestHelper::TUpdatesBuilder inserter = tester.Inserter(); - inserter.AddRow().Add(5).Add(5).AddNull(); - inserter.AddRow().Add(6).Add(6).AddNull(); - tester.Upsert(inserter); + TScenario<TString> FilterCompareScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec < " + Decimal229::DecCast("12.46") + " ORDER BY id", + "[[1];[2];[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec > " + Decimal229::DecCast("8.16") + " ORDER BY id", + "[[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec <= " + Decimal229::DecCast("12.46") + " ORDER BY id", + "[[1];[2];[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec >= " + Decimal229::DecCast("8.492") + " ORDER BY id", + "[[3];[4]]" }, + }, }; + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is NULL order by id", "[[#;5;[5]];[#;6;[6]]]", mode); - - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is not NULL order by id", expected, mode); + TScenario<TString> OrderByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "12.46" }, + { 2, 20, "3.14" }, + { 3, 30, "8.492" }, + { 4, 40, "8.16" }, + }, + { + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec", + "[[[\"3.14\"];2];[[\"8.16\"];4];[[\"8.492\"];3];[[\"12.46\"];1]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec DESC", + "[[[\"12.46\"];1];[[\"8.492\"];3];[[\"8.16\"];4];[[\"3.14\"];2]]" }, + }, }; - - insert(tester22); - insert(tester35); - insert(tester12); - - check(tester22); - check(tester35); - check(tester12); } - Y_UNIT_TEST_DUO(TestFilterCompare, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + TScenario<TString> GroupByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "3.14" }, + { 4, 40, "8.16" }, + { 5, 50, "12.46" }, + }, + { + { "SELECT dec, count(*) AS cnt FROM `" + tableName + "` GROUP BY dec ORDER BY dec", + "[[[\"3.14\"];2u];[[\"8.16\"];2u];[[\"12.46\"];1u]]" }, + }, + }; + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + TScenario<TString> AggregationScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, std::nullopt }, + { 5, 50, "12.46" }, + }, + { + { "SELECT min(dec) FROM `" + tableName + "`", "[[[\"3.14\"]]]" }, + { "SELECT max(dec) FROM `" + tableName + "`", "[[[\"12.46\"]]]" }, + { "SELECT count(dec) FROM `" + tableName + "`", "[[4u]]" }, + { "SELECT count(*) FROM `" + tableName + "`", "[[5u]]" }, + { "SELECT sum(dec) FROM `" + tableName + "`", "[[[\"32.252\"]]]" }, + }, + }; + } - auto check = [mode](const TDecimalTestCase& tester, ui32 precision, ui32 scale) { - TString expected1, expected2, expected3, expected4; - - if (precision == 12 && scale == 2) { - expected1 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]]]"; - expected2 = "[[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected3 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected4 = "[[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected1 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]]]"; - expected2 = "[[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected3 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected4 = "[[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec < cast(\"12.46\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected1, mode); + TJoinScenario<TString> JoinByDecimalScenario() { + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + return { + t1, + t2, + { + { 2, 20, "8.16" }, + { 4, 40, "12.46" }, + }, + { + { 1, 100, "12.46" }, + { 2, 200, "8.16" }, + { 3, 300, "12.46" }, + { 4, 400, "8.16" }, + }, + { + { "SELECT t1.id, t2.id, t1.dec FROM `" + t1 + "` AS t1 " + "JOIN `" + t2 + "` AS t2 ON t1.dec = t2.dec ORDER BY t1.id, t2.id", + R"([[2;2;["8.16"]];[2;4;["8.16"]];[4;1;["12.46"]];[4;3;["12.46"]]])" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec > cast(\"8.16\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected2, mode); + TScenario<TString> OrderByWithLimitScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "12.46" }, + { 2, 20, "3.14" }, + { 3, 30, "8.16" }, + { 4, 40, "8.492" }, + { 5, 50, "100" }, + }, + { + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec LIMIT 2", + "[[[\"3.14\"];2];[[\"8.16\"];3]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec DESC LIMIT 2", + "[[[\"100\"];5];[[\"12.46\"];1]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec LIMIT 2 OFFSET 2", + "[[[\"8.492\"];4];[[\"12.46\"];1]]" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec <= cast(\"12.46\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected3, mode); + TScenario<TString> GroupByWithNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, std::nullopt }, + { 3, 30, "3.14" }, + { 4, 40, std::nullopt }, + { 5, 50, "8.16" }, + }, + { + { "SELECT dec, count(*) AS cnt FROM `" + tableName + "` GROUP BY dec ORDER BY dec", + "[[#;2u];[[\"3.14\"];2u];[[\"8.16\"];1u]]" }, + { "SELECT count(dec), count(*) FROM `" + tableName + "`", "[[3u;5u]]" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec >= cast(\"8.492\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected4, mode); + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("8.16"), "[[[2]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("3.14"), "[[[1]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("12.46"), "[[[3]]]" }, + }, }; + } - check(tester22, 22, 9); - check(tester35, 35, 10); - check(tester12, 12, 2); + TCsvScenario CsvScenario() { + const TString tableName = "/Root/Table1"; + return { + tableName, + "1,10,3.14\n2,20,8.16\n3,30,8.492\n", + { + { "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY id", + "[[[\"3.14\"];1];[[\"8.16\"];2];[[\"8.492\"];3]]" }, + }, + }; } - Y_UNIT_TEST_DUO(TestOrderByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + template <ui32 Precision, ui32 Scale> + void RunFilterEqualForPrecision(EQueryMode scan, ETableKind table, ELoadKind load) { + using TTraits = TDecimalTraits<Precision, Scale>; + const TString tableName = TStringBuilder() << "/Root/ColumnTableTest_" << Precision << "_" << Scale; + const TString midValue = (Scale == 2) ? "8.49" : "8.492"; + TScenario<TString> scenario{ + tableName, + { + { 1, 4, "3.14" }, + { 2, 3, "8.16" }, + { 3, 2, midValue }, + { 4, 1, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec = " + TTraits::DecCast("3.14"), "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec != " + TTraits::DecCast("3.14") + " ORDER BY id", + TStringBuilder() << "[[2];[3];[4]]" }, + }, + }; + RunScenario<TTraits>(scenario, scan, table, load); + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + } // namespace - auto check = [mode](const TDecimalTestCase& tester) { - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - tester.CheckQuery("SELECT * FROM `/Root/Table1` order by dec", expected, mode); - }; + const TString tableName = "/Root/ColumnTableTest"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { + { 1, 4, "3.14" }, + { 2, 3, "8.16" }, + { 3, 2, "8.492" }, + { 4, 1, "12.46" }, + }, &col, &schema); - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` WHERE id=1", "[[1;[4]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` WHERE id=3", "[[3;[2]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` ORDER BY id", + "[[1;[4]];[2;[3]];[3;[2]];[4;[1]]]", Scan); + CheckOrExec(helper, "SELECT dec FROM `" + tableName + "` WHERE id=1", "[[[\"3.14\"]]]", Scan); + CheckOrExec(helper, "SELECT dec FROM `" + tableName + "` WHERE id=3", "[[[\"8.492\"]]]", Scan); } - Y_UNIT_TEST_DUO(TestGroupByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByDecimal, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByDecimal, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); - auto insert = [](TDecimalTestCase& tester) { - TTestHelper::TUpdatesBuilder inserter = tester.Inserter(); - inserter.AddRow().Add(5).Add(12).Add(TDecimalValue("8.492", tester.GetPrecision(), tester.GetScale())); - inserter.AddRow().Add(6).Add(30).Add(TDecimalValue("12.46", tester.GetPrecision(), tester.GetScale())); - tester.Upsert(inserter); - }; + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col1, col2; + TVector<TTestHelper::TColumnSchema> s1, s2; + if (Table == ETableKind::COLUMNSHARD) { + s1 = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), + }; + col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); + helper.CreateTable(col1); + s2 = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), + }; + col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); + helper.CreateTable(col2); + } else { + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + } - auto check = [mode](const TDecimalTestCase& tester) { - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1u];[[\"8.16\"];1u];[[\"8.49\"];2u];[[\"12.46\"];2u]]"; + Base::LoadData(helper, Table, Load, t1, { { 1, 100, "3.14" }, { 2, 200, "8.16" } }, &col1, &s1); + if (Table == ETableKind::COLUMNSHARD) { + Base::LoadData(helper, Table, Load, t2, + { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, &col2, &s2); + } else { + if (Load == ELoadKind::ARROW) { + Base::BulkUpsertRowTableArrowWithSecondColumn( + helper, t2, { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, "table1_id"); + } else if (Load == ELoadKind::YDB_VALUE) { + Base::BulkUpsertRowTableYdbValueWithSecondColumn( + helper, t2, { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, "table1_id"); } else { - expected = "[[[\"3.14\"];1u];[[\"8.16\"];1u];[[\"8.492\"];2u];[[\"12.46\"];2u]]"; + TStringBuilder csv; + csv << "1,1,8.16\n2,1,12.46\n3,2,8.16\n4,2,12.46\n"; + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(t2, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } + } - tester.CheckQuery("SELECT dec, count(*) FROM `/Root/Table1` group by dec order by dec", expected, mode); - }; + CheckOrExec(helper, + R"(SELECT t1.id, t1.dec, t2.dec FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.id = t2.table1_id ORDER BY t1.id, t2.dec)", + R"([[1;["3.14"];["8.16"]];[1;["3.14"];["12.46"]];[2;["8.16"];["8.16"]];[2;["8.16"];["12.46"]]])", Scan); + } - insert(tester22); - insert(tester35); - insert(tester12); + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByDecimal, JoinByDecimalScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestDecimalAsPrimaryKey, PkLookupScenario); + Y_UNIT_TEST_CSV_SCENARIO(TestCsv, CsvScenario); - check(tester22); - check(tester35); - check(tester12); - } + Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Load = Arg<1>(); - Y_UNIT_TEST_DUO(TestAggregation, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + auto runnerSettings = Decimal229::CreateSettings(); + runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); + TTestHelper helper(runnerSettings); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString ds = "/Root/RowSrc"; + const TString cs = "/Root/ColSrc"; - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT min(dec) FROM `/Root/Table1`", "[[[\"3.14\"]]]", mode); - tester.CheckQuery("SELECT max(dec) FROM `/Root/Table1`", "[[[\"12.46\"]]]", mode); - - TString expectedSum; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expectedSum = "[[[\"32.25\"]]]"; - } else { - expectedSum = "[[[\"32.252\"]]]"; - } + Base::CreateDataShardTable(helper, ds); - tester.CheckQuery("SELECT sum(dec) FROM `/Root/Table1`", expectedSum, mode); + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), }; - check(tester22); - check(tester35); - check(tester12); - } + TTestHelper::TColumnTable col; + col.SetName(cs).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col); - Y_UNIT_TEST_DUO(TestJoinById, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - tester22.PrepareTable2(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - tester35.PrepareTable2(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - tester12.PrepareTable2(); + helper.ExecuteQuery( + "UPSERT INTO `" + ds + "` (id, int, dec) VALUES " + "(1, 100, CAST(\"3.14\" AS Decimal(22,9))), " + "(2, 200, CAST(\"8.16\" AS Decimal(22,9)))"); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + if (Load == ELoadKind::ARROW) { + auto batch = Base::MakeArrowBatch({ + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + helper.BulkUpsert(col, batch); + } else if (Load == ELoadKind::YDB_VALUE) { + Base::BulkUpsertRowTableYdbValue(helper, cs, { + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + } else { + Base::BulkUpsertRowTableCSV(helper, cs, { + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + } - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery( - "SELECT t1.id, t1.dec, t2.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.id = t2.table1_id order by t1.id, t1.dec, " - "t2.dec", - R"([[1;["3.14"];["8.16"]];[1;["3.14"];["12.46"]];[2;["8.16"];["8.16"]];[2;["8.16"];["12.46"]]])", mode); - }; + CheckOrExec(helper, "SELECT id, int, dec FROM `" + ds + "` ORDER BY id", + "[[1;[100];[\"3.14\"]];[2;[200];[\"8.16\"]]]", Scan); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + cs + "` ORDER BY id", + "[[1;[100];[\"3.14\"]];[2;[200];[\"8.16\"]]]", Scan); + + helper.ExecuteQuery( + "UPSERT INTO `" + ds + "` (id, int, dec) VALUES " + "(3, 300, CAST(\"12.46\" AS Decimal(22,9))), " + "(1, 110, CAST(\"10.1\" AS Decimal(22,9)))"); + helper.ExecuteQuery( + "UPSERT INTO `" + cs + "` (id, int, dec) VALUES " + "(3, 300, CAST(\"12.46\" AS Decimal(22,9))), " + "(1, 110, CAST(\"10.1\" AS Decimal(22,9)))"); - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + ds + "` ORDER BY id", + "[[1;[110];[\"10.1\"]];[2;[200];[\"8.16\"]];[3;[300];[\"12.46\"]]]", Scan); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + cs + "` ORDER BY id", + "[[1;[110];[\"10.1\"]];[2;[200];[\"8.16\"]];[3;[300];[\"12.46\"]]]", Scan); + + helper.ExecuteQuery("DELETE FROM `" + ds + "` WHERE id = 2"); + helper.ExecuteQuery("DELETE FROM `" + cs + "` WHERE id = 2"); + CheckOrExec(helper, "SELECT id FROM `" + ds + "` ORDER BY id", "[[1];[3]]", Scan); + CheckOrExec(helper, "SELECT id FROM `" + cs + "` ORDER BY id", "[[1];[3]]", Scan); } - Y_UNIT_TEST_DUO(TestJoinByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - tester22.PrepareTable2(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - tester35.PrepareTable2(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - tester12.PrepareTable2(); + Y_UNIT_TEST(TestPMInfDecimal, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString tableName = "/Root/Table1"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, Table, tableName, &col, &schema); - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery( - "SELECT t1.id, t2.id, t1.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.dec = t2.dec order by t1.id, t2.id, t1.dec", - R"([[2;2;["8.16"]];[2;4;["8.16"]];[4;1;["12.46"]];[4;3;["12.46"]]])", mode); - }; + if (Load == ELoadKind::CSV) { + Base::LoadData(helper, Table, ELoadKind::CSV, tableName, { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + }, &col, &schema); + Base::LoadData(helper, Table, ELoadKind::YDB_VALUE, tableName, { + { 3, 30, "inf" }, + { 4, 40, "-inf" }, + }, &col, &schema); + } else { + Base::LoadData(helper, Table, Load, tableName, { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "inf" }, + { 4, 40, "-inf" }, + }, &col, &schema); + } - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT max(dec) FROM `" + tableName + "`", "[[[\"inf\"]]]", Scan); + CheckOrExec(helper, "SELECT min(dec) FROM `" + tableName + "`", "[[[\"-inf\"]]]", Scan); } - Y_UNIT_TEST_DUO(TestPMInfDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - auto inserter = tester22.Inserter(); - inserter.AddRow().Add(1).Add(5).Add(TDecimalValue("999999999999999999999", 22, 9)); - inserter.AddRow().Add(2).Add(6).Add(TDecimalValue("-999999999999999999999", 22, 9)); - tester22.Upsert(inserter); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - tester22.CheckQuery("SELECT max(dec) FROM `/Root/Table1`", "[[[\"inf\"]]]", mode); - tester22.CheckQuery("SELECT min(dec) FROM `/Root/Table1`", "[[[\"-inf\"]]]", mode); + Y_UNIT_TEST(TestFilterEqualMultiplePrecisions, EQueryMode, ETableKind, ELoadKind) { + RunFilterEqualForPrecision<22, 9>(Arg<0>(), Arg<1>(), Arg<2>()); + RunFilterEqualForPrecision<35, 10>(Arg<0>(), Arg<1>(), Arg<2>()); + RunFilterEqualForPrecision<12, 2>(Arg<0>(), Arg<1>(), Arg<2>()); } } diff --git a/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp b/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp index b9db096ed09..a64ef4a221f 100644 --- a/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp @@ -1,25 +1,8 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" #include <yql/essentials/types/dynumber/dynumber.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> - namespace NKikimr { namespace NKqp { @@ -29,178 +12,290 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<TString> Dyn; - }; + struct TDyNumberTraits { + using TValue = TString; + static constexpr const char* ColumnName = "dyn"; + static constexpr const char* SqlTypeName = "DyNumber"; - static TString MakeDyNumber(const TStringBuf& str) { - auto result = NDyNumber::ParseDyNumberString(str); - Y_ABORT_UNLESS(result.Defined(), "Failed to parse DyNumber: %s", TString(str).c_str()); - return TString(str); - } - - TKikimrSettings CreateKikimrSettingsWithDyNumberSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardDyNumber(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - - static void AppendDyNumber(arrow::BinaryBuilder& builder, const TStringBuf& str) { - Y_ABORT_UNLESS(builder.Append(str.data(), str.size()).ok()); - } + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardDyNumber(true); }); + } - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - dyn DyNumber, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static auto GetTypeId() { return NScheme::NTypeIds::DyNumber; } - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct() - .AddMember("id").Int32(r.Id) - .AddMember("int").Int64(r.IntVal) - .AddMember("dyn"); - if (r.Dyn.has_value()) { - builder.BeginOptional().DyNumber(*r.Dyn).EndOptional(); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().DyNumber(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::DyNumber); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Dyn.has_value()) { - builder << *r.Dyn; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } + } - builder << '\n'; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + arrow::BinaryBuilder dynBuilder; + for (auto&& r : rows) { + if (r.TypedVal.has_value()) { + Y_ABORT_UNLESS(dynBuilder.Append(r.TypedVal->data(), r.TypedVal->size()).ok()); + } else { + Y_ABORT_UNLESS(dynBuilder.AppendNull().ok()); + } + } + std::shared_ptr<arrow::Array> arr; + Y_ABORT_UNLESS(dynBuilder.Finish(&arr).ok()); + return arr; } - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::binary(); } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - } + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); + }; - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); + COLUMN_TYPE_TEST_USING(TDyNumberTraits); - arrow::BinaryBuilder dynBuilder; - for (auto&& r : rows) { - if (r.Dyn.has_value()) { - Y_ABORT_UNLESS(dynBuilder.Append(r.Dyn->data(), r.Dyn->size()).ok()); - } else { - Y_ABORT_UNLESS(dynBuilder.AppendNull().ok()); - } + static TString MakeDyNumber(const TStringBuf& str) { + auto result = NDyNumber::ParseDyNumberString(str); + Y_ABORT_UNLESS(result.Defined(), "Failed to parse DyNumber: %s", TString(str).c_str()); + return TString(str); + } + + static void AppendDyNumber(arrow::BinaryBuilder& builder, const TStringBuf& str) { + Y_ABORT_UNLESS(builder.Append(str.data(), str.size()).ok()); + } + + void TDyNumberTraits::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + arrow::BinaryBuilder dynB; + arrow::Int64Builder valB; + AppendDyNumber(dynB, "-100"); Y_ABORT_UNLESS(valB.Append(1).ok()); + AppendDyNumber(dynB, "0"); Y_ABORT_UNLESS(valB.Append(2).ok()); + AppendDyNumber(dynB, "3.14"); Y_ABORT_UNLESS(valB.Append(3).ok()); + AppendDyNumber(dynB, "100"); Y_ABORT_UNLESS(valB.Append(4).ok()); + AppendDyNumber(dynB, "999"); Y_ABORT_UNLESS(valB.Append(5).ok()); + std::shared_ptr<arrow::Array> dynArr; + std::shared_ptr<arrow::Array> valArr; + Y_ABORT_UNLESS(dynB.Finish(&dynArr).ok()); + Y_ABORT_UNLESS(valB.Finish(&valArr).ok()); + auto aSchema = arrow::schema({ + arrow::field("dyn", arrow::binary(), false), + arrow::field("val", arrow::int64()) + }); + auto batch = arrow::RecordBatch::Make(aSchema, 5, { dynArr, valArr }); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("-100").AddMember("val").BeginOptional().Int64(1).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("0").AddMember("val").BeginOptional().Int64(2).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("3.14").AddMember("val").BeginOptional().Int64(3).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("100").AddMember("val").BeginOptional().Int64(4).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("999").AddMember("val").BeginOptional().Int64(5).EndOptional().EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "-.1e3,1\n.0,2\n.314e1,3\n.1e3,4\n.999e3,5\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); } + } - std::shared_ptr<arrow::Array> dynArr; - Y_ABORT_UNLESS(dynBuilder.Finish(&dynArr).ok()); + TScenario<TString> FilterEqualScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, MakeDyNumber("2") }, + { 3, 30, MakeDyNumber("3.14") }, + { 4, 40, MakeDyNumber("100") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"1\" AS DyNumber)", "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"3.14\" AS DyNumber) ORDER BY id", "[[1];[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"100\" AS DyNumber) ORDER BY id", "[[1];[2];[3]]" }, + }, + }; + } - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field("int", arrow::int64()), - arrow::field("dyn", arrow::binary()) - }); + TScenario<TString> FilterNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, std::nullopt }, + { 3, 30, MakeDyNumber("3") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("0") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn IS NOT NULL ORDER BY id", "[[1];[3];[5]]" }, + }, + }; + } - return arrow::RecordBatch::Make(schema, rows.size(), { idArr, intArr, dynArr }); + TScenario<TString> FilterCompareScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-10") }, + { 2, 20, MakeDyNumber("0") }, + { 3, 30, MakeDyNumber("5") }, + { 4, 40, MakeDyNumber("10") }, + { 5, 50, MakeDyNumber("100") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn < CAST(\"5\" AS DyNumber) ORDER BY id", "[[1];[2]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"5\" AS DyNumber) ORDER BY id", "[[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn <= CAST(\"5\" AS DyNumber) ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"10\" AS DyNumber) ORDER BY id", "[[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"-10\" AS DyNumber) ORDER BY id", "[[2];[3];[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"0\" AS DyNumber) AND dyn <= CAST(\"10\" AS DyNumber) ORDER BY id", "[[2];[3];[4]]" }, + }, + }; } - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + TScenario<TString> OrderByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("100") }, + { 2, 20, MakeDyNumber("-50") }, + { 3, 30, MakeDyNumber("0") }, + { 4, 40, MakeDyNumber("3.14") }, + { 5, 50, MakeDyNumber("-1") }, + }, + { + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn", "[[2;[\"-.5e2\"]];[5;[\"-.1e1\"]];[3;[\"0\"]];[4;[\".314e1\"]];[1;[\".1e3\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC", "[[1;[\".1e3\"]];[4;[\".314e1\"]];[3;[\"0\"]];[5;[\"-.1e1\"]];[2;[\"-.5e2\"]]]" }, + }, + }; } - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch(rows); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + TScenario<TString> GroupByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, MakeDyNumber("2") }, + { 3, 30, MakeDyNumber("1") }, + { 4, 40, MakeDyNumber("2") }, + { 5, 50, MakeDyNumber("3") }, + }, + { + { "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", + "[[[\".1e1\"];2u];[[\".2e1\"];2u];[[\".3e1\"];1u]]" }, + }, + }; + } - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + TScenario<TString> AggregationScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-5") }, + { 2, 20, MakeDyNumber("10") }, + { 3, 30, MakeDyNumber("99") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("0") }, + }, + { + { "SELECT min(dyn) FROM `" + tableName + "`", "[[[\"-.5e1\"]]]" }, + { "SELECT max(dyn) FROM `" + tableName + "`", "[[[\".99e2\"]]]" }, + { "SELECT count(dyn) FROM `" + tableName + "`", "[[4u]]" }, + { "SELECT count(*) FROM `" + tableName + "`", "[[5u]]" }, + }, + }; + } - break; - } - } + TJoinScenario<TString> JoinByDyNumberScenario() { + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + return { + t1, + t2, + { + { 1, 10, MakeDyNumber("10") }, + { 2, 20, MakeDyNumber("20") }, + { 3, 30, MakeDyNumber("30") }, + }, + { + { 10, 100, MakeDyNumber("10") }, + { 20, 200, MakeDyNumber("30") }, + { 30, 300, MakeDyNumber("50") }, + }, + { + { "SELECT t1.id, t2.id, t1.dyn FROM `" + t1 + "` AS t1 " + "JOIN `" + t2 + "` AS t2 ON t1.dyn = t2.dyn " + "ORDER BY t1.id, t2.id", + "[[1;10;[\".1e2\"]];[3;20;[\".3e2\"]]]" }, + }, + }; } - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } + TScenario<TString> OrderByWithLimitScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-100") }, + { 2, 20, MakeDyNumber("-1") }, + { 3, 30, MakeDyNumber("0") }, + { 4, 40, MakeDyNumber("5") }, + { 5, 50, MakeDyNumber("42") }, + { 6, 60, MakeDyNumber("999") }, + }, + { + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 3", "[[1;[\"-.1e3\"]];[2;[\"-.1e1\"]];[3;[\"0\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC LIMIT 2", "[[6;[\".999e3\"]];[5;[\".42e2\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 2 OFFSET 2", "[[3;[\"0\"]];[4;[\".5e1\"]]]" }, + }, + }; } - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; + TScenario<TString> GroupByWithNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("10") }, + { 2, 20, std::nullopt }, + { 3, 30, MakeDyNumber("10") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("20") }, + { 6, 60, std::nullopt }, + }, + { + { "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", + "[[#;3u];[[\".1e2\"];2u];[[\".2e2\"];1u]]" }, + { "SELECT count(dyn), count(*) FROM `" + tableName + "`", "[[3u;6u]]" }, + }, + }; + } - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\".1e3\" AS DyNumber)", "[[[4]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\"0\" AS DyNumber)", "[[[2]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[[3]]]" }, + }, + }; } } // namespace @@ -211,11 +306,11 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { const auto Load = Arg<2>(); const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); + TTestHelper helper(TDyNumberTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { { 1, 10, MakeDyNumber("3.14") }, { 2, 20, MakeDyNumber("-100") }, { 3, 30, std::nullopt }, @@ -238,186 +333,19 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { CheckOrExec(helper, "SELECT dyn FROM `" + tableName + "` WHERE id=6", "[[[\"-.5e1\"]]]", Scan); } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, MakeDyNumber("2") }, - { 3, 30, MakeDyNumber("3.14") }, - { 4, 40, MakeDyNumber("100") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", - "[[3]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"1\" AS DyNumber)", - "[[1]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"3.14\" AS DyNumber) ORDER BY id", - "[[1];[2];[4]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"100\" AS DyNumber) ORDER BY id", - "[[1];[2];[3]]", Scan); - } - - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, std::nullopt }, - { 3, 30, MakeDyNumber("3") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("0") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` WHERE dyn IS NULL ORDER BY id", - "[[2];[4]]", Scan); - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` WHERE dyn IS NOT NULL ORDER BY id", - "[[1];[3];[5]]", Scan); - } - - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-10") }, - { 2, 20, MakeDyNumber("0") }, - { 3, 30, MakeDyNumber("5") }, - { 4, 40, MakeDyNumber("10") }, - { 5, 50, MakeDyNumber("100") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn < CAST(\"5\" AS DyNumber) ORDER BY id", - "[[1];[2]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"5\" AS DyNumber) ORDER BY id", - "[[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn <= CAST(\"5\" AS DyNumber) ORDER BY id", - "[[1];[2];[3]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"10\" AS DyNumber) ORDER BY id", - "[[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"-10\" AS DyNumber) ORDER BY id", - "[[2];[3];[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"0\" AS DyNumber) AND dyn <= CAST(\"10\" AS DyNumber) ORDER BY id", - "[[2];[3];[4]]", Scan); - } - - Y_UNIT_TEST(TestOrderBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("100") }, - { 2, 20, MakeDyNumber("-50") }, - { 3, 30, MakeDyNumber("0") }, - { 4, 40, MakeDyNumber("3.14") }, - { 5, 50, MakeDyNumber("-1") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn", - "[[2;[\"-.5e2\"]];[5;[\"-.1e1\"]];[3;[\"0\"]];[4;[\".314e1\"]];[1;[\".1e3\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC", - "[[1;[\".1e3\"]];[4;[\".314e1\"]];[3;[\"0\"]];[5;[\"-.1e1\"]];[2;[\"-.5e2\"]]]", Scan); - } - - Y_UNIT_TEST(TestGroupBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, MakeDyNumber("2") }, - { 3, 30, MakeDyNumber("1") }, - { 4, 40, MakeDyNumber("2") }, - { 5, 50, MakeDyNumber("3") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", - "[[[\".1e1\"];2u];[[\".2e1\"];2u];[[\".3e1\"];1u]]", Scan); - } - - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-5") }, - { 2, 20, MakeDyNumber("10") }, - { 3, 30, MakeDyNumber("99") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("0") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT min(dyn) FROM `" + tableName + "`", "[[[\"-.5e1\"]]]", Scan); - CheckOrExec(helper, "SELECT max(dyn) FROM `" + tableName + "`", "[[[\".99e2\"]]]", Scan); - CheckOrExec(helper, "SELECT count(dyn) FROM `" + tableName + "`", "[[4u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `" + tableName + "`", "[[5u]]", Scan); - } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderBy, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupBy, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); + TTestHelper helper(TDyNumberTraits::CreateSettings()); const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; @@ -444,7 +372,7 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); + Base::CreateDataShardTable(helper, t1); { auto& session = helper.GetSession(); auto res = session @@ -461,7 +389,7 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { } } - LoadData(helper, Table, Load, t1, { + Base::LoadData(helper, Table, Load, t1, { { 1, 10, MakeDyNumber("3.14") }, { 2, 20, MakeDyNumber("100") } }, &col1, &s1); @@ -546,189 +474,23 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { "[[1;[\".314e1\"];[\".5e2\"]];[1;[\".314e1\"];[\".6e2\"]];[2;[\".1e3\"];[\".7e2\"]]]", Scan); } - Y_UNIT_TEST(TestJoinByDyNumber, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { - { 1, 10, MakeDyNumber("10") }, - { 2, 20, MakeDyNumber("20") }, - { 3, 30, MakeDyNumber("30") } - }, &col1, &s1); - - LoadData(helper, Table, Load, t2, { - { 10, 100, MakeDyNumber("10") }, - { 20, 200, MakeDyNumber("30") }, - { 30, 300, MakeDyNumber("50") } - }, &col2, &s2); - - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.dyn FROM `" + t1 + "` AS t1 " - "JOIN `" + t2 + "` AS t2 ON t1.dyn = t2.dyn " - "ORDER BY t1.id, t2.id", - "[[1;10;[\".1e2\"]];[3;20;[\".3e2\"]]]", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-100") }, - { 2, 20, MakeDyNumber("-1") }, - { 3, 30, MakeDyNumber("0") }, - { 4, 40, MakeDyNumber("5") }, - { 5, 50, MakeDyNumber("42") }, - { 6, 60, MakeDyNumber("999") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 3", - "[[1;[\"-.1e3\"]];[2;[\"-.1e1\"]];[3;[\"0\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC LIMIT 2", - "[[6;[\".999e3\"]];[5;[\".42e2\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 2 OFFSET 2", - "[[3;[\"0\"]];[4;[\".5e1\"]]]", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("10") }, - { 2, 20, std::nullopt }, - { 3, 30, MakeDyNumber("10") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("20") }, - { 6, 60, std::nullopt } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", - "[[#;3u];[[\".1e2\"];2u];[[\".2e2\"];1u]]", Scan); - - CheckOrExec(helper, - "SELECT count(dyn), count(*) FROM `" + tableName + "`", - "[[3u;6u]]", Scan); - } - - Y_UNIT_TEST(TestDyNumberAsPrimaryKey, EQueryMode, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Load = Arg<1>(); - - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber).SetNullable(false), - TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64) - }; - - TTestHelper::TColumnTable testTable; - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"dyn"}).SetSharding({"dyn"}).SetSchema(schema); - helper.CreateTable(testTable); - - if (Load == ELoadKind::ARROW) { - arrow::BinaryBuilder dynB; - arrow::Int64Builder valB; - AppendDyNumber(dynB, "-100"); Y_ABORT_UNLESS(valB.Append(1).ok()); - AppendDyNumber(dynB, "0"); Y_ABORT_UNLESS(valB.Append(2).ok()); - AppendDyNumber(dynB, "3.14"); Y_ABORT_UNLESS(valB.Append(3).ok()); - AppendDyNumber(dynB, "100"); Y_ABORT_UNLESS(valB.Append(4).ok()); - AppendDyNumber(dynB, "999"); Y_ABORT_UNLESS(valB.Append(5).ok()); - std::shared_ptr<arrow::Array> dynArr, valArr; - Y_ABORT_UNLESS(dynB.Finish(&dynArr).ok()); - Y_ABORT_UNLESS(valB.Finish(&valArr).ok()); - auto aSchema = arrow::schema({ - arrow::field("dyn", arrow::binary(), false), - arrow::field("val", arrow::int64()) - }); - - auto batch = arrow::RecordBatch::Make(aSchema, 5, { dynArr, valArr }); - helper.BulkUpsert(testTable, batch); - } else if (Load == ELoadKind::YDB_VALUE) { - TValueBuilder builder; - builder.BeginList(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("-100").AddMember("val").BeginOptional().Int64(1).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("0").AddMember("val").BeginOptional().Int64(2).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("3.14").AddMember("val").BeginOptional().Int64(3).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("100").AddMember("val").BeginOptional().Int64(4).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("999").AddMember("val").BeginOptional().Int64(5).EndOptional().EndStruct(); - builder.EndList(); - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } else { - TStringBuilder csv; - csv << "-.1e3,1\n.0,2\n.314e1,3\n.1e3,4\n.999e3,5\n"; - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", EDataFormat::CSV, csv).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } - - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\".1e3\" AS DyNumber)", "[[[4]]]", Scan); - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\"0\" AS DyNumber)", "[[[2]]]", Scan); - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[[3]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByDyNumber, JoinByDyNumberScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestDyNumberAsPrimaryKey, PkLookupScenario); Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithDyNumberSupport(); + auto runnerSettings = TDyNumberTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -746,19 +508,19 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { "(2, 200, CAST(\"20\" AS DyNumber))"); if (Load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch({ + auto batch = Base::MakeArrowBatch({ { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); helper.BulkUpsert(col, batch); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, cs, { + Base::BulkUpsertRowTableYdbValue(helper, cs, { { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); } else { - BulkUpsertRowTableCSV(helper, cs, { + Base::BulkUpsertRowTableCSV(helper, cs, { { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); diff --git a/ydb/core/kqp/ut/olap/types/interval_ut.cpp b/ydb/core/kqp/ut/olap/types/interval_ut.cpp index 47a3805aa1a..b3c1c279f8b 100644 --- a/ydb/core/kqp/ut/olap/types/interval_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/interval_ut.cpp @@ -1,22 +1,5 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" namespace NKikimr { namespace NKqp { @@ -26,163 +9,43 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<i64> Ival; - }; - - constexpr i64 MaxValidInterval = (i64)(86400000000ULL * 49673U - 1); - constexpr i64 MinValidInterval = -MaxValidInterval; - - TKikimrSettings CreateKikimrSettingsWithIntervalSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardInterval(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - ival Interval, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct() - .AddMember("id").Int32(r.Id) - .AddMember("int").Int64(r.IntVal) - .AddMember("ival"); - if (r.Ival.has_value()) { - builder.BeginOptional().Interval(*r.Ival).EndOptional(); + struct TIntervalTraits { + using TValue = i64; + static constexpr const char* ColumnName = "ival"; + static constexpr const char* SqlTypeName = "Interval"; + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardInterval(true); }); + } + static auto GetTypeId() { return NScheme::NTypeIds::Interval; } + static void AppendYdbValue(TValueBuilder& builder, const std::optional<i64>& val) { + if (val.has_value()) { + builder.BeginOptional().Interval(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::Interval); } - - builder.EndStruct(); } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Ival.has_value()) { - builder << *r.Ival; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<i64>& val) { + if (val.has_value()) { + builder << *val; } - - builder << '\n'; } - - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> ints; - std::vector<std::optional<int64_t>> ivals; - ids.reserve(rows.size()); - ints.reserve(rows.size()); - ivals.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - ints.push_back(r.IntVal); - ivals.push_back(r.Ival); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(ints); - auto ivalArr = MakeInt64ArrayNullable(ivals); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field("int", arrow::int64()), - arrow::field("ival", arrow::int64()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, ivalArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch(rows); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<i64>>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<std::optional<int64_t>> vals; + vals.reserve(rows.size()); + for (auto&& r : rows) { + vals.push_back(r.TypedVal); } + return MakeInt64ArrayNullable(vals); } - } - - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::int64(); } + }; - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("ival").SetType(NScheme::NTypeIds::Interval), - }; + COLUMN_TYPE_TEST_USING(TIntervalTraits); - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } + constexpr i64 MaxValidInterval = (i64)(86400000000ULL * 49673U - 1); + constexpr i64 MinValidInterval = -MaxValidInterval; void CreateIntervalPkDataShardTable(TTestHelper& helper, const TString& name) { auto& session = helper.GetSession(); @@ -253,232 +116,217 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); } } - } // namespace - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt }, { 4, 40, (i64)0 }, - { 5, 50, MaxValidInterval }, { 6, 60, MinValidInterval } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[1;[10];[1000000]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=2", "[[2;[20];[-500000]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=3", "[[3;[30];#]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=4", "[[4;[40];[0]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=5", TStringBuilder() << "[[5;[50];[" << MaxValidInterval << "]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=6", TStringBuilder() << "[[6;[60];[" << MinValidInterval << "]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY id", - TStringBuilder() << "[[1;[10];[1000000]];[2;[20];[-500000]];[3;[30];#];[4;[40];[0]];[5;[50];[" << MaxValidInterval - << "]];[6;[60];[" << MinValidInterval << "]]]", Scan); - - CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=1", "[[[1000000]]]", Scan); - CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=3", "[[#]]", Scan); - - LoadData(helper, Table, Load, tableName, { { 7, 0, std::nullopt } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=7", "[[7;[0];#]]", Scan); - } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterEqualScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)1000000 }, { 2, 20, (i64)2000000 }, { 3, 30, (i64)1000000 }, { 4, 40, std::nullopt }, { 5, 50, (i64)-500000 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(1000000 AS Interval) ORDER BY id", - "[[1;[10];[1000000]];[3;[30];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(2000000 AS Interval) ORDER BY id", - "[[2;[20];[2000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(-500000 AS Interval) ORDER BY id", - "[[5;[50];[-500000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(999 AS Interval)", "[]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival != CAST(1000000 AS Interval) ORDER BY id", - "[[2;[20];[2000000]];[5;[50];[-500000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival != CAST(-500000 AS Interval) ORDER BY id", - "[[1;[10];[1000000]];[2;[20];[2000000]];[3;[30];[1000000]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(1000000 AS Interval) ORDER BY id", "[[1;[10];[1000000]];[3;[30];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(2000000 AS Interval) ORDER BY id", "[[2;[20];[2000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(-500000 AS Interval) ORDER BY id", "[[5;[50];[-500000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(999 AS Interval)", "[]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival != CAST(1000000 AS Interval) ORDER BY id", "[[2;[20];[2000000]];[5;[50];[-500000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival != CAST(-500000 AS Interval) ORDER BY id", "[[1;[10];[1000000]];[2;[20];[2000000]];[3;[30];[1000000]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterNullsScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)100 }, { 2, 20, std::nullopt }, { 3, 30, (i64)300 }, { 4, 40, std::nullopt }, { 5, 50, (i64)0 }, { 6, 60, std::nullopt } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival IS NULL ORDER BY id", - "[[2;[20];#];[4;[40];#];[6;[60];#]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival IS NOT NULL ORDER BY id", - "[[1;[10];[100]];[3;[30];[300]];[5;[50];[0]]]", Scan); - - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NOT NULL", "[[3u]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival IS NULL ORDER BY id", "[[2;[20];#];[4;[40];#];[6;[60];#]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival IS NOT NULL ORDER BY id", "[[1;[10];[100]];[3;[30];[300]];[5;[50];[0]]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NOT NULL", "[[3u]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterCompareScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)-3000000 }, { 2, 20, (i64)-1000000 }, { 3, 30, (i64)0 }, { 4, 40, (i64)1000000 }, { 5, 50, (i64)5000000 }, { 6, 60, std::nullopt } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(0 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival > CAST(0 AS Interval) ORDER BY id", - "[[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival <= CAST(0 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(0 AS Interval) ORDER BY id", - "[[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(-1000000 AS Interval) AND ival <= CAST(1000000 AS Interval) ORDER BY id", - "[[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival > CAST(-1000000 AS Interval) AND ival < CAST(5000000 AS Interval) ORDER BY id", - "[[3;[30];[0]];[4;[40];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(-1000000 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(99999999 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(0 AS Interval) ORDER BY id", "[[1;[10];[-3000000]];[2;[20];[-1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival > CAST(0 AS Interval) ORDER BY id", "[[4;[40];[1000000]];[5;[50];[5000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival <= CAST(0 AS Interval) ORDER BY id", "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(0 AS Interval) ORDER BY id", "[[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(-1000000 AS Interval) AND ival <= CAST(1000000 AS Interval) ORDER BY id", + "[[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival > CAST(-1000000 AS Interval) AND ival < CAST(5000000 AS Interval) ORDER BY id", + "[[3;[30];[0]];[4;[40];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(-1000000 AS Interval) ORDER BY id", "[[1;[10];[-3000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(99999999 AS Interval) ORDER BY id", + "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]" }, + }, + }; } - Y_UNIT_TEST(TestOrderByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> OrderByScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)300 }, { 2, 20, (i64)-100 }, { 3, 30, (i64)200 }, { 4, 40, (i64)-200 }, { 5, 50, (i64)0 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival", - "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[1;[10];[300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC", - "[[1;[10];[300]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]", Scan); - - LoadData(helper, Table, Load, tableName, { { 6, 60, (i64)200 } }, &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival, id", - "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[6;[60];[200]];[1;[10];[300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC, id DESC", - "[[1;[10];[300]];[6;[60];[200]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival", "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[1;[10];[300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC", "[[1;[10];[300]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]" }, + }, + { + { + { { 6, 60, (i64)200 } }, + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival, id", + "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[6;[60];[200]];[1;[10];[300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC, id DESC", + "[[1;[10];[300]];[6;[60];[200]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]" }, + }, + }, + }, + }; } - Y_UNIT_TEST(TestGroupByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> GroupByScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, (i64)100 }, { 4, 40, (i64)200 }, { 5, 50, (i64)300 }, { 6, 60, (i64)100 }, { 7, 70, (i64)-100 } }, - &col, &schema); + { + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[[-100];1u];[[100];3u];[[200];2u];[[300];1u]]" }, + { "SELECT ival, count(*) AS cnt, min(id) AS min_id FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[[-100];1u;7];[[100];3u;1];[[200];2u;2];[[300];1u;5]]" }, + }, + }; + } - CheckOrExec(helper, "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[[-100];1u];[[100];3u];[[200];2u];[[300];1u]]", Scan); + TScenario<i64> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)-500 }, { 2, 20, (i64)100 }, { 3, 30, (i64)999 }, + { 4, 40, std::nullopt }, { 5, 50, (i64)0 } }, + { + { "SELECT min(ival) FROM `/Root/Table1`", "[[[-500]]]" }, + { "SELECT max(ival) FROM `/Root/Table1`", "[[[999]]]" }, + { "SELECT count(ival) FROM `/Root/Table1`", "[[4u]]" }, + { "SELECT count(*) FROM `/Root/Table1`", "[[5u]]" }, + { "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table1`", "[[[-500];[999];4u;5u]]" }, + }, + }; + } - CheckOrExec(helper, "SELECT ival, count(*) AS cnt, min(id) AS min_id FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[[-100];1u;7];[[100];3u;1];[[200];2u;2];[[300];1u;5]]", Scan); + TScenario<i64> AggregationAllNullScenario() { + return { + "/Root/Table2", + { { 1, 0, std::nullopt }, { 2, 0, std::nullopt } }, + { + { "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table2`", "[[#;#;0u;2u]]" }, + }, + }; } - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + TJoinScenario<i64> JoinByIntervalScenario() { + return { + "/Root/JoinTable1", + "/Root/JoinTable2", + { { 1, 10, (i64)1000 }, { 2, 20, (i64)2000 }, { 3, 30, (i64)3000 } }, + { { 10, 100, (i64)1000 }, { 20, 200, (i64)2000 }, { 30, 300, (i64)4000 } }, + { + { "SELECT t1.id, t2.id, t1.ival FROM `/Root/JoinTable1` AS t1 " + "JOIN `/Root/JoinTable2` AS t2 ON t1.ival = t2.ival " + "ORDER BY t1.id, t2.id", + "[[1;10;[1000]];[2;20;[2000]]]" }, + }, + }; + } + + TScenario<i64> OrderByWithLimitScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)500 }, { 2, 20, (i64)-200 }, { 3, 30, (i64)100 }, + { 4, 40, (i64)-300 }, { 5, 50, (i64)1000 }, { 6, 60, (i64)0 } }, + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 1", "[[4;[40];[-300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 1", "[[5;[50];[1000]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 3", "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 3", "[[5;[50];[1000]];[1;[10];[500]];[3;[30];[100]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 2 OFFSET 2", "[[6;[60];[0]];[3;[30];[100]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 100", + "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]];[3;[30];[100]];[1;[10];[500]];[5;[50];[1000]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival, id LIMIT 2", "[[4;[40];[-300]];[2;[20];[-200]]]" }, + }, + }; + } + + TScenario<i64> GroupByWithNullsScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, std::nullopt }, + { 4, 40, (i64)100 }, { 5, 50, std::nullopt }, { 6, 60, (i64)200 }, + { 7, 70, std::nullopt }, { 8, 80, (i64)-100 } }, + { + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[#;3u];[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT ival, count(ival) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[#;0u];[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` WHERE ival IS NOT NULL GROUP BY ival ORDER BY ival", + "[[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]" }, + { "SELECT min(ival), max(ival) FROM `/Root/Table1`", "[[[-100];[200]]]" }, + }, + }; + } + + } // namespace + + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)-500 }, { 2, 20, (i64)100 }, { 3, 30, (i64)999 }, - { 4, 40, std::nullopt }, { 5, 50, (i64)0 } }, + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, + { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt }, { 4, 40, (i64)0 }, + { 5, 50, MaxValidInterval }, { 6, 60, MinValidInterval } }, &col, &schema); - CheckOrExec(helper, "SELECT min(ival) FROM `/Root/Table1`", "[[[-500]]]", Scan); - CheckOrExec(helper, "SELECT max(ival) FROM `/Root/Table1`", "[[[999]]]", Scan); - CheckOrExec(helper, "SELECT count(ival) FROM `/Root/Table1`", "[[4u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1`", "[[5u]]", Scan); - CheckOrExec(helper, "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table1`", - "[[[-500];[999];4u;5u]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[1;[10];[1000000]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=2", "[[2;[20];[-500000]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=3", "[[3;[30];#]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=4", "[[4;[40];[0]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=5", TStringBuilder() << "[[5;[50];[" << MaxValidInterval << "]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=6", TStringBuilder() << "[[6;[60];[" << MinValidInterval << "]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY id", + TStringBuilder() << "[[1;[10];[1000000]];[2;[20];[-500000]];[3;[30];#];[4;[40];[0]];[5;[50];[" << MaxValidInterval + << "]];[6;[60];[" << MinValidInterval << "]]]", Scan); - { - const TString tableName2 = "/Root/Table2"; - TTestHelper::TColumnTable col2; - TVector<TTestHelper::TColumnSchema> schema2; - PrepareBase(helper, Table, tableName2, &col2, &schema2); - LoadData(helper, Table, Load, tableName2, - { { 1, 0, std::nullopt }, { 2, 0, std::nullopt } }, - &col2, &schema2); + CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=1", "[[[1000000]]]", Scan); + CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=3", "[[#]]", Scan); - CheckOrExec(helper, "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table2`", - "[[#;#;0u;2u]]", Scan); - } + Base::LoadData(helper, Table, Load, tableName, { { 7, 0, std::nullopt } }, &col, &schema); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=7", "[[7;[0];#]]", Scan); + } + + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByInterval, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByInterval, GroupByScenario); + + Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + RunScenario<TIntervalTraits>(AggregationScenario(), Arg<0>(), Arg<1>(), Arg<2>()); + RunScenario<TIntervalTraits>(AggregationAllNullScenario(), Arg<0>(), Arg<1>(), Arg<2>()); } Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { @@ -489,7 +337,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -511,7 +359,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); + Base::CreateDataShardTable(helper, t1); auto& session = helper.GetSession(); auto res = session .ExecuteSchemeQuery(TStringBuilder() << R"( @@ -526,7 +374,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); } - LoadData(helper, Table, Load, t1, + Base::LoadData(helper, Table, Load, t1, { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt } }, &col1, &s1); @@ -606,122 +454,9 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { "[[1;[1000000];[2000000]];[1;[1000000];[3000000]];[2;[-500000];[-100000]]]", Scan); } - Y_UNIT_TEST(TestJoinByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString t1 = "/Root/JoinTable1"; - const TString t2 = "/Root/JoinTable2"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("ival").SetType(NScheme::NTypeIds::Interval), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - - s2 = s1; - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, - { { 1, 10, (i64)1000 }, { 2, 20, (i64)2000 }, { 3, 30, (i64)3000 } }, - &col1, &s1); - - LoadData(helper, Table, Load, t2, - { { 10, 100, (i64)1000 }, { 20, 200, (i64)2000 }, { 30, 300, (i64)4000 } }, - &col2, &s2); - - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.ival FROM `/Root/JoinTable1` AS t1 " - "JOIN `/Root/JoinTable2` AS t2 ON t1.ival = t2.ival " - "ORDER BY t1.id, t2.id", - "[[1;10;[1000]];[2;20;[2000]]]", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)500 }, { 2, 20, (i64)-200 }, { 3, 30, (i64)100 }, - { 4, 40, (i64)-300 }, { 5, 50, (i64)1000 }, { 6, 60, (i64)0 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 1", - "[[4;[40];[-300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 1", - "[[5;[50];[1000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 3", - "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 3", - "[[5;[50];[1000]];[1;[10];[500]];[3;[30];[100]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 2 OFFSET 2", - "[[6;[60];[0]];[3;[30];[100]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 100", - "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]];[3;[30];[100]];[1;[10];[500]];[5;[50];[1000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival, id LIMIT 2", - "[[4;[40];[-300]];[2;[20];[-200]]]", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, std::nullopt }, - { 4, 40, (i64)100 }, { 5, 50, std::nullopt }, { 6, 60, (i64)200 }, - { 7, 70, std::nullopt }, { 8, 80, (i64)-100 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[#;3u];[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, "SELECT ival, count(ival) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[#;0u];[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, - "SELECT ival, count(*) AS cnt FROM `/Root/Table1` WHERE ival IS NOT NULL GROUP BY ival ORDER BY ival", - "[[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", - "[[3u]]", Scan); - - CheckOrExec(helper, "SELECT min(ival), max(ival) FROM `/Root/Table1`", - "[[[-100];[200]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByInterval, JoinByIntervalScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); Y_UNIT_TEST(TestIntervalAsPrimaryKey, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); @@ -729,7 +464,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; PrepareIntervalPkTable(helper, Table, tableName, &col, &schema); @@ -764,14 +499,14 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithIntervalSupport(); + auto runnerSettings = TIntervalTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -856,10 +591,10 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, "/Root/Table1", &col, &schema); + Base::PrepareBase(helper, Table, "/Root/Table1", &col, &schema); { TStringBuilder builder; diff --git a/ydb/core/kqp/ut/olap/types/json_ut.cpp b/ydb/core/kqp/ut/olap/types/json_ut.cpp index 50649a8d967..15210705b18 100644 --- a/ydb/core/kqp/ut/olap/types/json_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/json_ut.cpp @@ -1699,6 +1699,29 @@ Y_UNIT_TEST_SUITE(KqpOlapJsonNativeScalars) { Variator::ToExecutor(Variator::SingleScript(script)).Execute(); } + // A full cycle test for doubles near max magnitude. The stored value stays exact; reading back renders + // it with fewer digits (17 in, 16 out) on BinaryJson level. + Y_UNIT_TEST(CompactionNearMaxDouble) { + const TString script = TStringBuilder() << NativeTableSetup() << R"( + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (1u, JsonDocument('{"n" : 1.7976931348623157e308}')), + (2u, JsonDocument('{"n" : -1.7976931348623157e308}')) + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]]] + ------ + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (3u, JsonDocument('{"n" : 1.5}')) + ------ + ONE_COMPACTION + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]];[3u;["{\"n\":1.5}"]]] + ------ + )" << NativeValueTypeCheck(EValueType::Double); + Variator::ToExecutor(Variator::SingleScript(script)).Execute(); + } + Y_UNIT_TEST(CompactionDivergentTypesFallback) { const TString script = TStringBuilder() << NativeTableSetup() << R"( DATA: diff --git a/ydb/core/kqp/ut/olap/types/uuid_ut.cpp b/ydb/core/kqp/ut/olap/types/uuid_ut.cpp index ee14eac65e0..4aedb83e226 100644 --- a/ydb/core/kqp/ut/olap/types/uuid_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/uuid_ut.cpp @@ -1,25 +1,8 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" #include <yql/essentials/types/uuid/uuid.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> - namespace NKikimr { namespace NKqp { @@ -28,106 +11,6 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<TString> Uid; - }; - - TKikimrSettings CreateKikimrSettingsWithUuidSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardUuid(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - uid Uuid, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - )" << secondName << R"( Int64, - uid Uuid, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void BulkUpsertRowTableYdbValueWithColumnName( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(columnName).Int64(r.IntVal).AddMember("uid"); - if (r.Uid.has_value()) { - builder.BeginOptional().Uuid(TUuidValue(std::string(*r.Uid))).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Uuid); - } - - builder.EndStruct(); - } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, "int"); - } - - void BulkUpsertRowTableYdbValueWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(secondName).Int64(r.IntVal).AddMember("uid"); - if (r.Uid.has_value()) { - builder.BeginOptional().Uuid(TUuidValue(std::string(*r.Uid))).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Uuid); - } - - builder.EndStruct(); - } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Uid.has_value()) { - builder << *r.Uid; - } - - builder << '\n'; - } - - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } std::shared_ptr<arrow::Array> MakeUuidArrayNullable(const std::vector<std::optional<TString>>& uuidStrings) { auto type = arrow::fixed_size_binary(16); @@ -146,147 +29,46 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { return out; } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - std::vector<std::optional<TString>> uuids; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - uuids.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - uuids.push_back(r.Uid); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); - auto uuidArr = MakeUuidArrayNullable(uuids); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(columnName, arrow::int64()), - arrow::field("uid", arrow::fixed_size_binary(16)) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, uuidArr }); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - return MakeArrowBatchWithColumnName(rows, "int"); - } + struct TUuidTraits { + using TValue = TString; + static constexpr const char* ColumnName = "uid"; + static constexpr const char* SqlTypeName = "Uuid"; - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> seconds; - std::vector<std::optional<TString>> uuids; - ids.reserve(rows.size()); - seconds.reserve(rows.size()); - uuids.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - seconds.push_back(r.IntVal); - uuids.push_back(r.Uid); + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardUuid(true); }); } - auto idArr = MakeInt32Array(ids); - auto secondArr = MakeInt64Array(seconds); - auto uuidArr = MakeUuidArrayNullable(uuids); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(secondName, arrow::int64()), - arrow::field("uid", arrow::fixed_size_binary(16)) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, secondArr, uuidArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableArrowWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } + static auto GetTypeId() { return NScheme::NTypeIds::Uuid; } - auto batch = MakeArrowBatchWithColumnName(rows, columnName); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().Uuid(TUuidValue(std::string(*val))).EndOptional(); + } else { + builder.EmptyOptional(EPrimitiveType::Uuid); } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + } - break; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } } - } - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + std::vector<std::optional<TString>> uuids; + uuids.reserve(rows.size()); + for (auto&& r : rows) { + uuids.push_back(r.TypedVal); + } + return MakeUuidArrayNullable(uuids); } - } - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::fixed_size_binary(16); } - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); + }; + + COLUMN_TYPE_TEST_USING(TUuidTraits); static const TString UUID1 = "550e8400-e29b-41d4-a716-446655440000"; static const TString UUID2 = "550e8400-e29b-41d4-a716-446655440001"; @@ -300,173 +82,210 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { static const TString UUID_D = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; static const TString UUID_E = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; -} // namespace - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, UUID1 }, { 2, 20, UUID4 }, { 3, 30, std::nullopt } }, &col, &schema); - - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=1", "[[1;[10]]]", Scan); - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=3", "[[3;[30]]]", Scan); - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` ORDER BY id", - "[[1;[10]];[2;[20]];[3;[30]]]", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` WHERE id=1", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1]])", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` WHERE id=3", - R"([[#;3]])", Scan); + void TUuidTraits::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + std::vector<std::optional<TString>> uuids = { UUID_A, UUID_B, UUID_C }; + auto uuidArr = MakeUuidArrayNullable(uuids); + using namespace NKikimr::NKqp::NTestArrow; + auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); + auto batch = MakeBatch( + { arrow::field("uid", arrow::fixed_size_binary(16), /*nullable*/ false), + arrow::field("val", arrow::int64()) }, + { uuidArr, valArr } + ); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_A))).AddMember("val").Int64(1).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_B))).AddMember("val").Int64(2).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_C))).AddMember("val").Int64(3).EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "11111111-1111-1111-1111-111111111111,1\n"; + csv << "22222222-2222-2222-2222-222222222222,2\n"; + csv << "33333333-3333-3333-3333-333333333333,3\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID1 }, { 2, 0, UUID4 }, { 3, 0, UUID5 } }, &col, &schema); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid))", - "[[1]]", Scan); - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid != CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid) ORDER BY id)", - "[[2];[3]]", Scan); - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("660e8400-e29b-41d4-a716-446655440000" AS Uuid))", - "[[2]]", Scan); + TScenario<TString> FilterEqualScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID1 }, { 2, 0, UUID4 }, { 3, 0, UUID5 } }, + { + { R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid))", "[[1]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid != CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid) ORDER BY id)", "[[2];[3]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("660e8400-e29b-41d4-a716-446655440000" AS Uuid))", "[[2]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID1 }, { 2, 0, std::nullopt }, { 3, 0, UUID5 }, { 4, 0, std::nullopt } }, &col, &schema); - - CheckOrExec(helper, "SELECT id FROM `/Root/Table1` WHERE uid IS NULL ORDER BY id", - "[[2];[4]]", Scan); - CheckOrExec(helper, "SELECT id FROM `/Root/Table1` WHERE uid IS NOT NULL ORDER BY id", - "[[1];[3]]", Scan); + TScenario<TString> FilterNullsScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID1 }, { 2, 0, std::nullopt }, { 3, 0, UUID5 }, { 4, 0, std::nullopt } }, + { + { "SELECT id FROM `/Root/Table1` WHERE uid IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `/Root/Table1` WHERE uid IS NOT NULL ORDER BY id", "[[1];[3]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_A }, { 2, 0, UUID_B }, { 3, 0, UUID_C }, { 4, 0, UUID_D } }, &col, &schema); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid < CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", - "[[1];[2]]", Scan); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid > CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", - "[[3];[4]]", Scan); + TScenario<TString> FilterCompareScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_A }, { 2, 0, UUID_B }, { 3, 0, UUID_C }, { 4, 0, UUID_D } }, + { + { R"(SELECT id FROM `/Root/Table1` WHERE uid < CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", "[[1];[2]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid > CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", "[[3];[4]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid <= CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", "[[1];[2]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid >= CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", "[[3];[4]]" }, + }, + }; + } - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid <= CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", - "[[1];[2]]", Scan); + TScenario<TString> OrderByScenario() { + return { + "/Root/Table1", + { { 1, 10, UUID_C }, { 2, 20, UUID_A }, { 3, 30, UUID_B } }, + { + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC", + R"([[["33333333-3333-3333-3333-333333333333"];1];[["22222222-2222-2222-2222-222222222222"];3];[["11111111-1111-1111-1111-111111111111"];2]])" }, + }, + }; + } - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid >= CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", - "[[3];[4]]", Scan); + TScenario<TString> GroupByScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_D }, { 2, 0, UUID_E }, { 3, 0, UUID_D }, { 4, 0, UUID_E }, { 5, 0, UUID1 } }, + { + { "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];2u]])" }, + }, + }; } - Y_UNIT_TEST(TestOrderBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); + TScenario<TString> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_A }, { 2, 0, UUID_C }, { 3, 0, UUID_B }, { 4, 0, std::nullopt } }, + { + { "SELECT min(uid) FROM `/Root/Table1`", R"([[["11111111-1111-1111-1111-111111111111"]]])" }, + { "SELECT max(uid) FROM `/Root/Table1`", R"([[["33333333-3333-3333-3333-333333333333"]]])" }, + { "SELECT count(uid) FROM `/Root/Table1`", "[[3u]]" }, + { "SELECT count(*) FROM `/Root/Table1`", "[[4u]]" }, + }, + }; + } - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, UUID_C }, { 2, 20, UUID_A }, { 3, 30, UUID_B } }, &col, &schema); + TJoinScenario<TString> JoinByUuidScenario() { + return { + "/Root/Table1", + "/Root/Table2", + { { 1, 10, UUID_D }, { 2, 20, UUID_E } }, + { { 1, 100, UUID_E }, { 2, 200, UUID_D }, { 3, 300, UUID_A } }, + { + { R"(SELECT t1.id, t2.id, t1.uid FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.uid = t2.uid ORDER BY t1.id, t2.id)", + R"([[1;2;["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[2;1;["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]])" }, + }, + }; + } - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])", Scan); + TScenario<TString> OrderByWithLimitScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_C }, { 2, 0, UUID_A }, { 3, 0, UUID_B }, { 4, 0, UUID_D } }, + { + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 2", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC LIMIT 1", + R"([[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];4]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 3", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])" }, + }, + }; + } - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC", - R"([[["33333333-3333-3333-3333-333333333333"];1];[["22222222-2222-2222-2222-222222222222"];3];[["11111111-1111-1111-1111-111111111111"];2]])", Scan); + TScenario<TString> GroupByWithNullsScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_D }, { 2, 0, std::nullopt }, { 3, 0, UUID_D }, { 4, 0, std::nullopt }, { 5, 0, UUID_E } }, + { + { "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", + R"([[#;2u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];1u]])" }, + }, + }; } - Y_UNIT_TEST(TestGroupBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("22222222-2222-2222-2222-222222222222" AS Uuid))", "[[[2]]]" }, + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("11111111-1111-1111-1111-111111111111" AS Uuid))", "[[[1]]]" }, + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("33333333-3333-3333-3333-333333333333" AS Uuid))", "[[[3]]]" }, + }, + }; + } + TCsvScenario CsvScenario() { const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_D }, { 2, 0, UUID_E }, { 3, 0, UUID_D }, { 4, 0, UUID_E }, { 5, 0, UUID1 } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];2u]])", Scan); + return { + tableName, + "1,0,550e8400-e29b-41d4-a716-446655440000\n" + "2,0,660e8400-e29b-41d4-a716-446655440000\n" + "3,0,770e8400-e29b-41d4-a716-446655440000\n", + { + { "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT uid, id FROM `" + tableName + "` ORDER BY id", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1];[["660e8400-e29b-41d4-a716-446655440000"];2];[["770e8400-e29b-41d4-a716-446655440000"];3]])" }, + }, + }; } - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { +} // namespace + + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); + TTestHelper helper(TUuidTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_A }, { 2, 0, UUID_C }, { 3, 0, UUID_B }, { 4, 0, std::nullopt } }, &col, &schema); + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, + { { 1, 10, UUID1 }, { 2, 20, UUID4 }, { 3, 30, std::nullopt } }, &col, &schema); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=1", "[[1;[10]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=3", "[[3;[30]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` ORDER BY id", + "[[1;[10]];[2;[20]];[3;[30]]]", Scan); CheckOrExec(helper, - "SELECT min(uid) FROM `/Root/Table1`", - R"([[["11111111-1111-1111-1111-111111111111"]]])", Scan); - CheckOrExec(helper, - "SELECT max(uid) FROM `/Root/Table1`", - R"([[["33333333-3333-3333-3333-333333333333"]]])", Scan); - CheckOrExec(helper, - "SELECT count(uid) FROM `/Root/Table1`", - "[[3u]]", Scan); + "SELECT uid, id FROM `/Root/Table1` WHERE id=1", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1]])", Scan); CheckOrExec(helper, - "SELECT count(*) FROM `/Root/Table1`", - "[[4u]]", Scan); + "SELECT uid, id FROM `/Root/Table1` WHERE id=3", + R"([[#;3]])", Scan); } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderBy, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupBy, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); @@ -474,7 +293,7 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); + TTestHelper helper(TUuidTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -495,19 +314,19 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); - CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); } - LoadData(helper, Table, Load, t1, { { 1, 100, UUID1 }, { 2, 200, UUID4 } }, &col1, &s1); + Base::LoadData(helper, Table, Load, t1, { { 1, 100, UUID1 }, { 2, 200, UUID4 } }, &col1, &s1); if (Table == ETableKind::COLUMNSHARD) { - LoadData(helper, Table, Load, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, &col2, &s2); + Base::LoadData(helper, Table, Load, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, &col2, &s2); } else { if (Load == ELoadKind::ARROW) { - BulkUpsertRowTableArrowWithSecondColumn( + Base::BulkUpsertRowTableArrowWithSecondColumn( helper, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, "table1_id"); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValueWithSecondColumn( + Base::BulkUpsertRowTableYdbValueWithSecondColumn( helper, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, "table1_id"); } else { TStringBuilder csv; @@ -524,150 +343,16 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { R"([[1;["550e8400-e29b-41d4-a716-446655440000"];["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[1;["550e8400-e29b-41d4-a716-446655440000"];["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]];[2;["660e8400-e29b-41d4-a716-446655440000"];["11111111-1111-1111-1111-111111111111"]]])", Scan); } - Y_UNIT_TEST(TestJoinByUuid, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { { 1, 10, UUID_D }, { 2, 20, UUID_E } }, &col1, &s1); - LoadData(helper, Table, Load, t2, { { 1, 100, UUID_E }, { 2, 200, UUID_D }, { 3, 300, UUID_A } }, &col2, &s2); - - CheckOrExec(helper, - R"(SELECT t1.id, t2.id, t1.uid FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.uid = t2.uid ORDER BY t1.id, t2.id)", - R"([[1;2;["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[2;1;["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]])", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_C }, { 2, 0, UUID_A }, { 3, 0, UUID_B }, { 4, 0, UUID_D } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 2", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3]])", Scan); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC LIMIT 1", - R"([[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];4]])", Scan); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 3", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_D }, { 2, 0, std::nullopt }, { 3, 0, UUID_D }, { 4, 0, std::nullopt }, { 5, 0, UUID_E } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", - R"([[#;2u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];1u]])", Scan); - } - - Y_UNIT_TEST(TestUuidAsPrimaryKey, EQueryMode, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Load = Arg<1>(); - - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid).SetNullable(false), - TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64) - }; - - TTestHelper::TColumnTable testTable; - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"uid"}).SetSharding({"uid"}).SetSchema(schema); - helper.CreateTable(testTable); - - if (Load == ELoadKind::ARROW) { - std::vector<std::optional<TString>> uuids = { UUID_A, UUID_B, UUID_C }; - auto uuidArr = MakeUuidArrayNullable(uuids); - using namespace NKikimr::NKqp::NTestArrow; - auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); - auto batch = MakeBatch( - { arrow::field("uid", arrow::fixed_size_binary(16), /*nullable*/ false), - arrow::field("val", arrow::int64()) }, - { uuidArr, valArr } - ); - helper.BulkUpsert(testTable, batch); - } else if (Load == ELoadKind::YDB_VALUE) { - TValueBuilder builder; - builder.BeginList(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_A))).AddMember("val").Int64(1).EndStruct(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_B))).AddMember("val").Int64(2).EndStruct(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_C))).AddMember("val").Int64(3).EndStruct(); - builder.EndList(); - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } else { - TStringBuilder csv; - csv << "11111111-1111-1111-1111-111111111111,1" << Endl; - csv << "22222222-2222-2222-2222-222222222222,2" << Endl; - csv << "33333333-3333-3333-3333-333333333333,3" << Endl; - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", EDataFormat::CSV, csv).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } - - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("22222222-2222-2222-2222-222222222222" AS Uuid))", - "[[[2]]]", Scan); - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("11111111-1111-1111-1111-111111111111" AS Uuid))", - "[[[1]]]", Scan); - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("33333333-3333-3333-3333-333333333333" AS Uuid))", - "[[[3]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByUuid, JoinByUuidScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestUuidAsPrimaryKey, PkLookupScenario); Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithUuidSupport(); + auto runnerSettings = TUuidTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); @@ -753,31 +438,7 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { R"([[["550e8400-e29b-41d4-a716-446655440000"];1;[10]];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2;[20]];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];4;[40]]])", Scan); } - Y_UNIT_TEST(TestCsv, EQueryMode, ETableKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - - { - TStringBuilder builder; - builder << "1,0,550e8400-e29b-41d4-a716-446655440000" << Endl; - builder << "2,0,660e8400-e29b-41d4-a716-446655440000" << Endl; - builder << "3,0,770e8400-e29b-41d4-a716-446655440000" << Endl; - const auto result = helper.GetKikimr().GetTableClient().BulkUpsert( - tableName, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `" + tableName + "` ORDER BY id", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1];[["660e8400-e29b-41d4-a716-446655440000"];2];[["770e8400-e29b-41d4-a716-446655440000"];3]])", Scan); - } + Y_UNIT_TEST_CSV_SCENARIO(TestCsv, CsvScenario); } } // namespace NKqp diff --git a/ydb/core/kqp/ut/olap/types/ya.make b/ydb/core/kqp/ut/olap/types/ya.make index 1be271a945d..6f7d9bf1253 100644 --- a/ydb/core/kqp/ut/olap/types/ya.make +++ b/ydb/core/kqp/ut/olap/types/ya.make @@ -39,6 +39,6 @@ PEERDIR( YQL_LAST_ABI_VERSION() -GENERATE_ENUM_SERIALIZATION(bool_test_enums.h) +GENERATE_ENUM_SERIALIZATION(column_type_test_enums.h) END() diff --git a/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp b/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp index c79b2b2493c..7c1282ef993 100644 --- a/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_returning_ut.cpp @@ -1806,6 +1806,120 @@ Y_UNIT_TEST_TWIN(UpdateReturningThenSelectInSameTx, EnableIndexStreamWrite) { } } +Y_UNIT_TEST(ReturningWithSmallChannelBuffer) { + auto appConfig = GetAppConfig(false); + appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1_KB); + appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(1_KB); + + auto serverSettings = TKikimrSettings(appConfig); + TKikimrRunner kikimr(serverSettings); + + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + auto settings = NYdb::NQuery::TExecuteQuerySettings() + .Syntax(NYdb::NQuery::ESyntax::YqlV1) + .ConcurrentResultSets(false); + + { + auto result = session.ExecuteQuery(Q_(R"( + --!syntax_v1 + CREATE TABLE test_returning_bp ( + key Int32, + value String, + PRIMARY KEY(key) + ); + )"), NYdb::NQuery::TTxControl::NoTx(), settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + const ui32 rowCount = 200; + const TString payload(500, 'x'); + + { + // Build a multi-row UPSERT ... RETURNING * query large enough to + // push TransformOutput past SoftLimit/HardLimit with 1KB channel buffer. + // Use DQ channel v1 so that ChannelBufferSize directly controls the + // output channel's MaxStoredBytes (v2 local buffers use a separate 8MB limit). + TStringBuilder query; + query << "pragma ydb.DqChannelVersion = \"1\";\n"; + query << "UPSERT INTO test_returning_bp (key, value) VALUES "; + for (ui32 i = 0; i < rowCount; ++i) { + if (i > 0) query << ", "; + query << "(" << i << ", \"" << payload << "\")"; + } + query << " RETURNING key, value;"; + + auto it = session.StreamExecuteQuery( + query, + NYdb::NQuery::TTxControl::BeginTx().CommitTx(), + settings + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_VALUES_EQUAL_C(res.RowsCount, rowCount, + "Expected " << rowCount << " rows in RETURNING result, got " << res.RowsCount); + } + + // Verify data was actually written + { + auto result = session.ExecuteQuery(Q_(R"( + --!syntax_v1 + SELECT COUNT(*) FROM test_returning_bp; + )"), NYdb::NQuery::TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([[200u]])", FormatResultSetYson(result.GetResultSet(0))); + } +} + +Y_UNIT_TEST(ReturningIndexedWithSmallChannelBuffer) { + // Regression test for TransformOutput backpressure with indexed writes. + // Indexed UPSERT ... RETURNING goes through TKqpForwardWriteActor (output transform) + // and must handle downstream backpressure without crashing. + auto appConfig = GetAppConfig(false); + appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1_KB); + appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(1_KB); + + auto kikimr = DefaultKikimrRunner({}, appConfig); + + { + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session, true); + } + + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + auto settings = NYdb::NQuery::TExecuteQuerySettings() + .Syntax(NYdb::NQuery::ESyntax::YqlV1) + .ConcurrentResultSets(false); + + const ui32 rowCount = 200; + const TString payload(500, 'x'); + + { + TStringBuilder query; + query << "pragma ydb.DqChannelVersion = \"1\";\n"; + query << "UPSERT INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES "; + for (ui32 i = 0; i < rowCount; ++i) { + if (i > 0) query << ", "; + query << "(" << (1000 + i) << ", " << (1000 + i) << ", \"" << payload << "\")"; + } + query << " RETURNING Key, Fk, Value;"; + + auto it = session.StreamExecuteQuery( + query, + NYdb::NQuery::TTxControl::BeginTx().CommitTx(), + settings + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_VALUES_EQUAL_C(res.RowsCount, rowCount, + "Expected " << rowCount << " rows in RETURNING result, got " << res.RowsCount); + } +} + } } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp index 84a7ba07175..8fc3e67375b 100644 --- a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp +++ b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp @@ -2021,9 +2021,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Execute)) .ExtractValueSync(); @@ -2072,10 +2070,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); auto params = paramsVector[i]; // clang-format off diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 70352f4f6d9..31681ea506a 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/core/base/tablet_resolver.h> #include <ydb/core/formats/arrow/arrow_helpers.h> #include <ydb/core/kqp/gateway/actors/scheme.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h> #include <ydb/core/kqp/gateway/kqp_gateway.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/kqp/ut/common/columnshard.h> #include <ydb/core/kqp/ut/common/olap_indexes_enums.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <ydb/core/protos/schemeshard/operations.pb.h> #include <ydb/core/testlib/cs_helper.h> #include <ydb/core/testlib/common_helper.h> @@ -8552,7 +8552,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(DisableExternalDataSourcesOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .EnableExternalDataSourcesOnServerless(false) .Create(); @@ -8589,21 +8589,21 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& dropTableSql = "DROP EXTERNAL TABLE MyExternalTable;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId(""); + auto settings = NWorkloadManager::TQueryRunnerSettings().PoolId(""); // Dedicated, enabled settings.Database(ydb->GetSettings().GetDedicatedTenantName()).NodeIndex(ydb->GetDedicatedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSourceSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createTableSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropTableSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSourceSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSourceSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createTableSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropTableSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSourceSql, settings)); // Shared, enabled settings.Database(ydb->GetSettings().GetSharedTenantName()).NodeIndex(ydb->GetSharedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSourceSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createTableSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropTableSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSourceSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSourceSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createTableSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropTableSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSourceSql, settings)); // Serverless, disabled settings.Database(ydb->GetSettings().GetServerlessTenantName()).NodeIndex(ydb->GetServerlessTenantInfo().NodeIdx); @@ -12427,7 +12427,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(DisableResourcePoolsOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .EnableResourcePoolsOnServerless(false) .Create(); @@ -12457,19 +12457,19 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& dropSql = "DROP RESOURCE POOL MyResourcePool;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId(""); + auto settings = NWorkloadManager::TQueryRunnerSettings().PoolId(""); // Dedicated, enabled settings.Database(ydb->GetSettings().GetDedicatedTenantName()).NodeIndex(ydb->GetDedicatedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Shared, enabled settings.Database(ydb->GetSettings().GetSharedTenantName()).NodeIndex(ydb->GetSharedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Serverless, disabled settings.Database(ydb->GetSettings().GetServerlessTenantName()).NodeIndex(ydb->GetServerlessTenantInfo().NodeIdx); @@ -12756,7 +12756,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(DisableResourcePoolClassifiersOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .EnableResourcePoolsOnServerless(false) .Create(); @@ -12785,23 +12785,23 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& dropSql = "DROP RESOURCE POOL CLASSIFIER MyResourcePoolClassifier;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId(""); + auto settings = NWorkloadManager::TQueryRunnerSettings().PoolId(""); // Dedicated, enabled settings.Database(ydb->GetSettings().GetDedicatedTenantName()).NodeIndex(ydb->GetDedicatedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test_pool WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test_pool WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Shared, enabled settings.Database(ydb->GetSettings().GetSharedTenantName()).NodeIndex(ydb->GetSharedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test_pool WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test_pool WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery("CREATE RESOURCE POOL test WITH (CONCURRENT_QUERY_LIMIT=10);", settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Serverless, disabled settings.Database(ydb->GetSettings().GetServerlessTenantName()).NodeIndex(ydb->GetServerlessTenantInfo().NodeIdx); @@ -12945,11 +12945,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TString FetchResourcePoolClassifiers(TTestActorRuntime& runtime, ui32 nodeIndex) { const TActorId edgeActor = runtime.AllocateEdgeActor(nodeIndex); - runtime.Send(NMetadata::NProvider::MakeServiceId(runtime.GetNodeId(nodeIndex)), edgeActor, new NMetadata::NProvider::TEvAskSnapshot(std::make_shared<TResourcePoolClassifierSnapshotsFetcher>()), nodeIndex); + runtime.Send(NMetadata::NProvider::MakeServiceId(runtime.GetNodeId(nodeIndex)), edgeActor, new NMetadata::NProvider::TEvAskSnapshot(std::make_shared<NWorkloadManager::TResourcePoolClassifierSnapshotsFetcher>()), nodeIndex); const auto response = runtime.GrabEdgeEvent<NMetadata::NProvider::TEvRefreshSubscriberData>(edgeActor); UNIT_ASSERT(response); - return response->Get()->GetSnapshotAs<TResourcePoolClassifierSnapshot>()->SerializeToString(); + return response->Get()->GetSnapshotAs<NWorkloadManager::TResourcePoolClassifierSnapshot>()->SerializeToString(); } TString FetchResourcePoolClassifiers(TKikimrRunner& kikimr) { @@ -12994,7 +12994,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(CreateResourcePoolClassifierOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .EnableResourcePoolsOnServerless(true) .Create(); @@ -13003,7 +13003,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& serverlessTenant = ydb->GetSettings().GetServerlessTenantName(); ydb->ExecuteQueryRetry("Wait EnableResourcePools on Serverless", R"( CREATE RESOURCE POOL test_pool WITH (CONCURRENT_QUERY_LIMIT=10);)", - NWorkload::TQueryRunnerSettings() + NWorkloadManager::TQueryRunnerSettings() .PoolId("") .Database(serverlessTenant) .NodeIndex(nodeIdx) @@ -13013,7 +13013,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { RANK=20, RESOURCE_POOL="test_pool" );)", - NWorkload::TQueryRunnerSettings() + NWorkloadManager::TQueryRunnerSettings() .PoolId("") .Database(serverlessTenant) .NodeIndex(nodeIdx) @@ -13201,7 +13201,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } Y_UNIT_TEST(DisableMetadataObjectsOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .EnableMetadataObjectsOnServerless(false) .Create(); @@ -13216,28 +13216,28 @@ Y_UNIT_TEST_SUITE(KqpScheme) { const auto& upsertSql = "UPSERT OBJECT MySecretObject (TYPE SECRET) WITH value = \"edcba\";"; const auto& dropSql = "DROP OBJECT MySecretObject (TYPE SECRET);"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId(""); + auto settings = NWorkloadManager::TQueryRunnerSettings().PoolId(""); // Dedicated, enabled settings.Database(ydb->GetSettings().GetDedicatedTenantName()).NodeIndex(ydb->GetDedicatedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Shared, enabled settings.Database(ydb->GetSettings().GetSharedTenantName()).NodeIndex(ydb->GetSharedTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); // Serverless, disabled settings.Database(ydb->GetSettings().GetServerlessTenantName()).NodeIndex(ydb->GetServerlessTenantInfo().NodeIdx); checkDisabled(ydb->ExecuteQuery(createSql, settings)); checkDisabled(ydb->ExecuteQuery(alterSql, settings)); checkDisabled(ydb->ExecuteQuery(upsertSql, settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings)); } Y_UNIT_TEST(CreateBackupCollectionDisabledByDefault) { @@ -14459,17 +14459,17 @@ END DO)", } Y_UNIT_TEST(StreamingQueriesOnServerless) { - auto ydb = NWorkload::TYdbSetupSettings() + auto ydb = NWorkloadManager::TYdbSetupSettings() .CreateSampleTenants(true) .Create(); const auto& tenantName = ydb->GetSettings().GetServerlessTenantName(); - const auto settings = NWorkload::TQueryRunnerSettings() + const auto settings = NWorkloadManager::TQueryRunnerSettings() .PoolId("") .Database(tenantName) .NodeIndex(ydb->GetServerlessTenantInfo().NodeIdx); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(fmt::format(R"( + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(fmt::format(R"( CREATE TOPIC MyTopic; CREATE EXTERNAL DATA SOURCE MySource WITH ( SOURCE_TYPE = "Ydb", @@ -14482,7 +14482,7 @@ END DO)", "database"_a = tenantName ), settings)); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( CREATE STREAMING QUERY MyStreamingQuery WITH ( RUN = TRUE ) AS DO BEGIN INSERT INTO MySource.MyTopic SELECT * FROM MySource.MyTopic END DO @@ -14500,7 +14500,7 @@ END DO)", const auto& result = ydb->ExecuteQuery( TStringBuilder() << "SELECT * FROM `.sys/streaming_queries` " << filter , settings); - NWorkload::TSampleQueries::CheckSuccess(result); + NWorkloadManager::TSampleQueries::CheckSuccess(result); UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 1); NYdb::TResultSetParser resultParser(result.ResultSets[0]); @@ -14534,7 +14534,7 @@ END DO)", checkSysView(queryText, false, TStringBuilder() << "WHERE Path > '" << queryName << "'"); checkSysView(queryText, false, TStringBuilder() << "WHERE Path < '" << queryName << "'"); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( ALTER STREAMING QUERY MyStreamingQuery SET ( FORCE = TRUE ) AS DO BEGIN INSERT INTO MySource.MyTopic SELECT /* hint */ * FROM MySource.MyTopic END DO @@ -14550,7 +14550,7 @@ END DO)", Sleep(TDuration::Seconds(2)); checkSysView(queryText); - NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( + NWorkloadManager::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(R"( DROP STREAMING QUERY MyStreamingQuery )", settings)); diff --git a/ydb/core/kqp/ut/scheme/ya.make b/ydb/core/kqp/ut/scheme/ya.make index 7974f7c2e1a..d9464d45057 100644 --- a/ydb/core/kqp/ut/scheme/ya.make +++ b/ydb/core/kqp/ut/scheme/ya.make @@ -25,7 +25,7 @@ PEERDIR( library/cpp/threading/local_executor ydb/core/kqp ydb/core/kqp/ut/common - ydb/core/kqp/workload_service/ut/common + ydb/services/workload_manager/ut/common ydb/core/tx/columnshard/hooks/testing ydb/public/sdk/cpp/src/client/arrow ydb/public/sdk/cpp/src/client/draft diff --git a/ydb/core/kqp/ut/tx/kqp_tx_commit_timestamp_cdc_ut.cpp b/ydb/core/kqp/ut/tx/kqp_tx_commit_timestamp_cdc_ut.cpp new file mode 100644 index 00000000000..84a59bb6b0b --- /dev/null +++ b/ydb/core/kqp/ut/tx/kqp_tx_commit_timestamp_cdc_ut.cpp @@ -0,0 +1,226 @@ +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NQuery; +using namespace NYdb::NTopic; + +namespace { + +TVector<NJson::TJsonValue> ReadCdcMessages(TKikimrRunner& kikimr, const TString& topicPath, size_t expectedCount) { + TTopicClient topicClient(kikimr.GetDriver()); + + TReadSessionSettings readSettings; + readSettings.ConsumerName("test_consumer"); + TTopicReadSettings topicReadSettings; + topicReadSettings.Path(topicPath); + readSettings.AppendTopics(topicReadSettings); + + auto readSession = topicClient.CreateReadSession(readSettings); + + TVector<NJson::TJsonValue> result; + const TInstant deadline = TInstant::Now() + TDuration::Seconds(30); + + while (result.size() < expectedCount && TInstant::Now() < deadline) { + const TDuration remain = deadline - TInstant::Now(); + if (!readSession->WaitEvent().Wait(remain)) { + break; + } + + for (auto& ev : readSession->GetEvents(false)) { + if (auto* startEvent = std::get_if<NTopic::TReadSessionEvent::TStartPartitionSessionEvent>(&ev)) { + startEvent->Confirm(); + } else if (auto* stopEvent = std::get_if<NTopic::TReadSessionEvent::TStopPartitionSessionEvent>(&ev)) { + stopEvent->Confirm(); + } else if (auto* data = std::get_if<NTopic::TReadSessionEvent::TDataReceivedEvent>(&ev)) { + for (const auto& m : data->GetMessages()) { + NJson::TJsonValue json; + if (NJson::ReadJsonTree(m.GetData(), &json)) { + result.push_back(json); + } + } + data->Commit(); + } else if (std::get_if<NTopic::TSessionClosedEvent>(&ev)) { + return result; + } + } + } + + return result; +} + +void ExecDdl(NQuery::TSession& session, const std::string& query, const char* what) { + auto result = session.ExecuteQuery(query, TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, + what << ": " << result.GetIssues().ToString()); +} + +} // namespace + +Y_UNIT_TEST_SUITE(KqpTxCommitTimestampCdc) { + Y_UNIT_TEST(SingleShardWrite) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + settings.SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecDdl(session, R"( + CREATE TABLE `/Root/CdcTest` ( + Key Uint64, + Value Text, + PRIMARY KEY (Key) + ); + )", "CREATE TABLE"); + + ExecDdl(session, R"( + ALTER TABLE `/Root/CdcTest` ADD CHANGEFEED `feed` WITH ( + MODE = 'UPDATES', FORMAT = 'JSON', VIRTUAL_TIMESTAMPS = TRUE + ); + )", "ADD CHANGEFEED"); + + ExecDdl(session, R"( + ALTER TOPIC `/Root/CdcTest/feed` ADD CONSUMER `test_consumer`; + )", "ADD CONSUMER"); + + auto result = session.ExecuteQuery(R"( + UPSERT INTO `/Root/CdcTest` (Key, Value) VALUES (1u, "hello"); + )", TTxControl::BeginTx(TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetCommitTimestamp().has_value(), "Commit timestamp should be present"); + const auto& commitTs = *result.GetCommitTimestamp(); + UNIT_ASSERT_C(commitTs.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(commitTs.TxId > 0, "TxId should be nonzero"); + + auto records = ReadCdcMessages(kikimr, "/Root/CdcTest/feed", 1); + UNIT_ASSERT_VALUES_EQUAL(records.size(), 1); + UNIT_ASSERT_C(records[0].Has("ts"), "CDC record should have 'ts' field"); + UNIT_ASSERT_VALUES_EQUAL(records[0]["ts"][0].GetUInteger(), commitTs.PlanStep); + UNIT_ASSERT_VALUES_EQUAL(records[0]["ts"][1].GetUInteger(), commitTs.TxId); + } + + Y_UNIT_TEST(ThreeInsertsOneShard) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + settings.SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecDdl(session, R"( + CREATE TABLE `/Root/CdcMultiInsert` ( + Key Uint64, + Value Text, + PRIMARY KEY (Key) + ) WITH (UNIFORM_PARTITIONS = 1); + )", "CREATE TABLE"); + + ExecDdl(session, R"( + ALTER TABLE `/Root/CdcMultiInsert` ADD CHANGEFEED `feed` WITH ( + MODE = 'UPDATES', FORMAT = 'JSON', VIRTUAL_TIMESTAMPS = TRUE + ); + )", "ADD CHANGEFEED"); + + ExecDdl(session, R"( + ALTER TOPIC `/Root/CdcMultiInsert/feed` ADD CONSUMER `test_consumer`; + )", "ADD CONSUMER"); + + auto beginResult = session.BeginTransaction(TTxSettings::StrictSerializableRW()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(beginResult.GetStatus(), EStatus::SUCCESS, beginResult.GetIssues().ToString()); + auto tx = beginResult.GetTransaction(); + + for (ui64 i = 1; i <= 3; ++i) { + TString query = TStringBuilder() + << "INSERT INTO `/Root/CdcMultiInsert` (Key, Value) VALUES (" + << i << "u, \"val" << i << "\");"; + auto execResult = session.ExecuteQuery(query, TTxControl::Tx(tx)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(execResult.GetStatus(), EStatus::SUCCESS, execResult.GetIssues().ToString()); + } + + auto commitResult = tx.Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::SUCCESS, commitResult.GetIssues().ToString()); + UNIT_ASSERT_C(commitResult.GetCommitTimestamp().has_value(), "Commit timestamp should be present"); + const auto& commitTs = *commitResult.GetCommitTimestamp(); + UNIT_ASSERT_C(commitTs.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(commitTs.TxId > 0, "TxId should be nonzero"); + + auto records = ReadCdcMessages(kikimr, "/Root/CdcMultiInsert/feed", 3); + UNIT_ASSERT_VALUES_EQUAL(records.size(), 3); + + for (size_t i = 0; i < records.size(); ++i) { + UNIT_ASSERT_C(records[i].Has("ts"), TStringBuilder() << "CDC record " << i << " should have 'ts' field"); + UNIT_ASSERT_VALUES_EQUAL_C( + records[i]["ts"][0].GetUInteger(), commitTs.PlanStep, + TStringBuilder() << "CDC record " << i << " PlanStep mismatch"); + UNIT_ASSERT_VALUES_EQUAL_C( + records[i]["ts"][1].GetUInteger(), commitTs.TxId, + TStringBuilder() << "CDC record " << i << " TxId mismatch"); + } + } + + Y_UNIT_TEST(ThreeUpsertsMultiShard) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + settings.SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecDdl(session, R"( + CREATE TABLE `/Root/CdcMultiShard` ( + Key Uint64, + Value Text, + PRIMARY KEY (Key) + ) WITH (UNIFORM_PARTITIONS = 4); + )", "CREATE TABLE"); + + ExecDdl(session, R"( + ALTER TABLE `/Root/CdcMultiShard` ADD CHANGEFEED `feed` WITH ( + MODE = 'UPDATES', FORMAT = 'JSON', VIRTUAL_TIMESTAMPS = TRUE + ); + )", "ADD CHANGEFEED"); + + ExecDdl(session, R"( + ALTER TOPIC `/Root/CdcMultiShard/feed` ADD CONSUMER `test_consumer`; + )", "ADD CONSUMER"); + + auto beginResult = session.BeginTransaction(TTxSettings::StrictSerializableRW()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(beginResult.GetStatus(), EStatus::SUCCESS, beginResult.GetIssues().ToString()); + auto tx = beginResult.GetTransaction(); + + const ui64 keys[] = {1, 1000000, 2000000}; + for (ui64 i = 0; i < 3; ++i) { + TString query = TStringBuilder() + << "UPSERT INTO `/Root/CdcMultiShard` (Key, Value) VALUES (" + << keys[i] << "u, \"val" << (i + 1) << "\");"; + auto execResult = session.ExecuteQuery(query, TTxControl::Tx(tx)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(execResult.GetStatus(), EStatus::SUCCESS, execResult.GetIssues().ToString()); + } + + auto commitResult = tx.Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::SUCCESS, commitResult.GetIssues().ToString()); + UNIT_ASSERT_C(commitResult.GetCommitTimestamp().has_value(), "Commit timestamp should be present"); + const auto& commitTs = *commitResult.GetCommitTimestamp(); + UNIT_ASSERT_C(commitTs.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(commitTs.TxId > 0, "TxId should be nonzero"); + + auto records = ReadCdcMessages(kikimr, "/Root/CdcMultiShard/feed", 3); + UNIT_ASSERT_VALUES_EQUAL(records.size(), 3); + + for (size_t i = 0; i < records.size(); ++i) { + UNIT_ASSERT_C(records[i].Has("ts"), TStringBuilder() << "CDC record " << i << " should have 'ts' field"); + UNIT_ASSERT_VALUES_EQUAL_C( + records[i]["ts"][0].GetUInteger(), commitTs.PlanStep, + TStringBuilder() << "CDC record " << i << " PlanStep mismatch"); + UNIT_ASSERT_VALUES_EQUAL_C( + records[i]["ts"][1].GetUInteger(), commitTs.TxId, + TStringBuilder() << "CDC record " << i << " TxId mismatch"); + } + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp b/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp index 5d9d8af2225..4f487a74863 100644 --- a/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_tx_ut.cpp @@ -868,6 +868,137 @@ Y_UNIT_TEST_SUITE(KqpTx) { UNIT_ASSERT(foundInsertWithImmediate); UNIT_ASSERT(foundPrepare); } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_ExecuteQuery) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto result = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (200u, "Commit1"); + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.GetCommitTimestamp().has_value(), "Commit timestamp should be present for StrictSerializableRW write commit"); + const auto& ts1 = *result.GetCommitTimestamp(); + UNIT_ASSERT_C(ts1.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(ts1.TxId > 0, "TxId should be nonzero"); + + auto result2 = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (201u, "Commit2"); + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + UNIT_ASSERT_C(result2.GetCommitTimestamp().has_value(), "Commit timestamp should be present for StrictSerializableRW write commit"); + const auto& ts2 = *result2.GetCommitTimestamp(); + UNIT_ASSERT_C(ts2.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(ts2.TxId > 0, "TxId should be nonzero"); + + UNIT_ASSERT_C(ts1 < ts2, "Second commit timestamp should be greater than first (lexicographic order)"); + } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_ExplicitCommit) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto beginResult = session.BeginTransaction(NYdb::NQuery::TTxSettings::StrictSerializableRW()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(beginResult.GetStatus(), EStatus::SUCCESS, beginResult.GetIssues().ToString()); + auto tx = beginResult.GetTransaction(); + + auto execResult = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (300u, "Explicit"); + )", NYdb::NQuery::TTxControl::Tx(tx)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(execResult.GetStatus(), EStatus::SUCCESS, execResult.GetIssues().ToString()); + + auto commitResult = tx.Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::SUCCESS, commitResult.GetIssues().ToString()); + UNIT_ASSERT_C(commitResult.GetCommitTimestamp().has_value(), "Commit timestamp should be present for StrictSerializableRW write commit"); + const auto& ts = *commitResult.GetCommitTimestamp(); + UNIT_ASSERT_C(ts.PlanStep > 0, "PlanStep should be nonzero"); + UNIT_ASSERT_C(ts.TxId > 0, "TxId should be nonzero"); + } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_ReadOnly) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto result = session.ExecuteQuery(R"( + SELECT * FROM `/Root/KeyValue` WHERE Key = 100u; + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(!result.GetCommitTimestamp().has_value(), "Commit timestamp should not be present for read-only query"); + } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_ReadOnly_ExplicitCommit) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto beginResult = session.BeginTransaction(NYdb::NQuery::TTxSettings::StrictSerializableRW()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(beginResult.GetStatus(), EStatus::SUCCESS, beginResult.GetIssues().ToString()); + auto tx = beginResult.GetTransaction(); + + auto execResult = session.ExecuteQuery(R"( + SELECT * FROM `/Root/KeyValue` WHERE Key = 100u; + )", NYdb::NQuery::TTxControl::Tx(tx)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(execResult.GetStatus(), EStatus::SUCCESS, execResult.GetIssues().ToString()); + + auto commitResult = tx.Commit().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::SUCCESS, commitResult.GetIssues().ToString()); + UNIT_ASSERT_C(!commitResult.GetCommitTimestamp().has_value(), "Commit timestamp should not be present for read-only explicit commit"); + } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_SerializableRW) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto result = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (400u, "Serializable"); + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(!result.GetCommitTimestamp().has_value(), "Commit timestamp should not be present for non-StrictSerializableRW isolation"); + } + + Y_UNIT_TEST(StrictSerializable_CommitTimestamp_Order) { + TKikimrSettings settings; + settings.SetEnableStrictSerializableIsolation(true); + auto kikimr = TKikimrRunner(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + auto result1 = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (500u, "Distinct1"); + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result1.GetStatus(), EStatus::SUCCESS, result1.GetIssues().ToString()); + UNIT_ASSERT_C(result1.GetCommitTimestamp().has_value(), "Commit timestamp should be present"); + const auto& ts1 = *result1.GetCommitTimestamp(); + + auto result2 = session.ExecuteQuery(R"( + UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (501u, "Distinct2"); + )", NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::StrictSerializableRW()).CommitTx()).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + UNIT_ASSERT_C(result2.GetCommitTimestamp().has_value(), "Commit timestamp should be present"); + const auto& ts2 = *result2.GetCommitTimestamp(); + + UNIT_ASSERT_C(ts1 < ts2, "Commit timestamps must be ordered and distinct"); + } } } // namespace NKqp diff --git a/ydb/core/kqp/ut/tx/ya.make b/ydb/core/kqp/ut/tx/ya.make index 3dc1e7152b8..900ffd20057 100644 --- a/ydb/core/kqp/ut/tx/ya.make +++ b/ydb/core/kqp/ut/tx/ya.make @@ -17,6 +17,7 @@ SRCS( kqp_sink_tx_ut.cpp kqp_snapshot_isolation_ut.cpp kqp_tx_ut.cpp + kqp_tx_commit_timestamp_cdc_ut.cpp kqp_rollback.cpp kqp_online_ro_ut.cpp ) diff --git a/ydb/core/kqp/workload_service/kqp_workload_service.h b/ydb/core/kqp/workload_service/kqp_workload_service.h deleted file mode 100644 index 75a36db5cb1..00000000000 --- a/ydb/core/kqp/workload_service/kqp_workload_service.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include <ydb/core/resource_pools/resource_pool_settings.h> - -#include <ydb/library/actors/core/actor.h> - - -namespace NKikimr::NKqp { - -NActors::IActor* CreateKqpWorkloadService(NMonitoring::TDynamicCounterPtr counters); - -} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/workload_service/ut/common/ya.make b/ydb/core/kqp/workload_service/ut/common/ya.make deleted file mode 100644 index 087874d1a15..00000000000 --- a/ydb/core/kqp/workload_service/ut/common/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -SRCS( - kqp_query_classifier_ut_common.h - kqp_workload_service_ut_common.cpp -) - -PEERDIR( - ydb/core/kqp/gateway/behaviour/resource_pool_classifier - ydb/core/kqp/ut/common - ydb/services/metadata -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/core/kqp/workload_service/ut/ya.make b/ydb/core/kqp/workload_service/ut/ya.make deleted file mode 100644 index de52737c141..00000000000 --- a/ydb/core/kqp/workload_service/ut/ya.make +++ /dev/null @@ -1,38 +0,0 @@ -UNITTEST_FOR(ydb/core/kqp/workload_service) - -FORK_SUBTESTS() - -SIZE(MEDIUM) -IF (SANITIZER_TYPE) - REQUIREMENTS(cpu:4) -ELSE() - REQUIREMENTS(cpu:2) -ENDIF() - -SRCS( - kqp_action_reject_ut.cpp - kqp_has_app_name_ut.cpp - kqp_has_full_scan_matcher_ut.cpp - kqp_has_full_scan_ut.cpp - kqp_has_path_ddl_ut.cpp - kqp_has_path_matcher_ut.cpp - kqp_has_path_ut.cpp - kqp_stream_query_classification_ut.cpp - kqp_member_name_ut.cpp - kqp_query_classifier_match_ut.cpp - kqp_query_classifier_ut.cpp - kqp_workload_service_actors_ut.cpp - kqp_workload_service_query_sessions_ut.cpp - kqp_workload_service_tables_ut.cpp - kqp_workload_service_ut.cpp -) - -PEERDIR( - ydb/core/kqp/workload_service/ut/common - - yql/essentials/sql/pg_dummy -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/core/kqp/workload_service/ya.make b/ydb/core/kqp/workload_service/ya.make deleted file mode 100644 index d02845691d1..00000000000 --- a/ydb/core/kqp/workload_service/ya.make +++ /dev/null @@ -1,43 +0,0 @@ -LIBRARY() - -SRCS( - kqp_has_full_scan_matcher.cpp - kqp_has_path_matcher.cpp - kqp_query_classifier.cpp - kqp_workload_service.cpp -) - -PEERDIR( - ydb/core/cms/console - - ydb/core/fq/libs/compute/common - - ydb/core/kqp/common - ydb/core/kqp/gateway/behaviour/resource_pool_classifier - ydb/core/kqp/query_data - ydb/core/kqp/workload_service/actors - - ydb/core/mind - - ydb/core/resource_pools - - ydb/library/actors/interconnect - ydb/library/aclib - ydb/library/yql/providers/pq/proto - - ydb/public/api/protos -) - -YQL_LAST_ABI_VERSION() - -END() - -RECURSE( - actors - common - tables -) - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/core/kqp/ya.make b/ydb/core/kqp/ya.make index d2cd8434ed2..5b281c3b21f 100644 --- a/ydb/core/kqp/ya.make +++ b/ydb/core/kqp/ya.make @@ -74,7 +74,6 @@ RECURSE( runtime session_actor tests - workload_service ) RECURSE_FOR_TESTS( diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index b54ac2960a5..9e9d4624b72 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -69,9 +69,7 @@ TBlobStorageController::TVSlotInfo::TVSlotInfo(TVSlotId vSlotId, TPDiskInfo *pdi Group = group; group->AddVSlot(this); } - pdisk->NumActiveSlots += TPDiskConfig::GetOwnerWeight( - group->GroupSizeInUnits, - pdisk->SlotSizeInUnits); + pdisk->NumActiveSlots += pdisk->GetOwnerWeight(group->GroupSizeInUnits); } } @@ -175,12 +173,10 @@ bool TBlobStorageController::TGroupInfo::FillInResources( const TPDiskInfo *pdisk = vslot->PDisk; const auto& metrics = pdisk->Metrics; - ui32 maxSlots = 0; - ui32 slotSizeInUnits = 0; - pdisk->ExtractInferredPDiskSettings(maxSlots, slotSizeInUnits); + const ui32 maxSlots = pdisk->GetEffectiveExpectedSlotCount(); ui64 vdiskSlotSize = 0; - const ui32 weight = TPDiskConfig::GetOwnerWeight(GroupSizeInUnits, slotSizeInUnits); + const ui32 weight = pdisk->GetOwnerWeight(GroupSizeInUnits); if (metrics.HasEnforcedDynamicSlotSize()) { vdiskSlotSize = metrics.GetEnforcedDynamicSlotSize() * weight; } else if (metrics.GetTotalSize()) { @@ -889,7 +885,6 @@ void TBlobStorageController::SetHostRecords(THostRecordMap hostRecords) { ClusterBalanceActorId = Register(CreateClusterBalancingActor(SelfId(), ClusterBalancingSettings)); } - PushStaticGroupsToSelfHeal(); Execute(CreateTxInitScheme()); } @@ -912,7 +907,7 @@ void TBlobStorageController::ValidateInternalState() { if (!vslot->IsBeingDeleted()) { const TGroupInfo* group = FindGroup(vslot->GroupId); Y_ABORT_UNLESS(group); - numActiveSlots += TPDiskConfig::GetOwnerWeight(group->GroupSizeInUnits, pdisk->SlotSizeInUnits); + numActiveSlots += pdisk->GetOwnerWeight(group->GroupSizeInUnits); } } Y_ABORT_UNLESS(pdisk->NumActiveSlots == numActiveSlots); diff --git a/ydb/core/mind/bscontroller/cmds_drive_status.cpp b/ydb/core/mind/bscontroller/cmds_drive_status.cpp index f9151299ff4..7f28b7123bc 100644 --- a/ydb/core/mind/bscontroller/cmds_drive_status.cpp +++ b/ydb/core/mind/bscontroller/cmds_drive_status.cpp @@ -133,6 +133,8 @@ namespace NKikimr::NBsController { if (cmd.GetPDiskType() != NKikimrBlobStorage::UNKNOWN_TYPE) { driveInfoMutable->PDiskType = cmd.GetPDiskType(); } + const TString context = TStringBuilder() << "DriveSerial# " << serial; + ValidatePDiskConfig(cmd.GetPDiskConfig(), context); TString config; if (!cmd.GetPDiskConfig().SerializeToString(&config)) { throw TExError() << "Couldn't serialize PDiskConfig for disk with serial number" << TErrorParams::DiskSerialNumber(serial); diff --git a/ydb/core/mind/bscontroller/cmds_host_config.cpp b/ydb/core/mind/bscontroller/cmds_host_config.cpp index 3e0cab6701e..599359dc293 100644 --- a/ydb/core/mind/bscontroller/cmds_host_config.cpp +++ b/ydb/core/mind/bscontroller/cmds_host_config.cpp @@ -8,6 +8,8 @@ namespace NKikimr::NBsController { TMaybe<TString> defaultPDiskConfig; if (cmd.HasDefaultHostPDiskConfig()) { + ValidatePDiskConfig(cmd.GetDefaultHostPDiskConfig(), + "DefaultHostPDiskConfig"); TString config; const bool success = cmd.GetDefaultHostPDiskConfig().SerializeToString(&config); Y_ABORT_UNLESS(success); @@ -28,6 +30,9 @@ namespace NKikimr::NBsController { } if (drive.HasPDiskConfig()) { + const TString context = TStringBuilder() << "HostConfigId# " << id + << " Path# " << drive.GetPath(); + ValidatePDiskConfig(drive.GetPDiskConfig(), context); TString config; const bool success = drive.GetPDiskConfig().SerializeToString(&config); Y_ABORT_UNLESS(success); diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp index 63d8befedad..e5cb792503e 100644 --- a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp +++ b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp @@ -529,8 +529,8 @@ namespace NKikimr::NBsController { TPDiskInfo* pdisk = PDisks.FindForUpdate(vslotId.ComprisingPDiskId()); Y_ABORT_UNLESS(pdisk); - pdisk->NumActiveSlots -= TPDiskConfig::GetOwnerWeight(oldSizeInUnits, pdisk->SlotSizeInUnits); - pdisk->NumActiveSlots += TPDiskConfig::GetOwnerWeight(newSizeInUnits, pdisk->SlotSizeInUnits); + pdisk->NumActiveSlots -= pdisk->GetOwnerWeight(oldSizeInUnits); + pdisk->NumActiveSlots += pdisk->GetOwnerWeight(newSizeInUnits); } // update the group size @@ -640,7 +640,8 @@ namespace NKikimr::NBsController { x->SetGuid(pdisk.Guid); x->SetNumStaticSlots(pdisk.StaticSlotUsage); x->SetDriveStatus(NKikimrBlobStorage::EDriveStatus::ACTIVE); - x->SetExpectedSlotCount(pdisk.ExpectedSlotCount); + x->SetExpectedSlotCount(pdisk.GetEffectiveExpectedSlotCount()); + x->SetExpectedSlotSize(pdisk.GetEffectiveExpectedSlotSize()); x->SetDecommitStatus(NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE); if (pdisk.PDiskMetrics) { x->MutablePDiskMetrics()->CopyFrom(*pdisk.PDiskMetrics); diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp index e1822d3abc1..dc1c443a5df 100644 --- a/ydb/core/mind/bscontroller/config.cpp +++ b/ydb/core/mind/bscontroller/config.cpp @@ -85,7 +85,9 @@ namespace NKikimr::NBsController { void ApplyPDiskDiff(const TPDiskId &pdiskId, const TPDiskInfo &prev, const TPDiskInfo &cur) { if (prev.Mood != cur.Mood || prev.ExpectedSlotCount != cur.ExpectedSlotCount || - prev.SlotSizeInUnits != cur.SlotSizeInUnits) { + prev.SlotSizeInUnits != cur.SlotSizeInUnits || + prev.ExpectedSlotSize != cur.ExpectedSlotSize || + prev.MaxSlots != cur.MaxSlots) { CreatePDiskEntry(pdiskId, cur); } } @@ -905,9 +907,7 @@ namespace NKikimr::NBsController { Y_ABORT_UNLESS(pdisk); const TGroupInfo *group = Groups.Find(mutableSlot->GroupId); Y_ABORT_UNLESS(group); - pdisk->NumActiveSlots -= TPDiskConfig::GetOwnerWeight( - group->GroupSizeInUnits, - pdisk->SlotSizeInUnits); + pdisk->NumActiveSlots -= pdisk->GetOwnerWeight(group->GroupSizeInUnits); if (UncommittedVSlots.erase(vslotId)) { const ui32 erased = pdisk->VSlotsOnPDisk.erase(vslotId.VSlotId); @@ -941,7 +941,7 @@ namespace NKikimr::NBsController { if (!vslot->IsBeingDeleted()) { const TGroupInfo *group = Groups.Find(vslot->GroupId); Y_ABORT_UNLESS(group); - numActiveSlots += TPDiskConfig::GetOwnerWeight(group->GroupSizeInUnits, pdisk.SlotSizeInUnits); + numActiveSlots += pdisk.GetOwnerWeight(group->GroupSizeInUnits); } } Y_ABORT_UNLESS(pdisk.NumActiveSlots == numActiveSlots); @@ -1159,7 +1159,8 @@ namespace NKikimr::NBsController { pb->SetBoxId(pdisk.BoxId); pb->SetNumStaticSlots(pdisk.StaticSlotUsage); pb->SetDriveStatus(pdisk.Status); - pb->SetExpectedSlotCount(pdisk.ExpectedSlotCount); + pb->SetExpectedSlotCount(pdisk.GetEffectiveExpectedSlotCount()); + pb->SetExpectedSlotSize(pdisk.GetEffectiveExpectedSlotSize()); pb->SetDriveStatusChangeTimestamp(pdisk.StatusTimestamp.GetValue()); pb->SetDecommitStatus(pdisk.DecommitStatus); pb->MutablePDiskMetrics()->CopyFrom(pdisk.Metrics); diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h index 21c52c2e68e..8addd7575f2 100644 --- a/ydb/core/mind/bscontroller/config.h +++ b/ydb/core/mind/bscontroller/config.h @@ -4,11 +4,19 @@ #include "error.h" #include "impl.h" +#include <ydb/core/blobstorage/base/pdisk_config_validation.h> #include <ydb/core/protos/blob_depot_config.pb.h> namespace NKikimr { namespace NBsController { + inline void ValidatePDiskConfig(const NKikimrBlobStorage::TPDiskConfig& config, + TStringBuf context) { + if (auto error = ::NKikimr::ValidatePDiskConfig(config, context)) { + throw TExError() << *error; + } + } + struct TConfigFitAction { std::set<TBoxId> Boxes; std::multiset<std::tuple<TBoxStoragePoolId, std::optional<TGroupId>>> PoolsAndGroups; // nullopt goes first and means 'cover all groups in the pool' diff --git a/ydb/core/mind/bscontroller/config_fit_groups.cpp b/ydb/core/mind/bscontroller/config_fit_groups.cpp index 51476b1a443..5b6f37eed49 100644 --- a/ydb/core/mind/bscontroller/config_fit_groups.cpp +++ b/ydb/core/mind/bscontroller/config_fit_groups.cpp @@ -746,6 +746,7 @@ namespace NKikimr { .NumSlots = numSlots, .MaxSlots = maxSlots, .SlotSizeInUnits = slotSizeInUnits, + .SlotSizeInBytes = info.GetEffectiveExpectedSlotSize(), .Groups = std::move(groups), .SpaceAvailable = availableSpace, .Operational = info.Operational, diff --git a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp index f5fae384ea2..a49f9c16f55 100644 --- a/ydb/core/mind/bscontroller/config_fit_pdisks.cpp +++ b/ydb/core/mind/bscontroller/config_fit_pdisks.cpp @@ -105,18 +105,16 @@ namespace NKikimr { } // run ExtractConfig as the very last step const ui32 oldSlotSizeInUnits = pdiskInfo->SlotSizeInUnits; + const bool oldHasFixedSlotSize = pdiskInfo->GetEffectiveExpectedSlotSize() != 0; pdiskInfo->ExtractConfig(defaultMaxSlots); - if (pdiskInfo->SlotSizeInUnits != oldSlotSizeInUnits) { - ui32 numActiveSlots = 0; - for (const auto& [vslotId, vslot] : pdiskInfo->VSlotsOnPDisk) { - if (vslot->IsBeingDeleted()) { - continue; - } - const TBlobStorageController::TGroupInfo *group = state.Groups.Find(vslot->GroupId); - Y_ABORT_UNLESS(group); - numActiveSlots += TPDiskConfig::GetOwnerWeight(group->GroupSizeInUnits, pdiskInfo->SlotSizeInUnits); - } - pdiskInfo->NumActiveSlots = numActiveSlots; + // NumActiveSlots is a sum of owner weights, so it must be recomputed when the + // config change affects the weight inputs: SlotSizeInUnits or the effective + // expected slot size being set/unset + if (pdiskInfo->SlotSizeInUnits != oldSlotSizeInUnits || + (pdiskInfo->GetEffectiveExpectedSlotSize() != 0) != oldHasFixedSlotSize) { + pdiskInfo->NumActiveSlots = pdiskInfo->ComputeNumActiveSlots([&](TGroupId groupId) { + return state.Groups.Find(groupId); + }); } } } diff --git a/ydb/core/mind/bscontroller/disk_metrics.cpp b/ydb/core/mind/bscontroller/disk_metrics.cpp index 9db78d89842..330a0f8382d 100644 --- a/ydb/core/mind/bscontroller/disk_metrics.cpp +++ b/ydb/core/mind/bscontroller/disk_metrics.cpp @@ -55,6 +55,12 @@ public: void Complete(const TActorContext&) override {} }; +void TBlobStorageController::RecomputePDiskNumActiveSlots(TPDiskInfo *pdisk) { + pdisk->NumActiveSlots = pdisk->ComputeNumActiveSlots([this](TGroupId groupId) { + return FindGroup(groupId); + }); +} + void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerUpdateDiskStatus::TPtr &ev) { TabletCounters->Cumulative()[NBlobStorageController::COUNTER_UPDATE_DISK_METRICS_COUNT].Increment(1); TRequestCounter counter(TabletCounters, NBlobStorageController::COUNTER_UPDATE_DISK_METRICS_USEC); @@ -149,6 +155,12 @@ void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerUpdateDiskStatu pdiskIds.push_back(pdiskId); } + // NumActiveSlots is maintained incrementally with owner weights that depend on + // whether the effective expected slot size is set; when a metrics update flips it + // (e.g. the PDisk started reporting ExpectedSlotSize inferred from global settings), + // the counter must be recomputed with the new weights + const bool hadFixedSlotSize = pdisk->GetEffectiveExpectedSlotSize() != 0; + if (pdisk->UpdatePDiskMetrics(m, now)) { // this PDisk just did obtain full metrics set, we can unblock any pending SelectGroups operations for (auto& [id, slot] : pdisk->VSlotsOnPDisk) { @@ -157,6 +169,9 @@ void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerUpdateDiskStatu } } } + if ((pdisk->GetEffectiveExpectedSlotSize() != 0) != hadFixedSlotSize) { + RecomputePDiskNumActiveSlots(pdisk); + } pdisk->UpdateOperational(true); SysViewChangedPDisks.insert(pdiskId); diff --git a/ydb/core/mind/bscontroller/group_mapper.cpp b/ydb/core/mind/bscontroller/group_mapper.cpp index 09a15a8a573..e89ada3d22c 100644 --- a/ydb/core/mind/bscontroller/group_mapper.cpp +++ b/ydb/core/mind/bscontroller/group_mapper.cpp @@ -47,17 +47,27 @@ namespace NKikimr::NBsController { return i32(MaxSlots) - NumSlots; } + bool HasFixedSlotSize() const { + return SlotSizeInBytes != 0; + } + + ui32 GetOwnerWeight(ui32 groupSizeInUnits) const { + return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, SlotSizeInUnits, SlotSizeInBytes); + } + // the less the better double GetPickerScore(ui32 groupSizeInUnits) const { double penalty = 0; - ui32 vu = groupSizeInUnits ?: 1; - ui32 pu = SlotSizeInUnits ?: 1; - if (vu > pu) { - // double-unit vdisk occupies two single pdisk slots - penalty += TImpl::GroupSizeInUnitsLargerThanPDiskPenalty; - } else if (vu < pu) { - // single-unit vdisk occupies double-unit pdisk slot (storage waste) - penalty += TImpl::GroupSizeInUnitsSmallerThanPDiskPenalty; + if (!HasFixedSlotSize()) { + ui32 vu = groupSizeInUnits ?: 1; + ui32 pu = SlotSizeInUnits ?: 1; + if (vu > pu) { + // double-unit vdisk occupies two single pdisk slots + penalty += TImpl::GroupSizeInUnitsLargerThanPDiskPenalty; + } else if (vu < pu) { + // single-unit vdisk occupies double-unit pdisk slot (storage waste) + penalty += TImpl::GroupSizeInUnitsSmallerThanPDiskPenalty; + } } if (!MaxSlots) { return NumSlots + penalty; @@ -170,6 +180,26 @@ namespace NKikimr::NBsController { } } + ui32 GetSlotsNeeded(const TPDiskInfo& pdisk) const { + return pdisk.GetOwnerWeight(GroupSizeInUnits); + } + + bool HasEnoughSpace(const TPDiskInfo& pdisk) const { + if (pdisk.SpaceAvailable < RequiredSpace) { + return false; + } + if (pdisk.SlotSizeInBytes && RequiredSpace > 0) { + const ui64 slotsNeeded = GetSlotsNeeded(pdisk); + if (slotsNeeded > Max<ui64>() / pdisk.SlotSizeInBytes) { + return false; + } + if (pdisk.SlotSizeInBytes * slotsNeeded < static_cast<ui64>(RequiredSpace)) { + return false; + } + } + return true; + } + bool DiskIsUsable(const TPDiskInfo& pdisk) const { if (!pdisk.Usable) { return false; // disk is not usable in this case @@ -180,13 +210,13 @@ namespace NKikimr::NBsController { if (RequireOperational && !pdisk.Operational) { return false; } - if (pdisk.SpaceAvailable < RequiredSpace) { + if (!HasEnoughSpace(pdisk)) { return false; } if (pdisk.BridgePileId != BridgePileId) { return false; } - if (auto slotsNeeded = i32(TPDiskConfig::GetOwnerWeight(GroupSizeInUnits, pdisk.SlotSizeInUnits)); pdisk.FreeSlots() < slotsNeeded) { + if (pdisk.FreeSlots() < i32(GetSlotsNeeded(pdisk))) { return false; } return true; @@ -1131,7 +1161,7 @@ namespace NKikimr::NBsController { s << std::exchange(minus, "") << "s[" << pdisk->NumSlots << "/" << pdisk->MaxSlots << "]"; } - if (pdisk->SpaceAvailable < diskManager.RequiredSpace) { + if (!diskManager.HasEnoughSpace(*pdisk)) { totalStats.NotEnoughSpace++; domainStats.NotEnoughSpace++; diskIsOk = false; @@ -1253,7 +1283,7 @@ namespace NKikimr::NBsController { const auto it = PDisks.find(pdiskId); Y_ABORT_UNLESS(it != PDisks.end()); TPDiskInfo& pdisk = it->second; - pdisk.NumSlots -= TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdisk.SlotSizeInUnits); + pdisk.NumSlots -= pdisk.GetOwnerWeight(groupSizeInUnits); pdisk.EraseGroup(groupId); } ui32 numZero = 0; @@ -1261,7 +1291,7 @@ namespace NKikimr::NBsController { if (!group[i]) { ++numZero; TPDiskInfo *pdisk = result->at(i); - pdisk->NumSlots += TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdisk->SlotSizeInUnits); + pdisk->NumSlots += pdisk->GetOwnerWeight(groupSizeInUnits); pdisk->InsertGroup(groupId); } } @@ -1359,14 +1389,14 @@ namespace NKikimr::NBsController { const auto it = PDisks.find(pdiskId); Y_ABORT_UNLESS(it != PDisks.end()); TPDiskInfo& pdisk = it->second; - pdisk.NumSlots -= TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdisk.SlotSizeInUnits);; + pdisk.NumSlots -= pdisk.GetOwnerWeight(groupSizeInUnits); pdisk.EraseGroup(groupId); } { const auto it = PDisks.find(*result); Y_ABORT_UNLESS(it != PDisks.end()); TPDiskInfo& pdisk = it->second; - pdisk.NumSlots += TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdisk.SlotSizeInUnits);; + pdisk.NumSlots += pdisk.GetOwnerWeight(groupSizeInUnits); pdisk.InsertGroup(groupId); groupDefinition[vdisk.FailRealm][vdisk.FailDomain][vdisk.VDisk] = *result; } diff --git a/ydb/core/mind/bscontroller/group_mapper.h b/ydb/core/mind/bscontroller/group_mapper.h index 02fbe88fada..0f50a4d3466 100644 --- a/ydb/core/mind/bscontroller/group_mapper.h +++ b/ydb/core/mind/bscontroller/group_mapper.h @@ -119,6 +119,7 @@ namespace NKikimr { ui32 NumSlots; const ui32 MaxSlots; const ui32 SlotSizeInUnits; + const ui64 SlotSizeInBytes; TStackVec<ui32, 16> Groups; i64 SpaceAvailable; const bool Operational; diff --git a/ydb/core/mind/bscontroller/group_mapper_ut.cpp b/ydb/core/mind/bscontroller/group_mapper_ut.cpp index b3d6fa459dc..60c298ec448 100644 --- a/ydb/core/mind/bscontroller/group_mapper_ut.cpp +++ b/ydb/core/mind/bscontroller/group_mapper_ut.cpp @@ -499,6 +499,7 @@ public: .NumSlots = pair.second.NumSlots, .MaxSlots = equalSlots || location.Rack < 8 ? maxSlots : 2 * maxSlots, .SlotSizeInUnits = pair.second.SlotSizeInUnits, + .SlotSizeInBytes = 0, .Groups{g.begin(), g.end()}, .SpaceAvailable = 0, .Operational = !nonoperationalDisks.contains(pair.first), @@ -632,6 +633,33 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) { context.DumpGroup(g2); } + Y_UNIT_TEST(SlotSizeInBytesLimitsRequiredSpace) { + NActorsInterconnect::TNodeLocation locationProto; + locationProto.SetDataCenter("1"); + locationProto.SetModule("1"); + locationProto.SetRack("1"); + locationProto.SetUnit("1"); + + TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureNone, 1, 1, 1)); + UNIT_ASSERT(mapper.RegisterPDisk({ + .PDiskId = TPDiskId(1, 1), + .Location = TNodeLocation(locationProto), + .Usable = true, + .NumSlots = 0, + .MaxSlots = 2, + .SlotSizeInUnits = 1, + .SlotSizeInBytes = 100, + .Groups{}, + .SpaceAvailable = 1000, + .Operational = true, + .Decommitted = false, + })); + + TGroupMapper::TGroupDefinition group; + TGroupMapperError error; + UNIT_ASSERT_C(!mapper.AllocateGroup(1, group, {}, {}, 2, 150, false, TBridgePileId(), error), error.ErrorMessage); + } + Y_UNIT_TEST(SimplestMirror3dc) { // Each node has 3 disks TTestContext context( diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index c6eaa82d1a6..df2385a0f91 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -361,6 +361,9 @@ public: TBoxId BoxId; ui32 ExpectedSlotCount = 0; bool HasExpectedSlotCount = false; + ui64 ExpectedSlotSize = 0; + bool HasExpectedSlotSize = false; + ui32 MaxSlots = 0; ui32 NumActiveSlots = 0; // sum of owners weights allocated on this PDisk ui32 SlotSizeInUnits = 0; TMap<Schema::VSlot::VSlotID::Type, TIndirectReferable<TVSlotInfo>::TPtr> VSlotsOnPDisk; // vslots over this PDisk @@ -474,6 +477,11 @@ public: void ExtractConfig(ui32 defaultMaxSlots) { ExpectedSlotCount = defaultMaxSlots; + HasExpectedSlotCount = false; + ExpectedSlotSize = 0; + HasExpectedSlotSize = false; + MaxSlots = 0; + SlotSizeInUnits = 0; NKikimrBlobStorage::TPDiskConfig pdiskConfig; if (pdiskConfig.ParseFromString(PDiskConfig)) { @@ -484,6 +492,16 @@ public: if (pdiskConfig.HasSlotSizeInUnits()) { SlotSizeInUnits = pdiskConfig.GetSlotSizeInUnits(); } + if (pdiskConfig.HasExpectedSlotSize() && pdiskConfig.GetExpectedSlotSize()) { + ExpectedSlotSize = pdiskConfig.GetExpectedSlotSize(); + HasExpectedSlotSize = true; + } + if (pdiskConfig.HasMaxSlots()) { + MaxSlots = pdiskConfig.GetMaxSlots(); + } + if (HasExpectedSlotSize && !HasExpectedSlotCount) { + ExpectedSlotCount = 0; + } } } @@ -589,6 +607,42 @@ public: } } + ui32 GetEffectiveExpectedSlotCount() const { + ui32 slotCount = 0; + ui32 slotSizeInUnits = 0; + ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); + return slotCount; + } + + ui64 GetEffectiveExpectedSlotSize() const { + return Metrics.HasExpectedSlotSize() ? Metrics.GetExpectedSlotSize() : ExpectedSlotSize; + } + + ui32 GetOwnerWeight(ui32 groupSizeInUnits) const { + // NOTE: uses the config-side SlotSizeInUnits, not the effective (metrics-preferred) + // one: for unit-size-inferred disks this over-counts occupancy of multi-unit groups + // (conservative). Switching to the effective value would change legacy accounting + // and requires extending the NumActiveSlots recompute triggers to units changes + return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, SlotSizeInUnits, GetEffectiveExpectedSlotSize()); + } + + // sum of owner weights over the live vslots with the current weight inputs; must be + // used to refresh NumActiveSlots whenever the weight inputs change (see GetOwnerWeight). + // The group resolver is a parameter because the authoritative group set differs by + // caller: committed controller state vs an in-flight TConfigState overlay + template<typename TGroupResolver> + ui32 ComputeNumActiveSlots(TGroupResolver&& findGroup) const { + ui32 numActiveSlots = 0; + for (const auto& [vslotId, vslot] : VSlotsOnPDisk) { + if (!vslot->IsBeingDeleted()) { + const auto *group = findGroup(vslot->GroupId); + Y_ABORT_UNLESS(group); + numActiveSlots += GetOwnerWeight(group->GroupSizeInUnits); + } + } + return numActiveSlots; + } + TString PathOrSerial() const { return Path ? Path : ExpectedSerial; } @@ -1801,6 +1855,8 @@ private: std::unique_ptr<TEvBlobStorage::TEvControllerConfigRequest> BuildConfigRequestFromStorageConfig( const NKikimrBlobStorage::TStorageConfig& storageConfig, const THostRecordMap& hostRecords, bool validationMode=false); + void RecomputePDiskNumActiveSlots(TPDiskInfo *pdisk); + void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr ev); void Handle(TEvBlobStorage::TEvControllerDistconfRequest::TPtr ev); @@ -2265,6 +2321,7 @@ public: UpdatePDisksCounters(); IssueInitialGroupContent(); InitializeSelfHealState(); + PushStaticGroupsToSelfHeal(); UpdateSystemViews(); UpdateSelfHealCounters(); SignalTabletActive(TActivationContext::AsActorContext()); @@ -2307,6 +2364,7 @@ public: pdisk->ExtractInferredPDiskSettings(effectiveSlotCount, effectiveSlotSizeInUnits); // Check if we should infer PDisk slot count based on global settings bool settingsShouldBeInferred = !pdisk->HasExpectedSlotCount && + !pdisk->HasExpectedSlotSize && StorageConfig && StorageConfig->HasBlobStorageConfig() && StorageConfig->GetBlobStorageConfig().HasInferPDiskSlotCountSettings() && (pdisk->Kind.Type() == NPDisk::DEVICE_TYPE_ROT ? @@ -2321,6 +2379,9 @@ public: if (!effectiveSlotCount) { continue; } + if (pdisk->GetEffectiveExpectedSlotSize()) { + continue; + } for (const auto& [_, vslot] : pdisk->VSlotsOnPDisk) { if (!vslot->Group) { @@ -2551,6 +2612,7 @@ public: std::optional<Schema::PDisk::DiskScope::Type> DiskScope; // null when not set in host config ui32 ExpectedSlotCount = 0; // explicit ui32 SlotSizeInUnits = 0; // explicit + ui64 ExpectedSlotSize = 0; // explicit // runtime info ui32 StaticSlotUsage = 0; @@ -2574,6 +2636,7 @@ public: if (pdisk.HasDiskScope()) { DiskScope = pdisk.GetDiskScope(); } + ExpectedSlotSize = cfg.GetExpectedSlotSize(); } const TPDiskId pdiskId(NodeId, PDiskId); @@ -2593,6 +2656,19 @@ public: slotSizeInUnits = SlotSizeInUnits; } } + + ui32 GetEffectiveExpectedSlotCount() const { + ui32 slotCount = 0; + ui32 slotSizeInUnits = 0; + ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); + return slotCount; + } + + ui64 GetEffectiveExpectedSlotSize() const { + return PDiskMetrics && PDiskMetrics->HasExpectedSlotSize() + ? PDiskMetrics->GetExpectedSlotSize() + : ExpectedSlotSize; + } }; std::map<TPDiskId, TStaticPDiskInfo> StaticPDisks; diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index 13bd7c56fb4..4dfee75874b 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -181,6 +181,7 @@ namespace NKikimr::NBsController { auto *cmd = record.MutableReassignGroupDisk(); VDiskIDFromVDiskID(*VDiskToReplace, cmd->MutableVDiskId()); cmd->SetConvertToDonor(DonorMode); + cmd->SetAllowUnusableDisks(true); cmd->SetIsSelfHealReasonDecommit(IsSelfHealReasonDecommit); cmd->SetFromSelfHeal(true); Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), ev.release()); diff --git a/ydb/core/mind/bscontroller/storage_stats_calculator.cpp b/ydb/core/mind/bscontroller/storage_stats_calculator.cpp index c99d84ac9e8..12aa00e3703 100644 --- a/ydb/core/mind/bscontroller/storage_stats_calculator.cpp +++ b/ydb/core/mind/bscontroller/storage_stats_calculator.cpp @@ -189,6 +189,7 @@ private: .NumSlots = pdisk.GetNumActiveSlots(), .MaxSlots = pdisk.GetExpectedSlotCount(), // either inferred or user-defined .SlotSizeInUnits = pdisk.GetSlotSizeInUnits(), // either inferred or user-defined + .SlotSizeInBytes = pdisk.GetExpectedSlotSize(), // either inferred or user-defined, 0 if not set .Groups = {}, .SpaceAvailable = 0, .Operational = true, @@ -226,7 +227,7 @@ private: pm.SetSlotSizeInUnits(pdisk.GetSlotSizeInUnits()); } vm.SetAllocatedSize(0); - disks.push_back({&pm, &vm, pdisk.GetExpectedSlotCount()}); + disks.push_back({&pm, &vm, pdisk.GetExpectedSlotCount(), pdisk.GetExpectedSlotSize()}); } } } diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp index 393663b17a7..d2b32dd29a0 100644 --- a/ydb/core/mind/bscontroller/sys_view.cpp +++ b/ydb/core/mind/bscontroller/sys_view.cpp @@ -61,13 +61,17 @@ void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vecto const auto& pdiskMetrics = *disk.PDiskMetrics; ui64 slotSize = 0; - if (pdiskMetrics.HasEnforcedDynamicSlotSize()) { + if (disk.ExpectedSlotSize) { + slotSize = disk.ExpectedSlotSize; + } else if (pdiskMetrics.HasEnforcedDynamicSlotSize()) { slotSize = pdiskMetrics.GetEnforcedDynamicSlotSize(); - } else if (pdiskMetrics.GetTotalSize()) { + } else if (pdiskMetrics.GetTotalSize() && disk.ExpectedSlotCount) { slotSize = pdiskMetrics.GetTotalSize() / disk.ExpectedSlotCount; } - slotSize *= TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdiskMetrics.GetSlotSizeInUnits()); + slotSize *= disk.ExpectedSlotSize + ? 1 + : TPDiskConfig::GetOwnerWeight(groupSizeInUnits, pdiskMetrics.GetSlotSizeInUnits()); if (slotSize) { totalSize = Min(totalSize ? totalSize : Max<ui64>(), slotSize); } @@ -343,6 +347,7 @@ void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageContro ui32 slotSizeInUnits = 0; pDiskInfo->ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); info->SetExpectedSlotCount(slotCount); + info->SetExpectedSlotSize(pDiskInfo->GetEffectiveExpectedSlotSize()); info->SetNumActiveSlots(pDiskInfo->NumActiveSlots + pDiskInfo->StaticSlotUsage); info->SetDecommitStatus(NKikimrBlobStorage::EDecommitStatus_Name(pDiskInfo->DecommitStatus)); info->SetMaintenanceStatus(NKikimrBlobStorage::TMaintenanceStatus::E_Name(pDiskInfo->MaintenanceStatus)); @@ -410,7 +415,12 @@ void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder<TBlobStorageContro std::vector<TGroupDiskInfo> disks; for (const auto& vslot : groupInfo->VDisksInGroup) { - disks.push_back({&vslot->PDisk->Metrics, &vslot->Metrics, vslot->PDisk->ExpectedSlotCount}); + disks.push_back({ + &vslot->PDisk->Metrics, + &vslot->Metrics, + vslot->PDisk->GetEffectiveExpectedSlotCount(), + vslot->PDisk->GetEffectiveExpectedSlotSize(), + }); } CalculateGroupUsageStats(info, disks, TBlobStorageGroupType(groupInfo->ErasureSpecies), groupInfo->GroupSizeInUnits); @@ -578,6 +588,7 @@ void TBlobStorageController::UpdateSystemViews() { pdisk.ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); pb->SetExpectedSlotCount(slotCount); + pb->SetExpectedSlotSize(pdisk.GetEffectiveExpectedSlotSize()); pb->SetSlotSizeInUnits(slotSizeInUnits); pb->SetNumActiveSlots(pdisk.StaticSlotUsage); } @@ -612,13 +623,14 @@ void TBlobStorageController::UpdateSystemViews() { for (TActorId actorId : info->GetDynamicInfo().ServiceIdForOrderNumber) { const auto& [nodeId, pdiskId, vdiskSlotId] = DecomposeVDiskServiceId(actorId); const TVSlotId vslotId(nodeId, pdiskId, vdiskSlotId); - TGroupDiskInfo disk{nullptr, nullptr, 0}; + TGroupDiskInfo disk{nullptr, nullptr, 0, 0}; if (const auto it = StaticVSlots.find(vslotId); it != StaticVSlots.end()) { disk.VDiskMetrics = it->second.VDiskMetrics ? &*it->second.VDiskMetrics : &zero; } if (const auto it = PDisks.find(vslotId.ComprisingPDiskId()); it != PDisks.end()) { disk.PDiskMetrics = &it->second->Metrics; - disk.ExpectedSlotCount = it->second->ExpectedSlotCount; + disk.ExpectedSlotCount = it->second->GetEffectiveExpectedSlotCount(); + disk.ExpectedSlotSize = it->second->GetEffectiveExpectedSlotSize(); } if (disk.VDiskMetrics && disk.PDiskMetrics) { disks.push_back(std::move(disk)); diff --git a/ydb/core/mind/bscontroller/sys_view.h b/ydb/core/mind/bscontroller/sys_view.h index cfc6439d3d2..88405e8c99e 100644 --- a/ydb/core/mind/bscontroller/sys_view.h +++ b/ydb/core/mind/bscontroller/sys_view.h @@ -55,6 +55,7 @@ struct TGroupDiskInfo { const NKikimrBlobStorage::TPDiskMetrics *PDiskMetrics; const NKikimrBlobStorage::TVDiskMetrics *VDiskMetrics; ui32 ExpectedSlotCount; + ui64 ExpectedSlotSize; }; void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vector<TGroupDiskInfo>& disks, TBlobStorageGroupType type, ui32 groupSizeInUnits); diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp index 7ffd7107469..1b07fd8f1b0 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp +++ b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp @@ -5,6 +5,7 @@ #include <ydb/core/mind/bscontroller/bsc.h> #include <ydb/core/mind/bscontroller/indir.h> #include <ydb/core/mind/bscontroller/impl.h> +#include <ydb/core/mind/bscontroller/sys_view.h> #include <ydb/core/mind/bscontroller/types.h> #include <ydb/core/mind/bscontroller/ut_helpers.h> #include <ydb/core/protos/blobstorage_config.pb.h> @@ -1661,6 +1662,271 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) { }); } + Y_UNIT_TEST(DefineHostConfigValidatesExpectedSlotSettings) { + TEnvironmentSetup env(1, 1); + RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) { + TFinalizer finalizer(env); + env.Prepare(dispatchName, setup, outActiveZone); + + auto invoke = [&](ui64 hostConfigId, auto configurePDiskConfig) { + NKikimrBlobStorage::TConfigRequest request; + auto* cmd = request.AddCommand()->MutableDefineHostConfig(); + cmd->SetHostConfigId(hostConfigId); + cmd->SetName(TStringBuilder() << "DefineHostConfigValidatesExpectedSlotSettings" << hostConfigId); + auto* drive = cmd->AddDrive(); + drive->SetPath(TStringBuilder() << "/dev/disk" << hostConfigId); + drive->SetType(NKikimrBlobStorage::ROT); + configurePDiskConfig(*drive->MutablePDiskConfig()); + return env.Invoke(request); + }; + + auto expectInvalid = [](const NKikimrBlobStorage::TConfigResponse& response, TStringBuf error) { + Cerr << (TStringBuilder() << response.DebugString() << Endl); + UNIT_ASSERT(!response.GetSuccess()); + UNIT_ASSERT_VALUES_EQUAL(response.StatusSize(), 1); + UNIT_ASSERT(!response.GetStatus(0).GetSuccess()); + UNIT_ASSERT_C(response.GetStatus(0).GetErrorDescription().Contains(error), + response.DebugString()); + }; + + expectInvalid(invoke(1, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetExpectedSlotCount(4); + config.SetExpectedSlotSize(100ull << 30); + }), "ExpectedSlotSize is mutually exclusive with ExpectedSlotCount and SlotSizeInUnits"); + + expectInvalid(invoke(2, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetSlotSizeInUnits(4); + config.SetExpectedSlotSize(100ull << 30); + }), "ExpectedSlotSize is mutually exclusive with ExpectedSlotCount and SlotSizeInUnits"); + + expectInvalid(invoke(3, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetExpectedSlotSize(100ull << 30); + }), "ExpectedSlotSize requires MaxSlots"); + + expectInvalid(invoke(6, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetMaxSlots(16); + }), "MaxSlots requires ExpectedSlotSize"); + + NKikimrBlobStorage::TConfigResponse response = invoke(4, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetExpectedSlotSize(100ull << 30); + config.SetMaxSlots(16); + }); + Cerr << (TStringBuilder() << response.DebugString() << Endl); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + UNIT_ASSERT_VALUES_EQUAL(response.StatusSize(), 1); + UNIT_ASSERT(response.GetStatus(0).GetSuccess()); + + response = invoke(5, [](NKikimrBlobStorage::TPDiskConfig& config) { + config.SetExpectedSlotCount(4); + config.SetSlotSizeInUnits(2); + }); + Cerr << (TStringBuilder() << response.DebugString() << Endl); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + UNIT_ASSERT_VALUES_EQUAL(response.StatusSize(), 1); + UNIT_ASSERT(response.GetStatus(0).GetSuccess()); + }); + } + + Y_UNIT_TEST(ExpectedSlotSizeWithoutSlotCountMetricsKeepsZeroSlotCount) { + NKikimrBlobStorage::TPDiskConfig config; + config.SetExpectedSlotSize(1ull << 30); + config.SetMaxSlots(16); + + TString serializedConfig; + UNIT_ASSERT(config.SerializeToString(&serializedConfig)); + + using TPDiskInfo = TBlobStorageController::TPDiskInfo; + using TPDiskTable = TPDiskInfo::Table; + + TPDiskInfo pdisk( + TBlobStorageController::THostId(TString("host"), 1), + TString("/dev/disk"), + 0, + 1, + TMaybe<TPDiskTable::SharedWithOs::Type>(), + TMaybe<TPDiskTable::ReadCentric::Type>(), + std::nullopt, + 1, + serializedConfig, + 0, + 16, + NKikimrBlobStorage::EDriveStatus::ACTIVE, + TInstant::Zero(), + NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, + TPDiskMood::Normal, + TString(), + TString(), + TString(), + 0, + true, + NKikimrBlobStorage::TMaintenanceStatus::NO_REQUEST); + + ui32 slotCount = Max<ui32>(); + ui32 slotSizeInUnits = Max<ui32>(); + UNIT_ASSERT(!pdisk.Metrics.HasTotalSize()); + pdisk.ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); + UNIT_ASSERT_VALUES_EQUAL(slotCount, 0); + UNIT_ASSERT_VALUES_EQUAL(slotSizeInUnits, 0); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotSize(), 1ull << 30); + + pdisk.Metrics.SetTotalSize(2400ull << 30); + pdisk.ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); + UNIT_ASSERT_VALUES_EQUAL(slotCount, 0); + UNIT_ASSERT_VALUES_EQUAL(slotSizeInUnits, 0); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotCount(), 0); + + pdisk.Metrics.SetSlotCount(64); + pdisk.ExtractInferredPDiskSettings(slotCount, slotSizeInUnits); + UNIT_ASSERT_VALUES_EQUAL(slotCount, 64); + UNIT_ASSERT_VALUES_EQUAL(slotSizeInUnits, 0); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotCount(), 64); + } + + Y_UNIT_TEST(GroupUsagePrefersExpectedSlotSizeOverDynamicSlotSize) { + NKikimrBlobStorage::TPDiskMetrics pdiskMetrics; + pdiskMetrics.SetEnforcedDynamicSlotSize(1000); + pdiskMetrics.SetSlotSizeInUnits(1); + + NKikimrBlobStorage::TVDiskMetrics vdiskMetrics; + vdiskMetrics.SetAllocatedSize(25); + + NKikimrSysView::TGroupInfo info; + CalculateGroupUsageStats( + &info, + {{&pdiskMetrics, &vdiskMetrics, 10, 100}}, + TBlobStorageGroupType(TBlobStorageGroupType::ErasureNone), + 2); + + UNIT_ASSERT_VALUES_EQUAL(info.GetAllocatedSize(), 25); + UNIT_ASSERT_VALUES_EQUAL(info.GetAvailableSize(), 75); + } + + Y_UNIT_TEST(ZeroExpectedSlotSizeDoesNotDisableDefaultSlotCount) { + NKikimrBlobStorage::TPDiskConfig config; + config.SetExpectedSlotSize(0); + + TString serializedConfig; + UNIT_ASSERT(config.SerializeToString(&serializedConfig)); + + using TPDiskInfo = TBlobStorageController::TPDiskInfo; + using TPDiskTable = TPDiskInfo::Table; + + TPDiskInfo pdisk( + TBlobStorageController::THostId(TString("host"), 1), + TString("/dev/disk"), + 0, + 1, + TMaybe<TPDiskTable::SharedWithOs::Type>(), + TMaybe<TPDiskTable::ReadCentric::Type>(), + std::nullopt, + 1, + serializedConfig, + 0, + 16, + NKikimrBlobStorage::EDriveStatus::ACTIVE, + TInstant::Zero(), + NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE, + TPDiskMood::Normal, + TString(), + TString(), + TString(), + 0, + true, + NKikimrBlobStorage::TMaintenanceStatus::NO_REQUEST); + + UNIT_ASSERT(!pdisk.HasExpectedSlotSize); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotSize(), 0); + UNIT_ASSERT_VALUES_EQUAL(pdisk.GetEffectiveExpectedSlotCount(), 16); + } + + Y_UNIT_TEST(NumActiveSlotsStaysConsistentWhenExpectedSlotSizeMetricsArrive) { + // NumActiveSlots is maintained incrementally with the owner weight computed at the + // moment a vslot is added or removed. The weight depends on the *effective* expected + // slot size, which flips from 0 to nonzero when the PDisk starts reporting + // ExpectedSlotSize in its metrics (the infer_pdisk_slot_count.<type>.slot_size case, + // where PDiskConfig itself carries no ExpectedSlotSize). The counter must be kept + // consistent with the new weights when that happens. + TEnvironmentSetup env(1, 1); + RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) { + TFinalizer finalizer(env); + env.Prepare(dispatchName, setup, outActiveZone); + + constexpr ui64 expectedSlotSize = 100ull << 30; + constexpr ui32 slotCount = 4; + + // A box with a single ROT drive without any explicit PDiskConfig. + NKikimrBlobStorage::TConfigRequest request; + env.DefineBox(1, "box", { + {"/dev/disk1", NKikimrBlobStorage::ROT, false, false, 0}, + }, env.GetNodes(), request); + + // Pool A: one group with GroupSizeInUnits=2. Its single vslot is accounted + // in NumActiveSlots with weight ceil(2/1) = 2 since no metrics arrived yet. + env.DefineStoragePool(1, 1, "pool-a", 1, NKikimrBlobStorage::ROT, {}, request, "none"); + request.MutableCommand(request.CommandSize() - 1)->MutableDefineStoragePool() + ->SetDefaultGroupSizeInUnits(2); + + const size_t baseConfigIndex = request.CommandSize(); + request.AddCommand()->MutableQueryBaseConfig(); + + NKikimrBlobStorage::TConfigResponse response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + + const auto& baseConfig = response.GetStatus(baseConfigIndex).GetBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(baseConfig.PDiskSize(), 1); + const ui32 pdiskNodeId = baseConfig.GetPDisk(0).GetNodeId(); + const ui32 pdiskId = baseConfig.GetPDisk(0).GetPDiskId(); + UNIT_ASSERT_VALUES_EQUAL(pdiskNodeId, env.Runtime->GetNodeId(0)); + + // The PDisk starts reporting ExpectedSlotSize and the materialized SlotCount in + // metrics, as it does when NodeWarden infers the slot count from a slot size. + // The effective expected slot size becomes nonzero and the owner weight of the + // already existing vslot flips from 2 to 1. BSC accepts disk status updates + // only over the pipe that carried TEvControllerRegisterNode of the same node, + // so register the node and send the metrics through one pipe. + { + const TActorId sender = env.Runtime->AllocateEdgeActor(0); + const TActorId pipeClient = env.Runtime->ConnectToPipe(env.TabletId, sender, 0, + GetPipeConfigWithRetries()); + env.Runtime->SendToPipe(pipeClient, sender, new TEvBlobStorage::TEvControllerRegisterNode( + pdiskNodeId, TVector<ui32>{}, TVector<ui32>{}, TVector<NPDisk::TDriveData>{})); + env.Runtime->GrabEdgeEventRethrow<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(sender); + + auto ev = MakeHolder<TEvBlobStorage::TEvControllerUpdateDiskStatus>(); + auto* m = ev->Record.AddPDisksMetrics(); + m->SetPDiskId(pdiskId); + m->SetSlotCount(slotCount); + m->SetExpectedSlotSize(expectedSlotSize); + env.Runtime->SendToPipe(pipeClient, sender, ev.Release()); + } + + // Make sure the metrics have been applied before proceeding. + { + NKikimrBlobStorage::TConfigRequest sync; + sync.AddCommand()->MutableQueryBaseConfig(); + NKikimrBlobStorage::TConfigResponse syncResponse = env.Invoke(sync); + UNIT_ASSERT_C(syncResponse.GetSuccess(), syncResponse.GetErrorDescription()); + const auto& syncBaseConfig = syncResponse.GetStatus(0).GetBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(syncBaseConfig.PDiskSize(), 1); + const auto& syncPDisk = syncBaseConfig.GetPDisk(0); + UNIT_ASSERT_VALUES_EQUAL(syncPDisk.GetPDiskMetrics().GetExpectedSlotSize(), expectedSlotSize); + UNIT_ASSERT_VALUES_EQUAL(syncPDisk.GetExpectedSlotCount(), slotCount); + UNIT_ASSERT_VALUES_EQUAL(syncPDisk.GetExpectedSlotSize(), expectedSlotSize); + } + + // The disk now has 4 fixed-size slots and the PDisk accounts the existing + // 2-unit group as a single owner, so exactly 3 more single-unit groups must + // fit. With a stale NumActiveSlots (still 2) the third group does not fit. + { + NKikimrBlobStorage::TConfigRequest more; + env.DefineStoragePool(1, 2, "pool-b", slotCount - 1, NKikimrBlobStorage::ROT, {}, more, "none"); + NKikimrBlobStorage::TConfigResponse moreResponse = env.Invoke(more); + UNIT_ASSERT_C(moreResponse.GetSuccess(), moreResponse.GetErrorDescription()); + } + }); + } + Y_UNIT_TEST(UnsupportedCommandError) { TEnvironmentSetup env(1, 1); RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) { diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index e71178384a4..ad1d1c15efe 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -9736,7 +9736,6 @@ Y_UNIT_TEST_SUITE(THiveTest) { } Y_UNIT_TEST(TestLockedTabletVolatileStateDependsOnMetricsFlag) { - return; // https://github.com/ydb-platform/ydb/issues/46172 for (bool lockedTabletsSendMetrics : {false, true}) { const ui64 hiveTablet = MakeDefaultHiveID(); const ui64 testerTablet = MakeTabletID(false, 1); @@ -9783,7 +9782,12 @@ Y_UNIT_TEST_SUITE(THiveTest) { : NKikimrHive::ETabletVolatileState::TABLET_VOLATILE_STATE_STOPPED, tabletState(tabletIdExt)); - RebootTablet(runtime, hiveTablet, runtime.AllocateEdgeActor(0)); + runtime.Register(CreateTabletKiller(hiveTablet)); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvStatus, runtime.GetNodeCount()); + runtime.DispatchEvents(options); + } SendLockTabletExecution(runtime, hiveTablet, tabletIdExt, 0, NKikimrProto::OK, owner, 60000, true); diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index d4c06eb6444..7d268747719 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -111,38 +111,10 @@ public: return !NeedToReleaseFromParent && State == ETabletState::ReadyToWork && !IsBootingSuppressed(); } - bool IsReadyToBoot() const { - return IsReadyToWork() && TTabletInfo::IsReadyToBoot(); - } - - bool IsReadyToStart(TInstant now) const { - return IsReadyToWork() && TTabletInfo::IsReadyToStart(now); - } - bool IsReadyToBlockStorage() const { return State == ETabletState::BlockStorage; } - bool IsStarting() const { - return IsReadyToWork() && TTabletInfo::IsStarting(); - } - - bool IsStartingOnNode(TNodeId nodeId) const { - return IsReadyToWork() && TTabletInfo::IsStartingOnNode(nodeId); - } - - bool IsRunning() const { - return IsReadyToWork() && TTabletInfo::IsRunning(); - } - - bool IsAlive() const { - return IsReadyToWork() && TTabletInfo::IsAlive(); - } - - bool IsAliveOnLocal(const TActorId& local) const { - return IsReadyToWork() && TTabletInfo::IsAliveOnLocal(local); - } - bool IsDeleting() const { return State == ETabletState::Deleting; } diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 55ec6e8442a..9fa41bcd477 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -150,17 +150,28 @@ void TTabletInfo::ChangeVolatileState(EVolatileState state) { VolatileStateChangeTime = TActivationContext::Now(); } +bool TTabletInfo::IsReadyToWork() const { + if (IsLeader()) { + return AsLeader().IsReadyToWork(); + } + return true; +} + +bool TTabletInfo::IsReadyToBoot() const { + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; +} + bool TTabletInfo::IsReadyToStart(TInstant now) const { if (IsFollower()) { if (!GetLeader().IsRunning()) { return false; } } - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; } bool TTabletInfo::IsStarting() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; } bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { @@ -168,28 +179,31 @@ bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { } bool TTabletInfo::IsRunning() const { - return Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; + return IsReadyToWork() && Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; } bool TTabletInfo::IsBooting() const { - return VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; + return IsReadyToWork() && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; } bool TTabletInfo::IsAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING); } bool TTabletInfo::CanBeAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_UNKNOWN); // KIKIMR-12558 } bool TTabletInfo::IsAliveOnLocal(const TActorId& local) const { - return Node != nullptr + return IsReadyToWork() + && Node != nullptr && Node->Local == local && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index c88c635e08a..2edbab742dd 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -204,10 +204,8 @@ public: TString FamilyString() const; void ChangeVolatileState(EVolatileState state); - bool IsReadyToBoot() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; - } - + bool IsReadyToWork() const; + bool IsReadyToBoot() const; bool IsReadyToStart(TInstant now) const; bool IsStarting() const; bool IsStartingOnNode(TNodeId nodeId) const; diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index 802fb89fb2e..d69f16cd2dd 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -1841,6 +1841,7 @@ std::future<void> TMon::Start(TActorSystem* actorSystem) { addPort->CertificateFile = Config.CertificateFile; addPort->PrivateKeyFile = Config.PrivateKeyFile; addPort->CaFile = Config.CaFile; + addPort->ClientCertificateRequired = Config.ClientCertificateRequired; addPort->Secure = !Config.Certificate.empty() || !Config.CertificateFile.empty(); addPort->MaxRequestsPerSecond = Config.MaxRequestsPerSecond; diff --git a/ydb/core/mon/mon.h b/ydb/core/mon/mon.h index 04152a18c31..ad2749934d0 100644 --- a/ydb/core/mon/mon.h +++ b/ydb/core/mon/mon.h @@ -51,6 +51,7 @@ public: TString CertificateFile; // certificate file path in PEM format (OpenSSL feature: may optionally contain both certificate chain and private key in the same PEM file if PrivateKeyFile is not set) TString PrivateKeyFile; // private key file path for the certificate in PEM format TString CaFile; // CA certificate file path for verifying client certificates (mTLS) + bool ClientCertificateRequired = false; ui32 MaxRequestsPerSecond = 0; TDuration InactivityTimeout = TDuration::Minutes(2); TString AllowOrigin; diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp new file mode 100644 index 00000000000..fc45a2b3871 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp @@ -0,0 +1,104 @@ +#include "block_range_field.h" + +namespace NYdb::NBS::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +void TBlockRangeField::Add(TBlockRange64 range) +{ + // Non-overlapping ranges sorted by End are also sorted by Start, so we + // can iterate forward and stop early. + + // Find first existing interval with End >= range.Start - 1 (adjacent or + // overlapping on the left side of the new range). + // When range.Start == 0, "range.Start - 1" would underflow → start from + // begin() to cover all intervals. + auto it = (range.Start > 0) + ? Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, range.Start - 1)) + : Intervals.begin(); + + ui64 mergedStart = range.Start; + ui64 mergedEnd = range.End; + + while (it != Intervals.end()) { + // For non-overlapping ranges sorted by End (= sorted by Start), we can + // stop when the next interval starts strictly after mergedEnd + 1. + // Guard against overflow when mergedEnd == MaxIndex: in that case every + // possible Start is <= mergedEnd, so no early exit is possible. + if (mergedEnd != TBlockRange64::MaxIndex && it->Start > mergedEnd + 1) { + break; + } + + mergedStart = Min(mergedStart, it->Start); + mergedEnd = Max(mergedEnd, it->End); + it = Intervals.erase(it); + } + + Intervals.insert(TBlockRange64::MakeClosedInterval(mergedStart, mergedEnd)); +} + +void TBlockRangeField::Remove(TBlockRange64 range) +{ + if (Intervals.empty()) { + return; + } + + // Find first interval with End >= range.Start (could overlap with range). + auto it = Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, range.Start)); + + while (it != Intervals.end()) { + // Since Start is monotonically increasing (non-overlapping + sorted by + // End), stop once Start is past range.End. + if (it->Start > range.End) { + break; + } + + const TBlockRange64 existing = *it; + it = Intervals.erase(it); + + // Keep the left tail if the existing interval starts before + // range.Start. + if (existing.Start < range.Start) { + Intervals.insert(TBlockRange64::MakeClosedInterval( + existing.Start, + range.Start - 1)); + } + + // Keep the right tail if the existing interval ends after range.End. + if (existing.End > range.End) { + Intervals.insert( + TBlockRange64::MakeClosedInterval(range.End + 1, existing.End)); + break; + } + } +} + +bool TBlockRangeField::Overlaps(TBlockRange64 other) const +{ + if (Intervals.empty()) { + return false; + } + + // First interval with End >= other.Start. + auto it = Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, other.Start)); + + if (it == Intervals.end()) { + return false; + } + + return it->Overlaps(other); +} + +void TBlockRangeField::Enumerate(TEnumerateFunc func) const +{ + for (const auto& range: Intervals) { + func(range); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h new file mode 100644 index 00000000000..2ca4f886a41 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h @@ -0,0 +1,39 @@ +#pragma once + +#include "public.h" + +#include "block_range.h" + +#include <util/generic/set.h> + +namespace NYdb::NBS::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +class TBlockRangeField +{ +public: + using TEnumerateFunc = std::function<void(TBlockRange64 item)>; + + void Add(TBlockRange64 range); + void Remove(TBlockRange64 range); + + [[nodiscard]] bool Overlaps(TBlockRange64 other) const; + + void Enumerate(TEnumerateFunc func) const; + +private: + struct TBlockRangeComparator + { + bool operator()(TBlockRange64 a, TBlockRange64 b) const + { + return a.End < b.End; + } + }; + + TSet<TBlockRange64, TBlockRangeComparator> Intervals; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp new file mode 100644 index 00000000000..cb848e43b97 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp @@ -0,0 +1,323 @@ +#include "block_range_field.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/vector.h> + +namespace NYdb::NBS::NBlockStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TVector<TBlockRange64> Collect(const TBlockRangeField& field) +{ + TVector<TBlockRange64> result; + field.Enumerate([&](TBlockRange64 r) { result.push_back(r); }); + return result; +} + +TBlockRange64 R(ui64 start, ui64 end) +{ + return TBlockRange64::MakeClosedInterval(start, end); +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TBlockRangeFieldTest) +{ + // ------------------------------------------------------------------------- + // Add – basic + + Y_UNIT_TEST(AddSingleRange) + { + TBlockRangeField f; + f.Add(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(AddTwoNonAdjacentRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 5), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(10, 15), v[1]); + } + + Y_UNIT_TEST(AddAdjacentRangesMerged) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(6, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 10), v[0]); + } + + Y_UNIT_TEST(AddOverlappingRangesMerged) + { + TBlockRangeField f; + f.Add(R(0, 10)); + f.Add(R(5, 15)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 15), v[0]); + } + + Y_UNIT_TEST(AddCoveredByExistingIsNoop) + { + TBlockRangeField f; + f.Add(R(0, 100)); + f.Add(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 100), v[0]); + } + + Y_UNIT_TEST(AddCoversMultipleRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + f.Add(R(20, 25)); + // New range covers all three and the gaps between them. + f.Add(R(0, 25)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 25), v[0]); + } + + Y_UNIT_TEST(AddMergesOnBothSides) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + // Bridge the gap. + f.Add(R(5, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 15), v[0]); + } + + Y_UNIT_TEST(AddSameRangeTwice) + { + TBlockRangeField f; + f.Add(R(3, 7)); + f.Add(R(3, 7)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(3, 7), v[0]); + } + + // ------------------------------------------------------------------------- + // Remove – basic + + Y_UNIT_TEST(RemoveFromEmpty) + { + TBlockRangeField f; + f.Remove(R(0, 10)); // must not crash + UNIT_ASSERT(Collect(f).empty()); + } + + Y_UNIT_TEST(RemoveExact) + { + TBlockRangeField f; + f.Add(R(0, 10)); + f.Remove(R(0, 10)); + UNIT_ASSERT(Collect(f).empty()); + } + + Y_UNIT_TEST(RemoveFromMiddle) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(5, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 4), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(11, 20), v[1]); + } + + Y_UNIT_TEST(RemoveLeftPart) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(0, 9)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(RemoveRightPart) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 9), v[0]); + } + + Y_UNIT_TEST(RemoveNonOverlapping) + { + TBlockRangeField f; + f.Add(R(10, 20)); + f.Remove(R(30, 40)); // no overlap, no change + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(RemoveSeveralRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + f.Add(R(20, 25)); + f.Remove(R(3, 22)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 2), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(23, 25), v[1]); + } + + // ------------------------------------------------------------------------- + // Overlaps + + Y_UNIT_TEST(OverlapsOnEmpty) + { + TBlockRangeField f; + UNIT_ASSERT(!f.Overlaps(R(0, 100))); + } + + Y_UNIT_TEST(OverlapsExact) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(10, 20))); + } + + Y_UNIT_TEST(OverlapsPartialLeft) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(5, 12))); + } + + Y_UNIT_TEST(OverlapsPartialRight) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(15, 30))); + } + + Y_UNIT_TEST(OverlapsCovering) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(0, 100))); + } + + Y_UNIT_TEST(OverlapsNoOverlapBefore) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(!f.Overlaps(R(0, 9))); + } + + Y_UNIT_TEST(OverlapsNoOverlapAfter) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(!f.Overlaps(R(21, 30))); + } + + Y_UNIT_TEST(OverlapsAdjacentNotOverlapping) + { + TBlockRangeField f; + f.Add(R(10, 20)); + // [9,9] touches start but doesn't overlap. + UNIT_ASSERT(!f.Overlaps(R(5, 9))); + // [21,21] touches end but doesn't overlap. + UNIT_ASSERT(!f.Overlaps(R(21, 25))); + } + + // ------------------------------------------------------------------------- + // Edge / boundary cases + + Y_UNIT_TEST(AddStartingAtZero) + { + TBlockRangeField f; + f.Add(R(0, 0)); + f.Add(R(1, 5)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 5), v[0]); + } + + Y_UNIT_TEST(RemoveSingleBlock) + { + TBlockRangeField f; + f.Add(R(0, 4)); + f.Remove(R(2, 2)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 1), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(3, 4), v[1]); + } + + Y_UNIT_TEST(ManyFragmentsAfterRemoves) + { + TBlockRangeField f; + f.Add(R(0, 99)); + // Remove every even block to create 50 gaps. + for (ui64 i = 0; i < 100; i += 2) { + f.Remove(R(i, i)); + } + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(50u, v.size()); + for (ui64 i = 0; i < 50; ++i) { + ui64 odd = i * 2 + 1; + UNIT_ASSERT_VALUES_EQUAL(R(odd, odd), v[i]); + } + } + + Y_UNIT_TEST(AddRestoresAfterRemoves) + { + TBlockRangeField f; + f.Add(R(0, 99)); + for (ui64 i = 0; i < 100; i += 2) { + f.Remove(R(i, i)); + } + // Adding back should merge everything. + for (ui64 i = 0; i < 100; i += 2) { + f.Add(R(i, i)); + } + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 99), v[0]); + } + + Y_UNIT_TEST(EnumerateOrderedByStart) + { + TBlockRangeField f; + f.Add(R(50, 60)); + f.Add(R(10, 20)); + f.Add(R(30, 40)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(3u, v.size()); + UNIT_ASSERT(v[0].Start < v[1].Start); + UNIT_ASSERT(v[1].Start < v[2].Start); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make b/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make index 74e5274eeec..8140f03c6d5 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make @@ -4,6 +4,7 @@ INCLUDE(${ARCADIA_ROOT}/ydb/core/nbs/cloud/storage/core/tests/recipes/small.inc) SRCS( block_range_algorithms_ut.cpp + block_range_field_ut.cpp block_range_map_ut.cpp block_range_ut.cpp diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/ya.make b/ydb/core/nbs/cloud/blockstore/libs/common/ya.make index e94dec6092d..6c7742a0e16 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/common/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/common/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( block_range_algorithms.cpp + block_range_field.cpp block_range_map.cpp block_range.cpp thread_checker.cpp diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ddisk_data_copier.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ddisk_data_copier.cpp index 6a256c23287..f1b861bd8ad 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ddisk_data_copier.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ddisk_data_copier.cpp @@ -150,7 +150,17 @@ void TDDiskDataCopier::StartCopyRange() DirtyMap->SetFlushWatermark(Destination, futureWatermark); auto readHint = DirtyMap->MakeReadHint(range); - Y_ABORT_UNLESS(!readHint.RangeHints.empty()); + if (readHint.RangeHints.empty()) { + auto waitReadyFuture = readHint.WaitReady; + Y_ABORT_UNLESS(!waitReadyFuture.HasValue()); + waitReadyFuture.Subscribe( + [self = shared_from_this()](const NThreading::TFuture<void>& f) + { + Y_UNUSED(f); + self->StartCopyRange(); + }); + return; + } const ui64 requestId = Random(); std::shared_ptr<TReadBlocksLocalRequest> readRequest = diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp index c124311c3d5..f56d55568fb 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp @@ -112,14 +112,18 @@ CreateWaitSessionCbForSyncWithPBuffer( } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->SyncWithPBuffer( vChunkIndex, pbufferHostIndex, ddiskHostIndex, segments, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGFlushResponseFuture& f) mutable { promise.SetValue(f.GetValue()); }); @@ -127,8 +131,8 @@ CreateWaitSessionCbForSyncWithPBuffer( for (size_t i = 0; i < segments.size(); ++i) { flushResponse.Errors.push_back(MakeError(E_CANCELLED)); } + promise.SetValue(std::move(flushResponse)); } - promise.SetValue(std::move(flushResponse)); }; return cb; @@ -315,14 +319,18 @@ TDirectBlockGroup::ReadBlocksFromDDisk( } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->ReadBlocksFromDDisk( vChunkIndex, hostIndex, range, guardedSglist, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGReadBlocksResponseFuture& f) mutable { promise.SetValue(UnsafeExtractValue(f)); }); @@ -519,14 +527,18 @@ TDirectBlockGroup::WriteBlocksToDDisk( return; } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->WriteBlocksToDDisk( vChunkIndex, hostIndex, range, guardedSglist, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGWriteBlocksResponseFuture& f) mutable { promise.SetValue(f.GetValue()); }); @@ -1211,9 +1223,11 @@ NThreading::TFuture<TListPBufferResponse> TDirectBlockGroup::ListPBuffers( } const auto& connection = PBufferConnections[hostIndex]; + // Hold a local copy of the connect future, + // do not put an address of changeable field into a wait. + auto connectFuture = connection.GetFuture(); // Switch co-routine context if needed. - const NProto::TError& connectError = - Executor->WaitFor(connection.GetFuture()); + const NProto::TError& connectError = Executor->WaitFor(connectFuture); if (HasError(connectError)) { return MakeFuture(TListPBufferResponse{.Error = connectError}); } @@ -1551,18 +1565,16 @@ void TDirectBlockGroup::OnConnectionEstablished( // Unblock waiters on ConnectFuture with the error. connection.ConnectPromise.SetValue(error); return; - // TODO (future phase): handle non-BLOCKED connect errors - // (ERROR/unavailability) via reconnect. - } else if (IsInitialized()) { + } else { LOG_ERROR( *ActorSystem, NKikimrServices::NBS_PARTITION, - "%s connection failed for %s (post-init): %s", + "%s connection failed %s: %s", LogTitle.GetWithTime().c_str(), PrintHostIndex(hostIndex).c_str(), FormatError(error).c_str()); - } else { - Y_ABORT("Unhandled branch of connect error"); + ReEstablishConnection(connectionType, hostIndex); + return; } // ConnectPromise resolves both "connection ready" and "session ready" in @@ -1579,12 +1591,16 @@ void TDirectBlockGroup::OnConnectionEstablished( } } -void TDirectBlockGroup::ReEstablishDDiskConnection( - THostIndex hostIndex, - TDuration reconnectDelay) +void TDirectBlockGroup::ReEstablishConnection( + EConnectionType connectionType, + THostIndex hostIndex) { Y_ABORT_UNLESS(ExecutorThreadChecker.Check()); - Y_ABORT_UNLESS(hostIndex < DDiskConnections.size()); + auto& connections = connectionType == EConnectionType::DDisk + ? DDiskConnections + : PBufferConnections; + Y_ABORT_UNLESS(hostIndex < connections.size()); + TDDiskConnection& connection = connections[hostIndex]; if (BlockedGenerationDetected) { LOG_WARN( @@ -1595,13 +1611,14 @@ void TDirectBlockGroup::ReEstablishDDiskConnection( return; } - DDiskConnections[hostIndex].ResetSession(); + connection.ResetSession(); + TDuration reconnectDelay = Oracle.GetHostReconnectDelay(hostIndex); Schedule( reconnectDelay, - [hostIndex, weakSelf = weak_from_this()]() + [hostIndex, weakSelf = weak_from_this(), connectionType]() { if (auto self = weakSelf.lock()) { - self->DoEstablishConnection(hostIndex, EConnectionType::DDisk); + self->DoEstablishConnection(hostIndex, connectionType); } }); } @@ -1619,9 +1636,9 @@ void TDirectBlockGroup::OnNodeDisconnected(THostIndex hostIndex, ui32 nodeId) nodeId); Oracle.OnDDiskDisconnected(hostIndex, TInstant::Now()); + // OnNodeDisconnected may be called only for DDisk - TDuration reconnectDelay = Oracle.GetDDiskReconnectDelay(hostIndex); - ReEstablishDDiskConnection(hostIndex, reconnectDelay); + ReEstablishConnection(EConnectionType::DDisk, hostIndex); } bool TDirectBlockGroup::HasPBufferQuorum() const diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h index 602c15a9f72..366615e0718 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h @@ -194,9 +194,9 @@ private: THostIndex hostIndex, ui64 seqNo, const NKikimrBlobStorage::NDDisk::TEvConnectResult& result); - void ReEstablishDDiskConnection( - THostIndex hostIndex, - TDuration reconnectDelay); + void ReEstablishConnection( + EConnectionType connectionType, + THostIndex hostIndex); void OnNodeDisconnected(THostIndex hostIndex, ui32 nodeId); [[nodiscard]] bool HasPBufferQuorum() const; diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp index 79af0b61f78..2ce680d6282 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp @@ -979,6 +979,199 @@ Y_UNIT_TEST_SUITE(TDirectBlockGroupTest) UNIT_ASSERT_VALUES_EQUAL(connectsBefore, connectsAfter); } + // A non-BLOCKED connect failure on DDisk (e.g. ERROR/unavailable) must NOT + // be terminal: the session is reset and a fresh Connect is retried with the + // next seq_no. + Y_UNIT_TEST_F(ShouldReconnectDDiskOnNonBlockedConnectError, TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& ddisks = transportPtr->GetDDiskIds(); + // host[0] connect is deferred; hosts 1..4 connect immediately -> the + // quorum (3 of 5) is reached without host[0]. + auto pendingConnect0 = + transportPtr->SetPendingConnect(EConnectionType::DDisk, ddisks[0]); + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + WaitReady(executor, initialReady); + + // Only the first (initial) connect attempt was issued for host[0]. + const size_t connectsBefore = + transportPtr + ->GetConnectCredentials(EConnectionType::DDisk, ddisks[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(1, connectsBefore); + + // Arm the next (reconnect) Connect for host[0] before delivering the + // failure. + auto reconnectPending = + transportPtr->SetPendingConnect(EConnectionType::DDisk, ddisks[0]); + + // Resolve the in-flight connect with a non-BLOCKED error. + pendingConnect0.SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + + // Drain OnConnectionEstablished (queues the reconnect) then the + // reconnect itself (issues a fresh Connect). + DrainExecutor(executor); + DrainExecutor(executor); + + // A reconnect Connect was issued and carries the next seq_no (=2). + const auto credentials = transportPtr->GetConnectCredentials( + EConnectionType::DDisk, + ddisks[0]); + UNIT_ASSERT_VALUES_EQUAL(connectsBefore + 1, credentials.size()); + UNIT_ASSERT_VALUES_EQUAL(2, credentials.back().DDiskSessionSeqNo); + + // No suicide on a non-BLOCKED connect error. + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + { + const auto state = + GetBlockedDetected(executor, dbg, 0, WaitTimeout); + UNIT_ASSERT(!state.DDiskSessionBroken); + UNIT_ASSERT(!state.BlockedGenerationDetected); + } + + // The retry succeeds -> the session is confirmed at seq_no=2. + reconnectPending.SetValue(TStorageTransportMock::MakeConnectResult()); + DrainExecutor(executor); + + const auto seqNos = ReadAllDDiskSeqNos(executor, dbg, WaitTimeout); + UNIT_ASSERT_VALUES_EQUAL(2, seqNos[0]); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + + // A non-BLOCKED connect failure on PBuffer is retried the same way. PBuffer + // has no session/lock, so only the reconnect (a fresh Connect) is asserted, + // and no suicide happens. + Y_UNIT_TEST_F(ShouldReconnectPBufferOnNonBlockedConnectError, TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& pbuffers = transportPtr->GetPBufferIds(); + // PBuffer[0] connect is deferred; the rest connect immediately -> the + // PBuffer quorum is reached without PBuffer[0]. All DDisks connect + // immediately -> locked quorum is reached too. + auto pendingConnect0 = transportPtr->SetPendingConnect( + EConnectionType::PBuffer, + pbuffers[0]); + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + WaitReady(executor, initialReady); + + // Only the first (initial) connect attempt was issued for PBuffer[0]. + const size_t connectsBefore = + transportPtr + ->GetConnectCredentials(EConnectionType::PBuffer, pbuffers[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(1, connectsBefore); + + // Arm the next (reconnect) Connect for PBuffer[0] before delivering the + // failure. + auto reconnectPending = transportPtr->SetPendingConnect( + EConnectionType::PBuffer, + pbuffers[0]); + + // Resolve the in-flight PBuffer connect with a non-BLOCKED error. + pendingConnect0.SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + + // Drain OnConnectionEstablished (queues the reconnect) then the + // reconnect itself (issues a fresh Connect). + DrainExecutor(executor); + DrainExecutor(executor); + + // A reconnect Connect was issued for PBuffer[0]. + const size_t connectsAfter = + transportPtr + ->GetConnectCredentials(EConnectionType::PBuffer, pbuffers[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(connectsBefore + 1, connectsAfter); + + // A PBuffer connect error never triggers a suicide. + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + + // The retry succeeds without any crash/suicide. + reconnectPending.SetValue(TStorageTransportMock::MakeConnectResult()); + DrainExecutor(executor); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + + // Startup must not fail permanently on a non-BLOCKED connect + // error: initial-ready stays unresolved while the DDisk connects keep + // failing, and resolves once the retried connects reach the locked quorum. + Y_UNIT_TEST_F( + ShouldReachInitialReadyAfterTransientConnectError, + TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& ddisks = transportPtr->GetDDiskIds(); + // Defer every DDisk connect so we control when the quorum is reached. + // PBuffers connect immediately -> PBuffer quorum is satisfied. + TVector<TStorageTransportMock::TConnectPromise> firstConnects; + for (const auto& ddiskId: ddisks) { + firstConnects.push_back(transportPtr->SetPendingConnect( + EConnectionType::DDisk, + ddiskId)); + } + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + + DrainExecutor(executor); + UNIT_ASSERT(!initialReady.HasValue()); + + // Arm the reconnect Connect for the first three DDisks (the quorum). + constexpr size_t quorum = QuorumDirectBlockGroupHostCount; + TVector<TStorageTransportMock::TConnectPromise> reconnects; + for (size_t i = 0; i < quorum; ++i) { + reconnects.push_back(transportPtr->SetPendingConnect( + EConnectionType::DDisk, + ddisks[i])); + } + + // First attempt fails on all three with a non-BLOCKED error. + for (size_t i = 0; i < quorum; ++i) { + firstConnects[i].SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + } + // Drain OnConnectionEstablished + queued reconnects, then the + // reconnects. + DrainExecutor(executor); + DrainExecutor(executor); + + // Still not ready: no DDisk session is locked yet. + UNIT_ASSERT(!initialReady.HasValue()); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + + // The retried connects succeed -> the locked quorum is reached and + // startup completes. + for (auto& reconnect: reconnects) { + reconnect.SetValue(TStorageTransportMock::MakeConnectResult()); + } + WaitReady(executor, initialReady); + + // The recovered sessions are confirmed at seq_no=2 (attempt + retry), + // and no suicide happened. + const auto seqNos = ReadAllDDiskSeqNos(executor, dbg, WaitTimeout); + for (size_t i = 0; i < quorum; ++i) { + UNIT_ASSERT_VALUES_EQUAL(2, seqNos[i]); + } + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + // A PBuffer error must NOT trigger the suicide: the BLOCKED check is // only wired into DDisk-addressed responses. Y_UNIT_TEST_F(ShouldNotSuicideOnPBufferError, TDBGFixture) diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp index 4e0ce88de08..bf0941a86ce 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp @@ -38,7 +38,7 @@ void TOracleMock::OnDDiskDisconnected(THostIndex hostIndex, TInstant now) Y_UNUSED(hostIndex, now); } -TDuration TOracleMock::GetDDiskReconnectDelay(THostIndex hostIndex) +TDuration TOracleMock::GetHostReconnectDelay(THostIndex hostIndex) { Y_UNUSED(hostIndex); return TDuration::MilliSeconds(1); diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h index c9d535839ed..214edafc69f 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h @@ -41,7 +41,7 @@ struct TOracleMock: public IOracle void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) override; void OnDDiskConnected(THostIndex hostIndex, TInstant now) override; - TDuration GetDDiskReconnectDelay(THostIndex hostIndex) override; + TDuration GetHostReconnectDelay(THostIndex hostIndex) override; [[nodiscard]] THostIndex SelectBestPBufferHost( THostMask hosts, diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp index d79c836bf0f..c29e14c7ce7 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp @@ -285,7 +285,7 @@ void TOracle::OnDDiskConnected(THostIndex hostIndex, TInstant now) HostsReconnectDelays[hostIndex].Reset(); } -TDuration TOracle::GetDDiskReconnectDelay(THostIndex hostIndex) +TDuration TOracle::GetHostReconnectDelay(THostIndex hostIndex) { return HostsReconnectDelays[hostIndex].GetDelayAndIncrease(); } diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h index 4e68e4eb54d..b78a53f216c 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h @@ -71,7 +71,7 @@ public: virtual void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) = 0; virtual void OnDDiskConnected(THostIndex hostIndex, TInstant now) = 0; - virtual TDuration GetDDiskReconnectDelay(THostIndex hostIndex) = 0; + virtual TDuration GetHostReconnectDelay(THostIndex hostIndex) = 0; // Picks the best host (by lowest inflight count) out of the provided set // of hosts. Ties are broken uniformly at random. @@ -135,7 +135,7 @@ public: void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) override; void OnDDiskConnected(THostIndex hostIndex, TInstant now) override; - [[nodiscard]] TDuration GetDDiskReconnectDelay( + [[nodiscard]] TDuration GetHostReconnectDelay( THostIndex hostIndex) override; [[nodiscard]] THostIndex SelectBestPBufferHost( diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp index 50537855b0e..055cdbd0738 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp @@ -58,6 +58,8 @@ void TVChunk::Start() { // ActorSystem thread + LogTitle.SetDiskId(PartitionDirectService->GetVolumeConfig()->DiskId); + Executor->ExecuteSimple( [weakSelf = weak_from_this()]() mutable { @@ -396,7 +398,6 @@ void TVChunk::DoStart() { Y_ABORT_UNLESS(ExecutorThreadChecker.Check()); - LogTitle.SetDiskId(PartitionDirectService->GetVolumeConfig()->DiskId); DirectBlockGroup->Register(weak_from_this()); LOG_DEBUG( diff --git a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp index 5890aa4f953..713fd40ff7a 100644 --- a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp +++ b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp @@ -21,6 +21,10 @@ namespace { using namespace NKikimr::NKqp; +TString MetadataUserToken() { + return NACLib::TSystemUsers::Metadata().SerializeAsString(); +} + TString MakeDeletePublicationSql() { return TStringBuilder() << R"( -- TFinalizePublicationActor::MakeDeletePublicationSql @@ -111,13 +115,11 @@ public: TString database, ui64 intPublicationId, EFinalizePublicationOp op, - TString userToken, TString callerSid) : ReplyTo(replyTo) , Database(std::move(database)) , IntPublicationId(intPublicationId) , Op(op) - , UserToken(std::move(userToken)) , CallerSid(std::move(callerSid)) {} @@ -158,19 +160,11 @@ private: Become(&TFinalizePublicationActor::StateDeleteOnly); } - void SetCreateSessionIdentity(TEvKqp::TEvCreateSessionRequest& ev) { - if (!UserToken.empty()) { - NACLibProto::TUserToken token; - if (token.ParseFromString(UserToken)) { - ev.Record.SetUserSID(token.GetUserSID()); - } - } - } - void StartKqpSession() { auto ev = MakeHolder<TEvKqp::TEvCreateSessionRequest>(); ev->Record.MutableRequest()->SetDatabase(Database); - SetCreateSessionIdentity(*ev); + // Registry and topic finalize ops run as metadata@system, like Begin/List query actors. + ev->Record.SetUserSID(BUILTIN_ACL_METADATA); Send(MakeKqpProxyID(SelfId().NodeId()), ev.Release()); Step = EStep::KqpCreateSession; Become(&TFinalizePublicationActor::StateKqp); @@ -178,9 +172,7 @@ private: void SendKqpBeginTx() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); ev->Record.MutableRequest()->SetSessionId(KqpSessionId); @@ -197,9 +189,7 @@ private: .Build(); auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); ev->Record.MutableRequest()->SetQuery(MakeDeletePublicationSql()); @@ -218,9 +208,7 @@ private: void SendKqpDeferredPublication() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_UNDEFINED); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_TOPIC); ev->Record.MutableRequest()->SetDatabase(Database); @@ -235,9 +223,7 @@ private: void SendKqpCommit() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); ev->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); @@ -369,7 +355,6 @@ private: const TString Database; const ui64 IntPublicationId; const EFinalizePublicationOp Op; - const TString UserToken; const TString CallerSid; TMaybe<TListDestinationsData> ListDestinationsData; @@ -386,10 +371,9 @@ NActors::IActor* CreateFinalizePublicationActor( const TString& database, ui64 intPublicationId, EFinalizePublicationOp op, - const TString& userToken, const TString& callerSid) { - return new TFinalizePublicationActor(replyTo, database, intPublicationId, op, userToken, callerSid); + return new TFinalizePublicationActor(replyTo, database, intPublicationId, op, callerSid); } } // namespace NKikimr::NPQ::NDeferredPublish diff --git a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h index b14130cd4e4..28c33b38dd2 100644 --- a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h +++ b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h @@ -11,7 +11,6 @@ NActors::IActor* CreateFinalizePublicationActor( const TString& database, ui64 intPublicationId, EFinalizePublicationOp op, - const TString& userToken, const TString& callerSid); } // namespace NKikimr::NPQ::NDeferredPublish diff --git a/ydb/core/persqueue/pqrb/mirror_describer.cpp b/ydb/core/persqueue/pqrb/mirror_describer.cpp index 47185c8846d..6cd1e3e3687 100644 --- a/ydb/core/persqueue/pqrb/mirror_describer.cpp +++ b/ydb/core/persqueue/pqrb/mirror_describer.cpp @@ -4,6 +4,8 @@ #include <google/protobuf/util/message_differencer.h> +#define YDB_LOG_THIS_FILE_COMPONENT Service + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("topic", TopicName) using namespace NPersQueue; @@ -34,7 +36,8 @@ void TMirrorDescriber::StartInit(const TActorContext& ctx) { } void TMirrorDescriber::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_N("killed"); + YDB_LOG_NOTICE("Killed", + {"logPrefix", NPQ_LOG_PREFIX}); CredentialsProvider = nullptr; Die(ctx); } @@ -44,10 +47,13 @@ void TMirrorDescriber::HandleChangeConfig(TEvPQ::TEvChangePartitionConfig::TPtr& Config, ev->Get()->Config.GetPartitionConfig().GetMirrorFrom() ); - LOG_D("got new config, equal with previous: " << equalConfigs); + YDB_LOG_DEBUG("Got new config, equal with", + {"logPrefix", NPQ_LOG_PREFIX}, + {"previous", equalConfigs}); if (!equalConfigs) { Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom(); - LOG_I("changing config"); + YDB_LOG_INFO("Changing config", + {"logPrefix", NPQ_LOG_PREFIX}); StartInit(ctx); } } @@ -56,13 +62,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: DescribeTopicRequestInFlight = false; const auto& description = ev->Get()->Description; if (!description.has_value()) { - LOG_E("cannot describe topic " << description.error()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"error", description.error()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } const NYdb::NTopic::TDescribeTopicResult& result = description.value(); if (!result.IsSuccess()) { - LOG_E("cannot describe topic " << result.GetIssues().ToString()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"issues", result.GetIssues()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } @@ -71,14 +81,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: descr.SerializeTo(req); return req.ShortUtf8DebugString(); }; - LOG_T("topic description: " << debugTopicDescriptionString(description.value().GetTopicDescription())); + YDB_LOG_TRACE("Topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"description", debugTopicDescriptionString(description.value().GetTopicDescription())}); ctx.Send(TabletActorId, ev->Release()); ctx.Schedule(DESCRIBE_RETRY_TIMEOUT_MAX, new TEvents::TEvWakeup()); } void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { if (DescribeTopicRequestInFlight) { - LOG_I("description request already inflight."); + YDB_LOG_INFO("Description request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } @@ -111,7 +124,8 @@ void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) { if (CredentialsRequestInFlight) { - LOG_W("credentials request already inflight."); + YDB_LOG_WARN("Credentials request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } CredentialsProvider = nullptr; @@ -145,13 +159,17 @@ void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /* void TMirrorDescriber::HandleCredentialsCreated(TEvPQ::TEvCredentialsCreated::TPtr& ev, const TActorContext& ctx) { CredentialsRequestInFlight = false; if (ev->Get()->Error) { - LOG_W("cannot initialize credentials provider: " << ev->Get()->Error.value()); + YDB_LOG_WARN("Cannot initialize credentials", + {"logPrefix", NPQ_LOG_PREFIX}, + {"provider", ev->Get()->Error.value()}); ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), CredentialsInitInterval, INIT_INTERVAL_MAX, ctx); return; } CredentialsProvider = ev->Get()->Credentials; - LOG_N("credentials provider created " << bool(CredentialsProvider)); + YDB_LOG_NOTICE("Credentials provider created", + {"logPrefix", NPQ_LOG_PREFIX}, + {"hasCredentialsProvider", bool(CredentialsProvider)}); CredentialsInitInterval = INIT_INTERVAL_START; ScheduleDescription(ctx); } diff --git a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp index 12b49163dbc..517fe6591ba 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp @@ -6,6 +6,8 @@ #include <algorithm> #include <ranges> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -35,9 +37,13 @@ void TPartitionScaleManager::HandleScaleStatusChange(const ui32 partitionId, NKi TMaybe<NKikimrPQ::TPartitionScaleParticipants> participants, TMaybe<TString> splitBoundary, const TActorContext& ctx) { - PQ_LOG_D("Handle HandleScaleStatusChange. Scale status: " << NKikimrPQ::EScaleStatus_Name(scaleStatus)); + YDB_LOG_DEBUG("Handle HandleScaleStatusChange. Scale", + {"logPrefix", LogPrefix()}, + {"status", NKikimrPQ::EScaleStatus_Name(scaleStatus)}); if (scaleStatus == NKikimrPQ::EScaleStatus::NEED_SPLIT) { - PQ_LOG_D("::HandleScaleStatusChange need to split partition " << partitionId); + YDB_LOG_DEBUG("::HandleScaleStatusChange need to split partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); TPartitionScaleOperationInfo op{ .PartitionId = partitionId, .PartitionScaleParticipants = std::move(participants), @@ -58,13 +64,15 @@ void TPartitionScaleManager::TrySendScaleRequest(const TActorContext& ctx) { auto splitMergeRequest = BuildScaleRequest(ctx); if (splitMergeRequest.Empty()) { - PQ_LOG_D("splitMergeRequest empty"); + YDB_LOG_DEBUG("SplitMergeRequest empty", + {"logPrefix", LogPrefix()}); return; } RequestInflight = true; RootPartitionsResetRequestInflight = !splitMergeRequest.SetBoundary.empty(); - PQ_LOG_D("send split request"); + YDB_LOG_DEBUG("Send split request", + {"logPrefix", LogPrefix()}); CurrentScaleRequest = ctx.Register(new TPartitionScaleRequest( TopicName, TopicPath, @@ -129,12 +137,12 @@ TPartitionScaleManager::TScaleRequest TPartitionScaleManager::BuildScaleRequest( auto mergesToApply = BuildMergeRequest(allowedSplitsCount); auto splitsToApply = BuildSplitRequest(allowedSplitsCount); - PQ_LOG_D(fmt::format("Scale request: #splits={}, #unprocessed={}, splitsLimit={}, #merges={}", - splitsToApply.Requests.size(), - splitsToApply.Unprocessed, - allowedSplitsCountLimit, - mergesToApply.Requests.size() - )); + YDB_LOG_DEBUG("Scale request", + {"logPrefix", LogPrefix()}, + {"splits", splitsToApply.Requests.size()}, + {"unprocessed", splitsToApply.Unprocessed}, + {"splitsLimit", allowedSplitsCountLimit}, + {"merges", mergesToApply.Requests.size()}); return { .Split = std::move(splitsToApply.Requests), @@ -166,19 +174,20 @@ TPartitionScaleManager::TRequests<TPartitionScaleManager::TPartitionBoundary> TP allowedSplitsCount -= cost; boundsToApply.push_back(std::move(part)); } else { - PQ_LOG_W(fmt::format("MaxActivePartitions ({}) is too low to recreate {} root partitions from the mirror source topic", - BalancerConfig.MaxActivePartitions, - RootPartitionsToCreate->size())); + YDB_LOG_WARN("MaxActivePartitions is too low to recreate root partitions from the mirror source topic", + {"logPrefix", LogPrefix()}, + {"maxActivePartitions", BalancerConfig.MaxActivePartitions}, + {"createPartitions", RootPartitionsToCreate->size()}); // don't send request at all, if there is not enough quota return { .Unprocessed = RootPartitionsToCreate->size(), }; } } - PQ_LOG_D(fmt::format("Set partition boundaries requsts: #modify={}, #create{}", - modifyPartitions, - createPartitions - )); + YDB_LOG_DEBUG("Set partition boundaries requsts", + {"logPrefix", LogPrefix()}, + {"modify", modifyPartitions}, + {"create", createPartitions}); } return { .Requests = std::move(boundsToApply), @@ -221,21 +230,29 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const ui32 partitionId = splitParameters.PartitionId; if (MirroredFromSomewhere) { if (!AppData()->FeatureFlags.GetEnableMirroredTopicSplitMerge()) { - PQ_LOG_D("split request for mirrored topic is disabled. Partition# " << partitionId); + YDB_LOG_DEBUG("Split request for mirrored topic is disabled", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } if (!splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("split request for mirrored topic doesn't have prescribed partition ids. Partition# " << partitionId); + YDB_LOG_NOTICE("Split request for mirrored topic doesn't have prescribed partition ids", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } const auto* node = PartitionGraph.GetPartition(partitionId); if (node == nullptr) { if (splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("attempt to split partition that was not created yet. Partition# " << partitionId); + YDB_LOG_NOTICE("Attempt to split partition that was not created yet", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } else { - PQ_LOG_ERROR("partition not found. Partition# " << partitionId); + YDB_LOG_ERROR("Partition not found", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } @@ -244,18 +261,25 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu auto to = node->To; auto mid = splitParameters.SplitBoundary.GetOrElse(MiddleOf(from, to)); if (mid.empty()) { - PQ_LOG_ERROR("wrong partition key range. Can't get mid. Partition# " << partitionId); + YDB_LOG_ERROR("Wrong partition key range. Can't get mid", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } if (splitParameters.PartitionScaleParticipants.Defined() && splitParameters.PartitionScaleParticipants->AdjacentPartitionIdsSize() != 0) { - PQ_LOG_ERROR("split request cannot have adjacent partitions. Partition# " << partitionId); + YDB_LOG_ERROR("Split request cannot have adjacent partitions", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } - PQ_LOG_D("partition split ranges. From# '" << ToHex(from) - << "'. To# '" << ToHex(to) << "'. Mid# '" << ToHex(mid) - << "'. Partition# " << partitionId); + YDB_LOG_DEBUG("Partition split ranges", + {"logPrefix", LogPrefix()}, + {"fromHex", ToHex(from)}, + {"toHex", ToHex(to)}, + {"midHex", ToHex(mid)}, + {"partition", partitionId}); TPartitionSplit split; split.set_partition(partitionId); @@ -264,7 +288,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu for (const auto& childPartitionId : splitParameters.PartitionScaleParticipants->GetChildPartitionIds()) { split.add_childpartitionids(childPartitionId); if (const auto* childNode = PartitionGraph.GetPartition(childPartitionId); childNode != nullptr) { - PQ_LOG_NOTICE(fmt::format("Child partition# {} already exists. Performing unordered split. Partition# {}", childPartitionId, partitionId)); + YDB_LOG_NOTICE("Child partition already exists. Performing unordered split", + {"logPrefix", LogPrefix()}, + {"childPartition", childPartitionId}, + {"partition", partitionId}); split.set_createrootlevelsibling(true); } } @@ -276,7 +303,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const auto& prescribedChildrenIds = splitParameters.PartitionScaleParticipants->GetChildPartitionIds(); if (!std::ranges::is_permutation(nodeChildrenIds, prescribedChildrenIds)) { const std::string mappingStr = fmt::format("([{}]->[{}])", fmt::join(nodeChildrenIds, ","), fmt::join(prescribedChildrenIds, ",")); - PQ_LOG_ERROR("trying to split partition into different set of children partitions " << mappingStr << ". Partition# " << partitionId); + YDB_LOG_ERROR("Trying to split partition into different set of children partitions", + {"logPrefix", LogPrefix()}, + {"mappingStr", mappingStr}, + {"partition", partitionId}); } } return {.Split = Nothing(), .Remove = true}; @@ -287,7 +317,9 @@ void TPartitionScaleManager::HandleScaleRequestResult(TPartitionScaleRequest::TE RequestInflight = false; LastResponseTime = ctx.Now(); auto result = ev->Get(); - PQ_LOG_D("HandleScaleRequestResult scale request result: " << result->Status); + YDB_LOG_DEBUG("HandleScaleRequestResult scale request", + {"logPrefix", LogPrefix()}, + {"result", result->Status}); if (result->Status == TEvTxUserProxy::TResultStatus::ExecComplete) { RequestTimeout = TDuration::Zero(); Backoff.Reset(); @@ -312,24 +344,27 @@ void TPartitionScaleManager::UpdateMirrorRootPartitionsSet() { NMirror::TMirrorGraphComparisonResult cmp = NMirror::ComparePartitionGraphs(PartitionGraph, MirrorTopicDescription->GetPartitions()); if (!cmp.RootPartitionsMismatch.has_value()) { - PQ_LOG_D("Topic has all root partitions from the source topic"); + YDB_LOG_DEBUG("Topic has all root partitions from the source topic", + {"logPrefix", LogPrefix()}); RootPartitionsToCreate.reset(); MirrorTopicError.reset(); return; } auto& rootPartitionsMismatch = cmp.RootPartitionsMismatch.value(); if (rootPartitionsMismatch.Error.has_value()) { - std::string msg = TStringBuilder() << "Incompatable configuration of root partitions between source and target topics:" << rootPartitionsMismatch.Error.value(); - PQ_LOG_ERROR(msg); + YDB_LOG_ERROR("Incompatable configuration of root partitions between source and target topics", + {"logPrefix", LogPrefix()}, + {"topics", rootPartitionsMismatch.Error.value()}); RootPartitionsToCreate.reset(); MirrorTopicError = std::move(*rootPartitionsMismatch.Error); return; } const size_t existingPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Modify, &NMirror::TPartitionWithBounds::Action); const size_t newPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Create, &NMirror::TPartitionWithBounds::Action); - PQ_LOG_I(fmt::format("Topic has less root partitions than the mirror source. New configuration has {}+{} partitions.", - existingPartitions, - newPartitions)); + YDB_LOG_INFO("Topic has less root partitions than the mirror source", + {"logPrefix", LogPrefix()}, + {"existing", existingPartitions}, + {"new", newPartitions}); RootPartitionsToCreate = std::move(rootPartitionsMismatch.AlterRootPartitions); MirrorTopicError.reset(); @@ -341,7 +376,8 @@ std::expected<void, std::string> TPartitionScaleManager::HandleMirrorTopicDescri } else { auto& description = ev->Get()->Description; if (!description.has_value() || !description.value().IsSuccess()) { - PQ_LOG_W("Ignoring invalid mirror source description"); + YDB_LOG_WARN("Ignoring invalid mirror source description", + {"logPrefix", LogPrefix()}); return {}; } MirrorTopicDescription.emplace(std::move(description->GetTopicDescription())); diff --git a/ydb/core/persqueue/pqrb/partition_scale_request.cpp b/ydb/core/persqueue/pqrb/partition_scale_request.cpp index e09adfe7b0e..30d1298c230 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_request.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_request.cpp @@ -3,6 +3,8 @@ #include <ydb/core/protos/schemeshard/operations.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -76,7 +78,8 @@ void TPartitionScaleRequest::FillProposeRequest(TEvTxUserProxy::TEvProposeTransa } logMessage << "."; } - PQ_LOG_D( logMessage); + YDB_LOG_DEBUG(logMessage, + {"logPrefix", LogPrefix()}); for(const auto& merge: Merges) { auto* newMerge = groupDescription.AddMerge(); @@ -124,7 +127,9 @@ void TPartitionScaleRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatus: for (auto& issue : ev->Get()->Record.GetIssues()) { issues << issue.ShortDebugString() + ", "; } - PQ_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split request: " << issues); + YDB_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split", + {"logPrefix", LogPrefix()}, + {"request", issues}); Send(ParentActorId, scaleRequestResult.release()); Die(ctx); } else { diff --git a/ydb/core/persqueue/pqrb/read_balancer.cpp b/ydb/core/persqueue/pqrb/read_balancer.cpp index 08ad274f8de..5eafeb10d21 100644 --- a/ydb/core/persqueue/pqrb/read_balancer.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer.cpp @@ -16,6 +16,8 @@ #include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/random_provider/random_provider.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("tablet_id", TabletID())("path", Path)("topic", Topic) namespace NKikimr { @@ -142,7 +144,9 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } return s; }; - PQ_LOG_D(getInitLog()); + YDB_LOG_DEBUG("BALANCER INIT DONE dump logPrefix, getInitLog", + {"logPrefix", LogPrefix()}, + {"getInitLog", getInitLog()}); for (auto &ev : UpdateEvents) { ctx.Send(ctx.SelfID, ev.Release()); @@ -161,7 +165,8 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } void TPersQueueReadBalancer::HandleWakeup(TEvents::TEvWakeup::TPtr& ev, const TActorContext &ctx) { - PQ_LOG_D("TPersQueueReadBalancer::HandleWakeup"); + YDB_LOG_DEBUG("TPersQueueReadBalancer::HandleWakeup", + {"logPrefix", LogPrefix()}); switch (ev->Get()->Tag) { case TPartitionScaleManager::TRY_SCALE_REQUEST_WAKE_UP_TAG: { @@ -209,9 +214,13 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvUpdateBalancerConfig::TPtr if (!WaitingResponse.empty()) { //got transaction infly WaitingResponse.push_back(ev->Sender); } else { //version already applied - PQ_LOG_D("BALANCER Topic " << Topic << "Tablet " << TabletID() - << " Config already applied version " << record.GetVersion() << " actor " << ev->Sender - << " txId " << record.GetTxId()); + YDB_LOG_DEBUG("BALANCER Topic Tablet Config already applied version actor txId", + {"logPrefix", LogPrefix()}, + {"topic", Topic}, + {"tabletID", TabletID()}, + {"version", record.GetVersion()}, + {"sender", ev->Sender}, + {"txId", record.GetTxId()}); THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; res->Record.SetStatus(NKikimrPQ::OK); res->Record.SetTxId(record.GetTxId()); @@ -374,7 +383,9 @@ TStringBuilder TPersQueueReadBalancer::LogPrefix() const { void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { auto tabletId = ev->Get()->TabletId; - PQ_LOG_D("TEvClientDestroyed " << tabletId); + YDB_LOG_DEBUG("TEvClientDestroyed", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); ClosePipe(tabletId, ctx); RequestTabletIfNeeded(tabletId, ctx, true); @@ -391,7 +402,10 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, ClosePipe(ev->Get()->TabletId, ctx); RequestTabletIfNeeded(ev->Get()->TabletId, ctx, true); - PQ_LOG_ERROR("TEvClientConnected Status " << ev->Get()->Status << ", TabletId " << tabletId); + YDB_LOG_ERROR("TEvClientConnected Status TabletId", + {"logPrefix", LogPrefix()}, + {"status", ev->Get()->Status}, + {"tabletId", tabletId}); return; } @@ -402,10 +416,16 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, it->second.Generation = ev->Get()->Generation; it->second.NodeId = ev->Get()->ServerId.NodeId(); - PQ_LOG_D("TEvClientConnected TabletId " << tabletId << ", NodeId " << ev->Get()->ServerId.NodeId() << ", Generation " << ev->Get()->Generation); + YDB_LOG_DEBUG("TEvClientConnected TabletId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"nodeId", ev->Get()->ServerId.NodeId()}, + {"generation", ev->Get()->Generation}); } else - PQ_LOG_I("TEvClientConnected Pipe is not found, TabletId " << tabletId); + YDB_LOG_INFO("TEvClientConnected Pipe is not found, TabletId", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); } void TPersQueueReadBalancer::ClosePipe(const ui64 tabletId, const TActorContext& ctx) @@ -453,7 +473,10 @@ void TPersQueueReadBalancer::RequestTabletIfNeeded(const ui64 tabletId, const TA StatsRequestTracker.Cookies[tabletId] = cookie; } - PQ_LOG_D("Send TEvPersQueue::TEvStatus TabletId: " << tabletId << " Cookie: " << cookie); + YDB_LOG_DEBUG("Send TEvPersQueue::TEvStatus", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"cookie", cookie}); NTabletPipe::SendData(ctx, pipeClient, new TEvPersQueue::TEvStatus("", true), cookie); } } @@ -531,11 +554,13 @@ void TPersQueueReadBalancer::CheckStat(const TActorContext& ctx) { UpdateCounters(ctx); TEvPersQueue::TEvPeriodicTopicStats* ev = GetStatsEvent(); - PQ_LOG_D("Send TEvPeriodicTopicStats PathId: " << PathId - << " Generation: " << Generation - << " StatsReportRound: " << StatsReportRound - << " DataSize: " << TopicMetricsHandler->GetTopicMetrics().TotalDataSize - << " UsedReserveSize: " << TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize); + YDB_LOG_DEBUG("Send TEvPeriodicTopicStats", + {"logPrefix", LogPrefix()}, + {"pathId", PathId}, + {"generation", Generation}, + {"statsReportRound", StatsReportRound}, + {"dataSize", TopicMetricsHandler->GetTopicMetrics().TotalDataSize}, + {"usedReserveSize", TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize}); NTabletPipe::SendData(ctx, GetPipeClient(SchemeShardId, ctx), ev); @@ -622,8 +647,12 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetPartitionsLocation::TPtr pResponse->SetNodeId(iter->second.NodeId.GetRef()); pResponse->SetGeneration(iter->second.Generation.GetRef()); - PQ_LOG_D("The partition location was added to response: TabletId " << tabletId << ", PartitionId " << partitionId - << ", NodeId " << pResponse->GetNodeId() << ", Generation " << pResponse->GetGeneration()); + YDB_LOG_DEBUG("The partition location was added to response: TabletId PartitionId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"partitionId", partitionId}, + {"pResponseNodeId", pResponse->GetNodeId()}, + {"pResponseGeneration", pResponse->GetGeneration()}); return true; }; @@ -713,7 +742,10 @@ void TPersQueueReadBalancer::Handle(NSchemeShard::TEvSchemeShard::TEvSubDomainPa if (SchemeShardId == msg->SchemeShardId && (!SubDomainPathId || SubDomainPathId->OwnerId != msg->SchemeShardId)) { - PQ_LOG_D("Discovered subdomain " << msg->LocalPathId << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain at RB", + {"logPrefix", LogPrefix()}, + {"localPathId", msg->LocalPathId}, + {"tabletID", TabletID()}); SubDomainPathId.emplace(msg->SchemeShardId, msg->LocalPathId); Execute(new TTxWriteSubDomainPathId(this), ctx); @@ -767,8 +799,11 @@ void TPersQueueReadBalancer::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated .GetDomainState() .GetDiskQuotaExceeded(); - PQ_LOG_D("Discovered subdomain " << msg->PathId << " state, outOfSpace = " << outOfSpace - << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain state, at RB", + {"logPrefix", LogPrefix()}, + {"pathId", msg->PathId}, + {"outOfSpace", outOfSpace}, + {"tabletID", TabletID()}); SubDomainOutOfSpace = outOfSpace; @@ -847,7 +882,9 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetReadSessionsInfo::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ctx); - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); MLPBalancer->Handle(ev); } @@ -874,13 +911,16 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ev, const TActorContext& ctx) { if (!SplitMergeEnabled(TabletConfig)) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled", + {"logPrefix", LogPrefix()}); return; } auto& record = ev->Get()->Record; auto* node = PartitionGraph.GetPartition(record.GetPartitionId()); if (!node) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: partition " << record.GetPartitionId() << " not found."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: partition not found", + {"logPrefix", LogPrefix()}, + {"partitionId", record.GetPartitionId()}); return; } @@ -893,7 +933,8 @@ void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ctx ); } else { - PQ_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized."); + YDB_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized", + {"logPrefix", LogPrefix()}); } } @@ -907,7 +948,8 @@ void TPersQueueReadBalancer::Handle(TPartitionScaleRequest::TEvPartitionScaleReq } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMirrorTopicDescription::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("Received TEvMirrorTopicDescription"); + YDB_LOG_DEBUG("Received TEvMirrorTopicDescription", + {"logPrefix", LogPrefix()}); if (!MirroringEnabled(TabletConfig)) { return; } @@ -928,13 +970,17 @@ void TPersQueueReadBalancer::BroadcastPartitionError(const TString& message, con } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetPartitionRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetPartitionRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPGetPartitionRequest", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); PendingMLPGetPartitionRequests.push_back(std::move(ev)); ProcessMLPGetPartitionRequests(ActorContext()); } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetRuntimeAttributesRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle", + {"logPrefix", LogPrefix()}, + {"mlpGetRuntimeAttributesRequest", ev->Get()->Record.ShortDebugString()}); if (StatsRequestTracker.StatsReceived) { return MLPBalancer->Handle(ev); } diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp index 98ba0856039..b91d5afe224 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp @@ -3,8 +3,7 @@ #include <ydb/core/persqueue/public/utils.h> -#define DEBUG(message) - +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER namespace NKikimr::NPQ::NBalancing { @@ -164,17 +163,23 @@ TString TPartitionFamily::LogPrefix() const { void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetStatus) { if (Status != EStatus::Active) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that isn't active"); + YDB_LOG_CRIT("Releasing the family that isn't active", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } if (!Session) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that does not have a session"); + YDB_LOG_CRIT("Releasing the family that does not have a session", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } - PQ_LOG_I(" release partitions [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) - << "]. Target status " << targetStatus); + YDB_LOG_INFO("Release partitions. Target status", + {"logPrefix", LogPrefix()}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}, + {"targetStatus", targetStatus}); Status = EStatus::Releasing; TargetStatus = targetStatus; @@ -193,25 +198,35 @@ void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetSta bool TPartitionFamily::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { if (!Session || Session->Pipe != sender) { - PQ_LOG_D("try unlock the partition " << partitionId << " from other sender"); + YDB_LOG_DEBUG("Try unlock the partition from other sender", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } if (Status != EStatus::Releasing) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but family status is " << Status); + YDB_LOG_CRIT("Try unlock partition but family status is", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"status", Status}); return false; } if (!LockedPartitions.erase(partitionId)) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but partition isn't locked." - << " Locked partitions are [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_CRIT("Try unlock partition but partition isn't locked. Locked partitions are", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } --Session->ReleasingPartitionCount; if (!LockedPartitions.empty()) { - PQ_LOG_D("partition " << partitionId << " was unlocked but wait else [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Partition was unlocked, but wait", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } @@ -238,7 +253,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct return false; case ETargetStatus::Free: - PQ_LOG_T(" is free."); + YDB_LOG_TRACE("Is free", + {"logPrefix", LogPrefix()}); Status = EStatus::Free; AfterRelease(); @@ -251,7 +267,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct auto it = Consumer.Families.find(MergeTo); if (it == Consumer.Families.end()) { - PQ_LOG_D(" has been released for merge but target family is not exists."); + YDB_LOG_DEBUG("Has been released for merge but target family is not exists", + {"logPrefix", LogPrefix()}); return true; } auto* targetFamily = it->second.get(); @@ -266,7 +283,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct } void TPartitionFamily::Destroy(const TActorContext&) { - PQ_LOG_D(" destroyed."); + YDB_LOG_DEBUG("Destroyed", + {"logPrefix", LogPrefix()}); if (Session) { Session->Families.erase(Id); @@ -301,11 +319,14 @@ void TPartitionFamily::AfterRelease() { void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) { if (Status != EStatus::Free) { - PQ_LOG_CRIT("try start reading but the family status is " << Status); + YDB_LOG_CRIT("Try start reading but the family status is", + {"logPrefix", LogPrefix()}, + {"status", Status}); return; } - PQ_LOG_T("start reading"); + YDB_LOG_TRACE("Start reading", + {"logPrefix", LogPrefix()}); Status = EStatus::Active; @@ -327,7 +348,9 @@ void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) } void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, const TActorContext& ctx) { - PQ_LOG_D("attaching partitions [" << JoinRange(", ", partitions.begin(), partitions.end()) << "]"); + YDB_LOG_DEBUG("Attaching partitions", + {"logPrefix", LogPrefix()}, + {"partitions", JoinRange(", ", partitions.begin(), partitions.end())}); std::unordered_set<ui32> existedPartitions; existedPartitions.insert(Partitions.begin(), Partitions.end()); @@ -376,13 +399,17 @@ void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, co } void TPartitionFamily::ActivatePartition(ui32 partitionId) { - PQ_LOG_D("activating partition " << partitionId); + YDB_LOG_DEBUG("Activating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(1, -1); } void TPartitionFamily::InactivatePartition(ui32 partitionId) { - PQ_LOG_D("inactivating partition " << partitionId); + YDB_LOG_DEBUG("Inactivating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(-1, 1); } @@ -401,7 +428,9 @@ void TPartitionFamily::InactivatePartition(ui32 partitionId) { } void TPartitionFamily::Merge(TPartitionFamily* other) { - PQ_LOG_D("merge family with " << other->DebugStr()); + YDB_LOG_DEBUG("Merge family with", + {"logPrefix", LogPrefix()}, + {"debug", other->DebugStr()}); Y_VERIFY(this != other); @@ -545,8 +574,12 @@ void TPartitionFamily::UpdateSpecialSessions() { void TPartitionFamily::LockPartition(ui32 partitionId, const TActorContext& ctx) { auto step = NextStep(); - PQ_LOG_I("lock partition " << partitionId << " for " << Session->DebugStr() - << " generation " << TabletGeneration() << " step " << step); + YDB_LOG_INFO("Lock partition for generation step", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"debug", Session->DebugStr()}, + {"tabletGeneration", TabletGeneration()}, + {"step", step}); ctx.Send(Session->Sender, MakeEvLockPartition(partitionId, step).release()); } @@ -636,7 +669,9 @@ ui32 TConsumer::NextStep() { void TConsumer::RegisterPartition(ui32 partitionId, const TActorContext& ctx) { auto [_, inserted] = Partitions.try_emplace(partitionId, TPartition()); if (inserted && IsReadable(partitionId)) { - PQ_LOG_D("register readable partition " << partitionId); + YDB_LOG_DEBUG("Register readable partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); CreateFamily({partitionId}, ctx); } @@ -666,7 +701,9 @@ TPartitionFamily* TConsumer::CreateFamily(std::vector<ui32>&& partitions, TParti UnreadableFamilies[id] = family; } - PQ_LOG_D("family created " << family->DebugStr()); + YDB_LOG_DEBUG("Family created", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); return family; } @@ -706,7 +743,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d std::vector<TPartitionFamily*> newFamilies; if (!family->IsLonely()) { - PQ_LOG_D("break up " << family->DebugStr() << " partition=" << partitionId); + YDB_LOG_DEBUG("Break up", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); std::unordered_set<ui32> partitions; partitions.insert(family->Partitions.begin(), family->Partitions.end()); @@ -786,7 +826,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d } } } else { - PQ_LOG_D("can't break up " << family->DebugStr() << " because partition=" << partitionId << " is not root of family"); + YDB_LOG_DEBUG("Can't break up because is not root of family", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); } } @@ -868,7 +911,9 @@ TPartitionFamily* TConsumer::FindFamily(ui32 partitionId) { } void TConsumer::RegisterReadingSession(TSession* session, const TActorContext& ctx) { - PQ_LOG_I("register reading session " << session->DebugStr()); + YDB_LOG_INFO("Register reading session", + {"logPrefix", LogPrefix()}, + {"debug", session->DebugStr()}); Sessions[session->Pipe] = session; @@ -952,7 +997,9 @@ void TConsumer::UnregisterReadingSession(TSession* session, const TActorContext& bool TConsumer::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_CRIT("unlocking the partition " << partitionId << " from unknown family."); + YDB_LOG_CRIT("Unlocking the partition from unknown family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } @@ -1034,7 +1081,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons }); if (partition.NeedReleaseChildren()) { - PQ_LOG_D("Attache partitions [" << JoinRange(", ", newPartitions.begin(), newPartitions.end()) << "] to " << family->DebugStr()); + YDB_LOG_DEBUG("Attache partitions", + {"logPrefix", LogPrefix()}, + {"newPartitions", JoinRange(", ", newPartitions.begin(), newPartitions.end())}, + {"family", family->DebugStr()}); for (auto id : newPartitions) { if (family->CanAttach(std::vector{id})) { auto* node = GetPartitionGraph().GetPartition(id); @@ -1066,7 +1116,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons } } } else { - PQ_LOG_D("Can't attache partition " << id << " to " << family->DebugStr()); + YDB_LOG_DEBUG("Can't attache partition", + {"logPrefix", LogPrefix()}, + {"id", id}, + {"family", family->DebugStr()}); } } } else { @@ -1083,19 +1136,28 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons void TConsumer::StartReading(ui32 partitionId, const TActorContext& ctx) { if (!GetPartitionInfo(partitionId)) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition has been deleted."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition has been deleted", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* partition = GetPartition(partitionId); if (!partition) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition does not exist."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition does not exist", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto wasInactive = partition->IsInactive(); if (partition->StartReading()) { - PQ_LOG_D("Reading of the partition " << partitionId << " was started by " << ConsumerName << ". We stop reading from child partitions."); + YDB_LOG_DEBUG("Reading of the partition was started by We stop reading from child partitions", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); auto* family = FindFamily(partitionId); if (!family) { @@ -1137,21 +1199,27 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: auto partitionId = r.GetPartitionId(); if (!IsReadable(partitionId)) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't family"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } if (!family->Session) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't reading session"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't reading session", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } @@ -1159,8 +1227,12 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: const bool wasInactive = partition.IsInactive(); if (partition.SetFinishedState(r.GetScaleAwareSDK(), r.GetStartedReadingFromEndOffset()) || wasInactive) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ", firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); if (ProccessReadingFinished(partitionId, wasInactive, ctx)) { ScheduleBalance(ctx); @@ -1168,9 +1240,13 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: } else if (!partition.IsInactive()) { auto delay = std::min<size_t>(1ul << partition.Iteration, Balancer.GetLifetimeSeconds()); // TODO use split/merge time - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ". Scheduled release of the partition for re-reading. Delay=" << delay << " seconds," - << " firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by Scheduled release of the partition for re-reading. seconds", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"delay", delay}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); ctx.Schedule(TDuration::Seconds(delay), new TEvPQ::TEvWakeupReleasePartition(ConsumerName, partitionId, partition.Cookie)); } @@ -1178,13 +1254,15 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: void TConsumer::ScheduleBalance(const TActorContext& ctx) { if (BalanceScheduled) { - PQ_LOG_T("rebalancing already was scheduled"); + YDB_LOG_TRACE("Rebalancing already was scheduled", + {"logPrefix", LogPrefix()}); return; } BalanceScheduled = true; - PQ_LOG_D("rebalancing was scheduled"); + YDB_LOG_DEBUG("Rebalancing was scheduled", + {"logPrefix", LogPrefix()}); ctx.Send(Balancer.TopicActor.SelfId(), new TEvPQ::TEvBalanceConsumer(ConsumerName)); } @@ -1254,9 +1332,14 @@ size_t GetMaxFamilySize(const std::unordered_map<size_t, const std::unique_ptr<T } void TConsumer::Balance(const TActorContext& ctx) { - PQ_LOG_D("balancing. Sessions=" << Sessions.size() << ", Families=" << Families.size() - << ", UnreadableFamilies=" << UnreadableFamilies.size() << " [" << DebugStr(UnreadableFamilies) - << "], RequireBalancing=" << FamiliesRequireBalancing.size() << " [" << DebugStr(FamiliesRequireBalancing) << "]"); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"sessions", Sessions.size()}, + {"families", Families.size()}, + {"unreadableFamilies", UnreadableFamilies.size()}, + {"unreadableFamiliesDebug", DebugStr(UnreadableFamilies)}, + {"requireBalancing", FamiliesRequireBalancing.size()}, + {"familiesRequireBalancingDebug", DebugStr(FamiliesRequireBalancing)}); if (Sessions.empty()) { return; @@ -1270,7 +1353,9 @@ void TConsumer::Balance(const TActorContext& ctx) { continue; } if (!family->SpecialSessions.contains(family->Session->Pipe)) { - PQ_LOG_D("rebalance " << family->DebugStr() << " because exists the special session for it"); + YDB_LOG_DEBUG("Rebalance because exists the special session for it", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); family->Release(ctx); } } @@ -1293,7 +1378,9 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (sit == sessions.end()) { - PQ_LOG_D("balancing of the " << family->DebugStr() << " failed because there are no suitable reading sessions."); + YDB_LOG_DEBUG("Balancing of the failed because there are no suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); continue; } @@ -1303,7 +1390,10 @@ void TConsumer::Balance(const TActorContext& ctx) { // Reorder sessions sessions.erase(sit); - PQ_LOG_D("balancing " << family->DebugStr() << " for " << session->DebugStr()); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"debug", session->DebugStr()}); family->StartReading(*session, ctx); // Reorder sessions @@ -1322,8 +1412,12 @@ void TConsumer::Balance(const TActorContext& ctx) { auto desiredFamilyCount = familyCount / commonSessions.size(); auto allowPlusOne = familyCount % commonSessions.size(); - PQ_LOG_D("start rebalancing. familyCount=" << familyCount << ", sessionCount=" << commonSessions.size() - << ", desiredFamilyCount=" << desiredFamilyCount << ", allowPlusOne=" << allowPlusOne); + YDB_LOG_DEBUG("Start rebalancing", + {"logPrefix", LogPrefix()}, + {"familyCount", familyCount}, + {"sessionCount", commonSessions.size()}, + {"desiredFamilyCount", desiredFamilyCount}, + {"allowPlusOne", allowPlusOne}); if (!OrderedSessions) { OrderedSessions.emplace(); @@ -1352,7 +1446,9 @@ void TConsumer::Balance(const TActorContext& ctx) { auto* family = it->second; if (!family->IsActive()) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is not active."); + YDB_LOG_DEBUG("Skip balancing because it is not active", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1365,14 +1461,18 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (family->Session->ActiveFamilyCount == 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is considered a session that does not read anything else."); + YDB_LOG_DEBUG("Skip balancing because it is considered a session that does not read anything else", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; } if (family->SpecialSessions.size() <= 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because there are no other suitable reading sessions."); + YDB_LOG_DEBUG("Skip balancing because there are no other suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1394,14 +1494,18 @@ void TConsumer::Balance(const TActorContext& ctx) { family->Release(ctx); it = FamiliesRequireBalancing.erase(it); } else { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is already being read by the best session."); + YDB_LOG_DEBUG("Skip balancing because it is already being read by the best session", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); ++it; } } } auto duration = TInstant::Now() - startTime; - PQ_LOG_D("balancing duration: " << duration); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"duration", duration}); } void TConsumer::Release(ui32 partitionId, const TActorContext& ctx) { @@ -1517,8 +1621,10 @@ const std::unordered_map<TActorId, std::unique_ptr<TSession>>& TBalancer::GetSes void TBalancer::UpdateConfig(const std::vector<ui32>& addedPartitions, const std::vector<ui32>& deletedPartitions, const TActorContext& ctx) { - PQ_LOG_D("updating configuration. Deleted partitions [" << JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end()) - << "]. Added partitions [" << JoinRange(", ", addedPartitions.begin(), addedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Updating configuration. Deleted partitions Added partitions", + {"logPrefix", LogPrefix()}, + {"deletedPartitions", JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end())}, + {"addedPartitions", JoinRange(", ", addedPartitions.begin(), addedPartitions.end())}); for (auto partitionId : deletedPartitions) { for (auto& [_, consumer] : Consumers) { @@ -1544,14 +1650,19 @@ bool TBalancer::SetCommittedState(const TString& consumerName, ui32 partitionId, } if (!consumer->IsReadable(partitionId)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("The offset of the partition was commited by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); return false; } auto wasInactive = consumer->IsInactive(partitionId); if (consumer->SetCommittedState(partitionId, generation, cookie)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName); + YDB_LOG_DEBUG("The offset of the partition was commited by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); if (consumer->ProccessReadingFinished(partitionId, wasInactive, ctx)) { consumer->ScheduleBalance(ctx); @@ -1575,13 +1686,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionStartedRequest::TPtr& ev auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1593,13 +1708,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionFinishedRequest::TPtr& e auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1614,17 +1733,28 @@ void TBalancer::Handle(TEvPersQueue::TEvPartitionReleased::TPtr& ev, const TActo auto* partitionInfo = GetPartitionInfo(partitionId); if (!partitionInfo) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender << " got deleted partition " << r); + YDB_LOG_CRIT("Client pipe got deleted partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"r", r}); return; } - PQ_LOG_I("client " << r.GetClientId() << " released partition from pipe " << sender - << " session " << r.GetSession() << " partition " << partitionId); + YDB_LOG_INFO("Client released partition from pipe session partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}, + {"partitionId", partitionId}); auto* consumer = GetConsumer(consumerName); if (!consumer) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender - << " is not connected and got release partitions request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got release partitions request for session", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}); return; } @@ -1646,11 +1776,17 @@ void TBalancer::Handle(TEvPQ::TEvWakeupReleasePartition::TPtr &ev, const TActorC } if (partition->Commited) { - PQ_LOG_D("skip releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout because offset is commited"); + YDB_LOG_DEBUG("Skip releasing partition of consumer by reading finished timeout because offset is commited", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); return; } - PQ_LOG_I("releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout"); + YDB_LOG_INFO("Releasing partition of consumer by reading finished timeout", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); consumer->Release(msg->PartitionId, ctx); } @@ -1666,22 +1802,31 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerConnected::TPtr& ev, const TActor auto& session = it->second; ++session->ServerActors; - PQ_LOG_I("pipe " << sender << " connected; active server actors: " << session->ServerActors); + YDB_LOG_INFO("Pipe connected; active server", + {"logPrefix", LogPrefix()}, + {"sender", sender}, + {"actors", session->ServerActors}); } void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected."); + YDB_LOG_DEBUG("Pipe disconnected", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Subscriptions.erase(ev->Get()->ClientId); auto it = Sessions.find(ev->Get()->ClientId); if (it == Sessions.end()) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected but there aren't sessions exists."); + YDB_LOG_DEBUG("Pipe disconnected but there aren't sessions exists", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); return; } - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected; active server actors: " - << (it != Sessions.end() ? it->second->ServerActors : -1)); + YDB_LOG_INFO("Pipe disconnected; active server", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}, + {"actors", (it != Sessions.end() ? it->second->ServerActors : -1)}); auto& session = it->second; if (--(session->ServerActors) > 0) { @@ -1689,8 +1834,11 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc } if (!session->SessionName.empty()) { - PQ_LOG_NOTICE("pipe " << ev->Get()->ClientId << " client " - << session->ClientId << " disconnected session " << session->SessionName); + YDB_LOG_NOTICE("Pipe client disconnected session", + {"logPrefix", LogPrefix()}, + {"eventClientId", ev->Get()->ClientId}, + {"sessionClientId", session->ClientId}, + {"sessionName", session->SessionName}); auto* consumer = GetConsumer(session->ClientId); if (consumer) { @@ -1706,7 +1854,9 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc Sessions.erase(it); } else { - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected no session"); + YDB_LOG_INFO("Pipe disconnected no session", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Sessions.erase(it); } @@ -1717,27 +1867,37 @@ void TBalancer::Handle(TEvPersQueue::TEvRegisterReadSession::TPtr& ev, const TAc auto& consumerName = r.GetClientId(); TActorId pipe = ActorIdFromProto(r.GetPipeClient()); - PQ_LOG_NOTICE("consumer \"" << consumerName << "\" register session for pipe " << pipe << " session " << r.GetSession()); + YDB_LOG_NOTICE("Consumer register session for pipe session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); if (consumerName.empty()) { - PQ_LOG_CRIT("ignored the session registration with empty consumer name."); + YDB_LOG_CRIT("Ignored the session registration with empty consumer name", + {"logPrefix", LogPrefix()}); return; } if (r.GetSession().empty()) { - PQ_LOG_CRIT("ignored the session registration with empty session name."); + YDB_LOG_CRIT("Ignored the session registration with empty session name", + {"logPrefix", LogPrefix()}); return; } if (!pipe) { - PQ_LOG_CRIT("ignored the session registration with empty Pipe."); + YDB_LOG_CRIT("Ignored the session registration with empty Pipe", + {"logPrefix", LogPrefix()}); return; } auto jt = Sessions.find(pipe); if (jt == Sessions.end()) { - PQ_LOG_CRIT("client \"" << consumerName << "\" pipe " << pipe - << " is not connected and got register session request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got register session request for session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); return; } @@ -1852,7 +2012,9 @@ void TBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorCo } void TBalancer::ProcessPendingStats(const TActorContext& ctx) { - PQ_LOG_D("ProcessPendingStats. PendingUpdates size " << PendingUpdates.size()); + YDB_LOG_DEBUG("ProcessPendingStats. PendingUpdates size", + {"logPrefix", LogPrefix()}, + {"pendingUpdatesSize", PendingUpdates.size()}); GetPartitionGraph().Travers([&](ui32 id) { for (auto& d : PendingUpdates[id]) { @@ -1868,7 +2030,9 @@ void TBalancer::ProcessPendingStats(const TActorContext& ctx) { void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TActorContext& ctx) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingSubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingSubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto status = Consumers.contains(record.GetConsumer()) ? @@ -1880,7 +2044,9 @@ void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TAct void TBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingUnsubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingUnsubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto& consumer = record.GetConsumer(); diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp index d62bd55c870..370de6c38d0 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp @@ -3,9 +3,6 @@ #include <library/cpp/monlib/service/pages/templates.h> #include <ydb/core/persqueue/common/common_app.h> -#define DEBUG(message) - - namespace NKikimr::NPQ::NBalancing { void TBalancer::RenderApp(NApp::TNavigationBar& __navigationBar) const { diff --git a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp index 61030f78738..2fdf743951d 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp @@ -3,6 +3,8 @@ #include <util/generic/ymath.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr::NPQ::NBalancing { namespace { @@ -192,7 +194,9 @@ void TMLPConsumer::Rebuild() { } } - PQ_LOG_D("Rebuild " << JoinSeq(",", PartitionsForBalancing) << " partitions for balancing"); + YDB_LOG_DEBUG("Rebuild partitions for balancing", + {"logPrefix", LogPrefix()}, + {"partitionsForBalancing", JoinSeq(",", PartitionsForBalancing)}); } const TMLPConsumer::TMetrics& TMLPConsumer::GetMetrics() const { @@ -212,7 +216,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' does not exist"; @@ -220,7 +226,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"; @@ -270,14 +278,18 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { const auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' does not exist"), 0, ev->Cookie); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"), 0, ev->Cookie); return; @@ -285,7 +297,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { auto it = Consumers.find(consumerName); if (it == Consumers.end()) { - PQ_LOG_D("Consumer '" << consumerName << "' is not initialized"); + YDB_LOG_DEBUG("Consumer is not initialized", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPGetRuntimeAttributesResponse(0, 0, 0), 0, ev->Cookie); return; } @@ -302,7 +316,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { } void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorContext&) { - PQ_LOG_D("Handle TEvPersQueue::TEvStatusResponse " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvStatusResponse", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); absl::flat_hash_map<TString, bool> mlpConsumers; for (const auto& consumer : GetConfig().GetConsumers()) { @@ -347,7 +363,9 @@ void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActo void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvReadingPartitionStatusRequest " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvReadingPartitionStatusRequest", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), true, // reading is finished @@ -363,7 +381,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, con void TMLPBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), std::nullopt, // reading is finished @@ -418,12 +438,16 @@ void TMLPBalancer::SetUseForReading(const TString& consumerName, ui64 cookie) { auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } diff --git a/ydb/core/persqueue/pqrb/read_balancer_log.h b/ydb/core/persqueue/pqrb/read_balancer_log.h index 96890d1d810..934eb09106e 100644 --- a/ydb/core/persqueue/pqrb/read_balancer_log.h +++ b/ydb/core/persqueue/pqrb/read_balancer_log.h @@ -7,12 +7,5 @@ namespace NKikimr::NPQ { inline TString LogPrefix() { return {}; } -#define PQ_LOG_T(stream) LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_NOTICE(stream) LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_commit_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_commit_ut.cpp index 580b450a6b7..580b450a6b7 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_commit_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_commit_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_consumer_split_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_consumer_split_ut.cpp index fd883d5bbab..fd883d5bbab 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_consumer_split_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_consumer_split_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_consumer_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_consumer_ut.cpp index 77b21be0e2b..77b21be0e2b 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_consumer_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_consumer_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_counters_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_counters_ut.cpp index 5bc45f60433..5bc45f60433 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_counters_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_counters_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_dlq_mover_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_dlq_mover_ut.cpp index e4d4be0986f..e4d4be0986f 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_dlq_mover_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_dlq_mover_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_message_enricher_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_message_enricher_ut.cpp index 184acc8e4e4..184acc8e4e4 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_message_enricher_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_message_enricher_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_storage_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_storage_ut.cpp index f687110d5ea..f687110d5ea 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_storage_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_storage_ut.cpp diff --git a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_ut.cpp b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_ut.cpp index dd92c80d123..dd92c80d123 100644 --- a/ydb/core/persqueue/pqtablet/partition/mlp/mlp_ut.cpp +++ b/ydb/core/persqueue/pqtablet/partition/mlp/ut/mlp_ut.cpp diff --git a/ydb/core/persqueue/public/schema/schema_operation.cpp b/ydb/core/persqueue/public/schema/schema_operation.cpp index 0f98bb13e42..99ab505f3cf 100644 --- a/ydb/core/persqueue/public/schema/schema_operation.cpp +++ b/ydb/core/persqueue/public/schema/schema_operation.cpp @@ -177,10 +177,9 @@ private: } void ReplyErrorAndDie(Ydb::StatusIds::StatusCode errorCode, TString&& errorMessage) { - YDB_LOG_DEBUG("Dump NPQLOGPREFIX, replyErrorAndDie, errorMessage", + YDB_LOG_DEBUG(errorMessage, {"logPrefix", NPQ_LOG_PREFIX}, - {"replyErrorAndDie", errorCode}, - {"errorMessage", errorMessage}); + {"replyErrorAndDie", errorCode}); Send(ParentId, new TEvSchemaOperationResponse(errorCode, std::move(errorMessage)), 0, Cookie); PassAway(); } diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index f1ca11529d9..5ef0c1ea189 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -178,7 +178,9 @@ class TPartitionWriter : public TActorBootstrapped<TPartitionWriter>, public TPa NKikimrPQ::TWriteId proto; auto* deferred = proto.MutableDeferredPublicationApi(); deferred->SetIntPublicationId(Opts.DeferredPublish->IntPublicationId); - deferred->SetExtPublicationId(Opts.DeferredPublish->ExtPublicationId); + if (Opts.DeferredPublish->ExtPublicationId) { + deferred->SetExtPublicationId(*Opts.DeferredPublish->ExtPublicationId); + } return TWriteId(std::move(proto)); } @@ -318,7 +320,7 @@ class TPartitionWriter : public TActorBootstrapped<TPartitionWriter>, public TPa WriteId = NPQ::GetWriteId(record.GetResponse().GetTopicOperations()); - YDB_LOG_DEBUG("Dump LOGPREFIX, sessionId, txId, writeId", + YDB_LOG_DEBUG("", {"logPrefix", LOG_PREFIX}, {"sessionId", Opts.SessionId}, {"txId", Opts.TxId}, diff --git a/ydb/core/persqueue/writer/writer.h b/ydb/core/persqueue/writer/writer.h index da80e75c920..14bc5606240 100644 --- a/ydb/core/persqueue/writer/writer.h +++ b/ydb/core/persqueue/writer/writer.h @@ -17,7 +17,7 @@ namespace NKikimr::NPQ { struct TDeferredPublishWriterOpts { ui64 IntPublicationId = 0; - TString ExtPublicationId; + TMaybe<TString> ExtPublicationId; }; struct TEvPartitionWriter { @@ -257,8 +257,8 @@ struct TPartitionWriterOpts { TPartitionWriterOpts& WithKafkaProducerInstanceId(const std::optional<NKafka::TProducerInstanceId>& value) { KafkaProducerInstanceId = value; return *this; } TPartitionWriterOpts& WithKafkaTransactionalId(const std::optional<TString>& value) { KafkaTransactionalId = value; return *this; } TPartitionWriterOpts& WithTrackProducerId(bool value) { TrackProducerId = value; return *this; } - TPartitionWriterOpts& WithDeferredPublish(ui64 intPublicationId, const TString& extPublicationId) { - DeferredPublish = TDeferredPublishWriterOpts{intPublicationId, extPublicationId}; + TPartitionWriterOpts& WithDeferredPublish(ui64 intPublicationId, TMaybe<TString> extPublicationId = Nothing()) { + DeferredPublish = TDeferredPublishWriterOpts{intPublicationId, std::move(extPublicationId)}; return *this; } }; diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index a83698da532..d3e00489294 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1059,12 +1059,20 @@ message TNodeWardenServiceSet { optional uint64 TotalResponseBytesPerSecond = 6; // quota for generated responses at network layer } + message TFailureInjectionConfig { + optional double FailureProbability = 1; // positive values enable failure injection + optional bool IncludeStaticGroups = 2; // also inject failures in request to static groups + optional uint64 RandomSeed = 3; + optional NKikimrProto.EReplyStatus ErrorReplyStatus = 4 [default = ERROR]; + } + repeated TPDisk PDisks = 1; repeated TVDisk VDisks = 2; repeated NKikimrBlobStorage.TGroupInfo Groups = 3; repeated uint64 AvailabilityDomains = 4; optional TReplBrokerConfig ReplBrokerConfig = 5; optional bool EnableProxyMock = 7; + optional TFailureInjectionConfig FailureInjectionConfig = 8; } message TNodeWardenCache { diff --git a/ydb/core/protos/blobstorage_config.proto b/ydb/core/protos/blobstorage_config.proto index d5686b8701e..7977e65041b 100644 --- a/ydb/core/protos/blobstorage_config.proto +++ b/ydb/core/protos/blobstorage_config.proto @@ -675,7 +675,11 @@ message TInferPDiskSlotCountForDriveTypeSettings { // where SlotSizeInUnits = 2^N is chosen to meet ExpectedSlotCount <= MaxSlots optional uint64 UnitSize = 1; // in bytes optional uint32 MaxSlots = 2; - optional bool PreferInferredSettingsOverExplicit = 3; // Unless true, PDisk.ExpectedSlotCount overrides inferred settings + // Unless true, explicit PDisk slot settings override inferred settings. + optional bool PreferInferredSettingsOverExplicit = 3; + // SlotSize is used as PDiskConfig ExpectedSlotSize for this drive type. + // It is mutually exclusive with UnitSize and requires MaxSlots as slot-count cap. + optional uint64 SlotSize = 4; // in bytes } message TInferPDiskSlotCountSettings { @@ -833,6 +837,7 @@ message TBaseConfig { reserved 21; // uint64 InferPDiskSlotCountFromUnitSize = 21 reserved 22; // uint32 InferPDiskSlotCountMax = 22 optional string DiskScope = 23; // when set allows for finer domain mapping + uint64 ExpectedSlotSize = 24; } message TVSlot { message TDonorDisk { diff --git a/ydb/core/protos/blobstorage_ddisk.proto b/ydb/core/protos/blobstorage_ddisk.proto index d5db486bc4c..b453c39f312 100644 --- a/ydb/core/protos/blobstorage_ddisk.proto +++ b/ydb/core/protos/blobstorage_ddisk.proto @@ -267,8 +267,11 @@ message TEvReadPersistentBufferResult { uint64 VChunkIndex = 4; uint32 OffsetInBytes = 5; uint32 SizeInBytes = 6; - // Reserved for checksum piggybacking: same raw XXH3_64(data) per MinSectorSize block of ReadResult - // payload as TEvWritePersistentBuffer.Checksums, no identity/salt mixed in; not filled yet. + // Same raw XXH3_64(data) per MinSectorSize block of ReadResult payload as + // TEvWritePersistentBuffer.Checksums, no identity/salt mixed in. Opt-in: populated only when the + // persisted record itself carries sender-supplied checksums (empty for legacy/checksum-less + // records). Forwarded verbatim by TWritePersistentBuffersRequestActor when re-replicating a + // record via TEvReadThenWritePersistentBuffers. repeated fixed64 Checksums = 7; } diff --git a/ydb/core/protos/blobstorage_disk.proto b/ydb/core/protos/blobstorage_disk.proto index 69180a428c4..9ab7ffcb43d 100644 --- a/ydb/core/protos/blobstorage_disk.proto +++ b/ydb/core/protos/blobstorage_disk.proto @@ -94,4 +94,5 @@ message TPDiskMetrics { optional uint32 SlotSizeInUnits = 13; optional double PDiskUsage = 14; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit optional NKikimrBlobStorage.TPDiskSpaceColor.E PDiskCapacityAlert = 15; // Max(SharedQuota.Color, CommonLog.Color) + optional uint64 ExpectedSlotSize = 16; // expected slot size for space limit calculation, 0 if not set } diff --git a/ydb/core/protos/blobstorage_distributed_config.proto b/ydb/core/protos/blobstorage_distributed_config.proto index e051fcb28b5..f0411a11f4d 100644 --- a/ydb/core/protos/blobstorage_distributed_config.proto +++ b/ydb/core/protos/blobstorage_distributed_config.proto @@ -198,6 +198,7 @@ message TEvNodeConfigInvokeOnRoot { bool IgnoreVSlotQuotaCheck = 6; bool IsSelfHealReasonDecommit = 7; bool FromSelfHeal = 8; + bool AllowUnusableDisks = 9; } // Regenerate configuration so the slain VDisk is no more reported as DESTROY one in the list. diff --git a/ydb/core/protos/blobstorage_pdisk_config.proto b/ydb/core/protos/blobstorage_pdisk_config.proto index c0e7c0fd138..da49cf40b3f 100644 --- a/ydb/core/protos/blobstorage_pdisk_config.proto +++ b/ydb/core/protos/blobstorage_pdisk_config.proto @@ -88,6 +88,8 @@ message TPDiskConfig { optional uint64 InsaneLogChunksMultiplier = 2000; // Log of cutThreshold * InsaneLogChunksMultiplier is insane + // ExpectedSlotSize is mutually exclusive with ExpectedSlotCount and SlotSizeInUnits in config. + // When ExpectedSlotSize is set, MaxSlots is required as an upper bound for inferred ExpectedSlotCount. optional uint64 ExpectedSlotCount = 2001; // Number of slots to calculate per-vdisk disk space limit. optional uint32 ChunkBaseLimit = 2002; // Free chunk permille that triggers Cyan color (e.g. 100 is 10%). Between 130 (default) and 13. @@ -100,4 +102,6 @@ message TPDiskConfig { optional bool UseBytesFlightControl = 2008; optional uint32 IoPieceSizeBytes = 2009; optional bool SortFreeChunksHDD = 2010; + optional uint64 ExpectedSlotSize = 2011; // Per-vdisk disk space limit. + optional uint32 MaxSlots = 2012; // Upper bound for ExpectedSlotCount inferred from ExpectedSlotSize. }; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 01c75df650f..8328a53aa75 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -393,13 +393,25 @@ message TSelfManagementConfig { optional bool AutomaticBoxManagement = 21 [default = true]; // invoke BSC DefineHostConfig/DefineBox automatically optional bool AutomaticBootstrap = 22; // whether bootstrap should be performed automatically; PROHIBITED for production optional bool AutomaticStaticGroupManagement = 23; // whether distconf/SelfHeal can change static group on its behalf - optional bool AutomaticStateStorageManagement = 24; // the same for state storage - optional bool AutomaticStateStorageBoardManagement = 25; // the same for state storage board - optional bool AutomaticSchemeBoardManagement = 26; // the same for scheme board + optional bool AutomaticStateStorageManagement = 24 [default = true]; // the same for state storage; if disabled, SelfHeal must not change current state storage config + optional bool AutomaticStateStorageBoardManagement = 25 [default = true]; // the same for state storage board; if disabled, SelfHeal must not change current state storage board config + optional bool AutomaticSchemeBoardManagement = 26 [default = true]; // the same for scheme board; if disabled, SelfHeal must not change current scheme board config // node ids onto which vdisks of static groups may be relocated during self-heal; // empty list means no restriction repeated uint32 StaticGroupSelfHealAllowedNodes = 27; + + // node ids onto which state storage may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageSelfHealAllowedNodes = 28; + + // node ids onto which state storage board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageBoardSelfHealAllowedNodes = 29; + + // node ids onto which state storage scheme board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 SchemeBoardSelfHealAllowedNodes = 30; } message TBlobStorageFormatConfig { @@ -536,6 +548,8 @@ message TInterconnectConfig { optional uint32 MaxRdmaRetryBackoffLevel = 61 [default = 8]; // 5s * 2^8 = 1280s, about 21 minutes by default optional bool UseUring = 62 [default = false]; optional bool EnableUringSQPOLL = 63 [default = false]; // only effective when UseUring is set + optional bool EnableInterconnectSessionV2 = 64 [default = false]; + optional bool ChecksumInterconnectSessionV2 = 65 [default = false]; // ballast is added to IC handshake frames to ensure correctness of jumbo frames transmission over network optional uint32 HandshakeBallastSize = 14; @@ -640,6 +654,7 @@ message TMonitoringConfig { optional bool RequireHealthcheckAuthentication = 22 [default = false]; optional string MonitoringCaFile = 23; repeated string CompressContentTypes = 24; // by default most text types are compressed + optional bool ClientCertificateRequired = 25 [default = false]; } message TRestartsCountConfig { @@ -3099,12 +3114,14 @@ message TClientCertificateAuthorization { repeated TClientCertificateDefinition ClientCertificateDefinitions = 1; optional bool RequestClientCertificate = 2 [default = false]; optional string DefaultGroup = 3 [default = "DefaultClientAuth@cert"]; + optional bool ClientCertificateRequired = 4 [default = false]; } message TKafkaProxyConfig { optional bool EnableKafkaProxy = 1 [default = false]; optional int32 ListeningPort = 2 [default = 9092]; + optional string ListeningAddress = 20 [default = "::"]; // You can either specify the path to the certificate, which will contain both the certificate and the private key, // or specify them separately in parameters Cert and Key. diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index c05170d7d96..4eeb18b8486 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -353,4 +353,6 @@ message TFeatureFlags { // RequireRestart: the in-memory counter is rebuilt from all FileStores at SchemeShard start, // the counter can be re-enabled, which requires restarting the SchemeShard tablet. optional bool DisableFileStoreSSDSystemSpaceAccounting = 302 [default = false, (RequireRestart) = true]; + optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = false]; + optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = false]; } diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 97c0af935ed..992347697f0 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -10,6 +10,7 @@ import "ydb/core/protos/kqp_tablemetadata.proto"; import "ydb/core/protos/kqp_physical.proto"; import "ydb/core/protos/kqp_stats.proto"; import "ydb/core/protos/data_events.proto"; +import "ydb/public/api/protos/ydb_common.proto"; import "ydb/public/api/protos/ydb_formats.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_table.proto"; @@ -322,6 +323,7 @@ message TQueryResponse { optional string QueryDiagnostics = 15; optional TQueryResponseExtraInfo ExtraInfo = 16; optional string EffectivePoolId = 17; + optional Ydb.VirtualTimestamp CommitTimestamp = 18; } message TEvQueryResponse { diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index 837b86af5b1..724d4cd2fa6 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -153,6 +153,7 @@ message TPDiskStateInfo { optional uint32 SlotSizeInUnits = 25; optional double PDiskUsage = 26; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit optional NKikimrBlobStorage.TPDiskSpaceColor.E PDiskCapacityAlert = 27; // Max(SharedQuota.Color, CommonLog.Color) + optional uint64 ExpectedSlotSize = 28 [(DefaultField) = true]; } message TEvPDiskStateRequest { diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto index 51b32e447f4..be32257fbc2 100644 --- a/ydb/core/protos/sys_view.proto +++ b/ydb/core/protos/sys_view.proto @@ -193,6 +193,7 @@ message TPDiskInfo { optional uint32 SlotSizeInUnits = 20; // either inferred or user-defined reserved 21; optional string MaintenanceStatus = 22; + optional uint64 ExpectedSlotSize = 23; // either inferred or user-defined, 0 if not set // metrics ? // physical location ? // config ? diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings.cpp b/ydb/core/resource_pools/resource_pool_classifier_settings.cpp index c6d7cc211d7..aad6cde67ef 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings.cpp +++ b/ydb/core/resource_pools/resource_pool_classifier_settings.cpp @@ -22,6 +22,14 @@ void TClassifierSettings::TParser::operator()(TString* setting) const { *setting = Value; } +void TClassifierSettings::TParser::operator()(std::optional<bool>* setting) const { + if (Value.empty()) { + setting->reset(); + } else { + *setting = FromString<bool>(Value); + } +} + void TClassifierSettings::TParser::operator()(std::optional<TString>* setting) const { if (Value.empty()) { setting->reset(); @@ -60,6 +68,13 @@ TString TClassifierSettings::TExtractor::operator()(TString* setting) const { return *setting; } +TString TClassifierSettings::TExtractor::operator()(std::optional<bool>* setting) const { + if (!*setting) { + return TString{}; + } + return **setting ? "true" : "false"; +} + TString TClassifierSettings::TExtractor::operator()(std::optional<TString>* setting) const { return setting->value_or(TString{}); } @@ -88,6 +103,7 @@ std::unordered_map<TString, TClassifierSettings::TProperty> TClassifierSettings: {"has_app_name", &HasAppName}, {"has_full_scan", &HasFullScan}, {"has_path", &HasPath}, + {"has_stream", &HasStream}, {"action", &Action} }; return properties; diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings.h b/ydb/core/resource_pools/resource_pool_classifier_settings.h index 71eff8f0ccf..88d77b28911 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings.h +++ b/ydb/core/resource_pools/resource_pool_classifier_settings.h @@ -17,11 +17,12 @@ enum class EClassifierAction { struct TClassifierSettings : public TSettingsBase { using TBase = TSettingsBase; - using TProperty = std::variant<i64*, TString*, std::optional<TString>*, std::optional<TRegexPredicate>*, std::optional<EClassifierAction>*>; + using TProperty = std::variant<i64*, TString*, std::optional<bool>*, std::optional<TString>*, std::optional<TRegexPredicate>*, std::optional<EClassifierAction>*>; struct TParser : public TBase::TParser { void operator()(i64* setting) const; void operator()(TString* setting) const; + void operator()(std::optional<bool>* setting) const; void operator()(std::optional<TString>* setting) const; void operator()(std::optional<TRegexPredicate>* setting) const; void operator()(std::optional<EClassifierAction>* setting) const; @@ -30,6 +31,7 @@ struct TClassifierSettings : public TSettingsBase { struct TExtractor : public TBase::TExtractor { TString operator()(i64* setting) const; TString operator()(TString* setting) const; + TString operator()(std::optional<bool>* setting) const; TString operator()(std::optional<TString>* setting) const; TString operator()(std::optional<TRegexPredicate>* setting) const; TString operator()(std::optional<EClassifierAction>* setting) const; @@ -46,6 +48,7 @@ struct TClassifierSettings : public TSettingsBase { std::optional<TString> HasAppName; std::optional<TRegexPredicate> HasFullScan; std::optional<TRegexPredicate> HasPath; + std::optional<bool> HasStream; std::optional<EClassifierAction> Action; }; diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp b/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp index debd32c057f..ffd4dde207d 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp +++ b/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp @@ -46,6 +46,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { std::visit(TClassifierSettings::TParser{"my_app"}, propertiesMap["has_app_name"]); std::visit(TClassifierSettings::TParser{"/Root/db/orders_*"}, propertiesMap["has_full_scan"]); std::visit(TClassifierSettings::TParser{"/Root/db/archive/*"}, propertiesMap["has_path"]); + std::visit(TClassifierSettings::TParser{"true"}, propertiesMap["has_stream"]); UNIT_ASSERT(settings.HasAppName.has_value()); UNIT_ASSERT_VALUES_EQUAL(*settings.HasAppName, "my_app"); @@ -55,6 +56,9 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { UNIT_ASSERT(settings.HasPath.has_value()); UNIT_ASSERT_VALUES_EQUAL(settings.HasPath->Pattern, "/Root/db/archive/*"); + + UNIT_ASSERT(settings.HasStream.has_value()); + UNIT_ASSERT_VALUES_EQUAL(settings.HasStream.value(), true); } Y_UNIT_TEST(SettingsExtracting) { @@ -78,11 +82,13 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { std::visit(TClassifierSettings::TParser{"ydb-cli"}, propertiesMap["has_app_name"]); std::visit(TClassifierSettings::TParser{"/Root/db/orders_*"}, propertiesMap["has_full_scan"]); std::visit(TClassifierSettings::TParser{"/Root/db/archive/*"}, propertiesMap["has_path"]); + std::visit(TClassifierSettings::TParser{"false"}, propertiesMap["has_stream"]); TClassifierSettings::TExtractor extractor; UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_app_name"]), "ydb-cli"); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_full_scan"]), "/Root/db/orders_*"); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_path"]), "/Root/db/archive/*"); + UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_stream"]), "false"); } Y_UNIT_TEST(OptionalStringEmptyResets) { @@ -111,6 +117,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_app_name"]), ""); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_full_scan"]), ""); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_path"]), ""); + UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_stream"]), ""); } Y_UNIT_TEST(ActionParsing) { diff --git a/ydb/core/sys_view/common/registry.h b/ydb/core/sys_view/common/registry.h index e74e88c1c20..d49ad9f41e5 100644 --- a/ydb/core/sys_view/common/registry.h +++ b/ydb/core/sys_view/common/registry.h @@ -260,6 +260,7 @@ struct Schema : NIceDb::Schema { struct SlotSizeInUnits : Column<19, NScheme::NTypeIds::Uint32> {}; // struct InferPDiskSlotCountFromUnitSize : Column<20, NScheme::NTypeIds::Uint64> {}; struct MaintenanceStatus : Column<21, NScheme::NTypeIds::Utf8> {}; + struct ExpectedSlotSize : Column<22, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<NodeId, PDiskId>; using TColumns = TableColumns< @@ -278,6 +279,7 @@ struct Schema : NIceDb::Schema { State, StatusChangeTimestamp, ExpectedSlotCount, + ExpectedSlotSize, NumActiveSlots, DecommitStatus, SlotSizeInUnits, @@ -779,6 +781,7 @@ struct Schema : NIceDb::Schema { struct Action : Column<7, NScheme::NTypeIds::Utf8> {}; struct HasFullScan : Column<8, NScheme::NTypeIds::Utf8> {}; struct HasPath : Column<9, NScheme::NTypeIds::Utf8> {}; + struct HasStream : Column<10, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<Name>; using TColumns = TableColumns< @@ -789,7 +792,8 @@ struct Schema : NIceDb::Schema { HasAppName, Action, HasFullScan, - HasPath>; + HasPath, + HasStream>; }; struct ShowCreate : Table<21> { diff --git a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp index 53ddd92b2a4..dc46181dff8 100644 --- a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp +++ b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp @@ -1,8 +1,8 @@ #include "resource_pool_classifiers.h" -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.h> +#include <ydb/services/workload_manager/actors/actors.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> #include <ydb/core/sys_view/common/events.h> #include <ydb/core/sys_view/common/registry.h> @@ -46,7 +46,7 @@ public: switch (ev->GetTypeRewrite()) { sFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleAck); hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle) - hFunc(NKqp::NWorkload::TEvFetchDatabaseResponse, Handle); + hFunc(NWorkloadManager::TEvFetchDatabaseResponse, Handle); hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution); cFunc(TEvents::TEvWakeup::EventType, HandleTimeout); cFunc(TEvents::TEvPoison::EventType, PassAway); @@ -66,60 +66,63 @@ private: if (!NMetadata::NProvider::TServiceOperator::IsEnabled()) { ReplyEmptyAndDie(); } - Register(NKqp::NWorkload::CreateDatabaseFetcherActor(SelfId(), TenantName, UserToken, NACLib::EAccessRights::GenericUse)); + Register(NWorkloadManager::CreateDatabaseFetcherActor(SelfId(), TenantName, UserToken, NACLib::EAccessRights::GenericUse)); } - void Handle(NKqp::NWorkload::TEvFetchDatabaseResponse::TPtr& ev) { + void Handle(NWorkloadManager::TEvFetchDatabaseResponse::TPtr& ev) { auto& event = *ev->Get(); if (event.Status != Ydb::StatusIds::SUCCESS) { ReplyErrorAndDie(event.Status, event.Issues.ToOneLineString()); return; } DatabaseId = event.DatabaseId; - Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), new NMetadata::NProvider::TEvAskSnapshot(std::make_shared<NKqp::TResourcePoolClassifierSnapshotsFetcher>())); + Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), new NMetadata::NProvider::TEvAskSnapshot(std::make_shared<NWorkloadManager::TResourcePoolClassifierSnapshotsFetcher>())); } void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - using TExtractor = std::function<TCell(const NKqp::TResourcePoolClassifierConfig&)>; + using TExtractor = std::function<TCell(const NWorkloadManager::TResourcePoolClassifierConfig&)>; using TSchema = Schema::ResourcePoolClassifiers; struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> { TExtractorsMap() { - insert({TSchema::Name::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::Name::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { return TCell(config.GetName().data(), config.GetName().size()); }}); - insert({TSchema::Rank::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::Rank::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { return TCell::Make<i64>(config.GetRank()); }}); - insert({TSchema::MemberName::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::MemberName::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& memberName = config.GetConfigJson()["member_name"].GetString(); return TCell(memberName.data(), memberName.size()); }}); - insert({TSchema::ResourcePool::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::ResourcePool::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& resourcePool = config.GetConfigJson()["resource_pool"].GetString(); return TCell(resourcePool.data(), resourcePool.size()); }}); - insert({TSchema::HasAppName::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::HasAppName::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& hasAppName = config.GetConfigJson()["has_app_name"].GetString(); return TCell(hasAppName.data(), hasAppName.size()); }}); - insert({TSchema::Action::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::Action::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& action = config.GetConfigJson()["action"].GetString(); return TCell(action.data(), action.size()); }}); - insert({TSchema::HasFullScan::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::HasFullScan::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& hasFullScan = config.GetConfigJson()["has_full_scan"].GetString(); return TCell(hasFullScan.data(), hasFullScan.size()); }}); - insert({TSchema::HasPath::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + insert({TSchema::HasPath::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { const auto& hasPath = config.GetConfigJson()["has_path"].GetString(); return TCell(hasPath.data(), hasPath.size()); }}); + insert({TSchema::HasStream::ColumnId, [] (const NWorkloadManager::TResourcePoolClassifierConfig& config) { + return TCell::Make<bool>(config.GetConfigJson()["has_stream"].GetBoolean()); + }}); } }; static TExtractorsMap extractors; - const auto& snapshot = ev->Get()->GetSnapshotAs<NKqp::TResourcePoolClassifierSnapshot>(); + const auto& snapshot = ev->Get()->GetSnapshotAs<NWorkloadManager::TResourcePoolClassifierSnapshot>(); const auto& config = snapshot->GetResourcePoolClassifierConfigs(); auto resourcePoolsIt = config.find(DatabaseId); if (resourcePoolsIt == config.end()) { diff --git a/ydb/core/sys_view/storage/pdisks.cpp b/ydb/core/sys_view/storage/pdisks.cpp index d4508000369..ac0b251cfe0 100644 --- a/ydb/core/sys_view/storage/pdisks.cpp +++ b/ydb/core/sys_view/storage/pdisks.cpp @@ -39,6 +39,7 @@ public: {T::State::ColumnId, {E::kInfoFieldNumber, V::kStateFieldNumber}}, {T::StatusChangeTimestamp::ColumnId, {E::kInfoFieldNumber, V::kStatusChangeTimestampFieldNumber}}, {T::ExpectedSlotCount::ColumnId, {E::kInfoFieldNumber, V::kExpectedSlotCountFieldNumber}}, + {T::ExpectedSlotSize::ColumnId, {E::kInfoFieldNumber, V::kExpectedSlotSizeFieldNumber}}, {T::NumActiveSlots::ColumnId, {E::kInfoFieldNumber, V::kNumActiveSlotsFieldNumber}}, {T::DecommitStatus::ColumnId, {E::kInfoFieldNumber, V::kDecommitStatusFieldNumber}}, {T::SlotSizeInUnits::ColumnId, {E::kInfoFieldNumber, V::kSlotSizeInUnitsFieldNumber}}, diff --git a/ydb/core/sys_view/streaming_queries/streaming_queries.cpp b/ydb/core/sys_view/streaming_queries/streaming_queries.cpp index ef935174e2b..aa8d27ba170 100644 --- a/ydb/core/sys_view/streaming_queries/streaming_queries.cpp +++ b/ydb/core/sys_view/streaming_queries/streaming_queries.cpp @@ -3,7 +3,7 @@ #include <ydb/core/kqp/common/events/script_executions.h> #include <ydb/core/kqp/common/kqp_script_executions.h> #include <ydb/core/kqp/gateway/behaviour/streaming_query/common/utils.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> +#include <ydb/services/workload_manager/actors/actors.h> #include <ydb/core/sys_view/common/registry.h> #include <ydb/core/sys_view/common/scan_actor_base_impl.h> #include <ydb/library/query_actor/query_actor.h> @@ -613,7 +613,7 @@ public: STFUNC(StateScan) { switch (ev->GetTypeRewrite()) { hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle); - hFunc(NKqp::NWorkload::TEvFetchDatabaseResponse, Handle); + hFunc(NWorkloadManager::TEvFetchDatabaseResponse, Handle); hFunc(TEvPrivate::TEvCheckStreamingQueriesTables, Handle); hFunc(TEvPrivate::TEvFetchStreamingQueriesResult, Handle); hFunc(TEvPrivate::TEvDescribeStreamingQueriesResult, Handle); @@ -633,7 +633,7 @@ public: ContinueScan(); } - void Handle(NKqp::NWorkload::TEvFetchDatabaseResponse::TPtr& ev) { + void Handle(NWorkloadManager::TEvFetchDatabaseResponse::TPtr& ev) { HasInflightOperation = false; if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { @@ -834,7 +834,7 @@ private: if (!DatabaseId) { HasInflightOperation = true; - const auto& databaseFetcher = Register(NKqp::NWorkload::CreateDatabaseFetcherActor(SelfId(), DatabaseName, MakeIntrusive<NACLib::TUserToken>(BUILTIN_ACL_METADATA, TVector<NACLib::TSID>{}))); + const auto& databaseFetcher = Register(NWorkloadManager::CreateDatabaseFetcherActor(SelfId(), DatabaseName, MakeIntrusive<NACLib::TUserToken>(BUILTIN_ACL_METADATA, TVector<NACLib::TSID>{}))); LOG_D("Start database fetcher " << databaseFetcher); return; } diff --git a/ydb/core/sys_view/streaming_queries/ya.make b/ydb/core/sys_view/streaming_queries/ya.make index 7ce9098465f..d819f52862f 100644 --- a/ydb/core/sys_view/streaming_queries/ya.make +++ b/ydb/core/sys_view/streaming_queries/ya.make @@ -9,7 +9,7 @@ PEERDIR( ydb/core/kqp/common/events ydb/core/kqp/gateway/behaviour/streaming_query/common ydb/core/kqp/runtime - ydb/core/kqp/workload_service/actors + ydb/services/workload_manager/actors ydb/core/sys_view/common ydb/library/actors/core ydb/library/query_actor diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index 079bb3be708..5280a01ab80 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -1101,6 +1101,7 @@ Y_UNIT_TEST_SUITE(SystemView) { BoxId, DecommitStatus, ExpectedSlotCount, + ExpectedSlotSize, Guid, Kind, MaintenanceStatus, @@ -1132,12 +1133,13 @@ Y_UNIT_TEST_SUITE(SystemView) { } } - TYsonFieldChecker check(ysonString, 19); + TYsonFieldChecker check(ysonString, 20); check.Uint64(0u); // AvailableSize check.Uint64(999u); // BoxId check.String("DECOMMIT_NONE"); // DecommitStatus check.Uint64(16); // ExpectedSlotCount + check.Uint64(0); // ExpectedSlotSize check.Uint64(123u); // Guid check.Uint64(0u); // Kind check.String("NO_REQUEST"); // MaintenanceStatus diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index dc2bb371c58..aee2907175d 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -76,6 +76,7 @@ #include <ydb/core/kqp/common/kqp.h> #include <ydb/core/kqp/rm_service/kqp_rm_service.h> #include <ydb/core/kqp/proxy_service/kqp_proxy_service.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.h> #include <ydb/core/metering/metering.h> #include <ydb/core/protos/replication.pb.h> @@ -1352,6 +1353,14 @@ namespace Tests { TActorId describeSchemaSecretsServiceId = Runtime->Register(describeSchemaSecretsService, nodeIdx, userPoolId); Runtime->RegisterService(NSecret::MakeDescribeSchemaSecretServiceId(Runtime->GetNodeId(nodeIdx)), describeSchemaSecretsServiceId, nodeIdx); } + + { + const auto& appData = Runtime->GetAppData(nodeIdx); + IActor* workloadManager = NWorkloadManager::CreateService(NWorkloadManager::GetWorkloadManagerCounters(appData.Counters)); + TActorId workloadManagerId = Runtime->Register(workloadManager, nodeIdx, userPoolId, TMailboxType::HTSwap, 0); + Runtime->RegisterService(NWorkloadManager::MakeServiceId(Runtime->GetNodeId(nodeIdx)), workloadManagerId, nodeIdx); + } + { auto kqpProxySharedResources = std::make_shared<NKqp::TKqpProxySharedResources>(); @@ -1622,6 +1631,7 @@ namespace Tests { NKafka::TListenerSettings settings; settings.Port = Settings->AppConfig->GetKafkaProxyConfig().GetListeningPort(); + settings.Address = Settings->AppConfig->GetKafkaProxyConfig().GetListeningAddress(); bool ssl = false; if (Settings->AppConfig->GetKafkaProxyConfig().HasSslCertificate()) { ssl = true; diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 9a63cfcd979..7b25d62c931 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -57,6 +57,7 @@ PEERDIR( ydb/core/kqp ydb/core/kqp/federated_query ydb/services/scheme_secret + ydb/services/workload_manager/service ydb/core/kqp/finalize_script_service ydb/core/kqp/proxy_service ydb/core/metering diff --git a/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp b/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp index e9f51e5b10d..76de9107fc2 100644 --- a/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp +++ b/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp @@ -90,12 +90,15 @@ class TGrabActor: public TActorBootstrapped<TGrabActor> { std::deque<TAutoPtr<IEventHandle>> Inputs; TMutex Mutex; std::unique_ptr<NColumnShard::NBackup::TExportDriver> Driver; - std::unique_ptr<NTable::IScan> Exporter; + + // non-owning pointer to the exporter actor + NTable::IScan* Exporter = nullptr; public: - TRuntimePtr Runtime; + // Non-owning pointer to the actor runtime + TTestActorRuntime* Runtime; - TGrabActor(TRuntimePtr runtime) + TGrabActor(TTestActorRuntime* runtime) : Runtime(runtime) { } @@ -107,12 +110,14 @@ public: auto exportFactory = std::make_shared<TDataShardExportFactory>(); - Exporter = + std::unique_ptr<NTable::IScan> exporter = NColumnShard::NBackup::CreateIScanExportUploader(SelfId(), MakeBackupTask("test"), exportFactory.get(), columns, 0).DetachResult(); - UNIT_ASSERT(Exporter); + UNIT_ASSERT(exporter); + Exporter = exporter.get(); Driver = std::make_unique<NColumnShard::NBackup::TExportDriver>(TActorContext::ActorSystem(), SelfId()); auto initialState = Exporter->Prepare(Driver.get(), MakeSchema()); UNIT_ASSERT_VALUES_EQUAL(initialState.Scan, NTable::EScan::Feed); + Y_UNUSED(exporter.release()); NTable::TLead lead; auto seekState = Exporter->Seek(lead, 0); @@ -193,7 +198,7 @@ Y_UNIT_TEST_SUITE(IScan) { runtime->SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_DEBUG); SetupTabletServices(*runtime); - auto grabActor = new TGrabActor(runtime); + auto grabActor = new TGrabActor(runtime.get()); runtime->Register(grabActor); while (true) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index 1ea9b91d814..e31512c860d 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -36,6 +36,7 @@ void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { self.BackgroundController.StartCompaction(GranuleMeta->GetPathId(), GetTaskIdentifier()); NeedGranuleStatusProvide = true; GranuleMeta->OnCompactionStarted(); + PortionsIndexSnapshot = GranuleMeta->GetPortionsIndex().GetPortionsSnapshot(); } void TCompactColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.h b/ydb/core/tx/columnshard/engines/changes/compaction.h index a1ec7967289..66a009b93b4 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction.h @@ -3,6 +3,7 @@ #include <ydb/core/tx/columnshard/common/path_id.h> #include <ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h> #include <ydb/core/tx/columnshard/engines/changes/with_appended.h> +#include <ydb/core/tx/columnshard/engines/storage/granule/portions_index.h> namespace NKikimr::NOlap { @@ -16,6 +17,7 @@ private: protected: std::vector<TPortionInfo::TConstPtr> SwitchedPortions; // Portions that would be replaced by new ones std::shared_ptr<TGranuleMeta> GranuleMeta; + NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot PortionsIndexSnapshot; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp index d5a0f48761e..cf56253be88 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp @@ -13,7 +13,7 @@ namespace NKikimr::NOlap::NCompaction::NSubColumns { std::shared_ptr<NArrow::NAccessor::IChunkedArray> TMergedBuilder::MaybeDictionaryEncode( const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& accessor, const ui32 filledRecordsCount, const EValueType valueType) const { - if (!NArrow::NAccessor::NSubColumns::DictionaryApplicableForValueType(valueType)) { + if (!NArrow::NAccessor::NSubColumns::CanBeDictionaryEncoded(valueType)) { return accessor; } const auto enumerateNotNull = [&accessor](const auto& consumer) { @@ -85,12 +85,11 @@ void TMergedBuilder::Initialize() { switch (ResultColumnStats.GetAccessorType(i)) { case NArrow::NAccessor::IChunkedArray::EType::Array: ColumnBuilders.emplace_back( - TPlainRuntimeBuilder(NArrow::NAccessor::NSubColumns::GetArrowTypeForValueType(valueType)), valueType); + TEncodingPlainBuilder(NArrow::NAccessor::NSubColumns::GetCodecForValueType(valueType), 0, 0), valueType); break; case NArrow::NAccessor::IChunkedArray::EType::SparsedArray: - // Native scalars are never sparsed, so a sparsed column is always binary-backed. - AFL_VERIFY(valueType == EValueType::BinaryJson || valueType == EValueType::String)("value_type", (ui32)valueType); - ColumnBuilders.emplace_back(TSparsedBuilder(nullptr, 0, 0), valueType); + ColumnBuilders.emplace_back( + TEncodingSparsedBuilder(NArrow::NAccessor::NSubColumns::GetCodecForValueType(valueType), 0, 0), valueType); break; case NArrow::NAccessor::IChunkedArray::EType::Undefined: case NArrow::NAccessor::IChunkedArray::EType::SerializedChunkedArray: diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h index 46730ba517c..ba053489d46 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h @@ -1,9 +1,8 @@ #pragma once #include "remap.h" -#include <ydb/core/formats/arrow/accessor/plain/accessor.h> -#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> #include <ydb/core/formats/arrow/accessor/sub_columns/accessor.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h> #include <ydb/core/formats/arrow/accessor/sub_columns/settings.h> #include <ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h> #include <ydb/core/tx/columnshard/engines/storage/chunks/column.h> @@ -12,7 +11,8 @@ namespace NKikimr::NOlap::NCompaction::NSubColumns { class TMergedBuilder { private: - using TSparsedBuilder = NArrow::NAccessor::TSparsedArray::TSparsedBuilder<arrow::BinaryType>; + using TEncodingSparsedBuilder = NArrow::NAccessor::NSubColumns::TEncodingSparsedBuilder; + using TEncodingPlainBuilder = NArrow::NAccessor::NSubColumns::TEncodingPlainBuilder; using TColumnsData = NArrow::NAccessor::NSubColumns::TColumnsData; using TOthersData = NArrow::NAccessor::NSubColumns::TOthersData; using TSettings = NArrow::NAccessor::NSubColumns::TSettings; @@ -29,106 +29,39 @@ private: const TSettings Settings; const TRemapColumns& Remapper; - // A plain array builder whose element type is chosen at construction type. - // Commonality with TPlainBuilder to be refactored. - class TPlainRuntimeBuilder { - private: - std::unique_ptr<arrow::ArrayBuilder> Builder; - std::optional<ui32> LastRecordIndex; - - void AppendGapNulls(const ui32 recordIndex) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex - 1)); - } else { - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordIndex)); - } - LastRecordIndex = recordIndex; - } - - public: - TPlainRuntimeBuilder(const std::shared_ptr<arrow::DataType>& type) - : Builder(NArrow::MakeBuilder(type)) - { - } - - void AddNativeValue(const ui32 recordIndex, const arrow::Array& array, const i64 position) { - AFL_VERIFY(Builder->type()->id() == array.type_id())("builder", Builder->type()->ToString())("array", array.type()->ToString()); - AppendGapNulls(recordIndex); - AFL_VERIFY(NArrow::Append(*Builder, array, position)); - } - - void AddBinaryValue(const ui32 recordIndex, const TStringBuf value) { - AFL_VERIFY(Builder->type()->id() == arrow::Type::BINARY)("type", Builder->type()->ToString()); - AppendGapNulls(recordIndex); - AFL_VERIFY(NArrow::Append<arrow::BinaryType>(*Builder, arrow::util::string_view(value.data(), value.size()))); - } - - std::shared_ptr<NArrow::NAccessor::IChunkedArray> Finish(const ui32 recordsCount) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordsCount)("last", LastRecordIndex)("count", recordsCount); - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordsCount - *LastRecordIndex - 1)); - } else { - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordsCount)); - } - return std::make_shared<NArrow::NAccessor::TTrivialArray>(NArrow::FinishBuilder(std::move(Builder))); - } - }; - class TGeneralAccessorBuilder { private: - std::variant<TSparsedBuilder, TPlainRuntimeBuilder> Builder; + std::variant<TEncodingSparsedBuilder, TEncodingPlainBuilder> Builder; const EValueType TargetValueType; YDB_READONLY(ui32, FilledRecordsCount, 0); YDB_READONLY(ui64, FilledRecordsSize, 0); public: - TGeneralAccessorBuilder(TSparsedBuilder&& builder, const EValueType targetValueType) + TGeneralAccessorBuilder(TEncodingSparsedBuilder&& builder, const EValueType targetValueType) : Builder(std::move(builder)) , TargetValueType(targetValueType) { } - TGeneralAccessorBuilder(TPlainRuntimeBuilder&& builder, const EValueType targetValueType) + TGeneralAccessorBuilder(TEncodingPlainBuilder&& builder, const EValueType targetValueType) : Builder(std::move(builder)) , TargetValueType(targetValueType) { } - // Returns the number of value bytes actually stored, since a re-encoded value's stored size differs from its size in the source. + // Returns the number of value bytes actually stored, since a re-encoded value's stored size differs from its + // size in the source. Both builders share the same interface, so one generic visitor covers both. ui32 AddRecord(const ui32 recordIndex, const TGeneralIterator& it) { const bool passthrough = it.GetValueType() == TargetValueType; - - struct TVisitor { - const ui32 RecordIndex; - const TGeneralIterator& It; - const bool Passthrough; - - ui32 operator()(TSparsedBuilder& builder) const { - // Sparsed columns are always binary-backed, so the passthrough value is its storage bytes. - if (Passthrough) { - builder.AddRecord(RecordIndex, It.GetStorageView()); - return It.GetValueSize(); - } else { - const auto bj = It.GetValueAsBinaryJson(); - builder.AddRecord(RecordIndex, TStringBuf(bj.data(), bj.size())); - return bj.size(); - } - } - - ui32 operator()(TPlainRuntimeBuilder& builder) const { - if (Passthrough) { - builder.AddNativeValue(RecordIndex, It.GetArray(), It.GetLocalIndex()); - return It.GetValueSize(); - } else { - const auto bj = It.GetValueAsBinaryJson(); - builder.AddBinaryValue(RecordIndex, TStringBuf(bj.data(), bj.size())); - return bj.size(); - } + const ui32 storedSize = std::visit([&](auto& builder) -> ui32 { + if (passthrough) { + builder.AddArrayElement(recordIndex, it.GetArray(), it.GetLocalIndex()); + return it.GetValueSize(); } - }; - - const ui32 storedSize = std::visit(TVisitor{ recordIndex, it, passthrough }, Builder); + const auto bj = it.GetValueAsBinaryJson(); + builder.AddFromBinaryJson(recordIndex, bj); + return bj.size(); + }, Builder); ++FilledRecordsCount; FilledRecordsSize += storedSize; return storedSize; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp index 60aabe3784b..372845d292a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp @@ -27,12 +27,8 @@ void TSubColumnsMerger::DoStart(const std::vector<std::shared_ptr<NArrow::NAcces AFL_VERIFY(stats.size()); AFL_VERIFY(statRecordsCount); auto commonStats = TDictStats::Merge(stats, GetSettings(), statRecordsCount); - auto splitted = commonStats.SplitByVolume(GetSettings(), statRecordsCount); - ResultColumnStats = splitted.ExtractColumns(); + ResultColumnStats = commonStats.SelectSeparatedColumns(GetSettings(), statRecordsCount); ResultColumnStats->CreateJsonPathAccessorTrieCache(); - // YDB_LOG_ERROR("", - // {"columns", ResultColumnStats->DebugJson()}, - // {"others", splitted.ExtractOthers().DebugJson()}); RemapKeyIndex.RegisterColumnStats(*ResultColumnStats); for (auto&& i : OrderedIterators) { i.Start(); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp index c1206d05bf0..07480a5a5fa 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp @@ -20,8 +20,7 @@ TRemapColumns::TOthersData::TFinishContext TRemapColumns::BuildRemapInfo( builder.Add(i.first, statsByKeyIndex[i.second].GetRecordsCount(), statsByKeyIndex[i.second].GetDataSize(), settings.IsSparsed(statsByKeyIndex[i.second].GetRecordsCount(), recordsCount) ? NArrow::NAccessor::IChunkedArray::EType::SparsedArray : NArrow::NAccessor::IChunkedArray::EType::Array, - // For now others always encode in BinaryJson - NArrow::NAccessor::NSubColumns::EValueType::BinaryJson); + NArrow::NAccessor::NSubColumns::OthersExplicitBinaryJson); remap[i.second] = idx++; } return TOthersData::TFinishContext(builder.Finish(), remap); diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 318e1b56a3b..2ccdec52fb5 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -66,10 +66,11 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc std::vector<TReadPortionInfoWithBlobs> portions = TReadPortionInfoWithBlobs::RestorePortions(accessors, Blobs, context.SchemaVersions); THashSet<ui64> usedPortionIds; + AFL_VERIFY(PortionsIndexSnapshot); std::vector<std::shared_ptr<ISubsetToMerge>> currentToMerge; for (auto&& i : portions) { AFL_VERIFY(usedPortionIds.emplace(i.GetPortionInfo().GetPortionId()).second); - currentToMerge.emplace_back(std::make_shared<TReadPortionToMerge>(std::move(i), GranuleMeta)); + currentToMerge.emplace_back(std::make_shared<TReadPortionToMerge>(std::move(i), GranuleMeta, PortionsIndexSnapshot)); } const auto buildPortionsToMerge = [&](const std::vector<std::shared_ptr<ISubsetToMerge>>& toMerge, const bool useDeletion) { @@ -94,7 +95,8 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc subsetsCount > 1) { auto merged = BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, false), resultFiltered, stats); if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta)); + appendedToMerge.emplace_back( + std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta, PortionsIndexSnapshot)); } toMerge.clear(); sumMemory = 0; @@ -110,7 +112,8 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, appendedToMerge.empty()), resultFiltered, stats); if (appendedToMerge.size()) { if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta)); + appendedToMerge.emplace_back( + std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta, PortionsIndexSnapshot)); } } else { context.Counters.OnCompactionCorrectMemory(totalSumMemory); diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp index 9791471ee14..a67e03dc53d 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp @@ -33,7 +33,7 @@ std::shared_ptr<NArrow::TColumnFilter> ISubsetToMerge::BuildPortionFilter(const } } } - if (GranuleMeta->GetPortionsIndex().HasOlderIntervals(pInfo, portionsInUsage)) { + if (NGranule::NPortionsIndex::TPortionsIndex::HasOlderIntervals(*PortionsIndexSnapshot, pInfo, portionsInUsage)) { filterDeleted = NArrow::TColumnFilter::BuildAllowFilter(); } } @@ -80,9 +80,9 @@ ui64 TWritePortionsToMerge::GetColumnMaxChunkMemory() const { return result; } -TWritePortionsToMerge::TWritePortionsToMerge( - std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta) - : TBase(granuleMeta) +TWritePortionsToMerge::TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, + const std::shared_ptr<TGranuleMeta>& granuleMeta, const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) + : TBase(granuleMeta, portionsIndexSnapshot) , WritePortions(std::move(portions)) { ui32 idx = 0; diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.h b/ydb/core/tx/columnshard/engines/changes/merge_subset.h index cafcf9207fd..228b48e2c73 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.h +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.h @@ -6,6 +6,7 @@ #include <ydb/core/tx/columnshard/engines/portions/read_with_blobs.h> #include <ydb/core/tx/columnshard/engines/portions/write_with_blobs.h> #include <ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h> +#include <ydb/core/tx/columnshard/engines/storage/granule/portions_index.h> namespace NKikimr::NOlap::NCompaction { @@ -30,15 +31,19 @@ private: protected: const std::shared_ptr<TGranuleMeta> GranuleMeta; + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot PortionsIndexSnapshot; std::shared_ptr<NArrow::TColumnFilter> BuildPortionFilter(const std::optional<NKikimr::NOlap::TGranuleShardingInfo>& shardingActual, const std::shared_ptr<NArrow::TGeneralContainer>& batch, const TPortionInfo& pInfo, const THashSet<ui64>& portionsInUsage, const bool useDeletionFilter) const; public: - ISubsetToMerge(const std::shared_ptr<TGranuleMeta>& granule) + ISubsetToMerge( + const std::shared_ptr<TGranuleMeta>& granule, const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) : GranuleMeta(granule) + , PortionsIndexSnapshot(portionsIndexSnapshot) { AFL_VERIFY(GranuleMeta); + AFL_VERIFY(PortionsIndexSnapshot); } virtual ~ISubsetToMerge() = default; @@ -62,8 +67,9 @@ private: const bool useDeletionFilter) const override; public: - TReadPortionToMerge(TReadPortionInfoWithBlobs&& rPortion, const std::shared_ptr<TGranuleMeta>& granuleMeta) - : TBase(granuleMeta) + TReadPortionToMerge(TReadPortionInfoWithBlobs&& rPortion, const std::shared_ptr<TGranuleMeta>& granuleMeta, + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) + : TBase(granuleMeta, portionsIndexSnapshot) , ReadPortion(std::move(rPortion)) { } @@ -89,7 +95,8 @@ private: virtual ui64 GetColumnMaxChunkMemory() const override; public: - TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta); + TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta, + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot); }; } // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp index e73e84a27d0..666f92d9c72 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp @@ -3,8 +3,8 @@ namespace NKikimr::NOlap::NGranule::NPortionsIndex { -bool TPortionsIndex::HasOlderIntervals(const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) const { - for (auto&& [_, p] : Portions) { +bool TPortionsIndex::HasOlderIntervals(const TPortions& portions, const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) { + for (const auto& p : portions) { if (p->GetPortionId() == inputPortion.GetPortionId()) { continue; } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h index ede88dfa60b..22724e7f82b 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h @@ -10,6 +10,10 @@ class TGranuleMeta; namespace NKikimr::NOlap::NGranule::NPortionsIndex { class TPortionsIndex { +public: + using TPortions = std::vector<TPortionInfo::TConstPtr>; + using TPortionsSnapshot = std::shared_ptr<const TPortions>; + private: THashMap<ui64, std::shared_ptr<TPortionInfo>> Portions; const TGranuleMeta& Owner; @@ -31,7 +35,16 @@ public: AFL_VERIFY(Portions.erase(p->GetPortionId())); } - bool HasOlderIntervals(const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) const; + TPortionsSnapshot GetPortionsSnapshot() const { + auto result = std::make_shared<TPortions>(); + result->reserve(Portions.size()); + for (const auto& [_, portion] : Portions) { + result->emplace_back(portion); + } + return result; + } + + static bool HasOlderIntervals(const TPortions& portions, const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions); }; } // namespace NKikimr::NOlap::NGranule::NPortionsIndex diff --git a/ydb/core/tx/replication/service/base_table_writer.cpp b/ydb/core/tx/replication/service/base_table_writer.cpp index f0a6c8da7b3..404ebfe0d08 100644 --- a/ydb/core/tx/replication/service/base_table_writer.cpp +++ b/ydb/core/tx/replication/service/base_table_writer.cpp @@ -1,5 +1,4 @@ #include "base_table_writer.h" -#include "logging.h" #include "service.h" #include "worker.h" @@ -14,6 +13,7 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/actor.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> #include <util/generic/hash.h> @@ -22,19 +22,17 @@ #include <util/generic/set.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::REPLICATION_SERVICE + namespace NKikimr::NReplication::NService { class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[TablePartitionWriter]" - << TableId - << "[" << TabletId << "]" - << SelfId() << " "; - } - - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TTablePartitionWriter"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}, + {"tabletId", TabletId}); } void GetProxyServices() { @@ -43,6 +41,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } STATEFN(StateGetProxyServices) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateGetProxyServices"}); switch (ev->GetTypeRewrite()) { hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, Handle); default: @@ -51,7 +51,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } void Handle(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); LeaderPipeCache = ev->Get()->Services.LeaderPipeCache; Ready(); @@ -63,6 +64,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } STATEFN(StateWaitingRecords) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWaitingRecords"}); switch (ev->GetTypeRewrite()) { hFunc(NChangeExchange::TEvChangeExchange::TEvRecords, Handle); default: @@ -71,7 +74,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } void Handle(NChangeExchange::TEvChangeExchange::TEvRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); auto event = MakeHolder<TEvDataShard::TEvApplyReplicationChanges>(); auto& tableId = *event->Record.MutableTableId(); @@ -100,6 +104,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } STATEFN(StateWaitingStatus) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWaitingStatus"}); switch (ev->GetTypeRewrite()) { hFunc(TEvDataShard::TEvApplyReplicationChangesResult, Handle); default: @@ -108,17 +114,18 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } void Handle(TEvDataShard::TEvApplyReplicationChangesResult::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; switch (record.GetStatus()) { case NKikimrTxDataShard::TEvApplyReplicationChangesResult::STATUS_OK: return Ready(); default: - LOG_E("Apply result" - << ": status# " << static_cast<ui32>(record.GetStatus()) - << ", reason# " << static_cast<ui32>(record.GetReason()) - << ", error# " << record.GetErrorDescription()); + YDB_LOG_ERROR("Apply result", + {"status", static_cast<ui32>(record.GetStatus())}, + {"reason", static_cast<ui32>(record.GetReason())}, + {"error", record.GetErrorDescription()}); if (IsHardError(record.GetReason())) { return Leave(true); } else { @@ -150,8 +157,8 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } void Leave(bool hardError = false) { - LOG_I("Leave" - << ": hard error# " << hardError); + YDB_LOG_INFO("Leave", + {"hardError", hardError}); Send(Parent, new NChangeExchange::TEvChangeExchangePrivate::TEvGone(TabletId, hardError)); PassAway(); @@ -164,6 +171,7 @@ class TTablePartitionWriter: public TActorBootstrapped<TTablePartitionWriter> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); Unlink(); TActorBootstrapped::PassAway(); } @@ -185,10 +193,13 @@ public: {} void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); GetProxyServices(); } STATEFN(StateBase) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateBase"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); sFunc(TEvents::TEvWakeup, Leave); @@ -200,7 +211,6 @@ private: const TActorId Parent; const ui64 TabletId; const TTableId TableId; - mutable TMaybe<TString> LogPrefix; TActorId LeaderPipeCache; ui64 SubscribeCookie = 0; @@ -216,15 +226,12 @@ class TLocalTableWriter , public NChangeExchange::IChangeSenderFactory , private NSchemeCache::TSchemeCacheHelpers { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[LocalTableWriter]" - << TablePathId - << SelfId() << " "; - } - - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTableWriter"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}, + {"tablePathId", TablePathId}); } static TSerializedTableRange GetFullRange(ui32 keyColumnsCount) { @@ -234,12 +241,12 @@ class TLocalTableWriter } void LogCritAndLeave(const TString& error) { - LOG_C(error); + YDB_LOG_CRIT(error); Leave(TEvWorker::TEvGone::SCHEME_ERROR, error); } void LogWarnAndRetry(const TString& error) { - LOG_W(error); + YDB_LOG_WARN(error); Retry(); } @@ -278,6 +285,7 @@ class TLocalTableWriter } void Resolve() override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); ResolveTable(); } @@ -291,8 +299,8 @@ class TLocalTableWriter void Handle(TEvWorker::TEvHandshake::TPtr& ev) { Worker = ev->Sender; - LOG_D("Handshake" - << ": worker# " << Worker); + YDB_LOG_DEBUG("Handshake", + {"worker", Worker}); ResolveTable(); } @@ -310,8 +318,8 @@ class TLocalTableWriter void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { const auto& result = ev->Get()->Request; - LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult", + {"result", (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")}); if (!CheckNotEmpty(result)) { return; @@ -386,8 +394,8 @@ class TLocalTableWriter void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { const auto& result = ev->Get()->Request; - LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySetResult" - << ": result# " << (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvResolveKeySetResult", + {"result", (result ? result->ToString(*AppData()->TypeRegistry) : "nullptr")}); if (!CheckNotEmpty(result)) { return; @@ -416,8 +424,8 @@ class TLocalTableWriter CreateSenders(NChangeExchange::MakePartitionIds(KeyDesc->GetPartitions())); if (!Initialized) { - LOG_D("Send handshake" - << ": worker# " << Worker); + YDB_LOG_DEBUG("Send handshake", + {"worker", Worker}); Send(Worker, new TEvWorker::TEvHandshake()); Initialized = true; } @@ -426,12 +434,14 @@ class TLocalTableWriter } IActor* CreateSender(ui64 partitionId) const override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); const auto tableId = TTableId(TablePathId, Schema->Version); return new TTablePartitionWriter(SelfId(), partitionId, tableId, Serializer->Clone()); } void Handle(TEvWorker::TEvData::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); Y_ABORT_UNLESS(PendingRecords.empty()); TVector<NChangeExchange::TEvChangeExchange::TEvEnqueueRecords::TRecordInfo> records(::Reserve(ev->Get()->Records.size())); @@ -490,7 +500,8 @@ class TLocalTableWriter } void Handle(TEvWorker::TEvTerminateWriter::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); Terminating = true; if (IsAllSendersReadyOrUninit()) { @@ -499,7 +510,8 @@ class TLocalTableWriter } void Handle(TEvService::TEvTxIdResult::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); TVector<NChangeExchange::TEvChangeExchange::TEvEnqueueRecords::TRecordInfo> records; @@ -565,7 +577,8 @@ class TLocalTableWriter } void Handle(NChangeExchange::TEvChangeExchange::TEvRequestRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); TVector<NChangeExchange::IChangeRecord::TPtr> records(::Reserve(ev->Get()->Records.size())); @@ -579,7 +592,8 @@ class TLocalTableWriter } void Handle(NChangeExchange::TEvChangeExchange::TEvRemoveRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); for (const auto& record : ev->Get()->Records) { PendingRecords.erase(record); @@ -596,7 +610,8 @@ class TLocalTableWriter } void Handle(NChangeExchange::TEvChangeExchangePrivate::TEvReady::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); OnReady(ev->Get()->PartitionId); if (Terminating && IsAllSendersReadyOrUninit()) { @@ -605,7 +620,8 @@ class TLocalTableWriter } void Handle(NChangeExchange::TEvChangeExchangePrivate::TEvGone::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Get()->HardError) { Leave(TEvWorker::TEvGone::SCHEME_ERROR, "Cannot apply changes"); @@ -620,13 +636,14 @@ class TLocalTableWriter template <typename... Args> void Leave(Args&&... args) { - LOG_I("Leave"); + YDB_LOG_INFO("Leave"); Send(Worker, new TEvWorker::TEvGone(std::forward<Args>(args)...)); PassAway(); } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); KillSenders(); TActor::PassAway(); } @@ -659,6 +676,8 @@ public: } STFUNC(StateWork) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvWorker::TEvHandshake, Handle); hFunc(TEvWorker::TEvData, Handle); diff --git a/ydb/core/tx/replication/service/logging.h b/ydb/core/tx/replication/service/logging.h deleted file mode 100644 index 5711316e2ea..00000000000 --- a/ydb/core/tx/replication/service/logging.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include <ydb/library/actors/core/log.h> -#include <ydb/library/services/services.pb.h> - -#define LOG_T(stream) LOG_TRACE_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_D(stream) LOG_DEBUG_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_E(stream) LOG_ERROR_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) -#define LOG_C(stream) LOG_CRIT_S (*TlsActivationContext, NKikimrServices::REPLICATION_SERVICE, GetLogPrefix() << stream) diff --git a/ydb/core/tx/replication/service/s3_writer.cpp b/ydb/core/tx/replication/service/s3_writer.cpp index 0fe0e5b33fa..4b6616561f2 100644 --- a/ydb/core/tx/replication/service/s3_writer.cpp +++ b/ydb/core/tx/replication/service/s3_writer.cpp @@ -1,5 +1,4 @@ #include "json_change_record.h" -#include "logging.h" #include "s3_writer.h" #include "worker.h" @@ -9,6 +8,7 @@ #include <ydb/core/wrappers/s3_wrapper.h> #include <ydb/library/actors/core/actor.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> #include <library/cpp/json/json_writer.h> @@ -16,12 +16,7 @@ #include <util/generic/maybe.h> #include <util/string/builder.h> -#define CB_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) -#define CB_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) -#define CB_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) -#define CB_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) -#define CB_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) -#define CB_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::CONTINUOUS_BACKUP, GetLogPrefix() << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::CONTINUOUS_BACKUP namespace { @@ -56,15 +51,12 @@ struct TS3Request { class TS3Writer : public TActor<TS3Writer> { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[S3Writer]" - << TableName - << SelfId() << " "; - } - - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "S3Writer"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}, + {"tableName", TableName}); } template <typename TResult> @@ -73,8 +65,9 @@ class TS3Writer return true; } - CB_LOG_E("Error at '" << marker << "'" - << ", error# " << result); + YDB_LOG_ERROR("Error", + {"marker", marker}, + {"error", result}); RetryOrLeave(result.GetError()); return false; @@ -111,8 +104,8 @@ class TS3Writer } void Leave(const TString& error) { - CB_LOG_I("Leave" - << ": error# " << error); + YDB_LOG_INFO("Leave", + {"error", error}); // TODO support different error kinds Send(Worker, new TEvWorker::TEvGone(TEvWorker::TEvGone::S3_ERROR)); @@ -127,8 +120,8 @@ class TS3Writer void Handle(TEvWorker::TEvHandshake::TPtr& ev) { Worker = ev->Sender; - CB_LOG_D("Handshake" - << ": worker# " << Worker); + YDB_LOG_DEBUG("Handshake", + {"worker", Worker}); S3Client = RegisterWithSameMailbox(NWrappers::CreateStorageWrapper(ExternalStorageConfig->ConstructStorageOperator())); @@ -159,7 +152,8 @@ class TS3Writer } void Handle(TEvWorker::TEvData::TPtr& ev) { - CB_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (!ev->Get()->Records) { Finished = true; @@ -185,7 +179,8 @@ class TS3Writer void Handle(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - CB_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (!CheckResult(result, TStringBuf("PutObject"))) { return; @@ -219,6 +214,8 @@ public: {} STFUNC(StateWork) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvWorker::TEvHandshake, Handle); hFunc(TEvWorker::TEvData, Handle); @@ -230,7 +227,6 @@ public: private: NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; - mutable TMaybe<TString> LogPrefix; const TString TableName; const TString WriterName; TActorId Worker; diff --git a/ydb/core/tx/replication/service/service.cpp b/ydb/core/tx/replication/service/service.cpp index 7b9b99ce972..19e4f7add5a 100644 --- a/ydb/core/tx/replication/service/service.cpp +++ b/ydb/core/tx/replication/service/service.cpp @@ -1,4 +1,3 @@ -#include "logging.h" #include "service.h" #include "table_writer.h" #include "topic_reader.h" @@ -16,6 +15,7 @@ #include <ydb/core/transfer/transfer_writer.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <util/generic/hash.h> #include <util/generic/hash_set.h> @@ -24,6 +24,8 @@ #include <tuple> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::REPLICATION_SERVICE + namespace NKikimr::NReplication::NService { class TSessionInfo { @@ -524,14 +526,12 @@ namespace NKikimr::NReplication { namespace NService { class TReplicationService: public TActorBootstrapped<TReplicationService> { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[Service]" - << SelfId() << " "; - } - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TReplicationService"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } void RunBoardPublisher() { @@ -547,7 +547,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvService::TEvHandshake::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; const auto& controller = record.GetController(); @@ -560,9 +561,9 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { auto& session = it->second; if (session.GetGeneration() > controller.GetGeneration()) { - LOG_W("Ignore stale controller" - << ": controller# " << controller.GetTabletId() - << ", generation# " << controller.GetGeneration()); + YDB_LOG_WARN("Ignore stale controller", + {"controller", controller.GetTabletId()}, + {"generation", controller.GetGeneration()}); return; } @@ -663,7 +664,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvService::TEvRunWorker::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; const auto& controller = record.GetController(); @@ -671,20 +673,20 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { auto it = Sessions.find(controller.GetTabletId()); if (it == Sessions.end()) { - LOG_W("Cannot run worker" - << ": controller# " << controller.GetTabletId() - << ", worker# " << id - << ", reason# " << R"("unknown session")"); + YDB_LOG_WARN("Cannot run worker", + {"controller", controller.GetTabletId()}, + {"worker", id}, + {"reason", R"("unknown session")"}); return; } auto& session = it->second; if (session.GetGeneration() != controller.GetGeneration()) { - LOG_W("Cannot run worker" - << ": controller# " << controller.GetTabletId() - << ", generation# " << controller.GetGeneration() - << ", worker# " << id - << ", reason# " << R"("generation mismatch")"); + YDB_LOG_WARN("Cannot run worker", + {"controller", controller.GetTabletId()}, + {"generation", controller.GetGeneration()}, + {"worker", id}, + {"reason", R"("generation mismatch")"}); return; } @@ -692,8 +694,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { return session.SendWorkerStatus(this, id, NKikimrReplication::TEvWorkerStatus::STATUS_RUNNING); } - LOG_I("Run worker" - << ": worker# " << id); + YDB_LOG_INFO("Run worker", + {"worker", id}); const auto& cmd = record.GetCommand(); // TODO: validate settings @@ -710,7 +712,7 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { const auto& writerSettings = cmd.GetTransferWriter(); const auto* transferWriterFactory = AppData()->TransferWriterFactory.get(); if (!transferWriterFactory) { - LOG_C("Run transfer but TransferWriterFactory does not exists."); + YDB_LOG_CRIT("Run transfer but TransferWriterFactory does not exists"); return; } autoCommit = false; @@ -733,7 +735,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvService::TEvStopWorker::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; const auto& controller = record.GetController(); @@ -741,20 +744,20 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { auto it = Sessions.find(controller.GetTabletId()); if (it == Sessions.end()) { - LOG_W("Cannot stop worker" - << ": controller# " << controller.GetTabletId() - << ", worker# " << id - << ", reason# " << R"("unknown session")"); + YDB_LOG_WARN("Cannot stop worker", + {"controller", controller.GetTabletId()}, + {"worker", id}, + {"reason", R"("unknown session")"}); return; } auto& session = it->second; if (session.GetGeneration() != controller.GetGeneration()) { - LOG_W("Cannot stop worker" - << ": controller# " << controller.GetTabletId() - << ", generation# " << controller.GetGeneration() - << ", worker# " << id - << ", reason# " << R"("generation mismatch")"); + YDB_LOG_WARN("Cannot stop worker", + {"controller", controller.GetTabletId()}, + {"generation", controller.GetGeneration()}, + {"worker", id}, + {"reason", R"("generation mismatch")"}); return; } @@ -762,14 +765,15 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { return session.SendWorkerStatus(this, id, NKikimrReplication::TEvWorkerStatus::STATUS_STOPPED); } - LOG_I("Stop worker" - << ": worker# " << id); + YDB_LOG_INFO("Stop worker", + {"worker", id}); WorkerActorIdToSession.erase(session.GetWorkerActorId(id)); session.StopWorker(this, id); } void Handle(TEvService::TEvGetTxId::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); auto* session = SessionFromWorker(ev->Sender); if (!session) { @@ -777,8 +781,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } if (!session->HasWorker(ev->Sender)) { - LOG_E("Cannot find worker" - << ": worker# " << ev->Sender); + YDB_LOG_ERROR("Cannot find worker", + {"worker", ev->Sender}); return; } @@ -786,25 +790,26 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvService::TEvTxIdResult::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; const auto& controller = record.GetController(); auto it = Sessions.find(controller.GetTabletId()); if (it == Sessions.end()) { - LOG_W("Cannot process tx id result" - << ": controller# " << controller.GetTabletId() - << ", reason# " << R"("unknown session")"); + YDB_LOG_WARN("Cannot process tx id result", + {"controller", controller.GetTabletId()}, + {"reason", R"("unknown session")"}); return; } auto& session = it->second; if (session.GetGeneration() != controller.GetGeneration()) { - LOG_W("Cannot process tx id result" - << ": controller# " << controller.GetTabletId() - << ", generation# " << controller.GetGeneration() - << ", reason# " << R"("generation mismatch")"); + YDB_LOG_WARN("Cannot process tx id result", + {"controller", controller.GetTabletId()}, + {"generation", controller.GetGeneration()}, + {"reason", R"("generation mismatch")"}); return; } @@ -812,7 +817,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvService::TEvHeartbeat::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); auto* session = SessionFromWorker(ev->Sender); if (!session) { @@ -820,19 +826,20 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } if (!session->HasWorker(ev->Sender)) { - LOG_E("Cannot find worker" - << ": worker# " << ev->Sender); + YDB_LOG_ERROR("Cannot find worker", + {"worker", ev->Sender}); return; } - LOG_I("Heartbeat" - << ": worker# " << ev->Sender - << ", version# " << TRowVersion::FromProto(ev->Get()->Record.GetVersion())); + YDB_LOG_INFO("Heartbeat", + {"worker", ev->Sender}, + {"version", TRowVersion::FromProto(ev->Get()->Record.GetVersion())}); session->Handle(this, ev); } void Handle(TEvWorker::TEvDataEnd::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto* session = SessionFromWorker(ev->Sender); if (!session) { @@ -840,19 +847,20 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } if (!session->HasWorker(ev->Sender)) { - LOG_E("Cannot find worker" - << ": worker# " << ev->Sender); + YDB_LOG_ERROR("Cannot find worker", + {"worker", ev->Sender}); return; } - LOG_I("Worker has ended" - << ": worker# " << ev->Sender); + YDB_LOG_INFO("Worker has ended", + {"worker", ev->Sender}); session->SendWorkerDataEnd(this, session->GetWorkerId(ev->Sender), ev->Get()->PartitionId, std::move(ev->Get()->AdjacentPartitionsIds), std::move(ev->Get()->ChildPartitionsIds)); } void Handle(TEvWorker::TEvStatsWakeup::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Get()->SessionToAdd) { SessionsToWake.insert(ev->Get()->SessionToAdd); @@ -879,7 +887,8 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void Handle(TEvWorker::TEvGone::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto* session = SessionFromWorker(ev->Sender); if (!session) { @@ -887,19 +896,20 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } if (!session->HasWorker(ev->Sender)) { - LOG_E("Cannot find worker" - << ": worker# " << ev->Sender); + YDB_LOG_ERROR("Cannot find worker", + {"worker", ev->Sender}); return; } - LOG_I("Worker has gone" - << ": worker# " << ev->Sender); + YDB_LOG_INFO("Worker has gone", + {"worker", ev->Sender}); WorkerActorIdToSession.erase(ev->Sender); session->StopWorker(this, ev->Sender, ToReason(ev->Get()->Status), ev->Get()->ErrorDescription); } void Handle(TEvWorker::TEvStatus::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto* session = SessionFromWorker(ev->Sender); if (session && session->HasWorker(ev->Sender)) { @@ -915,16 +925,16 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { TSessionInfo* SessionFromWorker(const TActorId& id) { auto wit = WorkerActorIdToSession.find(id); if (wit == WorkerActorIdToSession.end()) { - LOG_W("Unknown worker has gone" - << ": worker# " << id); + YDB_LOG_WARN("Unknown worker has gone", + {"worker", id}); return nullptr; } auto it = Sessions.find(wit->second); if (it == Sessions.end()) { - LOG_E("Cannot find session" - << ": worker# " << id - << ", session# " << wit->second); + YDB_LOG_ERROR("Cannot find session", + {"worker", id}, + {"session", wit->second}); return nullptr; } @@ -941,6 +951,7 @@ class TReplicationService: public TActorBootstrapped<TReplicationService> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); if (auto actorId = std::exchange(BoardPublisher, {})) { Send(actorId, new TEvents::TEvPoison()); } @@ -966,11 +977,14 @@ public: } void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); Become(&TThis::StateWork); RunBoardPublisher(); } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvService::TEvHandshake, Handle); hFunc(TEvService::TEvRunWorker, Handle); @@ -987,7 +1001,6 @@ public: } private: - mutable TMaybe<TString> LogPrefix; TActorId BoardPublisher; THashMap<ui64, TSessionInfo> Sessions; diff --git a/ydb/core/tx/replication/service/topic_reader.cpp b/ydb/core/tx/replication/service/topic_reader.cpp index f2455c093ec..7918356ebf2 100644 --- a/ydb/core/tx/replication/service/topic_reader.cpp +++ b/ydb/core/tx/replication/service/topic_reader.cpp @@ -1,4 +1,3 @@ -#include "logging.h" #include "topic_reader.h" #include "topic_reader_stats.h" #include "worker.h" @@ -9,31 +8,31 @@ #include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> #include <ydb/library/actors/core/actor.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> #include <util/generic/maybe.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::REPLICATION_SERVICE + namespace NKikimr::NReplication::NService { class TRemoteTopicReader: public TActor<TRemoteTopicReader> { - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[RemoteTopicReader]" - << "[" << Settings.GetBase().Topics_[0].Path_ << "]" - << "[" << Settings.GetBase().Topics_[0].PartitionIds_[0] << "]" - << SelfId() << " "; - } - - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "RemoteTopicReader"}, + {"actorActivityType", ActorActivityType()}, + {"path", Settings.GetBase().Topics_[0].Path_}, + {"partitionsIds", Settings.GetBase().Topics_[0].PartitionIds_[0]}, + {"selfId", SelfId()}); } void Handle(TEvWorker::TEvHandshake::TPtr& ev) { Worker = ev->Sender; CreatingReadSessionInProgress = true; - LOG_D("Handshake" - << ": worker# " << Worker); + YDB_LOG_DEBUG("Handshake", + {"worker", Worker}); Y_ABORT_UNLESS(!ReadSession); Send(YdbProxy, new TEvYdbProxy::TEvCreateTopicReaderRequest(Settings)); @@ -42,8 +41,8 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { void Handle(TEvYdbProxy::TEvCreateTopicReaderResponse::TPtr& ev) { ReadSession = ev->Get()->Result; CreatingReadSessionInProgress = false; - LOG_D("Create read session" - << ": session# " << ReadSession); + YDB_LOG_DEBUG("Create read session", + {"session", ReadSession}); if (StoppingInProgress) { return PassAway(); @@ -54,7 +53,8 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { } void Handle(TEvWorker::TEvPoll::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); Y_ABORT_UNLESS(ReadSession); auto settings = TEvYdbProxy::TReadTopicSettings() @@ -68,7 +68,8 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { } void Handle(TEvYdbProxy::TEvReadTopicResponse::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); Y_ABORT_UNLESS(!ReadQueue.empty()); TResponseDataTracker req{GetElapsedTicksAsSeconds(), Now() - ReadQueue.front().ReadStartTime, ev}; @@ -141,20 +142,23 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { } void Handle(TEvYdbProxy::TEvEndTopicPartition::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); auto& result = ev->Get()->Result; Send(Worker, new TEvWorker::TEvDataEnd(result.PartitionId, std::move(result.AdjacentPartitionsIds), std::move(result.ChildPartitionsIds))); } void Handle(TEvYdbProxy::TEvStartTopicReadingSession::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); ReadSessionId = ev->Get()->Result.ReadSessionId; } void Handle(TEvWorker::TEvCommit::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (!YdbProxy || !ReadSessionId) { return Leave(TEvWorker::TEvGone::UNAVAILABLE); @@ -166,10 +170,13 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { void Handle(TEvYdbProxy::TEvCommitOffsetResponse::TPtr& ev) { if (!ev->Get()->Result.IsSuccess()) { - LOG_W("Handle " << ev->Get()->ToString()); + YDB_LOG_WARN("Handle", + {"ev", ev->Get()->ToString()}); return Leave(TEvWorker::TEvGone::UNAVAILABLE); } else { - LOG_D("Handle " << CommittedOffset << " " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"committedOffset", CommittedOffset}, + {"ev", ev->Get()->ToString()}); if (CommittedOffset) { Send(ReadSession, CreateCommitOffsetRequest().release()); } @@ -188,7 +195,8 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { } void Handle(TEvYdbProxy::TEvTopicReaderGone::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (Settings.ReportStats_) { SendError(); } @@ -209,19 +217,21 @@ class TRemoteTopicReader: public TActor<TRemoteTopicReader> { case DecompressionDoneWakeupTag: return ProcessData(); default: - LOG_W("Handle Wakeup with unexpected tag " << ev->Get()->Tag); + YDB_LOG_WARN("Handle Wakeup with unexpected tag", + {"tag", ev->Get()->Tag}); } } template <typename... Args> void Leave(Args&&... args) { - LOG_I("Leave"); + YDB_LOG_INFO("Leave"); Send(Worker, new TEvWorker::TEvGone(std::forward<Args>(args)...)); PassAway(); } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); if (CreatingReadSessionInProgress) { StoppingInProgress = true; return; @@ -261,6 +271,8 @@ public: } STFUNC(StateWork) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix(), + {"actorStateFunc", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvWorker::TEvHandshake, Handle); hFunc(TEvWorker::TEvPoll, Handle); diff --git a/ydb/core/tx/replication/service/worker.cpp b/ydb/core/tx/replication/service/worker.cpp index 2afdfc2e7bf..0d4539faa5f 100644 --- a/ydb/core/tx/replication/service/worker.cpp +++ b/ydb/core/tx/replication/service/worker.cpp @@ -1,4 +1,3 @@ -#include "logging.h" #include "service.h" #include "topic_reader_stats.h" #include "worker.h" @@ -9,12 +8,15 @@ #include <ydb/core/tx/replication/ydb_proxy/topic_message.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> #include <util/generic/maybe.h> #include <util/string/builder.h> #include <util/string/join.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::REPLICATION_SERVICE + namespace NKikimr::NReplication::NService { TEvWorker::TEvPoll::TEvPoll(bool skipCommit) @@ -169,30 +171,27 @@ class TWorker: public TActorBootstrapped<TWorker> { } }; - TStringBuf GetLogPrefix() const { - if (!LogPrefix) { - LogPrefix = TStringBuilder() - << "[Worker]" - << SelfId() << " "; - } - - return LogPrefix.GetRef(); + NActors::NStructuredLog::TStructuredMessage GetLogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "Worker"}, + {"selfId", SelfId()}); } void Handle(TEvWorker::TEvHandshake::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Sender == Reader) { - LOG_I("Handshake with reader" - << ": sender# " << ev->Sender); + YDB_LOG_INFO("Handshake with reader", + {"sender", ev->Sender}); Reader.Registered(); if (!InFlightData && !TerminateWriter) { Send(Reader, new TEvWorker::TEvPoll()); } } else if (ev->Sender == Writer) { - LOG_I("Handshake with writer" - << ": sender# " << ev->Sender); + YDB_LOG_INFO("Handshake with writer", + {"sender", ev->Sender}); Writer.Registered(); if (InFlightData) { @@ -201,18 +200,19 @@ class TWorker: public TActorBootstrapped<TWorker> { Send(Writer, new TEvWorker::TEvTerminateWriter(TerminateWriter->PartitionId)); } } else { - LOG_W("Handshake from unknown actor" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Handshake from unknown actor", + {"sender", ev->Sender}); return; } } void Handle(TEvWorker::TEvPoll::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Sender != Writer) { - LOG_W("Poll from unknown actor" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Poll from unknown actor", + {"sender", ev->Sender}); return; } @@ -235,11 +235,12 @@ class TWorker: public TActorBootstrapped<TWorker> { } void Handle(TEvWorker::TEvCommit::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Sender != Writer) { - LOG_W("Commit from unknown actor" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Commit from unknown actor", + {"sender", ev->Sender}); return; } @@ -249,11 +250,12 @@ class TWorker: public TActorBootstrapped<TWorker> { } void Handle(TEvWorker::TEvData::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Sender != Reader) { - LOG_W("Data from unknown actor" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Data from unknown actor", + {"sender", ev->Sender}); return; } @@ -270,11 +272,12 @@ class TWorker: public TActorBootstrapped<TWorker> { } void Handle(TEvWorker::TEvTerminateWriter::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (ev->Sender != Reader) { - LOG_W("Terminate writer from unknown actor" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Terminate writer from unknown actor", + {"sender", ev->Sender}); return; } @@ -288,26 +291,27 @@ class TWorker: public TActorBootstrapped<TWorker> { void Handle(TEvWorker::TEvGone::TPtr& ev) { if (ev->Sender == Reader) { - LOG_I("Reader has gone" - << ": sender# " << ev->Sender - << ": " << ev->Get()->ToString()); + YDB_LOG_INFO("Reader has gone", + {"sender", ev->Sender}, + {"ev", ev->Get()->ToString()}); MaybeRecreateActor(ev, Reader); } else if (ev->Sender == Writer) { - LOG_I("Writer has gone" - << ": sender# " << ev->Sender - << ": " << ev->Get()->ToString()); + YDB_LOG_INFO("Writer has gone", + {"sender", ev->Sender}, + {"ev", ev->Get()->ToString()}); MaybeRecreateActor(ev, Writer); } else { - LOG_W("Unknown actor has gone" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Unknown actor has gone", + {"sender", ev->Sender}); } } void Handle(TEvWorker::TEvStatus::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); if (!ev->Get()->DetailedStats) { - LOG_W("Unexpected TEvWorker::TEvStatus with no stats, ignored" - << ": sender# " << ev->Sender); + YDB_LOG_WARN("Unexpected TEvWorker::TEvStatus with no stats, ignored", + {"sender", ev->Sender}); return; } Forward(ev); @@ -333,9 +337,9 @@ class TWorker: public TActorBootstrapped<TWorker> { } void Leave(TEvWorker::TEvGone::TPtr& ev) { - LOG_I("Leave" - << ": status# " << ev->Get()->Status - << ", error# " << ev->Get()->ErrorDescription); + YDB_LOG_INFO("Leave", + {"status", ev->Get()->Status}, + {"error", ev->Get()->ErrorDescription}); ev->Sender = SelfId(); Send(ev->Forward(Parent)); @@ -344,13 +348,15 @@ class TWorker: public TActorBootstrapped<TWorker> { } void Handle(TEvService::TEvTxIdResult::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); Send(ev->Forward(Writer)); } template <typename TEventPtr> void Forward(TEventPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); ev->Sender = SelfId(); Send(ev->Forward(Parent)); @@ -406,6 +412,7 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(GetLogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TEvWorker::TEvHandshake, Handle); hFunc(TEvWorker::TEvPoll, Handle); @@ -428,7 +435,6 @@ private: static constexpr TDuration LagReportInterval = TDuration::Seconds(7); const TActorId Parent; - mutable TMaybe<TString> LogPrefix; TActorInfo Reader; TActorInfo Writer; THolder<TEvWorker::TEvData> InFlightData; diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp index bafe47aeaca..2da9da71ff4 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp @@ -1,5 +1,7 @@ #include "local_partition_actor.h" -#include "logging.h" +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY namespace NKikimr::NReplication { @@ -17,6 +19,7 @@ TBaseLocalTopicPartitionActor::TBaseLocalTopicPartitionActor( void TBaseLocalTopicPartitionActor::Bootstrap() { LogPrefix = MakeLogPrefix(); + YDB_LOG_CREATE_CONTEXT(LogPrefix); DoDescribe(TopicPath); } @@ -34,7 +37,8 @@ TString TBaseLocalTopicPartitionActor::MakeAbsolutePath(TString path) const { void TBaseLocalTopicPartitionActor::DoDescribe(const TString& topicPath) { auto path = MakeAbsolutePath(topicPath); - LOG_D("Describe topic '" << path << "'"); + YDB_LOG_DEBUG("Describe topic", + {"path", path}); auto request = MakeHolder<TNavigate>(); request->DatabaseName = Database; @@ -45,7 +49,8 @@ void TBaseLocalTopicPartitionActor::DoDescribe(const TString& topicPath) { } void TBaseLocalTopicPartitionActor::Handle(TEvNavigateResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& result = ev->Get()->Request; static const TString errorMarker = "LocalYdbProxy"; @@ -108,6 +113,8 @@ TSchemeCacheHelpers::TCheckFailFunc TBaseLocalTopicPartitionActor::LeaveOnError( } STATEFN(TBaseLocalTopicPartitionActor::StateDescribe) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateDescribe"}); switch (ev->GetTypeRewrite()) { hFunc(TEvNavigateResult, Handle); hFunc(TEvents::TEvWakeup, HandleOnDescribe); @@ -119,7 +126,8 @@ STATEFN(TBaseLocalTopicPartitionActor::StateDescribe) { } void TBaseLocalTopicPartitionActor::DoCreatePipe() { - LOG_T("Create pipe to " << PartitionTabletId); + YDB_LOG_TRACE("Create pipe", + {"partitionTabletId", PartitionTabletId}); Attempt = 0; CreatePipe(); @@ -133,7 +141,8 @@ void TBaseLocalTopicPartitionActor::CreatePipe() { } void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& msg = *ev->Get(); if (msg.Status != NKikimrProto::OK) { @@ -143,17 +152,20 @@ void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TP return CreatePipe(); } - LOG_T("Pipe has been connected"); + YDB_LOG_TRACE("Pipe has been connected"); OnDescribeFinished(); } void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); OnError("Pipe destroyed"); } STATEFN(TBaseLocalTopicPartitionActor::StateCreatePipe) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateCreatePipe"}); switch (ev->GetTypeRewrite()) { hFunc(TEvTabletPipe::TEvClientConnected, Handle); hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h index f7cf7721a8c..111a37ccee4 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h @@ -21,7 +21,7 @@ protected: virtual void OnError(const TString& error) = 0; virtual void OnFatalError(const TString& error) = 0; virtual STATEFN(OnInitEvent) = 0; - virtual TString MakeLogPrefix() = 0; + virtual NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() = 0; }; class TBaseLocalTopicPartitionActor @@ -71,7 +71,7 @@ protected: TActorId PartitionPipeClient; size_t Attempt = 0; - TString LogPrefix; + NActors::NStructuredLog::TStructuredMessage LogPrefix; }; } diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp index c0981576f55..f8c47e393ad 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp @@ -1,11 +1,14 @@ #include "local_partition_actor.h" #include "local_proxy.h" -#include "logging.h" #include <ydb/core/persqueue/events/global.h> #include <ydb/core/persqueue/writer/common.h> #include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { class TLocalTopicPartitionCommitActor: public TBaseLocalTopicPartitionActor { @@ -41,8 +44,13 @@ protected: PassAway(); } - TString MakeLogPrefix() override { - return TStringBuilder() << "Committer[" << SelfId() << ":/" << Database << TopicPath <<" ] "; + NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() override { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionCommitActor"}, + {"selfId", SelfId()}, + {"database", Database}, + {"topicPath", TopicPath}, + ); } static std::unique_ptr<TEvYdbProxy::TEvCommitOffsetResponse> MakeResponse(NYdb::EStatus status, const TString& error) { @@ -60,14 +68,15 @@ protected: private: void DoCommitOffset() { - LOG_T("DoCommit"); + YDB_LOG_TRACE("DoCommit"); NTabletPipe::SendData(SelfId(), PartitionPipeClient, MakeCommitRequest().release()); Become(&TLocalTopicPartitionCommitActor::StateCommitOffset); } void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; @@ -94,6 +103,8 @@ private: } STATEFN(StateCommitOffset) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateCommitOffset"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, Handle); @@ -112,7 +123,8 @@ private: }; // TLocalTopicPartitionCommitActor void TLocalProxyActor::Handle(TEvYdbProxy::TEvCommitOffsetRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [topicName, partitionId, consumerName, offset, settings] = std::move(ev->Get()->GetArgs()); RegisterWithSameMailbox(new TLocalTopicPartitionCommitActor( diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp index 5bbd1f42f98..6037640f61f 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp @@ -1,12 +1,15 @@ #include "local_partition_actor.h" #include "local_proxy.h" -#include "logging.h" #include <ydb/core/persqueue/writer/common.h> #include <ydb/core/persqueue/events/global.h> #include <ydb/core/protos/grpc_pq_old.pb.h> #include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { class TLocalTopicPartitionReaderActor: public TBaseLocalTopicPartitionActor { @@ -52,8 +55,12 @@ protected: PassAway(); } - TString MakeLogPrefix() override { - return TStringBuilder() << "Reader[" << SelfId() << ":/" << Database << TopicPath <<" ] "; + NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() override { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionReaderActor"}, + {"selfId", SelfId()}, + {"database", Database}, + {"topicPath", TopicPath}); } static std::unique_ptr<TEvYdbProxy::TEvTopicReaderGone> MakeError(NYdb::EStatus status, const TString& error) { @@ -63,6 +70,8 @@ protected: } STATEFN(OnInitEvent) override { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "OnInitEvent"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvReadTopicRequest, HandleInit); hFunc(TEvYdbProxy::TEvCommitOffsetRequest, Handle); @@ -79,12 +88,14 @@ private: } void HandleInit(TEvYdbProxy::TEvReadTopicRequest::TPtr& ev) { - LOG_T("Handle on init " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle on init", + {"ev", ev->Get()->ToString()}); RequestsQueue.emplace_back(ev->Sender, ev->Cookie, GetSkipCommit(ev)); } void Handle(TEvYdbProxy::TEvCommitOffsetRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); } private: @@ -94,7 +105,8 @@ private: } void HandleOnInitOffset(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& record = ev->Get()->Record; if (record.GetErrorCode() == NPersQueue::NErrorCode::INITIALIZING) { @@ -137,6 +149,8 @@ private: } STATEFN(StateInitOffset) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateInitOffset"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, HandleOnInitOffset); hFunc(TEvents::TEvWakeup, HandleOnInitOffset); @@ -158,7 +172,8 @@ private: } void Handle(TEvYdbProxy::TEvReadTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); HandleInit(ev); Handle(RequestsQueue.front()); @@ -204,6 +219,8 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvReadTopicRequest, Handle); hFunc(TEvYdbProxy::TEvCommitOffsetRequest, Handle); @@ -226,7 +243,8 @@ private: } void HandleOnWaitData(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; @@ -316,6 +334,8 @@ private: } STATEFN(StateWaitData) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWaitData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, HandleOnWaitData); @@ -339,7 +359,8 @@ private: }; // TLocalTopicPartitionReaderActor void TLocalProxyActor::Handle(TEvYdbProxy::TEvCreateTopicReaderRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto args = std::move(ev->Get()->GetArgs()); auto& settings = std::get<TEvYdbProxy::TTopicReaderSettings>(args); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp index a087fb7cb92..2bbe23da69c 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp @@ -1,12 +1,14 @@ #include "local_proxy.h" #include "local_proxy_request.h" -#include "logging.h" #include <ydb/core/grpc_services/service_scheme.h> #include <ydb/core/grpc_services/service_topic.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yverify_stream/yverify_stream.h> #include <ydb/public/api/protos/ydb_table.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { TLocalProxyActor::TLocalProxyActor(const TString& database) @@ -15,7 +17,10 @@ TLocalProxyActor::TLocalProxyActor(const TString& database) } void TLocalProxyActor::Bootstrap() { - LogPrefix = TStringBuilder() << "[" << SelfId() << ":" << Database << "] "; + LogPrefix = YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionReaderActor"}, + {"selfId", SelfId()}, + {"database", Database}); Become(&TLocalProxyActor::StateWork); } @@ -65,7 +70,8 @@ auto CreateCallback(std::shared_ptr<T>&& ctx) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvAlterTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, settings] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -97,7 +103,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvAlterTopicRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, _] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -133,7 +140,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTopicRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, _] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -169,7 +177,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { - LOG_E("Handle " << ev->Get()->ToString()); + YDB_LOG_ERROR("Handle", + {"ev", ev->Get()->ToString()}); auto [path, settings] = std::move(ev->Get()->GetArgs()); @@ -182,6 +191,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { } STATEFN(TLocalProxyActor::StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWaitData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvCreateTopicReaderRequest, Handle); hFunc(TEvYdbProxy::TEvAlterTopicRequest, Handle); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h index 0781b99bbe6..b0c4089aaef 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h @@ -33,7 +33,7 @@ private: private: const TString Database; - TString LogPrefix; + NActors::NStructuredLog::TStructuredMessage LogPrefix; }; } diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h deleted file mode 100644 index 339a6597c23..00000000000 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include <ydb/library/actors/core/log.h> -#include <ydb/library/services/services.pb.h> - -#define CLOG_T(ctx, stream) LOG_TRACE_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_D(ctx, stream) LOG_DEBUG_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_I(ctx, stream) LOG_INFO_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_N(ctx, stream) LOG_NOTICE_S(ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_W(ctx, stream) LOG_WARN_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_E(ctx, stream) LOG_ERROR_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) - -#define LOG_T(stream) CLOG_T(*TlsActivationContext, stream) -#define LOG_D(stream) CLOG_D(*TlsActivationContext, stream) -#define LOG_I(stream) CLOG_I(*TlsActivationContext, stream) -#define LOG_N(stream) CLOG_N(*TlsActivationContext, stream) -#define LOG_W(stream) CLOG_W(*TlsActivationContext, stream) -#define LOG_E(stream) CLOG_E(*TlsActivationContext, stream) diff --git a/ydb/core/tx/scheme_board/backup.cpp b/ydb/core/tx/scheme_board/backup.cpp index c3a994bcb9e..2d3b88b40b7 100644 --- a/ydb/core/tx/scheme_board/backup.cpp +++ b/ydb/core/tx/scheme_board/backup.cpp @@ -20,6 +20,8 @@ #include <ranges> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_BACKUP + namespace NKikimr::NSchemeBoard { using namespace NJson; @@ -67,8 +69,11 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_BACKUP_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "backup"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TBackupActor"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } struct TPathDescriptionAggregate { @@ -123,6 +128,7 @@ public: } void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); try { OutputFile.ConstructInPlace(FilePath); } catch (...) { @@ -157,11 +163,10 @@ private: Send(MakeStateStorageProxyID(), new TEvStateStorage::TEvResolveSchemeBoard(path), 0, cookie); Schedule(DefaultTimeout, new TEvents::TEvWakeup(cookie)); - SBB_LOG_D("ProcessPaths" - << ", path: " << path - << ", cookie: " << cookie - << ", paths in progress: " << PathByCookie.size() - ); + YDB_LOG_DEBUG("ProcessPaths", + {"path", path}, + {"cookie", cookie}, + {"inProgress", PathByCookie.size()}); } SendProgressUpdate(); @@ -187,18 +192,22 @@ private: const ui64 cookie = ev->Cookie; auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_N("Unexpected cookie: " << cookie); + YDB_LOG_NOTICE("Unexpected cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } const TString& path = it->second; - SBB_LOG_D("Handle " << ev->Get()->ToString() << ", path: " << path); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"path", path}); const auto replicas = ev->Get()->GetPlainReplicas(); if (replicas.empty()) { - SBB_LOG_I("Empty replica list" - << ", path: " << path - ); + YDB_LOG_INFO("Empty replica list", + {"selfId", this->SelfId()}, + {"path", path}); EmptyReplicaList.emplace(path); return MarkPathCompleted(it); } @@ -232,12 +241,16 @@ private: const ui64 cookie = GetOriginalCookie(ev->Cookie); auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_D("Received description with inactive cookie: " << cookie); + YDB_LOG_DEBUG("Received description with inactive cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } const TString path = it->second; - SBB_LOG_D("Handle " << ev->Get()->ToString() << ", path: " << path); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"path", path}); TString jsonDescription = std::move(*ev->Get()->Record.MutableJson()); if (!jsonDescription.empty()) { @@ -251,13 +264,16 @@ private: const auto version = std::stoull(pathDescription["Self"]["PathVersion"].GetStringSafe()); auto aggregate = DescriptionsByCookie.find(cookie); if (aggregate == DescriptionsByCookie.end()) { - SBB_LOG_N("No description aggregate for cookie: " << cookie); + YDB_LOG_NOTICE("No description aggregate", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } if (!aggregate->second.AddDescription(std::move(jsonDescription), version, GetReplicaGroupIndex(ev->Cookie))) { - SBB_LOG_N("Invalid replica group: " << GetReplicaGroupIndex(ev->Cookie) - << ", cookie: " << ev->Cookie - ); + YDB_LOG_NOTICE("Invalid replica group", + {"selfId", this->SelfId()}, + {"group", GetReplicaGroupIndex(ev->Cookie)}, + {"cookie", ev->Cookie}); return; } if (aggregate->second.IsMajorityReached()) { @@ -269,7 +285,8 @@ private: MarkPathCompleted(it); } } else { - SBB_LOG_I("No version in path description"); + YDB_LOG_INFO("No version in path description", + {"selfId", this->SelfId()}); } } else { AddChildrenToPending(pathDescription, path); @@ -280,9 +297,10 @@ private: } } } catch (const std::exception& e) { - SBB_LOG_I("Parsing error: " << e.what() - << ", path: " << path - ); + YDB_LOG_INFO("Parsing error", + {"selfId", this->SelfId()}, + {"error", e.what()}, + {"path", path}); } } @@ -306,9 +324,11 @@ private: void AddChildrenToPending(const TJsonValue& pathDescription, const TString& path) { if (pathDescription.Has("Children")) { const auto& children = pathDescription["Children"]; - SBB_LOG_D("Queue children: " << JoinSeq(", ", children.GetArraySafe() | std::views::transform([](const TJsonValue& child) { - return child["Name"].GetStringSafe().Quote(); - }))); + YDB_LOG_DEBUG("Queue children", + {"selfId", this->SelfId()}, + {"children", JoinSeq(", ", children.GetArraySafe() | std::views::transform([](const TJsonValue& child) { + return child["Name"].GetStringSafe().Quote(); + }))}); for (const auto& child : children.GetArraySafe()) { if (child.Has("Name")) { TString childPath = TStringBuilder() << path << "/" << child["Name"].GetStringSafe(); @@ -328,12 +348,13 @@ private: auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { // assume already processed - SBB_LOG_D("Timeout with inactive cookie: " << cookie); + YDB_LOG_DEBUG("Timeout with inactive cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } - SBB_LOG_I("Timeout" - << ", path: " << it->second - ); + YDB_LOG_INFO("Timeout", + {"path", it->second}); Timeouts.emplace(it->second); MarkPathCompleted(it); if (RequireMajority) { @@ -345,28 +366,27 @@ private: const ui64 cookie = GetOriginalCookie(ev->Cookie); auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_N("Undelivered" - << ", unexpected cookie: " << cookie - ); + YDB_LOG_NOTICE("Undelivered", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } - SBB_LOG_I("Undelivered" - << ", path: " << it->second - ); + YDB_LOG_INFO("Undelivered", + {"selfId", this->SelfId()}, + {"path", it->second}); Undelivered.emplace(it->second); MarkPathCompleted(it); } void SendProgressUpdate() { - SBB_LOG_D("SendProgressUpdate" - << ", paths in progress: " << PathByCookie.size() - << ", processed paths: " << ProcessedPaths - << ", total paths: " << TotalPaths - << ", pending paths: " << PendingPaths.size() - << ", timeouts: " << Timeouts.size() - << ", delivery problems: " << Undelivered.size() - << ", empty replica list: " << EmptyReplicaList.size() - ); + YDB_LOG_DEBUG("SendProgressUpdate", + {"inProgress", PathByCookie.size()}, + {"processedPaths", ProcessedPaths}, + {"totalPaths", TotalPaths}, + {"pendingPaths", PendingPaths.size()}, + {"timeouts", Timeouts.size()}, + {"deliveryProblems", Undelivered.size()}, + {"emptyReplicaList", EmptyReplicaList.size()}); Send(Parent, new TSchemeBoardMonEvents::TEvBackupProgress(TotalPaths, ProcessedPaths)); } @@ -398,6 +418,7 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); hFunc(TSchemeBoardMonEvents::TEvDescribeResponse, Handle); @@ -433,8 +454,11 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_RESTORE_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "restore"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TRestoreActor"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } TRestoreActor(const TString& filePath, ui64 schemeShardId, ui64 generation, const TActorId& parent) @@ -446,6 +470,7 @@ public: } void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); try { InputFile.ConstructInPlace(FilePath); } catch (...) { @@ -476,7 +501,9 @@ private: TotalPaths++; descriptions.emplace_back(pathId, std::move(description)); if (TotalPaths % 1'000 == 0) { - SBB_LOG_D("Reading file, parsed paths descriptions: " << TotalPaths); + YDB_LOG_DEBUG("Reading file, parsed paths", + {"selfId", this->SelfId()}, + {"descriptions", TotalPaths}); SendProgressUpdate(); } } @@ -488,9 +515,9 @@ private: } void Populate(std::vector<std::pair<TPathId, NSchemeBoard::TTwoPartDescription>>&& descriptions) { - SBB_LOG_D("Populate" - << ", total paths: " << TotalPaths - ); + YDB_LOG_DEBUG("Populate", + {"selfId", this->SelfId()}, + {"paths", TotalPaths}); std::sort(descriptions.begin(), descriptions.end()); auto paths = descriptions | std::views::keys; PathsToProcess = TVector<TPathId>(paths.begin(), paths.end()); @@ -507,9 +534,12 @@ private: } void Handle(TSchemeBoardMonEvents::TEvInfoResponse::TPtr& ev) { - SBB_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", this->SelfId()}, + {"ev", ev->Get()->ToString()}); if (ev->Sender != Populator || !ev->Get()->Record.HasPopulatorResponse()) { - SBB_LOG_N("Unexpected info response"); + YDB_LOG_NOTICE("Unexpected info response", + {"selfId", this->SelfId()}); return; } @@ -545,6 +575,7 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvInfoResponse, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); diff --git a/ydb/core/tx/scheme_board/backup.h b/ydb/core/tx/scheme_board/backup.h index 7cfe216d6f5..cc6a3c9b069 100644 --- a/ydb/core/tx/scheme_board/backup.h +++ b/ydb/core/tx/scheme_board/backup.h @@ -11,13 +11,6 @@ struct TEvCommonResult; namespace NKikimr::NSchemeBoard { -#define SBB_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) - struct TCommonProgress { enum class EStatus { Idle, diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index c014544c089..32d6b4ac7d2 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -42,14 +42,11 @@ #include <google/protobuf/util/json_util.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_PROXY_SCHEME_CACHE + namespace NKikimr { namespace NSchemeBoard { -#define SBC_LOG_T(stream) SB_LOG_T(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_D(stream) SB_LOG_D(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_N(stream) SB_LOG_N(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_W(stream) SB_LOG_W(TX_PROXY_SCHEME_CACHE, stream) - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; using TEvNavigateResult = TEvTxProxySchemeCache::TEvNavigateKeySetResult; using TNavigate = NSchemeCache::TSchemeCacheNavigate; @@ -263,10 +260,10 @@ namespace { } void SendResult() { - SBC_LOG_D("Send result" - << ": self# " << this->SelfId() - << ", recipient# " << Context->Sender - << ", result# " << Context->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Send result", + {"self", this->SelfId()}, + {"recipient", Context->Sender}, + {"result", Context->Request->ToString(*AppData()->TypeRegistry)}); this->Send(Context->Sender, new TEvResult(Context->Request.Release()), 0, Context->Cookie); this->PassAway(); @@ -302,10 +299,10 @@ namespace { ) { if (Context->ResolvedDomainInfo->DomainKey != entry.DomainInfo->DomainKey) { - SBC_LOG_W("Path does not belong to the specified domain" - << ": self# " << this->SelfId() - << ", domain# " << Context->ResolvedDomainInfo->DomainKey - << ", path's domain# " << entry.DomainInfo->DomainKey); + YDB_LOG_WARN("Path does not belong to the specified domain", + {"self", this->SelfId()}, + {"contextDomain", Context->ResolvedDomainInfo->DomainKey}, + {"entryDomain", entry.DomainInfo->DomainKey}); SetErrorAndClear(Context.Get(), entry, false); } @@ -319,10 +316,10 @@ namespace { const ui32 access = GetAccess(entry); if (!securityObject->CheckAccess(access, *token)) { - SBC_LOG_W("Access denied" - << ": self# " << this->SelfId() - << ", for# " << token->GetUserSID() - << ", access# " << NACLib::AccessRightsToString(access)); + YDB_LOG_WARN("Access denied", + {"self", this->SelfId()}, + {"sid", token->GetUserSID()}, + {"access", NACLib::AccessRightsToString(access)}); const auto hasDescribeAccess = securityObject->CheckAccess(NACLib::DescribeSchema, *token); SetErrorAndClear(Context.Get(), entry, hasDescribeAccess); @@ -557,11 +554,11 @@ namespace { private: template <typename TPath> TActorId CreateSubscriber(const TPath& path, const ui64 tabletId, const ui64 domainOwnerId) const { - SBC_LOG_T("Create subscriber" - << ": self# " << SelfId() - << ", path# " << path - << ", tabletId# " << tabletId - << ", domainOwnerId# " << domainOwnerId); + YDB_LOG_TRACE("Create subscriber", + {"self", SelfId()}, + {"path", path}, + {"tabletId", tabletId}, + {"domainOwnerId", domainOwnerId}); return Register(CreateSchemeBoardSubscriber(SelfId(), path, domainOwnerId)); } @@ -1915,11 +1912,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = {}) const { - SBC_LOG_D("FillEntry for TNavigate" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); + YDB_LOG_DEBUG("FillEntry for TNavigate", + {"self", Owner->SelfId()}, + {"cacheItem", ToString()}, + {"entry", entry}, + {"props", props}); if (props.IsSync && props.Partial) { return SetError(context, entry, TNavigate::EStatus::LookupError); @@ -2145,11 +2142,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = {}) const { - SBC_LOG_D("FillEntry for TResolve" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); + YDB_LOG_DEBUG("FillEntry for TResolve", + {"self", Owner->SelfId()}, + {"cacheItem", ToString()}, + {"entry", entry}, + {"props", props}); TKeyDesc& keyDesc = *entry.KeyDescription; @@ -2416,10 +2413,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { template <typename TPath> TSubscriber CreateSubscriber(const TPath& path, const ui64 domainOwnerId) const { - SBC_LOG_T("Create subscriber" - << ": self# " << SelfId() - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId); + YDB_LOG_TRACE("Create subscriber", + {"self", SelfId()}, + {"path", path}, + {"domainOwnerId", domainOwnerId}); return TSubscriber( Register(CreateSchemeBoardSubscriber(SelfId(), path, domainOwnerId)), domainOwnerId, path @@ -2557,11 +2554,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TCacheItem* byPath = Cache.FindPtr(notify.Path); TCacheItem* byPathId = Cache.FindPtr(notify.PathId); - SBC_LOG_D("ResolveCacheItem" - << ": self# " << SelfId() - << ", notify# " << notify.ToString() - << ", by path# " << (byPath ? byPath->ToString() : "nullptr") - << ", by pathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItem", + {"self", SelfId()}, + {"notify", notify}, + {"path", (byPath ? byPath->ToString() : "nullptr")}, + {"pathId", (byPathId ? byPathId->ToString() : "nullptr")}); TCacheItem* commonResolve = ResolveCacheItemCommon(notify.PathId, notify.Path); if (commonResolve) { @@ -2579,20 +2576,22 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } - SBC_LOG_D("ResolveCacheItemForNotify: subdomain case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "subdomain"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (byPath->GetDomainId() != notifyDomainId && notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: recreation domain case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "recreation domain"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (PathIdLessThan(byPath->GetPathId(), notify.PathId)) { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); @@ -2603,51 +2602,57 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { if (byPath->GetPathId() == notifyDomainId) { //Update from TSS, GSS->TSS if (byPath->GetAbandonedSchemeShardIds().contains(notify.PathId.OwnerId)) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update is ignored, present GSS reverted implicitly that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from TSS is ignored, present GSS reverted implicitly that TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update overrides GSS by path" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from TSS overrides GSS by path"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } if (byPath->GetDomainId() == notify.PathId) { //Update from GSS, TSS->GSS if (abandonedSchemeShardIds.contains(byPath->GetPathId().OwnerId)) { //GSS reverts TSS - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update overrides TSS by path, GSS implicitly reverts that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS overrides TSS by path, GSS implicitly reverts that TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } if (!notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS that removes TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS that removes TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update is ignored, TSS is preferred" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS is ignored, TSS is preferred"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } if (byPath->GetDomainId() == notifyDomainId && notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: recreation migrated path case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "recreation migrated path"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (PathIdLessThan(byPath->GetPathId(), notify.PathId)) { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); @@ -2657,10 +2662,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } if (!notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: path has gone, update only by pathId" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "path has gone, update only by pathId"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } @@ -2684,9 +2690,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { const TResponseProps response = TResponseProps::FromEvent(ev); auto& notify = *ev->Get(); - SBC_LOG_D("HandleNotify" - << ": self# " << SelfId() - << ", notify# " << notify.ToString()); + YDB_LOG_DEBUG("HandleNotify", + {"self", SelfId()}, + {"notify", notify}); if (notify.Path && notify.PathId) { Y_ABORT_UNLESS(!response.IsSync); @@ -2695,11 +2701,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TCacheItem* cacheItem = ResolveCacheItemForNotify(notify); if (!cacheItem) { - SBC_LOG_W("HandleNotify doesn't find any cacheItem for Fill" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", isSync# " << response.IsSync); + YDB_LOG_WARN("HandleNotify doesn't find any cacheItem for Fill", + {"self", SelfId()}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"isSync", response.IsSync}); return; } @@ -2843,9 +2849,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // otherwise use original entry and remove fallback entry // - This approach allows to avoid adding new handlers to SchemeCache actor and changing ProcessInFlight behavior void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet" - << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvNavigateKeySet", + {"self", SelfId()}, + {"request", ev->Get()->Request->ToString(*AppData()->TypeRegistry)}); if (MaybeRunDbResolver(ev)) { return; @@ -2918,9 +2924,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void Handle(TEvTxProxySchemeCache::TEvResolveKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySet" - << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvResolveKeySet", + {"self", SelfId()}, + {"request", ev->Get()->Request->ToString(*AppData()->TypeRegistry)}); if (MaybeRunDbResolver(ev)) { return; @@ -2947,8 +2953,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void Handle(TEvTxProxySchemeCache::TEvInvalidateTable::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvInvalidateTable" - << ": self# " << SelfId()); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvInvalidateTable", + {"self", SelfId()}); Send(ev->Sender, new TEvTxProxySchemeCache::TEvInvalidateTableResult(ev->Get()->Sender), 0, ev->Cookie); } diff --git a/ydb/core/tx/scheme_board/helpers.h b/ydb/core/tx/scheme_board/helpers.h index 0adf8affda6..926035d8655 100644 --- a/ydb/core/tx/scheme_board/helpers.h +++ b/ydb/core/tx/scheme_board/helpers.h @@ -10,24 +10,6 @@ #include <util/generic/vector.h> #include <util/generic/set.h> -#if defined SB_LOG_T || \ - defined SB_LOG_D || \ - defined SB_LOG_I || \ - defined SB_LOG_N || \ - defined SB_LOG_W || \ - defined SB_LOG_E || \ - defined SB_LOG_CRIT -#error log macro redefinition -#endif - -#define SB_LOG_T(service, stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_D(service, stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_I(service, stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_N(service, stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_W(service, stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_E(service, stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_CRIT(service, stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) - namespace NKikimr { namespace NSchemeBoard { diff --git a/ydb/core/tx/scheme_board/monitoring.cpp b/ydb/core/tx/scheme_board/monitoring.cpp index d4e8d741a73..f818111345a 100644 --- a/ydb/core/tx/scheme_board/monitoring.cpp +++ b/ydb/core/tx/scheme_board/monitoring.cpp @@ -32,6 +32,8 @@ #include <util/string/join.h> #include <util/string/split.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_BACKUP + // additional html elements namespace NMonitoring { const char NavTag[] = "nav"; @@ -79,8 +81,15 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { static constexpr char ROOT[] = "scheme_board"; static constexpr TBackupLimits BackupLimits = TBackupLimits(); - static constexpr TStringBuf LogPrefix() { - return "monitoring"sv; + static constexpr auto ActorActivityType() { + return TActivity::SCHEME_BOARD_MONITORING_ACTOR; + } + + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TMonitoring"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } using TActivity = NKikimrServices::TActivity; @@ -1757,10 +1766,11 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { BackupProgress = TBackupProgress(); BackupProgress.Status = TBackupProgress::EStatus::Starting; - SBB_LOG_I("Starting backup to file: " << filePath - << ", in-flight limit: " << inFlightLimit - << ", require majority: " << requireMajority - ); + YDB_LOG_INFO("Starting backup", + {"selfId", this->SelfId()}, + {"filePath", filePath}, + {"inFlightLimit", inFlightLimit}, + {"majority", requireMajority}); Register(CreateSchemeBoardBackuper(filePath, inFlightLimit, requireMajority, SelfId())); @@ -1811,10 +1821,11 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { RestoreProgress = TRestoreProgress(); RestoreProgress.Status = TRestoreProgress::EStatus::Starting; - SBB_LOG_I("Starting restore from " << filePath - << " for SchemeShard ID: " << schemeShardId - << " of generation: " << generation - ); + YDB_LOG_INFO("Starting restore", + {"selfId", this->SelfId()}, + {"filePath", filePath}, + {"schemeShardId", schemeShardId}, + {"generation", generation}); Register(CreateSchemeBoardRestorer(filePath, schemeShardId, generation, SelfId())); @@ -1840,7 +1851,9 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { template <typename TProgress, typename TEventPtr> void Handle(TProgress& progress, TEventPtr& ev) { const auto& msg = *ev->Get(); - SBB_LOG_D("Handle " << msg.ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", this->SelfId()}, + {"ev", msg}); progress = TProgress(msg); } @@ -1861,10 +1874,6 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { } public: - static constexpr auto ActorActivityType() { - return TActivity::SCHEME_BOARD_MONITORING_ACTOR; - } - void Bootstrap() { if (auto* mon = AppData()->Mon) { auto* actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); @@ -1876,6 +1885,7 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvRegister, Handle); hFunc(TSchemeBoardMonEvents::TEvUnregister, Handle); diff --git a/ydb/core/tx/scheme_board/populator.cpp b/ydb/core/tx/scheme_board/populator.cpp index 6085cfad95d..e2a84317b29 100644 --- a/ydb/core/tx/scheme_board/populator.cpp +++ b/ydb/core/tx/scheme_board/populator.cpp @@ -27,15 +27,11 @@ #include <util/generic/vector.h> #include <util/generic/algorithm.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_POPULATOR + namespace NKikimr { namespace NSchemeBoard { -#define SBP_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_CRIT(stream) SB_LOG_CRIT(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) - namespace { using TDelayedUpdates = TVector<THolder<IEventHandle>>; @@ -91,7 +87,9 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { if (msg->HasDescription()) { if (msg->Description.Status != NKikimrScheme::StatusSuccess) { - SBP_LOG_E("Ignore description: " << msg->Description.ToString()); + YDB_LOG_ERROR("Ignore description", + {"selfId", SelfId()}, + {"description", msg->Description}); } else { CurPathId = msg->Description.PathId; update->SetDescribeSchemeResultSerialized(std::move(msg->Description.DescribeSchemeResultSerialized)); @@ -99,9 +97,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } if (msg->HasMigratedPath()) { - SBP_LOG_D("Ignore description of migrated path" - << ": owner# " << Owner - << ", localPathId# " << msg->MigratedPathId); + YDB_LOG_DEBUG("Ignore description of migrated path", + {"selfId", SelfId()}, + {"owner", Owner}, + {"localPathId", msg->MigratedPathId}); // this path should be described by another owner (tenant schemeshard) auto& migratedLocalPathIds = *update->Record.MutableMigratedLocalPathIds(); migratedLocalPathIds.SetBegin(msg->MigratedPathId); @@ -218,10 +217,11 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { template <typename TEvent, typename T> bool Check(TEvent& ev, T this_, T that, const TString& what) { if (this_ != that) { - SBP_LOG_E("Suspicious " << TypeName<TEvent>() - << ": sender# " << ev->Sender - << ", " << what << "# " << this_ - << ", other " << what << "# " << that); + YDB_LOG_ERROR(TStringBuilder() << "Suspicious " << what, + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"expected", this_}, + {"got", that}); return false; } @@ -239,8 +239,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvHandshakeResponse::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!CheckOwner(ev)) { return; @@ -248,20 +250,24 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { const ui64 generation = ev->Get()->Record.GetGeneration(); if (generation > Generation) { - SBP_LOG_CRIT("Keep calm"); + YDB_LOG_CRIT("Keep calm", + {"selfId", SelfId()}); Become(&TThis::StateCalm); } else { - SBP_LOG_N("Successful handshake" - << ": replica# " << ev->Sender); + YDB_LOG_NOTICE("Successful handshake", + {"selfId", SelfId()}, + {"replica", ev->Sender}); if (generation < Generation) { - SBP_LOG_N("Start full sync" - << ": replica# " << ev->Sender); + YDB_LOG_NOTICE("Start full sync", + {"selfId", SelfId()}, + {"replica", ev->Sender}); ProcessSync(); } else { - SBP_LOG_N("Resume sync" - << ": replica# " << ev->Sender - << ", fromPathId# " << LastAckedPathId.NextId()); + YDB_LOG_NOTICE("Resume sync", + {"selfId", SelfId()}, + {"replica", ev->Sender}, + {"fromPathId", LastAckedPathId.NextId()}); ResumeSync(LastAckedPathId.NextId()); } @@ -271,24 +277,30 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvDescribeResult::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); ProcessSync(ev->Get()); } void Handle(NInternalEvents::TEvUpdate::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); EnqueueUpdate(ev, true); } void Handle(NSchemeshardEvents::TEvUpdateAck::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (!CheckOwner(ev) || !CheckGeneration(ev)) { return; @@ -305,8 +317,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvCommitResponse::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!CheckOwner(ev) || !CheckGeneration(ev)) { return; @@ -539,16 +553,19 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(NInternalEvents::TEvRequestDescribe::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const TActorId replicaPopulator = ev->Sender; const TActorId replica = ev->Get()->Replica; if (ReplicaToReplicaPopulator[replica] != replicaPopulator) { - SBP_LOG_CRIT("Inconsistent replica populator" - << ": replica# " << replica - << ", replicaPopulator# " << replicaPopulator); + YDB_LOG_CRIT("Inconsistent replica populator", + {"selfId", SelfId()}, + {"replica", replica}, + {"replicaPopulator", replicaPopulator}); return; } @@ -634,8 +651,10 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(NInternalEvents::TEvRequestUpdate::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const TPathId pathId = ev->Get()->PathId; THolder<NInternalEvents::TEvUpdateBuilder> update; @@ -654,9 +673,11 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void DelayUpdate(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - SBP_LOG_D("DelayUpdate " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("DelayUpdate", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); DelayedUpdates.emplace_back(ev.Release()); } @@ -668,15 +689,17 @@ class TPopulator: public TMonitorableActor<TPopulator> { auto* msg = static_cast<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder*>(ev->Get()); auto& record = msg->Record; - SBP_LOG_D("Handle TEvSchemeShard::TEvDescribeSchemeResult { " << record.ShortDebugString() << " }" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", event size# " << msg->GetCachedByteSize() - << ", preserialized size# " << msg->PreSerializedData.size() - ); + YDB_LOG_DEBUG("Handle TEvSchemeShard::TEvDescribeSchemeResult", + {"selfId", SelfId()}, + {"ev", record.ShortDebugString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"eventSize", msg->GetCachedByteSize()}, + {"preserializedSize", msg->PreSerializedData.size()}); if (!record.HasStatus()) { - SBP_LOG_E("Description without status"); + YDB_LOG_ERROR("Description without status", + {"selfId", SelfId()}); return; } @@ -684,20 +707,21 @@ class TPopulator: public TMonitorableActor<TPopulator> { const bool isDeletion = record.GetStatus() == NKikimrScheme::StatusPathDoesNotExist; const ui64 version = isDeletion ? Max<ui64>() : NSchemeBoard::GetPathVersion(record); - SBP_LOG_N("Update description" - << ": owner# " << Owner - << ", pathId# " << pathId - << ", cookie# " << ev->Cookie - << ", is deletion# " << (isDeletion ? "true" : "false") - << ", version: " << (isDeletion ? 0 : version) - ); + YDB_LOG_NOTICE("Update description", + {"selfId", SelfId()}, + {"owner", Owner}, + {"pathId", pathId}, + {"cookie", ev->Cookie}, + {"isDeletion", (isDeletion ? "true" : "false")}, + {"version", (isDeletion ? 0 : version)}); if (isDeletion) { if (!Descriptions.contains(pathId)) { - SBP_LOG_N("Immediate ack for deleted path" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", pathId# " << pathId); + YDB_LOG_NOTICE("Immediate ack for deleted path", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"pathId", pathId}); auto ack = MakeHolder<NSchemeshardEvents::TEvUpdateAck>(Owner, Generation, pathId, Max<ui64>()); Send(ev->Sender, std::move(ack), 0, ev->Cookie); @@ -753,22 +777,28 @@ class TPopulator: public TMonitorableActor<TPopulator> { void Handle(NSchemeshardEvents::TEvUpdateAck::TPtr& ev) { const auto& record = ev->Get()->Record; - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto it = UpdateAcks.find(ev->Cookie); if (it == UpdateAcks.end()) { - SBP_LOG_D("Ack for unknown update (already acked?)" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Ack for unknown update", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"reason", "already acked"}); return; } TActorId* ackedReplica = ReplicaToReplicaPopulatorBackMap.FindPtr(ev->Sender); if (ackedReplica == nullptr) { - SBP_LOG_D("Ack from unknown replica (config changed?)" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Ack from unknown replica", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"reason", "config changed"}); return; } const TPathId pathId = ev->Get()->GetPathId(); @@ -779,11 +809,12 @@ class TPopulator: public TMonitorableActor<TPopulator> { && pathIt->first.first == pathId && pathIt->first.second <= version) { if (CheckQuorum(pathIt->second, *ackedReplica)) { - SBP_LOG_N("Ack update" - << ": ack to# " << it->second.AckTo - << ", cookie# " << ev->Cookie - << ", pathId# " << pathId - << ", version# " << pathIt->first.second); + YDB_LOG_NOTICE("Ack update", + {"selfId", SelfId()}, + {"to", it->second.AckTo}, + {"cookie", ev->Cookie}, + {"pathId", pathId}, + {"version", pathIt->first.second}); auto ack = MakeHolder<NSchemeshardEvents::TEvUpdateAck>(Owner, Generation, pathId, pathIt->first.second); Send(it->second.AckTo, std::move(ack), 0, ev->Cookie); @@ -803,14 +834,17 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& info = ev->Get()->Info; if (!info) { Y_ABORT_UNLESS(!GroupInfo); - SBP_LOG_E("Publish on unconfigured SchemeBoard"); + YDB_LOG_ERROR("Publish on unconfigured SchemeBoard", + {"selfId", SelfId()}); Become(&TThis::StateCalm); return; } @@ -914,7 +948,8 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void HandleUndelivered() { - SBP_LOG_E("Publish on unavailable SchemeBoard"); + YDB_LOG_ERROR("Publish on unavailable SchemeBoard", + {"selfId", SelfId()}); Become(&TThis::StateCalm); } diff --git a/ydb/core/tx/scheme_board/replica.cpp b/ydb/core/tx/scheme_board/replica.cpp index 24f53ffd506..bc755947ff2 100644 --- a/ydb/core/tx/scheme_board/replica.cpp +++ b/ydb/core/tx/scheme_board/replica.cpp @@ -27,15 +27,11 @@ #include <util/generic/variant.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_REPLICA + namespace NKikimr { namespace NSchemeBoard { -#define SBR_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) - class TReplica: public TMonitorableActor<TReplica> { using TDescribeSchemeResult = NKikimrScheme::TEvDescribeSchemeResult; using TCapabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities; @@ -313,9 +309,10 @@ public: << ": self# " << ToString() << ", other# " << other.ToString()); - SBR_LOG_T("Merge descriptions" - << ": self# " << ToLogString() - << ", other# " << other.ToLogString()); + YDB_LOG_TRACE("Merge descriptions", + {"selfId", SelfId()}, + {"self", ToLogString()}, + {"other", other.ToLogString()}); UntrackMemory(); other.UntrackMemory(); @@ -525,38 +522,40 @@ private: // register empty entry by path OR pathId template <typename TPath> TDescription& UpsertDescription(const TPath& path) { - SBR_LOG_I("Upsert description" - << ": path# " << path); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}); return Descriptions.Upsert(path, TDescription(this, path)); } // register empty entry by path AND pathId both TDescription& UpsertDescription(const TString& path, const TPathId& pathId) { - SBR_LOG_I("Upsert description" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId)); } // upsert description only by pathId TDescription& UpsertDescriptionByPathId(const TString& path, const TPathId& pathId, TOpaquePathDescription&& pathDescription) { - SBR_LOG_I("Upsert description" - << ": pathId# " << pathId - << ", pathDescription# " << pathDescription.ToString() - ); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"pathId", pathId}, + {"pathDescription", pathDescription}); return Descriptions.Upsert(pathId, TDescription(this, path, pathId, std::move(pathDescription))); } // upsert description by path AND pathId both TDescription& UpsertDescription(const TString& path, const TPathId& pathId, TOpaquePathDescription&& pathDescription) { - SBR_LOG_I("Upsert description" - << ": path# " << path - << ", pathId# " << pathId - << ", pathDescription# " << pathDescription.ToString() - ); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"pathDescription", pathDescription}); return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId, std::move(pathDescription))); } @@ -579,9 +578,10 @@ private: auto path = desc->GetPath(); - SBR_LOG_I("Delete description" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_INFO("Delete description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); if (TDescription* descByPath = Descriptions.FindPtr(path)) { if (descByPath != desc && !descByPath->IsEmpty()) { @@ -591,12 +591,13 @@ private: auto domainId = desc->GetDomainId(); if (curDomainId == pathId) { // Deletion from GSS - SBR_LOG_N("Delete description by GSS" - << ": path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + YDB_LOG_NOTICE("Delete description by GSS", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"domainId", domainId}, + {"curPathId", curPathId}, + {"curDomainId", curDomainId}); Descriptions.DeleteIndex(path); UpsertDescription(path, pathId); @@ -648,11 +649,12 @@ private: TDescription* desc = Descriptions.FindPtr(path); Y_ABORT_UNLESS(desc); - SBR_LOG_I("Subscribe" - << ": subscriber# " << subscriber - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId - << ", capabilities# " << capabilities.ShortDebugString()); + YDB_LOG_INFO("Subscribe", + {"selfId", SelfId()}, + {"subscriber", subscriber}, + {"path", path}, + {"domainOwnerId", domainOwnerId}, + {"capabilities", capabilities.ShortDebugString()}); desc->Subscribe(subscriber, path, domainOwnerId, capabilities); @@ -666,9 +668,10 @@ private: TDescription* desc = Descriptions.FindPtr(path); Y_ABORT_UNLESS(desc); - SBR_LOG_I("Unsubscribe" - << ": subscriber# " << subscriber - << ", path# " << path); + YDB_LOG_INFO("Unsubscribe", + {"selfId", SelfId()}, + {"subscriber", subscriber}, + {"path", path}); desc->Unsubscribe(subscriber); Subscribers.erase(subscriber); @@ -756,8 +759,10 @@ private: } void Handle(NInternalEvents::TEvHandshakeRequest::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& record = ev->Get()->Record; const ui64 owner = record.GetOwner(); @@ -765,17 +770,19 @@ private: TPopulatorInfo& info = Populators[owner]; if (generation < info.PendingGeneration) { - SBR_LOG_E("Reject handshake from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); + YDB_LOG_ERROR("Reject handshake from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", info.PendingGeneration}); return; } - SBR_LOG_N("Successful handshake" - << ": owner# " << owner - << ", generation# " << generation); + YDB_LOG_NOTICE("Successful handshake", + {"selfId", SelfId()}, + {"owner", owner}, + {"generation", generation}); info.PendingGeneration = generation; info.PopulatorActor = ev->Sender; @@ -784,11 +791,12 @@ private: } void Handle(NInternalEvents::TEvUpdate::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", event size# " << ev->Get()->GetCachedByteSize() - ); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"size", ev->Get()->GetCachedByteSize()}); const TString& path = ev->Get()->GetPath(); const TPathId& pathId = ev->Get()->GetPathId(); @@ -800,18 +808,20 @@ private: const auto populatorIt = Populators.find(owner); if (populatorIt == Populators.end()) { - SBR_LOG_E("Reject update from unknown populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); + YDB_LOG_ERROR("Reject update from unknown populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}); return; } if (generation != populatorIt->second.PendingGeneration) { - SBR_LOG_E("Reject update from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << populatorIt->second.PendingGeneration); + YDB_LOG_ERROR("Reject update from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", populatorIt->second.PendingGeneration}); return; } @@ -825,10 +835,11 @@ private: return AckUpdate(ev); } - SBR_LOG_N("Update description" - << ": path# " << path - << ", pathId# " << pathId - << ", deletion# " << (record.GetIsDeletion() ? "true" : "false")); + YDB_LOG_NOTICE("Update description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"deletion", (record.GetIsDeletion() ? "true" : "false")}); if (record.GetIsDeletion()) { SoftDeleteDescription(pathId, true); @@ -838,9 +849,10 @@ private: if (TDescription* desc = Descriptions.FindPtr(pathId)) { if (desc->IsExplicitlyDeleted()) { - SBR_LOG_N("Path was explicitly deleted, ignoring" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_NOTICE("Path was explicitly deleted, ignoring", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); return AckUpdate(ev); } @@ -885,12 +897,13 @@ private: const auto& domainId = pathDescription.SubdomainPathId; auto log = [&](const TString& message) { - SBR_LOG_N("" << message - << ": path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + YDB_LOG_NOTICE(message, + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"domainId", domainId}, + {"curPathId", curPathId}, + {"curDomainId", curDomainId}); }; if (curPathId == domainId) { // Update from TSS, GSS->TSS @@ -963,8 +976,10 @@ private: } void Handle(NInternalEvents::TEvCommitRequest::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& record = ev->Get()->Record; const ui64 owner = record.GetOwner(); @@ -972,26 +987,29 @@ private: auto it = Populators.find(owner); if (it == Populators.end()) { - SBR_LOG_E("Reject commit from unknown populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); + YDB_LOG_ERROR("Reject commit from unknown populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}); return; } TPopulatorInfo& info = it->second; if (generation != info.PendingGeneration) { - SBR_LOG_E("Reject commit from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); + YDB_LOG_ERROR("Reject commit from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", info.PendingGeneration}); return; } - SBR_LOG_N("Commit generation" - << ": owner# " << owner - << ", generation# " << generation); + YDB_LOG_NOTICE("Commit generation", + {"selfId", SelfId()}, + {"owner", owner}, + {"generation", generation}); info.Generation = info.PendingGeneration; info.IsCommited = true; @@ -1003,19 +1021,23 @@ private: } void Handle(TEvPrivate::TEvSendStrongNotifications::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}); const auto owner = ev->Get()->Owner; if (!IsPopulatorCommited(owner)) { - SBR_LOG_N("Populator is not commited" - << ": owner# " << owner); + YDB_LOG_NOTICE("Populator is not commited", + {"selfId", SelfId()}, + {"owner", owner}); return; } auto itSubscribers = WaitStrongNotifications.find(owner); if (itSubscribers == WaitStrongNotifications.end()) { - SBR_LOG_E("Invalid owner" - << ": owner# " << owner); + YDB_LOG_ERROR("Invalid owner", + {"selfId", SelfId()}, + {"owner", owner}); return; } @@ -1066,8 +1088,10 @@ private: const ui64 domainOwnerId = record.GetDomainOwnerId(); const auto& capabilities = record.GetCapabilities(); - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (record.HasPath()) { SubscribeBy(ev->Sender, record.GetPath(), domainOwnerId, capabilities); @@ -1080,8 +1104,10 @@ private: void Handle(NInternalEvents::TEvUnsubscribe::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (record.HasPath()) { UnsubscribeBy(ev->Sender, record.GetPath()); @@ -1094,8 +1120,10 @@ private: void Handle(NInternalEvents::TEvNotifyAck::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); auto it = Subscribers.find(ev->Sender); if (it == Subscribers.end()) { @@ -1132,9 +1160,11 @@ private: void Handle(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto it = Subscribers.find(ev->Sender); if (it == Subscribers.end()) { @@ -1237,7 +1267,9 @@ private: } void Handle(TEvStateStorage::TEvUpdateGroupConfig::TPtr ev) { - SBR_LOG_I("Handle " << ev->Get()->ToString()); + YDB_LOG_INFO("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}); Info = ev->Get()->SchemeBoardConfig; Y_ABORT_UNLESS(!ev->Get()->GroupConfig); @@ -1247,8 +1279,10 @@ private: void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { const ui32 nodeId = ev->Get()->NodeId; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": nodeId# " << nodeId); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"nodeId", nodeId}); auto it = Subscribers.lower_bound(TActorId(nodeId, 0, 0, 0)); while (it != Subscribers.end() && it->first.NodeId() == nodeId) { @@ -1267,7 +1301,8 @@ private: } void PassAway() override { - SBR_LOG_T("PassAway"); + YDB_LOG_TRACE("PassAway", + {"selfId", SelfId()}); for (const auto& [_, info] : Populators) { if (const auto& actorId = info.PopulatorActor) { Send(actorId, new TEvStateStorage::TEvReplicaShutdown()); @@ -1287,7 +1322,8 @@ public: } void Bootstrap() { - SBR_LOG_T("Bootstrap"); + YDB_LOG_TRACE("Bootstrap", + {"selfId", SelfId()}); TMonitorableActor::Bootstrap(); auto localNodeId = SelfId().NodeId(); auto whiteboardId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(localNodeId); diff --git a/ydb/core/tx/scheme_board/subscriber.cpp b/ydb/core/tx/scheme_board/subscriber.cpp index facc6f5fc9a..80e428324f4 100644 --- a/ydb/core/tx/scheme_board/subscriber.cpp +++ b/ydb/core/tx/scheme_board/subscriber.cpp @@ -26,19 +26,14 @@ #include <util/string/cast.h> #include <util/generic/xrange.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_SUBSCRIBER + namespace NKikimr { using EDeletionPolicy = ESchemeBoardSubscriberDeletionPolicy; namespace NSchemeBoard { -#define SBS_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) - namespace { template <typename T, typename U> @@ -355,14 +350,18 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { void Handle(NInternalEvents::TEvNotify::TPtr& ev) { auto& record = *ev->Get()->MutableRecord(); - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"path", Path}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); this->Send(ev->Sender, new NInternalEvents::TEvNotifyAck(record.GetVersion())); if (!IsValidNotification(Path, record)) { - SBS_LOG_E("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Suspicious", + {"path", Path}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } @@ -370,18 +369,20 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); CurrentSyncRequest = ev->Cookie; this->Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Cookie != CurrentSyncRequest) { return; @@ -412,6 +413,7 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); if (Replica.NodeId() != this->SelfId().NodeId()) { this->Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); } @@ -435,8 +437,12 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "replica"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TReplicaSubscriber"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", this->SelfId()}, + {"path", Path}); } explicit TReplicaSubscriber( @@ -453,6 +459,7 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); TMonitorableActor<TDerived>::Bootstrap(); this->Send(Replica, new NInternalEvents::TEvSubscribe(Path, DomainOwnerId), @@ -461,6 +468,9 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateWork"}); + switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvNotify, Handle); hFunc(NInternalEvents::TEvSyncVersionRequest, Handle); @@ -499,8 +509,8 @@ public: template <typename TPath, typename TDerived, typename TReplicaDerived> class TSubscriberProxy: public TMonitorableActor<TDerived> { void Sleep() { - SBS_LOG_T("Sleep" - << ": delay# " << Delay); + YDB_LOG_TRACE("Sleep", + {"delay", Delay}); ReplicaSubscriber = TActorId(); this->Send(Parent, new NInternalEvents::TEvNotifyBuilder(Path, true)); @@ -513,11 +523,10 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { return; } if (const auto& record = ev->Get()->GetRecord(); !ClusterStatesMatch(ClusterState, record)) { - SBS_LOG_D("Cluster state mismatch in replica notification" - << ": sender# " << ev->Sender - << ", subscriber cluster state# " << ClusterState - << ", replica cluster state# {" << record.GetClusterState().ShortDebugString() << "}" - ); + YDB_LOG_DEBUG("Cluster state mismatch in replica notification", + {"sender", ev->Sender}, + {"clusterState", ClusterState}, + {"state", record.GetClusterState().ShortDebugString()}); this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); return Sleep(); } @@ -532,18 +541,20 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { } void HandleSleep(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { - SBS_LOG_T("HandleSleep " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_TRACE("HandleSleep", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); this->Send(Parent, new NInternalEvents::TEvSyncVersionResponse(0), 0, ev->Cookie); } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_T("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", current sync request# " << CurrentSyncRequest); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"request", CurrentSyncRequest}); if (ev->Sender != ReplicaSubscriber || ev->Cookie != CurrentSyncRequest) { return; @@ -588,6 +599,7 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); if (ReplicaSubscriber) { this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); } @@ -608,8 +620,12 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "proxy"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TSubscriberProxy"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", this->SelfId()}, + {"path", Path}); } explicit TSubscriberProxy( @@ -633,6 +649,7 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); TMonitorableActor<TDerived>::Bootstrap(); ReplicaSubscriber = this->RegisterWithSameMailbox(new TReplicaDerived(this->SelfId(), Replica, Path, DomainOwnerId)); @@ -640,6 +657,8 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvNotify, Handle); hFunc(NInternalEvents::TEvSyncVersionRequest, Handle); @@ -653,6 +672,8 @@ public: } STFUNC(StateSleep) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateSleep"}); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvSyncVersionRequest, HandleSleep); @@ -772,8 +793,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { if (const auto* groupIdx = ProxyToGroupMap.FindPtr(proxy)) { responsesByGroup[*groupIdx]++; } else { - SBS_LOG_N("Previously received response sender is currently unknown" - << ": sender# " << proxy); + YDB_LOG_NOTICE("Previously received response sender is currently unknown", + {"sender", proxy}); } } for (size_t groupIdx : xrange(ProxyGroups.size())) { @@ -789,8 +810,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { } bool MaybeRunVersionSync() { - SBS_LOG_T("MaybeRunVersionSync" - << ": delayed sync request# " << DelayedSyncRequest); + YDB_LOG_TRACE("MaybeRunVersionSync", + {"request", DelayedSyncRequest}); if (!DelayedSyncRequest) { return false; } @@ -810,18 +831,21 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvNotify::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!IsValidNotification(Path, ev->Get()->GetRecord())) { - SBS_LOG_E("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Suspicious", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } if (!ProxyToGroupMap.contains(ev->Sender)) { - SBS_LOG_E("Unknown " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Unknown", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } @@ -844,27 +868,29 @@ class TSubscriber: public TMonitorableActor<TDerived> { TString reason; if (!State) { - SBS_LOG_N("Set up state" - << ": owner# " << Owner - << ", state# " << newestState); + YDB_LOG_NOTICE("Set up state", + {"owner", Owner}, + {"state", newestState}); State = newestState; } else if (State->LessThan(newestState, reason)) { - SBS_LOG_N("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); + YDB_LOG_NOTICE("New", + {"reason", reason}, + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); State = newestState; } else if (!State->Deleted && newestState.Deleted && newestState.Strong) { - SBS_LOG_N("Path was deleted" - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); + YDB_LOG_NOTICE("Path was deleted new", + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); State = newestState; } else { - SBS_LOG_I("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", other state# " << newestState); + YDB_LOG_INFO("Other", + {"reason", reason}, + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); return; } @@ -872,37 +898,39 @@ class TSubscriber: public TMonitorableActor<TDerived> { if (!record.GetIsDeletion()) { NKikimrScheme::TEvDescribeSchemeResult proto = selectedNotify->GetDescribeSchemeResult(); - SBS_LOG_T("Send notification to the owner" - << ": owner# " << Owner - << ", proto# " << proto.ShortDebugString()); + YDB_LOG_TRACE("Send notification to the owner", + {"owner", Owner}, + {"proto", proto.ShortDebugString()}); this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyUpdate>(record, std::move(proto))); } else { - SBS_LOG_T("Send deletion notification to the owner" - << ": owner# " << Owner); + YDB_LOG_TRACE("Send deletion notification to the owner", + {"owner", Owner}); this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyDelete>(record, record.GetStrong())); } } void Handle(NInternalEvents::TEvSyncRequest::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Sender != Owner) { - SBS_LOG_W("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", owner# " << Owner); + YDB_LOG_WARN("Suspicious", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"owner", Owner}); return; } EnqueueSyncRequest(ev); if (PendingSync) { - SBS_LOG_T("Pending sync"); + YDB_LOG_TRACE("Pending sync"); return; } if (!IsMajorityReached()) { - SBS_LOG_T("Haven't reached the majority"); + YDB_LOG_TRACE("Haven't reached the majority"); return; } @@ -914,42 +942,42 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Cookie != CurrentSyncRequest) { - SBS_LOG_D("Sync cookie mismatch" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", current cookie# " << CurrentSyncRequest); + YDB_LOG_DEBUG("Sync cookie mismatch", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"currentCookie", CurrentSyncRequest}); return; } auto it = PendingSync.find(ev->Sender); if (it == PendingSync.end()) { - SBS_LOG_D("Unexpected sync response" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Unexpected sync response", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); return; } if (!ProxyToGroupMap.contains(ev->Sender)) { - SBS_LOG_E("Sync sender is unknown" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_ERROR("Sync sender is unknown", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); return; } const auto& record = ev->Get()->Record; const bool clusterStatesMatch = ClusterStatesMatch(ClusterState, record); if (!clusterStatesMatch) { - SBS_LOG_I("Cluster State mismatch in sync version response" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", subscriber cluster state# " << ClusterState - << ", replica cluster state# {" << record.GetClusterState().ShortDebugString() << "}" - ); + YDB_LOG_INFO("Cluster State mismatch in sync version response", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"clusterState", ClusterState}, + {"state", record.GetClusterState().ShortDebugString()}); } PendingSync.erase(it); @@ -962,8 +990,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { for (const auto& [proxy, partial] : ReceivedSync) { const auto* groupIdx = ProxyToGroupMap.FindPtr(proxy); if (!groupIdx) { - SBS_LOG_N("Previously received sync sender is currently unknown" - << ": sender# " << proxy); + YDB_LOG_NOTICE("Previously received sync sender is currently unknown", + {"sender", proxy}); continue; } if (!partial) { @@ -977,13 +1005,13 @@ class TSubscriber: public TMonitorableActor<TDerived> { const ui32 size = ProxyGroups[groupIdx].Proxies.size(); const ui32 half = size / 2; if (!IsSyncFinished(successesByGroup[groupIdx], failuresByGroup[groupIdx], size, half)) { - SBS_LOG_D("Sync is in progress" - << ": cookie# " << ev->Cookie - << ", ring group# " << groupIdx - << ", size# " << size - << ", half# " << half - << ", successes# " << successesByGroup[groupIdx] - << ", failures# " << failuresByGroup[groupIdx]); + YDB_LOG_DEBUG("Sync is in progress", + {"cookie", ev->Cookie}, + {"group", groupIdx}, + {"size", size}, + {"half", half}, + {"successes", successesByGroup[groupIdx]}, + {"failures", failuresByGroup[groupIdx]}); return; } syncIsComplete &= successesByGroup[groupIdx] > half; @@ -996,14 +1024,16 @@ class TSubscriber: public TMonitorableActor<TDerived> { << ", failures# " << failuresByGroup[groupIdx] << ", partial# " << !syncIsComplete; if (syncIsComplete) { - SBS_LOG_D(finalMessage); + YDB_LOG_DEBUG("Dump logPrefix, SelfId, path, finalMessage", + {"finalMessage", finalMessage}); } else { - SBS_LOG_N(finalMessage); + YDB_LOG_NOTICE("Dump logPrefix, SelfId, path, finalMessage", + {"finalMessage", finalMessage}); } } if (!syncIsComplete) { - SBS_LOG_W("Sync is incomplete in one of the ring groups" - << ": cookie# " << ev->Cookie); + YDB_LOG_WARN("Sync is incomplete in one of the ring groups", + {"cookie", ev->Cookie}); } this->Send(Owner, new NInternalEvents::TEvSyncResponse(Path, !syncIsComplete), 0, ev->Cookie); @@ -1020,13 +1050,14 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); const auto& replicaGroups = ev->Get()->ReplicaGroups; if (replicaGroups.empty()) { Y_ABORT_UNLESS(ProxyGroups.empty()); - SBS_LOG_E("Subscribe on unconfigured SchemeBoard"); + YDB_LOG_ERROR("Subscribe on unconfigured SchemeBoard"); this->Become(&TDerived::StateCalm); return; } @@ -1038,7 +1069,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { } if (CurrentSyncRequest) { - SBS_LOG_I("Delay current sync request: " << CurrentSyncRequest); + YDB_LOG_INFO("Delay current sync", + {"request", CurrentSyncRequest}); DelayedSyncRequest = Max(DelayedSyncRequest, CurrentSyncRequest); CurrentSyncRequest = 0; } @@ -1130,7 +1162,7 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void HandleUndelivered() { - SBS_LOG_E("Subscribe on unavailable SchemeBoard"); + YDB_LOG_ERROR("Subscribe on unavailable SchemeBoard"); this->Become(&TDerived::StateCalm); } @@ -1158,8 +1190,13 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "main"sv; + ::NActors::NStructuredLog::TStructuredMessage GetLogContext() { + using namespace NActors::NStructuredLog; + return YDB_LOG_CREATE_MESSAGE( + {"selfId", this->SelfId()}, + {"actorName", "TSubscriber"}, + {"path", Path} + );; } explicit TSubscriber( @@ -1175,6 +1212,10 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "bootstrap"}, + ); TMonitorableActor<TDerived>::Bootstrap(); const TActorId proxy = MakeStateStorageProxyID(); @@ -1183,6 +1224,10 @@ public: } STATEFN(StateResolve) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateResolve"}, + ); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvSyncRequest, EnqueueSyncRequest); // from owner (cache) @@ -1196,6 +1241,10 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateWork"}, + ); switch (ev->GetTypeRewrite()) { hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); @@ -1210,6 +1259,10 @@ public: } STATEFN(StateCalm) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateCalm"}, + ); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); diff --git a/ydb/core/tx/schemeshard/index/index_build_info.h b/ydb/core/tx/schemeshard/index/index_build_info.h index 6eb8ca8a545..8a93f637ec4 100644 --- a/ydb/core/tx/schemeshard/index/index_build_info.h +++ b/ydb/core/tx/schemeshard/index/index_build_info.h @@ -896,7 +896,7 @@ public: return State == EState::Cancelled || State == EState::Rejected; } - bool IsFinished() const { + virtual bool IsFinished() const { return IsDone() || IsCancelled(); } @@ -1024,6 +1024,10 @@ struct TSetColumnConstraintOperationInfo: public TIndexBuildInfo { return OperationState == EOperationState::Done; } + bool IsFinished() const override { + return IsDone(); + } + bool IsCloseToCompletion() const { return OperationState == EOperationState::Done || OperationState == EOperationState::Unlocking diff --git a/ydb/core/tx/schemeshard/olap/columns/update.cpp b/ydb/core/tx/schemeshard/olap/columns/update.cpp index 71bbcbca79e..3c82e787f7a 100644 --- a/ydb/core/tx/schemeshard/olap/columns/update.cpp +++ b/ydb/core/tx/schemeshard/olap/columns/update.cpp @@ -375,6 +375,9 @@ bool TOlapColumnBase::IsAllowedDictionaryType(ui32 typeId) { case NYql::NProto::Date: case NYql::NProto::Datetime: case NYql::NProto::Timestamp: + case NYql::NProto::Interval: + case NYql::NProto::Uuid: + case NYql::NProto::DyNumber: return true; default: return false; diff --git a/ydb/core/tx/schemeshard/olap/operations/read_only_copy_table.cpp b/ydb/core/tx/schemeshard/olap/operations/read_only_copy_table.cpp index 17ed9a1c6d4..acca37298dc 100644 --- a/ydb/core/tx/schemeshard/olap/operations/read_only_copy_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/read_only_copy_table.cpp @@ -577,6 +577,9 @@ public: // create new path and inherit properties from src dstPath.MaterializeLeaf(srcPath.Base()->Owner, allocatedPathId, /*allowInactivePath*/ true); result->SetPathId(dstPath.Base()->PathId.LocalPathId); + + context.SS->TabletCounters->Simple()[COUNTER_COLUMN_TABLE_COUNT].Add(1); + dstPath.Base()->CreateTxId = OperationId.GetTxId(); dstPath.Base()->LastTxId = OperationId.GetTxId(); dstPath.Base()->PathState = TPathElement::EPathState::EPathStateCreate; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp index feb502e970a..0e0f50ba5f4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp @@ -192,6 +192,11 @@ struct TBackup { } static bool NeedToBill(const TPathId& pathId, TOperationContext& context) { + if (context.SS->ColumnTables.contains(pathId)) { + auto table = context.SS->ColumnTables.at(pathId); + return table->BackupSettings.GetNeedToBill(); + } + Y_ABORT_UNLESS(context.SS->Tables.contains(pathId)); auto table = context.SS->Tables.at(pathId); return table->BackupSettings.GetNeedToBill(); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 8fff773c8be..240c0750a78 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -8010,7 +8010,7 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, // Control op completed; finalize the tracked record. Execute(CreateTxFullBackupProgress(ui64(txId)), ctx); } - if (TxIdToSetColumnConstraintOperations.contains(txId)) { + if (TxIdToSetColumnConstraintOperations.contains(txId) || TxIdToDependentSetColumnConstraint.contains(txId)) { Execute(CreateTxReplyCompletedSetColumnConstraint(txId), ctx); executed = true; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 1fa19209ba2..90ae47b49ad 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1993,6 +1993,10 @@ public: THashMap<TString, std::shared_ptr<TSetColumnConstraintOperationInfo>> SetColumnConstraintOperationsByUid; TSet<std::pair<TInstant, TIndexBuildId>> SetColumnConstraintOperationsByTime; THashMap<TTxId, TIndexBuildId> TxIdToSetColumnConstraintOperations; + // txIds of concurrent operations (e.g. a backup CopyTable) that a + // SetColumnConstraint operation is currently waiting on before it can retry + // one of its own internal sub-transactions. + THashMap<TTxId, THashSet<TIndexBuildId>> TxIdToDependentSetColumnConstraint; void PersistCreateSetColumnConstraint(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); void PersistSetColumnConstraintState(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint.cpp index 18bb3286c51..868c0abf97c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint.cpp @@ -80,7 +80,12 @@ void FillSetColumnConstraint( // Map internal state to proto state proto.SetProgress(CalcProgress(operationInfo)); - if (operationInfo.IsCancelled) { + if (operationInfo.ValidationFailed) { + // Validation found NULL values (or hit an internal error) in the column(s) + // being altered. This is distinct from an explicit user cancellation. + proto.SetState(ProtoState::STATE_REJECTED); + } else if (operationInfo.IsCancelled) { + // Operation was cancelled by an explicit user request. proto.SetState(ProtoState::STATE_CANCELLED); } else { switch (operationInfo.OperationState) { @@ -96,9 +101,6 @@ void FillSetColumnConstraint( proto.SetState(ProtoState::STATE_APPLYING); break; case EState::Done: - // There is no way when IsValidationFailed equals `true`, - // because IsValidationFailed is true exactly when IsCancelled is true. - // Therefore, there cannot be STATE_CANCELLED. proto.SetState(ProtoState::STATE_DONE); break; case EState::Invalid: diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__cancel.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__cancel.cpp index c852cd1142f..e34b0cfb42d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__cancel.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__cancel.cpp @@ -66,11 +66,29 @@ public: operationInfo.MarkAsCancelled(TString("Cancelled by user request")); Self->PersistSetColumnConstraintCancellation(db, operationInfo); - // Note: The operation will continue through its natural stages: + // Note: cancellation is de facto acted upon only while the operation is in the + // Validating stage: // - TTxReplyValidateRowCondition ignores incoming validation messages due to IsCancelled flag // - AlterMainTableUnlockNullWritesPropose will NOT set NOT NULL constraint due to IsCancelled check // - The operation will complete naturally in Done state without setting the constraint - Progress(BuildId); + // + // Trying to react to cancellation on earlier or later stages introduces non-trivial + // logic that can easily lead to unwanted bugs (e.g. a stale reply for an already + // abandoned stage racing with a message for the newly entered one). We sidestep that + // entirely here: + // - Later stages (Finishing, Unlocking) are already forbidden for cancellation by the + // IsCloseToCompletion() check above. + // - Earlier stages (Locking, LockingNullWrites) simply ignore the fact of cancellation + // until the Validating stage is reached; the IsCancelled flag is already persisted, + // so once Validating starts (or is already running), the operation will pick it up + // and unwind normally. + // + // This is not a practical concern because Validating is by far the longest-running + // stage of this metadata operation, and it is also the very reason a user would want + // to cancel a SET NOT NULL metadata operation in the first place. + if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Validating) { + Progress(BuildId); + } return Reply(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__get.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__get.cpp index a83143cfb5b..dd7335b9d66 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__get.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__get.cpp @@ -56,14 +56,14 @@ public: FillSetColumnConstraint(*proto, operationInfo, Self); // Add issue for cancelled operations - if (operationInfo.IsCancelled && !operationInfo.CancellationReason.empty()) { + if (operationInfo.IsCancelled) { auto* issue = respRecord.AddIssues(); issue->set_message(operationInfo.CancellationReason); issue->set_issue_code(0); issue->set_severity(NYql::TSeverityIds::S_ERROR); } - if (operationInfo.ValidationFailed && operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Done) { + if (operationInfo.ValidationFailed) { TPath tablePath = TPath::Init(operationInfo.TablePathId, Self); auto* issue = respRecord.AddIssues(); issue->set_message(TStringBuilder() diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp index 288caedd7c8..b9810e66a98 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp @@ -205,45 +205,109 @@ public: << ", status# " << NKikimrScheme::EStatus_Name(record.GetStatus())); NIceDb::TNiceDb db(txc.DB); - - auto replyOnCreation = [&] { + bool shouldForget = false; + auto replyOnCreation = [&]() -> void { auto statusCode = TranslateStatusCode(record.GetStatus()); if (statusCode != Ydb::StatusIds::SUCCESS) { - // TODO(flown4qqqq): persist issue - // TODO(flown4qqqq): forget operation on error - // TODO(flown4qqqq): EraseBuildInfo(operationInfo); + shouldForget = true; } ReplyOnCreation(operationInfo, statusCode); - return true; }; - if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Locking) { - Y_ENSURE(txId == operationInfo.LockTxId); - operationInfo.LockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + // Most modifications of the main table are forbidden while SetColumnConstraint + // holds its lock, however copying the table (e.g. for a backup) is allowed to + // proceed concurrently (see TCopyTable::Propose: CheckLocks is skipped for + // IsBackup copies). While such a copy is in flight the table has + // PathStateCopying, and our own internal sub-transactions (which are gated by + // NotUnderOperation()) may transiently fail with StatusMultipleModifications. + // Just like build_index does, retry the very same proposal once the copy is + // done instead of failing the whole operation. + // Waiting for the conflicting operation's completion notification, not for + // an immediate self-retry - see the comment on shouldRetry() below for why. + bool waitingForDependency = false; - if (!replyOnCreation()) { + auto shouldRetry = [&]() { + if (record.GetStatus() != NKikimrScheme::StatusMultipleModifications) { return false; } - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites) { - Y_ENSURE(txId == operationInfo.LockNullWritesTxId); - operationInfo.LockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Finishing) { - Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); - operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Unlocking) { - Y_ENSURE(txId == operationInfo.UnlockTxId); - operationInfo.UnlockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); - } else { - Y_UNREACHABLE(); + + auto it = Self->PathsById.find(operationInfo.TablePathId); + if (it == Self->PathsById.end() || it->second->PathState != NKikimrSchemeOp::EPathStateCopying) { + return false; + } + + auto copyTxId = it->second->LastTxId; + LOG_I("TTxReplyModify : Waiting for txId " << copyTxId << " to retry SetColumnConstraint id# " << BuildId); + operationInfo.DependencyTxIds.insert(copyTxId); + Self->TxIdToDependentSetColumnConstraint[copyTxId].insert(BuildId); + Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(copyTxId))); + return true; + }; + + switch (operationInfo.OperationState) { + case TSetColumnConstraintOperationInfo::EOperationState::Locking: { + Y_ENSURE(txId == operationInfo.LockTxId); + operationInfo.LockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + replyOnCreation(); + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites: { + Y_ENSURE(txId == operationInfo.LockNullWritesTxId); + if (shouldRetry()) { + operationInfo.LockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.LockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Finishing: { + Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); + if (shouldRetry()) { + operationInfo.UnlockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Unlocking: { + Y_ENSURE(txId == operationInfo.UnlockTxId); + if (shouldRetry()) { + operationInfo.UnlockTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Invalid: + case TSetColumnConstraintOperationInfo::EOperationState::Validating: + case TSetColumnConstraintOperationInfo::EOperationState::Done: { + Y_UNREACHABLE(); + } + } + + if (!waitingForDependency) { + Progress(BuildId); + } + + if (shouldForget) { + Self->ForgetSetColumnConstraint(db, operationInfo); } - Progress(BuildId); return true; } @@ -271,6 +335,23 @@ public: const auto txId = CompletedTxId; auto* operationIdPtr = Self->TxIdToSetColumnConstraintOperations.FindPtr(txId); + + if (Self->TxIdToDependentSetColumnConstraint.contains(txId)) { + THashSet<TIndexBuildId> deps = std::move(Self->TxIdToDependentSetColumnConstraint.at(txId)); + Self->TxIdToDependentSetColumnConstraint.erase(txId); + for (auto& dependentBuildId : deps) { + LOG_I("TTxReplyCompleted txId: " << txId << " : trying to resume dependent SetColumnConstraint " << dependentBuildId); + if (auto* dependentOperationInfoPtr = Self->SetColumnConstraintOperations.FindPtr(dependentBuildId)) { + auto& dependentOperationInfo = **dependentOperationInfoPtr; + dependentOperationInfo.DependencyTxIds.erase(txId); + Progress(dependentBuildId); + } + } + if (!operationIdPtr) { + return true; + } + } + if (!operationIdPtr) { LOG_I("TTxReplyCompleted: operation not found, txId# " << txId); return true; @@ -357,6 +438,11 @@ public: return true; } + if (operationInfo.ValidationFailed) { + LOG_I("TTxReplyValidateRowCondition: operation is rejected, ignoring message, id# " << BuildId); + return true; + } + TShardIdx shardIdx; bool found = false; for (const auto& [idx, shardStatus] : operationInfo.ValidationShards) { @@ -384,7 +470,6 @@ public: } auto oldValidationFailedValue = operationInfo.ValidationFailed; - TString cancellationReason; auto& shardStatus = operationInfo.ValidationShards.at(shardIdx); shardStatus.ValidateStatus = record.GetStatus(); @@ -404,7 +489,6 @@ public: if (!record.GetIsValid()) { LOG_N("TTxReplyValidateRowCondition: validation failed on shard# " << shardIdx); operationInfo.ValidationFailed = true; - cancellationReason = "Validation failed: found NULL values in column(s) being set to NOT NULL"; } operationInfo.InProgressValidationShards.erase(shardIdx); @@ -417,7 +501,6 @@ public: << ", status# " << record.GetStatus()); operationInfo.ValidationFailed = true; - cancellationReason = "Validation failed: internal error"; operationInfo.InProgressValidationShards.erase(shardIdx); operationInfo.DoneValidationShards.insert(shardIdx); @@ -434,9 +517,6 @@ public: if (oldValidationFailedValue != operationInfo.ValidationFailed) { Self->PersistSetColumnConstraintValidationFailedValue(db, operationInfo); - - operationInfo.MarkAsCancelled(std::move(cancellationReason)); - Self->PersistSetColumnConstraintCancellation(db, operationInfo); } Self->PersistSetColumnConstraintShardDone(db, BuildId, shardIdx, operationInfo); @@ -638,9 +718,15 @@ public: Y_ENSURE(operationInfoPtr); auto& operationInfo = *operationInfoPtr->get(); + if (!operationInfo.DependencyTxIds.empty()) { + LOG_N("TTxProgressSetColumnConstraint: " << BuildId << ": waiting for dependencies"); + return true; + } + LOG_D("TTxProgressSetColumnConstraint::DoExecute, id# " << BuildId << "; OperationState = " << ToString(operationInfo.OperationState) - << "; IsCancelled = " << operationInfo.IsCancelled); + << "; IsCancelled = " << operationInfo.IsCancelled + << "; ValidationFailed = " << operationInfo.ValidationFailed); switch (operationInfo.OperationState) { case TSetColumnConstraintOperationInfo::EOperationState::Invalid: { @@ -662,15 +748,6 @@ public: break; } case TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites: { - // If cancelled, skip to Finishing to release locks without setting constraint - if (operationInfo.IsCancelled) { - LOG_I("TTxProgressSetColumnConstraint: operation cancelled in LockingNullWrites, jumping to Finishing, id# " << BuildId); - NIceDb::TNiceDb db(txc.DB); - ChangeState(BuildId, TSetColumnConstraintOperationInfo::EOperationState::Finishing); - Progress(BuildId); - break; - } - if (operationInfo.LockNullWritesTxId == InvalidTxId) { AllocateTxId(BuildId); } else if (operationInfo.LockNullWritesTxStatus == NKikimrScheme::StatusSuccess) { @@ -686,8 +763,13 @@ public: } case TSetColumnConstraintOperationInfo::EOperationState::Validating: { // If cancelled, skip to Finishing to release locks without setting constraint - if (operationInfo.IsCancelled) { - LOG_I("TTxProgressSetColumnConstraint: operation cancelled in Validating, jumping to Finishing, id# " << BuildId); + if (operationInfo.IsCancelled || operationInfo.ValidationFailed) { + if (operationInfo.IsCancelled) { + LOG_I("TTxProgressSetColumnConstraint: operation cancelled in Validating, jumping to Finishing, id# " << BuildId); + } else { + LOG_I("TTxProgressSetColumnConstraint: validation failed in Validating, jumping to Finishing, id# " << BuildId); + } + NIceDb::TNiceDb db(txc.DB); ChangeState(BuildId, TSetColumnConstraintOperationInfo::EOperationState::Finishing); Progress(BuildId); diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp index ab4ab2f3995..347ebfaaf76 100644 --- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp @@ -54,7 +54,8 @@ namespace { void Run(TTestBasicRuntime& runtime, TTestEnv& env, const std::variant<TVector<TString>, TTablesWithAttrs>& tablesVar, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "", - const TVector<TString>& cdcStreams = {}, bool checkAutoDropping = false) { + const TVector<TString>& cdcStreams = {}, bool checkAutoDropping = false, + const TVector<TString>& columnTables = {}) { TTablesWithAttrs tables; @@ -149,6 +150,11 @@ namespace { env.TestWaitNotification(runtime, txId, schemeshardId); } + for (const auto& table : columnTables) { + TestCreateColumnTable(runtime, schemeshardId, ++txId, dbName, table); + env.TestWaitNotification(runtime, txId, schemeshardId); + } + for (const auto& cdcStream : cdcStreams) { TestCreateCdcStream(runtime, schemeshardId, ++txId, dbName, cdcStream); env.TestWaitNotification(runtime, txId, schemeshardId); @@ -1250,6 +1256,24 @@ namespace { return info; } + void VerifyColumnTableS3ExportHasData(const TString& destinationPrefix) { + const TString prefix = destinationPrefix.empty() + ? TString() + : (destinationPrefix.StartsWith('/') ? destinationPrefix : "/" + destinationPrefix); + + UNIT_ASSERT(HasS3File(prefix + "/scheme.pb")); + + bool hasDataFile = false; + for (const auto& [path, content] : S3Mock().GetData()) { + // "metadata.json" contains "data" as a substring, so match a data-path component instead. + if (path.StartsWith(prefix + "/") && path.Contains("/data") && !content.empty()) { + hasDataFile = true; + break; + } + } + UNIT_ASSERT_C(hasDataFile, "Expected at least one non-empty data file under '" << (prefix.empty() ? "/" : prefix) << "'"); + } + ui64 StartColumnTableS3Export(ui64& txId, const TString& tableName, const TString& destinationPrefix) { const ui64 exportTxId = ++txId; TestExport(Runtime(), exportTxId, "/MyRoot", Sprintf(R"( @@ -5296,4 +5320,455 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `ExternalTable` ( NLs::CheckColumnTableMultiColumnStatistics("s1", {"data"}, {NKikimrSchemeOp::EMultiColumnStatisticsType::COUNT_MIN_SKETCH}), }); } + + Y_UNIT_TEST(ExportImportRowAndColumnTablesTogether) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + ui64 txId = 100; + + TestCreateTable(Runtime(), ++txId, "/MyRoot", R"( + Name: "RowTable" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + Env().TestWaitNotification(Runtime(), txId); + UpdateRow(Runtime(), "RowTable", 1, "rowValue"); + + const auto columnInfo = CreateColumnTableWithData(txId, "ColumnTable"); + Y_UNUSED(columnInfo); + + const ui64 exportTxId = ++txId; + TestExport(Runtime(), exportTxId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/RowTable" + destination_prefix: "RowExport" + } + items { + source_path: "/MyRoot/ColumnTable" + destination_prefix: "ColumnExport" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), exportTxId); + TestGetExport(Runtime(), exportTxId, "/MyRoot", Ydb::StatusIds::SUCCESS); + VerifyColumnTableS3ExportHasData("ColumnExport"); + + const ui64 importId = ++txId; + TestImport(Runtime(), importId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "RowExport" + destination_path: "/MyRoot/RowImported" + } + items { + source_prefix: "ColumnExport" + destination_path: "/MyRoot/ColumnImported" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), importId); + TestGetImport(Runtime(), importId, "/MyRoot", Ydb::StatusIds::SUCCESS); + + TestDescribeResult(DescribePath(Runtime(), "/MyRoot/RowImported"), {NLs::PathExist, NLs::IsTable}); + TestDescribeResult(DescribePath(Runtime(), "/MyRoot/ColumnImported"), { + NLs::PathExist, + NLs::IsColumnTable, + }); + + { + auto tableDesc = DescribePath(Runtime(), "/MyRoot/RowImported", true, false, true); + const auto& partitions = tableDesc.GetPathDescription().GetTablePartitions(); + UNIT_ASSERT(!partitions.empty()); + const auto rowContent = ReadTable( + Runtime(), partitions[0].GetDatashardId(), "RowImported", {"key"}, {"key", "value"}); + const auto rowList = NClient::TValue::Create(rowContent)["Result"]["List"]; + UNIT_ASSERT_VALUES_EQUAL(rowList.Size(), 1u); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(rowList[0]["key"]), 1u); + UNIT_ASSERT_VALUES_EQUAL(TString(rowList[0]["value"]), "rowValue"); + } + } + + Y_UNIT_TEST(ExportImportColumnTableAfterAddAndDropColumns) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + Runtime().SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); + ui64 txId = 100; + + TestCreateColumnTable(Runtime(), ++txId, "/MyRoot", R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + Columns { Name: "to_drop" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )"); + Env().TestWaitNotification(Runtime(), txId); + + ui64 pathId = 0; + ui64 shardId = 0; + { + auto describe = DescribePath(Runtime(), "/MyRoot/ColumnTable"); + TestDescribeResult(describe, {NLs::PathExist, NLs::IsColumnTable}); + pathId = describe.GetPathId(); + const auto& sharding = describe.GetPathDescription().GetColumnTableDescription().GetSharding(); + UNIT_ASSERT_VALUES_EQUAL(sharding.ColumnShardsSize(), 1); + shardId = sharding.GetColumnShards()[0]; + } + UNIT_ASSERT(pathId); + UNIT_ASSERT(shardId); + + auto writeRows = [&](const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, const std::pair<ui64, ui64>& rows) { + TActorId sender = Runtime().AllocateEdgeActor(); + const auto data = NTxUT::MakeTestBlob(rows, ydbSchema, {}, {"timestamp"}); + ui64 writeId = 0; + std::vector<ui64> writeIds; + ++txId; + NTxUT::WriteData(Runtime(), sender, shardId, ++writeId, pathId, data, ydbSchema, &writeIds, NEvWrite::EModificationType::Upsert, txId); + auto planStep = NTxUT::ProposeCommit(Runtime(), sender, shardId, txId, writeIds, txId); + NTxUT::PlanCommit(Runtime(), sender, shardId, planStep, {txId}); + }; + + const std::vector<NArrow::NTest::TTestColumn> schemaBeforeAlter = { + NArrow::NTest::TTestColumn("timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp)).SetNullable(false), + NArrow::NTest::TTestColumn("value", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("to_drop", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8)), + }; + writeRows(schemaBeforeAlter, {0, 50}); + + TestAlterColumnTable(Runtime(), ++txId, "/MyRoot", R"( + Name: "ColumnTable" + AlterSchema { + DropColumns { Name: "to_drop" } + AddColumns { Name: "added" Type: "Uint64" } + } + )"); + Env().TestWaitNotification(Runtime(), txId); + + const std::vector<NArrow::NTest::TTestColumn> schemaAfterAlter = { + NArrow::NTest::TTestColumn("timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp)).SetNullable(false), + NArrow::NTest::TTestColumn("value", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("added", NScheme::TTypeInfo(NScheme::NTypeIds::Uint64)), + }; + writeRows(schemaAfterAlter, {50, 100}); + + auto hasColumn = [](const NKikimrScheme::TEvDescribeSchemeResult& descr, const TString& columnName) { + const auto& schema = descr.GetPathDescription().GetColumnTableDescription().GetSchema(); + for (const auto& col : schema.GetColumns()) { + if (col.GetName() == columnName) { + return true; + } + } + return false; + }; + + { + auto descr = DescribePrivatePath(Runtime(), "/MyRoot/ColumnTable"); + UNIT_ASSERT(hasColumn(descr, "added")); + UNIT_ASSERT(!hasColumn(descr, "to_drop")); + } + + const ui64 exportTxId = ++txId; + TestExport(Runtime(), exportTxId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/ColumnTable" + destination_prefix: "AlteredColumnExport" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), exportTxId); + TestGetExport(Runtime(), exportTxId, "/MyRoot", Ydb::StatusIds::SUCCESS); + + const ui64 importId = ++txId; + TestImport(Runtime(), importId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "AlteredColumnExport" + destination_path: "/MyRoot/ColumnImported" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), importId); + TestGetImport(Runtime(), importId, "/MyRoot", Ydb::StatusIds::SUCCESS); + + { + auto descr = DescribePrivatePath(Runtime(), "/MyRoot/ColumnImported"); + UNIT_ASSERT(hasColumn(descr, "timestamp")); + UNIT_ASSERT(hasColumn(descr, "value")); + UNIT_ASSERT(hasColumn(descr, "added")); + UNIT_ASSERT(!hasColumn(descr, "to_drop")); + } + } + + Y_UNIT_TEST(ShouldWriteBillRecordOnColumnTableServerlessDb) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + + TVector<TString> billRecords; + Runtime().SetObserverFunc([&billRecords](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(Runtime(), Env(), TVector<TString>{}, Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/User/ColumnTable" + destination_prefix: "" + } + } + )", S3Port()), Ydb::StatusIds::SUCCESS, "/MyRoot/User", true, "", "", {}, false, { + R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )" + }); + + if (billRecords.empty()) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&billRecords](IEventHandle&) -> bool { + return !billRecords.empty(); + }); + Runtime().DispatchEvents(opts); + } + + UNIT_ASSERT_VALUES_EQUAL(billRecords.size(), 1); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"cloud_id\":\"CLOUD_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"folder_id\":\"FOLDER_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"resource_id\":\"DATABASE_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"schema\":\"ydb.serverless.requests.v1\""); + } + + Y_UNIT_TEST(ShouldNotWriteBillRecordOnColumnTableCommonDb) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + + TVector<TString> billRecords; + Runtime().SetObserverFunc([&billRecords](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(Runtime(), Env(), TVector<TString>{}, Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/User/ColumnTable" + destination_prefix: "" + } + } + )", S3Port()), Ydb::StatusIds::SUCCESS, "/MyRoot/User", false, "", "", {}, false, { + R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )" + }); + + UNIT_ASSERT(billRecords.empty()); + } + + Y_UNIT_TEST(ShouldFailExportColumnTableWhenBackupDisabled) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(false); + ui64 txId = 100; + + TestCreateColumnTable(Runtime(), ++txId, "/MyRoot", R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )"); + Env().TestWaitNotification(Runtime(), txId); + + TestExport(Runtime(), ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/ColumnTable" + destination_prefix: "DisabledExport" + } + } + )", S3Port()), "", "", Ydb::StatusIds::BAD_REQUEST); + } + + Y_UNIT_TEST(ExportImportMultiShardColumnTable) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + Runtime().SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); + ui64 txId = 100; + + TestCreateColumnTable(Runtime(), ++txId, "/MyRoot", R"( + Name: "MultiShardColumnTable" + ColumnShardCount: 2 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )"); + Env().TestWaitNotification(Runtime(), txId); + + ui64 pathId = 0; + TVector<ui64> shardIds; + { + auto describe = DescribePath(Runtime(), "/MyRoot/MultiShardColumnTable"); + TestDescribeResult(describe, {NLs::PathExist, NLs::IsColumnTable}); + pathId = describe.GetPathId(); + const auto& sharding = describe.GetPathDescription().GetColumnTableDescription().GetSharding(); + UNIT_ASSERT_VALUES_EQUAL(sharding.ColumnShardsSize(), 2); + for (ui64 id : sharding.GetColumnShards()) { + shardIds.push_back(id); + } + } + UNIT_ASSERT_VALUES_EQUAL(shardIds.size(), 2); + UNIT_ASSERT(pathId); + + { + TActorId sender = Runtime().AllocateEdgeActor(); + const std::vector<NArrow::NTest::TTestColumn> ydbSchema = { + NArrow::NTest::TTestColumn("timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp)).SetNullable(false), + NArrow::NTest::TTestColumn("value", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8)), + }; + for (size_t i = 0; i < shardIds.size(); ++i) { + const auto data = NTxUT::MakeTestBlob({i * 100, i * 100 + 50}, ydbSchema, {}, {"timestamp"}); + ui64 writeId = 0; + std::vector<ui64> writeIds; + ++txId; + NTxUT::WriteData(Runtime(), sender, shardIds[i], ++writeId, pathId, data, ydbSchema, &writeIds, NEvWrite::EModificationType::Upsert, txId); + auto planStep = NTxUT::ProposeCommit(Runtime(), sender, shardIds[i], txId, writeIds, txId); + NTxUT::PlanCommit(Runtime(), sender, shardIds[i], planStep, {txId}); + } + } + + const ui64 exportTxId = ++txId; + TestExport(Runtime(), exportTxId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/MultiShardColumnTable" + destination_prefix: "MultiShardExport" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), exportTxId); + TestGetExport(Runtime(), exportTxId, "/MyRoot", Ydb::StatusIds::SUCCESS); + VerifyColumnTableS3ExportHasData("MultiShardExport"); + + const ui64 importId = ++txId; + TestImport(Runtime(), importId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "MultiShardExport" + destination_path: "/MyRoot/MultiShardImported" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), importId); + TestGetImport(Runtime(), importId, "/MyRoot", Ydb::StatusIds::SUCCESS); + + { + auto describe = DescribePath(Runtime(), "/MyRoot/MultiShardImported"); + TestDescribeResult(describe, {NLs::PathExist, NLs::IsColumnTable}); + const auto& sharding = describe.GetPathDescription().GetColumnTableDescription().GetSharding(); + UNIT_ASSERT_VALUES_EQUAL(sharding.ColumnShardsSize(), 2); + } + } + + Y_UNIT_TEST(ShouldFailImportColumnTableWithInvalidScheme) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + ui64 txId = 100; + + CreateColumnTableWithData(txId, "ColumnTable"); + + const ui64 exportTxId = ++txId; + TestExport(Runtime(), exportTxId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/ColumnTable" + destination_prefix: "ColumnCorruptScheme" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), exportTxId); + TestGetExport(Runtime(), exportTxId, "/MyRoot", Ydb::StatusIds::SUCCESS); + + auto schemeIt = S3Mock().GetData().find("/ColumnCorruptScheme/scheme.pb"); + UNIT_ASSERT_C(schemeIt != S3Mock().GetData().end(), "scheme.pb must exist after export"); + schemeIt->second = "this is not a valid scheme protobuf"; + + const ui64 importId = ++txId; + TestImport(Runtime(), importId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "ColumnCorruptScheme" + destination_path: "/MyRoot/ColumnImported" + } + } + )", S3Port())); + Env().TestWaitNotification(Runtime(), importId); + TestGetImport(Runtime(), importId, "/MyRoot", Ydb::StatusIds::CANCELLED); + TestDescribeResult(DescribePath(Runtime(), "/MyRoot/ColumnImported"), {NLs::PathNotExist}); + } + + Y_UNIT_TEST(ShouldFailExportNonexistentColumnTable) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + ui64 txId = 100; + + TestExport(Runtime(), ++txId, "/MyRoot", Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/NoSuchColumnTable" + destination_prefix: "Missing" + } + } + )", S3Port()), "", "", Ydb::StatusIds::BAD_REQUEST); + } } diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp index 608e845200f..6e0a4bbfdc8 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp @@ -904,7 +904,17 @@ Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) { TTestData::ColumnTable() }, TTestData::Items(EPathTypeColumnTable)); } - + + Y_UNIT_TEST_WITH_REBOOTS_BUCKETS_TWIN(ShouldSucceedOnTablesAndColumnTables, 2, 1, false, IsFs) { + if (IsFs) { + return; // Column table FS export is not supported yet + } + RunExport<IsFs>(t, { + TTestData::Table(), + TTestData::ColumnTable() + }, {{"/MyRoot/Table", "table"}, {"/MyRoot/ColumnTable", "column_table"}}); + } + Y_UNIT_TEST_WITH_REBOOTS_BUCKETS_TWIN(CancelShouldSucceedOnSingleColumnTable, 2, 1, false, IsFs) { if (IsFs) { return; // It is not supported yet diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 7718617bf7c..154ba30e4e2 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -1,6 +1,7 @@ #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> #include <ydb/core/tx/schemeshard/ut_helpers/local_indexes.h> #include <ydb/core/tx/schemeshard/ut_helpers/olap_helpers.h> +#include <ydb/core/tx/schemeshard/ut_helpers/schemeshard_counters.h> #include <ydb/core/testlib/actors/block_events.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h> @@ -3344,4 +3345,44 @@ Y_UNIT_TEST_SUITE(TOlapNaming) { TestLs(runtime, "/MyRoot/MyDir/ColumnTable", false, NLs::PathNotExist); UNIT_ASSERT_VALUES_EQUAL(GetShardOwnerLocalPathId(runtime, localShardIdx), 0u); } + + Y_UNIT_TEST(DropReadOnlyCopyColumnTableCounter) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + static constexpr auto counterName = "SchemeShard/ColumnTables"; + + TestMkDir(runtime, ++txId, "/MyRoot", "MyDir"); + env.TestWaitNotification(runtime, txId); + + CheckSimpleCounter(runtime, counterName, 0); + + TestCreateColumnTable(runtime, ++txId, "/MyRoot/MyDir", defaultTableSchema); + env.TestWaitNotification(runtime, txId); + CheckSimpleCounter(runtime, counterName, 1); + + TestCopyColumnTable(runtime, ++txId, "/MyRoot/MyDir", "Copy1", "/MyRoot/MyDir/ColumnTable"); + env.TestWaitNotification(runtime, txId); + + TestCopyColumnTable(runtime, ++txId, "/MyRoot/MyDir", "Copy2", "/MyRoot/MyDir/ColumnTable"); + env.TestWaitNotification(runtime, txId); + + CheckSimpleCounter(runtime, counterName, 3); + + TestDropColumnTable(runtime, ++txId, "/MyRoot/MyDir", "Copy1"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/MyDir/Copy1", false, NLs::PathNotExist); + CheckSimpleCounter(runtime, counterName, 2); + + TestDropColumnTable(runtime, ++txId, "/MyRoot/MyDir", "Copy2"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/MyDir/Copy2", false, NLs::PathNotExist); + CheckSimpleCounter(runtime, counterName, 1); + + TestDropColumnTable(runtime, ++txId, "/MyRoot/MyDir", "ColumnTable"); + env.TestWaitNotification(runtime, txId); + TestLs(runtime, "/MyRoot/MyDir/ColumnTable", false, NLs::PathNotExist); + CheckSimpleCounter(runtime, counterName, 0); + } } diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp index a444ad78332..c0215f34965 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp @@ -1087,9 +1087,9 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { Ydb::Table::SetNotNullState::STATE_PREPARING, Ydb::Table::SetNotNullState::STATE_PREPARING, Ydb::Table::SetNotNullState::STATE_VALIDATING, - (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_CANCELLED : Ydb::Table::SetNotNullState::STATE_APPLYING), - (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_CANCELLED : Ydb::Table::SetNotNullState::STATE_APPLYING), - (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_CANCELLED : Ydb::Table::SetNotNullState::STATE_DONE) + (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_REJECTED : Ydb::Table::SetNotNullState::STATE_APPLYING), + (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_REJECTED : Ydb::Table::SetNotNullState::STATE_APPLYING), + (isShouldBeFailed ? Ydb::Table::SetNotNullState::STATE_REJECTED : Ydb::Table::SetNotNullState::STATE_DONE) }; runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) { @@ -1123,7 +1123,7 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); - // STATE_DONE/STATE_CANCELLED + // STATE_DONE/STATE_REJECTED answers.push_back(DoGetRequest(setConstraintTxId, runtime, root).GetState()); UNIT_ASSERT_VALUES_EQUAL_C( @@ -1385,7 +1385,7 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { } if (entry.GetId() == setConstraintTxId2) { foundOp2 = true; - UNIT_ASSERT_VALUES_EQUAL(static_cast<int>(entry.GetState()), static_cast<int>(Ydb::Table::SetNotNullState::STATE_CANCELLED)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<int>(entry.GetState()), static_cast<int>(Ydb::Table::SetNotNullState::STATE_REJECTED)); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 1); UNIT_ASSERT_VALUES_EQUAL(columns.Get(0), "value2"); } @@ -2383,5 +2383,459 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { Cerr << "=== Successfully cancelled at stage: " << stage.StageName << " ===" << Endl; } } + + Y_UNIT_TEST(MoveTableFailsWhileValidating) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Block validation requests to datashards so that the operation is stuck + // in the Validating stage (the table lock is already held at this point). + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + // While SetNotNull is stuck validating shards (table is locked), attempt to move + // the table. MoveTable must be rejected immediately at Propose (no TxState is even + // created for it), because CheckLocks() sees the path is locked by the SetNotNull + // operation and MoveTable's transaction carries no matching LockGuard. + ui64 moveTxId = ++txId; + TestMoveTable(runtime, moveTxId, tablePath, movedTablePath, + {NKikimrScheme::StatusMultipleModifications}); + + // The table must still be exactly where it was - move never actually started. + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathNotExist}); + + // Let SetNotNull finish normally. + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + // Now that the lock is released, the very same move must succeed. + TestMoveTable(runtime, ++txId, tablePath, movedTablePath); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + } + + Y_UNIT_TEST(SetNotNullFailsWhileMoveTableInProgress) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep MoveTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, MoveTable's local Propose has already committed - the source + // path is already marked EPathStateMoving. + TBlockEvents<TEvDataShard::TEvProposeTransaction> moveProposeBlocker(runtime); + + ui64 moveTxId = ++txId; + AsyncMoveTable(runtime, moveTxId, tablePath, movedTablePath); + + runtime.WaitFor("block move propose to datashards", [&]{ return moveProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (moving). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while moving): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_UNEQUAL_C( + response.GetStatus(), + Ydb::StatusIds::SUCCESS, + response.ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let MoveTable finish normally. + moveProposeBlocker.Stop(); + moveProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, moveTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + + // Now that the move is done, SetNotNull on the (now moved) table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + movedTablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, movedTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableNoBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the first scenario. + + For a standalone CopyTable, it is fine to fail the request and tell the user that the table is currently locked + by another long-running operation. In that scenario, CopyTable is a single explicit operation that the user can simply retry. + This is also the case where CopyTable is expected to check whether the table is locked by other schema operations. + */ + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx1 = ++txId; + TestConsistentCopyTables(runtime, copyTx1, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )", {NKikimrScheme::StatusMultipleModifications}); + env.TestWaitNotification(runtime, copyTx1); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathNotExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + ui64 copyTx2 = ++txId; + TestConsistentCopyTables(runtime, copyTx2, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )"); + env.TestWaitNotification(runtime, copyTx2); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableWithBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the second scenario. + + When CopyTable is invoked as part of Backup, it should not fail just because the table is locked by another + long-running operation. If ConsistentCopyTable gets blocked this way during a backup, it may lead to a large + number of retries. Each retry effectively means trying to copy all tables in the database again, which is a very + expensive and long-running operation. + + Therefore, the practical way to avoid such retries is to make the new long-running operation pause and wait + until the backup's CopyTable finishes. + */ + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx = ++txId; + TestConsistentCopyTables(runtime, copyTx, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: true + } + )"); + env.TestWaitNotification(runtime, copyTx); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", false}}); + + { + const auto describeResult = DescribePath(runtime, tablePath); + const auto& columns = describeResult.GetPathDescription().GetTable().GetColumns(); + for (const auto& column : columns) { + if (column.GetName() == "value") { + UNIT_ASSERT_VALUES_EQUAL(column.GetSetNotNullInProgress(), false); + } + } + } + } + + Y_UNIT_TEST_TWIN(SetNotNullFailsWhileCopyTableInProgress, IsBackup) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableCopy"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep CopyTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, CopyTable's local Propose has already committed - the source + // path is already marked EPathStateCopying. + TBlockEvents<TEvDataShard::TEvProposeTransaction> copyProposeBlocker(runtime); + + ui64 copyTxId = ++txId; + if (IsBackup) { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: true + } + )"); + } else { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: false + } + )"); + } + + runtime.WaitFor("block copy propose to datashards", [&]{ return copyProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (copying). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while copying): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let CopyTable finish normally. + copyProposeBlocker.Stop(); + copyProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, copyTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + // Now that the copy is done, SetNotNull on the source table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + } + + Y_UNIT_TEST(SetNotNullWithUsualSchemeOperation) { + // As an example of a 'usual' scheme operation, this test uses TRUNCATE TABLE. + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + + ui64 txId = 100; + + TString root = "/MyRoot"; + TString tablePath = root + "/Table"; + + TestCreateTable(runtime, ++txId, root, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvProposeTransaction> truncateTableBlocker(runtime); + const ui64 truncateTxId = ++txId; + AsyncTruncateTable(runtime, truncateTxId, root, "Table", TTestTxConfig::SchemeShard); + runtime.WaitFor("block truncate propose to datashards", [&]{ return truncateTableBlocker.size() > 0; }); + + ui64 setConstraintTxId = ++txId; + auto response = TestSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + truncateTableBlocker.Stop(); + truncateTableBlocker.Unblock(); + env.TestWaitNotification(runtime, truncateTxId); + } } // Y_UNIT_TEST_SUITE(SetNotNullTest) diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make index 13dfda64b9e..9df7e2698c9 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make @@ -1,6 +1,8 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) FORK_SUBTESTS() +SPLIT_FACTOR(60) +SIZE(MEDIUM) PEERDIR( library/cpp/getopt diff --git a/ydb/core/util/actorsys_test/testactorsys.h b/ydb/core/util/actorsys_test/testactorsys.h index d8bf7c420b4..7c76a4db0be 100644 --- a/ydb/core/util/actorsys_test/testactorsys.h +++ b/ydb/core/util/actorsys_test/testactorsys.h @@ -8,6 +8,9 @@ #include <ydb/library/actors/core/mailbox.h> #include <ydb/library/actors/core/scheduler_queue.h> #include <ydb/library/actors/core/executor_thread.h> +#include <ydb/library/actors/core/thread_context.h> +#include <ydb/library/actors/core/execution_stats.h> +#include <ydb/library/actors/core/mon_stats.h> #include <ydb/library/actors/interconnect/interconnect_common.h> #include <ydb/library/actors/interconnect/rdma/mem_pool.h> #include <ydb/library/actors/util/should_continue.h> @@ -107,6 +110,9 @@ class TTestActorSystem { std::unique_ptr<TMailboxTable> MailboxTable; std::unique_ptr<TExecutorThread> ExecutorThread; IExecutorPool* ExecutorPool = nullptr; + std::unique_ptr<TExecutorThreadStats> ExecutorThreadStats; + std::unique_ptr<TExecutionStats> ExecutionStats; + std::unique_ptr<TThreadContext> ThreadContext; std::unordered_map<ui32, TActorId> InterconnectProxy; TTestSchedulerThread *SchedulerThread; ui32 NextHint = 1; @@ -330,6 +336,13 @@ public: info.MailboxTable = std::make_unique<TMailboxTable>(); info.ExecutorThread = std::make_unique<TExecutorThread>(0, info.ActorSystem.get(), pool, "TestExecutor"); info.ExecutorPool = pool; + + info.ExecutorThreadStats = std::make_unique<TExecutorThreadStats>(); + info.ExecutionStats = std::make_unique<TExecutionStats>(); + info.ExecutionStats->Switch(info.ExecutorThreadStats.get()); + info.ThreadContext = std::make_unique<TThreadContext>( + info.ExecutorThread->GetWorkerId(), info.ExecutorPool, nullptr); + info.ThreadContext->ExecutionStats = info.ExecutionStats.get(); } void StartNode(ui32 nodeId) { @@ -734,16 +747,12 @@ public: info->SchedulerThread->AdjustClock(Clock); const auto nowTs = GetCycleCountFast(); - TExecutorThreadStats executorThreadStats; - TExecutionStats executionStats; - executionStats.Switch(&executorThreadStats); - TThreadContext threadCtx(info->ExecutorThread->GetWorkerId(), info->ExecutorPool, nullptr); - threadCtx.ExecutionStats = &executionStats; - threadCtx.SetMailboxScheduledTimestampTs(nowTs); - threadCtx.SetEventEnqueuedTimestampTs(nowTs); - threadCtx.SetActivationTimeUs(0); - threadCtx.SetEventDeliveryTimeUs(0); - TThreadContextGuard threadContextGuard(&threadCtx); + TThreadContext* threadCtx = info->ThreadContext.get(); + threadCtx->SetMailboxScheduledTimestampTs(nowTs); + threadCtx->SetEventEnqueuedTimestampTs(nowTs); + threadCtx->SetActivationTimeUs(0); + threadCtx->SetEventDeliveryTimeUs(0); + TThreadContextGuard threadContextGuard(threadCtx); // allocate context and store its reference in TLS TActorContext ctx(mbox, *info->ExecutorThread, GetCycleCountFast(), actorId); diff --git a/ydb/core/viewer/tests/canondata/result.json b/ydb/core/viewer/tests/canondata/result.json index 22fe623bdf5..d435695e120 100644 --- a/ydb/core/viewer/tests/canondata/result.json +++ b/ydb/core/viewer/tests/canondata/result.json @@ -852,6 +852,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -955,6 +957,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -1058,6 +1062,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -1162,6 +1168,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -1266,6 +1274,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -8755,6 +8765,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -9777,6 +9789,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -11093,6 +11107,8 @@ "ChangeTime": "not-zero-number-text", "Device": 1, "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -13072,6 +13088,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -13468,6 +13486,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -13864,6 +13884,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -14260,6 +14282,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", @@ -14656,6 +14680,8 @@ "ChangeTime": "not-zero-number-text", "Device": "Green", "EnforcedDynamicSlotSize": "not-zero-number-text", + "ExpectedSlotCount": 0, + "ExpectedSlotSize": "0", "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", diff --git a/ydb/core/viewer/viewer_cluster.h b/ydb/core/viewer/viewer_cluster.h index 2420d69cdb2..150c7f3a047 100644 --- a/ydb/core/viewer/viewer_cluster.h +++ b/ydb/core/viewer/viewer_cluster.h @@ -948,6 +948,9 @@ private: } static ui64 GetSlotSize(const NKikimrSysView::TPDiskInfo& pdiskInfo) { + if (pdiskInfo.GetExpectedSlotSize()) { + return pdiskInfo.GetExpectedSlotSize(); + } if (pdiskInfo.GetExpectedSlotCount()) { return pdiskInfo.GetTotalSize() / pdiskInfo.GetExpectedSlotCount(); } else { diff --git a/ydb/core/viewer/viewer_database_stats.h b/ydb/core/viewer/viewer_database_stats.h index 349f2f1c77b..12d3ad5b67c 100644 --- a/ydb/core/viewer/viewer_database_stats.h +++ b/ydb/core/viewer/viewer_database_stats.h @@ -12,6 +12,36 @@ using namespace NActors; using namespace NNodeWhiteboard; using TNavigate = NSchemeCache::TSchemeCacheNavigate; +struct TDatabaseStorageStats { + ui64 Total = 0; + bool UnknownSlotSize = false; + + void AddVDisk( + const NKikimrWhiteboard::TVDiskStateInfo& vdisk, + const NKikimrWhiteboard::TPDiskStateInfo& pdisk, + ui32 groupSizeInUnits) { + ui64 slotSize = pdisk.GetExpectedSlotSize(); + if (!slotSize) { + slotSize = pdisk.GetEnforcedDynamicSlotSize(); + } + if (!slotSize) { + const ui32 slotCount = pdisk.GetExpectedSlotCount(); + if (!slotCount) { + UnknownSlotSize = true; + Total += vdisk.GetAvailableSize(); + return; + } + slotSize = pdisk.GetTotalSize() / slotCount; + } + + const ui32 ownerWeight = TPDiskConfig::GetOwnerWeight( + groupSizeInUnits, + pdisk.GetSlotSizeInUnits(), + pdisk.GetExpectedSlotSize()); + Total += slotSize * ownerWeight; + } +}; + // Simple database/storage stats endpoint modeled after viewer_nodes class TJsonDatabaseStats : public TViewerPipeClient { using TBase = TViewerPipeClient; @@ -301,7 +331,6 @@ public: bool absentDatabaseNodeInfo = false; bool incompleteDatabaseStats = false; bool incompleteStorageStats = false; - bool unknownSlotSize = false; bool unknownPDisk = false; ui64 grpcRequestBytes = 0; ui64 grpcResponseBytes = 0; @@ -319,8 +348,8 @@ public: ui64 storageNetworkBytes = 0; ui64 diskReadBytes = 0; ui64 diskWriteBytes = 0; - ui64 storageTotal = 0; ui64 storageConsumed = 0; + TDatabaseStorageStats storageStats; std::unordered_map<TPDiskId, const NKikimrWhiteboard::TPDiskStateInfo&> pDisksIdx; Result.ClearProblems(); @@ -414,17 +443,10 @@ public: storageConsumed += record.GetAllocatedSize(); auto itPDisk = pDisksIdx.find(std::make_pair(nodeId, record.GetPDiskId())); if (itPDisk != pDisksIdx.end()) { - auto slotSize = itPDisk->second.GetEnforcedDynamicSlotSize(); - if (!slotSize) { - auto slotCount = itPDisk->second.GetExpectedSlotCount(); - if (!slotCount) { - unknownSlotSize = true; - storageTotal += record.GetAvailableSize(); - continue; - } - slotSize = itPDisk->second.GetTotalSize() / slotCount; - } - storageTotal += slotSize * TPDiskConfig::GetOwnerWeight(itGroup->second.GetInfo().GetGroupSizeInUnits(), itPDisk->second.GetSlotSizeInUnits()); + storageStats.AddVDisk( + record, + itPDisk->second, + itGroup->second.GetInfo().GetGroupSizeInUnits()); } else { unknownPDisk = true; } @@ -478,7 +500,7 @@ public: if (incompleteStorageStats) { Result.AddProblems("incomplete-storage-stats"); } - if (unknownSlotSize) { + if (storageStats.UnknownSlotSize) { Result.AddProblems("unknown-slot-size"); } if (unknownPDisk) { @@ -504,7 +526,7 @@ public: Result.SetStorageNetworkWrite(databaseToStorageBytes); Result.SetDiskRead(diskReadBytes); Result.SetDiskWrite(diskWriteBytes); - Result.SetStorageTotal(storageTotal); + Result.SetStorageTotal(storageStats.Total); Result.SetStorageConsumed(storageConsumed); } diff --git a/ydb/core/viewer/viewer_groups.h b/ydb/core/viewer/viewer_groups.h index 945c3850251..a1d24c04475 100644 --- a/ydb/core/viewer/viewer_groups.h +++ b/ydb/core/viewer/viewer_groups.h @@ -195,6 +195,7 @@ public: ui64 EnforcedDynamicSlotSize = 0; ui32 SlotCount = 0; ui32 SlotSizeInUnits = 0; + ui64 ExpectedSlotSize = 0; ui32 NumActiveSlots = 0; ui64 Category = 0; TString DecommitStatus; @@ -219,7 +220,9 @@ public: } ui64 GetSlotTotalSize() const { - if (EnforcedDynamicSlotSize) { + if (ExpectedSlotSize) { + return ExpectedSlotSize; + } else if (EnforcedDynamicSlotSize) { return EnforcedDynamicSlotSize; } else if (SlotCount) { return TotalSize / SlotCount; @@ -228,6 +231,10 @@ public: } } + ui32 GetOwnerWeight(ui32 groupSizeInUnits) const { + return TPDiskConfig::GetOwnerWeight(groupSizeInUnits, SlotSizeInUnits, ExpectedSlotSize); + } + float GetDiskSpaceUsage() const { return TotalSize ? 100.0 * (TotalSize - AvailableSize) / TotalSize : 0; } @@ -483,7 +490,7 @@ public: DiskSpace = std::max(DiskSpace, vdisk.DiskSpace); DiskSpaceUsage = std::max(DiskSpaceUsage, itPDisk->second.GetDiskSpaceUsage()); MaxPDiskUsage = std::max(MaxPDiskUsage, itPDisk->second.PDiskUsage); - ui64 slotSize = itPDisk->second.GetSlotTotalSize() * TPDiskConfig::GetOwnerWeight(GroupSizeInUnits, itPDisk->second.SlotSizeInUnits); + ui64 slotSize = itPDisk->second.GetSlotTotalSize() * itPDisk->second.GetOwnerWeight(GroupSizeInUnits); // when a vdisk overgrows its nominal slot, keep its real AvailableSize if (slotSize > vdisk.AllocatedSize) { ui64 slotAvailable = slotSize - vdisk.AllocatedSize; @@ -1608,6 +1615,7 @@ public: pDisk.EnforcedDynamicSlotSize = info.GetEnforcedDynamicSlotSize(); pDisk.SlotCount = info.GetExpectedSlotCount(); pDisk.SlotSizeInUnits = info.GetSlotSizeInUnits(); + pDisk.ExpectedSlotSize = info.GetExpectedSlotSize(); pDisk.NumActiveSlots = info.GetNumActiveSlots(); pDisk.Category = info.GetCategory(); pDisk.DecommitStatus = info.GetDecommitStatus(); @@ -1989,6 +1997,9 @@ public: if (pDisk.SlotCount < info.GetExpectedSlotCount()) { pDisk.SlotCount = info.GetExpectedSlotCount(); } + if (info.GetExpectedSlotSize()) { + pDisk.ExpectedSlotSize = info.GetExpectedSlotSize(); + } if (pDisk.NumActiveSlots < info.GetNumActiveSlots()) { pDisk.NumActiveSlots = info.GetNumActiveSlots(); } diff --git a/ydb/core/viewer/viewer_nodes.h b/ydb/core/viewer/viewer_nodes.h index cb5c40eeae3..77124a66ee5 100644 --- a/ydb/core/viewer/viewer_nodes.h +++ b/ydb/core/viewer/viewer_nodes.h @@ -410,6 +410,7 @@ class TJsonNodes : public TViewerPipeClient { pDiskState.SetTotalSize(pdisk.GetTotalSize()); pDiskState.SetAvailableSize(pdisk.GetAvailableSize()); pDiskState.SetExpectedSlotCount(pdisk.GetExpectedSlotCount()); + pDiskState.SetExpectedSlotSize(pdisk.GetExpectedSlotSize()); } } if (VDisks.empty() && !SysViewVDisks.empty()) { diff --git a/ydb/core/viewer/viewer_tenantinfo.h b/ydb/core/viewer/viewer_tenantinfo.h index 4aa39c928f3..154338f2db2 100644 --- a/ydb/core/viewer/viewer_tenantinfo.h +++ b/ydb/core/viewer/viewer_tenantinfo.h @@ -681,6 +681,9 @@ public: } static ui64 GetSlotSize(const NKikimrSysView::TPDiskInfo& pdiskInfo) { + if (pdiskInfo.GetExpectedSlotSize()) { + return pdiskInfo.GetExpectedSlotSize(); + } if (pdiskInfo.GetExpectedSlotCount()) { return pdiskInfo.GetTotalSize() / pdiskInfo.GetExpectedSlotCount(); } else { diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 41728137e72..f2193707474 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -16,6 +16,7 @@ #include "viewer_vdiskinfo.h" #include "viewer_pdiskinfo.h" #include "query_autocomplete_helper.h" +#include "viewer_database_stats.h" #include "viewer_groups.h" #include <library/cpp/testing/unittest/registar.h> @@ -725,6 +726,68 @@ Y_UNIT_TEST_SUITE(Viewer) { StorageSpaceTest("all", NKikimrWhiteboard::EFlag::Red, 10, 100, true, "Red"); } + Y_UNIT_TEST(DatabaseStatsStorageLimitWithExpectedSlotSize) + { + TDatabaseStorageStats stats; + NKikimrWhiteboard::TVDiskStateInfo vdisk; + NKikimrWhiteboard::TPDiskStateInfo pdisk; + pdisk.SetExpectedSlotSize(100); + pdisk.SetEnforcedDynamicSlotSize(1000); + pdisk.SetExpectedSlotCount(10); + pdisk.SetTotalSize(10000); + pdisk.SetSlotSizeInUnits(1); + + stats.AddVDisk(vdisk, pdisk, 4); + + UNIT_ASSERT_VALUES_EQUAL(stats.Total, 100); + UNIT_ASSERT(!stats.UnknownSlotSize); + } + + Y_UNIT_TEST(DatabaseStatsStorageLimitWithDynamicSlotSize) + { + TDatabaseStorageStats stats; + NKikimrWhiteboard::TVDiskStateInfo vdisk; + NKikimrWhiteboard::TPDiskStateInfo pdisk; + pdisk.SetEnforcedDynamicSlotSize(100); + pdisk.SetExpectedSlotCount(10); + pdisk.SetTotalSize(10000); + pdisk.SetSlotSizeInUnits(2); + + stats.AddVDisk(vdisk, pdisk, 4); + + UNIT_ASSERT_VALUES_EQUAL(stats.Total, 200); + UNIT_ASSERT(!stats.UnknownSlotSize); + } + + Y_UNIT_TEST(DatabaseStatsStorageLimitWithExpectedSlotCount) + { + TDatabaseStorageStats stats; + NKikimrWhiteboard::TVDiskStateInfo vdisk; + NKikimrWhiteboard::TPDiskStateInfo pdisk; + pdisk.SetExpectedSlotCount(10); + pdisk.SetTotalSize(1000); + pdisk.SetSlotSizeInUnits(3); + + stats.AddVDisk(vdisk, pdisk, 5); + + UNIT_ASSERT_VALUES_EQUAL(stats.Total, 200); + UNIT_ASSERT(!stats.UnknownSlotSize); + } + + Y_UNIT_TEST(DatabaseStatsStorageLimitWithUnknownSlotSize) + { + TDatabaseStorageStats stats; + NKikimrWhiteboard::TVDiskStateInfo vdisk; + vdisk.SetAvailableSize(75); + NKikimrWhiteboard::TPDiskStateInfo pdisk; + pdisk.SetTotalSize(1000); + + stats.AddVDisk(vdisk, pdisk, 1); + + UNIT_ASSERT_VALUES_EQUAL(stats.Total, 75); + UNIT_ASSERT(stats.UnknownSlotSize); + } + Y_UNIT_TEST(StorageGroupDiskSpaceDoesNotDependOnUsage) { TStorageGroups::TGroup group; @@ -767,6 +830,28 @@ Y_UNIT_TEST_SUITE(Viewer) { UNIT_ASSERT_DOUBLES_EQUAL(group.Usage, 50.0, 1e-6); } + Y_UNIT_TEST(StorageGroupUsageWithExpectedSlotSize) + { + TStorageGroups::TGroup group; + group.GroupSizeInUnits = 2; + auto& vdisk = group.VDisks.emplace_back(); + vdisk.VSlotId = TVSlotId(1, 1, 1); + vdisk.AllocatedSize = 25; + vdisk.AvailableSize = 900; + + TStorageGroups::TPDisk pdisk; + pdisk.ExpectedSlotSize = 100; + pdisk.EnforcedDynamicSlotSize = 1000; + pdisk.SlotSizeInUnits = 1; + pdisk.TotalSize = 10000; + pdisk.AvailableSize = 9000; + pdisk.SlotCount = 10; + + group.CalcAvailableAndDiskSpace({{TPDiskId(1, 1), pdisk}}); + UNIT_ASSERT_VALUES_EQUAL(group.Limit, 100); + UNIT_ASSERT_DOUBLES_EQUAL(group.Usage, 25.0, 1e-6); + } + Y_UNIT_TEST(StorageGroupUsageWithoutDynamicSlotSize) { TStorageGroups::TGroup group; diff --git a/ydb/docs/en/core/changelog-server.md b/ydb/docs/en/core/changelog-server.md index f70ac0c77ba..be8dd28efc8 100644 --- a/ydb/docs/en/core/changelog-server.md +++ b/ydb/docs/en/core/changelog-server.md @@ -4,7 +4,7 @@ ### Version 26.1.1.20 {#26-1-1-20} -Release date: TBD. +Release date: July 2, 2026. #### Functionality diff --git a/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md b/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md index 6a578cfe901..1ba4bab21e3 100644 --- a/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md +++ b/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md @@ -1,145 +1,180 @@ -# Connecting to {{ ydb-short-name }} Using the DBeaver Plugin +# Connecting to {{ ydb-short-name }} using the DBeaver plugin -[DBeaver](https://dbeaver.com) is a free cross-platform open-source database management tool that provides a visual interface for connecting to various databases and executing SQL queries. +[DBeaver](https://dbeaver.com) is a free, cross-platform, open-source database management tool that provides a visual interface for connecting to various databases and executing SQL queries. -[YDB DBeaver Plugin](https://github.com/ydb-platform/ydb-dbeaver-plugin) is a DBeaver extension with native support for {{ ydb-short-name }}. Unlike [connecting via a JDBC driver](dbeaver.md), the plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor, query execution plan visualization, session and cluster monitoring, access control (ACL) management, and other features. +[YDB DBeaver Plugin](https://github.com/ydb-platform/ydb-dbeaver-plugin) is a DBeaver extension with native support for {{ ydb-short-name }}. Unlike [connecting via a JDBC driver](dbeaver.md), the plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor, execution plan visualization, session and cluster monitoring, access control (ACL) management, and other features. -## Key Features of the Plugin {#features} +## Key features of the plugin {#features} -- Connecting to {{ ydb-name }} with all [authentication methods](../../security/authentication.md): anonymous, static, token-based, service account, metadata-based. +- Connecting to {{ ydb-name }} with all [authentication](../../security/authentication.md) methods: anonymous, static, token-based, service account, and metadata. - Hierarchical object navigator: tables, topics, external data sources, external tables, views. - System objects: [system views](../../dev/system-views.md) (`.sys`), [resource pools](../../concepts/glossary.md#resource-pool), and [resource pool classifiers](../../concepts/glossary.md#resource-pool-classifier). -- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting for over 150 keywords and built-in functions. -- Visualization of the query execution plan (`EXPLAIN` / `EXPLAIN ANALYZE`). -- Monitoring of active sessions via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). -- Cluster dashboard: CPU load, disk usage, memory usage, network traffic, node status (updated every 5 seconds). +- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting for 150+ keywords and built-in functions. +- Query execution plan visualization (`EXPLAIN` / `EXPLAIN ANALYZE`). +- Active session monitoring via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). +- Cluster dashboard: CPU load, disk usage, memory usage, network traffic, node status (updates every 5 seconds). - [Access control (ACL)](../../security/authorization.md#right) management: granting, revoking, and viewing permissions. -- Management of [streaming queries](../../concepts/glossary.md#streaming-query): viewing, modifying, starting, stopping. +- [Streaming query](../../concepts/glossary.md#streaming-query) management: viewing, modifying, starting, and stopping. - [Federated queries](../../concepts/query_execution/federated_query/index.md) via external data sources (S3, databases). -- [SQL Dialect Converter](../sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. -- Specialized editors for `JSON`, `JSONDOCUMENT`, `YSON` data types. +- [SQL query converter](../sql-translation/sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, and others) to YQL. +- Specialized editors for `JSON`, `JSONDOCUMENT`, and `YSON` data types. ## Requirements {#requirements} The plugin requires DBeaver Community Edition version 24.x or later. -## Installing the Plugin {#installation} +## Installing the plugin {#installation} -The plugin is installed via DBeaver's extension installation mechanism from a URL repository, which ensures automatic updates. +The plugin can be installed in two ways: + +- **Via P2 repository URL (recommended)**: DBeaver downloads the plugin from cloud storage and remembers the source for subsequent automatic updates. +- **From a ZIP archive (GitHub Releases)**: download the archive and install via a local file. Suitable for networks where external Eclipse repositories are unavailable, or for reproducible installation of a specific version. Automatic updates are not supported with this method. + +### Installation via P2 repository URL {#installation-url} 1. Open DBeaver. In the top menu, select **Help → Install New Software...**. - {% cut "Screenshot" %} + {% cut "Screenshot" %} + +  -  + {% endcut %} +2. Click the **Add...** button to the right of the **Work with:** field. - {% endcut %} + {% cut "Screenshot" %} -1. Click the **Add...** button to the right of the **Work with:** field. +  - {% cut "Screenshot" %} + {% endcut %} +3. In the **Add Repository** window that opens, specify a repository name (e.g., `YDB Plugin`) and paste the following URL into the **Location** field: -  - {% endcut %} + ```text + https://storage.yandexcloud.net/ydb-dbeaver-plugin + ``` -1. In the **Add Repository** window that opens, specify a repository name (for example, `YDB Plugin`) and paste the following URL into the **Location** field: - ```text - https://storage.yandexcloud.net/ydb-dbeaver-plugin - ``` + Click **Add**. DBeaver will load the repository metadata. - Click **Add**. DBeaver will load the repository metadata. + {% cut "Screenshot" %} - {% cut "Screenshot" %} +  -  + {% endcut %} - {% endcut %} + {% note warning %} -1. The **DBeaver YDB Support** category will appear in the component list. Select it and click **Next >**. + During installation, DBeaver downloads not only the plugin itself but also its dependencies (OSGi components) from Eclipse project servers and other external repositories. If these resources are not accessible from your network, contact your network administrator or use [installation from a ZIP archive](#installation-zip). - {% cut "Screenshot" %} + {% endnote %} +4. A category **DBeaver YDB Support** will appear in the component list. Check it and click **Next >**. -  + {% cut "Screenshot" %} - {% endcut %} +  -1. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are listed, then click **Next >**. + {% endcut %} +5. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are in the list, and click **Next >**. - {% cut "Screenshot" %} + {% cut "Screenshot" %} -  +  - {% endcut %} + {% endcut %} +6. DBeaver may show a warning about unsigned content. This is expected behavior — the plugin's JAR files are not signed with a commercial certificate. Click **Install Anyway**. -1. DBeaver may display a warning about unsigned content. This is expected behavior — the plugin's JAR files are not signed with a commercial certificate. Click **Install Anyway**. + {% note info %} - {% note info %} + Eclipse, on which DBeaver is based, verifies JAR file signatures to confirm authenticity. This open-source plugin is distributed without a signature; the source code is available in the [repository](https://github.com/ydb-platform/ydb-dbeaver-plugin). - Eclipse, on which DBeaver is based, checks JAR file signatures to verify authenticity. This open-source plugin is distributed without a signature; the source code is available in the [repository](https://github.com/ydb-platform/ydb-dbeaver-plugin). + {% endnote %} +7. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. - {% endnote %} + {% cut "Screenshot" %} -1. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. +  - {% cut "Screenshot" %} + {% endcut %} +8. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After restart, the plugin will be active. -  +### Installation from a ZIP archive (GitHub Releases) {#installation-zip} - {% endcut %} +1. Go to the [GitHub Releases page](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) and download the `ydb-dbeaver-plugin-*.zip` file of the required version. +2. Open DBeaver. In the top menu, select **Help → Install New Software...**. +3. Click the **Add...** button to the right of the **Work with:** field. +4. In the **Add Repository** window that opens, click **Archive...**, specify the path to the downloaded ZIP file, and click **Add**. +5. The **DBeaver YDB Support** category will appear in the component list. Check it. -1. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After restarting, the plugin will become active. + {% note warning %} -## Creating a Connection to {{ ydb-name }} {#connection} + Before clicking **Next >**, uncheck the **Contact all update sites during install to find required software** checkbox at the bottom of the **Install New Software** window. If this checkbox is enabled, DBeaver contacts all registered P2 repositories (`dbeaver.io`, `eclipse.org`, and others) when calculating dependencies, and if they are unavailable or respond slowly, the installation dialog hangs at the **Calculating requirements and dependencies** step. The plugin only uses DBeaver components already installed locally, so contacting other repositories during installation is not required. + + {% endnote %} +6. Click **Next >**. +7. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are present in the list, and click **Next >**. +8. DBeaver may show a warning about unsigned content. Click **Install Anyway**. +9. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. +10. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After the restart, the plugin becomes active. + +{% note warning %} + +When installing from a ZIP archive, automatic updates are not supported: DBeaver does not know where to look for new versions. To update: + +1. Download the new archive from the [GitHub Releases](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) page. +2. Open **Help → Installation Information**, select the plugin, and click **Uninstall**. +3. Install the new version by following the [instructions for installing from a ZIP archive](#installation-zip). + +{% endnote %} + +## Creating a connection to {{ ydb-name }} {#connection} To create a connection to {{ ydb-name }}, follow these steps: 1. In the top menu, select **Database → New Database Connection** (or press `Ctrl+Shift+N`). -1. In the search field, type `YDB`. Select **YDB** from the list and click **Next**. -1. The {{ ydb-name }} connection setup page will open. Fill in the fields: +2. In the search field, enter `YDB`. Select **YDB** from the list and click **Next**. +3. The connection settings page for {{ ydb-name }} will open. Fill in the fields: - | Field | Description | Example | - |------|----------|--------| - | **Host** | Host of the [endpoint](../../concepts/connect.md#endpoint) for the {{ ydb-name }} cluster | `ydb.example.com` | - | **Port** | Port (default is `2135`) | `2135` | - | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | - | **Monitoring URL** | URL of the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) with the database path, used for the dashboard (optional) | `http://ydb.example.com:8765/monitoring/tenant?name=%2FRoot%2Fdatabase` | - | **Use secure connection** | Use a secure connection (`grpcs://`) | ☑ | - | **Enable autocomplete API** | Autocomplete via {{ ydb-short-name }} API | ☑ | + | Field | Description | Example | + | --- | --- | --- | + | **Host** | Host of the {{ ydb-name }} cluster [endpoint](../../concepts/connect.md#endpoint) | `ydb.example.com` | + | **Port** | Port (default `2135`) | `2135` | + | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | + | **Monitoring URL** | URL of the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) with the database path, used for the dashboard (optional) | `http://ydb.example.com:8765/monitoring/tenant?name=%2FRoot%2Fdatabase` | + | **Use secure connection** | Use a secure connection (`grpcs://`) | ☑ | + | **Enable autocomplete API** | Autocomplete via {{ ydb-short-name }} API | ☑ | +4. Select the authentication method from the **Auth type** drop-down list (see [Authentication methods](#auth-methods)). +5. Click the **Test Connection** button to verify the settings. If the connection is successful, a dialog with the connection time in milliseconds will appear. +6. Click the **Finish** button. The connection will appear in the **Database Navigator** panel. -1. Select an authentication method from the **Auth type** dropdown list (see [Authentication Methods](#auth-methods)). -1. Click the **Test Connection** button to verify the settings. If the connection is successful, a dialog will appear with the connection time in milliseconds. -1. Click the **Finish** button. The connection will appear in the **Database Navigator** panel. +## Authentication methods {#auth-methods} -## Authentication Methods {#auth-methods} - -The plugin supports all [authentication methods](../../security/authentication.md) available in {{ ydb-short-name }}. The method is selected from the **Auth type** dropdown list on the connection setup page. +The plugin supports all [authentication](../../security/authentication.md) methods available in {{ ydb-short-name }}. The method is selected from the **Auth type** drop-down list on the connection settings page. ### Anonymous {#auth-anonymous} -Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled out. +Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled in. -### Static (Username and Password) {#auth-static} +### Static (login and password) {#auth-static} -Authentication using a username and password. Enter the username in the **User** field and the password in the **Password** field. Used if {{ ydb-short-name }} server has [username and password authentication](../../security/authentication.md#static-credentials) enabled. +Authentication by login and password. Enter the username in the **User** field and the password in the **Password** field. on the {{ ydb-short-name }} server. Used if [login and password authentication](../../security/authentication.md#static-credentials) is enabled {% note info %} -In managed installations of {{ ydb-name }}, username and password authentication is disabled: managed services use the centralized access control system of the cloud platform ([IAM](https://yandex.cloud/ru/docs/iam/)). +In managed installations of {{ ydb-name }}, login and password authentication is disabled: managed services use the cloud platform's centralized access control system ([IAM](https://yandex.cloud/en/docs/iam/)). {% endnote %} ### Token {#auth-token} -Authentication using an [IAM](https://yandex.cloud/ru/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/ru/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. +Authentication by [IAM](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. ### Service Account {#auth-service-account} -Authentication using the key of a [Yandex Cloud service account](https://yandex.cloud/ru/docs/iam/concepts/users/service-accounts). Specify the path to the JSON file with the key in the **SA Key File** field (use the **...** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/ru/docs/iam/operations/authentication/manage-authorized-keys). +Authentication by a [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **SA Key File** field (use the **...** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). Key file format: + ```json { "id": "aje...", @@ -148,15 +183,16 @@ Key file format: } ``` + ### Metadata {#auth-metadata} -Authentication via the [Yandex Cloud metadata service](https://yandex.cloud/ru/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine's metadata service. Used only when DBeaver is running on a Yandex Cloud virtual machine. +Authentication via [Yandex Cloud Metadata Service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine metadata service. It is used only when DBeaver is running on a Yandex Cloud virtual machine. -## Object Navigator {#object-navigator} +## Object navigator {#object-navigator} -After connecting, the {{ ydb-short-name }} object hierarchy is displayed in the **Database Navigator** panel. The root node is the connection, and inside it is the database path, which contains the following folders: +After connection, the **Database Navigator** panel displays the {{ ydb-short-name }} object hierarchy. The root node is the connection, inside it is the database path, which contains the following folders: -- **Tables** — tables organized by subdirectories according to the path in {{ ydb-short-name }} (for example, a table with the path `folder1/subfolder/mytable` will be nested under `folder1 → subfolder`). +- **Tables** — tables organized into subdirectories according to the path in {{ ydb-short-name }} (for example, a table at path `folder1/subfolder/mytable` will be nested in `folder1 → subfolder`). - **Topics** — [topics](../../concepts/datamodel/topic.md). - **Views** — [views](../../concepts/datamodel/view.md). - **External Data Sources** — [external data sources](../../concepts/glossary.md#external-data-source). @@ -164,95 +200,98 @@ After connecting, the {{ ydb-short-name }} object hierarchy is displayed in the - **System Views (.sys)** — [system views](../../dev/system-views.md), such as `partition_stats`, `query_sessions`. - **Resource Pools** — [resource pools](../../concepts/glossary.md#resource-pool). -## Working with the Plugin {#capabilities} +## Working with the plugin {#capabilities} -### YQL Editor {#yql-editor} +### YQL editor {#yql-editor} -Open the **SQL Editor** (`F3` or double-click on the connection). The editor supports: +Open the **SQL Editor** (`F3` or double-click the connection). The editor supports: -- Syntax highlighting for [YQL](../../yql/reference/index.md): keywords (`UPSERT`, `REPLACE`, `EVALUATE`, `PRAGMA`, `WINDOW`, and 145+ others), data types, built-in functions. -- Autocompletion of table, column, and function names. -- Query execution: `Ctrl+Enter` for the current query, `Ctrl+Shift+Enter` for the entire script. +- Syntax highlighting for [YQL](../../yql/reference/index.md): keywords (`UPSERT`, `REPLACE`, `EVALUATE`, `PRAGMA`, `WINDOW` and 145+ others), data types, built-in functions. +- Auto-completion of table names, columns, and functions. +- Query execution: `Ctrl+Enter` — current query, `Ctrl+Shift+Enter` — entire script. Example YQL query: + ```yql UPSERT INTO `users` (id, name, created_at) VALUES (1, "Alice", CurrentUtcDatetime()); ``` -### EXPLAIN and Execution Plan {#explain} + +### EXPLAIN and execution plan {#explain} Click **Explain** (or `Ctrl+Shift+E`) to get the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md). The plugin displays: -- **Text plan** — a tree of operations in text form. -- **Diagram** — a graphical representation as a DAG. -- **SVG plan** — an interactive visualization. +- **Text plan** — operation tree in text form. +- **Diagram** — graphical representation as a DAG. +- **SVG plan** — interactive visualization. -`EXPLAIN ANALYZE` additionally shows execution statistics (number of rows, execution time). +`EXPLAIN ANALYZE` additionally shows execution statistics (row count, execution time). -### Session Manager {#session-manager} +### Session manager {#session-manager} -Right-click on the connection and select **Manage Sessions**, or use the **Database → Manage Sessions** menu item. The opened view displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Hide Idle** checkbox hides sessions without an active query. +Select the menu item **Tools → Sessions Manager**, or right-click the connection and select the corresponding item. The opened view displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Hide Idle** checkbox hides sessions without an active query. -### Cluster Dashboard {#cluster-dashboard} +### Cluster dashboard {#cluster-dashboard} -Open the **Dashboard** tab in the connection editor (requires the **Monitoring URL** field to be filled in during connection setup). +Open the **Dashboard** tab in the connection editor (requires the **Monitoring URL** field to be filled in when configuring the connection). {% note warning %} -The dashboard is only available when working with self-hosted installations of {{ ydb-short-name }} where access to the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) is available. In Yandex Cloud Managed Service for {{ ydb-short-name }}, Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/ru/docs/ydb/operations/monitoring) for monitoring. +The dashboard is only available when working with self-hosted {{ ydb-short-name }} installations that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, the Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. {% endnote %} -The dashboard displays real-time data (updated every 5 seconds): +The dashboard displays in real time (updates every 5 seconds): -- CPU load by nodes. +- CPU load by node. - Disk space usage. - Memory usage. - Network traffic. -- Number of executing queries. -- Status of cluster nodes. +- Number of running queries. +- Cluster node status. -### Streaming Queries {#streaming-queries} +### Streaming queries {#streaming-queries} -In the navigator, expand the **Streaming Queries** folder. For each query, you can: +In the navigator, expand the **Streaming Queries** folder. For each query, the following are available: -- View the original YQL. +- View the source YQL. - View errors (issues). - View the execution plan. -- Perform actions: **Start**, **Stop**, **Alter**. +- Actions: **Start**, **Stop**, **Alter**. -### SQL Dialect Converter {#convert-dialect} +### SQL dialect converter {#convert-dialect} -The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. The converter is available on the **Convert Dialect** tab in the connection editor. +The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the context menu **Tools → Convert Dialect** in the connection editor. To convert a query: -1. In the **Source Dialect** dropdown list, select the source SQL dialect. The list of dialects is requested from the plugin's external service when the tab is first opened. -1. Paste the source SQL code into the **Input SQL** field. -1. Click **Convert**. The result will appear in the lower field. -1. Click **Copy** to copy the result to the clipboard. +1. In the **Source Dialect** dropdown, select the source SQL dialect. The list of dialects is fetched from the plugin's external service when the tab is first opened. +2. Paste the source SQL code into the **Input SQL** field. +3. Click **Convert**. The result appears in the lower field. +4. Click **Copy** to copy the result to the clipboard. -For more information on how the converter works, supported dialects, and limitations, see the [SQL Dialect Converter to YQL](../sql-dialect-converter.md) article. +For more details on how the converter works, supported dialects, and limitations, see the article [SQL dialect converter to YQL](../sql-translation/sql-dialect-converter.md). {% note warning %} -The plugin sends the original query to an external HTTPS service for conversion. Do not use the converter for queries containing confidential data. +To perform the conversion, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. {% endnote %} -### Creating Objects {#create-objects} +### Creating objects {#create-objects} -Right-click on a folder or object and select **Create New**: +Right-click a folder or object and select **Create New**: - **Create Table** — create a new table. - **Create Topic** — create a new topic. -- **Create Resource Pool** — create a resource pool. -## Updating the Plugin {#updates} +## Plugin update {#updates} + +Automatic updates are only supported when [installing via the P2 repository URL](#installation-url). DBeaver remembers the source (repository URL) and when a new version is published, it offers an update in one of two ways: -DBeaver uses the Eclipse P2 mechanism to detect and install updates. When you install the plugin, DBeaver remembers the source — the repository URL. When a new version is published, DBeaver compares the installed version with the version in the repository and offers an update in one of two ways: +1. Automatically on the next DBeaver launch (if update checking is enabled in **Window → Preferences → Install/Update → Automatic Updates**). +2. Manually via **Help → Check for Updates**: select the available update and follow the same steps as during the initial installation (license → unsigned content warning → restart). -1. Automatically on the next DBeaver startup (if update checks are enabled in **Window → Preferences → Install/Update → Automatic Updates**). -1. Manually via **Help → Check for Updates**: select the available update and follow the same steps as during the initial installation (license → warning about unsigned content → restart). +When [installing from a ZIP archive](#installation-zip), the update is performed manually: download the new archive from the [GitHub Releases](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) page, open **Help → Installation Information**, select the plugin, click **Uninstall**, then install the new version again. diff --git a/ydb/docs/en/core/integrations/gui/toc-ide.yaml b/ydb/docs/en/core/integrations/gui/toc-ide.yaml index 9a7aba07bc7..c8e6947778c 100644 --- a/ydb/docs/en/core/integrations/gui/toc-ide.yaml +++ b/ydb/docs/en/core/integrations/gui/toc-ide.yaml @@ -3,6 +3,8 @@ items: href: dbeaver.md - name: DBeaver Plugin href: dbeaver-plugin.md +- name: VS Code + href: vscode-plugin.md - name: DataGrip href: datagrip.md - name: Jupyter Notebook diff --git a/ydb/docs/en/core/integrations/gui/vscode-plugin.md b/ydb/docs/en/core/integrations/gui/vscode-plugin.md index 454f6dc9937..7bc93119669 100644 --- a/ydb/docs/en/core/integrations/gui/vscode-plugin.md +++ b/ydb/docs/en/core/integrations/gui/vscode-plugin.md @@ -1,110 +1,128 @@ # Connecting to {{ ydb-short-name }} using the VS Code plugin -[Visual Studio Code](https://code.visualstudio.com) is a free cross-platform open-source code editor that supports a wide ecosystem of extensions for working with databases, language servers, and development tools. +[Visual Studio Code](https://code.visualstudio.com) is a free, cross-platform, open-source code editor that supports a wide ecosystem of extensions for working with databases, language servers, and development tools. -[YDB for VS Code](https://github.com/ydb-platform/ydb-vscode-plugin) is a VS Code extension with native support for {{ ydb-short-name }}. The plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, visualization of execution plans, monitoring of sessions and the cluster, access control management (ACL), a built-in [MCP server](https://modelcontextprotocol.io/) for AI assistants, and other features. -## Key Features of the Plugin {#features} +[YDB for VS Code](https://github.com/ydb-platform/ydb-vscode-plugin) is a VS Code extension with native support for {{ ydb-short-name }}. The plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, visualization of execution plans, session and cluster monitoring, access control management (ACL), a built-in [MCP server](https://modelcontextprotocol.io/) for AI assistants, and other features. -- Connecting to {{ ydb-name }} with all [authentication methods](../../security/authentication.md): anonymous, static, token-based, service account, metadata-based. -- Hierarchical object navigator: tables ([row-oriented](../../concepts/glossary.md#row-oriented-table) and [column-oriented](../../concepts/glossary.md#column-oriented-table)), [topics](../../concepts/datamodel/topic.md), [views](../../concepts/datamodel/view.md), [external data sources](../../concepts/glossary.md#external-data-source), [external tables](../../concepts/glossary.md#external-table), [transfers](../../concepts/transfer.md), [streaming queries](../../concepts/glossary.md#streaming-query). +## Key features of the plugin {#features} + +- Connecting to {{ ydb-name }} with all [authentication](../../security/authentication.md) methods: anonymous, static, token-based, service account, and metadata. +- Hierarchical object navigator: tables ([row](../../concepts/glossary.md#row-oriented-table) and [column](../../concepts/glossary.md#column-oriented-table)), [topics](../../concepts/datamodel/topic.md), [views](../../concepts/datamodel/view.md), [external data sources](../../concepts/glossary.md#external-data-source), [external tables](../../concepts/glossary.md#external-table), [transfers](../../concepts/transfer.md), [streaming queries](../../concepts/glossary.md#streaming-query). - System objects: [system views](../../dev/system-views.md) (`.sys`), [resource pools](../../concepts/glossary.md#resource-pool). -- YQL editor with syntax highlighting, table and column autocompletion. -- Query execution and results visualization: table, JSON, chart. +- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, autocompletion of tables and columns. +- Query execution and result visualization: table, JSON, diagram. - Visualization of the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md) (`EXPLAIN`). -- Monitoring active sessions via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). -- Cluster dashboard based on [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md): CPU load, memory usage, network traffic (updated every 10 seconds). -- Managing [access rights (ACL)](../../security/authorization.md#right): viewing permissions for database objects. -- Generating DDL scripts ([`CREATE`](../../yql/reference/syntax/create_table/index.md)) for any database object. -- Managing [streaming queries](../../concepts/glossary.md#streaming-query): viewing, starting, stopping. -- [SQL query converter](../sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. +- Active session monitoring via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). +- Cluster dashboard based on [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md): CPU load, memory usage, network traffic (updates every 10 seconds). +- Management of [access rights (ACL)](../../security/authorization.md#right): viewing permissions on database objects. +- Generation of DDL scripts ( [`CREATE`](../../yql/reference/syntax/create_table/index.md)) for any database object. +- Management of [streaming queries](../../concepts/glossary.md#streaming-query): viewing, starting, stopping. +- [SQL query converter](../sql-translation/sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, and others) to YQL. - Built-in [MCP server](#mcp) — direct access to databases from AI assistants (Claude Code and others). -- [Semantic search for YQL documentation](#rag) (RAG) for AI-assisted query writing. +- [Semantic search in YQL documentation](#rag) (RAG) for AI-assisted query writing. + ## Requirements {#requirements} -The plugin requires Visual Studio Code version 1.75.0 or later. -## Plugin installation {#installation} +The plugin requires Visual Studio Code version 1.75.0 or newer. + +## Installing the plugin {#installation} The plugin can be installed from the VS Code Marketplace or from a `.vsix` file on the GitHub Releases page. -### Installing from the VS Code Marketplace {#install-marketplace} + +### Installation from VS Code Marketplace {#install-marketplace} 1. Open the Extensions panel in VS Code (`Ctrl+Shift+X` / `Cmd+Shift+X`). -1. In the search bar, enter `YDB for VS Code` and select the extension from the publisher `ydb-tech` ([direct link](https://marketplace.visualstudio.com/items?itemName=ydb-tech.ydb-vscode-plugin)). -1. Click **Install**. +2. In the search bar, enter `YDB for VS Code` and select the extension from publisher `ydb-tech` ( [direct link](https://marketplace.visualstudio.com/items?itemName=ydb-tech.ydb-vscode-plugin)). +3. Click **Install**. Alternatively, you can install the plugin from the Marketplace with a single command in the terminal: + + ```bash code --install-extension ydb-tech.ydb-vscode-plugin ``` + + ### Installation from a VSIX file {#install-vsix} This method is suitable if you need a specific version or do not have access to the Marketplace. 1. Go to the [GitHub Releases page](https://github.com/ydb-platform/ydb-vscode-plugin/releases) and download the `ydb-vscode-plugin-*.vsix` file of the required version. +2. Install the extension using one of the following methods: + + - **Via terminal:** -1. Install the extension using one of the methods: - - **Via the terminal:** - ```bash - code --install-extension ydb-vscode-plugin-X.X.X.vsix - ``` -Where `X.X.X` is the number of the downloaded version. + ```bash + code --install-extension ydb-vscode-plugin-X.X.X.vsix + ``` -- **Via the VS Code interface:** - 1. Open the extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). - 1. Click `...` (three dots) in the top right corner of the panel. - 1. Select **Install from VSIX...**. - 1. Specify the path to the downloaded `.vsix` file. + Where `X.X.X` — the number of the downloaded version. + - **Via VS Code interface:** + + 1. Open the Extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). + 2. Click `...` (three dots) in the top-right corner of the panel. + 3. Select **Install from VSIX...**. + 4. Specify the path to the downloaded `.vsix` file. + +After installation by any method, restart VS Code. The **YDB** icon will appear in the Activity Bar. -After installing using any of the methods, restart VS Code. The **YDB** icon will appear in the Activity Bar. ## Creating a connection to {{ ydb-name }} {#connection} 1. Click the **YDB** icon in the Activity Bar on the left. -1. In the **Connections** panel, click the **Add Connection** button (the `+` icon). -1. The connection creation form will open. Fill in the fields: -| Field | Description | Example | -|------|----------|--------| -| **Connection Name** | Arbitrary connection name | `my-ydb` | -| **Host** | Host of the [endpoint](../../concepts/connect.md#endpoint) of the {{ ydb-name }} cluster | `ydb.example.com` | -| **Port** | Port (default `2135`) | `2135` | -| **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | -| **Monitoring URL** | URL of [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md), used for the dashboard (filled in automatically based on the host, can be overridden) | `http://ydb.example.com:8765` | -| **Secure connection (grpcs)** | Use a secure connection (`grpcs://`) | ☑ | -| **Use RAG** | Enable [YQL documentation search](#rag) for this connection | ☑ | -1. If necessary, specify the path to the custom CA certificate (PEM) in the **CA Certificate File** field for connections with non-standard TLS. If the field is left empty, the built-in Yandex Cloud certificate is used. - -1. Select an authentication method from the **Auth type** dropdown list (see [Authentication methods](#auth-methods)). +2. In the **Connections** panel, click the **Add Connection** button (`+` icon). +3. The connection creation form will open. Fill in the fields: -1. Click **Test Connection** to check the settings. If the connection is successful, a success message will appear. + | Field | Description | Example | + | --- | --- | --- | + | **Connection Name** | Arbitrary connection name | `my-ydb` | + | **Host** | Endpoint of the [endpoint](../../concepts/connect.md#endpoint) of the {{ ydb-name }} cluster | `ydb.example.com` | + | **Port** | Port (default `2135`) | `2135` | + | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | + | **Monitoring URL** | URL of [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md), used for the dashboard (filled in automatically based on the host, can be overridden) | `http://ydb.example.com:8765` | + | **Secure connection (grpcs)** | Use a secure connection (`grpcs://`) | ☑ | + | **Use RAG** | Enable [search in YQL documentation](#rag) for this connection | ☑ | +4. If necessary, specify the path to a custom CA certificate (PEM) in the **CA Certificate File** field — for connections with non-standard TLS. If the field is left empty, the built-in Yandex Cloud certificate is used. +5. Select the authentication method from the **Auth type** drop-down list (see [Authentication methods](#auth-methods)). +6. Click **Test Connection** to verify the settings. If the connection is successful, a success message will appear. +7. Click **Save**. The connection will appear in the **Connections** panel. -1. Click **Save**. The connection will appear in the **Connections** panel. ## Authentication methods {#auth-methods} -The plugin supports all [authentication methods](../../security/authentication.md) available in {{ ydb-short-name }}. The method is selected in the **Auth type** dropdown list on the connection creation form. +The plugin supports all [authentication](../../security/authentication.md) methods available in {{ ydb-short-name }}. The method is selected in the **Auth type** dropdown on the connection creation form. + ### Anonymous {#auth-anonymous} Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled in. + ### Static Credentials (login and password) {#auth-static} -Authentication using login and password. Specify the username in the **Username** field and the password in the **Password** field. Used if {{ ydb-short-name }} server has [login and password authentication enabled](../../security/authentication.md#static-credentials). +Authentication by login and password. Specify the username in the **Username** field and the password in the **Password** field. Used if [login and password authentication](../../security/authentication.md#static-credentials) is enabled on the {{ ydb-short-name }} server. + {% note info %} -In managed installations of {{ ydb-name }}, authentication via login and password is disabled: managed services use the centralized access management system of the cloud platform ([IAM](https://yandex.cloud/en/docs/iam/)). +In managed installations of {{ ydb-name }}, login and password authentication is disabled: managed services use the cloud platform's centralized access control system ( [IAM](https://yandex.cloud/en/docs/iam/)). {% endnote %} + ### Access Token {#auth-token} -Authentication using [IAM-](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. +Authentication by [IAM](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. + {% note warning %} -The IAM token has a limited [lifespan — no more than 12 hours](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token#lifetime) — after which it must be obtained again and updated in the connection settings. For long-lived connections, use authentication via a [service account](#auth-service-account) or [metadata service](#auth-metadata). +The IAM token has a limited [lifetime — no more than 12 hours](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token#lifetime), after which it must be obtained again and updated in the connection settings. For long-lived connections, use authentication via a [service account](#auth-service-account) or [metadata service](#auth-metadata). {% endnote %} + ### Service Account Key File {#auth-service-account} -Authentication using the [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **Service Account Key File** field (use the **Browse** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). +Authentication by a [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **Service Account Key File** field (use the **Browse** button to select the file). For more details on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). + +Key file format: + -The format of the key file: ```json { "id": "aje...", @@ -112,14 +130,17 @@ The format of the key file: "private_key": "-----BEGIN RSA PRIVATE KEY-----\n..." } ``` + + ### Metadata Service {#auth-metadata} -Authentication via the [Yandex Cloud metadata service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin receives an IAM token from the virtual machine's metadata service. It is used only when VS Code is running on a Yandex Cloud virtual machine. -## Object Navigator {#object-navigator} +Authentication via [Yandex Cloud metadata service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine's metadata service. Used only when VS Code is running on a Yandex Cloud virtual machine. + +## Object navigator {#object-navigator} -After connecting, click on the connection in the **Connections** panel — the **Navigator** panel will open with the hierarchy of {{ ydb-short-name }} objects. The navigator contains the following sections: +After connecting, click the connection in the **Connections** panel — the **Navigator** panel opens with a hierarchy of {{ ydb-short-name }} objects. The navigator contains the following sections: -- **Tables** — tables (string and columnar), organized into subdirectories according to the path in {{ ydb-short-name }}. +- **Tables** — tables (row and column), organized by subdirectories according to the path in {{ ydb-short-name }}. - **System Views** — [system views](../../dev/system-views.md) (`.sys`), such as `query_sessions`, `partition_stats`. - **Views** — [views](../../concepts/datamodel/view.md). - **Topics** — [topics](../../concepts/datamodel/topic.md). @@ -129,80 +150,97 @@ After connecting, click on the connection in the **Connections** panel — the * - **Transfers** — data transfers. - **Streaming Queries** — [streaming queries](../../concepts/glossary.md#streaming-query). -Right-clicking on any object in the navigator opens a context menu with available actions. +Right-clicking any object in the navigator opens a context menu with available actions. + ## Working with the plugin {#capabilities} -### Query Workspace {#query-workspace} -Open the query workspace using `Ctrl+Shift+Q` (`Cmd+Shift+Q` on macOS) or click **Open Query Workspace** in the connection context menu. In the workspace, you can write and execute YQL queries, view history and results. +### Query workspace {#query-workspace} + +Open the query workspace via `Ctrl+Shift+Q` (`Cmd+Shift+Q` on macOS) or click **Open Query Workspace** in the connection context menu. In the workspace, you can write and execute YQL queries, view history and results. -To quickly open the editor with a pre-filled query, right-click on a table or view in the navigator and select: +To quickly open the editor with a pre-filled query, right-click a table or view in the navigator and select: - **Show Preview** — `SELECT` of the first 100 rows. - **Make Query** — `SELECT` with the object name. -### YQL Editor {#yql-editor} + +### YQL editor {#yql-editor} The editor supports: - Syntax highlighting for [YQL](../../yql/reference/index.md): keywords, data types, built-in functions. -- Autocompletion of table and column names (based on the connected database schema). +- Auto-completion of table and column names (based on the schema of the connected database). - Query execution: `Ctrl+Enter` (`Cmd+Enter` on macOS). -Example of a YQL query: +Example YQL query: + + ```yql UPSERT INTO `users` (id, name, created_at) VALUES (1, "Alice", CurrentUtcDatetime()); ``` -The execution results are displayed in the **Results** panel as a table, JSON, or chart (switch between them using tabs). -### EXPLAIN and query execution plan {#explain} + + +Execution results are displayed in the **Results** panel as a table, JSON, or diagram (switch via tabs). + +### EXPLAIN and execution plan {#explain} Select **Explain YQL Query** in the editor context menu or in the command palette (`Ctrl+Shift+P`) to get the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md). The plugin displays the plan's operation tree in text form. -### Session Manager {#session-manager} + +### Session manager {#session-manager} The **Sessions** panel (Activity Bar → YDB) displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Toggle Hide Idle** button hides sessions without an active query. -### Cluster Dashboard {#cluster-dashboard} + +### Cluster dashboard {#cluster-dashboard} The **Database Load** panel (Activity Bar → YDB) displays real-time cluster load (updated every 10 seconds): -- CPU usage (percentage and number of cores). -- Memory usage (percentage and volume). +- CPU load (% and number of cores). +- Memory usage (% and amount). - Network traffic. + {% note warning %} -The dashboard is available only when working with self-hosted installations of {{ ydb-short-name }} that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. +The dashboard is only available when working with self-hosted {{ ydb-short-name }} installations that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, the Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. {% endnote %} -### Streaming Queries {#streaming-queries} -In the navigator, expand the **Streaming Queries** section. For each query, the following is available: +### Streaming queries {#streaming-queries} + +In the navigator, expand the **Streaming Queries** section. For each query, the following are available: -- Viewing the original YQL (**View Source**). -- Starting and stopping (**Start** / **Stop**). -### SQL Dialect Converter {#convert-dialect} +- View the source YQL (**View Source**). +- Start and stop (**Start** / **Stop**). -The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the **Convert Dialect** panel on the Activity Bar. +### SQL dialect converter {#convert-dialect} + +The plugin allows converting an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the **Convert Dialect** panel on the Activity Bar. To convert a query: -1. Select the source SQL dialect in the **Source Dialect** dropdown list. -1. Paste the source SQL code into the **Input SQL** field. -1. Click **Convert**. The result will appear in the lower field. -1. Copy the result for use in the YQL editor. +1. In the **Source Dialect** dropdown, select the source SQL dialect. +2. Paste the source SQL code into the **Input SQL** field. +3. Click **Convert**. The result appears in the lower field. +4. Copy the result to use in the YQL editor. + +For more details on how the converter works, supported dialects, and limitations, see the article [SQL dialect converter to YQL](../sql-translation/sql-dialect-converter.md). -For more information about how the converter works, the supported dialects, and the limitations, see the article [SQL Dialect Converter to YQL](../sql-dialect-converter.md). {% note warning %} -To convert, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. +To perform the conversion, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. {% endnote %} -### Viewing Permissions {#permissions} -Right-click on an object in the navigator and select **View Permissions** to view the [access rights (ACL)](../../security/authorization.md#right) assigned to this object. -### DDL Generation {#ddl} +### View permissions {#permissions} + +Right-click an object in the navigator and select **View Permissions** to view the [access rights (ACL)](../../security/authorization.md#right) assigned to this object. + +### DDL generation {#ddl} + +Right-click a table, topic, or other object in the navigator and select **Create DDL** to get the `CREATE` script of the object. -Right-click on a table, topic, or another object in the navigator and select **Create DDL** to get the `CREATE` script for the object. ### Creating objects {#create-objects} -Right-click on the corresponding folder in the navigator to create a new object: +Right-click the appropriate folder in the navigator to create a new object: - **New Row Table** / **New Column Table** — create a row or column table. - **New Topic** — create a topic. @@ -211,100 +249,130 @@ Right-click on the corresponding folder in the navigator to create a new object: - **New CSV/JSON/Parquet External Table** — create an external table. - **New Transfer** — create a transfer. - **New Streaming Query** — create a streaming query. + ## Integration with AI assistants (MCP) {#mcp} -The plugin launches a built-in [MCP server](https://modelcontextprotocol.io/) (Model Context Protocol), which allows AI assistants (Claude Code and others) to directly execute queries to {{ ydb-short-name }} databases configured in the plugin. -### Configuring the port {#mcp-port} +The plugin runs a built-in [MCP server](https://modelcontextprotocol.io/) (Model Context Protocol), which allows AI assistants (Claude Code and others) to directly execute queries against {{ ydb-short-name }} databases configured in the plugin. + +### Port configuration {#mcp-port} + +The server runs on port **3333** (localhost only) by default. You can change the port in VS Code settings: + -The server starts on port **3333** (localhost only) by default. You can change the port in VS Code settings: ```json { "ydb.mcpPort": 3333 } ``` -If the port is already in use, the extension will display a warning and continue working without MCP. + + +If the port is already in use, the extension will show a warning and continue working without MCP. + ### Connecting Claude Code {#mcp-claude} -1. Make sure that the YDB extension is running in VS Code and at least one connection is added in the **Connections** panel (see [Creating a connection](#connection)). +1. Make sure the YDB extension is running in VS Code and at least one connection is added in the **Connections** panel (see [Creating a connection](#connection)). +2. Register the MCP server in Claude Code globally for the current user: + + + ```bash + claude mcp add --scope user --transport sse ydb http://localhost:3333/sse + ``` -1. Register the MCP server in Claude Code globally for the current user: - ```bash - claude mcp add --scope user --transport sse ydb http://localhost:3333/sse - ``` -The `--scope user` flag saves the configuration globally — the server will be accessible from any directory where Claude Code is launched. Without this flag, the `local` scope is used (by default), and registration will have to be repeated for each project. -1. Check the connection: - ```bash - claude mcp list - ``` + The `--scope user` flag saves the configuration globally — the server will be accessible from any directory where Claude Code is launched. Without this flag, the [scope `local`](https://docs.claude.com/en/docs/claude-code/mcp#mcp-installation-scopes) (default) is used, and registration will have to be repeated for each project. +3. Check the connection: + + + ```bash + claude mcp list + ``` + ### Available MCP tools {#mcp-tools} + | Tool | Parameters | Description | -|------------|-----------|----------| +| --- | --- | --- | | `ydb_list_connections` | — | List of all connections configured in the plugin | | `ydb_query` | `connection`, `sql` | Execute a YQL query | -| `ydb_describe_table` | `connection`, `path` | Get the table schema (columns, primary key) | +| `ydb_describe_table` | `connection`, `path` | Get table schema (columns, primary key) | | `ydb_list_directory` | `connection`, `path?` | List of objects in the database directory | | `ydb_list_all` | `connection`, `path?`, `limit?`, `offset?` | Recursive list of all objects | | `ydb_yql_help` | `query`, `connection?` | Search YQL documentation (requires [RAG](#rag) enabled) | -The `connection` parameter is the name of the connection as it appears in the **Connections** panel. -## YQL documentation search (RAG) {#rag} -The plugin supports semantic and keyword search of YQL documentation using the built-in RAG index (Retrieval-Augmented Generation). This allows AI assistants (via the `ydb_yql_help` tool) to find relevant fragments of the syntax reference when writing queries. +The `connection` parameter is the connection name as it appears in the **Connections** panel. + +## Search YQL documentation (RAG) {#rag} + +The plugin supports semantic and keyword search over YQL documentation using a built-in RAG index (Retrieval-Augmented Generation). This allows AI assistants (via the `ydb_yql_help` tool) to find relevant fragments of the syntax reference when writing queries. + ### Enabling RAG {#rag-enable} 1. When creating or editing a connection, set the **Use RAG** flag. -1. The plugin will automatically detect the {{ ydb-short-name }} version and load the corresponding index from the cloud on the first connection. +2. The plugin will automatically detect the {{ ydb-short-name }} version and download the corresponding index from the cloud on first connection. + ### Search modes {#rag-modes} - **Keyword search** — works without additional dependencies, used by default. -- **Semantic search (vector)** — requires locally running [Ollama](https://ollama.com) with the embedding model [`nomic-embed-text`](https://ollama.com/library/nomic-embed-text). See [Installing Ollama and the embedding model](#install-ollama). +- **Semantic search (vector)** — requires a locally running [Ollama](https://ollama.com) with the [`nomic-embed-text`](https://ollama.com/library/nomic-embed-text) embedding model. See [Installing Ollama and the embedding model](#install-ollama). + +The RAG status (Running / Not running) and Ollama status are displayed directly in the connection form. -The RAG status (Running / Not running) and the Ollama status are displayed directly in the connection form. ### Installing Ollama and the embedding model {#install-ollama} -[Ollama](https://ollama.com) is a local server for running language models and embedding models. The plugin accesses it via HTTP to convert documentation texts and search queries into vectors for semantic search. +[Ollama](https://ollama.com) is a local server for running language models and embedding models. The plugin communicates with it over HTTP to convert documentation texts and search queries into vectors for semantic search. 1. Install Ollama following the instructions on the [official download page](https://ollama.com/download) (macOS, Windows, and Linux are supported). +2. Make sure the Ollama service is running and accessible at `http://localhost:11434`: + + + ```bash + curl http://localhost:11434/api/tags + ``` + + + A JSON response with a list of models means the service is running. +3. Download the `nomic-embed-text` embedding model (~270 MB): -1. Make sure that the Ollama service is running and accessible at `http://localhost:11434`: - ```bash - curl http://localhost:11434/api/tags - ``` -A response in JSON format with a list of models means that the service is working. -1. Download the `nomic-embed-text` embedding model (~270 MB): - ```bash - ollama pull nomic-embed-text - ``` -1. Make sure the model is available: - ```bash - ollama list - ``` -The output should include the line `nomic-embed-text`. + ```bash + ollama pull nomic-embed-text + ``` + +4. Make sure the model is available: + + + ```bash + ollama list + ``` + + + The output should contain the line `nomic-embed-text`. +5. Open the connection form in the plugin and make sure the Ollama status is displayed as **Running**. If the status is **Not running**, check that the Ollama service is running and accessible at `http://localhost:11434`. -1. Open the connection form in the plugin and make sure that the Ollama status is displayed as **Running**. If the status is **Not running**, check that the Ollama service is running and available at `http://localhost:11434`. {% note info %} -The Ollama URL (`http://localhost:11434`) and model name (`nomic-embed-text`) are already set in the plugin by default — you only need to override the `ydb.ragOllamaUrl` and `ydb.ragOllamaModel` settings in case of a non-standard installation. +The Ollama URL (`http://localhost:11434`) and model name (`nomic-embed-text`) are already set by default in the plugin — you only need to override the `ydb.ragOllamaUrl` and `ydb.ragOllamaModel` settings for a non-standard installation. -It is not possible to use another embedding model: the indices published by the plugin in the cloud are based on `nomic-embed-text`, and if you change the model, the vectors of the query and documentation will become incompatible. +You cannot use a different embedding model: the indexes published by the plugin in the cloud are built specifically on `nomic-embed-text`, and changing the model will make the query and documentation vectors incompatible. {% endnote %} + ## Plugin update {#updates} The update method depends on how the plugin was installed. -### Updating from the VS Code Marketplace {#update-marketplace} + +### Updating from VS Code Marketplace {#update-marketplace} If the plugin is installed from the [Marketplace](#install-marketplace), VS Code updates it automatically. To update the plugin manually: 1. Open the Extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). -1. Enter `YDB for VS Code` in the search bar and open the installed extension. -1. If a new version is available, the **Update** button will appear — click it. +2. In the search bar, enter `YDB for VS Code` and open the installed extension. +3. If a new version is available, the **Update** button will appear — click it. + ### Updating from a VSIX file {#update-vsix} If the plugin is installed from a [VSIX file](#install-vsix), automatic updates are not available — VS Code does not track updates for manually installed extensions. -1. Download a new `.vsix` file from the [GitHub Releases](https://github.com/ydb-platform/ydb-vscode-plugin/releases) page. -1. Install it over the old version using the same method as during the initial installation — VS Code will automatically replace the previous version. +1. Download the new `.vsix` file from the [GitHub Releases](https://github.com/ydb-platform/ydb-vscode-plugin/releases) page. +2. Install it over the old version using the same method as the initial installation — VS Code will automatically replace the previous version. diff --git a/ydb/docs/en/core/integrations/index.md b/ydb/docs/en/core/integrations/index.md index 3e253291b7b..ecb78b24a18 100644 --- a/ydb/docs/en/core/integrations/index.md +++ b/ydb/docs/en/core/integrations/index.md @@ -4,7 +4,7 @@ This section provides basic information about {{ ydb-name }} integrations with t {% note info %} -In addition to its own native protocol, {{ ydb-name }} has a compatibility layer that allows external systems to connect to databases via [PostgreSQL](../postgresql/intro.md) or [Apache Kafka](../reference/kafka-api/index.md) network protocols. Thanks to the compatibility layer, many tools designed to work with these systems can also interact with {{ ydb-name }}. The compatibility level of each specific application should be clarified separately. +In addition to its own native protocol, {{ ydb-name }} has a compatibility layer that allows external systems to connect to databases via [PostgreSQL](../postgresql/intro.md) or [Apache Kafka](../reference/kafka-api/index.md) network protocols. Thanks to the compatibility layer, many tools designed to work with these systems can also interact with {{ ydb-name }}. The compatibility level of each specific application must be clarified separately. {% endnote %} @@ -44,10 +44,6 @@ In addition to its own native protocol, {{ ydb-name }} has a compatibility layer {% include notitle [Contents](orm/_includes/toc-table.md) %} -## Spring {#spring} - -{% include notitle [Contents](spring/_includes/toc-table.md) %} - ## Vector search {#vectorsearch} {% include notitle [Contents](vectorsearch/_includes/toc-table.md) %} diff --git a/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md b/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md index b7fd39554e5..6dddf148bd5 100644 --- a/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md +++ b/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md @@ -1,6 +1,6 @@ # SQL dialect conversion -| Tool | Instruction | +| Tool | Instructions | | --- | --- | -| Plugin [ydb-sqlglot-plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) for [SQLGlot](https://github.com/tobymao/sqlglot) | Instructions | -| Converter of SQL dialects to YQL | Instructions | +| Plugin [ydb-sqlglot-plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) for [SQLGlot](https://github.com/tobymao/sqlglot) | [Instructions](../sqlglot.md) | +| SQL dialect converter to YQL | [Instructions](../sql-dialect-converter.md) | diff --git a/ydb/docs/en/core/integrations/sql-translation/index.md b/ydb/docs/en/core/integrations/sql-translation/index.md new file mode 100644 index 00000000000..27ac349138e --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/index.md @@ -0,0 +1,3 @@ +# SQL dialect conversion + +{% include notitle [Contents](_includes/toc-table.md) %} diff --git a/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md b/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md new file mode 100644 index 00000000000..ee54fa46f3d --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md @@ -0,0 +1,78 @@ +# SQL dialect converter to YQL + +The SQL dialect converter is a service that converts SQL queries written in other dialects (PostgreSQL, MySQL, ClickHouse, and others) into [YQL](../../yql/reference/index.md) syntax. The service is designed to simplify migration of existing queries to {{ ydb-name }} and lower the entry barrier for developers already familiar with other DBMSs. + +The converter is available as a standalone web application at [ydb-dialect-converter.website.yandexcloud.net](https://ydb-dialect-converter.website.yandexcloud.net/), and is also built into the [DBeaver plugin for {{ ydb-short-name }}](../gui/dbeaver-plugin.md#convert-dialect) and the [VS Code plugin for {{ ydb-short-name }}](../gui/vscode-plugin.md). The user pastes a query in the source dialect, selects the dialect from the list, and gets a version of the same query in YQL. + +## How to use {#usage} + +1. Open the [converter web interface](https://ydb-dialect-converter.website.yandexcloud.net/). +2. Select the source SQL dialect from the drop-down list. +3. Paste the query into the **Input SQL query** field. If necessary, use the ready-made examples (`CTE Example`, `Create Table Example`, and others). +4. Click **Convert** — the YQL result will appear in the **Conversion result (YDB)** field. + +## How it works {#principles} + +The conversion is performed in three stages: + +1. **Parsing the source SQL.** The source query is parsed by the specific dialect parser into an abstract syntax tree (AST) — a structure that describes the query's meaning independently of syntax. +2. **AST transformation.** Constructs specific to the source dialect are replaced with their YQL equivalents. For example, type casting functions, date and string handling, `LIMIT`/`OFFSET` operators, and `WITH` constructs are adapted to YQL rules. +3. **YQL generation.** A textual representation of the query in YQL is generated from the modified AST. + +The stages are implemented using the [SQLGlot](https://github.com/tobymao/sqlglot) library and the [{{ ydb-short-name }} plugin for it](sqlglot.md), which adds a separate {{ ydb-short-name }} dialect to SQLGlot. SQLGlot is an open-source SQL parser and transpiler that supports over twenty dialects. It serves both as the parser for the source query and as the YQL generator. + +If you need not a one-time conversion in the GUI but to embed the transformation into your own code or data processing pipeline, use the plugin directly — see the [SQLGlot](sqlglot.md) article. + +{% note warning %} + +The source query is sent to an external HTTPS service. Do not send queries containing confidential data (personal data, trade secrets, identifiers of real objects in production) to the converter. + +{% endnote %} + +## Supported dialects {#dialects} + +The list of source dialects is determined by the set supported by SQLGlot. Among the main ones: + +- PostgreSQL +- MySQL +- ClickHouse +- Microsoft SQL Server (T-SQL) +- Oracle +- Snowflake +- BigQuery +- Presto / Trino +- Spark SQL / Databricks +- SQLite + +The current full list of dialects can be obtained via the converter API or viewed in the [SQLGlot source code](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects). The target dialect is always YQL. + +## Supported constructs {#supported-constructs} + +The converter covers typical constructs of analytical and OLTP queries: + +- SELECT operators: `SELECT`, `JOIN` (all types), `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, `LIMIT`, `OFFSET`. +- Window functions (`OVER`, `PARTITION BY`, `ROWS BETWEEN`). +- Subqueries and CTE (`WITH`). +- Type casting (`CAST`, `::`), arithmetic and logical operations. +- Functions for strings, numbers, dates, and time — mapped to the nearest equivalent in YQL. +- Aggregate functions (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, and others). +- DML operations: `INSERT`, `UPDATE`, `DELETE` (taking into account [transaction features of {{ ydb-short-name }}](../../concepts/transactions.md)). + +## Limitations {#limitations} + +Full automatic conversion is not always possible, since YQL and other SQL dialects differ in data models and operation semantics. In particular: + +- **Dialect-specific functions.** Functions that have no direct equivalent in YQL (for example, PostgreSQL arrays, `JSON_EXTRACT` MySQL with special path syntax) are translated to approximate equivalents or left for manual refinement. +- **Stored procedures and triggers.** Not supported, as {{ ydb-short-name }} uses a different model for executing code in the database. +- **Dialect-specific DDL.** `CREATE TABLE` for complex types may require manual adjustment considering [columnar and row-based tables](../../concepts/datamodel/table.md) in {{ ydb-short-name }}. +- **Optimizer hints.** Ignored: the {{ ydb-short-name }} optimizer has its own control mechanisms. + +The conversion result should be considered a draft: complex queries require manual review and adaptation before execution. + +## See also {#see-also} + +- [SQL dialect converter web interface](https://ydb-dialect-converter.website.yandexcloud.net/) +- [{#T}](sqlglot.md) +- [{#T}](../gui/dbeaver-plugin.md) +- [YQL Reference](../../yql/reference/index.md) +- [SQLGlot](https://github.com/tobymao/sqlglot) diff --git a/ydb/docs/en/core/integrations/sql-translation/sqlglot.md b/ydb/docs/en/core/integrations/sql-translation/sqlglot.md new file mode 100644 index 00000000000..8772c2cff6d --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/sqlglot.md @@ -0,0 +1,173 @@ +# SQLGlot + +[SQLGlot](https://github.com/tobymao/sqlglot) is a pure Python SQL parser, transpiler, optimizer, and formatter that supports over twenty dialects (PostgreSQL, MySQL, ClickHouse, BigQuery, Snowflake, Spark SQL, and others). SQLGlot parses a query into an abstract syntax tree (AST), which can be analyzed and transformed programmatically, and then generate back to SQL in any of the supported dialects. + +[ydb-sqlglot-plugin plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) adds the {{ ydb-short-name }} dialect to SQLGlot. After installation, SQLGlot can both parse [YQL](../../yql/reference/index.md) queries and generate YQL from queries written in other dialects. The conversion is bidirectional: any supported dialect ↔ YQL. + +Unlike the ready-made [SQL dialect converter](sql-dialect-converter.md), which operates as an external service and is built into graphical tools, the plugin is a library that you integrate into your own Python code. This provides two important advantages: + +- **Local processing.** Queries never leave your machine — no data is sent to an external HTTPS service. Suitable for working with confidential queries. +- **Programmatic access to AST.** In addition to transpilation, query parsing, column lineage analysis, optimization, and formatting are available — everything SQLGlot can do. + +## Features {#features} + +- Transpilation of queries from other dialects to YQL and back. +- Parsing a YQL query into AST for programmatic analysis and modification. +- Column lineage analysis (column lineage) — tracking which tables and columns each resulting column originates from. +- Query optimization and formatting using SQLGlot. +- Support for YQL-specific constructs: named expressions (`$variable`), module functions (`DateTime::GetYear()`), `FLATTEN`, lambda expressions, container types, and others. + +{% note info %} + +The full list of supported features and current limitations is provided in the [plugin repository README](https://github.com/ydb-platform/ydb-sqlglot-plugin). + +{% endnote %} + +## Installation {#install} + +The plugin is published on PyPI under the name `ydb-sqlglot-plugin`: + + +```bash +pip install ydb-sqlglot-plugin +``` + + +Requirements: + +- Python 3.9 or newer. +- SQLGlot version 28.6.0 or newer (installed automatically as a dependency). + +The plugin is distributed under the Apache 2.0 license. + +## Quick start {#quickstart} + +After installation, the {{ ydb-short-name }} dialect is available in SQLGlot automatically — no additional imports are needed. It has two equivalent names, `ydb` and `yql`; either of them can be specified in the `read` and `write` arguments. + +Converting a query from MySQL to YQL: + + +```python +import sqlglot + +result = sqlglot.transpile( + "SELECT * FROM users WHERE id = 1", + read="mysql", + write="ydb", +)[0] + +print(result) +# SELECT * FROM `users` WHERE id = 1 +``` + + +Reverse transformation — from YQL to PostgreSQL. The YQL named expression (`$t = (...)`) turns into a CTE (`WITH ... AS`): + + +```python +import sqlglot + +result = sqlglot.transpile( + "$t = (SELECT id FROM users); SELECT * FROM $t AS t", + read="ydb", + write="postgres", +)[0] + +print(result) +# WITH t AS (SELECT id FROM users) SELECT * FROM t AS t +``` + + +{% note tip %} + +The `sqlglot.transpile()` function returns a list of strings — one for each statement in the source text. If the query has one statement, take the first element (`[0]`). + +{% endnote %} + +## Usage examples {#usage} + +### Migrating queries from other DBMS {#migration} + +When migrating to {{ ydb-short-name }}, the plugin automatically converts the original dialect constructs to YQL rules. For example, names with schema references (`schema.table`) are transformed into a [{{ ydb-short-name }} path](../../concepts/datamodel/dir.md) with backticks: + + +```python +import sqlglot + +print(sqlglot.transpile("SELECT * FROM analytics.events", read="postgres", write="ydb")[0]) +# SELECT * FROM `analytics/events` +``` + + +Correlated subqueries not directly supported in {{ ydb-short-name }} are, where possible, rewritten into `JOIN`, and `WITH` constructs into named YQL expressions. + +### Parsing a query into AST {#parse} + +If you need not only transpilation but also query structure analysis, parse it into a tree using `sqlglot.parse_one()`: + + +```python +import sqlglot +from sqlglot import exp + +tree = sqlglot.parse_one("SELECT id, name FROM `users` WHERE age > 18", dialect="ydb") + +# List all tables accessed by the query +for table in tree.find_all(exp.Table): + print(table.name) +# users + +# Generate the query back — in any dialect +print(tree.sql(dialect="clickhouse")) +``` + + +### Column lineage analysis {#lineage} + +Since YQL is parsed into the standard SQLGlot AST, the built-in SQLGlot column lineage analysis works for YQL queries. This allows you to trace which source tables and columns each resulting column comes from: + + +```python +from sqlglot.lineage import lineage + +node = lineage( + "total", + "SELECT SUM(amount) AS total FROM orders", + dialect="ydb", +) + +print(node.name) +# total +``` + + +Lineage analysis is useful when building data documentation tools, assessing the impact of schema changes (impact analysis), and auditing queries. + +## Function and type mapping {#mappings} + +The plugin contains mapping tables between standard SQL constructs and their counterparts in {{ ydb-short-name }}. + +Data types are mapped to {{ ydb-short-name }} equivalents: + +| Type in the source dialect | {{ ydb-short-name }} | +| --- | --- | +| `TINYINT` | `Int8` | +| `INT` | `Int32` | +| `BIGINT` | `Int64` | +| `VARCHAR`, `TEXT` | `Utf8` | +| `TIMESTAMP` | `Timestamp` | + +Functions are matched by value family: + +- **Date and time:** `DATE_TRUNC`, `EXTRACT`, interval operations. +- **Rows:** `CONCAT`, `UPPER`, `LOWER`, `LENGTH`, and others. +- **Collections:** `ARRAY`, `ARRAY_FILTER`, `UNNEST` → `FLATTEN BY`. +- **Conditional and numeric:** `NULLIF`, `ROUND`, `COUNT()`. +- **JSON:** `JSON_VALUE`, `JSON_QUERY` with support for related constructs. + +## See also {#see-also} + +- [{#T}](sql-dialect-converter.md) +- [ydb-sqlglot-plugin on GitHub](https://github.com/ydb-platform/ydb-sqlglot-plugin) +- [SQLGlot](https://github.com/tobymao/sqlglot) +- [YQL Reference](../../yql/reference/index.md) diff --git a/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml b/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml new file mode 100644 index 00000000000..4974c8b8eb5 --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml @@ -0,0 +1,5 @@ +items: +- name: SQLGlot + href: sqlglot.md +- name: SQL dialect converter to YQL + href: sql-dialect-converter.md diff --git a/ydb/docs/en/core/integrations/toc_i.yaml b/ydb/docs/en/core/integrations/toc_i.yaml index 10ed0517d1e..02258884ee5 100644 --- a/ydb/docs/en/core/integrations/toc_i.yaml +++ b/ydb/docs/en/core/integrations/toc_i.yaml @@ -39,15 +39,25 @@ items: include: mode: link path: orm/toc-orm.yaml +- name: Vector search + href: vectorsearch/index.md + include: + mode: link + path: vectorsearch/toc-vectorsearch.yaml +- name: SQL dialect conversion + href: sql-translation/index.md + include: + mode: link + path: sql-translation/toc-sql-translation.yaml - name: Spring href: spring/index.md include: mode: link path: spring/toc-spring.yaml -- name: Vector search - href: vectorsearch/index.md +- name: Spring + href: spring/index.md include: mode: link - path: vectorsearch/toc-vectorsearch.yaml + path: spring/toc-spring.yaml - name: SQL Dialect Converter to YQL href: sql-dialect-converter.md diff --git a/ydb/docs/ru/core/changelog-server.md b/ydb/docs/ru/core/changelog-server.md index be45fd905cc..db7a6d558ea 100644 --- a/ydb/docs/ru/core/changelog-server.md +++ b/ydb/docs/ru/core/changelog-server.md @@ -4,7 +4,7 @@ ### Версия 26.1.1.20 {#26-1-1-20} -Дата выхода: уточняется. +Дата выхода: 02 июля 2026. #### Функциональность diff --git a/ydb/docs/ru/core/reference/ydb-sdk/feature-parity.md b/ydb/docs/ru/core/reference/ydb-sdk/feature-parity.md index 508869de21c..9928161d522 100644 --- a/ydb/docs/ru/core/reference/ydb-sdk/feature-parity.md +++ b/ydb/docs/ru/core/reference/ydb-sdk/feature-parity.md @@ -6,12 +6,12 @@ | Функционал | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Строка подключения вида 'grpc[s]://endpoint:port/database' | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/491) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/651) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1771) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/460) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/513) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/401) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/331) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/238) | +| Строка подключения вида 'grpc[s]://endpoint:port/database' | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/491) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/651) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1771) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/460) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/513) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/401) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/331) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/238) | | Поддержка SSL/TLS (системные сертификаты) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Поддержка SSL/TLS (кастомные сертификаты) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/312) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/270) | {{ sign_plus }} | -| Configure/enable GRPC KeepAlive (keeping the connection alive in the background) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/439) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/499) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/313) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/271) | {{ sign_plus }} | -| Регулярный прогон тестов SLO на последней версии кода | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/600) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/446) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/314) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/235) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/155) | -| Шаблоны Issue в GitHub | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/601) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/394) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/272) | {{ sign_plus }} | +| Поддержка SSL/TLS (кастомные сертификаты) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/312) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/270) | {{ sign_plus }} | +| Configure/enable GRPC KeepAlive (keeping the connection alive in the background) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/439) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/499) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/313) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/271) | {{ sign_plus }} | +| Регулярный прогон тестов SLO на последней версии кода | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/600) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/446) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/314) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/235) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/155) | +| Шаблоны Issue в GitHub | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/601) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/394) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/272) | {{ sign_plus }} | {wide-content} @@ -20,18 +20,18 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| | Инициализация балансировщика через Discovery/ListEndpoints | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Отключение клиентской балансировки (все запросы в начальный Endpoint) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/467) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/591) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/395) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/447) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/273) | {{ sign_plus }} | -| Фоновый Discovery/ListEndpoints (раз в минуту по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/156) | -| Поддержка множества IP адресов в DNS-записи начального Endpoint, часть из которых может быть недоступна (DNS балансировка) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/469) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/440) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/448) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/311) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/233) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/196) | -| Пессимизация соединений на транспортных ошибках | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/201) | -| Принудительный Discovery/ListEndpoints если пессимизировано более половины нод | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/449) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/202) | -| Автоматическое определение ближайшего ДЦ / зоны доступности по TCP-пингам | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/592) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/396) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/450) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/315) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/236) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/203) | -| Aвтоматическое определение ближайшего ДЦ / зоны доступности по ответу Discovery/ListEndpoints | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1746) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/397) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/451) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/316) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/237) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/204) | -| Равномерный случайный выбор нод (по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/205) | -| Балансировка среди всех нод всех ДЦ (по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/206) | -| Балансировка среди всех нод конкретного ДЦ / зоны доступности (например, “a”, “vla”) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/441) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/317) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/238) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/207) | -| Балансировка среди всех нод всех локального ДЦ | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/442) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/318) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/239) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/208) | -| Все запросы отправлять через соединение к начальному endpoint вместо клиентской балансировки (для использования в serverless) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/513) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/668) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1796) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/488) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/528) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/420) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/345) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/251) | +| Отключение клиентской балансировки (все запросы в начальный Endpoint) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/467) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/591) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/395) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/447) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/273) | {{ sign_plus }} | +| Фоновый Discovery/ListEndpoints (раз в минуту по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/156) | +| Поддержка множества IP адресов в DNS-записи начального Endpoint, часть из которых может быть недоступна (DNS балансировка) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/469) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/440) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/448) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/311) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/233) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/196) | +| Пессимизация соединений на транспортных ошибках | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/201) | +| Принудительный Discovery/ListEndpoints если пессимизировано более половины нод | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/449) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/202) | +| Автоматическое определение ближайшего ДЦ / зоны доступности по TCP-пингам | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/592) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/396) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/450) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/315) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/236) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/203) | +| Aвтоматическое определение ближайшего ДЦ / зоны доступности по ответу Discovery/ListEndpoints | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1746) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/397) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/451) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/316) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/237) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/204) | +| Равномерный случайный выбор нод (по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/205) | +| Балансировка среди всех нод всех ДЦ (по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/206) | +| Балансировка среди всех нод конкретного ДЦ / зоны доступности (например, “a”, “vla”) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/441) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/317) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/238) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/207) | +| Балансировка среди всех нод всех локального ДЦ | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/442) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/318) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/239) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/208) | +| Все запросы отправлять через соединение к начальному endpoint вместо клиентской балансировки (для использования в serverless) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/513) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/668) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1796) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/488) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/528) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/420) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/345) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/251) | {wide-content} @@ -40,9 +40,9 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| | Anonymous (по умолчанию) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Static (user - password) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/454) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/319) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/157) | +| Static (user - password) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/454) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/319) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/157) | | Token: IAM, Access token | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Service account (Yandex.Cloud specific) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/240) | {{ sign_plus }} | +| Service account (Yandex.Cloud specific) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/240) | {{ sign_plus }} | | Metadata (Yandex.Cloud specific) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {wide-content} @@ -54,20 +54,20 @@ | Int/Uint(8,16,32,64) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | | Float, Double | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | | Bool | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| String, Bytes | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/198) | +| String, Bytes | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/198) | | Utf8, Text | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| NULL, Optional, Void | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/309) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/199) | +| NULL, Optional, Void | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/309) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/199) | | Struct | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| List | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/310) | {{ sign_plus }} | {{ sign_plus }} | -| Set | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/470) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/602) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1751) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/443) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/445) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/328) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/234) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/197) | -| Tuple | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/220) | -| Variant\<Struct>,Variant\<Tuple\> | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/329) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/246) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/159) | -| Date, DateTime, Timestamp, Interval | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/160) | -| TzDate, TzDateTime, TzTimestamp | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/330) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/247) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/161) | -| DyNumber | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/331) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/248) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/162) | -| Decimal (120 bits) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/249) | {{ sign_plus }} | -| Json, JsonDocument, Yson | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/200) | -| Широкие типы дат (Date32, Datetime64, Timestamp64, Interval64) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/236) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/649) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1769) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/458) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/511) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/399) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/329) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/236) | +| List | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/310) | {{ sign_plus }} | {{ sign_plus }} | +| Set | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/470) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/602) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1751) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/443) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/445) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/328) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/234) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/197) | +| Tuple | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/220) | +| Variant\<Struct>,Variant\<Tuple\> | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/329) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/246) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/159) | +| Date, DateTime, Timestamp, Interval | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/160) | +| TzDate, TzDateTime, TzTimestamp | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/330) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/247) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/161) | +| DyNumber | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/331) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/248) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/162) | +| Decimal (120 bits) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/249) | {{ sign_plus }} | +| Json, JsonDocument, Yson | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/200) | +| Широкие типы дат (Date32, Datetime64, Timestamp64, Interval64) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/236) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/649) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1769) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/458) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/511) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/399) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/329) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/236) | {wide-content} @@ -77,8 +77,8 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| ListEndpoints | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/502) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/662) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1782) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/471) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/524) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/412) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/342) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/249) | -| WhoAmI | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/503) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/663) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1783) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/472) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/525) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/413) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/343) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/250) | +| ListEndpoints | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/502) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/662) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1782) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/471) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/524) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/412) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/342) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/249) | +| WhoAmI | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/503) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/663) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1783) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/472) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/525) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/413) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/343) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/250) | {wide-content} @@ -86,25 +86,25 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Пул сессий | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/492) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/652) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1772) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/461) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/514) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/402) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/332) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/239) | -| Ограничение количества одновременных сессий на клиенте | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/493) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/653) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1773) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/462) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/515) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/403) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/333) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/240) | -| Несгораемый остаток сессий в пуле | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/494) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/654) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1774) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/463) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/516) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/404) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/334) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/241) | -| Прогрев пула до указанного значения количества сессий при создании пула | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/495) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/655) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1775) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/464) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/517) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/405) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/335) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/242) | -| Фоновый attach-стрим используется для очистки пула от плохих сессий | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/496) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/656) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1776) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/465) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/518) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/406) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/336) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/243) | -| Ретраер на пуле сессий (объект для повторов - сессия) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/497) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/657) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1777) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/466) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/519) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/407) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/337) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/244) | -| Ретраер на пуле сессий (объект для повторов - транзакция) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/499) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/659) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1779) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/468) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/521) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/409) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/339) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/246) | -| Поддержка серверной балансировки сессий (запрос CreateSession должен содержать в metadata-заголовке "x-ydb-client-capabilities" значение "session-balancer") | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/498) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/658) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1778) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/467) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/520) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/408) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/338) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/245) | -| Транзакции топик-таблица | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/500) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/660) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1780) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/469) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/522) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/410) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/340) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/247) | -| \* CreateSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/479) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/639) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1760) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/448) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/502) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/390) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/320) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/227) | -| \* DeleteSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/480) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/640) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1761) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/449) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/503) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/391) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/321) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/228) | -| \* AttachSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/481) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/641) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1762) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/450) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/504) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/392) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/322) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/229) | -| \* CommitTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/482) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/642) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1763) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/451) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/505) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/393) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/323) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/230) | -| \* BeginTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/483) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/643) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1764) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/506) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/394) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/324) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/231) | -| \* RollbackTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/484) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/644) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1765) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/507) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/395) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/325) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/232) | -| \* ExecuteQuery | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/485) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/645) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1766) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/454) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/508) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/396) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/326) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/233) | -| \* ExecuteScript | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/486) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/646) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1767) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/509) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/397) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/327) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/234) | -| \* FetchScriptResults | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/487) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/647) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1768) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/456) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/510) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/398) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/328) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/235) | -| Выполнение запросов без явного создания сессии | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/524) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/674) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1804) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/502) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/530) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/448) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/355) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/252) | +| Пул сессий | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/492) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/652) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1772) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/461) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/514) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/402) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/332) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/239) | +| Ограничение количества одновременных сессий на клиенте | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/493) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/653) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1773) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/462) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/515) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/403) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/333) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/240) | +| Несгораемый остаток сессий в пуле | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/494) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/654) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1774) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/463) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/516) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/404) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/334) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/241) | +| Прогрев пула до указанного значения количества сессий при создании пула | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/495) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/655) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1775) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/464) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/517) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/405) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/335) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/242) | +| Фоновый attach-стрим используется для очистки пула от плохих сессий | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/496) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/656) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1776) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/465) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/518) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/406) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/336) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/243) | +| Ретраер на пуле сессий (объект для повторов - сессия) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/497) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/657) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1777) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/466) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/519) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/407) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/337) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/244) | +| Ретраер на пуле сессий (объект для повторов - транзакция) | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/499) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/659) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1779) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/468) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/521) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/409) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/339) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/246) | +| Поддержка серверной балансировки сессий (запрос CreateSession должен содержать в metadata-заголовке "x-ydb-client-capabilities" значение "session-balancer") | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/498) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/658) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1778) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/467) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/520) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/408) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/338) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/245) | +| Транзакции топик-таблица | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/500) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/660) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1780) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/469) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/522) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/410) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/340) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/247) | +| \* CreateSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/479) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/639) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1760) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/448) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/502) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/390) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/320) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/227) | +| \* DeleteSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/480) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/640) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1761) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/449) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/503) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/391) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/321) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/228) | +| \* AttachSession | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/481) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/641) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1762) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/450) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/504) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/392) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/322) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/229) | +| \* CommitTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/482) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/642) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1763) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/451) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/505) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/393) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/323) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/230) | +| \* BeginTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/483) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/643) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1764) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/452) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/506) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/394) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/324) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/231) | +| \* RollbackTransaction | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/484) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/644) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1765) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/453) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/507) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/395) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/325) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/232) | +| \* ExecuteQuery | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/485) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/645) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1766) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/454) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/508) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/396) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/326) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/233) | +| \* ExecuteScript | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/486) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/646) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1767) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/509) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/397) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/327) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/234) | +| \* FetchScriptResults | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/487) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/647) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1768) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/456) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/510) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/398) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/328) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/235) | +| Выполнение запросов без явного создания сессии | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/524) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/674) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1804) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/502) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/530) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/448) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/355) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/252) | {wide-content} @@ -112,11 +112,11 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| MakeDirectory | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/332) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/276) | {{ sign_plus }} | -| RemoveDirectory | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/333) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/277) | {{ sign_plus }} | +| MakeDirectory | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/332) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/276) | {{ sign_plus }} | +| RemoveDirectory | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/333) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/277) | {{ sign_plus }} | | ListDirectory | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| ModifyPermissions | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/403) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/334) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/278) | {{ sign_plus }} | -| DescribePath | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/335) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/279) | {{ sign_plus }} | +| ModifyPermissions | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/403) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/334) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/278) | {{ sign_plus }} | +| DescribePath | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/335) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/279) | {{ sign_plus }} | {wide-content} @@ -125,42 +125,42 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| | Пул сессий | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Ограничение количества одновременных сессий на клиенте | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/209) | -| Несгораемый остаток сессий в пуле | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/320) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/241) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/210) | -| Прогрев пула до указанного значения количества сессий при создании пула | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/454) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1747) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/398) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/321) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/242) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/211) | -| Фоновый KeepAlive для простаивающих сессий в пуле | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/593) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1748) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/212) | -| Фоновое закрытие простаивающих сессий в пуле (лишние сессии) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/456) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/322) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/243) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/213) | -| Автоматическое выбрасывание сессии из пула при получении ошибок BAD_SESSION / SESSION_BUSY | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/214) | -| Ретраер на пуле сессий (объект для повторов - сессия) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/216) | -| Ретраер на пуле сессий (объект для повторов - транзакция на сессии) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/595) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/400) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/458) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/324) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/217) | -| Поддержка graceful shutdown сессий ("session-close" в metadata "x-ydb-server-hints" - означает надо "забыть" сессию и больше ее не использовать) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/459) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/325) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/274) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/218) | -| Поддержка серверной балансировки сессий (запрос CreateSession должен содержать в metadata-заголовке "x-ydb-client-capabilities" значение "session-balancer") | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/401) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/460) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/326) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/275) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/219) | -| Транзакции топик-таблица | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/501) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/661) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1781) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/470) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/523) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/411) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/341) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/248) | +| Ограничение количества одновременных сессий на клиенте | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/209) | +| Несгораемый остаток сессий в пуле | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/320) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/241) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/210) | +| Прогрев пула до указанного значения количества сессий при создании пула | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/454) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1747) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/398) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/321) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/242) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/211) | +| Фоновый KeepAlive для простаивающих сессий в пуле | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/593) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1748) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/212) | +| Фоновое закрытие простаивающих сессий в пуле (лишние сессии) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/456) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/322) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/243) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/213) | +| Автоматическое выбрасывание сессии из пула при получении ошибок BAD_SESSION / SESSION_BUSY | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/214) | +| Ретраер на пуле сессий (объект для повторов - сессия) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/216) | +| Ретраер на пуле сессий (объект для повторов - транзакция на сессии) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/455) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/595) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/400) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/458) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/324) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/217) | +| Поддержка graceful shutdown сессий ("session-close" в metadata "x-ydb-server-hints" - означает надо "забыть" сессию и больше ее не использовать) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/459) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/325) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/274) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/218) | +| Поддержка серверной балансировки сессий (запрос CreateSession должен содержать в metadata-заголовке "x-ydb-client-capabilities" значение "session-balancer") | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/401) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/460) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/326) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/275) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/219) | +| Транзакции топик-таблица | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/501) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/661) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1781) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/470) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/523) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/411) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/341) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/248) | | CreateSession | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | | DeleteSession | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| KeepAlive | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/163) | -| CreateTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/336) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/280) | {{ sign_plus }} | -| DropTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/337) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/281) | {{ sign_plus }} | -| AlterTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/338) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/282) | {{ sign_plus }} | -| CopyTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/462) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/339) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/283) | {{ sign_plus }} | -| CopyTables | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/404) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/463) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/340) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/284) | {{ sign_plus }} | -| DescribeTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/341) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/285) | {{ sign_plus }} | -| ExplainDataQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/464) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/342) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/286) | {{ sign_plus }} | -| PrepareDataQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/343) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/287) | {{ sign_plus }} | +| KeepAlive | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/163) | +| CreateTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/336) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/280) | {{ sign_plus }} | +| DropTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/337) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/281) | {{ sign_plus }} | +| AlterTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/338) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/282) | {{ sign_plus }} | +| CopyTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/462) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/339) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/283) | {{ sign_plus }} | +| CopyTables | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/404) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/463) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/340) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/284) | {{ sign_plus }} | +| DescribeTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/341) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/285) | {{ sign_plus }} | +| ExplainDataQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/464) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/342) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/286) | {{ sign_plus }} | +| PrepareDataQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/343) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/287) | {{ sign_plus }} | | ExecuteDataQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| \* Серверный кэш по умолчанию для всех запросов с параметрами (KeepInCache) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/457) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/288) | {{ sign_plus }} | -| \* Отдельная опция для включения/выключения серверного кэша для конкретного запроса | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/289) | {{ sign_plus }} | -| \* Truncated result как ошибка (по дефолту) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/458) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/597) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/444) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/344) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/290) | {{ sign_plus }} | -| \* Truncated result как ошибка (как опция opt-in, opt-out) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/459) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/598) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/445) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/345) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/291) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/164) | -| ExecuteSchemeQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/465) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| BeginTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/346) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/292) | {{ sign_plus }} | -| CommitTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/347) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/293) | {{ sign_plus }} | -| RollbackTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/348) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/294) | {{ sign_plus }} | -| DescribeTableOptions | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/405) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/466) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/349) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/295) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/165) | +| \* Серверный кэш по умолчанию для всех запросов с параметрами (KeepInCache) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/457) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/288) | {{ sign_plus }} | +| \* Отдельная опция для включения/выключения серверного кэша для конкретного запроса | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/289) | {{ sign_plus }} | +| \* Truncated result как ошибка (по дефолту) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/458) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/597) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/444) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/344) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/290) | {{ sign_plus }} | +| \* Truncated result как ошибка (как опция opt-in, opt-out) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/459) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/598) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/445) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/345) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/291) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/164) | +| ExecuteSchemeQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/465) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | +| BeginTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/346) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/292) | {{ sign_plus }} | +| CommitTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/347) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/293) | {{ sign_plus }} | +| RollbackTransaction | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/348) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/294) | {{ sign_plus }} | +| DescribeTableOptions | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/405) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/466) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/349) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/295) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/165) | | StreamExecuteScanQuery | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| StreamReadTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/250) | {{ sign_plus }} | -| BulkUpsert | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/350) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/251) | {{ sign_plus }} | -| Управление атрибутами таблиц | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/619) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/826) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/2158) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/651) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/608) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/637) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/410) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/259) | +| StreamReadTable | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/250) | {{ sign_plus }} | +| BulkUpsert | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/350) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/251) | {{ sign_plus }} | +| Управление атрибутами таблиц | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/619) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/826) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/2158) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/651) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/608) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/637) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/410) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/259) | {wide-content} @@ -168,8 +168,8 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Consumed Units из метаданных ответа на grpc-запрос (чтобы пользователь мог получить это) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1752) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/351) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/252) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/166) | -| Получение OperationId операции для long-polling статуса выполнения операции | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/406) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/467) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/253) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/167) | +| Consumed Units из метаданных ответа на grpc-запрос (чтобы пользователь мог получить это) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1752) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/351) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/252) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/166) | +| Получение OperationId операции для long-polling статуса выполнения операции | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/406) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/467) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/253) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/167) | {wide-content} @@ -177,9 +177,9 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| ExecuteYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/603) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/407) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/468) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/352) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/296) | {{ sign_plus }} | -| ExplainYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/604) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/408) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/469) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/353) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/297) | {{ sign_plus }} | -| StreamExecuteYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/605) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/409) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/470) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/354) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/298) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/168) | +| ExecuteYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/603) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/407) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/468) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/352) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/296) | {{ sign_plus }} | +| ExplainYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/604) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/408) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/469) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/353) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/297) | {{ sign_plus }} | +| StreamExecuteYql | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/605) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/409) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/470) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/354) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/298) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/168) | {wide-content} @@ -187,11 +187,11 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| CreateNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/606) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/410) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/471) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/355) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/299) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/169) | -| AlterNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/607) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/411) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/472) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/356) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/300) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/170) | -| DropNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/608) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/412) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/473) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/357) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/301) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/171) | -| DescribeNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/609) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/413) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/474) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/358) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/302) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/172) | -| Session (leader election, распределенный лок) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/610) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1753) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/414) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/475) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/359) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/303) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/173) | +| CreateNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/606) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/410) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/471) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/355) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/299) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/169) | +| AlterNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/607) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/411) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/472) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/356) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/300) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/170) | +| DropNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/608) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/412) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/473) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/357) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/301) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/171) | +| DescribeNode | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/609) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/413) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/474) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/358) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/302) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/172) | +| Session (leader election, распределенный лок) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/610) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1753) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/414) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/475) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/359) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/303) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/173) | {wide-content} @@ -199,13 +199,13 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| CreateTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/415) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/476) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/360) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/254) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/174) | -| DescribeTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/416) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/477) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/361) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/255) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/175) | -| AlterTopic | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/599) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/417) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/478) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/362) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/256) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/176) | -| DropTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/418) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/479) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/363) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/257) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/177) | -| StreamWrite | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/419) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/480) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/364) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/258) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/178) | -| StreamRead | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/420) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/481) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/365) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/259) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/179) | -| CommitOffset | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/490) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/650) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1770) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/459) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/512) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/400) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/330) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/237) | +| CreateTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/415) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/476) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/360) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/254) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/174) | +| DescribeTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/416) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/477) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/361) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/255) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/175) | +| AlterTopic | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/599) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/417) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/478) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/362) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/256) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/176) | +| DropTopic | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/418) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/479) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/363) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/257) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/177) | +| StreamWrite | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/419) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/480) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/364) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/258) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/178) | +| StreamRead | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/420) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/481) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/365) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/259) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/179) | +| CommitOffset | [](https://github.com/ydb-platform/ydb-cpp-sdk/issues/490) | [](https://github.com/ydb-platform/ydb-python-sdk/issues/650) | [](https://github.com/ydb-platform/ydb-go-sdk/issues/1770) | [](https://github.com/ydb-platform/ydb-java-sdk/issues/459) | [](https://github.com/ydb-platform/ydb-js-sdk/issues/512) | [](https://github.com/ydb-platform/ydb-dotnet-sdk/issues/400) | [](https://github.com/ydb-platform/ydb-rs-sdk/issues/330) | [](https://github.com/ydb-platform/ydb-php-sdk/issues/237) | {wide-content} @@ -213,12 +213,12 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| CreateResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/611) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/421) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/482) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/366) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/260) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/221) | -| AlterResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/612) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/422) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/483) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/367) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/261) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/222) | -| DropResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/613) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/423) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/484) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/368) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/262) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/223) | -| ListResources | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/614) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/424) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/485) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/369) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/263) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/224) | -| DescribeResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/615) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/425) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/486) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/370) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/264) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/225) | -| AcquireResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/616) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/426) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/487) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/371) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/265) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/226) | +| CreateResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/611) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/421) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/482) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/366) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/260) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/221) | +| AlterResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/612) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/422) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/483) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/367) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/261) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/222) | +| DropResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/613) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/423) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/484) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/368) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/262) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/223) | +| ListResources | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/614) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/424) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/485) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/369) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/263) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/224) | +| DescribeResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/615) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/425) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/486) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/370) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/264) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/225) | +| AcquireResource | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/616) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/426) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/487) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/371) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/265) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/226) | {wide-content} @@ -226,10 +226,10 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Логирование событий SDK | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/461) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/619) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Метрики SDK в Solomon / Monitoring | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/617) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/427) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/488) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/372) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/266) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/180) | -| Метрики SDK в Prometheus | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/460) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/618) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/428) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/489) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/373) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/267) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/181) | -| Трассировка событий SDK в OpenTelemetry | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/462) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/620) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1754) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/429) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/490) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/374) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/268) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/182) | +| Логирование событий SDK | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/461) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/619) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | +| Метрики SDK в Solomon / Monitoring | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/617) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/427) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/488) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/372) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/266) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/180) | +| Метрики SDK в Prometheus | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/460) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/618) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/428) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/489) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/373) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/267) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/181) | +| Трассировка событий SDK в OpenTelemetry | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/462) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/620) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1754) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/429) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/490) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/374) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/268) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/182) | {wide-content} @@ -237,24 +237,24 @@ | Feature | C\+\+ | Python | Go | Java | NodeJS | C# | Rust | PHP | |:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Аутентификация с помощью токена | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/471) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/622) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Анонимная аутентификация | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/472) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/623) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/304) | {{ sign_plus }} | -| Аутентификация с помощью переменных окружения | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/473) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/624) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/376) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/305) | {{ sign_plus }} | -| Аутентификация с помощью metadata-сервиса | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/474) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/625) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/306) | {{ sign_plus }} | -| Аутентификация с помощью файла сервисного аккаунта | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/475) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/626) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/377) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/307) | {{ sign_plus }} | -| Аутентификация с помощью логина и пароля | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/476) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/627) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/184) | -| Basic (series) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/628) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | -| Bulk Upsert | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/468) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/629) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/378) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/308) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/185) | -| Containers (Struct, Variant, List, Tuple) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/464) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/630) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/431) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/492) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/379) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/309) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/186) | -| Pagination | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/631) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/493) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/380) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/310) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/187) | -| Partition policies | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/465) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/632) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/432) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/494) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/381) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/311) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/188) | -| Read table | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/477) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/633) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/433) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/382) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/312) | {{ sign_plus }} | -| Secondary index Workaround | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/634) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1755) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/495) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/383) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/313) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/189) | -| Secondary index builtin | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/635) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1756) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/434) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/496) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/384) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/314) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/190) | -| TTL | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/636) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/435) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/497) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/385) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/315) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/191) | -| TTL Readtable | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/637) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/436) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/498) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/386) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/316) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/192) | -| URL Shortener (serverless yandex function) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/478) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/638) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/446) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/387) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/317) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/193) | -| Topic reader | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1757) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/437) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/500) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/388) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/318) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/194) | -| Topic writer | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/466) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1758) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/438) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/501) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/389) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/319) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/195) | +| Аутентификация с помощью токена | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/471) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/622) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | +| Анонимная аутентификация | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/472) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/623) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/304) | {{ sign_plus }} | +| Аутентификация с помощью переменных окружения | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/473) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/624) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/376) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/305) | {{ sign_plus }} | +| Аутентификация с помощью metadata-сервиса | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/474) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/625) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/306) | {{ sign_plus }} | +| Аутентификация с помощью файла сервисного аккаунта | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/475) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/626) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/377) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/307) | {{ sign_plus }} | +| Аутентификация с помощью логина и пароля | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/476) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/627) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/184) | +| Basic (series) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/628) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | +| Bulk Upsert | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/468) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/629) | {{ sign_plus }} | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/378) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/308) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/185) | +| Containers (Struct, Variant, List, Tuple) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/464) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/630) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/431) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/492) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/379) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/309) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/186) | +| Pagination | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/631) | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/493) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/380) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/310) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/187) | +| Partition policies | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/465) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/632) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/432) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/494) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/381) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/311) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/188) | +| Read table | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/477) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/633) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/433) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/382) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/312) | {{ sign_plus }} | +| Secondary index Workaround | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/634) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1755) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/495) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/383) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/313) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/189) | +| Secondary index builtin | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/635) | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1756) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/434) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/496) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/384) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/314) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/190) | +| TTL | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/636) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/435) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/497) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/385) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/315) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/191) | +| TTL Readtable | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/637) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/436) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/498) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/386) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/316) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/192) | +| URL Shortener (serverless yandex function) | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/478) | [](https://github\.com/ydb-platform/ydb-python-sdk/issues/638) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/446) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/387) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/317) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/193) | +| Topic reader | {{ sign_plus }} | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1757) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/437) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/500) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/388) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/318) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/194) | +| Topic writer | [](https://github\.com/ydb-platform/ydb-cpp-sdk/issues/466) | {{ sign_plus }} | [](https://github\.com/ydb-platform/ydb-go-sdk/issues/1758) | [](https://github\.com/ydb-platform/ydb-java-sdk/issues/438) | [](https://github\.com/ydb-platform/ydb-js-sdk/issues/501) | [](https://github\.com/ydb-platform/ydb-dotnet-sdk/issues/389) | [](https://github\.com/ydb-platform/ydb-rs-sdk/issues/319) | [](https://github\.com/ydb-platform/ydb-php-sdk/issues/195) | {wide-content} diff --git a/ydb/library/actors/core/event.cpp b/ydb/library/actors/core/event.cpp index c2df22e98ba..aef8da0d3b0 100644 --- a/ydb/library/actors/core/event.cpp +++ b/ydb/library/actors/core/event.cpp @@ -53,6 +53,15 @@ namespace NActors { return new TEventSerializedData; } + void IEventHandle::Preserialize() { + if (Event && !Buffer) { + TAllocChunkSerializer serializer; + Event->SerializeToArcadiaStream(&serializer); + Buffer = serializer.Release(Event->CreateSerializationInfo(false)); + Event.Reset(); + } + } + #ifndef NDEBUG void IEventHandle::DoTrackNextEvent() { TrackNextEvent = true; diff --git a/ydb/library/actors/core/event.h b/ydb/library/actors/core/event.h index 36880d3e965..c599a3866a8 100644 --- a/ydb/library/actors/core/event.h +++ b/ydb/library/actors/core/event.h @@ -313,6 +313,7 @@ namespace NActors { TIntrusivePtr<TEventSerializedData> GetChainBuffer(); TIntrusivePtr<TEventSerializedData> ReleaseChainBuffer(); + void Preserialize(); ui32 GetSize() const { if (Buffer) { diff --git a/ydb/library/actors/core/event_pb.cpp b/ydb/library/actors/core/event_pb.cpp index 1ea150f9e0d..14c9c904a0d 100644 --- a/ydb/library/actors/core/event_pb.cpp +++ b/ydb/library/actors/core/event_pb.cpp @@ -80,10 +80,11 @@ namespace NActors { Y_ABORT_UNLESS(size >= 0); NSan::CheckMemIsInitialized(data, size); while (size) { - if (const size_t bytesToAppend = Min<size_t>(size, TotalSizeRemain, Buffer.size())) { + if (const size_t bytesToAppend = Min<size_t>(size, TotalSizeRemain)) { const void *produce = data; if ((reinterpret_cast<uintptr_t>(data) & 63) + bytesToAppend <= 64 && - (Chunks.empty() || data != Chunks.back().first + Chunks.back().second)) { + (Chunks.empty() || data != Chunks.back().first + Chunks.back().second) && + Buffer.size() >= bytesToAppend) { memcpy(Buffer.data(), data, bytesToAppend); produce = Buffer.data(); Buffer = Buffer.SubSpan(bytesToAppend, Max<size_t>()); @@ -188,11 +189,12 @@ namespace NActors { return Chunks; } - void TCoroutineChunkSerializer::SetSerializingEvent(const IEventBase *event) { + void TCoroutineChunkSerializer::SetSerializingEvent(const IEventBase *event, bool withCachedSizes) { Y_ABORT_UNLESS(Event == nullptr); Event = event; TotalSerializedDataSize = 0; AbortFlag = false; + WithCachedSizes = withCachedSizes; } void TCoroutineChunkSerializer::Abort() { diff --git a/ydb/library/actors/core/event_pb.h b/ydb/library/actors/core/event_pb.h index b46f1f7d307..7b2d51d99dd 100644 --- a/ydb/library/actors/core/event_pb.h +++ b/ydb/library/actors/core/event_pb.h @@ -89,7 +89,7 @@ namespace NActors { TCoroutineChunkSerializer(); ~TCoroutineChunkSerializer(); - void SetSerializingEvent(const IEventBase *event); + void SetSerializingEvent(const IEventBase *event, bool withCachedSizes); void DiscardEvent() { Event = nullptr; }; void Abort(); std::span<TChunk> FeedBuf(void* data, size_t size); @@ -116,6 +116,9 @@ namespace NActors { bool WriteString(const TString *s) override; NProtoBuf::io::CodedOutputStream *GetCodedOutputStream() override { + if (!WithCachedSizes) { + return nullptr; + } if (!CodedOutputStream) { CodedOutputStream.reset(new NProtoBuf::io::CodedOutputStream(this, false)); } @@ -140,6 +143,7 @@ namespace NActors { bool AbortFlag; bool SerializationSuccess; bool Finished = false; + bool WithCachedSizes = false; std::unique_ptr<NProtoBuf::io::CodedOutputStream> CodedOutputStream; }; diff --git a/ydb/library/actors/core/servicemap.h b/ydb/library/actors/core/servicemap.h index d72e50cae5f..209d8cf13e3 100644 --- a/ydb/library/actors/core/servicemap.h +++ b/ydb/library/actors/core/servicemap.h @@ -2,23 +2,60 @@ #include "defs.h" +#include <util/generic/algorithm.h> + +#include <array> +#include <atomic> +#include <cstring> +#include <type_traits> + namespace NActors { // wait-free one writer multi reader hash-tree for service mapping purposes // on fast updates on same key - could lead to false-negatives, we don't care as such cases are broken from service-map app logic template <typename TKey, typename TValue, typename THash, ui64 BaseSize = 256 * 1024, ui64 ExtCount = 4, ui64 ExtBranching = 4> class TServiceMap : TNonCopyable { + // CounterIn/CounterOut validate the whole snapshot, but every part still + // has to be accessed atomically to make a discarded torn read well-defined. + template <typename T> + class TAtomicStorage : TNonCopyable { + static_assert(std::is_trivially_copyable_v<T>); + static_assert(sizeof(T) % sizeof(ui64) == 0); + + static constexpr size_t PartCount = sizeof(T) / sizeof(ui64); + std::array<std::atomic<ui64>, PartCount> Parts{}; + + public: + T Load() const noexcept { + std::array<ui64, PartCount> parts; + for (size_t i = 0; i != PartCount; ++i) { + parts[i] = Parts[i].load(std::memory_order_relaxed); + } + + T value; + std::memcpy(&value, parts.data(), sizeof(value)); + return value; + } + + void Store(const T& value) noexcept { + std::array<ui64, PartCount> parts; + std::memcpy(parts.data(), &value, sizeof(value)); + + for (size_t i = 0; i != PartCount; ++i) { + Parts[i].store(parts[i], std::memory_order_relaxed); + } + } + }; + struct TEntry : TNonCopyable { ui32 CounterIn; ui32 CounterOut; - TKey Key; - TValue Value; + TAtomicStorage<TKey> Key; + TAtomicStorage<TValue> Value; TEntry() : CounterIn(0) , CounterOut(0) - , Key() - , Value() { } }; @@ -39,8 +76,8 @@ namespace NActors { for (ui32 i = 0; i != ExtCount; ++i) { const TEntry& entry = branch->Entries[i]; const ui32 counterIn = AtomicLoad(&entry.CounterIn); - if (counterIn != 0 && entry.Key == key) { - ret = entry.Value; + if (counterIn != 0 && entry.Key.Load() == key) { + ret = entry.Value.Load(); const ui32 counterOut = AtomicLoad(&entry.CounterOut); if (counterOut == counterIn) return true; @@ -64,7 +101,7 @@ namespace NActors { return; } } else { - if (entry.Key == key) { + if (entry.Key.Load() == key) { oldEntry = &entry; if (!zeroEntry || *zeroEntry) return; @@ -131,12 +168,12 @@ namespace NActors { // now we got both entries, first - push new one const ui32 counter = AtomicUi32Increment(&Counter); AtomicStore(&zeroEntry->CounterOut, counter); - zeroEntry->Key = key; - zeroEntry->Value = value; + zeroEntry->Key.Store(key); + zeroEntry->Value.Store(value); AtomicStore(&zeroEntry->CounterIn, counter); if (oldEntry != nullptr) { - const TValue ret = oldEntry->Value; + const TValue ret = oldEntry->Value.Load(); AtomicStore<ui32>(&oldEntry->CounterOut, 0); AtomicStore<ui32>(&oldEntry->CounterIn, 0); return ret; diff --git a/ydb/library/actors/core/ut/event_pb_payload_ut.cpp b/ydb/library/actors/core/ut/event_pb_payload_ut.cpp index aa3d37fe57b..9493e5758b5 100644 --- a/ydb/library/actors/core/ut/event_pb_payload_ut.cpp +++ b/ydb/library/actors/core/ut/event_pb_payload_ut.cpp @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(TEventProtoWithPayload) { TString chunkerRes; TCoroutineChunkSerializer chunker; - chunker.SetSerializingEvent(&msg); + chunker.SetSerializingEvent(&msg, true); while (!chunker.IsComplete()) { char buffer[4096]; auto range = chunker.FeedBuf(buffer, sizeof(buffer)); @@ -96,7 +96,7 @@ Y_UNIT_TEST_SUITE(TEventProtoWithPayload) { for (size_t chunkSize = 1; chunkSize <= 32; ++chunkSize) { TString actual; TCoroutineChunkSerializer chunker; - chunker.SetSerializingEvent(&msg); + chunker.SetSerializingEvent(&msg, true); while (!chunker.IsComplete()) { TString buffer(chunkSize, '\0'); for (const auto& [data, size] : chunker.FeedBuf(buffer.begin(), buffer.size())) { diff --git a/ydb/library/actors/core/ut/event_pb_ut.cpp b/ydb/library/actors/core/ut/event_pb_ut.cpp index 1658b98c577..e8dd15a2a36 100644 --- a/ydb/library/actors/core/ut/event_pb_ut.cpp +++ b/ydb/library/actors/core/ut/event_pb_ut.cpp @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(TEventSerialization) { for (int i = 0; i < 4; ++i) { TMockEvent event; event.msg = &bm; - chunker.SetSerializingEvent(&event); + chunker.SetSerializingEvent(&event, true); char buf1[87]; TString bmChunkedSerialized; while (!chunker.IsComplete()) { diff --git a/ydb/library/actors/core/ut/servicemap_ut.cpp b/ydb/library/actors/core/ut/servicemap_ut.cpp new file mode 100644 index 00000000000..5f32b68ffef --- /dev/null +++ b/ydb/library/actors/core/ut/servicemap_ut.cpp @@ -0,0 +1,57 @@ +#include "../actorid.h" +#include "../servicemap.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <atomic> +#include <latch> +#include <thread> +#include <vector> + +namespace NActors { + +Y_UNIT_TEST_SUITE(ServiceMap) { + + Y_UNIT_TEST(ConcurrentUpdateAndFind) { + constexpr size_t readerCount = 4; + constexpr size_t updateCount = 1'000'000; + + TServiceMap<TActorId, TActorId, TActorId::THash, 1, 2, 2> map; + const TActorId serviceId(1, TStringBuf("service")); + const TActorId actorId1(1, 0, 1, 0); + const TActorId actorId2(1, 0, 2, 0); + map.Update(serviceId, actorId1); + + std::atomic<bool> stop = false; + std::atomic<size_t> invalidReads = 0; + std::latch start(readerCount + 1); + std::vector<std::thread> readers; + readers.reserve(readerCount); + + for (size_t i = 0; i != readerCount; ++i) { + readers.emplace_back([&] { + start.arrive_and_wait(); + while (!stop.load(std::memory_order_relaxed)) { + const TActorId actorId = map.Find(serviceId); + if (actorId && actorId != actorId1 && actorId != actorId2) { + invalidReads.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + + start.arrive_and_wait(); + for (size_t i = 0; i != updateCount; ++i) { + map.Update(serviceId, i % 2 ? actorId1 : actorId2); + } + stop.store(true, std::memory_order_relaxed); + + for (std::thread& reader : readers) { + reader.join(); + } + + UNIT_ASSERT_VALUES_EQUAL(invalidReads.load(), 0); + } +} + +} // namespace NActors diff --git a/ydb/library/actors/core/ut/ya.make b/ydb/library/actors/core/ut/ya.make index 6c18ed3f37a..c68796ac696 100644 --- a/ydb/library/actors/core/ut/ya.make +++ b/ydb/library/actors/core/ut/ya.make @@ -38,6 +38,7 @@ SRCS( mon_ut.cpp scheduler_actor_ut.cpp mailbox_lockfree_ut.cpp + servicemap_ut.cpp struct_log_ut.cpp ) diff --git a/ydb/library/actors/http/http_proxy.h b/ydb/library/actors/http/http_proxy.h index 3e7e715e29c..e0d9eab3e69 100644 --- a/ydb/library/actors/http/http_proxy.h +++ b/ydb/library/actors/http/http_proxy.h @@ -78,6 +78,7 @@ struct TEvHttpProxy { TString PrivateKeyFile; TString SslCertificatePem; TString CaFile; + bool ClientCertificateRequired = false; std::vector<TString> CompressContentTypes; ui32 MaxRequestsPerSecond = 0; ui32 MaxRecycledRequestsCount = DEFAULT_MAX_RECYCLED_REQUESTS_COUNT; diff --git a/ydb/library/actors/http/http_proxy_acceptor.cpp b/ydb/library/actors/http/http_proxy_acceptor.cpp index 048c1762baf..e80ca9b4ba2 100644 --- a/ydb/library/actors/http/http_proxy_acceptor.cpp +++ b/ydb/library/actors/http/http_proxy_acceptor.cpp @@ -66,9 +66,16 @@ protected: int err = 0; if (Endpoint->Secure) { if (!event->Get()->SslCertificatePem.empty()) { - Endpoint->SecureContext = TSslHelpers::CreateServerContext(event->Get()->SslCertificatePem, event->Get()->CaFile); + Endpoint->SecureContext = TSslHelpers::CreateServerContext( + event->Get()->SslCertificatePem, + event->Get()->CaFile, + event->Get()->ClientCertificateRequired); } else { - Endpoint->SecureContext = TSslHelpers::CreateServerContext(event->Get()->CertificateFile, event->Get()->PrivateKeyFile, event->Get()->CaFile); + Endpoint->SecureContext = TSslHelpers::CreateServerContext( + event->Get()->CertificateFile, + event->Get()->PrivateKeyFile, + event->Get()->CaFile, + event->Get()->ClientCertificateRequired); } if (Endpoint->SecureContext == nullptr) { err = -1; diff --git a/ydb/library/actors/http/http_proxy_ssl.h b/ydb/library/actors/http/http_proxy_ssl.h index f7dda95fb66..a7200ca8a8e 100644 --- a/ydb/library/actors/http/http_proxy_ssl.h +++ b/ydb/library/actors/http/http_proxy_ssl.h @@ -52,7 +52,33 @@ struct TSslHelpers { return CreateSslCtx(SSLv23_client_method()); } - static TSslHolder<SSL_CTX> CreateServerContext(const TString& certificate, const TString& key, const TString& caFile) { + static bool ConfigureClientCertificateVerification( + SSL_CTX* ctx, + const TString& caFile, + bool clientCertificateRequired) + { + if (caFile.empty()) { + return true; + } + if (SSL_CTX_load_verify_locations(ctx, caFile.c_str(), nullptr) != 1) { + return false; + } + int mode = SSL_VERIFY_PEER; + if (clientCertificateRequired) { + mode |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT; + } + // SSL_VERIFY_PEER option requests the client certificate during TLS handshake (mTLS), + // but doesn't fail if not provided + SSL_CTX_set_verify(ctx, mode, nullptr); + return true; + } + + static TSslHolder<SSL_CTX> CreateServerContext( + const TString& certificate, + const TString& key, + const TString& caFile, + bool clientCertificateRequired) + { TSslHolder<SSL_CTX> ctx = CreateSslCtx(SSLv23_server_method()); SSL_CTX_set_ecdh_auto(ctx.Get(), 1); int res; @@ -67,14 +93,8 @@ struct TSslHelpers { // TODO(xenoxeno): more diagnostics? return nullptr; } - if (!caFile.empty()) { - if (SSL_CTX_load_verify_locations(ctx.Get(), caFile.c_str(), nullptr) != 1) { - // TODO(yurikiselev): more diagnostics? - return nullptr; - } - // SSL_VERIFY_PEER option requests the client certificate during TLS handshake (mTLS), - // but doesn't fail if not provided - SSL_CTX_set_verify(ctx.Get(), SSL_VERIFY_PEER, nullptr); + if (!ConfigureClientCertificateVerification(ctx.Get(), caFile, clientCertificateRequired)) { + return nullptr; } return ctx; @@ -117,7 +137,11 @@ struct TSslHelpers { return true; } - static TSslHolder<SSL_CTX> CreateServerContext(const TString& pem, const TString& caFile) { + static TSslHolder<SSL_CTX> CreateServerContext( + const TString& pem, + const TString& caFile, + bool clientCertificateRequired) + { TSslHolder<SSL_CTX> ctx = CreateSslCtx(SSLv23_server_method()); SSL_CTX_set_ecdh_auto(ctx.Get(), 1); if (!LoadX509Chain(ctx, pem)) { @@ -126,14 +150,8 @@ struct TSslHelpers { if (!LoadPrivateKey(ctx, pem)) { return nullptr; } - if (!caFile.empty()) { - if (SSL_CTX_load_verify_locations(ctx.Get(), caFile.c_str(), nullptr) != 1) { - // TODO(yurikiselev): more diagnostics? - return nullptr; - } - // SSL_VERIFY_PEER option requests the client certificate during TLS handshake (mTLS), - // but doesn't fail if not provided - SSL_CTX_set_verify(ctx.Get(), SSL_VERIFY_PEER, nullptr); + if (!ConfigureClientCertificateVerification(ctx.Get(), caFile, clientCertificateRequired)) { + return nullptr; } return ctx; diff --git a/ydb/library/actors/http/http_ut.cpp b/ydb/library/actors/http/http_ut.cpp index 600f7e3b056..a9e102b5316 100644 --- a/ydb/library/actors/http/http_ut.cpp +++ b/ydb/library/actors/http/http_ut.cpp @@ -1619,7 +1619,8 @@ Y_UNIT_TEST_SUITE(THttpProxyWithMTls) { TMtlsTestSetup( const bool useRealThreads = false, const bool secureConnection = true, - TAutoPtr<TLogBackend> customLogBackend = nullptr + TAutoPtr<TLogBackend> customLogBackend = nullptr, + const bool clientCertificateRequired = false ) : ActorSystem(1, useRealThreads) { @@ -1664,6 +1665,7 @@ Y_UNIT_TEST_SUITE(THttpProxyWithMTls) { add->CertificateFile = ServerCertFile.Name(); add->PrivateKeyFile = ServerKeyFile.Name(); add->CaFile = CaCertFile.Name(); // enables mTLS + add->ClientCertificateRequired = clientCertificateRequired; } ActorSystem.Send(new NActors::IEventHandle(ProxyId, ActorSystem.AllocateEdgeActor(), add.Release()), 0, true); TAutoPtr<NActors::IEventHandle> handle; @@ -1686,9 +1688,9 @@ Y_UNIT_TEST_SUITE(THttpProxyWithMTls) { }); TAutoPtr<NActors::IEventHandle> handle; - NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request1 = setup.ActorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); - UNIT_ASSERT_EQUAL(request1->Request->URL, "/test"); - UNIT_ASSERT(!request1->Request->MTlsClientCertificate.empty()); + NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = setup.ActorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); + UNIT_ASSERT_EQUAL(request->Request->URL, "/test"); + UNIT_ASSERT(!request->Request->MTlsClientCertificate.empty()); clientThread.join(); } @@ -1734,6 +1736,43 @@ Y_UNIT_TEST_SUITE(THttpProxyWithMTls) { clientThread.join(); } + Y_UNIT_TEST(RequiredNoClientCertificate) { + TAutoPtr<TLogBackend> backend(new TSignalingLogBackend("connection closed - error in Accept")); + auto* signalingBackend = dynamic_cast<TSignalingLogBackend*>(backend.Get()); + bool expectedMessageLogged = false; + + { + TMtlsTestSetup setup(/* useRealThreads */ true, /* secureConnection */ true, std::move(backend), /* clientCertificateRequired */ true); + + const TString httpRequest = "GET /test HTTP/1.1\r\nHost: 127.0.0.1:" + ToString(setup.Port) + "\r\nConnection: close\r\n\r\n"; + std::thread clientThread([&setup, httpRequest]() { + NHttp::NTest::SendTlsRequest(setup.Port, "", "", setup.CaCertFile.Name(), httpRequest); + }); + clientThread.join(); + + signalingBackend->WaitFor(TDuration::Seconds(2)); + expectedMessageLogged = signalingBackend->Seen(); + } + + UNIT_ASSERT_C(expectedMessageLogged, "No connection error happened without client certificate"); + } + + Y_UNIT_TEST(RequiredValidClientCertificate) { + TMtlsTestSetup setup(/* useRealThreads */ false, /* secureConnection */ true, nullptr, /* clientCertificateRequired */ true); + + const TString httpRequest = "GET /test HTTP/1.1\r\nHost: 127.0.0.1:" + ToString(setup.Port) + "\r\nConnection: close\r\n\r\n"; + std::thread clientThread([&setup, httpRequest]() { + NHttp::NTest::SendTlsRequest(setup.Port, setup.ClientCertFile.Name(), setup.ClientKeyFile.Name(), setup.CaCertFile.Name(), httpRequest); + }); + + TAutoPtr<NActors::IEventHandle> handle; + NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = setup.ActorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); + UNIT_ASSERT_EQUAL(request->Request->URL, "/test"); + UNIT_ASSERT(!request->Request->MTlsClientCertificate.empty()); + + clientThread.join(); + } + Y_UNIT_TEST(NotSecureConnection) { TMtlsTestSetup setup(/* useRealThreads */ false, /* secureConnection */ false); diff --git a/ydb/library/actors/interconnect/events/events.h b/ydb/library/actors/interconnect/events/events.h index a54f3710781..e50d2edfce1 100644 --- a/ydb/library/actors/interconnect/events/events.h +++ b/ydb/library/actors/interconnect/events/events.h @@ -68,6 +68,12 @@ namespace NActors { // wake for the direct-session (v1) lock-free registration queue EvProcessDirectSessionQueue, + // internal event for v2 queue dispatching + EvRegisterCallback, + EvRegisterSession, + EvUnregisterSession, + EvStop, + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // nonlocal messages; their indices must be preserved in order to work properly while doing rolling update //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/library/actors/interconnect/interconnect_channel.cpp b/ydb/library/actors/interconnect/interconnect_channel.cpp index 5d0aee6b6a2..18822a17833 100644 --- a/ydb/library/actors/interconnect/interconnect_channel.cpp +++ b/ydb/library/actors/interconnect/interconnect_channel.cpp @@ -174,7 +174,7 @@ namespace NActors { State = EState::BODY; IEventBase *base = event.Event.Get(); if (event.EventSerializedSize) { - Chunker.SetSerializingEvent(base); + Chunker.SetSerializingEvent(base, /*withCachedSizes=*/ true); } SerializationInfoContainer = base->CreateSerializationInfo(Params.UseExternalDataChannel); SerializationInfo = &SerializationInfoContainer; diff --git a/ydb/library/actors/interconnect/interconnect_common.cpp b/ydb/library/actors/interconnect/interconnect_common.cpp index 669aa2ca2c7..9b692301292 100644 --- a/ydb/library/actors/interconnect/interconnect_common.cpp +++ b/ydb/library/actors/interconnect/interconnect_common.cpp @@ -1,7 +1,35 @@ #include "interconnect_common.h" +#include "interconnect_uring_engine.h" namespace NActors { + TInterconnectProxyCommon::TInterconnectProxyCommon() = default; + TInterconnectProxyCommon::~TInterconnectProxyCommon() = default; + + void TInterconnectProxyCommon::SetUringEngineV2(TIntrusivePtr<IUringEngine> engine) { + TGuard<TMutex> guard(UringEngineLock); + UringEngineV2 = std::move(engine); + } + + TIntrusivePtr<IUringEngine> TInterconnectProxyCommon::GetUringEngineV2() const { + TGuard<TMutex> guard(UringEngineLock); + return UringEngineV2; + } + + TIntrusivePtr<IUringEngine> TInterconnectProxyCommon::EnsureUringEngineV2(TActorSystem* actorSystem, ui32 numShards, + NMonitoring::TDynamicCounterPtr counters) { + TGuard<TMutex> guard(UringEngineLock); + if (!UringEngineV2) { + UringEngineV2 = CreateUringEngine(actorSystem, numShards, counters); + if (UringEngineV2 && actorSystem) { + // Stop the reaper threads while the actor system is still up, so no completion is posted + // to a torn-down system. + actorSystem->DeferPreStop([engine = UringEngineV2] { engine->Stop(); }); + } + } + return UringEngineV2; + } + double TInterconnectProxyCommon::CalculateNetworkUtilization() { const ui64 sessions = NumSessionsWithDataInQueue.load(); const ui64 ts = GetCycleCountFast(); diff --git a/ydb/library/actors/interconnect/interconnect_common.h b/ydb/library/actors/interconnect/interconnect_common.h index 0e92ca91cb9..0dafd746c57 100644 --- a/ydb/library/actors/interconnect/interconnect_common.h +++ b/ydb/library/actors/interconnect/interconnect_common.h @@ -9,7 +9,9 @@ #include <library/cpp/monlib/metrics/metric_registry.h> #include <util/generic/map.h> #include <util/generic/set.h> +#include <util/generic/ptr.h> #include <util/system/datetime.h> +#include <util/system/mutex.h> #include "event_filter.h" @@ -86,6 +88,7 @@ namespace NActors { // Enables negotiation and usage of TInterconnectSessionTCPv2 (no session continuation, no encryption). // v2 is used only when both peers have this enabled and encryption is not in effect. bool EnableInterconnectSessionV2 = false; + bool ChecksumInterconnectSessionV2 = false; }; struct TWhiteboardSessionStatus { @@ -129,6 +132,8 @@ namespace NActors { using TUpdateWhiteboardCallback = std::function<void(const TWhiteboardSessionStatus& data)>; + class IUringEngine; // shared v2 io_uring data-plane engine (see interconnect_uring_engine.h) + struct TInterconnectProxyCommon : TAtomicRefCount<TInterconnectProxyCommon> { TActorId NameserviceId; NMonitoring::TDynamicCounterPtr MonCounters; @@ -181,6 +186,26 @@ namespace NActors { std::shared_ptr<NInterconnect::NRdma::IMemPool> RdmaMemPool; + // Shared v2 io_uring data-plane engine for the node (created once at startup when v2 + io_uring + // are enabled). Sessions fetch it and call it directly. Guarded because it is published on one + // thread and read from session mailbox threads. + mutable TMutex UringEngineLock; + TIntrusivePtr<IUringEngine> UringEngineV2; + + void SetUringEngineV2(TIntrusivePtr<IUringEngine> engine); + TIntrusivePtr<IUringEngine> GetUringEngineV2() const; + + // Returns the node's v2 io_uring engine, creating it lazily on first use and registering it to + // be stopped when the actor system stops (via DeferPreStop). Returns null only when io_uring is + // unavailable. This keeps the engine working without any per-deployment initializer wiring. + TIntrusivePtr<IUringEngine> EnsureUringEngineV2(TActorSystem* actorSystem, ui32 numShards, + NMonitoring::TDynamicCounterPtr counters); + + // Out-of-line so translation units that construct/destroy Common do not need the complete + // IUringEngine type (it is only complete in interconnect_common.cpp). + TInterconnectProxyCommon(); + ~TInterconnectProxyCommon(); + using TPtr = TIntrusivePtr<TInterconnectProxyCommon>; }; diff --git a/ydb/library/actors/interconnect/interconnect_direct_session.h b/ydb/library/actors/interconnect/interconnect_direct_session.h index 43bf8a1f424..b9fa93d6e69 100644 --- a/ydb/library/actors/interconnect/interconnect_direct_session.h +++ b/ydb/library/actors/interconnect/interconnect_direct_session.h @@ -167,82 +167,4 @@ namespace NActors { THashMap<TActorId, TIntrusivePtr<IReceiveCallback>, TActorId::THash> Callbacks; }; - // Direct session backing a newer (v2) TInterconnectSessionTCPv2 connection. The outgoing data plane - // (serialization / socket handoff) is intentionally left as a stub for now, so this implementation - // keeps a simple mutex-guarded registration table. An atomic counter lets DeliverIncoming skip the - // mutex entirely when no receivers are registered. - class TDirectSessionV2 final : public IDirectSession { - public: - bool Send(TAutoPtr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback = nullptr) override { - { - TGuard<TMutex> guard(Mutex); - if (!Connected) { - return false; - } - if (replyCallback) { - RegisterLocked(ev->Sender, std::move(replyCallback)); - } - } - // TODO(v2 data plane): hand the event off to the session's outgoing stream. - Y_UNUSED(ev); - return true; - } - - void RegisterReceiveCallback(const TActorId& localActorId, TIntrusivePtr<IReceiveCallback> callback) override { - TGuard<TMutex> guard(Mutex); - if (!Connected) { - return; - } - RegisterLocked(localActorId, std::move(callback)); - } - - void UnregisterReceiveCallback(const TActorId& localActorId) override { - TGuard<TMutex> guard(Mutex); - if (Callbacks.erase(localActorId)) { - CallbackCount.store(Callbacks.size(), std::memory_order_release); - } - } - - bool DeliverIncoming(TAutoPtr<IEventHandle>& ev) { - if (CallbackCount.load(std::memory_order_acquire) == 0) { - return false; - } - TIntrusivePtr<IReceiveCallback> callback; - { - TGuard<TMutex> guard(Mutex); - if (!Connected) { - return false; - } - // just received -> Recipient and RecipientRewrite are guaranteed to match - const auto it = Callbacks.find(ev->Recipient); - if (it == Callbacks.end()) { - return false; - } - callback = it->second; - } - callback->Receive(std::move(ev)); - return true; - } - - void Shutdown() { - TGuard<TMutex> guard(Mutex); - Connected = false; - Callbacks.clear(); - CallbackCount.store(0, std::memory_order_release); - } - - private: - // Mutex must be held and Connected must be true. - void RegisterLocked(const TActorId& localActorId, TIntrusivePtr<IReceiveCallback> callback) { - Callbacks[localActorId] = std::move(callback); - CallbackCount.store(Callbacks.size(), std::memory_order_release); - } - - private: - mutable TMutex Mutex; - bool Connected = true; - std::atomic<size_t> CallbackCount = 0; - THashMap<TActorId, TIntrusivePtr<IReceiveCallback>, TActorId::THash> Callbacks; - }; - } diff --git a/ydb/library/actors/interconnect/interconnect_handshake.cpp b/ydb/library/actors/interconnect/interconnect_handshake.cpp index e68098167fa..680ea15c7f9 100644 --- a/ydb/library/actors/interconnect/interconnect_handshake.cpp +++ b/ydb/library/actors/interconnect/interconnect_handshake.cpp @@ -1,6 +1,7 @@ #include "interconnect_handshake.h" #include "handshake_broker.h" #include "interconnect_tcp_proxy.h" +#include "uring_context.h" // TUringContext::IsSupported() gates v2 (io_uring data plane) #include "rdma/link_manager.h" #include "rdma/events.h" @@ -960,9 +961,12 @@ namespace NActors { request.SetRequestXxhash(true); request.SetRequestXdcShuffle(true); request.SetRequestAllowDisablingPayloadChecksums(true); - // v2 session is incompatible with encryption; only request it when encryption is disabled locally + // v2 session is incompatible with encryption and needs the io_uring data plane; only + // request it when encryption is disabled locally and io_uring is available (buffer rings + // are used when present, with a fallback to ordinary buffers on older kernels) request.SetRequestSessionV2(Common->Settings.EnableInterconnectSessionV2 && - Common->Settings.EncryptionMode == EEncryptionMode::DISABLED); + Common->Settings.EncryptionMode == EEncryptionMode::DISABLED && + TUringContext::IsAvailable()); request.SetHandshakeId(*HandshakeId); ui32 pending = 0; @@ -1360,9 +1364,11 @@ namespace NActors { Params.UseXdcShuffle = request.GetRequestXdcShuffle(); Params.UseKernelLiveness = MainChannel.IsKernelLivenessReady(); Params.AllowDisablingPayloadChecksums = request.GetRequestAllowDisablingPayloadChecksums(); - // v2 session is used only when both peers enabled it and encryption is not in effect + // v2 session is used only when both peers enabled it, encryption is not in effect, and + // this side has the io_uring data plane available Params.UseSessionV2 = request.GetRequestSessionV2() && - Common->Settings.EnableInterconnectSessionV2 && !Params.Encryption; + Common->Settings.EnableInterconnectSessionV2 && !Params.Encryption && + TUringContext::IsAvailable(); if (Params.UseExternalDataChannel) { if (request.HasHandshakeId()) { diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp index fea8d1ca9e8..8d19db2503c 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp +++ b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp @@ -2,13 +2,61 @@ #include "interconnect_tcp_proxy.h" #include <util/stream/str.h> +#include <util/system/env.h> +#include <util/string/cast.h> namespace NActors { + namespace { + // number of io_uring rings/reaper threads the shared engine is created with (overridable for tests) + ui32 UringEngineShards() { + const TString s = GetEnv("YDB_IC_V2_SHARDS"); + return s.empty() ? 4 : FromString<ui32>(s); + } + } + + class TInterconnectSessionTCPv2::TDirectSessionV2 final : public IDirectSession { + TIntrusivePtr<IUringEngine> Engine; + ui64 Conn; + std::atomic_bool Stopped{false}; + + public: + TDirectSessionV2(TIntrusivePtr<IUringEngine> engine, ui64 conn) + : Engine(std::move(engine)) + , Conn(conn) + {} + + bool Send(TAutoPtr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) override { + if (Stopped) { + return false; + } + Engine->Send(Conn, std::unique_ptr<IEventHandle>(ev.Release()), std::move(replyCallback)); + return true; + } + + void RegisterReceiveCallback(const TActorId& localActorId, TIntrusivePtr<IReceiveCallback> callback) override { + if (Stopped) { + return; + } + Engine->RegisterReceiveCallback(Conn, localActorId, std::move(callback)); + } + + void UnregisterReceiveCallback(const TActorId& localActorId) override { + if (Stopped) { + return; + } + Engine->RegisterReceiveCallback(Conn, localActorId, nullptr); + } + + void Shutdown() { + Stopped.store(true); + } + }; + TInterconnectSessionTCPv2::TInterconnectSessionTCPv2(TInterconnectProxyTCP* const proxy) : TActor(&TInterconnectSessionTCPv2::StateFunc) , Proxy(proxy) - { } + {} void TInterconnectSessionTCPv2::Init(const TSessionParams& params) { Params = params; @@ -18,7 +66,6 @@ namespace NActors { Proxy->Metrics->SetConnected(0); SetPrefix(Sprintf("SessionV2 %s [node %" PRIu32 "]", SelfId().ToString().data(), Proxy->PeerNodeId)); LOG_INFO_IC_SESSION("ICS90", "v2 session created"); - DirectSession = std::make_shared<TDirectSessionV2>(); } void TInterconnectSessionTCPv2::SetNewConnection(TEvHandshakeDone::TPtr& ev) { @@ -30,12 +77,32 @@ namespace NActors { i64(*ev->Get()->Socket)); Socket = std::move(ev->Get()->Socket); - XdcSocket = std::move(ev->Get()->XdcSocket); + XdcSocket = std::move(ev->Get()->XdcSocket); // unused by v2 (no external data channel) Proxy->Metrics->SetConnected(1); - // NOTE: data-plane setup (input session, poller registration, traffic generation) is stubbed. - // Subscribers receive the direct interface through TEvNodeConnected upon subscription. + // Register the socket with the shared io_uring engine and start driving traffic. Register is a + // direct, synchronous call (the engine arms recv under its shard mutex). The engine is created + // lazily on first use, so no per-deployment initializer wiring is required. + Engine = Proxy->Common->EnsureUringEngineV2(TActivationContext::ActorSystem(), UringEngineShards(), + Proxy->Common->MonCounters->GetSubgroup("subsystem", "uring")); + if (!Engine) { + LOG_ERROR_IC_SESSION("ICS99", "v2 io_uring engine is unavailable"); + return Terminate(TDisconnectReason::LostConnection()); + } + + auto onDisconnectCallback = [selfId = SelfId(), as = TActivationContext::ActorSystem()](TDisconnectReason reason) { + as->Send(selfId, new TEvPrivate::TEvTerminate(reason)); + }; + SetNonBlock(*Socket, false); + EngineHandle = Engine->Register(Socket, SelfId(), Proxy->Common->Settings.ChecksumInterconnectSessionV2, + Params.PeerScopeId, onDisconnectCallback); + if (!EngineHandle) { + LOG_ERROR_IC_SESSION("ICS99", "v2 io_uring engine failed to register the connection"); + return Terminate(TDisconnectReason::LostConnection()); + } + + DirectSession = std::make_shared<TDirectSessionV2>(Engine, EngineHandle); } void TInterconnectSessionTCPv2::Terminate(TDisconnectReason reason) { @@ -48,6 +115,7 @@ namespace NActors { DirectSession->Shutdown(); } + // Shutting down the socket forces any in-flight writev/recv to complete promptly. if (Socket) { Socket->Shutdown(SHUT_RDWR); } @@ -62,6 +130,8 @@ namespace NActors { Proxy->Metrics->SetConnected(0); + Engine->Unregister(EngineHandle); + TActor::PassAway(); } @@ -85,7 +155,9 @@ namespace NActors { } void TInterconnectSessionTCPv2::CloseInputSession() { - // no input session in the current stub + // v2 has no separate input session; a request to close the input means the connection is being + // torn down + Terminate(TDisconnectReason::UserRequest()); } void TInterconnectSessionTCPv2::AddSubscriber(const TActorId& actorId, ui64 cookie) { @@ -96,14 +168,18 @@ namespace NActors { return new TEvInterconnect::TEvNodeConnected(Proxy->PeerNodeId, DirectSession); } + void TInterconnectSessionTCPv2::EnqueueOutgoing(TAutoPtr<IEventHandle> ev) { + ev->Preserialize(); + Engine->Send(EngineHandle, std::unique_ptr<IEventHandle>(ev.Release())); + } + void TInterconnectSessionTCPv2::Forward(STATEFN_SIG) { Proxy->ValidateEvent(ev, "Forward"); if (ev->Flags & IEventHandle::FlagSubscribeOnSession) { AddSubscriber(ev->Sender, ev->Cookie); Send(ev->Sender, MakeNodeConnectedEvent(), 0, ev->Cookie); } - // data-plane stub: the payload event is dropped for now - LOG_DEBUG_IC_SESSION("ICS95", "v2 stub dropping forwarded event to %s", ev->Recipient.ToString().data()); + EnqueueOutgoing(std::move(ev)); } void TInterconnectSessionTCPv2::ForwardWithSubscribe(STATEFN_SIG) { @@ -112,7 +188,7 @@ namespace NActors { Y_ABORT_UNLESS(msg->Event); AddSubscriber(msg->Event->Sender, msg->Event->Cookie); Send(msg->Event->Sender, MakeNodeConnectedEvent(), 0, msg->Event->Cookie); - // data-plane stub: the wrapped payload event is dropped for now + EnqueueOutgoing(TAutoPtr<IEventHandle>(msg->Event.Release())); } void TInterconnectSessionTCPv2::HandleSubscribe(STATEFN_SIG) { @@ -135,8 +211,15 @@ namespace NActors { ev->Get()->Output(str); str << "<div class=\"panel panel-info\">" "<div class=\"panel-heading\">Session (v2)</div>" - "<div class=\"panel-body\">TInterconnectSessionTCPv2: data plane not yet implemented</div>" - "</div>"; + "<div class=\"panel-body\">"; + str << "<table class=\"table\">"; +// str << "<tr><td>Registered</td><td>" << (Registered ? "true" : "false") << "</td></tr>"; + str << "<tr><td>EngineHandle</td><td>" << EngineHandle << "</td></tr>"; +// str << "<tr><td>OutstandingWrites</td><td>" << PendingBatches.size() << "</td></tr>"; + str << "<tr><td>BytesSent</td><td>" << BytesSent << "</td></tr>"; + str << "<tr><td>BytesReceived</td><td>" << BytesReceived << "</td></tr>"; + str << "</table>"; + str << "</div></div>"; TActivationContext::Send(new IEventHandle(ev->Recipient, ev->Sender, new NMon::TEvHttpInfoRes(str.Str()))); } diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h index 06e02020260..29b6899a63a 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h +++ b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h @@ -8,8 +8,11 @@ #include "interconnect_session_iface.h" #include "interconnect_direct_session.h" +#include "interconnect_uring_engine.h" +#include "v2_event_serializer.h" #include "logging/logging.h" +#include <deque> #include <memory> #include <unordered_map> @@ -23,8 +26,12 @@ namespace NActors { // * exposes a thread-safe direct send/receive interface (IDirectSession) to subscribers via // TEvInterconnect::TEvNodeConnected. // - // NOTE: the data-plane operations (packetization, socket IO, input session, pinging, metrics, ...) - // are intentionally left as stubs in this first integration step. + // Its data plane serializes events with TEventSerializer/TEventDeserializer and moves bytes through + // a single shared io_uring ring owned by TInterconnectUringEngineActor. Outgoing events (both the + // normal actor-system Forward path and IDirectSession::Send) are serialized and handed to the engine + // one buffer at a time (single send in flight); incoming bytes are fed to the deserializer and each + // reconstructed event is offered to the direct-session receive table before falling back to normal + // actor-system delivery. class TInterconnectSessionTCPv2 : public TActor<TInterconnectSessionTCPv2> , public TInterconnectLoggingBase @@ -62,6 +69,18 @@ namespace NActors { private: friend class TActor<TInterconnectSessionTCPv2>; + struct TEvPrivate { + enum { + EvTerminate = EventSpaceBegin(TEvents::ES_PRIVATE), + }; + + struct TEvTerminate : TEventLocal<TEvTerminate, EvTerminate> { + TDisconnectReason Reason; + + TEvTerminate(TDisconnectReason reason) : Reason(reason) {} + }; + }; + STATEFN(StateFunc) { STRICT_STFUNC_BODY( fFunc(TEvInterconnect::EvForward, Forward) @@ -71,6 +90,7 @@ namespace NActors { fFunc(TEvents::TEvUnsubscribe::EventType, HandleUnsubscribe) cFunc(TEvents::TEvPoisonPill::EventType, HandlePoison) cFunc(TEvInterconnect::EvForwardDelayed, IgnoreForwardDelayed) + hFunc(TEvPrivate::TEvTerminate, [&](auto& ev) { Terminate(ev->Get()->Reason); }); ) } @@ -81,9 +101,12 @@ namespace NActors { void HandlePoison(); void IgnoreForwardDelayed() {} + void EnqueueOutgoing(TAutoPtr<IEventHandle> ev); + void AddSubscriber(const TActorId& actorId, ui64 cookie); IEventBase* MakeNodeConnectedEvent() const; + private: TInterconnectProxyTCP* const Proxy; TSessionParams Params; @@ -91,10 +114,18 @@ namespace NActors { TIntrusivePtr<NInterconnect::TStreamSocket> XdcSocket; // thread-safe direct send/receive handle shared with users via TEvNodeConnected + class TDirectSessionV2; std::shared_ptr<TDirectSessionV2> DirectSession; + // io_uring data plane + TUringEnginePtr Engine; + ui64 EngineHandle = 0; + + ui64 BytesSent = 0; + ui64 BytesReceived = 0; + // subscribers awaiting connection state notifications (actor id -> cookie) - std::unordered_map<TActorId, ui64, TActorId::THash> Subscribers; + THashMap<TActorId, ui64> Subscribers; }; } diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.cpp b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp new file mode 100644 index 00000000000..9cfbea6c36a --- /dev/null +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp @@ -0,0 +1,903 @@ +#include "interconnect_uring_engine.h" + +#ifdef __linux__ + +#include "uring_recv_buffer_pool.h" +#include "uring_context.h" // for TUringContext::IsSupported() + +#include "v2_event_serializer.h" +#include "interconnect_direct_session.h" + +#include <ydb/library/actors/core/actorsystem.h> +#include <ydb/library/actors/core/actor.h> + +#include <ydb/library/actors/util/funnel_queue.h> + +// Must be included AFTER YDB headers because linux/uapi headers pulled by +// liburing may define macros that clash with project headers. +#include <ydb/library/uring/liburing_linux.h> + +#include <util/system/env.h> +#include <util/system/hp_timer.h> + +#include <sys/socket.h> +#include <sys/uio.h> + +#include <cerrno> +#include <deque> +#include <memory> +#include <thread> +#include <unordered_map> +#include <vector> + +namespace NActors { + + namespace { + constexpr ui32 RingQueueDepth = 4096; + constexpr unsigned CqeBatchSize = 64; + constexpr size_t ReadBufferSize = 262144; + constexpr size_t MinReadBufferSize = 65536; + constexpr size_t WriteBufferSize = 262144; + constexpr size_t MinWriteBufferSize = 65536; + constexpr size_t MaxSpansPerWrite = 64; + constexpr size_t SerializeWindowSize = 65536; + constexpr size_t MinSerializeWindowSize = 4096; + } + + struct TEventPayload { + ui64 Conn; + TIntrusivePtr<IReceiveCallback> Callback; + }; + static_assert(sizeof(TEventPayload) <= sizeof(TActorId)); + + class TIncomingEventQueue { + IEventHandle Stub{0, 0, {}, {}, nullptr, 0}; + std::atomic<IEventHandle*> Head{&Stub}; + std::atomic<IEventHandle*> Tail{&Stub}; + + public: + TIncomingEventQueue() { + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + } + + bool Push(std::unique_ptr<IEventHandle> ev) { + IEventHandle *last = ev.release(); + last->NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(last, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(last), std::memory_order_release); + return prev == &Stub; // if it was the first event + } + + bool IsEmpty() const { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head = Head.load(std::memory_order_acquire); + return tail == &Stub && !next && tail == head; + } + + std::unique_ptr<IEventHandle> Pop() { + for (;;) { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head; + + if (tail == &Stub) { + if (!next) { + if (head = Head.load(std::memory_order_acquire); tail != head) { + continue; + } else { + return nullptr; + } + } + Tail.store(next, std::memory_order_relaxed); + tail = next; + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + } + + if (next) { + Tail.store(next, std::memory_order_relaxed); + return std::unique_ptr<IEventHandle>(tail); + } + + head = Head.load(std::memory_order_acquire); + if (tail != head) { + continue; + } + + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(&Stub, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(&Stub), std::memory_order_release); + + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + if (next) { + Tail.store(next, std::memory_order_relaxed); + return std::unique_ptr<IEventHandle>(tail); + } + } + } + }; + + class TUringEngine final : public IUringEngine { + TActorSystem* const ActorSystem; + std::atomic_bool Stopping{false}; + + struct TRegisteredSession : TEventDeserializer::IEventProcessor { + const ui32 ShardIdx; + const TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + const TActorId SessionId; + const std::function<void(TDisconnectReason)> OnDisconnectCallback; + TActorSystem* const ActorSystem; + TEventSerializer Serializer; + TEventDeserializer Deserializer; + TRcBuf ReadBuffer; + bool Terminated = false; + bool ReadPending = false; + bool WritePending = false; + bool UnregisterRequested = false; + TRcBuf WriteBuffer; + std::deque<TContiguousSpan> OutgoingSpans; + iovec Iov[MaxSpansPerWrite]; + size_t IovLen = 0; + size_t UnsentBytes = 0; + + THashMap<TActorId, TIntrusivePtr<IReceiveCallback>> ReceiveCallbacks; + NMonitoring::TDynamicCounters::TCounterPtr EventsReceived; + + TEventDeserializer _TempDeser; + + TRegisteredSession(ui32 shardIdx, TIntrusivePtr<NInterconnect::TStreamSocket> socket, TActorId sessionId, + bool checksumming, TScopeId peerScopeId, std::function<void(TDisconnectReason)> onDisconnectCallback, + TActorSystem *actorSystem) + : ShardIdx(shardIdx) + , Socket(std::move(socket)) + , SessionId(sessionId) + , OnDisconnectCallback(std::move(onDisconnectCallback)) + , ActorSystem(actorSystem) + , Serializer(checksumming) + , Deserializer(peerScopeId) + , _TempDeser(peerScopeId) + {} + + void Disconnect(TDisconnectReason reason) { + OnDisconnectCallback(reason); + Terminated = true; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // deserialization/receiving + + TMutableContiguousSpan GetReadSpan() { + if (ReadBuffer.size() < MinReadBufferSize) { + ReadBuffer = TRcBuf::Uninitialized(ReadBufferSize); + } + return ReadBuffer.UnsafeGetContiguousSpanMut(); + } + + void ApplyBytesRead(size_t num) { + TRcBuf chunk = {TRcBuf::Piece, ReadBuffer.data(), num, ReadBuffer}; + Deserializer.Push(std::move(chunk), this, SessionId); + Y_ABORT_UNLESS(num <= ReadBuffer.size()); + const size_t remain = ReadBuffer.size() - num; + ReadBuffer.TrimFront(remain - remain % 64); // make only this number of bytes remaining in buffer + } + + void PushEvent(std::unique_ptr<IEventHandle> ev) override { + if (const auto it = ReceiveCallbacks.find(ev->Recipient); it != ReceiveCallbacks.end()) { + it->second->Receive(ev.release()); + } else { + ActorSystem->Send(ev.release()); + } + ++*EventsReceived; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // serialization/sending + + bool Serialize() { + if (UnsentBytes >= MinSerializeWindowSize) { // we have some bytes to send by now, don't trigger serialization + return false; + } + + Serializer.ResetCounters(); + + while (UnsentBytes < SerializeWindowSize && OutgoingSpans.size() < MaxSpansPerWrite) { + if (WriteBuffer.size() < MinWriteBufferSize) { // (re)allocate write buffer + WriteBuffer = TRcBuf::Uninitialized(WriteBufferSize); + } else { // align write buffer to 64-byte boundary + WriteBuffer.TrimFront(WriteBuffer.size() - WriteBuffer.size() % 64); + } + const size_t numBytesProduced = Serializer.ProduceOutputStream(WriteBuffer, &OutgoingSpans, + SerializeWindowSize - UnsentBytes); + + if (!numBytesProduced) { + break; + } + UnsentBytes += numBytesProduced; + } + + return true; + } + + bool PrepareIovec() { + // Build the iovec WITHOUT consuming spans: writev may complete partially, so a span is only + // dropped once the bytes it covers have actually been confirmed sent (see ApplyBytesWritten). + IovLen = 0; + for (const TContiguousSpan& span : OutgoingSpans) { + if (IovLen >= MaxSpansPerWrite) { + break; + } + Iov[IovLen++] = { + .iov_base = const_cast<char*>(span.data()), + .iov_len = span.size(), + }; + } + return IovLen != 0; + } + + void ApplyBytesWritten(size_t num) { + // Advance past exactly the bytes the kernel accepted. A writev can be short (e.g. under + // backpressure or on a real network), so drop only fully-sent spans and trim the span that + // straddles the boundary; the rest stay queued and are retried by the next writev. + for (size_t remaining = num; remaining && !OutgoingSpans.empty(); OutgoingSpans.pop_front()) { +// static struct TNullProcessor : IEventProcessor { void PushEvent(std::unique_ptr<IEventHandle>) override {} } nullProcessor; +// _TempDeser.Push(TRcBuf::Copy(OutgoingSpans.front().SubSpan(0, remaining)), &nullProcessor, {}); + + if (TContiguousSpan& front = OutgoingSpans.front(); front.size() <= remaining) { + remaining -= front.size(); + } else { + front = TContiguousSpan(front.data() + remaining, front.size() - remaining); + break; + } + } + + Y_ABORT_UNLESS(num <= UnsentBytes, "num# %zu UnsentBytes# %zu", num, UnsentBytes); + UnsentBytes -= num; + + Serializer.CommitProducedBytes(num); + } + }; + + class TShard { + enum EOperationType { + kOpPipe = 1, + kOpRead, + kOpWrite, + }; + static const ui64 kOpMask = (1 << 3) - 1; + + TIncomingEventQueue IncomingEventQueue; + std::thread Worker; + + io_uring Ring; + i64 ItemsToSubmit = 0; + std::atomic_bool WaitingForCQ{false}; + + int ReadPipe; + int WritePipe; + char ReadPipeBuffer[256]; + + struct TSessionHash { + size_t operator()(const std::unique_ptr<TRegisteredSession>& p) const { return THash<void*>{}(p.get()); } + size_t operator()(const TRegisteredSession *p) const { return THash<void*>{}(p); } + }; + + struct TSessionEqual { + using T = std::unique_ptr<TRegisteredSession>; + bool operator()(const T& x, const T& y) const { return x == y; } + bool operator()(const TRegisteredSession *x, const T& y) const { return x == y.get(); } + bool operator()(const T& x, const TRegisteredSession *y) const { return x.get() == y; } + }; + + THashSet<std::unique_ptr<TRegisteredSession>, TSessionHash, TSessionEqual> Sessions; + + NMonitoring::TDynamicCounters::TCounterPtr SessionsRegistered; + NMonitoring::TDynamicCounters::TCounterPtr SessionsUnregistered; + NMonitoring::TDynamicCounters::TCounterPtr EventsSent; + NMonitoring::TDynamicCounters::TCounterPtr EventsReceived; + NMonitoring::TDynamicCounters::TCounterPtr DirectReceiveCallbacksRegistered; + NMonitoring::TDynamicCounters::TCounterPtr DirectReceiveCallbacksUnregistered; + NMonitoring::TDynamicCounters::TCounterPtr BytesSent; + NMonitoring::TDynamicCounters::TCounterPtr BytesCopied; + NMonitoring::TDynamicCounters::TCounterPtr BytesAliased; + NMonitoring::TDynamicCounters::TCounterPtr BytesReceived; + NMonitoring::TDynamicCounters::TCounterPtr SQEAllocated; + NMonitoring::TDynamicCounters::TCounterPtr SubmitCount; + NMonitoring::TDynamicCounters::TCounterPtr CQEProcessed; + NMonitoring::TDynamicCounters::TCounterPtr PipeWakeups; + NMonitoring::TDynamicCounters::TCounterPtr PushedAsFirst; + NMonitoring::TDynamicCounters::TCounterPtr PushedTotal; + NMonitoring::TDynamicCounters::TCounterPtr ReadUnavail; + NMonitoring::TDynamicCounters::TCounterPtr WriteUnavail; + + NMonitoring::TDynamicCounters::TCounterPtr ActiveTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr CompleteWaitTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr SubmitWaitTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr ApplyBytesReadTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr ApplyBytesWrittenTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr SerializeBufferTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr SerializeEventTotalTime; + NMonitoring::TDynamicCounters::TCounterPtr ProduceOutputStreamOtherTotalTime; + + NMonitoring::THistogramPtr CommandDeliveryTime; + NMonitoring::THistogramPtr CompletionWaitTime; + NMonitoring::THistogramPtr CommandExecTime; + NMonitoring::THistogramPtr SubmitExecTime; + NMonitoring::THistogramPtr SerializeTime; + NMonitoring::THistogramPtr CompletionsProcessedAtOnce; + NMonitoring::THistogramPtr SubmissionsProcessedAtOnce; + + THPTimer ActiveTimer; + NMonitoring::TDynamicCounters::TCounterPtr *CurrentActivityTime = &ActiveTotalTime; + + private: + class TActivityMeasure { + TShard& Shard; + NMonitoring::TDynamicCounters::TCounterPtr *PrevActivityTime; + + public: + TActivityMeasure(TShard& shard, NMonitoring::TDynamicCounters::TCounterPtr *activityTime) + : Shard(shard) + , PrevActivityTime(std::exchange(shard.CurrentActivityTime, activityTime)) + { +// **PrevActivityTime += shard.ActiveTimer.PassedReset() * 1e9; + } + + ~TActivityMeasure() { + const ui64 delta = 0; +// const ui64 delta = Shard.ActiveTimer.PassedReset() * 1e9; + if (Shard.CurrentActivityTime) { + **Shard.CurrentActivityTime += delta; + } + Shard.CurrentActivityTime = PrevActivityTime; + } + }; + +#define ACTIVITY(NAME) if (TActivityMeasure __measure{*this, NAME}; false); else + + public: + static NMonitoring::IHistogramCollectorPtr TimeCollector() { + return NMonitoring::ExponentialHistogram(22, 2); + } + + TShard(const NMonitoring::TDynamicCounterPtr& shardCounters) +#define COUNTER(NAME, DERIV) NAME(shardCounters->GetCounter(#NAME, DERIV)) + : COUNTER(SessionsRegistered, true) + , COUNTER(SessionsUnregistered, true) + , COUNTER(EventsSent, true) + , COUNTER(EventsReceived, true) + , COUNTER(DirectReceiveCallbacksRegistered, true) + , COUNTER(DirectReceiveCallbacksUnregistered, true) + , COUNTER(BytesSent, true) + , COUNTER(BytesCopied, true) + , COUNTER(BytesAliased, true) + , COUNTER(BytesReceived, true) + , COUNTER(SQEAllocated, true) + , COUNTER(SubmitCount, true) + , COUNTER(CQEProcessed, true) + , COUNTER(PipeWakeups, true) + , COUNTER(PushedAsFirst, true) + , COUNTER(PushedTotal, true) + , COUNTER(ReadUnavail, true) + , COUNTER(WriteUnavail, true) +#define TOTAL_TIME(NAME) NAME(shardCounters->GetCounter("TotalTime/" #NAME, true)) + , TOTAL_TIME(ActiveTotalTime) + , TOTAL_TIME(CompleteWaitTotalTime) + , TOTAL_TIME(SubmitWaitTotalTime) + , TOTAL_TIME(ApplyBytesReadTotalTime) + , TOTAL_TIME(ApplyBytesWrittenTotalTime) + , TOTAL_TIME(SerializeBufferTotalTime) + , TOTAL_TIME(SerializeEventTotalTime) + , TOTAL_TIME(ProduceOutputStreamOtherTotalTime) + , CommandDeliveryTime(shardCounters->GetNamedHistogram("sensor", "CommandDeliveryTime", TimeCollector())) + , CompletionWaitTime(shardCounters->GetNamedHistogram("sensor", "CompletionWaitTime", TimeCollector())) + , CommandExecTime(shardCounters->GetNamedHistogram("sensor", "CommandExecTime", TimeCollector())) + , SubmitExecTime(shardCounters->GetNamedHistogram("sensor", "SubmitExecTime", TimeCollector())) + , SerializeTime(shardCounters->GetNamedHistogram("sensor", "SerializeTime", TimeCollector())) + , CompletionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "CompletionsProcessedAtOnce", NMonitoring::ExponentialHistogram(10, 2))) + , SubmissionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "SubmissionsProcessedAtOnce", NMonitoring::ExponentialHistogram(12, 2))) +#undef TOTAL_TIME +#undef COUNTER + { + // initialize ring for this shard + if (io_uring_queue_init(RingQueueDepth, &Ring, IORING_SETUP_SQPOLL) < 0) { + Y_ABORT("failed to initialize ring"); + } + + // create pipe to kick worker thread when first commands arrive + int fds[2]; + if (pipe(fds) == -1) { + Y_ABORT("pipe() failed: %s", strerror(errno)); + } + ReadPipe = fds[0]; + WritePipe = fds[1]; + + // start worker thread + Worker = std::thread(std::bind(&TShard::WorkerThread, this)); + } + + ~TShard() { + Stop(); // joins the worker thread, so no completion will be dispatched after this point + io_uring_queue_exit(&Ring); // cancels any still-armed read/write without dispatching them + DrainQueue(); // free commands that were enqueued after the worker stopped (teardown races) + close(ReadPipe); + close(WritePipe); + // remaining registered sessions are freed as the Sessions container is destroyed + } + + void Register(std::unique_ptr<TRegisteredSession> session) { + ++*SessionsRegistered; + SendInternal(reinterpret_cast<ui64>(session.release()), static_cast<ui32>(ENetwork::EvRegisterSession), + {}, nullptr); + } + + void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { + ++*EventsSent; + SendImpl(conn, std::move(ev), std::move(replyCallback)); + } + + void Unregister(ui64 conn) { + ++*SessionsUnregistered; + SendInternal(conn, static_cast<ui32>(ENetwork::EvUnregisterSession), {}, nullptr); + } + + void RegisterReceiveCallback(ui64 conn, TActorId localActorId, TIntrusivePtr<IReceiveCallback> callback) { + ++*(callback ? DirectReceiveCallbacksRegistered : DirectReceiveCallbacksUnregistered); + SendInternal(conn, static_cast<ui32>(ENetwork::EvRegisterCallback), localActorId, std::move(callback)); + } + + void Stop() { + if (Worker.joinable()) { + SendInternal(0, static_cast<ui32>(ENetwork::EvStop), {}, nullptr); + Worker.join(); + // The worker is stopped, so it is now safe to touch the sessions directly. Shut every + // socket down so the peer observes the disconnect promptly instead of only when this shard + // (and thus the sockets it still references) is finally destroyed. The session objects + // themselves stay alive until the shard is destroyed. + for (const auto& session : Sessions) { + if (session->Socket) { + session->Socket->Shutdown(SHUT_RDWR); + } + } + } + } + + private: + // Pops and frees any commands still sitting in the queue after the worker has stopped. Mirrors + // the ownership handling of the worker loop: destroys the embedded TEventPayload and reclaims a + // TRegisteredSession handed off via an unprocessed EvRegisterSession. + void DrainQueue() { + while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { + auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); + if (ev->Type == static_cast<ui32>(ENetwork::EvRegisterSession)) { + std::unique_ptr<TRegisteredSession> reclaim(reinterpret_cast<TRegisteredSession*>(payload.Conn)); + } + payload.~TEventPayload(); + } + } + + void SendImpl(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { + new(const_cast<TActorId*>(&ev->InterconnectSession)) TEventPayload{ + .Conn = conn, + .Callback = std::move(replyCallback), + }; + const bool first = IncomingEventQueue.Push(std::move(ev)); + if (first) { + ++*PushedAsFirst; + } + ++*PushedTotal; + if (first && WaitingForCQ.load()) { + // this is the first command and we are currently waiting on CQ, so we have to push it through syscall + // writing to special fd + char temp = 0; + int res; + while ((res = write(WritePipe, &temp, 1)) != 1) { + if (res == -1) { + if (errno == EINTR) { + continue; + } else { + Y_ABORT("write() to pipe failed: %s", strerror(errno)); + } + } else { + Y_ABORT_UNLESS(res == 0); + Y_ABORT("write() to pipe failed: zero bytes written"); + } + } + ++*PipeWakeups; + } + } + + void SendInternal(ui64 conn, ui32 type, TActorId sender, TIntrusivePtr<IReceiveCallback> callback) { + SendImpl(conn, std::make_unique<IEventHandle>(type, 0, TActorId(), sender, nullptr, GetCycleCountFast()), + std::move(callback)); + } + + // GetSQE returns next available SQ entry, setting up ItemsToSubmit counter in order to commence submission + // on the end of the worker loop + io_uring_sqe *GetSQE(TRegisteredSession *session, EOperationType op) { + io_uring_sqe *sqe = io_uring_get_sqe(&Ring); + if (!sqe) { // submit queue is full: try to submit something to free it up + DoSubmit(); + sqe = io_uring_get_sqe(&Ring); + } + if (sqe) { + ++ItemsToSubmit; + uintptr_t sessionId = reinterpret_cast<uintptr_t>(session); + Y_ABORT_UNLESS((sessionId & kOpMask) == 0); + io_uring_sqe_set_data64(sqe, sessionId | op); + Y_DEBUG_ABORT_UNLESS(op == kOpPipe ? session == nullptr : session != nullptr); + ++*SQEAllocated; + } + return sqe; + } + + // DoSubmit performs actual io_uring submit operation for all allocated entries during the worker loop + void DoSubmit() { + THPTimer timer; + + ACTIVITY(&SubmitWaitTotalTime) { + for (;;) { + int res = io_uring_submit(&Ring); + if (res == -EINTR) { + continue; + } + if (res < 0) { + Y_ABORT("io_uring_submit() failed: %s", strerror(-res)); + } + break; + } + } + + ++*SubmitCount; + SubmitExecTime->Collect(timer.Passed() * 1e6); + SubmissionsProcessedAtOnce->Collect(ItemsToSubmit, 1u); + ItemsToSubmit = 0; + } + + void PutPipeReadRequest() { + io_uring_sqe *sqe = GetSQE(nullptr, kOpPipe); + Y_ABORT_UNLESS(sqe, "failed to obtain pipe SQE: SQ overflow"); // TODO(alexvru): handle this somehow + io_uring_prep_read(sqe, ReadPipe, ReadPipeBuffer, sizeof(ReadPipeBuffer), -1); + } + + void WorkerThread() { + pthread_setname_np(pthread_self(), "IC_uring"); + ActiveTimer.Reset(); + + // prepare read request in order for sender threads to wake this one up when waiting on CQ + PutPipeReadRequest(); + + for (;;) { + // submit any pending SQ's (if we have any) + if (ItemsToSubmit) { + DoSubmit(); + } + + // wait for something to happen + WaitingForCQ.store(true); + if (IncomingEventQueue.IsEmpty()) { + io_uring_cqe *cqe; + THPTimer timer; + ACTIVITY(&CompleteWaitTotalTime) { + if (int res = io_uring_wait_cqe(&Ring, &cqe); res && res != -EINTR) { + Y_ABORT("io_uring_wait_cqe() failed: %s", strerror(-res)); + } + } + CompletionWaitTime->Collect(timer.Passed() * 1e6); + } + WaitingForCQ.store(false); + + // process pending CQ events + io_uring_cqe *cqes[CqeBatchSize]; + i64 completionsProcessedAtOnce = 0; + while (const unsigned n = io_uring_peek_batch_cqe(&Ring, cqes, CqeBatchSize)) { + for (unsigned i = 0; i < n; ++i) { + DispatchCompletion(*cqes[i]); + } + io_uring_cq_advance(&Ring, n); + completionsProcessedAtOnce += n; + if (n < CqeBatchSize) { + break; + } + } + if (completionsProcessedAtOnce) { + CompletionsProcessedAtOnce->Collect(completionsProcessedAtOnce, 1u); + } + + // process pending events and commands + while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { + auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); + const ui64 conn = payload.Conn; + TIntrusivePtr<IReceiveCallback> callback = std::move(payload.Callback); + payload.~TEventPayload(); + + const ui64 cycleCountOnEnter = GetCycleCountFast(); + bool isInternal = true; + + switch (ev->Type) { + case static_cast<ui32>(ENetwork::EvRegisterCallback): + if (TRegisteredSession& session = GetSession(conn); callback) { + session.ReceiveCallbacks[ev->Sender] = std::move(callback); + } else { + session.ReceiveCallbacks.erase(ev->Sender); + } + break; + + case static_cast<ui32>(ENetwork::EvRegisterSession): { + std::unique_ptr<TRegisteredSession> session(reinterpret_cast<TRegisteredSession*>(conn)); + const auto [it, inserted] = Sessions.emplace(std::move(session)); + Y_ABORT_UNLESS(inserted); + (*it)->EventsReceived = EventsReceived; + IssueReadForSession(**it); + break; + } + + case static_cast<ui32>(ENetwork::EvUnregisterSession): { + TRegisteredSession& session = GetSession(conn); + // Do NOT free the session while it still has an armed recv or an in-flight + // writev: their io_uring completions carry a raw pointer to this object and + // would dereference freed memory. Mark it terminated (so no new ops are armed) + // and erase only once both are drained. The session actor has already shut the + // socket down before requesting unregistration, so the pending ops complete + // promptly (EOF/EPIPE). + session.Terminated = true; + session.UnregisterRequested = true; + MaybeEraseSession(session); + break; + } + + case static_cast<ui32>(ENetwork::EvStop): + return; + + default: { + TRegisteredSession& session = GetSession(conn); + if (callback) { // register callback coming along with the message + session.ReceiveCallbacks[ev->Sender] = std::move(callback); + } + session.Serializer.Push(std::move(ev)); + IssueWritesForSession(session); + isInternal = false; + break; + } + } + + const ui64 cycleCountOnExit = GetCycleCountFast(); + + if (isInternal) { + CommandDeliveryTime->Collect((cycleCountOnEnter - ev->Cookie) * 1'000'000 / NHPTimer::GetCyclesPerSecond()); + } + + CommandExecTime->Collect((cycleCountOnExit - cycleCountOnEnter) * 1'000'000 / NHPTimer::GetCyclesPerSecond()); + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // io_uring completion handlers + + void DispatchCompletion(io_uring_cqe cqe) { + auto *session = reinterpret_cast<TRegisteredSession*>(uintptr_t(cqe.user_data) & ~uintptr_t(kOpMask)); + switch (static_cast<EOperationType>(cqe.user_data & kOpMask)) { + case kOpPipe: + // this operation is used just to break wait-CQE syscall and exit to process some commands; but + // we have to re-arm the pipe + Y_DEBUG_ABORT_UNLESS(session == nullptr); + PutPipeReadRequest(); + break; + + case kOpRead: + Y_ABORT_UNLESS(!(cqe.flags & IORING_CQE_F_MORE)); // not expecting multiple completions + Y_DEBUG_ABORT_UNLESS(session != nullptr); + DispatchRead(*session, cqe.res); // TODO(alexvru): maybe handle NONEMPTY (if it won't break equality) + break; + + case kOpWrite: + Y_ABORT_UNLESS(!(cqe.flags & IORING_CQE_F_MORE)); // not expecting multiple completions + Y_DEBUG_ABORT_UNLESS(session != nullptr); + DispatchWrite(*session, cqe.res); + break; + } + ++*CQEProcessed; + } + + void DispatchRead(TRegisteredSession& session, i32 res) { + Y_DEBUG_ABORT_UNLESS(session.ReadPending); + session.ReadPending = false; + + if (session.Terminated) { + // teardown in progress: don't process further data or re-arm; just let the session drain + // toward erasure below + } else if (res == -EAGAIN) { + ++*ReadUnavail; + IssueReadForSession(session); + } else if (res < 0) { + session.Disconnect(TDisconnectReason::FromErrno(-res)); + } else if (res == 0) { + session.Disconnect(TDisconnectReason::EndOfStream()); + } else { + *BytesReceived += res; + ACTIVITY(&ApplyBytesReadTotalTime) { + session.ApplyBytesRead(res); + } + IssueReadForSession(session); + } + + MaybeEraseSession(session); // NB: may free `session`; must be the last use + } + + void IssueReadForSession(TRegisteredSession& session) { + if (session.Terminated) { + return; + } + Y_DEBUG_ABORT_UNLESS(!session.ReadPending); + TMutableContiguousSpan span = session.GetReadSpan(); + io_uring_sqe *sqe = GetSQE(&session, kOpRead); + Y_ABORT_UNLESS(sqe); + io_uring_prep_read(sqe, *session.Socket, span.data(), span.size(), -1); + session.ReadPending = true; + } + + void DispatchWrite(TRegisteredSession& session, i32 res) { + Y_ABORT_UNLESS(session.WritePending); + session.WritePending = false; + + if (session.Terminated) { + // teardown in progress: don't retry the write or re-arm; just let the session drain + // toward erasure below + } else if (res == -EAGAIN) { + ++*WriteUnavail; + SubmitIovec(session); + } else if (res < 0) { + session.Disconnect(TDisconnectReason::FromErrno(-res)); + } else if (res == 0) { + session.Disconnect(TDisconnectReason::EndOfStream()); + } else { + *BytesSent += res; + ACTIVITY(&ApplyBytesWrittenTotalTime) { + session.ApplyBytesWritten(res); + } + IssueWritesForSession(session); + } + + MaybeEraseSession(session); // NB: may free `session`; must be the last use + } + + void IssueWritesForSession(TRegisteredSession& session) { + if (session.WritePending || session.Terminated || !session.Serializer.IsTrafficPending()) { + return; + } + ACTIVITY(nullptr) { + if (session.Serialize()) { + *SerializeBufferTotalTime += session.Serializer.GetSerializeBufferTime(); + *SerializeEventTotalTime += session.Serializer.GetSerializeEventTime(); + *ProduceOutputStreamOtherTotalTime += session.Serializer.GetOtherTime(); + *BytesCopied += session.Serializer.GetBytesCopied(); + *BytesAliased += session.Serializer.GetBytesAliased(); + } + } + if (session.PrepareIovec()) { + SubmitIovec(session); + } + } + + void SubmitIovec(TRegisteredSession& session) { + io_uring_sqe *sqe = GetSQE(&session, kOpWrite); + Y_ABORT_UNLESS(sqe); + io_uring_prep_writev(sqe, *session.Socket, session.Iov, session.IovLen, -1); + session.WritePending = true; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // commands from outer threads + + TRegisteredSession& GetSession(ui64 conn) const { + TRegisteredSession *ptr = reinterpret_cast<TRegisteredSession*>(conn); + Y_ABORT_UNLESS(Sessions.find(ptr) != Sessions.end()); + return *ptr; + } + + // Frees an unregistered session once it has no io_uring operation in flight. It is unsafe to + // erase earlier because any pending read/write completion references the session by raw pointer. + void MaybeEraseSession(TRegisteredSession& session) { + if (session.UnregisterRequested && !session.ReadPending && !session.WritePending) { + auto it = Sessions.find(&session); + Y_ABORT_UNLESS(it != Sessions.end()); + Sessions.erase(it); + } + } + }; + + std::vector<std::unique_ptr<TShard>> Shards; + std::atomic_uint64_t NextShardIdx; + + NMonitoring::TDynamicCounterPtr UringCounters; + + public: + TUringEngine(TActorSystem *actorSystem, ui32 numShards, NMonitoring::TDynamicCounterPtr counters) + : ActorSystem(actorSystem) + , UringCounters(std::move(counters)) + { + Shards.reserve(numShards); + for (ui32 i = 0; i < numShards; ++i) { + Shards.push_back(std::make_unique<TShard>(UringCounters->GetSubgroup("shard", "0" /*ToString(i)*/))); + } + } + + ~TUringEngine() { + Stop(); + } + + ui64 Register(TIntrusivePtr<NInterconnect::TStreamSocket> socket, const TActorId& sessionActorId, + bool checksumming, TScopeId peerScopeId, std::function<void(TDisconnectReason)> onDisconnectCallback) override { + if (Stopping) { + return 0; // engine is shutting down; caller treats 0 as a failed registration and terminates + } + const ui32 shardIdx = NextShardIdx++ % Shards.size(); + auto session = std::make_unique<TRegisteredSession>(shardIdx, std::move(socket), sessionActorId, + checksumming, peerScopeId, std::move(onDisconnectCallback), ActorSystem); + const ui64 conn = reinterpret_cast<ui64>(session.get()); + Shards[shardIdx]->Register(std::move(session)); + return conn; + } + + TShard& GetShard(ui64 conn) const { + return *Shards.at(reinterpret_cast<TRegisteredSession*>(conn)->ShardIdx); + } + + void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) override { + // Sessions may still forward events during actor-system teardown (DeferPreStop runs Stop() before + // executor threads are joined), so drop rather than abort. Checking Stopping before touching the + // shard/conn keeps this safe: Stop() only joins workers, it never frees shards or sessions. + if (Stopping) { + return; + } + GetShard(conn).Send(conn, std::move(ev), std::move(replyCallback)); + } + + void Unregister(ui64 conn) override { + if (Stopping) { + return; + } + GetShard(conn).Unregister(conn); + } + + void RegisterReceiveCallback(ui64 conn, TActorId localActorId, TIntrusivePtr<IReceiveCallback> callback) override { + if (Stopping) { + return; + } + GetShard(conn).RegisterReceiveCallback(conn, localActorId, std::move(callback)); + } + + void Stop() override { + // Quiesce the reaper/worker threads (so no completion is posted to a torn-down actor system) but + // keep shards and their registered sessions alive: executor threads may still be running and may + // call in with live conn pointers. The memory is released later in the destructor, once the actor + // system is fully stopped and no more calls can arrive. + if (!Stopping.exchange(true)) { + for (auto& shard : Shards) { + shard->Stop(); + } + } + } + }; + + TUringEnginePtr CreateUringEngine(TActorSystem* actorSystem, ui32 numShards, NMonitoring::TDynamicCounterPtr counters) { + if (!TUringContext::IsAvailable()) { + return nullptr; + } + if (numShards < 1) { + numShards = 1; + } + return MakeIntrusive<TUringEngine>(actorSystem, numShards, std::move(counters)); + } + +} // namespace NActors + +#else // !__linux__ + +namespace NActors { + TUringEnginePtr CreateUringEngine(TActorSystem*, ui32) { + return nullptr; + } +} + +#endif diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.h b/ydb/library/actors/interconnect/interconnect_uring_engine.h new file mode 100644 index 00000000000..d60dbd2a3e3 --- /dev/null +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.h @@ -0,0 +1,48 @@ +#pragma once + +#include <ydb/library/actors/core/actorid.h> +#include <ydb/library/actors/core/event_local.h> +#include <ydb/library/actors/util/rc_buf.h> + +#include <library/cpp/monlib/dynamic_counters/counters.h> + +#include "events/events.h" +#include "interconnect_stream.h" +#include "types.h" + +#include <memory> +#include <vector> + +namespace NActors { + + class TActorSystem; + class IReceiveCallback; + + class IUringEngine : public TThrRefBase { + public: + // Registers a connected socket, arms receiving, and returns an opaque non-zero handle (0 on + // failure). The engine holds a reference to the socket until Unregister fully drains. Thread-safe. + virtual ui64 Register(TIntrusivePtr<NInterconnect::TStreamSocket> socket, const TActorId& sessionActorId, + bool checksumming, TScopeId peerScopeId, std::function<void(TDisconnectReason)> onDisconnectCallback) = 0; + + // Put a message into outgoing queue. Thread-safe. + virtual void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> callback = nullptr) = 0; + + // Cancels the connection's IO and tears it down; TEvUringV2Unregistered is posted once every + // outstanding operation has drained. Thread-safe. + virtual void Unregister(ui64 conn) = 0; + + // Register receive callback (for IDirectSession). Null callback deregisters it. + virtual void RegisterReceiveCallback(ui64 conn, TActorId localActorId, TIntrusivePtr<IReceiveCallback> callback) = 0; + + // Stops every reaper thread. Must be called before the actor system is torn down. + virtual void Stop() = 0; + }; + + using TUringEnginePtr = TIntrusivePtr<IUringEngine>; + + // Creates the engine with `numShards` rings/reaper threads (clamped to >= 1). Returns nullptr when + // io_uring is unavailable (non-Linux or a kernel that cannot create a ring). + TUringEnginePtr CreateUringEngine(TActorSystem* actorSystem, ui32 numShards, NMonitoring::TDynamicCounterPtr counters); + +} diff --git a/ydb/library/actors/interconnect/load.cpp b/ydb/library/actors/interconnect/load.cpp index 1f2e2db20c1..727ae35f1f3 100644 --- a/ydb/library/actors/interconnect/load.cpp +++ b/ydb/library/actors/interconnect/load.cpp @@ -364,12 +364,13 @@ namespace NInterconnect { msg << " Throughput# "; const TDuration duration = now - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); const ui64 traffic = *Traffic; msg << "{window# " << duration << " bytes# " << ThroughputBytes << " samples# " << ThroughputSamples - << " b/s# " << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()) - << " common# " << ui64((traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()) + << " b/s# " << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)) + << " common# " << (durationUs ? ui64((traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)) << "}"; if (RunFirstSample == TMonotonic::Zero()) { @@ -478,6 +479,7 @@ namespace NInterconnect { TString RenderHTML(bool finished, const TActorContext& ctx) { const auto duration = ctx.Monotonic() - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); TStringStream msg; @@ -498,8 +500,8 @@ namespace NInterconnect { TABLED() { str << duration; } TABLED() { str << ThroughputBytes; } TABLED() { str << ThroughputSamples; } - TABLED() { str << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()); } - TABLED() { str << ui64((*Traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()); } + TABLED() { str << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)); } + TABLED() { str << (durationUs ? ui64((*Traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)); } } } } diff --git a/ydb/library/actors/interconnect/uring_context.cpp b/ydb/library/actors/interconnect/uring_context.cpp index 30bfc45db3a..2e4978b66bc 100644 --- a/ydb/library/actors/interconnect/uring_context.cpp +++ b/ydb/library/actors/interconnect/uring_context.cpp @@ -238,4 +238,21 @@ namespace NActors { return supported; } + bool TUringContext::IsAvailable() { + static const bool available = [] { + // Minimal capability: can we create a plain ring at all? The v2 data plane needs only this; + // it uses provided buffer rings when IsSupported() also holds, and falls back to ordinary + // per-recv buffers otherwise. + struct io_uring ring; + struct io_uring_params params = {}; + const int ret = io_uring_queue_init_params(8, &ring, ¶ms); + if (ret != 0) { + return false; + } + io_uring_queue_exit(&ring); + return true; + }(); + return available; + } + } // namespace NActors diff --git a/ydb/library/actors/interconnect/uring_context.h b/ydb/library/actors/interconnect/uring_context.h index 0c776214899..4f156c2c127 100644 --- a/ydb/library/actors/interconnect/uring_context.h +++ b/ydb/library/actors/interconnect/uring_context.h @@ -85,7 +85,14 @@ namespace NActors { struct io_uring* GetRing() { return Ring.get(); } + // True when the provided-buffer-ring API (io_uring_setup_buf_ring, kernel >= 5.19) works. This + // is the hard dependency of the v1 multishot-recv data path. static bool IsSupported(); + + // True when io_uring is usable at all (a plain ring can be created). This is the minimum for the + // v2 data plane, which uses buffer rings when available (IsSupported) and falls back to ordinary + // per-recv buffers on older kernels (>= 5.1) where only IsAvailable() holds. + static bool IsAvailable(); #else TUringContext(TActorId, TActorId) {} ~TUringContext() = default; @@ -121,6 +128,7 @@ namespace NActors { void* GetRing() { return nullptr; } static bool IsSupported() { return false; } + static bool IsAvailable() { return false; } #endif using TPtr = TIntrusivePtr<TUringContext>; diff --git a/ydb/library/actors/interconnect/ut/lib/node.h b/ydb/library/actors/interconnect/ut/lib/node.h index a06005e848d..fd55506571e 100644 --- a/ydb/library/actors/interconnect/ut/lib/node.h +++ b/ydb/library/actors/interconnect/ut/lib/node.h @@ -11,6 +11,7 @@ #include <ydb/library/actors/interconnect/interconnect_tcp_server.h> #include <ydb/library/actors/interconnect/interconnect_tcp_proxy.h> #include <ydb/library/actors/interconnect/interconnect_proxy_wrapper.h> +#include <ydb/library/actors/interconnect/interconnect_uring_engine.h> #include <ydb/library/actors/interconnect/poller/uring_poller_actor.h> #include <ydb/library/actors/interconnect/rdma/mem_pool.h> #include <ydb/library/actors/interconnect/rdma/cq_actor/cq_actor.h> @@ -174,6 +175,9 @@ public: auto sp = MakeHolder<TActorSystemSetup>(std::move(setup)); ActorSystem.Reset(new TActorSystem(sp, nullptr, loggerSettings)); ActorSystem->Start(); + // The v2 io_uring engine (shared, off-actor, with its own reaper threads) is created lazily on + // the first v2 session and stopped via the actor system's DeferPreStop hook -- no explicit + // wiring here or in production initializers is required. } ~TNode() { @@ -183,6 +187,8 @@ public: void Stop() { if (ActorSystem) { + // The v2 engine (if created) is stopped via the actor system's DeferPreStop hook during + // Stop(), so its reaper threads are joined before the executors shut down. ActorSystem->Stop(); ActorSystem.Reset(); } diff --git a/ydb/library/actors/interconnect/ut/v2_event_serializer_ut.cpp b/ydb/library/actors/interconnect/ut/v2_event_serializer_ut.cpp index d6deaf81c35..2710e42976b 100644 --- a/ydb/library/actors/interconnect/ut/v2_event_serializer_ut.cpp +++ b/ydb/library/actors/interconnect/ut/v2_event_serializer_ut.cpp @@ -64,19 +64,14 @@ void CheckIndexedEvent(IEventHandle& outEv, ui16 channel, size_t dataLen, bool w } } -std::vector<TContiguousSpan> Serialize(TEventSerializer& ser, std::vector<TRcBuf>& bufs) { - std::vector<TContiguousSpan> spans; +std::deque<TContiguousSpan> Serialize(TEventSerializer& ser, std::vector<TRcBuf>& bufs) { + std::deque<TContiguousSpan> spans; for (;;) { if (bufs.empty() || bufs.back().size() < 1024) { bufs.push_back(TRcBuf::Uninitialized(65536)); } - TRcBuf& buffer = bufs.back(); - - TMutableContiguousSpan bufferSpan = buffer.UnsafeGetContiguousSpanMut(); - const size_t produced = ser.ProduceOutputStream(&bufferSpan, &spans); - UNIT_ASSERT_EQUAL(bufferSpan.data() + bufferSpan.size(), buffer.data() + buffer.size()); - buffer.TrimFront(bufferSpan.size()); + const size_t produced = ser.ProduceOutputStream(bufs.back(), &spans); if (!produced) { break; } @@ -101,17 +96,17 @@ void CheckSerializeThenDeserialize(bool withPayload, bool buffer, ui32 metaLengt h->Cookie, nullptr, std::move(h->TraceId)); } - TEventSerializer ser; + TEventSerializer ser(true); ser.Push(std::move(h)); std::vector<TRcBuf> bufs; - std::vector<TContiguousSpan> spans = Serialize(ser, bufs); + std::deque<TContiguousSpan> spans = Serialize(ser, bufs); - TEventDeserializer deser; + TEventDeserializer deser(TScopeId{}); TEventProcessor processor; for (TContiguousSpan span : spans) { UNIT_ASSERT(processor.Events.empty()); - deser.Push(TRcBuf::Copy(span), &processor); + deser.Push(TRcBuf::Copy(span), &processor, {}); } UNIT_ASSERT(processor.Events.size() == 1); auto& outEv = *processor.Events.front(); @@ -164,6 +159,134 @@ Y_UNIT_TEST_SUITE(EventSerializerV2) { CheckSerializeThenDeserialize(true, true, 1000); } + // Feed the serialized stream to the deserializer in tiny fragments (1..3 bytes), so chunk headers and + // bodies straddle buffer boundaries exactly as they do with real TCP segmentation (which loopback, + // delivering big buffers, never triggers). Every event must still round-trip intact and in order. + Y_UNIT_TEST(DeserializeFromFragmentedStream) { + constexpr size_t numEvents = 300; + constexpr ui16 numChannels = 4; + + auto channelOf = [](ui64 i) -> ui16 { return i % numChannels; }; + auto dataLenOf = [](ui64 i) -> size_t { return (i % 11) * 37 + 1; }; + auto withPayloadOf = [](ui64 i) -> bool { return i % 2; }; + + TEventSerializer ser(false); + for (ui64 i = 0; i < numEvents; ++i) { + ser.Push(MakeIndexedEvent(channelOf(i), i, dataLenOf(i), withPayloadOf(i))); + } + + std::vector<TRcBuf> bufs; + std::deque<TContiguousSpan> spans = Serialize(ser, bufs); + + TString stream; + for (const TContiguousSpan& s : spans) { + stream.append(s.data(), s.size()); + } + + TEventDeserializer deser(TScopeId{}); + TEventProcessor processor; + std::vector<std::pair<ui16, ui64>> delivered; + + size_t pos = 0; + while (pos < stream.size()) { + const size_t frag = Min<size_t>(1 + (pos % 3), stream.size() - pos); + deser.Push(TRcBuf::Copy(TContiguousSpan(stream.data() + pos, frag)), &processor, TActorId()); + pos += frag; + while (!processor.Events.empty()) { + auto ev = std::move(processor.Events.front()); + processor.Events.pop_front(); + const ui64 index = ev->Cookie; + CheckIndexedEvent(*ev, channelOf(index), dataLenOf(index), withPayloadOf(index)); + delivered.push_back({ev->GetChannel(), index}); + } + } + + UNIT_ASSERT_VALUES_EQUAL(delivered.size(), numEvents); + std::array<ui64, numChannels> nextPerChannel{}; + for (const auto& [channel, index] : delivered) { + UNIT_ASSERT_VALUES_EQUAL(channelOf(index), channel); + UNIT_ASSERT_VALUES_EQUAL(index, nextPerChannel[channel] * numChannels + channel); + ++nextPerChannel[channel]; + } + } + + // A large event on one channel interleaves with many small events on another, so their serialized bytes + // end up scattered across many pipelined batches. Acknowledging (committing) the batches one at a time + // must never release an event whose bytes still live in a not-yet-committed batch -- otherwise the + // aliased payload memory is freed while a later batch still references it. That is a use-after-free that + // only manifests once the network actually sends those later bytes (loopback sends inline, so it hides + // it). We surface it by reading every not-yet-committed batch's spans after each commit -- under ASAN a + // premature release fails here; it also corrupts the round-trip which we verify at the end. + Y_UNIT_TEST(CommitDoesNotReleaseEventsWithBytesStillInFlight) { + auto channelOf = [](ui64 i) -> ui16 { return i == 0 ? 1 : 2; }; + auto dataLenOf = [](ui64 i) -> size_t { return i == 0 ? 60000 : 1000; }; + + TEventSerializer ser(false); + constexpr ui64 numEvents = 41; + ser.Push(MakeIndexedEvent(channelOf(0), 0, dataLenOf(0), /*withPayload=*/true)); + for (ui64 i = 1; i < numEvents; ++i) { + ser.Push(MakeIndexedEvent(channelOf(i), i, dataLenOf(i), /*withPayload=*/true)); + } + + struct TBatch { + TRcBuf Scratch; + std::deque<TContiguousSpan> Spans; + size_t Bytes; + }; + std::vector<TBatch> batches; + for (;;) { + TRcBuf scratch = TRcBuf::Uninitialized(4096); // small scratch -> many pipelined batches + std::deque<TContiguousSpan> spans; + size_t total = 0; + for (;;) { + const size_t produced = ser.ProduceOutputStream(scratch, &spans); + total += produced; + if (produced == 0 || scratch.size() < sizeof(TEventSerializer::TChunkHeader) + sizeof(ui32)) { + break; + } + } + if (total == 0) { + break; + } + batches.push_back({std::move(scratch), std::move(spans), total}); + } + + // gather the full stream up front (all spans still reference live memory here) for a round-trip check + TString stream; + for (const TBatch& b : batches) { + for (const TContiguousSpan& s : b.Spans) { + stream.append(s.data(), s.size()); + } + } + + // commit the batches one at a time; after each commit, touch every span that has not been committed + // yet -- these must all still point to live memory + volatile ui64 acc = 0; + for (size_t i = 0; i < batches.size(); ++i) { + ser.CommitProducedBytes(batches[i].Bytes); + for (size_t j = i + 1; j < batches.size(); ++j) { + for (const TContiguousSpan& s : batches[j].Spans) { + for (size_t k = 0; k < s.size(); ++k) { + acc += static_cast<ui8>(s.data()[k]); + } + } + } + } + Y_UNUSED(acc); + + // the stream captured before any commit must still round-trip to the original events + TEventDeserializer deser(TScopeId{}); + TEventProcessor processor; + deser.Push(TRcBuf::Copy(TContiguousSpan(stream.data(), stream.size())), &processor, TActorId()); + UNIT_ASSERT_VALUES_EQUAL(processor.Events.size(), numEvents); + while (!processor.Events.empty()) { + auto ev = std::move(processor.Events.front()); + processor.Events.pop_front(); + const ui64 index = ev->Cookie; + CheckIndexedEvent(*ev, channelOf(index), dataLenOf(index), /*withPayload=*/true); + } + } + // Push a lot of events into one channel and only a few into another one, then serialize the whole thing into a // stream and deserialize it back. Verifies that: // * every event is delivered exactly once, in FIFO order within each channel; @@ -178,7 +301,7 @@ Y_UNIT_TEST_SUITE(EventSerializerV2) { constexpr size_t floodDataLen = 512; constexpr size_t otherDataLen = 8; - TEventSerializer ser; + TEventSerializer ser(true); for (ui64 i = 0; i < numFlood; ++i) { ser.Push(MakeIndexedEvent(floodChannel, i, floodDataLen, true /*withPayload*/)); } @@ -187,10 +310,10 @@ Y_UNIT_TEST_SUITE(EventSerializerV2) { } std::vector<TRcBuf> bufs; - std::vector<TContiguousSpan> spans = Serialize(ser, bufs); + std::deque<TContiguousSpan> spans = Serialize(ser, bufs); // feed the produced stream to the deserializer chunk by chunk and record the exact delivery order - TEventDeserializer deser; + TEventDeserializer deser(TScopeId{}); TEventProcessor processor; struct TDelivered { @@ -200,7 +323,7 @@ Y_UNIT_TEST_SUITE(EventSerializerV2) { std::vector<TDelivered> delivered; for (TContiguousSpan span : spans) { - deser.Push(TRcBuf::Copy(span), &processor); + deser.Push(TRcBuf::Copy(span), &processor, {}); while (!processor.Events.empty()) { auto ev = std::move(processor.Events.front()); processor.Events.pop_front(); diff --git a/ydb/library/actors/interconnect/ut/v2_session_ut.cpp b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp new file mode 100644 index 00000000000..8d93cc19d46 --- /dev/null +++ b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp @@ -0,0 +1,863 @@ +#include <ydb/library/actors/interconnect/ut/lib/ic_test_cluster.h> +#include <ydb/library/actors/interconnect/ut/lib/test_events.h> +#include <ydb/library/actors/interconnect/interconnect_direct_session.h> +#include <ydb/library/actors/interconnect/uring_context.h> + +#include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/core/interconnect.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/threading/future/future.h> + +#include <util/generic/algorithm.h> +#include <util/generic/scope.h> +#include <util/system/env.h> +#include <util/system/mutex.h> + +#include <atomic> + +using namespace NActors; + +namespace { + + // Echo actor on the peer node: replies to every TEvTest with a TEvTestResponse addressed back to + // the sender, echoing the sequence number. + class TEchoActor: public TActor<TEchoActor> { + public: + TEchoActor() + : TActor(&TThis::StateFunc) + {} + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvTest, Handle) + ) + + void Handle(TEvTest::TPtr& ev) { + Send(ev->Sender, new TEvTestResponse(ev->Get()->Record.GetSequenceNumber()), 0, ev->Cookie); + } + }; + + // Collects TEvTestResponse delivered through the normal actor system. + class TResponseCollectorActor: public TActor<TResponseCollectorActor> { + public: + TResponseCollectorActor() + : TActor(&TThis::StateFunc) + {} + + size_t GetCount() const { + return Count.load(std::memory_order_acquire); + } + + TVector<ui64> GetReceived() const { + TGuard<TMutex> guard(Lock); + return Received; + } + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvTestResponse, Handle) + ) + + void Handle(TEvTestResponse::TPtr& ev) { + { + TGuard<TMutex> guard(Lock); + Received.push_back(ev->Get()->Record.GetConfirmedSequenceNumber()); + } + Count.fetch_add(1, std::memory_order_release); + } + + mutable TMutex Lock; + TVector<ui64> Received; + std::atomic<size_t> Count = 0; + }; + + // Collects incoming TEvTest payloads (used to verify large/multi-chunk transfers end to end). + class TPayloadCollectorActor: public TActor<TPayloadCollectorActor> { + public: + TPayloadCollectorActor() + : TActor(&TThis::StateFunc) + {} + + size_t GetCount() const { + return Count.load(std::memory_order_acquire); + } + + TString GetLastPayload() const { + TGuard<TMutex> guard(Lock); + return LastPayload; + } + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvTest, Handle) + ) + + void Handle(TEvTest::TPtr& ev) { + { + TGuard<TMutex> guard(Lock); + LastPayload = ev->Get()->Record.GetPayload(); + } + Count.fetch_add(1, std::memory_order_release); + } + + mutable TMutex Lock; + TString LastPayload; + std::atomic<size_t> Count = 0; + }; + + // Direct-session receive callback recording the sequence numbers of the replies it receives. + class TCollectingReceiveCallback: public IReceiveCallback { + public: + void Receive(TAutoPtr<IEventHandle> ev) override { + ui64 seq = Max<ui64>(); + if (ev->GetTypeRewrite() == TEvTestResponse::EventType) { + seq = ev->Get<TEvTestResponse>()->Record.GetConfirmedSequenceNumber(); + } + { + TGuard<TMutex> guard(Lock); + Received.push_back(seq); + } + Count.fetch_add(1, std::memory_order_release); + } + + size_t GetCount() const { + return Count.load(std::memory_order_acquire); + } + + TVector<ui64> GetReceived() const { + TGuard<TMutex> guard(Lock); + return Received; + } + + private: + mutable TMutex Lock; + TVector<ui64> Received; + std::atomic<size_t> Count = 0; + }; + + // Echoes each TEvTest back to its sender as a TEvTestResponse, preserving channel and cookie. + class TLoadEchoActor: public TActor<TLoadEchoActor> { + public: + TLoadEchoActor() + : TActor(&TThis::StateFunc) + {} + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvTest, Handle) + ) + + void Handle(TEvTest::TPtr& ev) { + Send(ev->Sender, new TEvTestResponse(ev->Get()->Record.GetSequenceNumber()), + IEventHandle::MakeFlags(ev->GetChannel(), 0), ev->Cookie); + } + }; + + // Mimics the interconnect load test: keeps up to InFlyMax messages (each carrying a rope payload on + // a given channel) in flight and only generates more as responses return; fulfils the promise once + // Total responses have been received. + class TLoadDriverActor: public TActorBootstrapped<TLoadDriverActor> { + public: + TLoadDriverActor(const TActorId& responder, ui32 total, ui32 inFlyMax, ui32 payloadSize, ui16 channel, + NThreading::TPromise<ui32> done) + : Responder(responder) + , Total(total) + , InFlyMax(inFlyMax) + , PayloadSize(payloadSize) + , Channel(channel) + , Done(std::move(done)) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + Generate(); + } + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvTestResponse, Handle) + hFunc(TEvents::TEvUndelivered, HandleUndelivered) + ) + + void SendOne(ui64 seq) { + auto ev = MakeHolder<TEvTest>(seq); + if (PayloadSize) { + TString payload = TString::Uninitialized(PayloadSize); + memset(payload.Detach(), '*', payload.size()); + ev->AddPayload(TRope(std::move(payload))); + } + Send(Responder, ev.Release(), IEventHandle::MakeFlags(Channel, 0) | IEventHandle::FlagTrackDelivery, seq); + ++InFly; + } + + // A tracked event may bounce back as undelivered while the interconnect connection is still being + // (re)established (e.g. handshake races at startup); just retry it -- this is how a real client uses + // FlagTrackDelivery. + void HandleUndelivered(TEvents::TEvUndelivered::TPtr& ev) { + --InFly; + SendOne(ev->Cookie); + } + + void Generate() { + while (InFly < InFlyMax && Sent < Total) { + SendOne(Sent); + ++Sent; + } + } + + void Handle(TEvTestResponse::TPtr& ev) { + // Get() parses the response -> detects receive-side corruption on the driver's connection. + const ui64 seq = ev->Get()->Record.GetConfirmedSequenceNumber(); + Y_ABORT_UNLESS(seq < Total, "corrupted response: seq# %" PRIu64 " total# %" PRIu32, seq, Total); + --InFly; + ++Received; + if (Received >= Total) { + if (!Finished) { + Finished = true; + Done.SetValue(Received); + } + } else { + Generate(); + } + } + + const TActorId Responder; + const ui32 Total; + const ui32 InFlyMax; + const ui32 PayloadSize; + const ui16 Channel; + NThreading::TPromise<ui32> Done; + ui32 Sent = 0; + ui32 Received = 0; + ui32 InFly = 0; + bool Finished = false; + }; + + class TDirectSessionGrabber: public TActorBootstrapped<TDirectSessionGrabber> { + public: + TDirectSessionGrabber(ui32 peerNodeId, NThreading::TPromise<std::shared_ptr<IDirectSession>> promise) + : PeerNodeId(peerNodeId) + , Promise(std::move(promise)) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + Send(TActivationContext::ActorSystem()->InterconnectProxy(PeerNodeId), new TEvInterconnect::TEvConnectNode); + } + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvInterconnect::TEvNodeConnected, Handle) + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, HandleDisconnected) + ) + + void Handle(TEvInterconnect::TEvNodeConnected::TPtr& ev) { + if (!Resolved) { + Resolved = true; + Promise.SetValue(ev->Get()->DirectSession); + } + } + + void HandleDisconnected() {} + + const ui32 PeerNodeId; + NThreading::TPromise<std::shared_ptr<IDirectSession>> Promise; + bool Resolved = false; + }; + + // Subscribes to a peer's connection state and counts connect/disconnect notifications, re-subscribing + // after every disconnect so a whole connect/disconnect/reconnect churn is observable. + class TConnectionMonitorActor: public TActorBootstrapped<TConnectionMonitorActor> { + public: + explicit TConnectionMonitorActor(ui32 peerNodeId) + : PeerNodeId(peerNodeId) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + Subscribe(); + } + + size_t Connects() const { return ConnectCount.load(std::memory_order_acquire); } + size_t Disconnects() const { return DisconnectCount.load(std::memory_order_acquire); } + + private: + STRICT_STFUNC(StateFunc, + hFunc(TEvInterconnect::TEvNodeConnected, Handle) + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, HandleDisconnected) + ) + + void Subscribe() { + Send(TActivationContext::ActorSystem()->InterconnectProxy(PeerNodeId), new TEvInterconnect::TEvConnectNode); + } + + void Handle(TEvInterconnect::TEvNodeConnected::TPtr&) { + ConnectCount.fetch_add(1, std::memory_order_release); + } + + void HandleDisconnected() { + DisconnectCount.fetch_add(1, std::memory_order_release); + Subscribe(); // re-arm so the next connection is observed too + } + + const ui32 PeerNodeId; + std::atomic<size_t> ConnectCount{0}; + std::atomic<size_t> DisconnectCount{0}; + }; + + std::unique_ptr<TTestICCluster> MakeV2Cluster(ui32 tcpSocketBufferSize = 0) { + auto customizer = [tcpSocketBufferSize](ui32, TInterconnectSettings& settings) { + settings.EnableInterconnectSessionV2 = true; + settings.ChecksumInterconnectSessionV2 = true; + if (tcpSocketBufferSize) { + settings.TCPSocketBufferSize = tcpSocketBufferSize; + } + }; + return std::make_unique<TTestICCluster>( + /*numNodes=*/2, TChannelsConfig(), /*tiSettings=*/nullptr, /*loggerSettings=*/nullptr, + TTestICCluster::EMPTY, /*checkerFactory=*/TTestICCluster::TCheckerFactory{}, + /*deadPeerTimeout=*/TDuration::Seconds(2), /*inflight=*/TNode::DefaultInflight(), customizer); + } + + std::shared_ptr<IDirectSession> GrabDirectSession(TTestICCluster& cluster, ui32 fromNode, ui32 peerNode) { + auto promise = NThreading::NewPromise<std::shared_ptr<IDirectSession>>(); + auto future = promise.GetFuture(); + cluster.RegisterActor(new TDirectSessionGrabber(peerNode, promise), fromNode); + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(10)), "timed out waiting for TEvNodeConnected"); + return future.GetValueSync(); + } + + template <typename TCallback> + void WaitFor(TDuration timeout, TCallback&& cb, TStringBuf what) { + const TInstant deadline = TInstant::Now() + timeout; + while (TInstant::Now() < deadline) { + if (cb()) { + return; + } + Sleep(TDuration::MilliSeconds(10)); + } + UNIT_FAIL(TStringBuilder() << "condition not met: " << what); + } + + void AssertV2InUse(TTestICCluster& cluster, ui32 me, ui32 peer) { + WaitFor(TDuration::Seconds(10), [&] { + auto httpResp = cluster.GetSessionDbg(me, peer); + if (!httpResp.Wait(TDuration::Seconds(2))) { + return false; + } + return httpResp.GetValueSync().Contains("Session (v2)"); + }, "v2 session is in use"); + } + +} // namespace + +Y_UNIT_TEST_SUITE(InterconnectSessionV2) { + + // Normal actor-system traffic must round-trip over a v2 session. + Y_UNIT_TEST(ActorSystemRoundTrip) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId echoId = cluster->RegisterActor(new TEchoActor, 2); + + auto* collector = new TResponseCollectorActor; + const TActorId collectorId = cluster->RegisterActor(collector, 1); + + constexpr ui64 N = 32; + for (ui64 i = 0; i < N; ++i) { + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(echoId, collectorId, new TEvTest(i))); + } + + WaitFor(TDuration::Seconds(20), [&] { return collector->GetCount() >= N; }, + "all actor-system replies received over v2"); + + auto received = collector->GetReceived(); + Sort(received); + UNIT_ASSERT_VALUES_EQUAL(received.size(), N); + for (ui64 i = 0; i < N; ++i) { + UNIT_ASSERT_VALUES_EQUAL(received[i], i); + } + + AssertV2InUse(*cluster, 1, 2); + AssertV2InUse(*cluster, 2, 1); + } + + // IDirectSession Send + registered receive callback must round-trip over a v2 session. + Y_UNIT_TEST(DirectSessionRoundTrip) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId echoId = cluster->RegisterActor(new TEchoActor, 2); + auto directSession = GrabDirectSession(*cluster, 1, 2); + UNIT_ASSERT_C(directSession, "v2 session did not provide a direct session handle"); + + const TActorId localId(1, 0, 0xD1EC7A, 0); + auto callback = MakeIntrusive<TCollectingReceiveCallback>(); + directSession->RegisterReceiveCallback(localId, callback); + + constexpr ui64 N = 32; + for (ui64 i = 0; i < N; ++i) { + UNIT_ASSERT(directSession->Send(new IEventHandle(echoId, localId, new TEvTest(i)))); + } + + WaitFor(TDuration::Seconds(20), [&] { return callback->GetCount() >= N; }, + "all direct-session replies received over v2"); + + auto received = callback->GetReceived(); + Sort(received); + UNIT_ASSERT_VALUES_EQUAL(received.size(), N); + for (ui64 i = 0; i < N; ++i) { + UNIT_ASSERT_VALUES_EQUAL(received[i], i); + } + } + + // A large payload must survive chunk splitting/reassembly across the v2 wire format. + Y_UNIT_TEST(LargePayloadRoundTrip) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + auto* collector = new TPayloadCollectorActor; + const TActorId collectorId = cluster->RegisterActor(collector, 2); + + // well over the 16383-byte per-chunk limit to force multi-chunk splitting + TString payload = TString::Uninitialized(200000); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast<char>('a' + (i % 26)); + } + + const TActorId sender(1, 0, 0xBEEF, 0); + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(collectorId, sender, new TEvTest(1, payload))); + + WaitFor(TDuration::Seconds(20), [&] { return collector->GetCount() >= 1; }, + "large payload event received over v2"); + + UNIT_ASSERT_VALUES_EQUAL(collector->GetLastPayload().size(), payload.size()); + UNIT_ASSERT_EQUAL(collector->GetLastPayload(), payload); + } + + // Reproduces the interconnect load test pattern: bounded in-flight window, rope payloads on a + // non-default channel, sustained bidirectional traffic. A stall here manifests as a timeout. + Y_UNIT_TEST(LoadLikeRoundTripWithPayload) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + constexpr ui32 kTotal = 20000; + cluster->RegisterActor(new TLoadDriverActor(responder, kTotal, /*inFlyMax=*/16, /*payloadSize=*/4096, + /*channel=*/1, promise), 1); + + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), "load-like round trip stalled"); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), kTotal); + AssertV2InUse(*cluster, 1, 2); + } + + // Several channels active at once (a small-inline "system" channel alongside large rope-payload + // channels), mimicking real traffic where a distconf event shares the v2 session with the load. The + // echo actor parses every event via Get(), so any framing/interleaving corruption throws. + Y_UNIT_TEST(LoadLikeRoundTripMultiChannel) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + constexpr ui32 kChannels = 6; + constexpr ui32 kPerChannel = 5000; + std::vector<NThreading::TFuture<ui32>> futures; + for (ui16 ch = 0; ch < kChannels; ++ch) { + auto promise = NThreading::NewPromise<ui32>(); + futures.push_back(promise.GetFuture()); + const ui32 payload = (ch % 3 == 0) ? 40 : (ch % 3 == 1 ? 4096 : 20000); + cluster->RegisterActor(new TLoadDriverActor(responder, kPerChannel, /*inFlyMax=*/8, payload, + /*channel=*/ch, promise), 1); + } + for (ui16 ch = 0; ch < kChannels; ++ch) { + UNIT_ASSERT_C(futures[ch].Wait(TDuration::Seconds(90)), "multi-channel load stalled"); + UNIT_ASSERT_VALUES_EQUAL(futures[ch].GetValueSync(), kPerChannel); + } + } + + // Many connections forced into a single engine shard (one shared buffer ring / reaper), which is how + // a real multi-node cluster looks but a 2-node test never does. Each node drives payload load to every + // other node; corruption on any connection surfaces as a parse failure in Get(). + Y_UNIT_TEST(LoadLikeRoundTripSharedShardManyPeers) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + SetEnv("YDB_IC_V2_SHARDS", "1"); // force all connections onto one shard/buffer-ring/reaper + Y_DEFER { + UnsetEnv("YDB_IC_V2_SHARDS"); + }; + + constexpr ui32 kNodes = 5; + auto customizer = [](ui32, TInterconnectSettings& settings) { + settings.EnableInterconnectSessionV2 = true; + }; + auto cluster = std::make_unique<TTestICCluster>( + kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, + TTestICCluster::TCheckerFactory{}, TDuration::Seconds(2), TNode::DefaultInflight(), customizer); + + // an echo responder on every node + std::vector<TActorId> responders(kNodes + 1); + for (ui32 n = 1; n <= kNodes; ++n) { + responders[n] = cluster->RegisterActor(new TLoadEchoActor, n); + } + + // every node drives payload load to every other node concurrently + constexpr ui32 kPerPair = 500; + std::vector<NThreading::TFuture<ui32>> futures; + for (ui32 from = 1; from <= kNodes; ++from) { + for (ui32 to = 1; to <= kNodes; ++to) { + if (from == to) { + continue; + } + auto promise = NThreading::NewPromise<ui32>(); + futures.push_back(promise.GetFuture()); + const ui32 payload = ((from + to) % 2) ? 8192 : 100; + cluster->RegisterActor(new TLoadDriverActor(responders[to], kPerPair, /*inFlyMax=*/8, payload, + /*channel=*/static_cast<ui16>(from % 4), promise), from); + } + } + for (auto& f : futures) { + UNIT_ASSERT_C(f.Wait(TDuration::Seconds(30)), "shared-shard multi-peer load stalled"); + UNIT_ASSERT_VALUES_EQUAL(f.GetValueSync(), kPerPair); + } + } + + // Same as above but over the ordinary-buffer fallback recv path (no buffer ring) -- exactly what a + // kernel-5.15 dev cluster uses. Reproduces multi-connection + fallback + heavy load together. + Y_UNIT_TEST(LoadLikeRoundTripSharedShardFallbackManyPeers) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + SetEnv("YDB_IC_V2_SHARDS", "1"); + SetEnv("YDB_IC_V2_DISABLE_BUF_RING", "1"); + Y_DEFER { + UnsetEnv("YDB_IC_V2_SHARDS"); + UnsetEnv("YDB_IC_V2_DISABLE_BUF_RING"); + }; + + constexpr ui32 kNodes = 5; + auto customizer = [](ui32, TInterconnectSettings& settings) { + settings.EnableInterconnectSessionV2 = true; + }; + auto cluster = std::make_unique<TTestICCluster>( + kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, + TTestICCluster::TCheckerFactory{}, TDuration::Seconds(2), TNode::DefaultInflight(), customizer); + + std::vector<TActorId> responders(kNodes + 1); + for (ui32 n = 1; n <= kNodes; ++n) { + responders[n] = cluster->RegisterActor(new TLoadEchoActor, n); + } + + constexpr ui32 kPerPair = 500; + std::vector<NThreading::TFuture<ui32>> futures; + for (ui32 from = 1; from <= kNodes; ++from) { + for (ui32 to = 1; to <= kNodes; ++to) { + if (from == to) { + continue; + } + auto promise = NThreading::NewPromise<ui32>(); + futures.push_back(promise.GetFuture()); + const ui32 payload = ((from + to) % 2) ? 8192 : 100; + cluster->RegisterActor(new TLoadDriverActor(responders[to], kPerPair, /*inFlyMax=*/8, payload, + /*channel=*/static_cast<ui16>(from % 4), promise), from); + } + } + for (auto& f : futures) { + UNIT_ASSERT_C(f.Wait(TDuration::Seconds(60)), "shared-shard fallback multi-peer load stalled"); + UNIT_ASSERT_VALUES_EQUAL(f.GetValueSync(), kPerPair); + } + } + + // Load-like pattern over the ordinary-buffer fallback path (kernels without buffer rings, e.g. 5.15), + // which is what a real dev cluster is likely using. + Y_UNIT_TEST(LoadLikeRoundTripFallback) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + SetEnv("YDB_IC_V2_DISABLE_BUF_RING", "1"); + Y_DEFER { + UnsetEnv("YDB_IC_V2_DISABLE_BUF_RING"); + }; + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + constexpr ui32 kTotal = 20000; + cluster->RegisterActor(new TLoadDriverActor(responder, kTotal, /*inFlyMax=*/16, /*payloadSize=*/4096, + /*channel=*/1, promise), 1); + + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), "load-like round trip stalled on fallback path"); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), kTotal); + } + + // Same load-like pattern but with a tiny TCP socket buffer to force backpressure -- frequent partial + // writes and short reads, which loopback with default buffers never triggers. + Y_UNIT_TEST(LoadLikeRoundTripWithBackpressure) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(/*tcpSocketBufferSize=*/8192); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + constexpr ui32 kTotal = 20000; + cluster->RegisterActor(new TLoadDriverActor(responder, kTotal, /*inFlyMax=*/16, /*payloadSize=*/16384, + /*channel=*/1, promise), 1); + + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), "load-like round trip stalled under backpressure"); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), kTotal); + } + + // Force the ordinary-buffer fallback (kernels without the provided-buffer-ring API) and make sure + // both the actor-system path and a large multi-recv payload still round-trip over v2. + Y_UNIT_TEST(FallbackWithoutBufRing) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + SetEnv("YDB_IC_V2_DISABLE_BUF_RING", "1"); + Y_DEFER { + UnsetEnv("YDB_IC_V2_DISABLE_BUF_RING"); + }; + + auto cluster = MakeV2Cluster(); + const TActorId echoId = cluster->RegisterActor(new TEchoActor, 2); + + auto* collector = new TResponseCollectorActor; + const TActorId collectorId = cluster->RegisterActor(collector, 1); + + constexpr ui64 N = 16; + for (ui64 i = 0; i < N; ++i) { + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(echoId, collectorId, new TEvTest(i))); + } + WaitFor(TDuration::Seconds(20), [&] { return collector->GetCount() >= N; }, + "actor-system replies received over v2 fallback path"); + + auto received = collector->GetReceived(); + Sort(received); + UNIT_ASSERT_VALUES_EQUAL(received.size(), N); + for (ui64 i = 0; i < N; ++i) { + UNIT_ASSERT_VALUES_EQUAL(received[i], i); + } + + AssertV2InUse(*cluster, 1, 2); + + // payload well above the 64 KB fallback recv buffer, so reassembly must span several recvs + auto* payloadCollector = new TPayloadCollectorActor; + const TActorId pcId = cluster->RegisterActor(payloadCollector, 2); + TString payload = TString::Uninitialized(300000); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast<char>('a' + (i % 26)); + } + const TActorId sender(1, 0, 0xBEEF, 0); + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(pcId, sender, new TEvTest(1, payload))); + + WaitFor(TDuration::Seconds(20), [&] { return payloadCollector->GetCount() >= 1; }, + "large payload received over v2 fallback path"); + UNIT_ASSERT_VALUES_EQUAL(payloadCollector->GetLastPayload().size(), payload.size()); + UNIT_ASSERT_EQUAL(payloadCollector->GetLastPayload(), payload); + } + + // + // Teardown / disconnection coverage. + // + // These exercise the paths that run when a v2 session goes away: the engine must drain its in-flight + // io_uring read/write before freeing the per-connection state (otherwise a completion dereferences + // freed memory on the shard worker thread), the session actor must tear down cleanly, and a fresh + // session must be establishable afterwards. + // + + // A destroyed cluster (actor-system teardown) with idle-but-established v2 sessions in both directions + // must shut down cleanly: engine reaper threads joined, sessions freed, no crash or hang. + Y_UNIT_TEST(ActorSystemTeardownIdleSessions) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId echo1 = cluster->RegisterActor(new TEchoActor, 1); + const TActorId echo2 = cluster->RegisterActor(new TEchoActor, 2); + + // establish a v2 session in each direction + auto* c1 = new TResponseCollectorActor; + auto* c2 = new TResponseCollectorActor; + const TActorId c1Id = cluster->RegisterActor(c1, 1); + const TActorId c2Id = cluster->RegisterActor(c2, 2); + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(echo2, c1Id, new TEvTest(0))); + cluster->GetNode(2)->GetActorSystem()->Send(new IEventHandle(echo1, c2Id, new TEvTest(0))); + WaitFor(TDuration::Seconds(20), [&] { return c1->GetCount() >= 1 && c2->GetCount() >= 1; }, + "v2 sessions established both ways"); + AssertV2InUse(*cluster, 1, 2); + AssertV2InUse(*cluster, 2, 1); + + // teardown: the engine must be stopped (reaper threads joined) with the still-armed recv in flight + cluster.reset(); + } + + // Destroying the cluster while a load is actively flowing means the engine is torn down with reads and + // writes in flight and connections still registered. Must not crash or hang. + Y_UNIT_TEST(ActorSystemTeardownUnderLoad) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + // a large, essentially unbounded load so it is guaranteed still running at teardown + auto promise = NThreading::NewPromise<ui32>(); + cluster->RegisterActor(new TLoadDriverActor(responder, /*total=*/1'000'000, /*inFlyMax=*/32, + /*payloadSize=*/4096, /*channel=*/1, promise), 1); + + // let traffic actually start flowing (session up, io_uring ops in flight) + Sleep(TDuration::MilliSeconds(300)); + + // teardown mid-flight + cluster.reset(); + } + + // Stopping the peer node must make the local side observe a disconnect (EOF on the socket propagates + // through the engine to the session, which notifies subscribers). + Y_UNIT_TEST(PeerStopTriggersDisconnect) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId echoId = cluster->RegisterActor(new TEchoActor, 2); + + auto* monitor = new TConnectionMonitorActor(2); + cluster->RegisterActor(monitor, 1); + + // trigger the connection and wait until it is up + auto* collector = new TResponseCollectorActor; + const TActorId collectorId = cluster->RegisterActor(collector, 1); + cluster->GetNode(1)->GetActorSystem()->Send(new IEventHandle(echoId, collectorId, new TEvTest(0))); + WaitFor(TDuration::Seconds(20), [&] { return collector->GetCount() >= 1 && monitor->Connects() >= 1; }, + "v2 session established and observed by monitor"); + + cluster->StopNode(2); + + WaitFor(TDuration::Seconds(20), [&] { return monitor->Disconnects() >= 1; }, + "local side observes disconnect after peer stop"); + } + + // Forcibly terminating the session (TEvPoisonSession -> IInterconnectSession::Terminate, i.e. the + // synchronous Unregister-while-recv-armed path) must not crash, and a brand new session must round-trip + // afterwards. + Y_UNIT_TEST(ForcedSessionTeardownAndReconnect) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + auto runLoad = [&](ui32 total, TStringBuf what) { + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + cluster->RegisterActor(new TLoadDriverActor(responder, total, /*inFlyMax=*/16, /*payloadSize=*/4096, + /*channel=*/1, promise), 1); + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), what); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), total); + }; + + // phase 1: a session is created and used + runLoad(2000, "initial load stalled"); + AssertV2InUse(*cluster, 1, 2); + + // forcibly tear the session down + cluster->GetNode(1)->Send(cluster->InterconnectProxy(2, 1), new TEvInterconnect::TEvPoisonSession); + + // phase 2: a fresh session must be established transparently and carry the load + runLoad(2000, "load after forced teardown stalled -- reconnect broken"); + AssertV2InUse(*cluster, 1, 2); + } + + // Shutting the peer socket (TEvClosePeerSocket -> the engine sees EOF -> session self-terminates) must + // also recover with a new session. + Y_UNIT_TEST(ClosePeerSocketAndReconnect) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + + auto* monitor = new TConnectionMonitorActor(2); + cluster->RegisterActor(monitor, 1); + + auto runLoad = [&](ui32 total, TStringBuf what) { + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + cluster->RegisterActor(new TLoadDriverActor(responder, total, /*inFlyMax=*/16, /*payloadSize=*/4096, + /*channel=*/1, promise), 1); + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), what); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), total); + }; + + runLoad(2000, "initial load stalled"); + WaitFor(TDuration::Seconds(20), [&] { return monitor->Connects() >= 1; }, "connection observed"); + + cluster->GetNode(1)->Send(cluster->InterconnectProxy(2, 1), new TEvInterconnect::TEvClosePeerSocket); + WaitFor(TDuration::Seconds(20), [&] { return monitor->Disconnects() >= 1; }, + "disconnect observed after ClosePeerSocket"); + + runLoad(2000, "load after ClosePeerSocket stalled -- reconnect broken"); + } + + // Repeatedly forcing the session down while traffic is actively flowing is the strongest reproducer of + // the "free the connection while a read/write is in flight" teardown bug. After the churn the connection + // must still be fully usable. + Y_UNIT_TEST(RepeatedDisconnectChurnUnderLoad) { + if (!TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + auto cluster = MakeV2Cluster(); + const TActorId echoId = cluster->RegisterActor(new TEchoActor, 2); + auto* collector = new TResponseCollectorActor; + const TActorId collectorId = cluster->RegisterActor(collector, 1); + + // churn: keep some best-effort traffic in flight and repeatedly tear the session down mid-transfer + for (int round = 0; round < 15; ++round) { + for (ui64 i = 0; i < 200; ++i) { + cluster->GetNode(1)->GetActorSystem()->Send( + new IEventHandle(echoId, collectorId, new TEvTest(i))); + } + Sleep(TDuration::MilliSeconds(15)); + cluster->GetNode(1)->Send(cluster->InterconnectProxy(2, 1), new TEvInterconnect::TEvPoisonSession); + } + + // recovery: after all the churn a clean load must complete fully over a freshly established session + const TActorId responder = cluster->RegisterActor(new TLoadEchoActor, 2); + auto promise = NThreading::NewPromise<ui32>(); + auto future = promise.GetFuture(); + constexpr ui32 kTotal = 3000; + cluster->RegisterActor(new TLoadDriverActor(responder, kTotal, /*inFlyMax=*/16, /*payloadSize=*/4096, + /*channel=*/1, promise), 1); + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(60)), "connection did not recover after disconnect churn"); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), kTotal); + } +} diff --git a/ydb/library/actors/interconnect/ut/xdc_shuffle_ut.cpp b/ydb/library/actors/interconnect/ut/xdc_shuffle_ut.cpp index 136bf607e12..2793863ac13 100644 --- a/ydb/library/actors/interconnect/ut/xdc_shuffle_ut.cpp +++ b/ydb/library/actors/interconnect/ut/xdc_shuffle_ut.cpp @@ -42,7 +42,7 @@ Y_UNIT_TEST_SUITE(InterconnectXdcShuffle) { }; TCoroutineChunkSerializer chunker; - chunker.SetSerializingEvent(&event); + chunker.SetSerializingEvent(&event, true); TString headerBuffer(info.Sections[0].Size, '\0'); auto chunks = chunker.FeedBuf(headerBuffer.begin(), headerBuffer.size()); diff --git a/ydb/library/actors/interconnect/ut/ya.make b/ydb/library/actors/interconnect/ut/ya.make index 1c2c6c64889..04195440915 100644 --- a/ydb/library/actors/interconnect/ut/ya.make +++ b/ydb/library/actors/interconnect/ut/ya.make @@ -23,6 +23,7 @@ SRCS( #uring_ut.cpp xdc_shuffle_ut.cpp v2_event_serializer_ut.cpp + v2_session_ut.cpp ) PEERDIR( diff --git a/ydb/library/actors/interconnect/v2_event_serializer.cpp b/ydb/library/actors/interconnect/v2_event_serializer.cpp index c0dc49088b2..543ac0f2f20 100644 --- a/ydb/library/actors/interconnect/v2_event_serializer.cpp +++ b/ydb/library/actors/interconnect/v2_event_serializer.cpp @@ -1,54 +1,66 @@ #include "v2_event_serializer.h" +#include <util/stream/format.h> +#include <util/string/builder.h> +#include <util/string/hex.h> + namespace NActors { + TEventSerializer::TEventSerializer(bool checksumming) + : Checksumming(checksumming) + {} + void TEventSerializer::Push(std::unique_ptr<IEventHandle> ev) { const ui16 channel = ev->GetChannel(); TPerChannelQueue& queue = GetQueue(channel); const bool first = queue.Events.empty(); queue.Events.push_back(std::move(ev)); if (first) { - // calculate quota for new channel: either it is default quota if we have no active transmissions, or the - // greatest one amongst other channels - const ui16 quota = PerChannelQuotaHeap.empty() - ? DefaultQuota - : PerChannelQuotaHeap.front().Quota; - - // this new quota must be nonzero - Y_ABORT_UNLESS(quota); - // place this new quota into non-zero part of the heap PerChannelQuotaHeap.push_back(TPerChannelQuota{ .Channel = channel, - .Quota = quota, + .Quota = DefaultQuota, }); std::ranges::push_heap(PerChannelQuotaHeap, std::less<ui16>{}, &TPerChannelQuota::Quota); } } - size_t TEventSerializer::ProduceOutputStream(TMutableContiguousSpan *buffer, std::vector<TContiguousSpan> *out) { + size_t TEventSerializer::ProduceOutputStream(TRcBuf& buffer, std::deque<TContiguousSpan> *out, size_t maxBytesToProduce) { + Timer.Reset(); + size_t totalBytesProduced = 0; + ui64 bufferProduced = 0; + + Y_ABORT_UNLESS(buffer.size() >= TEventSerializer::MinUsefulQuota); + + const ui64 bytesProducedOnEntry = CumulativeProduced; // we can't emit anything useful once the output buffer can't hold at least a chunk header along with a whole // some useful data, so we stop here to avoid spinning without making any progress - while (!PerChannelQuotaHeap.empty() && buffer->size() >= MinUsefulQuota) { + while (!PerChannelQuotaHeap.empty()) { // if even the channel with the most quota can't emit a whole event header, replenish quota for every channel // back to the default value; this keeps the bandwidth distributed equally and guarantees that the channel we // are about to serve always has enough quota to make progress if (PerChannelQuotaHeap.front().Quota < MinUsefulQuota) { for (auto& item : PerChannelQuotaHeap) { - item.Quota = DefaultQuota; + item.Quota += DefaultQuota; } } // get the channel/quota pair for the channel with the most quota available TPerChannelQuota& q = PerChannelQuotaHeap.front(); + Y_DEBUG_ABORT_UNLESS(q.Quota <= TChunkHeader::LengthMask); // serialize part of data for this channel TPerChannelQueue& queue = GetQueue(q.Channel); - const size_t numBytesProduced = ProduceOutputStreamForQueue(queue, q.Quota, buffer, out); + const size_t numBytesProduced = ProduceOutputStreamForQueue(queue, Min<size_t>(maxBytesToProduce, q.Quota), + buffer, out, &bufferProduced); + if (!numBytesProduced) { // in case we did not make any progress (not enough space in buffer) + break; + } Y_ABORT_UNLESS(numBytesProduced <= q.Quota); totalBytesProduced += numBytesProduced; + maxBytesToProduce -= numBytesProduced; // update quota std::ranges::pop_heap(PerChannelQuotaHeap, std::less<ui16>{}, &TPerChannelQuota::Quota); @@ -59,52 +71,58 @@ namespace NActors { // adjust quota PerChannelQuotaHeap.back().Quota -= numBytesProduced; std::ranges::push_heap(PerChannelQuotaHeap, std::less<ui16>{}, &TPerChannelQuota::Quota); - Y_ABORT_UNLESS(numBytesProduced); // ensure we had progress } } + Y_DEBUG_ABORT_UNLESS(bytesProducedOnEntry + totalBytesProduced == CumulativeProduced); + + if (bufferProduced) { + RefcountItems.push_back({ + .EndOffset = bufferProduced, + .Scratch = buffer, + }); + } + +// OtherTime += Timer.PassedReset() * 1e9; return totalBytesProduced; } void TEventSerializer::CommitProducedBytes(size_t numBytes) { - while (numBytes) { - if (RefcountItems.empty()) { - OverproducedBytes += numBytes; - break; - } - - TRefcountItem& item = RefcountItems.front(); - if (numBytes < item.NumBytesRemaining) { - item.NumBytesRemaining -= numBytes; - break; - } else { - numBytes -= item.NumBytesRemaining; - RefcountItems.pop_front(); - } + CumulativeCommitted += numBytes; + Y_ABORT_UNLESS(CumulativeCommitted <= CumulativeProduced); + while (!RefcountItems.empty() && RefcountItems.front().EndOffset <= CumulativeCommitted) { + RefcountItems.pop_front(); } } - size_t TEventSerializer::ProduceOutputStreamForQueue(TPerChannelQueue& queue, size_t maxBytesToProduce, - TMutableContiguousSpan *buffer, std::vector<TContiguousSpan> *out) { - const TContiguousSpan bufferSpan = *buffer; // remember original buffer span + size_t TEventSerializer::ProduceOutputStreamForQueue(TPerChannelQueue& queue, size_t maxBytesToProduce, TRcBuf& buffer, + std::deque<TContiguousSpan> *out, ui64 *bufferProduced) { + const TContiguousSpan bufferSpan = buffer.GetContiguousSpan(); // remember original buffer span size_t numBytesProduced = 0; // this function is used to generate output span storing reference either to buffer, or to aliased memory range - auto produceOutputSpan = [&](TContiguousSpan span) { + auto produceOutputSpan = [&](TContiguousSpan span, bool addToChecksum) { + if (addToChecksum) { + XXH3_64bits_update(&queue.ChecksumState, span.data(), span.size()); + } + if (span.data() + span.size() <= bufferSpan.data() || span.data() >= bufferSpan.data() + bufferSpan.size()) { // we got span referenced outside original buffer; check if we can copy it into the buffer, if it is // small enough and buffer has the space to do it - if (span.size() <= 64 && buffer->size() >= span.size()) { - memcpy(buffer->data(), span.data(), span.size()); - span = {buffer->data(), span.size()}; - *buffer = buffer->SubSpan(span.size(), Max<size_t>()); + const uintptr_t spanBegin = reinterpret_cast<uintptr_t>(span.data()); + const uintptr_t spanEnd = reinterpret_cast<uintptr_t>(span.data() + span.size() - 1); + const uintptr_t mask = ~uintptr_t(63); // check if it fits the same cacheline + if (buffer.size() >= span.size() && (spanBegin & mask) == (spanEnd & mask)) { + memcpy(buffer.UnsafeGetDataMut(), span.data(), span.size()); + span = {buffer.data(), span.size()}; + buffer.TrimFront(buffer.size() - span.size()); } } Y_ABORT_UNLESS(span.size() <= maxBytesToProduce); maxBytesToProduce -= span.size(); numBytesProduced += span.size(); - queue.EventProducedSize += span.size(); + CumulativeProduced += span.size(); if (out->empty()) { out->push_back(span); } else if (TContiguousSpan& last = out->back(); last.data() + last.size() != span.data()) { @@ -112,41 +130,43 @@ namespace NActors { } else { // concatenate last span with the new one last = {last.data(), last.size() + span.size()}; } + + if (span.data() + span.size() <= bufferSpan.data() || span.data() >= bufferSpan.data() + bufferSpan.size()) { + BytesAliased += span.size(); + } else { + BytesCopied += span.size(); + } }; // this function allocated specified amount of space in provided buffer and returns reference to it, also // producing output span with allocated data auto takeInBuffer = [&](size_t numBytes) -> void* { Y_ABORT_UNLESS(numBytes <= maxBytesToProduce); - Y_ABORT_UNLESS(numBytes <= buffer->size()); - TMutableContiguousSpan res = buffer->SubSpan(0, numBytes); - *buffer = buffer->SubSpan(numBytes, Max<size_t>()); - produceOutputSpan(res); + Y_ABORT_UNLESS(numBytes <= buffer.size()); + TMutableContiguousSpan res(buffer.UnsafeGetDataMut(), numBytes); + buffer.TrimFront(buffer.size() - numBytes); + produceOutputSpan(res, false); + *bufferProduced = CumulativeProduced; return res.data(); }; - while (maxBytesToProduce && !queue.Events.empty()) { + while (Min(buffer.size(), maxBytesToProduce) >= MinUsefulQuota && !queue.Events.empty()) { IEventHandle& ev = *queue.Events.front(); - // prepare chunk header depending on the state - TChunkHeader *chunkHeader = nullptr; - if (queue.SerializeStage != ESerializeStage::kInitial) { - if (buffer->size() < sizeof(TChunkHeader) || maxBytesToProduce < MinUsefulQuota) { - break; // not even a chance to put something useful + TChunkHeader *header = nullptr; + auto addEventChunkBytes = [&](const char *ptr, size_t numBytes) { + Y_DEBUG_ABORT_UNLESS(numBytes); + if (!header) { + header = static_cast<TChunkHeader*>(takeInBuffer(sizeof(TChunkHeader))); + *header = { + .TypeLength = TChunkHeader::kEventChunk, + .Channel = ev.GetChannel(), + }; } - - // allocate chunk header and fill it in - chunkHeader = static_cast<TChunkHeader*>(takeInBuffer(sizeof(TChunkHeader))); - *chunkHeader = { - .TypeLength = - queue.SerializeStage == ESerializeStage::kHeader - ? TChunkHeader::kEventHeader - : TChunkHeader::kEventChunk, - .Channel = ev.GetChannel(), - }; - } - - bool quit = false; + Y_DEBUG_ABORT_UNLESS(header->GetLength() + numBytes <= TChunkHeader::LengthMask); + header->TypeLength += numBytes; + produceOutputSpan({ptr, numBytes}, Checksumming); + }; switch (queue.SerializeStage) { case ESerializeStage::kInitial: @@ -159,7 +179,7 @@ namespace NActors { } else if (ev.HasEvent()) { IEventBase *event = ev.GetBase(); queue.SerializeStage = ESerializeStage::kChunkSerializer; - queue.CoroutineChunkSerializer.SetSerializingEvent(event); + queue.CoroutineChunkSerializer.SetSerializingEvent(event, /*withCachedSizes=*/ false); queue.EvSerInfoHolder = event->CreateSerializationInfo(true); queue.EvSerInfo = &queue.EvSerInfoHolder; } else { @@ -167,12 +187,16 @@ namespace NActors { queue.EvSerInfoHolder = {}; queue.EvSerInfo = &queue.EvSerInfoHolder; } + if (Checksumming) { + XXH3_64bits_reset(&queue.ChecksumState); + } // fill in event header queue.EventHeader = { .Type = ev.Type, .Flags = ev.Flags | (queue.EvSerInfo->IsExtendedFormat ? IEventHandle::FlagExtendedFormat : 0), .Cookie = ev.Cookie, + .Checksum = 0, .Sender = ev.Sender, .Recipient = ev.Recipient, }; @@ -180,48 +204,62 @@ namespace NActors { break; case ESerializeStage::kBufferSerializer: +// OtherTime += Timer.PassedReset() * 1e9; + while (maxBytesToProduce && queue.Iter.Valid()) { - const size_t numBytes = Min(maxBytesToProduce, queue.Iter.ContiguousSize()); - produceOutputSpan(TContiguousSpan(queue.Iter.ContiguousData(), numBytes)); - chunkHeader->TypeLength += numBytes; + const size_t numBytes = Min(maxBytesToProduce - (header ? 0 : sizeof(TChunkHeader)), + queue.Iter.ContiguousSize()); + addEventChunkBytes(queue.Iter.ContiguousData(), numBytes); queue.Iter += numBytes; } if (!queue.Iter.Valid()) { queue.SerializeStage = ESerializeStage::kHeader; } + +// SerializeBufferTime += Timer.PassedReset() * 1e9; break; case ESerializeStage::kChunkSerializer: { +// OtherTime += Timer.PassedReset() * 1e9; + // serialize as much as we can - bool progress = false; - for (const auto [data, size] : queue.CoroutineChunkSerializer.FeedBuf(buffer, maxBytesToProduce)) { - produceOutputSpan(TContiguousSpan(data, size)); - chunkHeader->TypeLength += size; - Y_ABORT_UNLESS(size); - progress = true; + TMutableContiguousSpan span = buffer.UnsafeGetContiguousSpanMut().SubSpan(sizeof(TChunkHeader), + Max<size_t>()); // reserve space for TChunkHeader which we write first thing if we have some data + for (const auto [data, size] : queue.CoroutineChunkSerializer.FeedBuf(&span, maxBytesToProduce - + sizeof(TChunkHeader))) { + addEventChunkBytes(data, size); + } + Y_DEBUG_ABORT_UNLESS(buffer.data() + buffer.size() == span.data() + span.size()); + Y_DEBUG_ABORT_UNLESS(span.size() <= buffer.size()); // ensure span did not reduce + if (header) { + buffer.TrimFront(span.size()); } // check if we have finished serializing this event if (queue.CoroutineChunkSerializer.IsComplete()) { queue.SerializeStage = ESerializeStage::kHeader; - } else if (!progress) { - // we didn't serialize anything but the chunk header, so we should exit the loop after dropping - // the header - Y_ABORT_UNLESS(!chunkHeader->GetLength()); - Y_ABORT_UNLESS(!buffer->size()); - quit = true; } + +// SerializeEventTime += Timer.PassedReset() * 1e9; break; } case ESerializeStage::kHeader: { + if (Checksumming && !queue.EventHeaderOffset) { + XXH3_64bits_update(&queue.ChecksumState, &queue.EventHeader, sizeof(queue.EventHeader)); + queue.EventHeader.Checksum = XXH3_64bits_digest(&queue.ChecksumState); + } + const size_t numDataBytes = Min( - buffer->size(), - maxBytesToProduce, + buffer.size() - sizeof(TChunkHeader), + maxBytesToProduce - sizeof(TChunkHeader), sizeof(TEventHeader) - queue.EventHeaderOffset ); - chunkHeader->TypeLength += numDataBytes; - Y_ABORT_UNLESS(chunkHeader->GetLength() == numDataBytes); + Y_DEBUG_ABORT_UNLESS(numDataBytes); + *static_cast<TChunkHeader*>(takeInBuffer(sizeof(TChunkHeader))) = { + .TypeLength = static_cast<ui16>(TChunkHeader::kEventHeader | numDataBytes), + .Channel = ev.GetChannel(), + }; void *ptr = takeInBuffer(numDataBytes); memcpy(ptr, reinterpret_cast<const char*>(&queue.EventHeader) + queue.EventHeaderOffset, numDataBytes); @@ -229,11 +267,10 @@ namespace NActors { if (queue.EventHeaderOffset == sizeof(TEventHeader)) { RefcountItems.push_back({ - .NumBytesRemaining = std::exchange(queue.EventProducedSize, 0), + .EndOffset = CumulativeProduced, .Buffer = std::exchange(queue.Buffer, nullptr), .Event{ev.ReleaseBase().Release()}, }); - CommitProducedBytes(std::exchange(OverproducedBytes, 0)); queue.Events.pop_front(); queue.SerializeStage = ESerializeStage::kInitial; queue.EventHeaderOffset = 0; @@ -241,53 +278,16 @@ namespace NActors { break; } } - - if (chunkHeader && !chunkHeader->GetLength()) { - // drop useless chunk header (when we have produced empty event, for instance); note that the chunk - // header may have been concatenated with a preceding output span, so we trim it off the tail instead of - // assuming it occupies a whole span - Y_ABORT_UNLESS(!out->empty()); - TContiguousSpan& last = out->back(); - Y_ABORT_UNLESS(last.size() >= sizeof(TChunkHeader)); - Y_ABORT_UNLESS(last.data() + last.size() == buffer->data()); - Y_ABORT_UNLESS(last.data() + last.size() - sizeof(TChunkHeader) == reinterpret_cast<const char*>(chunkHeader)); - if (last.size() == sizeof(TChunkHeader)) { - out->pop_back(); - } else { - last = {last.data(), last.size() - sizeof(TChunkHeader)}; - } - *buffer = {buffer->data() - sizeof(TChunkHeader), buffer->size() + sizeof(TChunkHeader)}; - maxBytesToProduce += sizeof(TChunkHeader); - numBytesProduced -= sizeof(TChunkHeader); - queue.EventProducedSize -= sizeof(TChunkHeader); - } else if (chunkHeader) { - // find chunk header in out - for (size_t i = 0; i < out->size(); ++i) { - const void *begin = (*out)[i].data(); - const void *end = (*out)[i].data() + (*out)[i].size(); - if (chunkHeader >= begin && chunkHeader + 1 <= end) { - size_t numBytesAfter = (const char*)end - (const char*)(chunkHeader + 1); - while (++i < out->size()) { - const void *begin = (*out)[i].data(); - const void *end = (*out)[i].data() + (*out)[i].size(); - Y_ABORT_UNLESS(!(chunkHeader >= begin && chunkHeader + 1 <= end)); - - numBytesAfter += (*out)[i].size(); - } - Y_ABORT_UNLESS(numBytesAfter == chunkHeader->GetLength()); - } - } - } - - if (quit) { - break; - } } return numBytesProduced; } - void TEventDeserializer::Push(TRcBuf buffer, IEventProcessor *eventProcessor) { + TEventDeserializer::TEventDeserializer(TScopeId peerScopeId) + : PeerScopeId(peerScopeId) + {} + + void TEventDeserializer::Push(TRcBuf buffer, IEventProcessor *eventProcessor, TActorId sessionId) { // put incoming buffer to the queue's end Accum.Insert(Accum.End(), std::move(buffer)); @@ -319,9 +319,21 @@ namespace NActors { if (queue.EventHeaderOffset == sizeof(TEventHeader)) { queue.EventHeaderOffset = 0; + if (queue.EventHeader.Checksum) { + XXH3_state_t state; + XXH3_64bits_reset(&state); + for (auto iter = queue.Accum.begin(); iter.Valid(); iter.AdvanceToNextContiguousBlock()) { + XXH3_64bits_update(&state, iter.ContiguousData(), iter.ContiguousSize()); + } + const ui64 expected = std::exchange(queue.EventHeader.Checksum, 0); + XXH3_64bits_update(&state, &queue.EventHeader, sizeof(queue.EventHeader)); + const ui64 calculated = XXH3_64bits_digest(&state); + Y_ABORT_UNLESS(calculated == expected); + } + queue.EvSerInfo.IsExtendedFormat = queue.EventHeader.Flags & IEventHandle::FlagExtendedFormat; eventProcessor->PushEvent(std::make_unique<IEventHandle>( - TActorId(), // session id will be filled later + sessionId, queue.EventHeader.Type, queue.EventHeader.Flags & ~IEventHandle::FlagExtendedFormat, queue.EventHeader.Recipient, @@ -330,7 +342,7 @@ namespace NActors { std::exchange(queue.Accum, {}), std::exchange(queue.EvSerInfo, {})), queue.EventHeader.Cookie, - TScopeId(), + PeerScopeId, NWilson::TTraceId(queue.EventHeader.TraceId))); } diff --git a/ydb/library/actors/interconnect/v2_event_serializer.h b/ydb/library/actors/interconnect/v2_event_serializer.h index 4a12f0b090d..a5721c8a1b3 100644 --- a/ydb/library/actors/interconnect/v2_event_serializer.h +++ b/ydb/library/actors/interconnect/v2_event_serializer.h @@ -7,6 +7,9 @@ #include <ydb/library/actors/wilson/wilson_trace.h> #include <ydb/library/actors/util/rc_buf.h> +#define XXH_INLINE_ALL +#include <contrib/libs/xxhash/xxhash.h> + #include <deque> namespace NActors { @@ -18,6 +21,7 @@ namespace NActors { ui32 Type; ui32 Flags; ui64 Cookie; + ui64 Checksum; // checksum (optional) for the whole event, including this header (with zero checksum) TActorId Sender; TActorId Recipient; NWilson::TTraceId::TSerializedTraceId TraceId; @@ -40,6 +44,8 @@ namespace NActors { #pragma pack(pop) private: + const bool Checksumming; + struct TPerChannelQuota { ui16 Channel; // channel number ui16 Quota; // quota in bytes to produce @@ -69,37 +75,70 @@ namespace NActors { ESerializeStage SerializeStage = ESerializeStage::kInitial; TEventSerializationInfo EvSerInfoHolder; const TEventSerializationInfo *EvSerInfo; - size_t EventProducedSize = 0; ui16 Quota = 0; // must be the same as TPerChannelQuota for this channel + XXH3_state_t ChecksumState; }; std::array<TPerChannelQueue, NumDefaultChannels> PerChannelQueue; THashMap<ui16, TPerChannelQueue> PerChannelQueueMap; - // refcounted objects tracking + // Refcounted objects tracking. An event's serialized bytes may be scattered across the output + // stream (interleaved with other channels) and across several pipelined write batches, so we can + // only release an event once *all* of its bytes have been sent. We track that with absolute stream + // offsets: an event's memory is freed once the total number of committed (sent) bytes reaches the + // stream offset just past the event's last produced byte. Releasing on a plain FIFO byte count is + // wrong -- committed bytes belonging to one channel would be charged against a different event at + // the head of the queue, freeing it while a later, still-in-flight batch aliases its memory. struct TRefcountItem { - size_t NumBytesRemaining = 0; + ui64 EndOffset = 0; // total produced bytes at the moment this event was fully produced TIntrusivePtr<TEventSerializedData> Buffer; std::unique_ptr<IEventBase> Event; + TRcBuf Scratch; }; std::deque<TRefcountItem> RefcountItems; - size_t OverproducedBytes = 0; + ui64 CumulativeProduced = 0; // total bytes ever produced into the output stream + ui64 CumulativeCommitted = 0; // total bytes ever reported as sent via CommitProducedBytes + + THPTimer Timer; + ui64 SerializeBufferTime = 0; + ui64 SerializeEventTime = 0; + ui64 OtherTime = 0; + ui64 BytesCopied = 0; + ui64 BytesAliased = 0; public: + TEventSerializer(bool checksumming); + void Push(std::unique_ptr<IEventHandle> ev); + bool IsTrafficPending() const { return !PerChannelQuotaHeap.empty(); } + // this function generates output stream for transmission; it returns number of bytes added to output spans - size_t ProduceOutputStream(TMutableContiguousSpan *buffer, std::vector<TContiguousSpan> *out); + size_t ProduceOutputStream(TRcBuf& buffer, std::deque<TContiguousSpan> *out, size_t maxBytesToProduce = Max<size_t>()); // notification issued when N produced bytes have been sent to the other party void CommitProducedBytes(size_t numBytes); + void ResetCounters() { + SerializeBufferTime = 0; + SerializeEventTime = 0; + OtherTime = 0; + BytesCopied = 0; + BytesAliased = 0; + } + + ui64 GetSerializeBufferTime() const { return SerializeBufferTime; } + ui64 GetSerializeEventTime() const { return SerializeEventTime; } + ui64 GetOtherTime() const { return OtherTime; } + ui64 GetBytesCopied() const { return BytesCopied; } + ui64 GetBytesAliased() const { return BytesAliased; } + private: TPerChannelQueue& GetQueue(ui16 channel) { return channel < NumDefaultChannels ? PerChannelQueue[channel] : PerChannelQueueMap[channel]; } - size_t ProduceOutputStreamForQueue(TPerChannelQueue& queue, size_t maxBytesToProduce, - TMutableContiguousSpan *buffer, std::vector<TContiguousSpan> *out); + size_t ProduceOutputStreamForQueue(TPerChannelQueue& queue, size_t maxBytesToProduce, TRcBuf& buffer, + std::deque<TContiguousSpan> *out, ui64 *bufferProduced); }; class TEventDeserializer { @@ -108,6 +147,8 @@ namespace NActors { using TEventHeader = TEventSerializer::TEventHeader; using TChunkHeader = TEventSerializer::TChunkHeader; + const TScopeId PeerScopeId; + struct TPerChannelQueue { TRope Accum; TEventSerializationInfo EvSerInfo; @@ -126,7 +167,8 @@ namespace NActors { }; public: - void Push(TRcBuf buffer, IEventProcessor *eventProcessor); + TEventDeserializer(TScopeId peerScopeId); + void Push(TRcBuf buffer, IEventProcessor *eventProcessor, TActorId sessionId); private: TPerChannelQueue& GetQueue(ui16 channel) { diff --git a/ydb/library/actors/interconnect/ya.make b/ydb/library/actors/interconnect/ya.make index be4d56a91a2..c2c3e765a8f 100644 --- a/ydb/library/actors/interconnect/ya.make +++ b/ydb/library/actors/interconnect/ya.make @@ -47,6 +47,8 @@ SRCS( interconnect_tcp_session.h interconnect_tcp_session_v2.cpp interconnect_tcp_session_v2.h + interconnect_uring_engine.cpp + interconnect_uring_engine.h interconnect_zc_processor.cpp interconnect_zc_processor.h load.cpp diff --git a/ydb/library/grpc/server/grpc_server.cpp b/ydb/library/grpc/server/grpc_server.cpp index 46ed7726bba..bd4c5b2f572 100644 --- a/ydb/library/grpc/server/grpc_server.cpp +++ b/ydb/library/grpc/server/grpc_server.cpp @@ -154,7 +154,9 @@ void TGRpcServer::Start() { sslOps.pem_root_certs = std::move(Options_.SslData->Root); sslOps.pem_key_cert_pairs.push_back(keycert); - if (Options_.SslData->DoRequestClientCertificate) { + if (Options_.SslData->ClientCertificateRequired) { + sslOps.client_certificate_request = GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY; + } else if (Options_.SslData->DoRequestClientCertificate) { sslOps.client_certificate_request = GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY; } diff --git a/ydb/library/grpc/server/grpc_server.h b/ydb/library/grpc/server/grpc_server.h index 9a816c4f56c..cf0f1d116ec 100644 --- a/ydb/library/grpc/server/grpc_server.h +++ b/ydb/library/grpc/server/grpc_server.h @@ -33,6 +33,7 @@ struct TSslData { TString Key; TString Root; bool DoRequestClientCertificate = false; + bool ClientCertificateRequired = false; }; struct IExternalListener diff --git a/ydb/library/yaml_config/protos/blobstorage_config.proto b/ydb/library/yaml_config/protos/blobstorage_config.proto index 24461cf4478..4911e1c9420 100644 --- a/ydb/library/yaml_config/protos/blobstorage_config.proto +++ b/ydb/library/yaml_config/protos/blobstorage_config.proto @@ -69,8 +69,14 @@ message Drive { bool read_centric = 4; // nontransparent user-defined kind used for filtering when picking up groups uint64 kind = 5; + // expected_slot_count and expected_slot_size are mutually exclusive. + // expected_slot_size requires max_slots. // Number of slots to calculate per-vdisk disk space limit. uint64 expected_slot_count = 6; + // Per-vdisk disk space limit. + uint64 expected_slot_size = 7; + // Upper bound for expected_slot_count inferred from expected_slot_size. + uint32 max_slots = 8; } message HostConfig { diff --git a/ydb/library/yaml_config/protos/config.proto b/ydb/library/yaml_config/protos/config.proto index 7afaa1a5ff4..bf131375019 100644 --- a/ydb/library/yaml_config/protos/config.proto +++ b/ydb/library/yaml_config/protos/config.proto @@ -73,11 +73,14 @@ message TExtendedHostConfigDrive { optional uint64 Kind = 5 [(NMarkers.CopyTo) = "THostConfigDrive"]; optional NKikimrBlobStorage.TPDiskConfig PDiskConfig = 6 [(NMarkers.CopyTo) = "THostConfigDrive"]; + // ExpectedSlotSize is mutually exclusive with ExpectedSlotCount and SlotSizeInUnits, and requires MaxSlots. optional uint64 ExpectedSlotCount = 7; optional uint32 SlotSizeInUnits = 8; reserved 9; // optional uint64 InferPDiskSlotCountFromUnitSize = 9 [(NMarkers.CopyTo) = "THostConfigDrive"]; // deprecated reserved 10; // optional uint32 InferPDiskSlotCountMax = 10 [(NMarkers.CopyTo) = "THostConfigDrive"]; // deprecated optional string DiskScope = 11 [(NMarkers.CopyTo) = "THostConfigDrive"]; + optional uint64 ExpectedSlotSize = 12; + optional uint32 MaxSlots = 13; } message THosts { diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 6a154b8ece5..5b0b90dbd98 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -679,12 +679,34 @@ namespace NKikimr::NYaml { drive.SetPath(Sprintf("SectorMap:%d:64", sectorMapIndex)); drive.SetType("SSD"); } + const bool hasExpectedSlotSize = drive.HasExpectedSlotSize() && drive.GetExpectedSlotSize(); + const bool hasMaxSlots = drive.HasMaxSlots() && drive.GetMaxSlots(); + if (hasExpectedSlotSize + && (drive.GetExpectedSlotCount() || drive.GetSlotSizeInUnits())) { + ythrow yexception() << "expected_slot_size is mutually exclusive with expected_slot_count" + << " and slot_size_in_units" + << " for drive with path '" << drive.GetPath() << "'"; + } + if (hasExpectedSlotSize && !hasMaxSlots) { + ythrow yexception() << "expected_slot_size requires max_slots" + << " for drive with path '" << drive.GetPath() << "'"; + } + if (hasMaxSlots && !hasExpectedSlotSize) { + ythrow yexception() << "max_slots requires expected_slot_size" + << " for drive with path '" << drive.GetPath() << "'"; + } if (drive.HasExpectedSlotCount()) { drive.MutablePDiskConfig()->SetExpectedSlotCount(drive.GetExpectedSlotCount()); } if (drive.HasSlotSizeInUnits()) { drive.MutablePDiskConfig()->SetSlotSizeInUnits(drive.GetSlotSizeInUnits()); } + if (drive.HasExpectedSlotSize()) { + drive.MutablePDiskConfig()->SetExpectedSlotSize(drive.GetExpectedSlotSize()); + } + if (drive.HasMaxSlots()) { + drive.MutablePDiskConfig()->SetMaxSlots(drive.GetMaxSlots()); + } } } } diff --git a/ydb/library/yaml_config/yaml_config_parser_ut.cpp b/ydb/library/yaml_config/yaml_config_parser_ut.cpp index 0d1617c3d55..0ead4015336 100644 --- a/ydb/library/yaml_config/yaml_config_parser_ut.cpp +++ b/ydb/library/yaml_config/yaml_config_parser_ut.cpp @@ -188,6 +188,83 @@ feature_flags: UNIT_CHECK_GENERATED_EXCEPTION(Parse(config, false), yexception); } + Y_UNIT_TEST(ExpectedSlotSizeIsMutuallyExclusiveWithLegacySlotSettings) { + TString withExpectedSlotCount = R"( +host_configs: +- host_config_id: 1 + drive: + - path: disk1 + type: SSD + expected_slot_count: 4 + expected_slot_size: 100000 +hosts: +- host: fqdn1 + host_config_id: 1 +)"; + UNIT_CHECK_GENERATED_EXCEPTION(Parse(withExpectedSlotCount, true), yexception); + + TString withSlotSizeInUnits = R"( +host_configs: +- host_config_id: 1 + drive: + - path: disk1 + type: SSD + slot_size_in_units: 2 + expected_slot_size: 100000 +hosts: +- host: fqdn1 + host_config_id: 1 +)"; + UNIT_CHECK_GENERATED_EXCEPTION(Parse(withSlotSizeInUnits, true), yexception); + } + + Y_UNIT_TEST(ExpectedSlotSizeRequiresMaxSlots) { + TString withoutMaxSlots = R"( +host_configs: +- host_config_id: 1 + drive: + - path: disk1 + type: SSD + expected_slot_size: 100000 +hosts: +- host: fqdn1 + host_config_id: 1 +)"; + UNIT_CHECK_GENERATED_EXCEPTION(Parse(withoutMaxSlots, true), yexception); + + TString withMaxSlots = R"( +host_configs: +- host_config_id: 1 + drive: + - path: disk1 + type: SSD + expected_slot_size: 100000 + max_slots: 8 +hosts: +- host: fqdn1 + host_config_id: 1 +)"; + NKikimrConfig::TAppConfig cfg = Parse(withMaxSlots, true); + const auto& drive = cfg.GetBlobStorageConfig().GetDefineHostConfig(0).GetDrive(0); + UNIT_ASSERT_VALUES_EQUAL(drive.GetPDiskConfig().GetExpectedSlotSize(), 100000); + UNIT_ASSERT_VALUES_EQUAL(drive.GetPDiskConfig().GetMaxSlots(), 8); + } + + Y_UNIT_TEST(MaxSlotsRequiresExpectedSlotSize) { + TString withoutExpectedSlotSize = R"( +host_configs: +- host_config_id: 1 + drive: + - path: disk1 + type: SSD + max_slots: 8 +hosts: +- host: fqdn1 + host_config_id: 1 +)"; + UNIT_CHECK_GENERATED_EXCEPTION(Parse(withoutExpectedSlotSize, true), yexception); + } + Y_UNIT_TEST(StoragePoolTypesWithDefaultDomainName) { TString config = R"( storage_pool_types: diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h index 11508ad599f..58ee724406e 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_async_io.h @@ -179,6 +179,11 @@ struct IDqComputeActorAsyncOutput { virtual void PassAway() = 0; // The same signature as IActor::PassAway() + // Called by compute actor when the output consumer (TransformOutput) has been + // drained and can accept more data. Only meaningful for output transforms that + // push data into an IDqOutputConsumer. + virtual void OnOutputConsumerReady() {} + // You must also destroy all internal UnboxedValues inside destructor (same as in PassAway) // But you should explicitly bind MKQL allocator here, because it is called from actor system thread. virtual ~IDqComputeActorAsyncOutput() = default; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index 83629c0933d..cde6a46f085 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -476,6 +476,19 @@ protected: DrainAsyncOutput(outputIndex, info); } + for (auto& [outputIndex, transform] : OutputTransformsMap) { + if (!transform.OutputBuffer || !transform.AsyncOutput) { + continue; + } + const auto level = transform.OutputBuffer->GetFillLevel(); + if (level != EDqFillLevel::NoLimit) { + transform.OutputConsumerWasLimited = true; + } else if (transform.OutputConsumerWasLimited) { + transform.OutputConsumerWasLimited = false; + transform.AsyncOutput->OnOutputConsumerReady(); + } + } + CheckRunStatus(); } @@ -1090,6 +1103,7 @@ protected: struct TAsyncOutputTransformInfo : public TAsyncOutputInfoBase { IDqOutputConsumer::TPtr OutputBuffer; + bool OutputConsumerWasLimited = false; }; protected: diff --git a/ydb/library/yql/dq/actors/spilling/spilling_file.cpp b/ydb/library/yql/dq/actors/spilling/spilling_file.cpp index 055dd2d346c..bab5517834e 100644 --- a/ydb/library/yql/dq/actors/spilling/spilling_file.cpp +++ b/ydb/library/yql/dq/actors/spilling/spilling_file.cpp @@ -205,7 +205,7 @@ private: ui32 NodeId; TString SpillingSessionId; - TEvRemoveOldTmp(TFsPath tmpRoot, ui32 nodeId, TString spillingSessionId) + TEvRemoveOldTmp(TFsPath tmpRoot, ui32 nodeId, TString spillingSessionId) : TmpRoot(std::move(tmpRoot)), NodeId(nodeId), SpillingSessionId(std::move(spillingSessionId)) {} }; }; @@ -243,7 +243,7 @@ public: Become(&TDqLocalFileSpillingService::BrokenState); return; } - + Send(SelfId(), MakeHolder<TEvPrivate::TEvRemoveOldTmp>(rootToRemoveOldTmp, nodeId, sessionId)); Become(&TDqLocalFileSpillingService::WorkState); @@ -483,7 +483,7 @@ private: LOG_E(error); Send(ev->Sender, new TEvDqSpilling::TEvError(error)); - } + } } void HandleWork(TEvPrivate::TEvWriteFileResponse::TPtr& ev) { @@ -536,6 +536,7 @@ private: Send(msg.Client, new TEvDqSpilling::TEvError(*fd.Error)); fd.Ops.clear(); + fd.HasActiveOp = false; CloseFile(it, fd.Error); return; } @@ -630,7 +631,7 @@ private: LOG_E(error); Send(ev->Sender, new TEvDqSpilling::TEvError(error)); - } + } } void HandleWork(TEvPrivate::TEvReadFileResponse::TPtr& ev) { @@ -684,6 +685,7 @@ private: Send(msg.Client, new TEvDqSpilling::TEvError(*fd.Error)); fd.Ops.clear(); + fd.HasActiveOp = false; CloseFile(it, fd.Error); return; } @@ -792,7 +794,7 @@ private: LOG_I("[RemoveOldTmp] removing at root: " << root); - const auto isDirOldTmp = [&nodePrefix, &nodeIdString, &sessionId](const TString& dirName) -> bool { + const auto isDirOldTmp = [&nodePrefix, &nodeIdString, &sessionId](const TString& dirName) -> bool { // dirName: node_<nodeId>_<sessionId> TVector<TString> parts; StringSplitter(dirName).Split('_').Limit(3).Collect(&parts); @@ -805,13 +807,13 @@ private: try { TDirIterator iter(root, TDirIterator::TOptions().SetMaxLevel(1)); - + TVector<TString> oldTmps; for (const auto& dirEntry : iter) { if (dirEntry.fts_info == FTS_DP) { continue; } - + const auto dirName = dirEntry.fts_name; if (isDirOldTmp(dirName)) { LOG_D("[RemoveOldTmp] found old temporary at " << (root / dirName)); diff --git a/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp b/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp index ae7cfe55621..71ee1269fe5 100644 --- a/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp +++ b/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp @@ -496,6 +496,103 @@ Y_UNIT_TEST_SUITE(DqSpillingFileTests) { UNIT_ASSERT_C(err.Contains("No such file or directory"), err); UNIT_ASSERT_C(err.Contains(expected), err); } + + // The read error must trigger a proper cleanup: the file has to be closed and its + // descriptor released. If the close operation is not dispatched (e.g. because the + // active-op flag was left set), EvCloseFileResponse never arrives and the file stays + // stuck in the active list, keeping the file descriptor counter above zero. + { + std::atomic<bool> closed = false; + runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) { + if (event->GetRecipientRewrite() == spillingSvc) { + if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */) { + closed = true; + } + } + return TTestActorRuntimeBase::EEventAction::PROCESS; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + return (bool) closed; + }; + runtime.DispatchEvents(options, TDuration::Seconds(1)); + UNIT_ASSERT_C(closed, "file was not closed after read error"); + } + + { + THttpRequest httpReq(HTTP_METHOD_GET); + NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr); + + runtime.Send(new IEventHandle(spillingSvc, tester, new NMon::TEvHttpInfo(monReq))); + auto resp = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(tester, TDuration::Seconds(1)); + UNIT_ASSERT(((NMon::TEvHttpInfoRes*) resp->Get())->Answer.Contains("Used file descriptors (compute): 0")); + } + } + + Y_UNIT_TEST(WriteError) { + TTestActorRuntime runtime; + runtime.Initialize(); + + auto spillingSvc = runtime.StartSpillingService(1000, 100, 25); + auto tester = runtime.AllocateEdgeActor(); + auto spillingActor = runtime.StartSpillingActor(tester); + + runtime.WaitBootstrap(); + + auto nodePath = TFsPath("node_" + std::to_string(spillingSvc.NodeId()) + "_" + runtime.GetSpillingSessionId()); + const TFsPath spillingDir = runtime.GetSpillingRoot() / nodePath; + const TFsPath firstFile = spillingDir / "1_test_0"; + const TFsPath secondFile = spillingDir / "1_test_1"; + + // Write the first blob. Because MaxFilePartSize is small and RemoveBlobsAfterRead is on, + // this blob lands in its own file part "1_test_0". + { + auto ev = new TEvDqSpilling::TEvWrite(0, CreateRope(20, 'a')); + runtime.Send(new IEventHandle(spillingActor, tester, ev)); + + auto resp = runtime.GrabEdgeEvent<TEvDqSpilling::TEvWriteResult>(tester); + UNIT_ASSERT_VALUES_EQUAL(0, resp->Get()->BlobId); + } + UNIT_ASSERT(NFs::Exists(firstFile.GetPath())); + + // File part names are predictable, so occupy the name of the would-be next file part + // ("1_test_1") with a directory. Opening it for writing will fail and produce an IO error. + secondFile.MkDirs(); + + // Write the second blob. It requires a fresh file part ("1_test_1"), whose creation now + // fails with an IO error. + { + auto ev = new TEvDqSpilling::TEvWrite(1, CreateRope(20, 'b')); + runtime.Send(new IEventHandle(spillingActor, tester, ev)); + + auto resp = runtime.GrabEdgeEvent<TEvDqSpilling::TEvError>(tester); + UNIT_ASSERT_C(resp->Get()->Message.Contains("1_test_1"), resp->Get()->Message); + } + + // The write error must trigger cleanup of the whole file, including the already-written + // "1_test_0" part. Wait for the close operation to complete. + { + std::atomic<bool> closed = false; + runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) { + if (event->GetRecipientRewrite() == spillingSvc) { + if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */) { + closed = true; + } + } + return TTestActorRuntimeBase::EEventAction::PROCESS; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + return (bool) closed; + }; + runtime.DispatchEvents(options, TDuration::Seconds(1)); + UNIT_ASSERT_C(closed, "file was not closed after write error"); + } + + // The successfully written part must have been removed from disk during cleanup. + UNIT_ASSERT_C(!NFs::Exists(firstFile.GetPath()), "partially written file was not cleaned up"); } Y_UNIT_TEST(ThreadPoolQueueOverflow) { diff --git a/ydb/library/yql/dq/runtime/dq_channel_service.cpp b/ydb/library/yql/dq/runtime/dq_channel_service.cpp index 4bb02dcb1fe..6446cd1d05a 100644 --- a/ydb/library/yql/dq/runtime/dq_channel_service.cpp +++ b/ydb/library/yql/dq/runtime/dq_channel_service.cpp @@ -1042,6 +1042,7 @@ void TNodeState::PushDataChunk(TDataChunk&& data, std::shared_ptr<TOutputDescrip item->Leading = descriptor->Leading.exchange(false); Queue.push_back(item); SendMessage(item); + SendCount++; InflightBytes += bytes; *OutputBufferInflightBytes += bytes; (*OutputBufferInflightMessages)++; @@ -1080,7 +1081,7 @@ void TNodeState::PushDataChunk(TDataChunk&& data, std::shared_ptr<TOutputDescrip } void TNodeState::SendMessage(std::shared_ptr<TOutputItem> item) { - Y_ENSURE(PeerActorId); + Y_ENSURE(InputNodeActorId); auto ev = MakeHolder<TEvDqCompute::TEvChannelDataV2>(); ev->Record.SetGenMajor(GenMajor); @@ -1132,20 +1133,20 @@ void TNodeState::SendMessage(std::shared_ptr<TOutputItem> item) { FailureDoubleSend.store(failCount - 1); auto ev2 = MakeHolder<TEvDqCompute::TEvChannelDataV2>(); ev2->Record = ev->Record; - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, ev2.Release(), flags, item->SeqNo)); + ActorSystem->Send(new NActors::IEventHandle(InputNodeActorId, NodeActorId, ev2.Release(), flags, item->SeqNo)); } #endif - LOG_T(LogPrefix << "SEND MSG, G=" << GenMajor << '.' << GenMinor << ", SeqNo=" << item->SeqNo - << ", ChannelSeqNo=" << item->ChannelSeqNo - << ", ChannelId=" << item->Descriptor->Info.ChannelId << ", Leading=" << item->Leading << ", Bytes=" << item->Data.Bytes); - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, ev.Release(), flags, item->SeqNo)); + LOG_T(LogPrefix << "SEND DATA, G=" << GenMajor << '.' << GenMinor << ", SeqNo=" << item->SeqNo + << ", ChannelSeqNo=" << item->ChannelSeqNo << ", ChannelId=" << item->Descriptor->Info.ChannelId + << ", Leading=" << item->Leading << ", Finished=" << item->Data.Finished << ", Bytes=" << item->Data.Bytes); + ActorSystem->Send(new NActors::IEventHandle(InputNodeActorId, NodeActorId, ev.Release(), flags, item->SeqNo)); #if !defined(NDEBUG) } #endif item->State.store(TOutputItem::EState::Sent); } -void TNodeState::FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor) { +void TNodeState::FailInputs(const NActors::TActorId& outputNodeActorId, ui64 outputNodeGenMajor) { if (InputDescriptors.empty()) { return; } @@ -1153,11 +1154,11 @@ void TNodeState::FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMa std::vector<TChannelInfo> failedBuffers; for (auto& [info, descriptor] : InputDescriptors) { - if (!descriptor->IsFinished() && descriptor->PeerGenMajor) { - if (descriptor->PeerActorId != peerActorId || descriptor->PeerGenMajor != peerGenMajor) { + if (!descriptor->IsFinished() && descriptor->OutputNodeGenMajor) { + if (descriptor->OutputNodeActorId != outputNodeActorId || descriptor->OutputNodeGenMajor != outputNodeGenMajor) { descriptor->AbortChannel( - TStringBuilder() << "PeerActorId=" << descriptor->PeerActorId << ", PeerGenMajor=" << descriptor->PeerGenMajor - << " DO NOT MATCH peerActorId=" << peerActorId << ", peerGenMajor=" << peerGenMajor + TStringBuilder() << "OutputNodeActorId=" << descriptor->OutputNodeActorId << ", OutputNodeGenMajor=" << descriptor->OutputNodeGenMajor + << " DO NOT MATCH outputNodeActorId=" << outputNodeActorId << ", outputNodeGenMajor=" << outputNodeGenMajor << ", Session=" << LogPrefix << ", Log=" << GetReconciliationLog() ); failedBuffers.push_back(info); @@ -1207,14 +1208,14 @@ void TNodeState::SendAck(THolder<TEvDqCompute::TEvChannelAckV2>& evAck, ui64 coo flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, cookie)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, cookie)); } void TNodeState::SendAckWithError(ui64 cookie, const TString& message) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::ERROR); evAck->Record.SetSeqNo(ConfirmedSeqNo); evAck->Record.SetMessage(message); @@ -1245,12 +1246,12 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { return; } - if (descriptor->PeerGenMajor) { - if (descriptor->PeerActorId != PeerActorId || descriptor->PeerGenMajor != PeerGenMajor.load()) { + if (descriptor->OutputNodeGenMajor) { + if (descriptor->OutputNodeActorId != OutputNodeActorId || descriptor->OutputNodeGenMajor != OutputNodeGenMajor.load()) { descriptor->Terminate(); - TString errorMessage = TStringBuilder() << "MISMATCH G=" << descriptor->PeerGenMajor - << ", Peer=" << descriptor->PeerGenMajor << " vs G=" << PeerGenMajor.load() - << ", Peer=" << PeerActorId + TString errorMessage = TStringBuilder() << "MISMATCH ID G=" << descriptor->OutputNodeGenMajor + << ", Peer=" << descriptor->OutputNodeGenMajor << " vs G=" << OutputNodeGenMajor.load() + << ", Peer=" << OutputNodeActorId << ", ChannelId=" << info.ChannelId << ", OA=" << info.OutputActorId << ", IA=" << info.InputActorId << ", Bytes=" << record.GetBytes() << ", L=" << record.GetLeading() @@ -1264,8 +1265,8 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { return; } } else { - descriptor->PeerActorId = PeerActorId; - descriptor->PeerGenMajor = PeerGenMajor.load(); + descriptor->OutputNodeActorId = OutputNodeActorId; + descriptor->OutputNodeGenMajor = OutputNodeGenMajor.load(); } TDataChunk data(TChunkedBuffer(), record.GetRows(), record.GetTransportVersion(), @@ -1322,8 +1323,8 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); evAck->Record.SetSeqNo(ConfirmedSeqNo); @@ -1346,25 +1347,18 @@ void TNodeState::HandleDisconnected(NActors::TEvInterconnect::TEvNodeDisconnecte void TNodeState::HandleUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - if (ev->Get()->Reason == NActors::TEvents::TEvUndelivered::ReasonActorUnknown) { - std::lock_guard lock(Mutex); - if (Reconciliation.load() == 0) { // ignore errors in recovery - if (ev->Get()->SourceType == TEvDqCompute::TEvChannelUpdateV2::EventType) { - LOG_D(LogPrefix << "UNDELIVERED/UNKNOWN UPDATE, PeerActorId " << PeerActorId << ", Sender=" << ev->Sender); - } else { - LOG_W(LogPrefix << "UNDELIVERED/UNKNOWN, PeerActorId " << PeerActorId << ", Sender=" << ev->Sender); - PeerActorId = NActors::TActorId{}; - StartReconciliation(true, 'U'); - } - } - return; - } - switch (ev->Get()->SourceType) { case TEvDqCompute::TEvChannelDataV2::EventType: { - LOG_W(LogPrefix << "UNDELIVERED/OTHER"); std::lock_guard lock(Mutex); - StartReconciliation(false, 'O'); + if (ev->Get()->Reason == NActors::TEvents::TEvUndelivered::ReasonActorUnknown) { + if (Reconciliation.load() == 0) { // ignore errors in recovery + LOG_W(LogPrefix << "UNDELIVERED/UNKNOWN, InputNodeActorId " << InputNodeActorId << ", Sender=" << ev->Sender); + StartReconciliation(true, 'U'); + } + } else { + LOG_W(LogPrefix << "UNDELIVERED/OTHER"); + StartReconciliation(false, 'O'); + } break; } case TEvDqCompute::TEvChannelAckV2::EventType: { @@ -1380,45 +1374,32 @@ void TNodeState::HandleUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { } } -void TNodeState::ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor, ui64 seqNo) { - std::lock_guard lock(Mutex); - if (!Connected) { - PeerActorId = sender; - PeerGenMajor.store(genMajor); - PeerGenMinor.store(genMinor); - ConfirmedSeqNo = seqNo; - Connected = true; - LOG_D(LogPrefix << "CONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); +void TNodeState::ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor) { + if (OutputNodeActorId == sender && OutputNodeGenMajor.load() == genMajor) { + LOG_D(LogPrefix << "RECONNECTED, OutputNodeActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); } else { - if (PeerActorId == sender && PeerGenMajor.load() == genMajor) { - LOG_D(LogPrefix << "RECONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); - } else { - PeerActorId = sender; - PeerGenMajor.store(genMajor); - LOG_W(LogPrefix << "RECONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); - } - PeerGenMinor.store(genMinor); - ConfirmedSeqNo = seqNo; - FailInputs(PeerActorId, PeerGenMajor.load()); - } - for (auto& [_, descriptor] : InputDescriptors) { - if (descriptor->EarlyFinished.load() || descriptor->PushStats.Bytes.load()) { - SendUpdateProgress(descriptor); - } + OutputNodeActorId = sender; + OutputNodeGenMajor.store(genMajor); + ConfirmedSeqNo = 0; + LOG_W(LogPrefix << "RECONNECTED, OutputNodeActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); } + OutputNodeGenMinor.store(genMinor); + FailInputs(OutputNodeActorId, OutputNodeGenMajor.load()); } void TNodeState::HandleDiscovery(TEvDqCompute::TEvChannelDiscoveryV2::TPtr& ev) { LastPeerActivity.store(TInstant::Now()); auto& record = ev->Get()->Record; - ConnectSession(ev->Sender, record.GetGenMajor(), record.GetGenMinor(), record.GetSeqNo()); + + std::lock_guard lock(Mutex); + ConnectSession(ev->Sender, record.GetGenMajor(), record.GetGenMinor()); auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); - evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); + evAck->Record.SetStatus(record.GetSeqNo() <= ConfirmedSeqNo ? NYql::NDqProto::TEvChannelAckV2::OK : NYql::NDqProto::TEvChannelAckV2::RESEND); evAck->Record.SetSeqNo(ConfirmedSeqNo); ui32 flags = NActors::IEventHandle::FlagTrackDelivery; @@ -1426,67 +1407,46 @@ void TNodeState::HandleDiscovery(TEvDqCompute::TEvChannelDiscoveryV2::TPtr& ev) flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); + + for (auto& [_, descriptor] : InputDescriptors) { + if (descriptor->EarlyFinished.load() || descriptor->PushStats.Bytes.load()) { + SendUpdateProgress(descriptor); + } + } } void TNodeState::HandleData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { - LastPeerActivity.store(TInstant::Now()); auto& record = ev->Get()->Record; - LOG_T(LogPrefix << "RECV MSG, PG=" << PeerGenMajor.load() << '.' << PeerGenMinor.load() << ", SeqNo=" << record.GetSeqNo() - << ", ChannelSeqNo=" << record.GetChannelSeqNo() - << ", ChannelId=" << record.GetChannelId() << ", Bytes=" << record.GetBytes()); - - auto prevPeerGenMajor = PeerGenMajor.exchange(record.GetGenMajor()); - if (PeerActorId != ev->Sender || (prevPeerGenMajor && prevPeerGenMajor != record.GetGenMajor())) { - auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - - auto reconciliationLog = GetReconciliationLog(); - evAck->Record.SetGenMajor(record.GetGenMajor()); - evAck->Record.SetGenMinor(record.GetGenMinor()); - evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::FAIL); - evAck->Record.SetSeqNo(ConfirmedSeqNo); - evAck->Record.SetMessage(reconciliationLog); - LOG_E(LogPrefix << "FAIL, PeerActorId=" << PeerActorId << ", ev->Sender" << ev->Sender - << ", PeerGenMajor=" << prevPeerGenMajor << ", record.GetGenMajor()=" << record.GetGenMajor() - << ", Log=" << reconciliationLog); - - ui32 flags = NActors::IEventHandle::FlagTrackDelivery; - if (!Subscribed.exchange(true)) { - flags |= NActors::IEventHandle::FlagSubscribeOnSession; - } + auto genMajor = record.GetGenMajor(); + auto genMinor = record.GetGenMinor(); + auto seqNo = record.GetSeqNo(); - ActorSystem->Send(new NActors::IEventHandle(ev->Sender, NodeActorId, evAck.Release(), flags, ev->Cookie)); - return; + if (OutputNodeActorId != ev->Sender || OutputNodeGenMajor.load() != genMajor || OutputNodeGenMinor.load() != genMinor) { + LOG_D(LogPrefix << "OBSOLETE DATA, OutputNodeActorId=" << OutputNodeActorId << ", G=" << OutputNodeGenMajor.load() << '.' << OutputNodeGenMinor.load() + << " vs Sender=" << ev->Sender << ", G=" << genMajor << '.' << genMinor + << ", SeqNo=" << seqNo); + } else { + LOG_T(LogPrefix << "RECV DATA, G=" << genMajor << '.' << genMinor << ", SeqNo=" << seqNo + << ", ChannelSeqNo=" << record.GetChannelSeqNo() << ", ChannelId=" << record.GetChannelId() + << ", Leading=" << record.GetLeading() << ", Finished=" << record.GetFinished() + << ", Bytes=" << record.GetBytes()); } - PeerGenMinor.store(std::max<ui64>(record.GetGenMinor(), PeerGenMinor.load())); - - auto seqNo = record.GetSeqNo(); - if (seqNo <= ConfirmedSeqNo) { LOG_W(LogPrefix << "DATA/IGNORED, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); return; } - switch (seqNo - ConfirmedSeqNo) { - case 1: { - break; - } - case 2: { - // allow 1 out of order message - LOG_W(LogPrefix << "DATA/OOO, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); - OutOfOrderMessage = ev.Release(); - return; - } - default: { - if (!ResendAsked.exchange(true)) { - LOG_W(LogPrefix << "DATA/RESEND, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); - } + if (seqNo - ConfirmedSeqNo > 1) { + if (!ResendAsked.exchange(true)) { + LOG_W(LogPrefix << "DATA/RESEND, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); + auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::RESEND); evAck->Record.SetSeqNo(ConfirmedSeqNo + 1); @@ -1495,27 +1455,15 @@ void TNodeState::HandleData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); - return; + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); } + return; } // happy path - ResendAsked.store(false); ConfirmedSeqNo++; HandleChannelData(ev); - - if (OutOfOrderMessage) { - auto& record = OutOfOrderMessage->Get()->Record; - - if (record.GetSeqNo() == ConfirmedSeqNo + 1) { - ConfirmedSeqNo++; - HandleChannelData(OutOfOrderMessage); - } - - OutOfOrderMessage.Reset(); - } } void TNodeState::SendFromWaiters(ui64 deltaBytes) { @@ -1540,6 +1488,7 @@ void TNodeState::SendFromWaiters(ui64 deltaBytes) { (*OutputBufferWaiterCount)--; /* +now may need to send very last msg from terminated descriptor if (WaitersQueue.top()->IsTerminatedOrAborted()) { auto waitQueueBytes = WaitersQueue.top()->WaitQueueBytes.load(); @@ -1599,6 +1548,7 @@ void TNodeState::SendFromWaiters(ui64 deltaBytes) { item->Leading = waiter->Leading.exchange(false); Queue.push_back(item); SendMessage(item); + SendCount++; inflightBytes += bytes; InflightBytes += bytes; *OutputBufferInflightBytes += bytes; @@ -1645,7 +1595,7 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { auto genMajor = record.GetGenMajor(); auto genMinor = record.GetGenMinor(); if (GenMajor != genMajor || GenMinor != genMinor) { - LOG_W(LogPrefix << "ACK/IGNORED, G=" << GenMajor << '.' << GenMinor << ", ack.G=" << genMajor << '.' << genMinor); + LOG_W(LogPrefix << "ACK/IGNORED, G=" << GenMajor << '.' << GenMinor << ", ack.G=" << genMajor << '.' << genMinor << ", Sender=" << ev->Sender); return; } @@ -1658,9 +1608,9 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { return; } - if (PeerActorId == NActors::TActorId{}) { - PeerActorId = ev->Sender; - LOG_D(LogPrefix << "PEER/ACK, PeerActorId=" << ev->Sender); + if (InputNodeActorId == NActors::TActorId{}) { + InputNodeActorId = ev->Sender; + LOG_D(LogPrefix << "PEER/ACK, InputNodeActorId=" << ev->Sender); } if (seqNo > SeqNo) { @@ -1701,11 +1651,8 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { } } else { if (status == NYql::NDqProto::TEvChannelAckV2::RESEND) { - // if we're reconcilating, ignore next RESENDs - if (record.GetGenMinor() == GenMinor) { - LOG_W(LogPrefix << "SEQ/RESEND, SeqNo=" << seqNo); - StartReconciliation(false, 'R'); - } + LOG_W(LogPrefix << "SEQ/RESEND, SeqNo=" << seqNo); + StartReconciliation(false, 'R'); return; } @@ -1735,10 +1682,12 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { if (Reconciliation.exchange(0) > 0) { ReconciliationCount = 0; + ReconSent.store(TInstant::Zero()); LOG_I(LogPrefix << "RECONCILED, Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", InflightBytes=" << InflightBytes.load() << '-' << deltaBytes); if (!Queue.empty()) { for (auto item : Queue) { SendMessage(item); + ResendCount++; item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Repeat from SeqNo=" << Queue.front()->SeqNo << ", item->SeqNo=" << item->SeqNo); } } @@ -1817,8 +1766,8 @@ void TNodeState::SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descripto auto evUpdate = MakeHolder<TEvDqCompute::TEvChannelUpdateV2>(); - evUpdate->Record.SetGenMajor(PeerGenMajor.load()); - evUpdate->Record.SetGenMinor(PeerGenMinor.load()); + evUpdate->Record.SetGenMajor(OutputNodeGenMajor.load()); + evUpdate->Record.SetGenMinor(OutputNodeGenMinor.load()); // evUpdate->Record.SetSeqNo(ConfirmedSeqNo); NActors::ActorIdToProto(descriptor->Info.OutputActorId, evUpdate->Record.MutableSrcActorId()); @@ -1839,7 +1788,7 @@ void TNodeState::SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descripto << ", EarlyFinished=" << descriptor->EarlyFinished.load() << ", PopBytes=" << descriptor->PopStats.Bytes.load() << ", Finishing=" << descriptor->Finishing.load()); - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evUpdate.Release(), flags)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evUpdate.Release(), flags)); } std::shared_ptr<TOutputDescriptor> TNodeState::GetOrCreateOutputDescriptor(const TChannelFullInfo& info, IMemoryQuotaManager::TPtr quotaManager, bool bound, bool leading) { @@ -2035,10 +1984,12 @@ void TNodeState::HandleReconciliation(TEvPrivate::TEvReconciliation::TPtr& ev) { void TNodeState::StartReconciliation(bool major, char logSymbol) { if (Reconciliation.load() == 0 || (major && (GenMinor > 1))) { + ReconCount++; if (major) { GenMajor++; GenMinor = 1; SeqNo = 0; + InputNodeActorId = NActors::TActorId{}; } else { GenMinor++; } @@ -2087,70 +2038,70 @@ void TNodeState::DoReconciliation(char logSymbol) { << ", Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", Log=" << reconciliationLog); } else { - LOG_D(LogPrefix << "RECONCILIATION, G" << GenMajor << '.' << GenMinor + LOG_D(LogPrefix << "RECONCILIATION, G=" << GenMajor << '.' << GenMinor << ", Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", Log=" << reconciliationLog); } ui32 delta = 0; - std::deque<std::shared_ptr<TOutputItem>> RebuiltQueue; - - auto seqNo = SeqNo; + if (GenMinor == 1) { // => major reconciliation + std::deque<std::shared_ptr<TOutputItem>> RebuiltQueue; + while (!Queue.empty()) { - while (!Queue.empty()) { + auto& item = Queue.front(); - auto& item = Queue.front(); + if (item->Leading) { + auto prevGenMajor = item->Descriptor->GenMajor.exchange(GenMajor); + if (prevGenMajor != GenMajor) { + LOG_W(LogPrefix << "CHANGE OD G=" << prevGenMajor << " to " << GenMajor << " by RECONCILIATION" + << " Channel: " << item->Descriptor->Info.ChannelId + << ", SrcStageId: " << item->Descriptor->Info.SrcStageId + << ", DstStageId: " << item->Descriptor->Info.DstStageId + << ", Log=" << reconciliationLog); + } + } - if (item->Leading) { - auto prevGenMajor = item->Descriptor->GenMajor.exchange(GenMajor); - if (prevGenMajor != GenMajor) { - LOG_W(LogPrefix << "CHANGE OG G=" << prevGenMajor << " to " << GenMajor << " by RECONCILIATION" - << " Channel: " << item->Descriptor->Info.ChannelId - << ", SrcStageId: " << item->Descriptor->Info.SrcStageId - << ", DstStageId: " << item->Descriptor->Info.DstStageId - << ", Log=" << reconciliationLog); + if (item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Reconciliation, Log=" << reconciliationLog)) { + item->SeqNo = ++SeqNo; + RebuiltQueue.push_back(std::move(item)); + } else { + delta += item->Data.Bytes; } - } - if (item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Reconciliation, Log=" << reconciliationLog)) { - item->SeqNo = ++SeqNo; - RebuiltQueue.push_back(std::move(item)); - } else { - delta += item->Data.Bytes; + Queue.pop_front(); } - - Queue.pop_front(); + Queue.swap(RebuiltQueue); } - Queue.swap(RebuiltQueue); InflightBytes -= delta; - SendDiscovery(MakeChannelServiceActorID(NodeId), seqNo); + SendDiscovery(); + ReconSent.store(TInstant::Now()); ActorSystem->Schedule(reconciliationTimeout, new NActors::IEventHandle(NodeActorId, NodeActorId, new TEvPrivate::TEvReconciliation(GenMajor, GenMinor, ReconciliationCount))); } -void TNodeState::SendDiscovery(NActors::TActorId actorId, ui64 seqNo) { +void TNodeState::SendDiscovery() { auto evDiscovery = MakeHolder<TEvDqCompute::TEvChannelDiscoveryV2>(); evDiscovery->Record.SetGenMajor(GenMajor); evDiscovery->Record.SetGenMinor(GenMinor); - evDiscovery->Record.SetSeqNo(seqNo); + evDiscovery->Record.SetSeqNo(SeqNo); ui32 flags = NActors::IEventHandle::FlagTrackDelivery; if (!Subscribed.exchange(true)) { flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(actorId, NodeActorId, evDiscovery.Release(), flags)); + ActorSystem->Send(new NActors::IEventHandle(MakeChannelServiceActorID(NodeId), NodeActorId, evDiscovery.Release(), flags)); } TString TNodeState::GetDebugInfo() { std::lock_guard lock(Mutex); TStringBuilder builder; - builder << "TNodeState, NodeId=" << NodeActorId.NodeId() << ", Peer NodeId=" << PeerActorId.NodeId() + builder << "TNodeState, NodeId=" << NodeActorId.NodeId() << ", Peer NodeId=" << NodeId << ", SeqNo=" << SeqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo << ", InflightBytes=" << InflightBytes.load() << ", Reconciliation=" << Reconciliation.load() << Endl; @@ -2175,7 +2126,7 @@ void TDebugNodeState::HandleNullMode(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto& record = ev->Get()->Record; - PeerGenMinor.store(std::max<ui64>(record.GetGenMinor(), PeerGenMinor.load())); + OutputNodeGenMinor.store(record.GetGenMinor()); auto seqNo = record.GetSeqNo(); @@ -2196,8 +2147,8 @@ void TDebugNodeState::HandleNullMode(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); evAck->Record.SetSeqNo(ConfirmedSeqNo); @@ -2396,6 +2347,7 @@ IDqInputChannel::TPtr TDqChannelService::GetInputChannel(const TDqChannelSetting } void TDqChannelService::NotifyCleanup() { + LOG_T("CLEANUP"); ActorSystem->Schedule(Limits.CleanupPeriod, new NActors::IEventHandle(ServiceActorId, ServiceActorId, new TEvPrivate::TEvCleanup())); std::lock_guard lock(Mutex); for (auto& [_, nodeState] : NodeStates) { @@ -2611,18 +2563,22 @@ void TChannelServiceActor::Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { TABLEH_ATTRS({{"title", "FailureReconciliation"}}) {str << "FR";} #endif TABLEH_ATTRS({{"title", "FailureDestroy"}}) {str << "F~";} - TABLEH_ATTRS({{"title", "GenMajor"}}) {str << "GM";} - TABLEH_ATTRS({{"title", "GenMinor"}}) {str << "gm";} + TABLEH_ATTRS({{"title", "Gen"}}) {str << "G";} + TABLEH_ATTRS({{"title", "Queue.front()->SeqNo"}}) {str << "f/SeqNo";} TABLEH() {str << "SeqNo";} - TABLEH_ATTRS({{"title", "Queue.front()->SeqNo"}}) {str << "front()";} TABLEH_ATTRS({{"title", "InflightBytes"}}) {str << "InflightB";} + TABLEH_ATTRS({{"title", "SendCount"}}) {str << "Send";} + TABLEH_ATTRS({{"title", "ResendCount"}}) {str << "Resend";} + TABLEH_ATTRS({{"title", "ReconCount"}}) {str << "Recon";} + TABLEH() {str << "ReconSent";} TABLEH_ATTRS({{"title", "WaitersQueueSize"}}) {str << "W/Queue";} TABLEH_ATTRS({{"title", "WaitersMessages"}}) {str << "W/Msg";} - TABLEH_ATTRS({{"title", "PeerGenMajor"}}) {str << "PM";} - TABLEH_ATTRS({{"title", "PeerGenMinor"}}) {str << "pm";} + TABLEH_ATTRS({{"title", "OutputNodeGen"}}) {str << "OG";} TABLEH_ATTRS({{"title", "ConfirmedSeqNo"}}) {str << "C/SeqNo";} - TABLEH() {str << "PeerActorId";} + TABLEH() {str << "OutputNodeActorId";} + TABLEH() {str << "InputNodeActorId";} TABLEH() {str << "NodeActorId";} + TABLEH() {str << "ReconciliationLog";} } } TABLEBODY() { @@ -2656,22 +2612,26 @@ void TChannelServiceActor::Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { str << "X"; } } - TABLED() {str << state->GenMajor;} - TABLED() {str << state->GenMinor;} - TABLED() {str << state->SeqNo;} + TABLED() {str << state->GenMajor << '.' << state->GenMinor;} TABLED() { if (!state->Queue.empty()) { str << state->Queue.front()->SeqNo; } } + TABLED() {str << state->SeqNo;} TABLED() {str << state->InflightBytes.load();} + TABLED() {str << state->SendCount.load();} + TABLED() {str << state->ResendCount.load();} + TABLED() {str << state->ReconCount.load();} + TABLED() {str << state->ReconSent.load();} TABLED() {str << state->WaitersQueue.size();} TABLED() {str << state->WaiterMessages.load();} - TABLED() {str << state->PeerGenMajor.load();} - TABLED() {str << state->PeerGenMinor.load();} + TABLED() {str << state->OutputNodeGenMajor.load() << '.' << state->OutputNodeGenMinor.load();} TABLED() {str << state->ConfirmedSeqNo;} - TABLED() {str << state->PeerActorId;} + TABLED() {str << state->OutputNodeActorId;} + TABLED() {str << state->InputNodeActorId;} TABLED() {str << state->NodeActorId;} + TABLED() {str << state->GetReconciliationLog();} } } } diff --git a/ydb/library/yql/dq/runtime/dq_channel_service_impl.h b/ydb/library/yql/dq/runtime/dq_channel_service_impl.h index 032f9412e1b..b1e43f6194d 100644 --- a/ydb/library/yql/dq/runtime/dq_channel_service_impl.h +++ b/ydb/library/yql/dq/runtime/dq_channel_service_impl.h @@ -439,8 +439,8 @@ public: TChannelFullInfo Info; NActors::TActorSystem* ActorSystem; - ui64 PeerGenMajor = 0; - NActors::TActorId PeerActorId; + ui64 OutputNodeGenMajor = 0; + NActors::TActorId OutputNodeActorId; bool IsBound = false; TDqThreadSafeStats PushStats; TDqThreadSafeStats PopStats; @@ -586,13 +586,13 @@ public: void TerminateOutputDescriptor(const std::shared_ptr<TOutputDescriptor>& descriptor); void TerminateInputDescriptor(const std::shared_ptr<TInputDescriptor>& descriptor); void HandleCleanup(); - void FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor); + void FailInputs(const NActors::TActorId& outputNodeActorId, ui64 outputNodeGenMajor); void FailOutputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor); void SendAck(THolder<TEvDqCompute::TEvChannelAckV2>& evAck, ui64 cookie); void SendAckWithError(ui64 cookie, const TString& message); void HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev); void SendFromWaiters(ui64 deltaBytes); - void ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor, ui64 seqNo); + void ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor); virtual TString GetDebugInfo(); void UpdateProgress(std::shared_ptr<TInputDescriptor>& descriptor); void SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descriptor); @@ -602,7 +602,7 @@ public: void DoReconciliation(char logSymbol); void AddReconciliationLog(char logSymbol); TString GetReconciliationLog(); - void SendDiscovery(NActors::TActorId actorId, ui64 seqNo); + void SendDiscovery(); NActors::TActorId NodeActorId; TString LogPrefix; @@ -615,7 +615,6 @@ public: mutable std::unordered_map<TChannelInfo, std::shared_ptr<TInputDescriptor>> InputDescriptors; mutable std::queue<std::pair<TChannelInfo, TInstant>> UnboundInputs; mutable std::queue<std::pair<TChannelInfo, TInstant>> UnboundOutputs; - bool Connected = false; std::weak_ptr<TNodeState> Self; // Sender ui64 GenMajor = 1; @@ -623,12 +622,12 @@ public: ui64 ReconciliationCount = 0; ui64 SeqNo = 0; std::atomic<ui64> InflightBytes = 0; + NActors::TActorId InputNodeActorId; // Receiver - NActors::TActorId PeerActorId; - std::atomic<ui64> PeerGenMajor = 0; - std::atomic<ui64> PeerGenMinor = 0; + NActors::TActorId OutputNodeActorId; + std::atomic<ui64> OutputNodeGenMajor = 0; + std::atomic<ui64> OutputNodeGenMinor = 0; ui64 ConfirmedSeqNo = 0; - TEvDqCompute::TEvChannelDataV2::TPtr OutOfOrderMessage; // ... const TDqChannelLimits Limits; const ui64 MaxInflightMessages = 8192; @@ -660,6 +659,10 @@ public: std::atomic<bool> ResendAsked = false; std::deque<char> ReconciliationLog; TChannelInfo LastLostInfo = TChannelInfo(0, NActors::TActorId{}, NActors::TActorId{}); + std::atomic<ui64> SendCount = 0; + std::atomic<ui64> ResendCount = 0; + std::atomic<ui64> ReconCount = 0; + std::atomic<TInstant> ReconSent; }; class TDebugNodeState : public TNodeState { diff --git a/ydb/library/yql/providers/dq/actors/actor_helpers.h b/ydb/library/yql/providers/dq/actors/actor_helpers.h index 67cdbbd2460..69eaee8cce7 100644 --- a/ydb/library/yql/providers/dq/actors/actor_helpers.h +++ b/ydb/library/yql/providers/dq/actors/actor_helpers.h @@ -10,6 +10,13 @@ namespace NYql { +enum class EActorFutureCallbackFailureReason { + None, + Undelivered, + NodeDisconnected, + Timeout, +}; + enum EExecutorPoolType { Main, FullResultWriter, @@ -25,17 +32,27 @@ struct TRichActorFutureCallback : public TRichActor<TRichActorFutureCallback<Eve static constexpr char ActorName[] = "YQL_DQ_ACTOR_FUTURE_CALLBACK"; - TRichActorFutureCallback(TCallback&& callback, TFailure&& failure, TDuration timeout) + TRichActorFutureCallback( + TCallback&& callback, + TFailure&& failure, + TDuration timeout, + TString context = {}, + NActors::TActorId targetActorId = {}) : TBase(&TRichActorFutureCallback::StateWaitForEvent) , Callback(std::move(callback)) , Failure(std::move(failure)) , Timeout(timeout) + , Context(std::move(context)) + , TargetActorId(targetActorId) { } private: const TCallback Callback; const TFailure Failure; const TDuration Timeout; + const TString Context; + const NActors::TActorId TargetActorId; + EActorFutureCallbackFailureReason FailureReason = EActorFutureCallbackFailureReason::None; bool TimerStarted = false; NActors::TSchedulerCookieHolder TimerCookieHolder; @@ -47,11 +64,26 @@ private: }) hFunc(NActors::TEvInterconnect::TEvNodeDisconnected, [this] (NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) mutable { this->Unsubscribe(ev->Get()->NodeId); + FailureReason = EActorFutureCallbackFailureReason::NodeDisconnected; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: interconnect node disconnected" + << " context=" << Context + << " disconnectedNodeId=" << ev->Get()->NodeId + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); TimerStarted = true; OnFailure(); }) hFunc(NActors::TEvents::TEvUndelivered, [this] (NActors::TEvents::TEvUndelivered::TPtr& ev) mutable { + const auto* undelivered = ev->Get(); this->Unsubscribe(ev->Sender.NodeId()); + FailureReason = EActorFutureCallbackFailureReason::Undelivered; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: event undelivered (fast failure, not a timeout)" + << " context=" << Context + << " undeliveredReason=" << static_cast<int>(undelivered->Reason) + << " sourceEventType=" << undelivered->SourceType + << " unsure=" << undelivered->Unsure + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); TimerStarted = true; OnFailure(); }) @@ -69,6 +101,14 @@ private: void OnFailure() { if (TimerStarted) { + if (FailureReason == EActorFutureCallbackFailureReason::None) { + FailureReason = EActorFutureCallbackFailureReason::Timeout; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: response timeout" + << " context=" << Context + << " timeout=" << Timeout + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); + } Failure(); this->PassAway(); } else { diff --git a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp index 8c143ec3dad..95c31712355 100644 --- a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp +++ b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp @@ -85,7 +85,10 @@ private: Fail((ui64)-1, "GWM Disconnected"); }) hFunc(TEvents::TEvUndelivered, [this](TEvents::TEvUndelivered::TPtr& ev) { - Fail(ev->Cookie, "Undelivered"); + Fail(ev->Cookie, TStringBuilder() + << "Undelivered: sender: " << ev->Sender + << ", reason: " << ev->Get()->Reason + << ", source type: " << ev->Get()->SourceType); }) }) @@ -299,7 +302,8 @@ private: QueryStat.GetCounterName("Actor", {{"ClusterName", maybeRequestInfo->second.ClusterName}}, "CreateFailTimeUs"), delta); } - TString message = "Disconnected from worker: `" + workerInfo + "', reason: " + reason; + TString message = TStringBuilder() + << "Disconnected from worker: `" << workerInfo << "`, reason: " << reason; YQL_CLOG(ERROR, ProviderDq) << message; auto response = MakeHolder<TEvAllocateWorkersResponse>(message, NYql::NDqProto::StatusIds::UNAVAILABLE); QueryStat.FlushCounters(response->Record); diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 6216abea8ad..6f3b24cddca 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -979,7 +979,8 @@ private: return true; }); IDqGateway::TDqProgressWriter progressWriter = MakeDqProgressWriter(publicIds); - bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false); + const bool initialFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false); + bool enableFullResultWrite = initialFullResultWrite; if (enableFullResultWrite) { const auto type = result.Input().Ref().GetTypeAnn(); const auto integration = GetDqIntegrationForFullResTable(State); @@ -991,6 +992,11 @@ private: && integration->PrepareFullResultTableParams(result.Ref(), ctx, graphParams, secureParams, TColumnOrder(columns)); settings->EnableFullResultWrite = enableFullResultWrite; } + YQL_CLOG(INFO, ProviderDq) << "FullResultWrite decision: initial=" << initialFullResultWrite + << " effective=" << enableFullResultWrite + << " discard=" << fillSettings.Discard + << " hasDqGateway=" << (bool)State->DqGateway + << " hasYtFullResultParam=" << graphParams.contains("yt.full_result_table"); TString lambda; bool untrustedUdfFlag; @@ -1086,10 +1092,13 @@ private: } return WrapFutureCallback<false>(future, [localRun, startTime, type, rowSpec, skiffType, fillSettings, level, settings, enableFullResultWrite, columns, graphParams, state = State, skiffConverter = SkiffConverter](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - YQL_CLOG(DEBUG, ProviderDq) << state->SessionId << " WrapFutureCallback"; - auto duration = TInstant::Now() - startTime; - YQL_CLOG(INFO, ProviderDq) << "Execution Result complete, duration: " << duration; + YQL_CLOG(INFO, ProviderDq) << state->SessionId << " Execution Result complete, duration: " << duration + << " localRun=" << localRun + << " enableFullResultWrite=" << enableFullResultWrite + << " truncated=" << res.Truncated + << " rowsCount=" << res.RowsCount + << " dataBytes=" << res.Data.size(); if (state->Metrics) { state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds()); state->Metrics->SetCounter( diff --git a/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp b/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp index 23e559c28b9..bfbc2f79713 100644 --- a/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp +++ b/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp @@ -1,6 +1,5 @@ #include "ydb/core/testlib/basics/appdata.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> diff --git a/ydb/public/api/protos/ydb_query.proto b/ydb/public/api/protos/ydb_query.proto index b4dc75df786..e48f4fcb056 100644 --- a/ydb/public/api/protos/ydb_query.proto +++ b/ydb/public/api/protos/ydb_query.proto @@ -142,6 +142,10 @@ message CommitTransactionRequest { message CommitTransactionResponse { StatusIds.StatusCode status = 1; repeated Ydb.Issue.IssueMessage issues = 2; + + // Commit timestamp (PlanStep, TxId) for StrictSerializableRW write transactions. + // Present only on SUCCESS and when the transaction had write effects. + Ydb.VirtualTimestamp commit_timestamp = 3; } message RollbackTransactionRequest { @@ -255,6 +259,10 @@ message ExecuteQueryResponsePart { TransactionMeta tx_meta = 6; VirtualTimestamp snapshot_timestamp = 7; + + // Commit timestamp (PlanStep, TxId) for StrictSerializableRW write transactions. + // Present only in the final (trailing) part on SUCCESS and when the transaction had write effects. + VirtualTimestamp commit_timestamp = 8; } message ExecuteScriptRequest { diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index d185fcb6b4b..25deb7ee33e 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -488,6 +488,7 @@ message SetNotNullState { STATE_APPLYING = 3; STATE_DONE = 4; STATE_CANCELLED = 5; + STATE_REJECTED = 6; } } diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto index 0b4db6267c9..e14efd37cd0 100644 --- a/ydb/public/api/protos/ydb_topic.proto +++ b/ydb/public/api/protos/ydb_topic.proto @@ -722,12 +722,14 @@ message TransactionIdentity { } // Identity of a deferred topic publication for StreamWrite. -// int_publication_id is assigned by the server in BeginPublication. -// ext_publication_id is repeated from BeginPublication on each write batch. +// int_publication_id is assigned by the server in BeginPublication and is authoritative. +// ext_publication_id is optional and informational only (diagnostics / TWriteId display). +// When set, the server does not require a non-empty value and does not check that it +// matches BeginPublication. Omitting the field and setting an empty string are both allowed. message DeferredPublishIdentity { uint64 int_publication_id = 1; - string ext_publication_id = 2 [(Ydb.length).le = 2048]; + optional string ext_publication_id = 2 [(Ydb.length).le = 2048]; } // Add offsets to transaction request sent from client to server. diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index ea48c937654..f9168caf16c 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -22,6 +22,8 @@ namespace NYdb::inline Dev { +using NCredentials::NDetail::TOwningFacilityCredentialsProvider; + constexpr std::chrono::milliseconds BACKOFF_START{50}; constexpr std::chrono::milliseconds BACKOFF_MAX{10000}; constexpr std::chrono::milliseconds PERIODIC_TICK{100}; @@ -77,19 +79,16 @@ private: std::weak_ptr<ICoreFacility> responseFacility, TCredentialsProviderPtr authTokenProvider) : Rpc_(rpc) - , Ticket_("") , NextTicketUpdate_(SysTimePoint{}) , IamEndpoint_(iamEndpoint) , RequestFiller_(requestFiller) , Context_(std::nullopt) - , LastRequestError_("") , NeedStop_(false) , BackoffTimeout_(BACKOFF_START) , Lock_() , ResponseFacility_(std::move(responseFacility)) , AuthTokenProvider_(authTokenProvider) - , FirstTokenReady_(NThreading::NewPromise<void>()) - , FirstTokenReadySet_(false) + , AuthInfo_(NThreading::NewPromise<std::string>()) { std::shared_ptr<grpc::ChannelCredentials> creds = nullptr; if (IamEndpoint_.EnableSsl) { @@ -112,7 +111,7 @@ private: void StartPeriodicTask() { auto facility = ResponseFacility_.lock(); if (!facility) { - FailFirstToken("IAM-token provider response facility is not available"); + Fail("IAM-token provider response facility is not available"); return; } @@ -125,7 +124,7 @@ private: return false; } if (status != EStatus::SUCCESS) { - self->FailFirstToken(TStringBuilder() + self->Fail(TStringBuilder() << "IAM-token provider periodic task failed with status " << static_cast<int>(status)); return false; @@ -135,53 +134,30 @@ private: PERIODIC_TICK ); } catch (...) { - FailFirstToken(TStringBuilder() + Fail(TStringBuilder() << "Failed to start IAM-token provider periodic task: " << CurrentExceptionMessage()); } } - std::string GetTicket() { + NThreading::TFuture<std::string> GetAuthInfoAsync() { std::lock_guard guard(Lock_); - if (Ticket_.empty()) { - ythrow yexception() << "IAM-token not ready yet. " << LastRequestError_; - } - return Ticket_; - } - - void WaitForToken() { - std::unique_lock guard(Lock_); - TokenReady_.wait_for(guard, - std::chrono::microseconds(2 * IamEndpoint_.RequestTimeout.MicroSeconds()), - [this]() { - return NeedStop_ || !Ticket_.empty(); - } - ); - } - - NThreading::TFuture<void> GetReadyFuture() const { - return FirstTokenReady_.GetFuture(); + return AuthInfo_.GetFuture(); } void Stop() { - bool setStoppedException = false; + NThreading::TPromise<std::string> promise; { std::unique_lock guard(Lock_); - if (NeedStop_) { - return; - } NeedStop_ = true; - setStoppedException = MarkFirstTokenReadyLocked(); - TokenReady_.notify_all(); + promise = AuthInfo_; if (Context_.has_value()) { Context_->TryCancel(); } ContextReady_.wait(guard, [this]() { return !Context_.has_value(); }); } - if (setStoppedException) { - FirstTokenReady_.SetException( - std::make_exception_ptr(yexception() << "IAM-token provider stopped before token was ready")); - } + promise.TrySetException(std::make_exception_ptr( + yexception() << "IAM-token provider stopped before token was ready")); Stub_.reset(); Channel_.reset(); } @@ -189,21 +165,17 @@ private: private: using SysDuration = SysClock::duration; - bool MarkFirstTokenReadyLocked() { - return !std::exchange(FirstTokenReadySet_, true); - } - - void FailFirstToken(std::string error) { - bool setException = false; + void Fail(std::string error) { + NThreading::TPromise<std::string> promise; { std::lock_guard guard(Lock_); - if ((setException = MarkFirstTokenReadyLocked())) { - LastRequestError_ = error; + NeedStop_ = true; + promise = AuthInfo_; + if (Context_) { + Context_->TryCancel(); } } - if (setException) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << error)); - } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); } static SysDuration ToBoundedSysDuration(const TDuration& d) { @@ -251,16 +223,11 @@ private: } if (auto self = weakSelf.lock()) { - bool failFirstToken; { std::lock_guard guard(self->Lock_); - failFirstToken = self->MarkFirstTokenReadyLocked(); self->ResetContextImpl(); } - if (failFirstToken) { - self->FirstTokenReady_.SetException(std::make_exception_ptr( - yexception() << "IAM-token provider response facility is not available")); - } + self->Fail("IAM-token provider response facility is not available"); } }; @@ -268,42 +235,36 @@ private: try { RequestFiller_(req); + Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } catch (...) { - std::optional<std::string> firstTokenError; - const auto now = SysClock::now(); - { - std::lock_guard guard(Lock_); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) - << ". Failed to prepare IAM request: " << CurrentExceptionMessage(); - if (MarkFirstTokenReadyLocked()) { - firstTokenError = LastRequestError_; - } - ResetContextImpl(); - RescheduleOnFailure(); - } - if (firstTokenError) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); - } - return; + std::lock_guard guard(Lock_); + ResetContextImpl(); + RescheduleOnFailure(); } - - Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } - void FillContext(std::unique_lock<std::mutex>& guard) { + bool FillContext(std::unique_lock<std::mutex>& guard) { std::optional<std::string> authToken; if (AuthTokenProvider_) { guard.unlock(); try { - authToken = AuthTokenProvider_->GetAuthInfo(); + if (!AuthTokenInfo_.Initialized()) { + AuthTokenInfo_ = AuthTokenProvider_->GetAuthInfoAsync(); + } + if (!AuthTokenInfo_.IsReady()) { + guard.lock(); + return false; + } + authToken = AuthTokenInfo_.GetValue(); + AuthTokenInfo_ = {}; } catch (...) { + AuthTokenInfo_ = {}; guard.lock(); throw; } guard.lock(); if (NeedStop_) { - return; + return false; } } @@ -317,6 +278,7 @@ private: if (authToken) { context.AddMetadata("authorization", "Bearer " + *authToken); } + return true; } void ResetContextImpl() { @@ -332,8 +294,9 @@ private: } bool OnPeriodicTick() { - std::optional<std::string> firstTokenError; + std::optional<std::string> terminalError; bool updateTicket = false; + bool authPending = false; { std::unique_lock guard(Lock_); if (NeedStop_) { @@ -342,16 +305,15 @@ private: if (Context_.has_value() || SysClock::now() < NextTicketUpdate_) { return true; } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise<std::string>(); + } try { - FillContext(guard); + authPending = !FillContext(guard); } catch (...) { - const auto now = SysClock::now(); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) + terminalError = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) << ". Failed to prepare IAM request context: " << CurrentExceptionMessage(); - if (MarkFirstTokenReadyLocked()) { - firstTokenError = LastRequestError_; - } ResetContextImpl(); } if (NeedStop_) { @@ -359,13 +321,16 @@ private: return false; } if (!Context_.has_value()) { - RescheduleOnFailure(); + if (!authPending && !terminalError) { + RescheduleOnFailure(); + } } else { updateTicket = true; } } - if (firstTokenError) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); + if (terminalError) { + Fail(*terminalError); + return false; } if (updateTicket) { UpdateTicket(); @@ -374,38 +339,52 @@ private: } void ProcessIamResponse(grpc::Status&& status, TResponse&& result) { - bool setFirstTokenReady = false; + std::optional<std::string> token; + std::optional<std::string> terminalError; + NThreading::TPromise<std::string> promise; { std::lock_guard guard(Lock_); if (!status.ok()) { - LastRequestError_ = TStringBuilder() + const std::string error = TStringBuilder() << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) << ". GrpcStatusCode: " << static_cast<int>(status.error_code()) << " Message: \"" << status.error_message() << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; - RescheduleOnFailure(); + if (IsRetryable(status.error_code())) { + RescheduleOnFailure(); + } else { + terminalError = error; + } + } else if (result.iam_token().empty()) { + terminalError = "IAM-token service returned an empty token"; } else { - LastRequestError_ = ""; - Ticket_ = result.iam_token(); + token = result.iam_token(); + promise = AuthInfo_; const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); RescheduleOnSuccess(expiresAt); - - setFirstTokenReady = MarkFirstTokenReadyLocked(); - TokenReady_.notify_all(); } ResetContextImpl(); } - if (setFirstTokenReady) { - FirstTokenReady_.SetValue(); + if (token) { + promise.TrySetValue(std::move(*token)); + } else if (terminalError) { + Fail(*terminalError); } } + static bool IsRetryable(grpc::StatusCode code) { + return code == grpc::StatusCode::CANCELLED || code == grpc::StatusCode::UNKNOWN || + code == grpc::StatusCode::DEADLINE_EXCEEDED || code == grpc::StatusCode::RESOURCE_EXHAUSTED || + code == grpc::StatusCode::ABORTED || code == grpc::StatusCode::INTERNAL || + code == grpc::StatusCode::UNAVAILABLE; + } + void RescheduleOnFailure() { // call with Lock_ const auto now = SysClock::now(); const auto retryDelay = std::min(BackoffTimeout_, BACKOFF_MAX); @@ -431,21 +410,18 @@ private: std::shared_ptr<typename TService::Stub> Stub_; TAsyncRpc Rpc_; - std::string Ticket_; SysTimePoint NextTicketUpdate_; const TIamEndpoint IamEndpoint_; const TRequestFiller RequestFiller_; std::optional<grpc::ClientContext> Context_; std::condition_variable ContextReady_; - std::condition_variable TokenReady_; - std::string LastRequestError_; bool NeedStop_; std::chrono::milliseconds BackoffTimeout_; std::mutex Lock_; std::weak_ptr<ICoreFacility> ResponseFacility_; TCredentialsProviderPtr AuthTokenProvider_; - NThreading::TPromise<void> FirstTokenReady_; - bool FirstTokenReadySet_; + NThreading::TFuture<std::string> AuthTokenInfo_; + NThreading::TPromise<std::string> AuthInfo_; }; public: @@ -453,14 +429,10 @@ public: const TRequestFiller& requestFiller, TAsyncRpc rpc, std::weak_ptr<ICoreFacility> responseFacility, - TCredentialsProviderPtr authTokenProvider = nullptr, - bool waitForToken = true) + TCredentialsProviderPtr authTokenProvider = nullptr) : Impl_(std::make_shared<TImpl>(endpoint, requestFiller, rpc, std::move(responseFacility), authTokenProvider)) { Impl_->StartPeriodicTask(); - if (waitForToken) { - Impl_->WaitForToken(); - } } ~TGrpcIamCredentialsProvider() { @@ -468,90 +440,43 @@ public: } std::string GetAuthInfo() const override { - return Impl_->GetTicket(); + return GetAuthInfoAsync().GetValueSync(); } - bool IsValid() const override { - return true; - } - - NThreading::TFuture<void> GetReadyFuture() const { - return Impl_->GetReadyFuture(); - } - -private: - std::shared_ptr<TImpl> Impl_; -}; - -// Adapter that keeps a self-owned ICoreFacility alive for the lifetime of an inner credentials -// provider. Used by deprecated no-arg ICredentialsProviderFactory::CreateProvider() paths where -// the caller hasn't supplied a facility. -class TOwningFacilityCredentialsProvider : public ICredentialsProvider { -public: - TOwningFacilityCredentialsProvider(std::shared_ptr<ICoreFacility> facility, - TCredentialsProviderPtr inner) - : Facility_(std::move(facility)) - , Inner_(std::move(inner)) - {} - - std::string GetAuthInfo() const override { - return Inner_->GetAuthInfo(); + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { + return Impl_->GetAuthInfoAsync(); } bool IsValid() const override { - return Inner_->IsValid(); + return true; } private: - // Field declaration order matters: Inner_ is destroyed first so that its Stop() can still - // drive the facility's queue (cancel the in-flight gRPC context, drain the response callback), - // and only then is Facility_ destroyed. - std::shared_ptr<ICoreFacility> Facility_; - TCredentialsProviderPtr Inner_; + std::shared_ptr<TImpl> Impl_; }; -namespace NPrivate { - -template <typename TProvider, typename TParams> -NThreading::TFuture<TCredentialsProviderPtr> CreateGrpcIamCredentialsProviderAsync( - const TParams& params, - std::weak_ptr<ICoreFacility> facility, - std::shared_ptr<ICoreFacility> ownedFacility = {}) -{ - auto inner = std::make_shared<TProvider>(params, std::move(facility), false); - auto ready = inner->GetReadyFuture(); - TCredentialsProviderPtr provider = std::move(inner); - if (ownedFacility) { - provider = std::make_shared<TOwningFacilityCredentialsProvider>( - std::move(ownedFacility), std::move(provider)); - } - return ready.Return(std::move(provider)); -} - -} // namespace NPrivate - template<typename TRequest, typename TResponse, typename TService> class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider<TRequest, TResponse, TService> { public: - TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr<ICoreFacility> responseFacility, bool waitForToken = true) + TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr<ICoreFacility> responseFacility) : TGrpcIamCredentialsProvider<TRequest, TResponse, TService>(params, [jwtParams = params.JwtParams](TRequest& req) { req.set_jwt(MakeSignedJwt(jwtParams)); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function<void(grpc::Status)> cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr, waitForToken) {} + }, std::move(responseFacility)) {} }; template<typename TRequest, typename TResponse, typename TService> class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider<TRequest, TResponse, TService> { public: - TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr<ICoreFacility> responseFacility, bool waitForToken = true) + TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr<ICoreFacility> responseFacility) : TGrpcIamCredentialsProvider<TRequest, TResponse, TService>(params, [token = params.OAuthToken](TRequest& req) { req.set_yandex_passport_oauth_token(TStringType{token}); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function<void(grpc::Status)> cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr, waitForToken) {} + }, std::move(responseFacility)) {} }; template<typename TRequest, typename TResponse, typename TService> @@ -574,12 +499,6 @@ public: }); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync() const override { - auto facility = CreateSimpleCoreFacility(); - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamJwtCredentialsProvider<TRequest, TResponse, TService>>(Params_, facility, facility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { return std::make_shared<TIamJwtCredentialsProvider<TRequest, TResponse, TService>>(Params_, std::move(facility)); } @@ -595,11 +514,6 @@ public: Params_.JwtParams.PrivKey); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const override { - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamJwtCredentialsProvider<TRequest, TResponse, TService>>(Params_, std::move(facility)); - } - private: TIamJwtParams Params_; }; @@ -622,12 +536,6 @@ public: }); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync() const override { - auto facility = CreateSimpleCoreFacility(); - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamOAuthCredentialsProvider<TRequest, TResponse, TService>>(Params_, facility, facility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { return std::make_shared<TIamOAuthCredentialsProvider<TRequest, TResponse, TService>>(Params_, std::move(facility)); } @@ -640,11 +548,6 @@ public: Params_.OAuthToken); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const override { - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamOAuthCredentialsProvider<TRequest, TResponse, TService>>(Params_, std::move(facility)); - } - private: TIamOAuth Params_; }; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h index 62464535561..91ed6635b04 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/client.h @@ -8,6 +8,7 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/params/params.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/tx/tx.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h> @@ -293,19 +294,25 @@ public: const std::optional<TTransaction>& GetTransaction() const { return Transaction_; } - TExecuteQueryPart(TStatus&& status, std::optional<TExecStats>&& queryStats, std::optional<TTransaction>&& tx) + const std::optional<NScheme::TVirtualTimestamp>& GetCommitTimestamp() const { return CommitTimestamp_; } + + TExecuteQueryPart(TStatus&& status, std::optional<TExecStats>&& queryStats, std::optional<TTransaction>&& tx, + std::optional<NScheme::TVirtualTimestamp>&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} TExecuteQueryPart(TStatus&& status, TResultSet&& resultSet, int64_t resultSetIndex, - std::optional<TExecStats>&& queryStats, std::optional<TTransaction>&& tx) + std::optional<TExecStats>&& queryStats, std::optional<TTransaction>&& tx, + std::optional<NScheme::TVirtualTimestamp>&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) , ResultSetIndex_(resultSetIndex) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: @@ -313,6 +320,7 @@ private: int64_t ResultSetIndex_ = 0; std::optional<TExecStats> Stats_; std::optional<TTransaction> Transaction_; + std::optional<NScheme::TVirtualTimestamp> CommitTimestamp_; }; class TExecuteQueryResult : public TStatus { @@ -325,22 +333,27 @@ public: std::optional<TTransaction> GetTransaction() const {return Transaction_; } + const std::optional<NScheme::TVirtualTimestamp>& GetCommitTimestamp() const { return CommitTimestamp_; } + TExecuteQueryResult(TStatus&& status) : TStatus(std::move(status)) {} TExecuteQueryResult(TStatus&& status, std::vector<TResultSet>&& resultSets, - std::optional<TExecStats>&& stats, std::optional<TTransaction>&& tx) + std::optional<TExecStats>&& stats, std::optional<TTransaction>&& tx, + std::optional<NScheme::TVirtualTimestamp>&& commitTimestamp = {}) : TStatus(std::move(status)) , ResultSets_(std::move(resultSets)) , Stats_(std::move(stats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: std::vector<TResultSet> ResultSets_; std::optional<TExecStats> Stats_; std::optional<TTransaction> Transaction_; + std::optional<NScheme::TVirtualTimestamp> CommitTimestamp_; }; } // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h index 5f0fe3462b3..91d9c54b56e 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/query/query.h @@ -6,6 +6,7 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/retry/retry.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/fluent_settings_helpers.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/operation/operation.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/request_settings.h> @@ -131,6 +132,12 @@ struct TDeleteSessionSettings : public TRequestSettings<TDeleteSessionSettings> class TCommitTransactionResult : public TStatus { public: TCommitTransactionResult(TStatus&& status); + TCommitTransactionResult(TStatus&& status, std::optional<NScheme::TVirtualTimestamp>&& commitTimestamp); + + const std::optional<NScheme::TVirtualTimestamp>& GetCommitTimestamp() const { return CommitTimestamp_; } + +private: + std::optional<NScheme::TVirtualTimestamp> CommitTimestamp_; }; using TAsyncBeginTransactionResult = NThreading::TFuture<TBeginTransactionResult>; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h index b638b824bb6..99e921a9105 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h @@ -1,6 +1,7 @@ #pragma once #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/driver/driver.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h> namespace Ydb { class VirtualTimestamp; @@ -57,25 +58,6 @@ enum class ESchemeEntryType : i32 { Secret = 26, }; -struct TVirtualTimestamp { - uint64_t PlanStep = 0; - uint64_t TxId = 0; - - TVirtualTimestamp() = default; - TVirtualTimestamp(uint64_t planStep, uint64_t txId); - TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); - - std::string ToString() const; - void Out(IOutputStream& out) const; - - bool operator<(const TVirtualTimestamp& rhs) const; - bool operator<=(const TVirtualTimestamp& rhs) const; - bool operator>(const TVirtualTimestamp& rhs) const; - bool operator>=(const TVirtualTimestamp& rhs) const; - bool operator==(const TVirtualTimestamp& rhs) const; - bool operator!=(const TVirtualTimestamp& rhs) const; -}; - struct TSchemeEntry { std::string Name; std::string Owner; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h index e4d428fb02e..398fe89df76 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h @@ -19,7 +19,7 @@ public: // Add task to execute periodicaly // Task should return false to stop execution virtual void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) = 0; - // Post task on SDK response executor. + // Post task on SDK response executor, never inline. virtual void PostToResponseQueue(TPostTaskCb&& f) = 0; }; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h index 67068921f84..8f31135be55 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -5,6 +5,7 @@ #include <functional> #include <library/cpp/threading/future/future.h> +#include <exception> #include <memory> #include <string> #include <utility> @@ -16,9 +17,17 @@ public: virtual ~ICredentialsProvider() = default; virtual std::string GetAuthInfo() const = 0; virtual bool IsValid() const = 0; + virtual NThreading::TFuture<std::string> GetAuthInfoAsync() const { + try { + return NThreading::MakeFuture(GetAuthInfo()); + } catch (...) { + return NThreading::MakeErrorFuture<std::string>(std::current_exception()); + } + } }; using TCredentialsProviderPtr = std::shared_ptr<ICredentialsProvider>; +class ICoreFacility; // Implementation detail for SDK credentials factories. Symbols in NCredentials::NDetail are not // part of the public YDB C++ SDK API and may change or be removed without notice. @@ -26,6 +35,35 @@ namespace NCredentials::NDetail { using TCredentialsProviderCreator = std::function<TCredentialsProviderPtr()>; +class TOwningFacilityCredentialsProvider final : public ICredentialsProvider { +public: + TOwningFacilityCredentialsProvider(std::shared_ptr<ICoreFacility> facility, + TCredentialsProviderPtr inner, + bool forwardAsync = false) + : Facility_(std::move(facility)) + , Inner_(std::move(inner)) + , ForwardAsync_(forwardAsync) + {} + + std::string GetAuthInfo() const override { + return Inner_->GetAuthInfo(); + } + + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { + return ForwardAsync_ ? Inner_->GetAuthInfoAsync() : ICredentialsProvider::GetAuthInfoAsync(); + } + + bool IsValid() const override { + return Inner_->IsValid(); + } + +private: + // Reverse destruction keeps Facility_ alive while Inner_ stops. + std::shared_ptr<ICoreFacility> Facility_; + TCredentialsProviderPtr Inner_; + const bool ForwardAsync_; +}; + // Process-wide weak cache for no-argument factory paths whose providers own their facilities. // Facility-bound providers must not use it: their callbacks belong to the supplied facility. TCredentialsProviderPtr GetOrCreateCachedProvider( @@ -34,21 +72,15 @@ TCredentialsProviderPtr GetOrCreateCachedProvider( } // namespace NCredentials::NDetail -class ICoreFacility; class ICredentialsProviderFactory { public: virtual ~ICredentialsProviderFactory() = default; // deprecated, use CreateProvider(std::weak_ptr<ICoreFacility> facility) instead virtual TCredentialsProviderPtr CreateProvider() const = 0; - virtual NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync() const { - return NThreading::MakeFuture(CreateProvider()); - } + // The facility must outlive the returned provider. virtual TCredentialsProviderPtr CreateProvider([[maybe_unused]] std::weak_ptr<ICoreFacility> facility) const { return CreateProvider(); } - virtual NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const { - return NThreading::MakeFuture(CreateProvider(std::move(facility))); - } virtual std::string GetClientIdentity() const; }; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h new file mode 100644 index 00000000000..ce71b9010f3 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h @@ -0,0 +1,32 @@ +#pragma once + +#include <ydb/public/api/protos/ydb_common.pb.h> + +#include <util/stream/output.h> + +#include <tuple> + +namespace NYdb::inline Dev { +namespace NScheme { + +struct TVirtualTimestamp { + uint64_t PlanStep = 0; + uint64_t TxId = 0; + + TVirtualTimestamp() = default; + TVirtualTimestamp(uint64_t planStep, uint64_t txId); + TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); + + std::string ToString() const; + void Out(IOutputStream& out) const; + + bool operator<(const TVirtualTimestamp& rhs) const; + bool operator<=(const TVirtualTimestamp& rhs) const; + bool operator>(const TVirtualTimestamp& rhs) const; + bool operator>=(const TVirtualTimestamp& rhs) const; + bool operator==(const TVirtualTimestamp& rhs) const; + bool operator!=(const TVirtualTimestamp& rhs) const; +}; + +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/iam/iam.cpp b/ydb/public/sdk/cpp/src/client/iam/iam.cpp index 06bbbccc54e..72383cc0bfe 100644 --- a/ydb/public/sdk/cpp/src/client/iam/iam.cpp +++ b/ydb/public/sdk/cpp/src/client/iam/iam.cpp @@ -15,120 +15,156 @@ using namespace yandex::cloud::iam::v1; namespace NYdb::inline Dev { -class TIAMCredentialsProvider : public ICredentialsProvider { +class TIAMCredentialsProvider : public ICredentialsProvider, public std::enable_shared_from_this<TIAMCredentialsProvider> { public: - TIAMCredentialsProvider(const TIamHost& params) + TIAMCredentialsProvider(const TIamHost& params, std::weak_ptr<ICoreFacility> facility) : HttpClient_(TSimpleHttpClient(TString(params.Host), params.Port)) , Request_("/computeMetadata/v1/instance/service-accounts/default/token") , NextTicketUpdate_(TInstant::Zero()) , RefreshPeriod_(params.RefreshPeriod) - { - GetTicket(); - } + , Facility_(std::move(facility)) + , AuthInfo_(NThreading::NewPromise<std::string>()) + {} - std::string GetAuthInfo() const override { - std::string ticket; - TInstant nextTicketUpdate; - auto now = TInstant::Now(); - std::optional<TString> lastErrorMessage; - { - std::lock_guard lock(Lock_); - if (LastErrorMessage_.has_value() && now > ExpiresAt_) { - Ticket_.clear(); - } - ticket = Ticket_; - nextTicketUpdate = NextTicketUpdate_; - lastErrorMessage = LastErrorMessage_; + void Start() { + auto facility = Facility_.lock(); + if (!facility) { + Fail("IAM-token provider response facility is not available"); + return; } - if (now >= nextTicketUpdate) { - GetTicket(); - { - std::lock_guard lock(Lock_); - if (LastErrorMessage_.has_value() && now > ExpiresAt_) { - Ticket_.clear(); + try { + facility->AddPeriodicTask([weak = weak_from_this()](NYdb::NIssue::TIssues&&, EStatus status) { + if (auto self = weak.lock()) { + return self->OnPeriodicTick(status); } - ticket = Ticket_; - lastErrorMessage = LastErrorMessage_; - } - } - if (ticket.empty() && lastErrorMessage.has_value()) { - throw yexception() << *lastErrorMessage; + return false; + }, PERIODIC_TICK); + } catch (...) { + Fail(CurrentExceptionMessage()); } - return ticket; + } + + std::string GetAuthInfo() const override { + return GetAuthInfoAsync().GetValueSync(); + } + + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { + std::lock_guard lock(Lock_); + return AuthInfo_.GetFuture(); } bool IsValid() const override { return true; } + ~TIAMCredentialsProvider() { Fail("IAM-token provider stopped"); } + private: TSimpleHttpClient HttpClient_; std::string Request_; mutable std::mutex Lock_; - mutable std::string Ticket_; mutable TInstant NextTicketUpdate_; - mutable TInstant ExpiresAt_ = TInstant::Zero(); - mutable std::optional<TString> LastErrorMessage_; TDuration RefreshPeriod_; + std::weak_ptr<ICoreFacility> Facility_; + mutable NThreading::TPromise<std::string> AuthInfo_; + mutable bool Stopped_ = false; - void GetTicket() const { - try { - TStringStream out; - TSimpleHttpClient::THeaders headers; - headers["Metadata-Flavor"] = "Google"; - HttpClient_.DoGet(Request_, &out, headers); - NJson::TJsonValue resp; - NJson::ReadJsonTree(&out, &resp, true); - - auto respMap = resp.GetMap(); + void Fail(std::string error) const { + NThreading::TPromise<std::string> promise; + { + std::lock_guard lock(Lock_); + Stopped_ = true; + promise = AuthInfo_; + } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); + } - std::string ticket; - if (auto it = respMap.find("access_token"); it == respMap.end()) - ythrow yexception() << "Result doesn't contain access_token"; - else if (ticket = it->second.GetStringSafe(); ticket.empty()) - ythrow yexception() << "Got empty ticket"; + bool OnPeriodicTick(EStatus status) const { + if (status != EStatus::SUCCESS) { + Fail("IAM-token provider periodic task failed"); + return false; + } - const auto now = TInstant::Now(); - TInstant nextUpdate; - TDuration expiresIn; - TInstant expiresAt = TInstant::Max(); - if (auto it = respMap.find("expires_in"); it != respMap.end()) { - auto seconds = it->second.GetUInteger(); - if (seconds > 0) { - expiresIn = TDuration::Seconds(seconds); - expiresAt = now + expiresIn; - } - } else if (auto it = respMap.find("expiry"); it != respMap.end()) { - try { - TInstant expiry; - if (TInstant::TryParseIso8601(it->second.GetStringSafe(), expiry) && expiry > now) { - expiresIn = expiry - now; - expiresAt = expiry; - } - } catch (...) { - expiresAt = now; - } + NThreading::TPromise<std::string> promise; + { + std::lock_guard lock(Lock_); + if (Stopped_) { + return false; } - if (expiresIn > TDuration::Zero()) { - const auto halfLife = expiresIn / 2; - const auto interval = std::max(std::min(halfLife, RefreshPeriod_), TDuration::MilliSeconds(100)); - nextUpdate = now + interval; - } else { - nextUpdate = now + std::min(RefreshPeriod_, TDuration::Minutes(30)); + if (TInstant::Now() < NextTicketUpdate_) { + return true; } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise<std::string>(); + } + promise = AuthInfo_; + } + try { + auto [ticket, nextUpdate] = GetTicket(); { std::lock_guard lock(Lock_); - Ticket_ = std::move(ticket); NextTicketUpdate_ = nextUpdate; - ExpiresAt_ = expiresAt; - LastErrorMessage_.reset(); } + promise.TrySetValue(std::move(ticket)); } catch (...) { + const auto error = std::current_exception(); + if (!IsRetryable(error)) { + Fail(CurrentExceptionMessage()); + return false; + } std::lock_guard lock(Lock_); NextTicketUpdate_ = TInstant::Now() + std::min(RefreshPeriod_, TDuration::Seconds(10)); - LastErrorMessage_ = CurrentExceptionMessage(); } + return true; + } + + static bool IsRetryable(const std::exception_ptr& error) { + try { + std::rethrow_exception(error); + } catch (const THttpRequestException& e) { + const int code = e.GetStatusCode(); + return code == 0 || code == HTTP_REQUEST_TIME_OUT || code == HTTP_AUTHENTICATION_TIMEOUT || + code == HTTP_TOO_MANY_REQUESTS || (code >= 500 && code < 600); + } catch (const TSystemError&) { + return true; + } catch (...) { + return false; + } + } + + std::pair<std::string, TInstant> GetTicket() const { + TStringStream out; + TSimpleHttpClient::THeaders headers; + headers["Metadata-Flavor"] = "Google"; + HttpClient_.DoGet(Request_, &out, headers); + NJson::TJsonValue resp; + NJson::ReadJsonTree(&out, &resp, true); + + auto respMap = resp.GetMap(); + std::string ticket; + if (auto it = respMap.find("access_token"); it == respMap.end()) + ythrow yexception() << "Result doesn't contain access_token"; + else if (ticket = it->second.GetStringSafe(); ticket.empty()) + ythrow yexception() << "Got empty ticket"; + + const auto now = TInstant::Now(); + TDuration expiresIn; + if (auto it = respMap.find("expires_in"); it != respMap.end()) { + const auto seconds = it->second.GetUInteger(); + if (seconds > 0) { + expiresIn = TDuration::Seconds(seconds); + } + } else if (auto it = respMap.find("expiry"); it != respMap.end()) { + TInstant expiry; + if (TInstant::TryParseIso8601(it->second.GetStringSafe(), expiry) && expiry > now) { + expiresIn = expiry - now; + } + } + const auto interval = expiresIn > TDuration::Zero() + ? std::max(std::min(expiresIn / 2, RefreshPeriod_), TDuration::MilliSeconds(100)) + : std::min(RefreshPeriod_, TDuration::Minutes(30)); + return {std::move(ticket), now + interval}; } }; @@ -140,21 +176,17 @@ public: return NCredentials::NDetail::GetOrCreateCachedProvider( GetClientIdentity(), [this] { - return std::make_shared<TIAMCredentialsProvider>(Params_); + auto facility = CreateSimpleCoreFacility(); + auto provider = CreateProvider(facility); + return std::make_shared<TOwningFacilityCredentialsProvider>( + std::move(facility), std::move(provider)); }); } - // Keep the facility-taking path driver-scoped; only the no-arg path is process-wide cached. - TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility>) const final { - return std::make_shared<TIAMCredentialsProvider>(Params_); - } - - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync() const final { - return CreateProviderAsync(std::weak_ptr<ICoreFacility>{}); - } - - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const final { - return CreateProviderInBackground(Params_, std::move(facility)); + TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const final { + auto provider = std::make_shared<TIAMCredentialsProvider>(Params_, std::move(facility)); + provider->Start(); + return provider; } std::string GetClientIdentity() const final { @@ -164,30 +196,6 @@ public: } private: - static NThreading::TFuture<TCredentialsProviderPtr> CreateProviderInBackground( - TIamHost params, - std::weak_ptr<ICoreFacility> facility) - { - auto promise = NThreading::NewPromise<TCredentialsProviderPtr>(); - auto createProvider = [params = std::move(params), promise]() mutable { - try { - promise.TrySetValue(std::make_shared<TIAMCredentialsProvider>(params)); - } catch (...) { - promise.TrySetException(std::current_exception()); - } - }; - try { - if (auto core = facility.lock()) { - core->PostToResponseQueue(std::move(createProvider)); - } else { - createProvider(); - } - } catch (...) { - promise.TrySetException(std::current_exception()); - } - return promise.GetFuture(); - } - TIamHost Params_; }; diff --git a/ydb/public/sdk/cpp/src/client/iam_private/common/iam.h b/ydb/public/sdk/cpp/src/client/iam_private/common/iam.h index 676172f40a6..7882ea19c16 100644 --- a/ydb/public/sdk/cpp/src/client/iam_private/common/iam.h +++ b/ydb/public/sdk/cpp/src/client/iam_private/common/iam.h @@ -2,8 +2,6 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/common/generic_provider.h> -#include <library/cpp/threading/future/core/coroutine_traits.h> - namespace NYdb::inline Dev { template<typename TRequest, typename TResponse, typename TService> @@ -27,88 +25,45 @@ private: class TCredentialsProvider : public TGrpcIamCredentialsProvider<TRequest, TResponse, TService> { public: - // TDriver path: a shared facility (TGRpcConnectionsImpl) supports multiple periodic tasks, - // so we can hand the same weak_ptr to the nested auth provider here. - TCredentialsProvider(const TIamServiceParams& params, std::weak_ptr<ICoreFacility> responseFacility) - : TGrpcIamCredentialsProvider<TRequest, TResponse, TService>(params, - MakeRequestFiller(params), - MakeRpc(), - responseFacility, - params.SystemServiceAccountCredentials->CreateProvider(responseFacility)) - {} - - // Standalone (no-arg) path: the caller has already built a self-owning auth provider - // backed by its OWN facility. We must not share `outerFacility` with the auth provider - // because TSimpleCoreFacility allows only one periodic task. TCredentialsProvider(const TIamServiceParams& params, - std::weak_ptr<ICoreFacility> outerFacility, - TCredentialsProviderPtr authProvider, - bool waitForToken = true) + std::weak_ptr<ICoreFacility> responseFacility, + TCredentialsProviderPtr authProvider = {}) : TGrpcIamCredentialsProvider<TRequest, TResponse, TService>(params, MakeRequestFiller(params), MakeRpc(), - std::move(outerFacility), - std::move(authProvider), - waitForToken) + responseFacility, + authProvider ? std::move(authProvider) : + params.SystemServiceAccountCredentials->CreateProvider(responseFacility)) {} }; - static NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsyncImpl( - TIamServiceParams params, - NThreading::TFuture<TCredentialsProviderPtr> authProvider, - std::weak_ptr<ICoreFacility> facility, - std::shared_ptr<ICoreFacility> ownedFacility = {}) - { - auto serviceProvider = std::make_shared<TCredentialsProvider>( - params, std::move(facility), co_await authProvider, false); - auto ready = serviceProvider->GetReadyFuture(); - TCredentialsProviderPtr provider = std::move(serviceProvider); - if (ownedFacility) { - provider = std::make_shared<TOwningFacilityCredentialsProvider>( - std::move(ownedFacility), std::move(provider)); - } - co_return co_await ready.Return(std::move(provider)); - } - public: TIamServiceCredentialsProviderFactory(const TIamServiceParams& params) : Params_(params) {} - // Deprecated. Kept for backward compatibility — see comment on TIamJwtCredentialsProviderFactory. - // The nested auth provider gets its own facility (via a recursive no-arg CreateProvider() that - // owns its private facility). Sharing a TSimpleCoreFacility between two gRPC - // IAM providers would abort: each one registers a periodic task and the facility allows only one. TCredentialsProviderPtr CreateProvider() const override final { return NCredentials::NDetail::GetOrCreateCachedProvider( GetClientIdentity(), [this] { - auto authProvider = Params_.SystemServiceAccountCredentials->CreateProvider(); - auto outerFacility = CreateSimpleCoreFacility(); + auto authProvider = NCredentials::NDetail::GetOrCreateCachedProvider( + "async:" + Params_.SystemServiceAccountCredentials->GetClientIdentity(), [this] { + auto facility = CreateSimpleCoreFacility(); + return std::make_shared<TOwningFacilityCredentialsProvider>(facility, + Params_.SystemServiceAccountCredentials->CreateProvider(facility), true); + }); + auto facility = CreateSimpleCoreFacility(); auto serviceProvider = std::make_shared<TCredentialsProvider>( - Params_, std::weak_ptr<ICoreFacility>(outerFacility), std::move(authProvider)); + Params_, facility, std::move(authProvider)); return std::make_shared<TOwningFacilityCredentialsProvider>( - std::move(outerFacility), std::move(serviceProvider)); + std::move(facility), std::move(serviceProvider)); }); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync() const override { - auto outerFacility = CreateSimpleCoreFacility(); - return CreateProviderAsyncImpl( - Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(), - outerFacility, outerFacility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { return std::make_shared<TCredentialsProvider>(Params_, std::move(facility)); } - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const override { - return CreateProviderAsyncImpl( - Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(facility), - facility); - } - std::string GetClientIdentity() const override final { return NIam::NDetail::MakeClientIdentity( "TIamServiceCredentialsProviderFactory", diff --git a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp index a4b031b29af..b44b0b99e8d 100644 --- a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp +++ b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp @@ -62,29 +62,24 @@ TDbDriverState::TDbDriverState( void TDbDriverState::InitCredentials( std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory ) { - Credentials = credentialsProviderFactory->CreateProviderAsync(weak_from_this()).Apply( - [](const NThreading::TFuture<TCredentialsProviderPtr>& future) { - TCredentials result{future.GetValue()}; + Credentials.Provider = credentialsProviderFactory->CreateProvider(weak_from_this()); #ifndef YDB_GRPC_UNSECURE_AUTH - result.CallCredentials = grpc::MetadataCredentialsFromPlugin( - std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(result.Provider))); + Credentials.CallCredentials = grpc::MetadataCredentialsFromPlugin( + std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(Credentials.Provider))); #endif - return result; - }); - CredentialsReady = Credentials.IgnoreResult(); } NThreading::TFuture<void> TDbDriverState::GetCredentialsReady() const { - return CredentialsReady; + return Credentials.Provider->GetAuthInfoAsync().IgnoreResult(); } std::shared_ptr<ICredentialsProvider> TDbDriverState::GetCredentialsProvider() const { - return Credentials.HasValue() ? Credentials.GetValue().Provider : nullptr; + return Credentials.Provider; } #ifndef YDB_GRPC_UNSECURE_AUTH std::shared_ptr<grpc::CallCredentials> TDbDriverState::GetCallCredentials() const { - return Credentials.HasValue() ? Credentials.GetValue().CallCredentials : nullptr; + return Credentials.CallCredentials; } #endif diff --git a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.h b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.h index 17cc3596b34..05b8b7dc045 100644 --- a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.h +++ b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.h @@ -84,8 +84,7 @@ private: #endif }; - NThreading::TFuture<void> CredentialsReady; - NThreading::TFuture<TCredentials> Credentials; + TCredentials Credentials; mutable std::once_flag ClientTlsValidationOnceFlag_; mutable bool ClientTlsCredentialsValid_ = true; mutable std::string ClientTlsValidationDetail_; diff --git a/ydb/public/sdk/cpp/src/client/impl/internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/src/client/impl/internal/grpc_connections/grpc_connections.h index 6f2302303b7..7c56960e22c 100644 --- a/ydb/public/sdk/cpp/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/ydb/public/sdk/cpp/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -335,7 +335,7 @@ public: if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), [this, requestWrapper = std::move(requestWrapper), userResponseCb = std::move(userResponseCb), - rpc, dbState, requestSettings, context = std::move(context)] + rpc, dbState, requestSettings, context] (std::optional<TPlainStatus> status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { userResponseCb(nullptr, std::move(*status)); @@ -592,7 +592,7 @@ public: if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), - [this, request, responseCb = std::move(responseCb), rpc, dbState, requestSettings, context = std::move(context)] + [this, request, responseCb = std::move(responseCb), rpc, dbState, requestSettings, context] (std::optional<TPlainStatus> status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { responseCb(std::move(*status), nullptr); @@ -699,7 +699,7 @@ public: if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), - [this, connectedCallback = std::move(connectedCallback), rpc, dbState, requestSettings, context = std::move(context)] + [this, connectedCallback = std::move(connectedCallback), rpc, dbState, requestSettings, context] (std::optional<TPlainStatus> status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { connectedCallback(std::move(*status), nullptr); diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp index 1da3278a16d..de5adc71936 100644 --- a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.cpp @@ -29,7 +29,6 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , InitSeqNoPromise(NThreading::NewPromise<ui64>()) , WakeupInterval( Settings.BatchFlushInterval_ != TDuration::Zero() ? @@ -1184,19 +1183,62 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: try to update token"); auto credentialsProvider = DbDriverState->GetCredentialsProvider(); - if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished || Aborting) return; - TClientMessage clientMessage; - auto* updateRequest = clientMessage.mutable_update_token_request(); - auto token = credentialsProvider->GetAuthInfo(); - if (token == PrevToken) + auto authInfo = credentialsProvider->GetAuthInfoAsync(); + if (authInfo.IsReady()) { + UpdateTokenImpl(authInfo); return; + } + UpdateTokenInProgress = true; + try { + authInfo.Subscribe([cbContext = SelfContext](const auto& future) { + if (auto self = cbContext->LockShared()) try { + self->Connections->ScheduleCallback(TDuration::Zero(), [cbContext, future](bool ok) { + if (auto self = cbContext->LockShared()) { + if (!ok) { + self->UpdateTokenInProgress = false; + return; + } + std::lock_guard guard(self->Lock); + self->UpdateTokenImpl(future); + } + }); + } catch (...) { + self->UpdateTokenInProgress = false; + } + }); + } catch (...) { + UpdateTokenInProgress = false; + } +} + +void TWriteSessionImpl::UpdateTokenImpl(const NThreading::TFuture<std::string>& future) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTokenInProgress = false; + if (!SessionEstablished || Aborting) { + return; + } + + std::string token; + try { + token = future.GetValue(); + } catch (...) { + CloseImpl(EStatus::CLIENT_UNAUTHENTICATED, CurrentExceptionMessage()); + return; + } + if (token == PrevToken) { + return; + } + UpdateTokenInProgress = true; - updateRequest->set_token(TStringType{token}); PrevToken = token; LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: updating token"); + TClientMessage clientMessage; + clientMessage.mutable_update_token_request()->set_token(TStringType{token}); Processor->Write(std::move(clientMessage)); } diff --git a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h index 947095722af..1a56d5dde44 100644 --- a/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h +++ b/ydb/public/sdk/cpp/src/client/persqueue_public/impl/write_session_impl.h @@ -325,6 +325,7 @@ private: TStringBuilder LogPrefix() const; void UpdateTokenIfNeededImpl(); + void UpdateTokenImpl(const NThreading::TFuture<std::string>& future); void WriteInternal(TContinuationToken&& continuationToken, std::string_view data, std::optional<ECodec> codec, ui32 originalSize, std::optional<ui64> seqNo = std::nullopt, std::optional<TInstant> createTimestamp = std::nullopt); @@ -387,7 +388,7 @@ private: std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory; TDbDriverStatePtr DbDriverState; std::string PrevToken; - bool UpdateTokenInProgress = false; + std::atomic_bool UpdateTokenInProgress = false; TInstant LastTokenUpdate = TInstant::Zero(); std::shared_ptr<TWriteSessionEventsQueue> EventsQueue; NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. diff --git a/ydb/public/sdk/cpp/src/client/query/client.cpp b/ydb/public/sdk/cpp/src/client/query/client.cpp index 61af1e38afb..e8bdda91f2e 100644 --- a/ydb/public/sdk/cpp/src/client/query/client.cpp +++ b/ydb/public/sdk/cpp/src/client/query/client.cpp @@ -260,7 +260,12 @@ public: obs->End(commitTxStatus.GetStatus(), commitTxStatus.GetEndpoint()); - TCommitTransactionResult commitTxResult(std::move(commitTxStatus)); + std::optional<NScheme::TVirtualTimestamp> commitTimestamp; + if (response->has_commit_timestamp()) { + commitTimestamp = NScheme::TVirtualTimestamp(response->commit_timestamp()); + } + + TCommitTransactionResult commitTxResult(std::move(commitTxStatus), std::move(commitTimestamp)); promise.SetValue(std::move(commitTxResult)); } else { obs->End(status.Status, status.Endpoint); diff --git a/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp index 6911525c777..79589c3b923 100644 --- a/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp +++ b/ydb/public/sdk/cpp/src/client/query/impl/exec_query.cpp @@ -88,6 +88,7 @@ public: std::optional<TExecStats> stats; std::optional<TTransaction> tx; + std::optional<NScheme::TVirtualTimestamp> commitTimestamp; if (self->Response_.has_exec_stats()) { stats = TExecStats(std::move(*self->Response_.mutable_exec_stats())); } @@ -96,16 +97,21 @@ public: tx = TTransaction(self->Session_.value(), self->Response_.tx_meta().id()); } + if (self->Response_.has_commit_timestamp()) { + commitTimestamp = NScheme::TVirtualTimestamp(self->Response_.commit_timestamp()); + } + if (self->Response_.has_result_set()) { promise.SetValue({ std::move(status), TResultSet(std::move(*self->Response_.mutable_result_set())), self->Response_.result_set_index(), std::move(stats), - std::move(tx) + std::move(tx), + std::move(commitTimestamp) }); } else { - promise.SetValue({std::move(status), std::move(stats), std::move(tx)}); + promise.SetValue({std::move(status), std::move(stats), std::move(tx), std::move(commitTimestamp)}); } } }; @@ -160,6 +166,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector<Ydb::ResultSet> ResultSets_; std::optional<TExecStats> Stats_; std::optional<TTransaction> Tx_; + std::optional<NScheme::TVirtualTimestamp> CommitTimestamp_; std::vector<std::string> ArrowSchemas_; std::vector<std::vector<std::string>> BytesData_; std::vector<bool> ResultSetSeen_; @@ -174,6 +181,10 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { self->Stats_ = st; } + if (const auto& ct = part.GetCommitTimestamp()) { + self->CommitTimestamp_ = ct; + } + if (!part.IsSuccess()) { std::optional<TExecStats> stats; std::swap(self->Stats_, stats); @@ -182,12 +193,14 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector<NYdb::NIssue::TIssue> issues; std::vector<Ydb::ResultSet> resultProtos; std::optional<TTransaction> tx; + std::optional<NScheme::TVirtualTimestamp> commitTimestamp; std::vector<std::string> arrowSchemas; std::vector<std::vector<std::string>> bytesData; std::swap(self->Issues_, issues); std::swap(self->ResultSets_, resultProtos); std::swap(self->Tx_, tx); + std::swap(self->CommitTimestamp_, commitTimestamp); std::swap(self->ArrowSchemas_, arrowSchemas); std::swap(self->BytesData_, bytesData); @@ -204,10 +217,11 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues(std::move(issues))), std::move(resultSets), std::move(stats), - std::move(tx) + std::move(tx), + std::move(commitTimestamp) )); } else { - self->Promise_.SetValue(TExecuteQueryResult(std::move(part), {}, std::move(stats), {})); + self->Promise_.SetValue(TExecuteQueryResult(std::move(part), {}, std::move(stats), {})); // No commit timestamp on error } return; diff --git a/ydb/public/sdk/cpp/src/client/query/impl/ya.make b/ydb/public/sdk/cpp/src/client/query/impl/ya.make index 3cd591433bf..0f14ffdd428 100644 --- a/ydb/public/sdk/cpp/src/client/query/impl/ya.make +++ b/ydb/public/sdk/cpp/src/client/query/impl/ya.make @@ -14,6 +14,7 @@ PEERDIR( ydb/public/sdk/cpp/src/client/impl/session ydb/public/sdk/cpp/src/client/impl/observability ydb/public/sdk/cpp/src/client/proto + ydb/public/sdk/cpp/src/client/types ) END() diff --git a/ydb/public/sdk/cpp/src/client/query/query.cpp b/ydb/public/sdk/cpp/src/client/query/query.cpp index f4b8cddceab..d0b547fdd2f 100644 --- a/ydb/public/sdk/cpp/src/client/query/query.cpp +++ b/ydb/public/sdk/cpp/src/client/query/query.cpp @@ -66,4 +66,9 @@ TCommitTransactionResult::TCommitTransactionResult(TStatus&& status) : TStatus(std::move(status)) {} +TCommitTransactionResult::TCommitTransactionResult(TStatus&& status, std::optional<NScheme::TVirtualTimestamp>&& commitTimestamp) + : TStatus(std::move(status)) + , CommitTimestamp_(std::move(commitTimestamp)) +{} + } // namespace NYdb::NQuery diff --git a/ydb/public/sdk/cpp/src/client/query/ya.make b/ydb/public/sdk/cpp/src/client/query/ya.make index 69507e40c1a..27ccdc7d704 100644 --- a/ydb/public/sdk/cpp/src/client/query/ya.make +++ b/ydb/public/sdk/cpp/src/client/query/ya.make @@ -16,6 +16,7 @@ PEERDIR( ydb/public/sdk/cpp/src/client/metrics ydb/public/sdk/cpp/src/client/query/impl ydb/public/sdk/cpp/src/client/result + ydb/public/sdk/cpp/src/client/types ydb/public/sdk/cpp/src/client/types/operation ) diff --git a/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp b/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp index 479929ef268..d3b342cfec9 100644 --- a/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp +++ b/ydb/public/sdk/cpp/src/client/scheme/scheme.cpp @@ -29,52 +29,6 @@ void TPermissions::SerializeTo(::Ydb::Scheme::Permissions& proto) const { } } -TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) - : PlanStep(planStep) - , TxId(txId) -{} - -TVirtualTimestamp::TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto) - : TVirtualTimestamp(proto.plan_step(), proto.tx_id()) -{} - -std::string TVirtualTimestamp::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TVirtualTimestamp::Out(IOutputStream& out) const { - out << "{ plan_step: " << PlanStep - << ", tx_id: " << TxId - << " }"; -} - -bool TVirtualTimestamp::operator<(const TVirtualTimestamp& rhs) const { - return PlanStep < rhs.PlanStep && TxId < rhs.TxId; -} - -bool TVirtualTimestamp::operator<=(const TVirtualTimestamp& rhs) const { - return PlanStep <= rhs.PlanStep && TxId <= rhs.TxId; -} - -bool TVirtualTimestamp::operator>(const TVirtualTimestamp& rhs) const { - return PlanStep > rhs.PlanStep && TxId > rhs.TxId; -} - -bool TVirtualTimestamp::operator>=(const TVirtualTimestamp& rhs) const { - return PlanStep >= rhs.PlanStep && TxId >= rhs.TxId; -} - -bool TVirtualTimestamp::operator==(const TVirtualTimestamp& rhs) const { - return PlanStep == rhs.PlanStep && TxId == rhs.TxId; -} - -bool TVirtualTimestamp::operator!=(const TVirtualTimestamp& rhs) const { - return !(*this == rhs); -} - static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) { switch (entry) { case ::Ydb::Scheme::Entry::DIRECTORY: diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp index df6eb1da819..eea32b545e7 100644 --- a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.cpp @@ -114,7 +114,6 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , MaxBlockMessageCount(Settings.BatchFlushMessageCount_) , InitSeqNoPromise(NThreading::NewPromise<uint64_t>()) , WakeupInterval( @@ -1587,11 +1586,53 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefixImpl() << "Write session: try to update token"); auto credentialsProvider = DbDriverState->GetCredentialsProvider(); - if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) { + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished || Aborting) { return; } - auto token = credentialsProvider->GetAuthInfo(); + auto authInfo = credentialsProvider->GetAuthInfoAsync(); + if (authInfo.IsReady()) { + UpdateTokenImpl(authInfo); + return; + } + UpdateTokenInProgress = true; + try { + authInfo.Subscribe([cbContext = SelfContext](const auto& future) { + if (auto self = cbContext->LockShared()) try { + self->Connections->ScheduleCallback(TDuration::Zero(), [cbContext, future](bool ok) { + if (auto self = cbContext->LockShared()) { + if (!ok) { + self->UpdateTokenInProgress = false; + return; + } + std::lock_guard guard(self->Lock); + self->UpdateTokenImpl(future); + } + }); + } catch (...) { + self->UpdateTokenInProgress = false; + } + }); + } catch (...) { + UpdateTokenInProgress = false; + } +} + +void TWriteSessionImpl::UpdateTokenImpl(const NThreading::TFuture<std::string>& future) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTokenInProgress = false; + if (!SessionEstablished || Aborting) { + return; + } + + std::string token; + try { + token = future.GetValue(); + } catch (...) { + CloseImpl(EStatus::CLIENT_UNAUTHENTICATED, CurrentExceptionMessage()); + return; + } if (token == PrevToken) { return; } diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h index 5f7ec7d92b5..a711a8bf376 100644 --- a/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h +++ b/ydb/public/sdk/cpp/src/client/topic/impl/write_session_impl.h @@ -377,6 +377,7 @@ private: TStringBuilder LogPrefixImpl() const; void UpdateTokenIfNeededImpl(); + void UpdateTokenImpl(const NThreading::TFuture<std::string>& future); void WriteInternal(TContinuationToken&& continuationToken, TWriteMessage&& message); @@ -447,7 +448,7 @@ private: std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory; TDbDriverStatePtr DbDriverState; std::string PrevToken; - bool UpdateTokenInProgress = false; + std::atomic_bool UpdateTokenInProgress = false; TInstant LastTokenUpdate = TInstant::Zero(); std::shared_ptr<TWriteSessionEventsQueue> EventsQueue; NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. diff --git a/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.cpp b/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.cpp index 2047ab4d292..db3c5ace047 100644 --- a/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.cpp +++ b/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.cpp @@ -2,7 +2,6 @@ #include <util/generic/yexception.h> #include <util/stream/output.h> -#include <util/system/yassert.h> namespace NYdb::inline Dev { @@ -30,9 +29,6 @@ TSimpleCoreFacility::~TSimpleCoreFacility() { void TSimpleCoreFacility::AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) { std::lock_guard lock(Mutex_); - Y_ABORT_UNLESS(!PeriodicStarted_); - PeriodicStarted_ = true; - auto periodicCb = std::make_shared<TPeriodicCb>(std::move(cb)); EnqueueTaskNoLock( TClock::now(), @@ -46,15 +42,11 @@ void TSimpleCoreFacility::PostToResponseQueue(TPostTaskCb&& f) { if (!f) { return; } - { - std::lock_guard lock(Mutex_); - if (!Stop_) { - EnqueueTaskNoLock(TClock::now(), std::move(f)); - Cv_.notify_one(); - return; - } + std::lock_guard lock(Mutex_); + if (!Stop_) { + EnqueueTaskNoLock(TClock::now(), std::move(f)); + Cv_.notify_one(); } - f(); } void TSimpleCoreFacility::EnqueueTaskNoLock(TTimePoint executeAt, TPostTaskCb&& task) { diff --git a/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.h b/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.h index 7e7fd9ff28f..fe76229a271 100644 --- a/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.h +++ b/ydb/public/sdk/cpp/src/client/types/core_facility/simple_core_facility.h @@ -51,7 +51,6 @@ private: std::condition_variable Cv_; std::priority_queue<TScheduledTask, std::vector<TScheduledTask>, TScheduledTaskLess> Queue_; bool Stop_ = false; - bool PeriodicStarted_ = false; std::uint64_t NextSeqNo_ = 0; std::thread WorkerThread_; diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp index 221bd92d65c..0a2bf8b010a 100644 --- a/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp +++ b/ydb/public/sdk/cpp/src/client/types/credentials/login/login.cpp @@ -37,75 +37,99 @@ std::chrono::system_clock::time_point GetTokenExpiresAt(const std::string& token } return {}; } + +bool IsRetryable(EStatus status) { + return status == EStatus::INTERNAL_ERROR || status == EStatus::ABORTED || status == EStatus::UNAVAILABLE || + status == EStatus::OVERLOADED || status == EStatus::GENERIC_ERROR || status == EStatus::TIMEOUT || + status == EStatus::CANCELLED || status == EStatus::UNDETERMINED || status == EStatus::SESSION_BUSY || + status == EStatus::TRANSPORT_UNAVAILABLE || status == EStatus::CLIENT_RESOURCE_EXHAUSTED || + status == EStatus::CLIENT_DEADLINE_EXCEEDED || status == EStatus::CLIENT_INTERNAL_ERROR || + status == EStatus::CLIENT_CANCELLED || status == EStatus::CLIENT_DISCOVERY_FAILED || + status == EStatus::CLIENT_LIMITS_REACHED; } -class TLoginCredentialsProvider : public ICredentialsProvider { +bool IsRetryableOperation(EStatus status) { + return status == EStatus::ABORTED || status == EStatus::UNAVAILABLE || status == EStatus::OVERLOADED || + status == EStatus::TIMEOUT || status == EStatus::SESSION_BUSY; +} +} + +class TLoginCredentialsProvider : public ICredentialsProvider, public std::enable_shared_from_this<TLoginCredentialsProvider> { public: TLoginCredentialsProvider(std::weak_ptr<ICoreFacility> facility, TLoginCredentialsParams params); - virtual std::string GetAuthInfo() const override; - virtual bool IsValid() const override; - NThreading::TFuture<void> PrepareTokenAsync(); + std::string GetAuthInfo() const override; + NThreading::TFuture<std::string> GetAuthInfoAsync() const override; + bool IsValid() const override; + void Start(); + ~TLoginCredentialsProvider() { Fail("Login credentials provider stopped"); } private: - void PrepareToken(); + bool OnPeriodicTick(EStatus status); void RequestToken(); void FinishRequest(Ydb::Auth::LoginResponse* response, TPlainStatus status, bool facilityAvailable); - bool IsOk() const; - void ParseToken(); - std::string GetToken() const; - std::string GetError() const; - std::string GetTokenOrError() const; - - enum class EState { - Empty, - Requesting, - Done, - }; + void Fail(std::string error); + static std::string GetError(const TPlainStatus& status, const Ydb::Auth::LoginResponse& response); std::weak_ptr<ICoreFacility> Facility_; TLoginCredentialsParams Params_; - EState State_ = EState::Empty; - std::mutex Mutex_; - std::atomic<ui64> TokenReceived_ = 1; - std::atomic<ui64> TokenParsed_ = 0; - std::optional<std::string> Token_; - std::optional<std::string> Error_; - TInstant TokenExpireAt_; + mutable std::mutex Mutex_; TInstant TokenRequestAt_; - TPlainStatus Status_; - Ydb::Auth::LoginResponse Response_; - NThreading::TPromise<void> TokenReadyPromise_; + bool Requesting_ = false; + bool HasToken_ = false; + bool Stopped_ = false; + mutable NThreading::TPromise<std::string> AuthInfo_; }; TLoginCredentialsProvider::TLoginCredentialsProvider(std::weak_ptr<ICoreFacility> facility, TLoginCredentialsParams params) : Facility_(facility) , Params_(std::move(params)) - , TokenReadyPromise_(NThreading::NewPromise<void>()) -{ - auto strongFacility = facility.lock(); - if (strongFacility) { - auto periodicTask = [facility, this](NYdb::NIssue::TIssues&&, EStatus status) -> bool { - if (status != EStatus::SUCCESS) { - return false; - } - - auto strongFacility = facility.lock(); - if (!strongFacility) { - return false; - } - - if (!TokenRequestAt_) { - return true; - } + , AuthInfo_(NThreading::NewPromise<std::string>()) +{} - if (TInstant::Now() >= TokenRequestAt_) { - RequestToken(); +void TLoginCredentialsProvider::Start() { + auto facility = Facility_.lock(); + if (!facility) { + Fail("Login credentials provider response facility is not available"); + return; + } + { + std::lock_guard lock(Mutex_); + Requesting_ = true; + } + try { + facility->AddPeriodicTask([weak = weak_from_this()](NYdb::NIssue::TIssues&&, EStatus status) { + if (auto self = weak.lock()) { + return self->OnPeriodicTick(status); } + return false; + }, 1s); + } catch (...) { + Fail(CurrentExceptionMessage()); + return; + } + RequestToken(); +} +bool TLoginCredentialsProvider::OnPeriodicTick(EStatus status) { + if (status != EStatus::SUCCESS) { + Fail("Login credentials provider periodic task failed"); + return false; + } + { + std::lock_guard lock(Mutex_); + if (Stopped_) { + return false; + } + if (Requesting_ || TInstant::Now() < TokenRequestAt_) { return true; - }; - strongFacility->AddPeriodicTask(std::move(periodicTask), 1min); + } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise<std::string>(); + } + Requesting_ = true; } + RequestToken(); + return true; } bool TLoginCredentialsProvider::IsValid() const { @@ -113,22 +137,21 @@ bool TLoginCredentialsProvider::IsValid() const { } std::string TLoginCredentialsProvider::GetAuthInfo() const { - if (TokenParsed_ == TokenReceived_) { - return GetTokenOrError(); - } else { - const_cast<TLoginCredentialsProvider*>(this)->PrepareToken(); // will block here - return GetTokenOrError(); - } + return GetAuthInfoAsync().GetValueSync(); +} + +NThreading::TFuture<std::string> TLoginCredentialsProvider::GetAuthInfoAsync() const { + std::lock_guard lock(Mutex_); + return AuthInfo_.GetFuture(); } void TLoginCredentialsProvider::RequestToken() { auto strongFacility = Facility_.lock(); if (strongFacility) { - TokenRequestAt_ = {}; - - auto responseCb = [facility = Facility_, this](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { - auto strongFacility = facility.lock(); - FinishRequest(resp, std::move(status), static_cast<bool>(strongFacility)); + auto responseCb = [facility = Facility_, weak = weak_from_this()](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { + if (auto self = weak.lock()) { + self->FinishRequest(resp, std::move(status), !facility.expired()); + } }; Ydb::Auth::LoginRequest request; @@ -137,9 +160,13 @@ void TLoginCredentialsProvider::RequestToken() { TRpcRequestSettings rpcSettings; rpcSettings.Deadline = TDeadline::AfterDuration(60s); - TGRpcConnectionsImpl::RunOnDiscoveryEndpoint<Ydb::Auth::V1::AuthService, Ydb::Auth::LoginRequest, Ydb::Auth::LoginResponse>( - strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, - rpcSettings); + try { + TGRpcConnectionsImpl::RunOnDiscoveryEndpoint<Ydb::Auth::V1::AuthService, Ydb::Auth::LoginRequest, Ydb::Auth::LoginResponse>( + strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, + rpcSettings); + } catch (...) { + Fail(CurrentExceptionMessage()); + } } else { FinishRequest(nullptr, {}, false); } @@ -150,107 +177,78 @@ void TLoginCredentialsProvider::FinishRequest( TPlainStatus status, bool facilityAvailable) { - std::optional<std::string> error; - { - std::lock_guard lock(Mutex_); - State_ = EState::Done; - ++TokenReceived_; - if (facilityAvailable) { - Status_ = std::move(status); - if (response) { - Response_ = std::move(*response); - } - ParseToken(); - } else { - Token_.reset(); - Error_ = "Login credentials provider response facility is not available"; - TokenParsed_ = TokenReceived_.load(); - } - error = Error_; + if (!facilityAvailable) { + Fail("Login credentials provider response facility is not available"); + return; } - if (error) { - TokenReadyPromise_.TrySetException(std::make_exception_ptr(yexception() << *error)); - } else { - TokenReadyPromise_.TrySetValue(); - } -} -void TLoginCredentialsProvider::PrepareToken() { - PrepareTokenAsync().Wait(); - std::lock_guard lock(Mutex_); - ParseToken(); -} - -NThreading::TFuture<void> TLoginCredentialsProvider::PrepareTokenAsync() { - bool requestToken = false; - auto future = TokenReadyPromise_.GetFuture(); - { - std::unique_lock<std::mutex> lock(Mutex_); - if (State_ == EState::Empty) { - State_ = EState::Requesting; - requestToken = true; + Ydb::Auth::LoginResponse emptyResponse; + const auto& responseValue = response ? *response : emptyResponse; + const auto operationStatus = static_cast<EStatus>(responseValue.operation().status()); + if (!status.Ok() || operationStatus != EStatus::SUCCESS) { + bool retry; + { + std::lock_guard lock(Mutex_); + retry = (!status.Ok() && IsRetryable(status.Status)) || + (HasToken_ && status.Ok() && IsRetryableOperation(operationStatus)); + if (retry) { + Requesting_ = false; + TokenRequestAt_ = TInstant::Now() + TDuration::Seconds(1); + } } - } - if (requestToken) { - RequestToken(); - } - return future; -} - -bool TLoginCredentialsProvider::IsOk() const { - return State_ == EState::Done - && Status_.Ok() - && Response_.operation().status() == Ydb::StatusIds::SUCCESS; -} - -void TLoginCredentialsProvider::ParseToken() { // works under mutex - if (TokenParsed_ != TokenReceived_) { - if (IsOk()) { - Token_ = GetToken(); - Error_.reset(); - TInstant now = TInstant::Now(); - TokenExpireAt_ = ToInstant(GetTokenExpiresAt(Token_.value())); - TokenRequestAt_ = now + TDuration::Minutes((TokenExpireAt_ - now).Minutes() / 2); - } else { - Token_.reset(); - Error_ = GetError(); + if (retry) { + return; } - TokenParsed_ = TokenReceived_.load(); + Fail(GetError(status, responseValue)); + return; } -} -std::string TLoginCredentialsProvider::GetToken() const { Ydb::Auth::LoginResult result; - Response_.operation().result().UnpackTo(&result); - return result.token(); + if (!responseValue.operation().result().UnpackTo(&result) || result.token().empty()) { + Fail("Login service returned an empty token"); + return; + } + auto token = result.token(); + const auto now = TInstant::Now(); + NThreading::TPromise<std::string> promise; + { + std::lock_guard lock(Mutex_); + Requesting_ = false; + HasToken_ = true; + TokenRequestAt_ = now + (ToInstant(GetTokenExpiresAt(token)) - now) / 2; + promise = AuthInfo_; + } + promise.TrySetValue(std::move(token)); } -std::string TLoginCredentialsProvider::GetError() const { - if (Status_.Ok()) { - if (Response_.operation().issues_size() > 0) { - return Response_.operation().issues(0).message(); - } else { - return Ydb::StatusIds_StatusCode_Name(Response_.operation().status()); - } - } else { - TStringBuilder str; - str << "Couldn't get token for provided credentials from " << Status_.Endpoint - << " with status " << Status_.Status << "."; - for (const auto& issue : Status_.Issues) { - str << Endl << "Issue: " << issue; - } - return str; +void TLoginCredentialsProvider::Fail(std::string error) { + NThreading::TPromise<std::string> promise; + { + std::lock_guard lock(Mutex_); + Stopped_ = true; + Requesting_ = false; + promise = AuthInfo_; } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); } -std::string TLoginCredentialsProvider::GetTokenOrError() const { - if (Token_) { - return Token_.value(); +std::string TLoginCredentialsProvider::GetError( + const TPlainStatus& status, + const Ydb::Auth::LoginResponse& response) +{ + if (status.Ok()) { + if (response.operation().issues_size() > 0) { + return response.operation().issues(0).message(); + } + return Ydb::StatusIds_StatusCode_Name(response.operation().status()); } - if (Error_) { - ythrow yexception() << Error_.value(); + TStringBuilder str; + str << "Couldn't get token for provided credentials from " << status.Endpoint + << " with status " << status.Status << "."; + for (const auto& issue : status.Issues) { + str << Endl << "Issue: " << issue; } - ythrow yexception() << "Wrong state of credentials provider"; + return str; } class TLoginCredentialsProviderFactory : public ICredentialsProviderFactory { @@ -258,7 +256,6 @@ public: TLoginCredentialsProviderFactory(TLoginCredentialsParams params); virtual std::shared_ptr<ICredentialsProvider> CreateProvider() const override; virtual std::shared_ptr<ICredentialsProvider> CreateProvider(std::weak_ptr<ICoreFacility> facility) const override; - virtual NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const override; private: TLoginCredentialsParams Params_; @@ -274,13 +271,9 @@ std::shared_ptr<ICredentialsProvider> TLoginCredentialsProviderFactory::CreatePr } std::shared_ptr<ICredentialsProvider> TLoginCredentialsProviderFactory::CreateProvider(std::weak_ptr<ICoreFacility> facility) const { - return std::make_shared<TLoginCredentialsProvider>(std::move(facility), Params_); -} - -NThreading::TFuture<TCredentialsProviderPtr> TLoginCredentialsProviderFactory::CreateProviderAsync(std::weak_ptr<ICoreFacility> facility) const { auto provider = std::make_shared<TLoginCredentialsProvider>(std::move(facility), Params_); - TCredentialsProviderPtr result = provider; - return provider->PrepareTokenAsync().Return(std::move(result)); + provider->Start(); + return provider; } std::shared_ptr<ICredentialsProviderFactory> CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params) { diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp index df877f5dce0..367cfbc92dc 100644 --- a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -1,4 +1,5 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h> #include <library/cpp/cgiparam/cgiparam.h> #include <library/cpp/http/misc/httpcodes.h> @@ -85,18 +86,11 @@ bool IsRetryableError(TKeepAliveHttpClient::THttpCode code) { || code == HTTP_GATEWAY_TIME_OUT; } -ERetryErrorClass RetryPolicyClass(TKeepAliveHttpClient::THttpCode code) { - return IsRetryableError(code) ? ERetryErrorClass::LongRetry : ERetryErrorClass::ShortRetry; // In case when we already have token we know that all params are correct and all errors are temporary -} - -ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErrors) { +ERetryErrorClass RetryPolicyClass(const std::exception* ex) { if (const TTokenExchangeError* err = dynamic_cast<const TTokenExchangeError*>(ex)) { if (IsRetryableError(err->HttpCode)) { return ERetryErrorClass::LongRetry; } - if (retryAllErrors) { - return ERetryErrorClass::ShortRetry; - } } if (dynamic_cast<const TSystemError*>(ex)) { return ERetryErrorClass::ShortRetry; @@ -106,8 +100,7 @@ ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErr return ERetryErrorClass::NoRetry; } -using TRetryPolicy = IRetryPolicy<TKeepAliveHttpClient::THttpCode>; -using TSyncRetryPolicy = IRetryPolicy<const std::exception*, bool>; +using TRetryPolicy = IRetryPolicy<const std::exception*>; struct TPrivateOauth2TokenExchangeParams: public TOauth2TokenExchangeParams { TPrivateOauth2TokenExchangeParams(const TOauth2TokenExchangeParams& params) @@ -172,48 +165,65 @@ private: } }; -class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this<TOauth2TokenExchangeProviderImpl> -{ +class TOauth2TokenExchangeProviderImpl final : public ICredentialsProvider { struct TTokenExchangeResult { std::string Token; - TInstant TokenDeadline; TInstant TokenRefreshTime; }; public: - explicit TOauth2TokenExchangeProviderImpl(const TPrivateOauth2TokenExchangeParams& params) + TOauth2TokenExchangeProviderImpl(const TPrivateOauth2TokenExchangeParams& params, + std::weak_ptr<ICoreFacility> responseFacility) : Params(params) + , ResponseFacility(std::move(responseFacility)) + , AuthInfo(NThreading::NewPromise<std::string>()) { - ExchangeTokenSync(TInstant::Now()); + ResponseFacility.expired() ? Stop() : Start(); + } + +private: + void Start() { + try { + WorkerThread = std::thread([this] { Run(); }); + } catch (...) { + Fail(std::current_exception()); + } } void Stop() { + NThreading::TPromise<std::string> promise; { std::unique_lock<std::mutex> lock(StopMutex); Stopping = true; StopVar.notify_all(); } + with_lock (Lock) { + promise = AuthInfo; + } + promise.TrySetException(std::make_exception_ptr(yexception() << PROV_ERR "stopped")); - if (RefreshTokenThread.joinable()) { - RefreshTokenThread.join(); + if (WorkerThread.joinable()) { + WorkerThread.join(); } } - std::string GetAuthInfo() const { - const TInstant now = TInstant::Now(); - std::string token; +public: + ~TOauth2TokenExchangeProviderImpl() { + Stop(); + } + + std::string GetAuthInfo() const override { + return GetAuthInfoAsync().GetValueSync(); + } + + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { with_lock (Lock) { - if (Token.empty() || now >= TokenDeadline) { // Update sync. This can be if we have repeating error during token refresh process. In this case we will try for the last time and throw an error - ExchangeTokenSync(now, true); - token = Token; - } else { - if (now >= TokenRefreshTime) { - TryRefreshToken(); - } - token = Token; // Still valid - } + return AuthInfo.GetFuture(); } - return token; + } + + bool IsValid() const override { + return true; } private: @@ -344,7 +354,6 @@ private: const TDuration expireDelta = TDuration::Seconds(expireTime); TTokenExchangeResult result; - result.TokenDeadline = now + expireDelta; result.TokenRefreshTime = now + expireDelta / 2; result.Token = TStringBuilder() << "Bearer " << token; return result; @@ -361,31 +370,63 @@ private: return ProcessExchangeTokenResponse(now, statusCode, responseStream); } - void ExchangeTokenSync(TInstant now, bool retryAllErrors = false) const { // Is run under lock - TSyncRetryPolicy::IRetryState::TPtr retryState; + void Run() { while (true) { - try { - TTokenExchangeResult result = ExchangeToken(now); - Token = result.Token; - TokenDeadline = result.TokenDeadline; - TokenRefreshTime = result.TokenRefreshTime; - break; - } catch (const std::exception& ex) { - if (!retryState) { - retryState = TSyncRetryPolicy::GetFixedIntervalPolicy( // retry more aggresively when we are in sync mode - SyncRetryPolicyClass, - TDuration::MilliSeconds(100), // delay // default - TDuration::MilliSeconds(300), // long delay // default - std::numeric_limits<size_t>::max(), // max retries // default - Params.SyncUpdateTimeout_ // max time - )->CreateRetryState(); + TRetryPolicy::IRetryState::TPtr retryState; + TTokenExchangeResult result; + while (true) { + if (IsStopping()) { + return; } - if (auto interval = retryState->GetNextRetryDelay(&ex, retryAllErrors)) { - Sleep(*interval); - } else { - throw; + try { + result = ExchangeToken(TInstant::Now()); + break; + } catch (const std::exception& ex) { + if (!retryState) { + retryState = TRetryPolicy::GetExponentialBackoffPolicy( + RetryPolicyClass, + TDuration::MilliSeconds(10), + TDuration::MilliSeconds(200), + TDuration::Seconds(30))->CreateRetryState(); + } + auto delay = retryState->GetNextRetryDelay(&ex); + if (!delay) { + Fail(std::current_exception()); + return; + } + std::unique_lock<std::mutex> lock(StopMutex); + if (StopVar.wait_for(lock, TChronoDuration(delay->GetValue()), [this] { return Stopping; })) { + return; + } + } catch (...) { + Fail(std::current_exception()); + return; } } + + NThreading::TPromise<std::string> promise; + with_lock (Lock) { + promise = AuthInfo; + } + Complete([promise, token = std::move(result.Token)]() mutable { + promise.TrySetValue(std::move(token)); + }); + + { + std::unique_lock<std::mutex> lock(StopMutex); + const auto delay = result.TokenRefreshTime - TInstant::Now(); + if (delay > TDuration::Zero() && + StopVar.wait_for(lock, TChronoDuration(delay.GetValue()), [this] { return Stopping; })) + { + return; + } + if (Stopping) { + return; + } + } + with_lock (Lock) { + AuthInfo = NThreading::NewPromise<std::string>(); + } } } @@ -394,85 +435,32 @@ private: return Stopping; } - void RefreshToken() const { - TRetryPolicy::IRetryState::TPtr retryState; - TInstant deadline; + void Fail(std::exception_ptr error) const { + NThreading::TPromise<std::string> promise; with_lock (Lock) { - deadline = TokenDeadline; + promise = AuthInfo; } - while (!IsStopping()) { - const TInstant now = TInstant::Now(); - try { - TTokenExchangeResult result = ExchangeToken(now); - with_lock (Lock) { - Token = result.Token; - TokenDeadline = result.TokenDeadline; - TokenRefreshTime = result.TokenRefreshTime; - break; - } - } catch (const std::exception& ex) { // If this error will repeat, we finally will get it syncronously in GetAuthInfo() and pass to client - if (!retryState) { - retryState = TRetryPolicy::GetExponentialBackoffPolicy( - RetryPolicyClass, - TDuration::MilliSeconds(10), // min delay // default - TDuration::MilliSeconds(200), // min long delay // default - TDuration::Seconds(30), // max delay // default - std::numeric_limits<size_t>::max(), // max retries // default - deadline - TInstant::Now() // max time - )->CreateRetryState(); - } else { - TKeepAliveHttpClient::THttpCode code = HTTP_CODE_MAX; - if (const auto* err = dynamic_cast<const TTokenExchangeError*>(&ex)) { - code = err->HttpCode; - } - if (auto delay = retryState->GetNextRetryDelay(code)) { - std::unique_lock<std::mutex> lock(StopMutex); - const bool stopping = StopVar.wait_for( - lock, - TChronoDuration(delay->GetValue()), - [this]() { - return Stopping; - } - ); - if (stopping) { - break; - } - } else { - break; - } - } - } - } - TokenIsRefreshing.store(false); + Complete([promise, error = std::move(error)]() mutable { + promise.TrySetException(std::move(error)); + }); } - void TryRefreshToken() const { // Is run under lock - if (TokenIsRefreshing.load()) { - return; - } - if (RefreshTokenThread.joinable()) { - RefreshTokenThread.join(); - } - - TokenIsRefreshing.store(true); - RefreshTokenThread = std::thread( - [w = weak_from_this()]() { - if (auto p = w.lock()) { - p->RefreshToken(); - } + void Complete(TPostTaskCb&& callback) const noexcept { + try { + if (auto facility = ResponseFacility.lock()) { + facility->PostToResponseQueue(std::move(callback)); } - ); + } catch (...) { + } } private: TPrivateOauth2TokenExchangeParams Params; + std::weak_ptr<ICoreFacility> ResponseFacility; - TAdaptiveLock Lock; - mutable std::atomic<bool> TokenIsRefreshing = false; - mutable std::thread RefreshTokenThread; - mutable std::string Token; - mutable TInstant TokenDeadline; - mutable TInstant TokenRefreshTime; + mutable TAdaptiveLock Lock; + mutable NThreading::TPromise<std::string> AuthInfo; + std::thread WorkerThread; // Stop bool Stopping = false; @@ -480,42 +468,31 @@ private: mutable std::condition_variable StopVar; }; -class TOauth2TokenExchangeProvider: public ICredentialsProvider { -public: - explicit TOauth2TokenExchangeProvider(const TPrivateOauth2TokenExchangeParams& params) - : Impl(std::make_shared<TOauth2TokenExchangeProviderImpl>(params)) - { - } - - std::string GetAuthInfo() const override { - return Impl->GetAuthInfo(); - } - - bool IsValid() const override { - return true; - } - - ~TOauth2TokenExchangeProvider() { // The last link tp provider is gone - Impl->Stop(); - } - -private: - std::shared_ptr<TOauth2TokenExchangeProviderImpl> Impl; -}; - class TOauth2TokenExchangeFactory: public ICredentialsProviderFactory { public: explicit TOauth2TokenExchangeFactory(const TOauth2TokenExchangeParams& params) - : Provider(std::make_shared<TOauth2TokenExchangeProvider>(params)) + : Params(params) { } TCredentialsProviderPtr CreateProvider() const override { + std::lock_guard lock(Lock); + if (!Provider) { + auto facility = CreateSimpleCoreFacility(); + Provider = std::make_shared<NCredentials::NDetail::TOwningFacilityCredentialsProvider>( + facility, std::make_shared<TOauth2TokenExchangeProviderImpl>(Params, facility)); + } return Provider; } + TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const override { + return std::make_shared<TOauth2TokenExchangeProviderImpl>(Params, std::move(facility)); + } + private: - std::shared_ptr<TOauth2TokenExchangeProvider> Provider; + TPrivateOauth2TokenExchangeParams Params; + mutable std::mutex Lock; + mutable TCredentialsProviderPtr Provider; }; } // namespace diff --git a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make index 9962d6bb675..a318032d249 100644 --- a/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make +++ b/ydb/public/sdk/cpp/src/client/types/credentials/oauth2_token_exchange/ya.make @@ -16,6 +16,7 @@ PEERDIR( library/cpp/string_utils/base64 library/cpp/uri ydb/public/sdk/cpp/src/client/types + ydb/public/sdk/cpp/src/client/types/core_facility ydb/public/sdk/cpp/src/client/types/credentials ) diff --git a/ydb/public/sdk/cpp/src/client/types/virtual_timestamp.cpp b/ydb/public/sdk/cpp/src/client/types/virtual_timestamp.cpp new file mode 100644 index 00000000000..8a80a50a2f3 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/types/virtual_timestamp.cpp @@ -0,0 +1,58 @@ +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h> + +#include <util/stream/output.h> +#include <util/string/printf.h> + +#include <tuple> + +namespace NYdb::inline Dev { +namespace NScheme { + +TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) + : PlanStep(planStep) + , TxId(txId) +{} + +TVirtualTimestamp::TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto) + : TVirtualTimestamp(proto.plan_step(), proto.tx_id()) +{} + +std::string TVirtualTimestamp::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TVirtualTimestamp::Out(IOutputStream& out) const { + out << "{ plan_step: " << PlanStep + << ", tx_id: " << TxId + << " }"; +} + +bool TVirtualTimestamp::operator<(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) < std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator<=(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) <= std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator>(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) > std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator>=(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) >= std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator==(const TVirtualTimestamp& rhs) const { + return PlanStep == rhs.PlanStep && TxId == rhs.TxId; +} + +bool TVirtualTimestamp::operator!=(const TVirtualTimestamp& rhs) const { + return !(*this == rhs); +} + +} // namespace NScheme +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/src/client/types/ya.make b/ydb/public/sdk/cpp/src/client/types/ya.make index 98b9db6ea2a..e6a7e901e49 100644 --- a/ydb/public/sdk/cpp/src/client/types/ya.make +++ b/ydb/public/sdk/cpp/src/client/types/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + virtual_timestamp.cpp ydb.cpp ) diff --git a/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h b/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h index 99469c65fa1..89543bf4b54 100644 --- a/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h +++ b/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h @@ -640,7 +640,7 @@ public: // template<typename Meta> inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaInt16 version, const typename Meta::Type& value) { - if (VersionCheck<Meta::PresentVersions.Min, Meta::PresentVersions.Max>(version)) { + if (VersionCheck<Meta::PresentVersions.Min, Meta::PresentVersions.Max>(version)) { if (VersionCheck<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>(version)) { if (!IsDefaultValue<Meta>(value)) { ++collector.NumTaggedFields; @@ -653,7 +653,7 @@ inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaIn template<typename Meta> inline void Read(TKafkaReadable& readable, TKafkaInt16 version, typename Meta::Type& value) { - if (!VersionNone<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>() + if (!VersionNone<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>() && VersionCheck<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>(version)) { return; } else { diff --git a/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.cpp b/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.cpp index a07a54fef8c..e6bd55103e2 100644 --- a/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.cpp +++ b/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.cpp @@ -10,6 +10,11 @@ void TIamTokenServiceStub::SetResponseToken(const std::string& token, int64_t ex ExpiresAtSeconds_ = expiresAtSeconds; } +void TIamTokenServiceStub::SetStatus(grpc::Status status) { + std::lock_guard lock(Lock_); + Status_ = std::move(status); +} + grpc::Status TIamTokenServiceStub::Create( grpc::ServerContext*, const yandex::cloud::iam::v1::CreateIamTokenRequest* request, @@ -19,6 +24,9 @@ grpc::Status TIamTokenServiceStub::Create( ++RequestCount_; LastRequest_ = *request; HasLastRequest_ = true; + if (!Status_.ok()) { + return Status_; + } response->set_iam_token(IamToken_); response->mutable_expires_at()->set_seconds(ExpiresAtSeconds_); response->mutable_expires_at()->set_nanos(0); diff --git a/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.h b/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.h index dede93ca1a0..c02001d459b 100644 --- a/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.h +++ b/ydb/public/sdk/cpp/tests/common/iam_mocks/iam_grpc_mock_server.h @@ -21,6 +21,7 @@ namespace NYdb::NTest { class TIamTokenServiceStub final : public yandex::cloud::iam::v1::IamTokenService::Service { public: void SetResponseToken(const std::string& token, int64_t expiresAtSeconds = 4102444800); + void SetStatus(grpc::Status status); grpc::Status Create( grpc::ServerContext*, @@ -34,6 +35,7 @@ public: private: mutable std::mutex Lock_; std::string IamToken_; + grpc::Status Status_; int64_t ExpiresAtSeconds_ = 4102444800; int RequestCount_ = 0; yandex::cloud::iam::v1::CreateIamTokenRequest LastRequest_; diff --git a/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp index 330e048cdf1..d69edfdb76d 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp @@ -124,32 +124,54 @@ namespace { std::atomic_int& ProviderCount_; }; + class TDeferredAuthProvider final : public ICredentialsProvider { + public: + TDeferredAuthProvider() + : AuthInfo_(NThreading::NewPromise<std::string>()) + {} + + std::string GetAuthInfo() const override { + return AuthInfo_.GetFuture().GetValueSync(); + } + + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { + return AuthInfo_.GetFuture(); + } + + bool IsValid() const override { + return true; + } + + void SetReady() { + AuthInfo_.SetValue("token"); + } + + private: + NThreading::TPromise<std::string> AuthInfo_; + }; + class TDeferredCredentialsFactory final : public ICredentialsProviderFactory { public: TDeferredCredentialsFactory() - : Provider_(NThreading::NewPromise<TCredentialsProviderPtr>()) + : Provider_(std::make_shared<TDeferredAuthProvider>()) {} TCredentialsProviderPtr CreateProvider() const override { - return CreateInsecureCredentialsProviderFactory()->CreateProvider(); - } - - NThreading::TFuture<TCredentialsProviderPtr> CreateProviderAsync(std::weak_ptr<ICoreFacility>) const override { - return Provider_.GetFuture(); + return Provider_; } void SetReady() { - Provider_.SetValue(CreateProvider()); + Provider_->SetReady(); } private: - NThreading::TPromise<TCredentialsProviderPtr> Provider_; + std::shared_ptr<TDeferredAuthProvider> Provider_; }; } // namespace Y_UNIT_TEST_SUITE(DeferredCredentialsTest) { - Y_UNIT_TEST(RequestWaitsForCredentials) { + Y_UNIT_TEST(RequestWaitsForAuthInfo) { auto factory = std::make_shared<TDeferredCredentialsFactory>(); auto driver = TDriver(TDriverConfig() .SetEndpoint("localhost:100") diff --git a/ydb/public/sdk/cpp/tests/unit/client/iam/grpc_iam_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/iam/grpc_iam_ut.cpp index f0393f26110..fb3346e7f21 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/iam/grpc_iam_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/iam/grpc_iam_ut.cpp @@ -33,19 +33,19 @@ TEST(GrpcIamCredentialsProvider, TeardownWhileIamCreatePendingCompletes) { TIamOAuth params = MakeOAuthParams(server.Endpoint()); params.RequestTimeout = TDuration::MilliSeconds(400); - auto work = [¶ms] { - auto facility = std::make_shared<TSimpleCoreFacility>(); - TIamOAuthCredentialsProvider<CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService> provider( - params, - facility); - (void)provider; - }; - - std::future<void> done = std::async(std::launch::async, work); + auto facility = std::make_shared<TSimpleCoreFacility>(); + auto provider = std::make_shared<TIamOAuthCredentialsProvider< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>>(params, facility); + auto authInfo = provider->GetAuthInfoAsync(); + ASSERT_FALSE(authInfo.IsReady()); ASSERT_TRUE(iamService.WaitUntilRpcEntered(std::chrono::seconds(5))) << "server should have accepted the IAM Create call"; + std::future<void> done = std::async(std::launch::async, [provider = std::move(provider)]() mutable { + provider.reset(); + }); + ASSERT_EQ(done.wait_for(std::chrono::seconds(20)), std::future_status::ready) << "provider destructor must finish while an IAM Create is still blocked on the server " "(in-flight RPC vs channel teardown)."; @@ -67,16 +67,15 @@ TEST(GrpcIamCredentialsProvider, TeardownWhileIamCreatePendingCompletesViaFactor auto factory = std::make_shared<TIamOAuthCredentialsProviderFactory< CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>>(params); - auto work = [&factory] { - auto provider = factory->CreateProvider(); - (void)provider; - }; - - std::future<void> done = std::async(std::launch::async, work); + auto provider = factory->CreateProvider(); ASSERT_TRUE(iamService.WaitUntilRpcEntered(std::chrono::seconds(5))) << "server should have accepted the IAM Create call"; + std::future<void> done = std::async(std::launch::async, [provider = std::move(provider)]() mutable { + provider.reset(); + }); + ASSERT_EQ(done.wait_for(std::chrono::seconds(20)), std::future_status::ready) << "factory wrapper teardown must finish while an IAM Create is still blocked on the server"; done.get(); @@ -107,6 +106,7 @@ TEST(GrpcIamCredentialsProviderFactory, NoArgProvidersAreCachedAcrossFactoryInst for (size_t i = 1; i < oauthProviders.size(); ++i) { EXPECT_EQ(firstOAuthProvider, oauthProviders[i].get()); } + EXPECT_EQ(firstOAuthProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 1); using TJwtFactory = TIamJwtCredentialsProviderFactory< @@ -116,17 +116,41 @@ TEST(GrpcIamCredentialsProviderFactory, NoArgProvidersAreCachedAcrossFactoryInst auto secondJwtProvider = std::make_shared<TJwtFactory>(jwtParams)->CreateProvider(); EXPECT_EQ(firstJwtProvider, secondJwtProvider); + EXPECT_EQ(firstJwtProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 2); auto differentOAuthParams = oauthParams; differentOAuthParams.OAuthToken = "different-oauth-token"; auto differentOAuthProvider = std::make_shared<TOAuthFactory>(differentOAuthParams)->CreateProvider(); EXPECT_NE(firstOAuthProvider, differentOAuthProvider); + EXPECT_EQ(differentOAuthProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 3); server.Stop(); } +TEST(GrpcIamCredentialsProviderFactory, ReadyCallbackCanReleaseNoArgProvider) { + TIamTokenServiceStub iamStub; + iamStub.SetResponseToken("unit-test-iam-token"); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto provider = std::make_shared<TIamOAuthCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>>( + MakeOAuthParams(server.Endpoint()))->CreateProvider(); + auto released = std::make_shared<std::promise<void>>(); + auto authInfo = provider->GetAuthInfoAsync(); + ASSERT_TRUE(authInfo.IsReady()); + authInfo.Subscribe( + [provider = std::move(provider), released](const auto&) mutable { + provider.reset(); + released->set_value(); + }); + + ASSERT_EQ(released->get_future().wait_for(std::chrono::seconds(10)), std::future_status::ready); + server.Stop(); +} + namespace { class TSlowBlockingAuthProvider final : public ICredentialsProvider { @@ -134,6 +158,7 @@ public: std::string GetAuthInfo() const override { std::unique_lock lock(Mutex_); if (++CallCount_ > 1) { + BlockedCv_.notify_all(); BlockedCv_.wait(lock, [this] { return Released_; }); } return "slow-auth-token"; @@ -152,16 +177,8 @@ public: } bool WaitUntilBlocked(std::chrono::milliseconds timeout) const { - const auto deadline = std::chrono::steady_clock::now() + timeout; - while (std::chrono::steady_clock::now() < deadline) { - std::lock_guard lock(Mutex_); - if (CallCount_ > 1 && !Released_) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - } - std::lock_guard lock(Mutex_); - return CallCount_ > 1 && !Released_; + std::unique_lock lock(Mutex_); + return BlockedCv_.wait_for(lock, timeout, [this] { return CallCount_ > 1; }); } private: @@ -171,30 +188,36 @@ private: bool Released_ = false; }; -class TFailThenSucceedAuthProvider final : public ICredentialsProvider { +class TDeferredAuthProvider final : public ICredentialsProvider { public: - explicit TFailThenSucceedAuthProvider(int failCount) - : FailCount_(failCount) + TDeferredAuthProvider() + : AuthInfo_(NThreading::NewPromise<std::string>()) {} std::string GetAuthInfo() const override { - if (CallCount_.fetch_add(1) < FailCount_) { - ythrow yexception() << "auth failure"; - } - return "auth-token"; + return AuthInfo_.GetFuture().GetValueSync(); + } + + NThreading::TFuture<std::string> GetAuthInfoAsync() const override { + ++CallCount_; + return AuthInfo_.GetFuture(); } bool IsValid() const override { return true; } - int GetCallCount() const { + int CallCount() const { return CallCount_.load(); } + void SetReady() { + AuthInfo_.SetValue("auth-token"); + } + private: - const int FailCount_; mutable std::atomic<int> CallCount_{0}; + NThreading::TPromise<std::string> AuthInfo_; }; } // namespace @@ -250,13 +273,13 @@ TEST(GrpcIamCredentialsProvider, StopDuringFillContextDoesNotHang) { server.Stop(); } -TEST(GrpcIamCredentialsProvider, FillContextAuthExceptionSurvivesAndRecovers) { +TEST(GrpcIamCredentialsProvider, WaitsForNestedAuthWithoutPollingIt) { TIamTokenServiceStub iamStub; iamStub.SetResponseToken("unit-test-iam-token"); TIamGrpcServer server(&iamStub); ASSERT_TRUE(server.Start()); - auto authProvider = std::make_shared<TFailThenSucceedAuthProvider>(2); + auto authProvider = std::make_shared<TDeferredAuthProvider>(); TIamOAuth params = MakeOAuthParams(server.Endpoint()); auto facility = std::make_shared<TSimpleCoreFacility>(); @@ -273,9 +296,64 @@ TEST(GrpcIamCredentialsProvider, FillContextAuthExceptionSurvivesAndRecovers) { facility, authProvider); - EXPECT_EQ(provider.GetAuthInfo(), "unit-test-iam-token"); + auto future = provider.GetAuthInfoAsync(); + for (int i = 0; i < 100 && authProvider->CallCount() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_EQ(authProvider->CallCount(), 1); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(authProvider->CallCount(), 1); + + authProvider->SetReady(); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 1); - EXPECT_GE(authProvider->GetCallCount(), 3); + + server.Stop(); +} + +TEST(GrpcIamCredentialsProvider, RetriesTransientFailure) { + TIamTokenServiceStub iamStub; + iamStub.SetResponseToken("unit-test-iam-token"); + iamStub.SetStatus(grpc::Status(grpc::StatusCode::UNAVAILABLE, "retry")); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto facility = std::make_shared<TSimpleCoreFacility>(); + auto provider = std::make_shared<TIamOAuthCredentialsProvider< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>>( + MakeOAuthParams(server.Endpoint()), facility); + auto future = provider->GetAuthInfoAsync(); + for (int i = 0; i < 1000 && iamStub.GetRequestCount() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_GT(iamStub.GetRequestCount(), 0); + ASSERT_FALSE(future.IsReady()); + + iamStub.SetStatus(grpc::Status::OK); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "unit-test-iam-token"); + EXPECT_GT(iamStub.GetRequestCount(), 1); + + server.Stop(); +} + +TEST(GrpcIamCredentialsProvider, DoesNotRetryTerminalFailure) { + TIamTokenServiceStub iamStub; + iamStub.SetStatus(grpc::Status(grpc::StatusCode::PERMISSION_DENIED, "terminal")); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto facility = std::make_shared<TSimpleCoreFacility>(); + auto provider = std::make_shared<TIamOAuthCredentialsProvider< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>>( + MakeOAuthParams(server.Endpoint()), facility); + auto future = provider->GetAuthInfoAsync(); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_THROW(future.GetValue(), yexception); + const int requests = iamStub.GetRequestCount(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(iamStub.GetRequestCount(), requests); server.Stop(); } diff --git a/ydb/public/sdk/cpp/tests/unit/client/iam/http_iam_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/iam/http_iam_ut.cpp index a4a667e983d..baae8eec0d5 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/iam/http_iam_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/iam/http_iam_ut.cpp @@ -1,4 +1,5 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/iam/iam.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h> #include <ydb/public/sdk/cpp/tests/common/iam_mocks/iam_http_mock_server.h> @@ -65,7 +66,7 @@ TEST(IamCredentialsProvider, NoExpiryFieldFallback) { TEST(IamCredentialsProvider, ServerError) { TMetadataServer server; server.SetStrictMode(false); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); + server.SetResponse(HTTP_BAD_REQUEST, ""); TIamHost params = MakeMetadataParams(server.Port); @@ -75,7 +76,7 @@ TEST(IamCredentialsProvider, ServerError) { EXPECT_THROW(provider->GetAuthInfo(), yexception); } -TEST(IamCredentialsProvider, GracePeriodOnRefreshError) { +TEST(IamCredentialsProvider, RetriesTransientError) { TMetadataServer server; server.SetStrictMode(false); server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 3600)); @@ -83,57 +84,22 @@ TEST(IamCredentialsProvider, GracePeriodOnRefreshError) { TIamHost params = MakeMetadataParams(server.Port); params.RefreshPeriod = TDuration::MilliSeconds(100); - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + auto facility = CreateSimpleCoreFacility(); + auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(facility); EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - int countBeforeRefresh = server.GetRequestCount(); - Sleep(TDuration::MilliSeconds(150)); - + const int countBeforeRefresh = server.GetRequestCount(); server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - EXPECT_GT(server.GetRequestCount(), countBeforeRefresh); -} - -TEST(IamCredentialsProvider, ThrowAfterTokenExpiredOnRefreshError) { - TMetadataServer server; - server.SetStrictMode(false); - server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 1)); - - TIamHost params = MakeMetadataParams(server.Port); - params.RefreshPeriod = TDuration::MilliSeconds(100); - - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - - Sleep(TDuration::MilliSeconds(150)); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - - Sleep(TDuration::Seconds(1)); - EXPECT_THROW(provider->GetAuthInfo(), yexception); -} - -TEST(IamCredentialsProvider, RecoveryAfterRefreshError) { - TMetadataServer server; - server.SetStrictMode(false); - server.SetResponse(HTTP_OK, MakeTokenResponse("token-1", 3600)); - - TIamHost params = MakeMetadataParams(server.Port); - params.RefreshPeriod = TDuration::MilliSeconds(100); - - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); - EXPECT_EQ(provider->GetAuthInfo(), "token-1"); - - Sleep(TDuration::MilliSeconds(150)); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "token-1"); - - server.SetResponse(HTTP_OK, MakeTokenResponse("token-2", 3600)); - int countBeforeRecovery = server.GetRequestCount(); - Sleep(TDuration::MilliSeconds(150)); + for (int i = 0; i < 1000 && server.GetRequestCount() == countBeforeRefresh; ++i) { + Sleep(TDuration::MilliSeconds(10)); + } + ASSERT_GT(server.GetRequestCount(), countBeforeRefresh); + auto future = provider->GetAuthInfoAsync(); + EXPECT_FALSE(future.IsReady()); - EXPECT_EQ(provider->GetAuthInfo(), "token-2"); - EXPECT_GT(server.GetRequestCount(), countBeforeRecovery); + server.SetResponse(HTTP_OK, MakeTokenResponse("new-token", 3600)); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "new-token"); } TEST(IamCredentialsProvider, ConcurrentAccess) { diff --git a/ydb/public/sdk/cpp/tests/unit/client/iam_private/grpc_iam_service_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/iam_private/grpc_iam_service_ut.cpp index 29b21e145b3..73b1164d347 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/iam_private/grpc_iam_service_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/iam_private/grpc_iam_service_ut.cpp @@ -107,9 +107,7 @@ public: callback(std::move(issues), EStatus::CLIENT_CANCELLED); } - void PostToResponseQueue(TPostTaskCb&& callback) override { - callback(); - } + void PostToResponseQueue(TPostTaskCb&&) override {} }; using TTestOAuthFactory = TIamOAuthCredentialsProviderFactory< @@ -183,28 +181,31 @@ TEST(IamServiceCredentialsProvider, NoArgProviderIsCachedAcrossFactoryInstances) secondServiceParams.TargetServiceAccountId = "another-target"; auto differentProvider = CreateIamServiceCredentialsProviderFactory(secondServiceParams)->CreateProvider(); EXPECT_NE(firstProvider, differentProvider); + EXPECT_EQ(differentProvider->GetAuthInfo(), "outer-service-token"); EXPECT_EQ(stub.GetCreateRequestCount(), 1); EXPECT_EQ(stub.GetCreateForServiceRequestCount(), 2); server.Stop(); } -TEST(IamCredentialsProvider, AsyncCreationFailsWithExpiredFacility) { - auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr<ICoreFacility>{}); +TEST(IamCredentialsProvider, AuthInfoFailsWithExpiredFacility) { + auto provider = MakeOAuthFactory().CreateProvider(std::weak_ptr<ICoreFacility>{}); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.IsReady()); EXPECT_THROW(future.GetValue(), std::exception); } -TEST(IamCredentialsProvider, AsyncCreationFailsWhenPeriodicTaskIsRejected) { +TEST(IamCredentialsProvider, AuthInfoFailsWhenPeriodicTaskIsRejected) { auto facility = std::make_shared<TFailingCoreFacility>(); - auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr<ICoreFacility>(facility)); + auto provider = MakeOAuthFactory().CreateProvider(std::weak_ptr<ICoreFacility>(facility)); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.IsReady()); EXPECT_THROW(future.GetValue(), std::exception); } -TEST(IamCredentialsProvider, AsyncCreationRetriesTransientIamFailure) { +TEST(IamCredentialsProvider, AuthInfoRetriesTransientIamFailure) { TFlakyIamServiceStub stub; TIamGrpcServer server(&stub); ASSERT_TRUE(server.Start()); @@ -215,22 +216,17 @@ TEST(IamCredentialsProvider, AsyncCreationRetriesTransientIamFailure) { params.OAuthToken = "token"; params.RequestTimeout = TDuration::Seconds(2); - auto future = TTestOAuthFactory(params).CreateProviderAsync(); + auto provider = TTestOAuthFactory(params).CreateProvider(); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); - auto provider = future.GetValue(); - EXPECT_EQ(provider->GetAuthInfo(), "oauth-token"); + EXPECT_EQ(future.GetValue(), "oauth-token"); EXPECT_GE(stub.Attempts(), 2u); server.Stop(); } -// Regression test for the deprecated no-arg CreateProvider() on the IAM service-account -// factory with a nested gRPC JWT auth provider. Before the fix, both providers shared a single -// TSimpleCoreFacility, each registered a periodic refresh task, and TSimpleCoreFacility's -// single-task invariant tripped Y_ABORT_UNLESS and killed the process. The fix gives the -// nested auth provider its own facility (via a recursive no-arg CreateProvider()), so the two -// periodic tasks land on separate facilities. +// The no-arg service provider keeps nested gRPC authentication on a separate facility. TEST(IamServiceCredentialsProvider, NoArgCreateProviderWithGrpcInnerCreds) { TIamServiceStub stub; TIamGrpcServer server(&stub); diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp index 7b2d0dd973e..724dedcbc33 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp @@ -1,5 +1,6 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/credentials.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/from_file.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h> #include <ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.h> #include <library/cpp/string_utils/base64/base64.h> @@ -12,6 +13,8 @@ #include <util/string/builder.h> #include <util/system/tempfile.h> +#include <future> + using namespace NYdb; extern const std::string TestRSAPrivateKeyContent; @@ -108,6 +111,19 @@ struct TTestConfigFile : public TJsonFiller<TTestConfigFile> { }; Y_UNIT_TEST_SUITE(TestTokenExchange) { + bool WaitRequest(TTestTokenExchangeServer& server, TDuration timeout) { + const auto deadline = TInstant::Now() + timeout; + do { + bool received = false; + server.WithLock([&] { received = server.Check.InputParams.has_value(); }); + if (received) { + return true; + } + Sleep(TDuration::MilliSeconds(10)); + } while (TInstant::Now() < deadline); + return false; + } + void Exchanges(bool fromConfig) { TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); @@ -405,28 +421,6 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { server.Check.ExpectedInputParams.erase("scope"); - server.Check.ExpectedErrorPart = "can not connect to"; - server.Check.ExpectRequest = false; - if (fromConfig) { - server.RunFromConfig( - TTestConfigFile() - .Field("token-endpoint", "https://localhost:42/aaa") - .SubMap("subject-credentials") - .Field("type", "Fixed") - .Field("token", "test_token") - .Field("token-type", "test_token_type") - .Build() - .Build() - ); - } else { - server.Run( - TOauth2TokenExchangeParams() - .TokenEndpoint("https://localhost:42/aaa") - .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) - ); - } - server.Check.ExpectRequest = true; - // parsing response server.Check.StatusCode = HTTP_FORBIDDEN; server.Check.Response = R"(not json)"; @@ -488,16 +482,13 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { server.WithLock( [&]() { + server.Check.Reset(); server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 1})"; } ); - Sleep(TDuration::Seconds(1)); - server.Run( - [&]() { - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - } - ); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); } Y_UNIT_TEST(UpdatesToken) { @@ -540,138 +531,44 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer the_only_token"); } - Y_UNIT_TEST(UpdatesTokenInBackgroud) { - TCredentialsProviderFactoryPtr factory; - TInstant startTime; - - TTestTokenExchangeServer server; - server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); - server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); - server.Check.ExpectedInputParams.emplace("actor_token", "test_token"); - server.Check.ExpectedInputParams.emplace("actor_token_type", "test_token_type"); - - for (int i = 0; i < 2; ++i) { - server.WithLock( - [&]() { - server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 2})"; - } - ); - if (!factory) { - server.Run( - [&]() { - factory = CreateOauth2TokenExchangeCredentialsProviderFactory( - TOauth2TokenExchangeParams() - .TokenEndpoint(server.GetEndpoint()) - .ActorTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - startTime = TInstant::Now(); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); - } - ); - } - - server.WithLock( - [&]() { - server.Check.Reset(); - if (i == 0) { - server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 2})"; - } else { - server.Check.Response = R"({"access_token": "token_3", "token_type": "bearer", "expires_in": 2})"; - } - } - ); - - SleepUntil(startTime + TDuration::Seconds(1) + TDuration::MilliSeconds(5)); - const std::string token = factory->CreateProvider()->GetAuthInfo(); - TInstant halfTimeTokenValid = TInstant::Now(); - if (halfTimeTokenValid < startTime + TDuration::Seconds(2)) { // valid => got cached token, but async update must be run after half time token is valid - if (i == 0) { - UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_1"); - } else { - UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_2"); - } - do { - Sleep(TDuration::MilliSeconds(10)); - bool gotRequest = false; - server.WithLock( - [&]() { - if (server.Check.InputParams) { // InputParams are created => got the request - gotRequest = true; - } - } - ); - if (gotRequest) { - startTime = TInstant::Now(); // for second iteration - break; - } - } while (TInstant::Now() <= startTime + TDuration::Seconds(30)); - server.CheckExpectations(); - server.WithLock( - [&]() { - server.Check.Reset(); - server.Check.Response = R"(invalid response)"; // update must finish asyncronously - } - ); - Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token - if (i == 0) { // Finally check that we got updated token - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - } else { - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_3"); - } - Cerr << "Checked backgroud update on " << i << " iteration" << Endl; - } - } - } - Y_UNIT_TEST(UpdatesTokenAndRetriesErrors) { TCredentialsProviderFactoryPtr factory; + auto facility = CreateSimpleCoreFacility(); + TCredentialsProviderPtr provider; TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); - server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 6})"; + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 2})"; server.Run( [&]() { factory = CreateOauth2TokenExchangeCredentialsProviderFactory( TOauth2TokenExchangeParams() .TokenEndpoint(server.GetEndpoint()) .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + provider = factory->CreateProvider(facility); + UNIT_ASSERT_VALUES_EQUAL(provider->GetAuthInfo(), "Bearer token_1"); } ); server.WithLock( [&]() { server.Check.Reset(); - server.Check.StatusCode = HTTP_BAD_REQUEST; // all errors are temporary, because the first attempt is always successful (in constructor) + server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; server.Check.Response = R"({"error": "tmp", "error_description": "temporary error"})"; } ); - Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); - - auto waitRequest = [&](TDuration howLong) { - TInstant startTime = TInstant::Now(); - bool gotRequest = false; - do { - Sleep(TDuration::MilliSeconds(10)); - server.WithLock( - [&]() { - if (server.Check.InputParams) { // InputParams are created => got the request - gotRequest = true; - } - } - ); - if (gotRequest) { - break; - } - } while (TInstant::Now() <= startTime + howLong); - return gotRequest; - }; - - UNIT_ASSERT(waitRequest(TDuration::Seconds(30))); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(30))); + auto future = provider->GetAuthInfoAsync(); + UNIT_ASSERT(!future.IsReady()); + auto released = std::make_shared<std::promise<void>>(); + future.Subscribe([provider = std::move(provider), released](const auto&) mutable { + provider.reset(); + released->set_value(); + }); server.WithLock( [&]() { @@ -681,24 +578,15 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { } ); - UNIT_ASSERT(waitRequest(TDuration::Seconds(10))); - Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - - server.WithLock( - [&]() { - server.Check.Reset(); - server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; - server.Check.Response = R"({})"; - } - ); - - Sleep(TDuration::Seconds(2)); - UNIT_ASSERT_EXCEPTION(factory->CreateProvider()->GetAuthInfo(), std::runtime_error); + UNIT_ASSERT(future.Wait(TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), "Bearer token_2"); + UNIT_ASSERT(released->get_future().wait_for(std::chrono::seconds(10)) == std::future_status::ready); } Y_UNIT_TEST(ShutdownWhileRefreshingToken) { TCredentialsProviderFactoryPtr factory; + auto facility = CreateSimpleCoreFacility(); + TCredentialsProviderPtr provider; TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); @@ -713,7 +601,8 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { TOauth2TokenExchangeParams() .TokenEndpoint(server.GetEndpoint()) .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + provider = factory->CreateProvider(facility); + UNIT_ASSERT_VALUES_EQUAL(provider->GetAuthInfo(), "Bearer token_1"); } ); @@ -725,14 +614,16 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { } ); - Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); - - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(30))); + auto future = provider->GetAuthInfoAsync(); + UNIT_ASSERT(!future.IsReady()); const TInstant shutdownStart = TInstant::Now(); - factory = nullptr; + provider = nullptr; const TInstant shutdownStop = TInstant::Now(); - Cerr << "Shutdown: " << (shutdownStop - shutdownStart) << Endl; + UNIT_ASSERT(shutdownStop - shutdownStart < TDuration::Seconds(1)); + UNIT_ASSERT(future.IsReady()); + UNIT_ASSERT_EXCEPTION(future.GetValue(), yexception); } Y_UNIT_TEST(ExchangesFromFileConfig) { diff --git a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp index 1e745f8e618..26da1c120be 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp @@ -17,9 +17,7 @@ void TTestTokenExchangeServer::Run(const NYdb::TOauth2TokenExchangeParams& param std::string token; Run([&]() { auto factory = CreateOauth2TokenExchangeCredentialsProviderFactory(params); - if (!expectedToken.empty()) { - token = factory->CreateProvider()->GetAuthInfo(); - } + token = factory->CreateProvider()->GetAuthInfo(); }, checkExpectations); @@ -32,9 +30,7 @@ void TTestTokenExchangeServer::RunFromConfig(const std::string& fileName, const std::string token; Run([&]() { auto factory = NYdb::CreateOauth2TokenExchangeFileCredentialsProviderFactory(fileName, explicitTokenEndpoint); - if (!expectedToken.empty()) { - token = factory->CreateProvider()->GetAuthInfo(); - } + token = factory->CreateProvider()->GetAuthInfo(); }, checkExpectations); diff --git a/ydb/public/sdk/cpp/tests/unit/client/query/virtual_timestamp_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/query/virtual_timestamp_ut.cpp new file mode 100644 index 00000000000..34589da9439 --- /dev/null +++ b/ydb/public/sdk/cpp/tests/unit/client/query/virtual_timestamp_ut.cpp @@ -0,0 +1,42 @@ +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/virtual_timestamp.h> + +#include <library/cpp/testing/unittest/registar.h> + +using NYdb::NScheme::TVirtualTimestamp; + +Y_UNIT_TEST_SUITE(VirtualTimestamp) { + Y_UNIT_TEST(LexicographicLess) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) < TVirtualTimestamp(2, 1)); + UNIT_ASSERT(TVirtualTimestamp(1, 50) < TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(2, 1) < TVirtualTimestamp(1, 100))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) < TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(LexicographicGreater) { + UNIT_ASSERT(TVirtualTimestamp(2, 1) > TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) > TVirtualTimestamp(1, 50)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) > TVirtualTimestamp(2, 1))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) > TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(LessOrEqual) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) <= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 50) <= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) <= TVirtualTimestamp(2, 1)); + UNIT_ASSERT(!(TVirtualTimestamp(2, 1) <= TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(GreaterOrEqual) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) >= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) >= TVirtualTimestamp(1, 50)); + UNIT_ASSERT(TVirtualTimestamp(2, 1) >= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 50) >= TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(Equality) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) == TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) == TVirtualTimestamp(1, 101))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) == TVirtualTimestamp(2, 100))); + UNIT_ASSERT(TVirtualTimestamp(1, 100) != TVirtualTimestamp(2, 100)); + } +} diff --git a/ydb/public/sdk/cpp/tests/unit/client/query/ya.make b/ydb/public/sdk/cpp/tests/unit/client/query/ya.make index c36998d57d6..f0572e36a5a 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/query/ya.make +++ b/ydb/public/sdk/cpp/tests/unit/client/query/ya.make @@ -13,6 +13,7 @@ SRCS( client_session_ut.cpp deferred_session_creation_ut.cpp query_stats_ut.cpp + virtual_timestamp_ut.cpp ) PEERDIR( @@ -23,6 +24,7 @@ PEERDIR( ydb/public/sdk/cpp/src/client/impl/session ydb/public/sdk/cpp/src/client/query/impl ydb/public/sdk/cpp/src/client/query + ydb/public/sdk/cpp/src/client/types ) END() diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp index a3555725f02..ba828b2907d 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + namespace NKikimr { namespace NGRpcProxy { @@ -78,7 +80,8 @@ void TClustersUpdater::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TA } ctx.Schedule(TDuration::Seconds(AppData(ctx)->PQConfig.GetClustersUpdateTimeoutSec()), new TEvPQClustersUpdater::TEvUpdateClusters()); } else { - LOG_ERROR_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "can't update clusters " << record); + YDB_LOG_ERROR_CTX(ctx, "Can't update clusters", + {"record", record}); ctx.Schedule(TDuration::Seconds(CLUSTERS_UPDATER_TIMEOUT_ON_ERROR), new TEvPQClustersUpdater::TEvUpdateClusters()); } } diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index 54e10ae2006..0c20b05f00f 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -24,6 +24,8 @@ #include <algorithm> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_READ_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -39,8 +41,8 @@ using namespace NSchemeCache; #ifdef PQ_LOG_PREFIX #undef PQ_LOG_PREFIX #endif -#define PQ_LOG_PREFIX "session cookie " << Cookie << " client " << InternalClientId << " session " << Session +#define PQ_LOG_PREFIX (TStringBuilder() << "session cookie " << Cookie << " client " << InternalClientId << " session " << Session) //11 tries = 10,23 seconds, then each try for 5 seconds , so 21 retries will take near 1 min static const NTabletPipe::TClientRetryPolicy RetryPolicyForPipes = { @@ -352,7 +354,8 @@ void TReadSessionActor::Die(const TActorContext& ctx) { if (t.second->PipeClient) NTabletPipe::CloseClient(ctx, t.second->PipeClient); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Is DEAD", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); if (SessionsActive) { --(*SessionsActive); @@ -408,7 +411,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorCont CloseSession(TStringBuilder() << "double commit of cookie " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx); return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from client for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request from client", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readId", readId}); MakeCommit(ctx); } @@ -427,7 +432,10 @@ void TReadSessionActor::MakeCommit(const TActorContext& ctx) { if (readId == ReadIdCommitted) return; NextCommits.erase(NextCommits.begin(), it); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from " << ReadIdCommitted + 1 << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdCommittedPlus1", ReadIdCommitted + 1}, + {"readId", readId}); auto& commit = Commits[readId]; commit.StartReadId = ReadIdCommitted + 1; @@ -486,8 +494,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const TActorC if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL get status request from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL get status request from client for generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"generation", ev->Get()->Generation}); return; } @@ -514,13 +525,22 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvLocked::TPtr& ev, const TActorCont if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL lock from client for " << topic - << ":" << ev->Get()->Partition << " at offset " << ev->Get()->ReadOffset << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL lock from client for at offset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"generation", ev->Get()->Generation}); return; } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got lock from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " at readOffset " << ev->Get()->ReadOffset << " commitOffset " << ev->Get()->CommitOffset << " generation " << ev->Get()->Generation); + YDB_LOG_INFO_CTX(ctx, "Got lock from client for at readOffset commitOffset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"commitOffset", ev->Get()->CommitOffset}, + {"generation", ev->Get()->Generation}); //proxy request to partition - allow initing //TODO: add here VerifyReadOffset too and check it against Committed position @@ -582,8 +602,10 @@ void TReadSessionActor::AnswerForCommitsIfCan(const TActorContext& ctx) { for (ui64 i = it->second.StartReadId; i <= readId; ++i){ result.MutableCommit()->AddCookie(i); } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits from " << it->second.StartReadId - << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Replying for commits", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"startReadId", it->second.StartReadId}, + {"readId", readId}); ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; @@ -706,7 +728,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo topicsToResolve.insert(t); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "From", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"init", event->Request}, + {"peerName", PeerName}); ClientsideLocksAllowed = init.GetClientsideLocksAllowed(); BalanceRightNow = init.GetBalancePartitionRightNow() || CommitsDisabled; @@ -716,7 +741,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo } if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, "session without AuthInfo : " << ExternalClientId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo", + {"externalClientId", ExternalClientId}, + {"peerName", PeerName}); if (SessionsWithoutAuth) { ++(*SessionsWithoutAuth); } @@ -811,8 +838,10 @@ void TReadSessionActor::CreateInitAndAuthActor(const TActorContext& ctx) { void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) { TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_DEBUG_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -827,7 +856,9 @@ void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& void TReadSessionActor::RegisterSession(const TActorId& pipe, const TString& topic, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " register session to " << topic); + YDB_LOG_INFO_CTX(ctx, "Register session", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}); THolder<TEvPersQueue::TEvRegisterReadSession> request; request.Reset(new TEvPersQueue::TEvRegisterReadSession); auto& req = request->Record; @@ -940,7 +971,10 @@ void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const LastACLCheckTimestamp = ctx.Now(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone); + YDB_LOG_DEBUG_CTX(ctx, "Auth ok, got topics, init done", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topicAndTablets", ev->Get()->TopicAndTablets.size()}, + {"initDone", InitDone}); AuthInitActor = TActorId(); AuthInflight = false; @@ -1046,10 +1080,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path)); if (converterIter.IsEnd()) { - LOG_ALERT_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for event = " << record.ShortDebugString() << " path not recognized" - ); + YDB_LOG_ALERT_CTX(ctx, "Ignored ev lock for event path not recognized", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"record", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse lock event: " << record.ShortDebugString() << ", reason: topic not found", NPersQueue::NErrorCode::ERROR, ctx @@ -1062,21 +1095,18 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto jt = Topics.find(intName); if (jt == Topics.end() || pipe != jt->second->PipeClient) { //this is message from old version of pipe - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for topic = " << converterIter->second->GetPrintableString() - << " path recognized, but topic is unknown, this is unexpected" - ); + YDB_LOG_DEBUG_CTX(ctx, "Ignored ev lock for topic path recognized, but topic is unknown, this is unexpected", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", converterIter->second->GetPrintableString()}); return; } auto partitionGraph = jt->second->GetPartitionGraph(); auto* partitionNode = partitionGraph->GetPartition(record.GetPartition()); if (!partitionNode) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " lock for unknown partition = " << record.GetPartition() - ); + YDB_LOG_DEBUG_CTX(ctx, "Lock for unknown partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.GetPartition()}); Locks.push_back(ev->Release()); if (!AuthInflight) { SendAuthRequest(ctx); @@ -1124,7 +1154,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T it->second.PartitionsLocked.Inc(); it->second.PartitionsInfly.Inc(); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " lock: " << record); + YDB_LOG_INFO_CTX(ctx, "Lock partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"ev", record}); ctx.Send(actorId, new TEvPQProxy::TEvLockPartition(0, 0, false, !ClientsideLocksAllowed)); } @@ -1137,8 +1169,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T auto it = Partitions.find(std::make_pair(evTopic->GetClientsideName(), ev->Get()->Partition)); Y_ABORT_UNLESS(it != Partitions.end()); if (!it->second.LockGeneration) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " the unlocked partition " << ev->Get()->Partition << " status has been requested"); + YDB_LOG_ALERT_CTX(ctx, "The unlocked partition status has been requested", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", ev->Get()->Partition}); CloseSession( TStringBuilder() << "Internal server error, the unlocked partition " << ev->Get()->Partition << " status has been requested", NPersQueue::NErrorCode::ERROR, ctx @@ -1212,7 +1245,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(topic)); if (converterIter.IsEnd()) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Failed to parse balancer response: " << record.ShortDebugString()); + YDB_LOG_ALERT_CTX(ctx, "Failed to parse balancer", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"response", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse release event: " << record.ShortDebugString() << ", path not recognized", NPersQueue::NErrorCode::ERROR, ctx @@ -1232,7 +1267,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons } auto onUnknownPartition = [&](auto& marker) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Releasing unknown partition: " << record.ShortDebugString() << " " << marker); + YDB_LOG_ALERT_CTX(ctx, "Releasing unknown", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.ShortDebugString()}, + {"marker", marker}); CloseSession( TStringBuilder() << "Internal server error, releasing unknown partition: " << record.ShortDebugString() << " " << marker, NPersQueue::NErrorCode::ERROR, ctx @@ -1265,7 +1303,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons it->second.PartitionsToBeReleased.Inc(); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " releasing " << jt->first.first << ":" << jt->first.second); + YDB_LOG_INFO_CTX(ctx, "Releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", jt->first.first}, + {"partition", jt->first.second}); jt->second.Releasing = true; ctx.Send(actorId, new TEvPQProxy::TEvReleasePartition()); @@ -1319,7 +1360,10 @@ void TReadSessionActor::InformBalancerAboutRelease(const THashMap<std::pair<TStr req.SetTopic(it->first.first); req.SetPartition(it->first.second); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->first.first << ":" << it->first.second); + YDB_LOG_INFO_CTX(ctx, "Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", it->first.first}, + {"partition", it->first.second}); NTabletPipe::SendData(ctx, jt->second->PipeClient, request.Release()); } @@ -1347,17 +1391,21 @@ void TReadSessionActor::CloseSession(const TString& errorReason, const NPersQueu error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed with error reason: " << errorReason); + YDB_LOG_INFO_CTX(ctx, "Closed with error", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"reason", errorReason}); if (!Handler->IsShuttingDown()) { ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; Handler->Reply(std::move(result)); } else { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " GRps is shutting dows, skip reply"); + YDB_LOG_WARN_CTX(ctx, "GRps is shutting down, skip reply", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed"); + YDB_LOG_INFO_CTX(ctx, "Closed", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } Die(ctx); @@ -1457,7 +1505,9 @@ bool TReadSessionActor::ProcessReleasePartition(const THashMap<std::pair<TString bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorContext& ctx) { for (auto& t : Topics) { if (t.second->TabletID == tablet) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " balancer for topic " << t.first << " is dead, restarting all from this topic"); + YDB_LOG_INFO_CTX(ctx, "Balancer for topic is dead, restarting all from this topic", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", t.first}); //Drop all partitions from this topic for (auto it = Partitions.begin(); it != Partitions.end();) { @@ -1520,7 +1570,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContex ProcessAuth(event->Request.GetCredentials()); event->Request.ClearCredentials(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got read request: " << event->Request.GetRead() << " with guid: " << event->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Got read with", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", event->Request.GetRead()}, + {"guid", event->Guid}); Reads.emplace_back(event.Release()); @@ -1562,9 +1615,12 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TAct TFormedReadResponse::TPtr formedResponse = it->second; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " read done guid " << formedResponse->Guid - << " " << key.first << ":" << key.second - << " size " << event->Response.ByteSize()); + YDB_LOG_DEBUG_CTX(ctx, "Read done guid size", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}, + {"keyFirst", key.first}, + {"keySecond", key.second}, + {"byteSize", event->Response.ByteSize()}); const i64 diff = formedResponse->ApplyResponse(std::move(event->Response)); if (event->FromDisk) { @@ -1609,7 +1665,10 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo const bool hasMessages = HasMessages(formedResponse->Response.GetBatchedData()); if (hasMessages) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " assign read id " << ReadIdToResponse << " to read request " << formedResponse->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Assign read id to read request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdToResponse", ReadIdToResponse}, + {"guid", formedResponse->Guid}); formedResponse->Response.MutableBatchedData()->SetCookie(ReadIdToResponse); // reply to client if (ProtocolVersion < NPersQueue::TReadRequest::Batching) { @@ -1618,7 +1677,9 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo diff -= formedResponse->Response.ByteSize(); // Bytes will be tracked inside handler Handler->Reply(std::move(formedResponse->Response)); } else { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " empty read result " << formedResponse->Guid << ", start new reading"); + YDB_LOG_DEBUG_CTX(ctx, "Empty read result start new reading", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}); } BytesInflight_ -= diff; @@ -1764,11 +1825,16 @@ bool TReadSessionActor::ProcessReads(const TActorContext& ctx) { readR->SetMaxTimeLagMs(MaxTimeLagMs); readR->SetReadTimestampMs(ReadTimestampMs); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX - << " performing read request: " << (*readR) << " with guid " << read->Guid - << " from " << part.Topic->GetPrintableString() << ", partition:" << part.Partition - << " count " << ccount << " size " << csize - << " partitionsAsked " << partitionsAsked << " maxTimeLag " << MaxTimeLagMs << "ms"); + YDB_LOG_DEBUG_CTX(ctx, "Performing read with guid from count size partitionsAsked maxTimeLag ms", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", (*readR)}, + {"guid", read->Guid}, + {"topic", part.Topic->GetPrintableString()}, + {"partition", part.Partition}, + {"ccount", ccount}, + {"csize", csize}, + {"partitionsAsked", partitionsAsked}, + {"maxTimeLagMs", MaxTimeLagMs}); Y_ABORT_UNLESS(!it->second.Reading); @@ -1807,10 +1873,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionReady::TPtr& ev, const TA if (!ActualPartitionActor(ev->Sender)) return; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << ev->Get()->Topic->GetPrintableString() - << " partition:" << ev->Get()->Partition << " ready for read with readOffset " - << ev->Get()->ReadOffset << " endOffset " << ev->Get()->EndOffset << " WTime " - << ev->Get()->WTime << " sizeLag " << ev->Get()->SizeLag); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset WTime sizeLag", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic->GetPrintableString()}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"endOffset", ev->Get()->EndOffset}, + {"WTime", ev->Get()->WTime}, + {"sizeLag", ev->Get()->SizeLag}); const auto it = PartitionToReadResponse.find(ev->Sender); // check whether this partition is taking part in read response auto& container = it != PartitionToReadResponse.end() ? it->second->PartitionsBecameAvailable : AvailablePartitions; @@ -1834,7 +1904,8 @@ void TReadSessionActor::HandleWakeup(const TActorContext& ctx) { ForceACLCheck = false; RequestNotChecked = false; Y_ABORT_UNLESS(!AuthInitActor); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout"); + YDB_LOG_DEBUG_CTX(ctx, "Checking auth because of timeout", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); SendAuthRequest(ctx); } @@ -1935,17 +2006,29 @@ void TPartitionActor::Bootstrap(const TActorContext&) { void TPartitionActor::CheckRelease(const TActorContext& ctx) { const bool hasUncommittedData = ReadOffset > ClientCommitOffset && ReadOffset > ClientReadOffset; if (NeedRelease) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " checking release readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_DEBUG_CTX(ctx, "Checking release readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } if (NeedRelease && ReadGuid.empty() && CommitsInfly.empty() && !hasUncommittedData && !Released) { Released = true; ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReleased(Topic, Partition)); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " check release done - releasing; readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_INFO_CTX(ctx, "Check release done - releasing; readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } } @@ -1991,9 +2074,14 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc Y_ABORT_UNLESS(!Session.empty()); commit->SetSessionId(Session); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" - << Partition << " committing to position " << offset << " prev " << CommittedOffset - << " end " << EndOffset << " by cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Committing to position prev end by cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); @@ -2004,17 +2092,22 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { if (CommitsInfly.empty()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for nothing"); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting for nothing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}); return; } ui64 readId = CommitsInfly.front().first; if (cookie != readId) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}, + {"readId", readId}); return; } @@ -2025,17 +2118,23 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { CommitsInfly.pop_front(); if (readId != Max<ui64>()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit done to position endOffset with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); while (!CommitsInfly.empty() && CommitsInfly.front().second == Max<ui64>()) { //this is cookies that have no effect on this partition readId = CommitsInfly.front().first; CommitsInfly.pop_front(); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " commit done with no effect with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Partition commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); } CheckRelease(ctx); @@ -2043,8 +2142,12 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { } void TPartitionActor::SendPartitionReady(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << Topic->GetPrintableString() << " partition:" << Partition - << " ready for read with readOffset " << ReadOffset << " endOffset " << EndOffset); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}); if (FirstRead) { ctx.Send(ParentId, new TEvPQProxy::TEvReadingStarted(Topic->GetInternalName(), Partition)); FirstRead = false; @@ -2072,8 +2175,12 @@ void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reaso ctx.Schedule(TDuration::MilliSeconds(RESTART_PIPE_DELAY_MS), new TEvPQProxy::TEvRestartPipe()); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " schedule pipe restart attempt " << PipeGeneration << " reason: " << reason); + YDB_LOG_INFO_CTX(ctx, "Schedule pipe restart attempt", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"reason", reason}); } @@ -2092,13 +2199,23 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); Y_ABORT_UNLESS(TabletID); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " RequestInfly " << RequestInfly << " ReadOffset " << ReadOffset << " EndOffset " << EndOffset - << " InitDone " << InitDone << " WaitForData " << WaitForData); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt RequestInfly ReadOffset EndOffset InitDone WaitForData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"requestInfly", RequestInfly}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"initDone", InitDone}, + {"waitForData", WaitForData}); if (RequestInfly) { //got read infly - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " resend " << CurrentRequest); + YDB_LOG_INFO_CTX(ctx, "Resend", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"currentRequest", CurrentRequest}); TAutoPtr<TEvPersQueue::TEvRequest> event(new TEvPersQueue::TEvRequest); event->Record = CurrentRequest; @@ -2152,16 +2269,21 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " initDone " << InitDone << " event " << PartitionResponseToLog(result)); + YDB_LOG_DEBUG_CTX(ctx, "InitDone event", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"initDone", InitDone}, + {"partitionResponseToLog", PartitionResponseToLog(result)}); if (!InitDone) { if (result.GetCookie() != INIT_COOKIE) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited response in init with cookie " << result.GetCookie()); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response in init with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}); return; } Y_ABORT_UNLESS(RequestInfly); @@ -2183,9 +2305,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo ClientHasAnyCommits = resp.GetClientHasAnyCommits(); InitDone = true; PipeGeneration = 0; //reset tries counter - all ok - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " INIT DONE " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " << CommittedOffset); + YDB_LOG_INFO_CTX(ctx, "INIT DONE EndOffset readOffset committedOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}); if (!StartReading) { @@ -2207,9 +2333,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo const auto& res = result.GetCmdReadResult(); if (result.GetCookie() != (ui64)ReadOffset) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " unwaited read-response with cookie " << result.GetCookie() << "; waiting for " << ReadOffset << "; current read guid is " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "Partition unwaited read-response with cookie waiting for current read guid is", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); return; } @@ -2250,11 +2380,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo TString sourceId = ""; if (!r.GetSourceId().empty()) { if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "read bad sourceId from topic " << Topic->GetPrintableString() - << " partition:" << Partition - << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "' ReadGuid " << ReadGuid); + YDB_LOG_ERROR_CTX(ctx, "Read bad sourceId from topic offset seqNo sourceId ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"readGuid", ReadGuid}); } sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); } @@ -2311,13 +2444,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo if (proto.HasCodec()) { const auto codec = proto.GetCodec(); if (codec < Min<int>() || codec > Max<int>() || !NPersQueueCommon::ECodec_IsValid(codec)) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "data chunk (topic " << Topic->GetInternalName() << ", partition " << Partition - << ", offset " << r.GetOffset() << ", seqNo " << r.GetSeqNo() << ", sourceId " - << r.GetSourceId() << ") codec (id " << codec - << ") is not valid NPersQueueCommon::ECodec, loss of data compression codec information" - ); + YDB_LOG_ERROR_CTX(ctx, "Data chunk codec is not a valid NPersQueueCommon::ECodec; compression codec info may be lost", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetInternalName()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"codec", codec}); } message->SetCodec((NPersQueueCommon::ECodec)proto.GetCodec()); } @@ -2341,11 +2475,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo WaitDataInPartition(ctx); } - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " after read state " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "After read state EndOffset ReadOffset ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); ReadGuid = TString(); @@ -2364,10 +2500,12 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo void TPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { TEvTabletPipe::TEvClientConnected *msg = ev->Get(); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " pipe creation result: " << msg->Status); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt pipe creation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"result", msg->Status}); if (msg->Status != NKikimrProto::OK) { RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << msg->TabletId, NPersQueue::NErrorCode::ERROR); @@ -2381,12 +2519,13 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const void TPartitionActor::Handle(TEvPQProxy::TEvReleasePartition::TPtr&, const TActorContext& ctx) { - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " (partition)releasing " << Topic->GetPrintableString() << " partition:" << Partition - << " ReadOffset " << ReadOffset << " ClientCommitOffset " << ClientCommitOffset - << " CommittedOffst " << CommittedOffset - ); + YDB_LOG_INFO_CTX(ctx, "(partition)releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}); NeedRelease = true; CheckRelease(ctx); } @@ -2456,13 +2595,15 @@ void TPartitionActor::InitStartReading(const TActorContext& ctx) { Y_ABORT_UNLESS(AllPrepareInited); Y_ABORT_UNLESS(!WaitForData); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " Start reading " << Topic->GetPrintableString() << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " - << CommittedOffset << " clientCommittedOffset " << ClientCommitOffset - << " clientReadOffset " << ClientReadOffset - ); + YDB_LOG_INFO_CTX(ctx, "Start reading EndOffset readOffset committedOffset clientCommittedOffset clientReadOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"clientReadOffset", ClientReadOffset}); Counters.PartitionsToBeLocked.Dec(); LockCounted = false; @@ -2547,9 +2688,10 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) { cmd->SetGeneration(Generation); cmd->SetStep(Step); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " INITING " << Topic->GetPrintableString() << " partition:" << Partition); + YDB_LOG_INFO_CTX(ctx, "INITING", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); Y_ABORT_UNLESS(!RequestInfly); @@ -2588,11 +2730,11 @@ void TPartitionActor::WaitDataInPartition(const TActorContext& ctx) { event->Record.SetDeadline(deadline); event->Record.SetClientId(InternalClientId); - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait data in partition inited, cookie " << WaitDataCookie - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait data in partition inited", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"waitDataCookie", WaitDataCookie}); NTabletPipe::SendData(ctx, PipeClient, event.Release()); @@ -2609,10 +2751,11 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con auto it = WaitDataInfly.find(ev->Get()->Record.GetCookie()); if (it == WaitDataInfly.end()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " unwaited response for WaitData " << ev->Get()->Record); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response for WaitData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"record", ev->Get()->Record}); return; } WaitDataInfly.erase(it); @@ -2626,13 +2769,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con Y_ABORT_UNLESS(EndOffset <= record.GetEndOffset()); //end offset could not be changed if no data arrived, but signal will be sended anyway after timeout Y_ABORT_UNLESS(ReadOffset >= EndOffset); //otherwise no WaitData were needed - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait for data done: " << " readOffset " << ReadOffset << " EndOffset " << EndOffset - << " newEndOffset " << record.GetEndOffset() << " commitOffset " << CommittedOffset - << " clientCommitOffset " << ClientCommitOffset << " cookie " << ev->Get()->Record.GetCookie() - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait for data done: readOffset EndOffset newEndOffset commitOffset clientCommitOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"recordEndOffset", record.GetEndOffset()}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"cookie", ev->Get()->Record.GetCookie()}); EndOffset = record.GetEndOffset(); SizeLag = record.GetSizeLag(); @@ -2664,13 +2810,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " READ FROM " << Topic->GetPrintableString() << " partition:" << Partition - << " event " << ev->Get()->Request << " readOffset " << ReadOffset - << " EndOffset " << EndOffset << " ClientCommitOffset " << ClientCommitOffset - << " committedOffset " << CommittedOffset << " Guid " << ev->Get()->Guid - ); + YDB_LOG_DEBUG_CTX(ctx, "READ FROM event readOffset EndOffset ClientCommitOffset committedOffset Guid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"request", ev->Get()->Request}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}, + {"guid", ev->Get()->Guid}); Y_ABORT_UNLESS(!NeedRelease); Y_ABORT_UNLESS(!Released); @@ -2727,12 +2876,13 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex const ui64 offset = ev->Get()->Offset; Y_ABORT_UNLESS(offset != Max<ui64>()); // has concreete offset if (offset < ClientCommitOffset) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done to too small position " << offset - << " committedOffset " << ClientCommitOffset << " cookie " << readId - ); + YDB_LOG_ERROR_CTX(ctx, "Commit done to too small position committedOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"clientCommitOffset", ClientCommitOffset}, + {"readId", readId}); } Y_ABORT_UNLESS(offset >= ClientCommitOffset); @@ -2740,11 +2890,11 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex if (!hasProgress) {//nothing to commit for this partition if (CommitsInfly.empty()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done with no effect with cookie " << readId - ); + YDB_LOG_DEBUG_CTX(ctx, "Commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); CheckRelease(ctx); } else { diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp index d9a9abc1dcb..3a409920805 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp @@ -17,6 +17,8 @@ #include <util/string/vector.h> #include <util/string/escape.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -133,7 +135,9 @@ void TWriteSessionActor::Die(const TActorContext& ctx) { BytesInflightTotal.Dec(BytesInflightTotal_); } - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Session is DEAD", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); if (PartitionChooser) { ctx.Send(PartitionChooser, new TEvents::TEvPoison()); @@ -161,7 +165,9 @@ void TWriteSessionActor::CheckFinish(const TActorContext& ctx) { } void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " got TEvDone"); + YDB_LOG_INFO_CTX(ctx, "Session got TEvDone", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); WritesDone = true; CheckFinish(ctx); } @@ -231,7 +237,11 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor event->Request.ClearCredentials(); Y_PROTOBUF_SUPPRESS_NODISCARD Auth.SerializeToString(&AuthStr); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << ", Database: '" << Database << "' from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "Session request Database", + {"cookie", Cookie}, + {"init", init}, + {"database", Database}, + {"peerName", PeerName}); UserAgent = init.GetVersion(); LogSession(ctx); @@ -402,8 +412,10 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { //ACLCheckInProgress is still true - no recheck will be done - LOG_WARN_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session without AuthInfo : " << DiscoveryConverter->GetPrintableString() - << " sourceId " << SourceId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo sourceId", + {"discoveryConverter", DiscoveryConverter->GetPrintableString()}, + {"sourceId", SourceId}, + {"peerName", PeerName}); SessionsWithoutAuth.Inc(); if (FirstACLCheck) { FirstACLCheck = false; @@ -445,9 +457,10 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " - << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_INFO_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -459,7 +472,9 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check"); + YDB_LOG_INFO_CTX(ctx, "Session describe result for acl check", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); CheckACL(ctx); } @@ -548,13 +563,17 @@ void TWriteSessionActor::CloseSession(const TString& errorReason, const NPersQue error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, - "session error cookie: " << Cookie << " reason: \"" << errorReason << "\" code: " - << EErrorCode_Name(errorCode) << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session error reason", + {"cookie", Cookie}, + {"errorReason", errorReason}, + {"code", EErrorCode_Name(errorCode)}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(result)); } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session closed cookie: " << Cookie << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session closed", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); } Die(ctx); @@ -586,8 +605,11 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev Y_ABORT_UNLESS(FullConverter); init->SetTopic(FullConverter->GetClientsideName()); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition - << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session inited", + {"cookie", Cookie}, + {"partition", Partition}, + {"maxSeqNo", maxSeqNo}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(response)); @@ -886,9 +908,14 @@ void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const void TWriteSessionActor::LogSession(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie - << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v0 " - << " topic=" << DiscoveryConverter->GetPrintableString() << " durationSec=" << (ctx.Now() - StartTime).Seconds()); + YDB_LOG_INFO_CTX(ctx, "Write session", + {"cookie", Cookie}, + {"proto", "v0"}, + {"sessionId", OwnerCookie}, + {"userAgent", UserAgent}, + {"ip", PeerName}, + {"topic", DiscoveryConverter->GetPrintableString()}, + {"durationSec", (ctx.Now() - StartTime).Seconds()}); LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60); } diff --git a/ydb/services/persqueue_v1/actors/write_request_info.h b/ydb/services/persqueue_v1/actors/write_request_info.h index 41eaab6694e..0a637d607a4 100644 --- a/ydb/services/persqueue_v1/actors/write_request_info.h +++ b/ydb/services/persqueue_v1/actors/write_request_info.h @@ -91,7 +91,9 @@ TMaybe<NPQ::TDeferredPublishWriterOpts> TWriteRequestInfoImpl<TEvWrite>::GetDefe const auto& deferredPublish = request.deferred_publish(); return NPQ::TDeferredPublishWriterOpts{ deferredPublish.int_publication_id(), - deferredPublish.ext_publication_id(), + deferredPublish.has_ext_publication_id() + ? TMaybe<TString>(deferredPublish.ext_publication_id()) + : Nothing(), }; } } diff --git a/ydb/services/persqueue_v1/actors/write_session_actor.cpp b/ydb/services/persqueue_v1/actors/write_session_actor.cpp index 412d293170f..e4c0b6f2b8b 100644 --- a/ydb/services/persqueue_v1/actors/write_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/write_session_actor.cpp @@ -1571,14 +1571,8 @@ void TWriteSessionActor<UseMigrationProtocol>::Handle(typename TEvWrite::TPtr& e ctx); return; } - if (deferredPublish.ext_publication_id().empty()) { - CloseSession( - "WriteRequest.deferred_publish.ext_publication_id must not be empty", - PersQueue::ErrorCode::BAD_REQUEST, - ctx); - return; - } - if (deferredPublish.ext_publication_id().size() > NPQ::NDeferredPublish::MaxDeferredPublishStringLength) { + if (deferredPublish.has_ext_publication_id() + && deferredPublish.ext_publication_id().size() > NPQ::NDeferredPublish::MaxDeferredPublishStringLength) { CloseSession( "WriteRequest.deferred_publish.ext_publication_id is too long", PersQueue::ErrorCode::BAD_REQUEST, @@ -1587,16 +1581,19 @@ void TWriteSessionActor<UseMigrationProtocol>::Handle(typename TEvWrite::TPtr& e } const ui64 intPublicationId = deferredPublish.int_publication_id(); - const auto knownExt = DeferredPublicationExtByInt.FindPtr(intPublicationId); - if (knownExt && *knownExt != deferredPublish.ext_publication_id()) { - YDB_LOG_WARN_CTX(ctx, "Deferred publish ext_publication_id mismatch", - {"cookie", Cookie}, - {"sessionId", OwnerCookie}, - {"intPublicationId", intPublicationId}, - {"expectedExtPublicationId", *knownExt}, - {"actualExtPublicationId", deferredPublish.ext_publication_id()}); - } else if (!knownExt) { - DeferredPublicationExtByInt[intPublicationId] = deferredPublish.ext_publication_id(); + if (deferredPublish.has_ext_publication_id()) { + const auto& extPublicationId = deferredPublish.ext_publication_id(); + const auto knownExt = DeferredPublicationExtByInt.FindPtr(intPublicationId); + if (knownExt && *knownExt != extPublicationId) { + YDB_LOG_WARN_CTX(ctx, "Deferred publish ext_publication_id mismatch", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}, + {"intPublicationId", intPublicationId}, + {"expectedExtPublicationId", *knownExt}, + {"actualExtPublicationId", extPublicationId}); + } else if (!knownExt) { + DeferredPublicationExtByInt[intPublicationId] = extPublicationId; + } } } } diff --git a/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp b/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp index cfbe9e4c964..7bff5703560 100644 --- a/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp +++ b/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp @@ -431,17 +431,6 @@ void GrantPublicationTableWrite(NPersQueue::TTestServer& server, const TString& } } -void GrantPublicationRegistryDelete(NPersQueue::TTestServer& server, const TString& subject) { - GrantPublicationTableWrite(server, subject); - if (SchemePathExists(server, "/Root/.metadata/topic_deferred_publications")) { - server.AnnoyingClient->TestGrant( - "/Root/.metadata", - "topic_deferred_publications", - subject, - NACLib::EAccessRights::GenericWrite); - } -} - void InsertDestinationRow( NPersQueue::TTestServer& server, const TString& authTicket, @@ -812,7 +801,7 @@ void InitStreamWriteSession( Ydb::Topic::StreamWriteMessage::FromClient MakeStreamWriteRequest( ui64 seqNo, const TString& data, - const TMaybe<std::pair<ui64, TString>>& deferredPublish = Nothing(), + const TMaybe<std::pair<ui64, TMaybe<TString>>>& deferredPublish = Nothing(), const TMaybe<std::pair<TString, TString>>& tx = Nothing()) { Ydb::Topic::StreamWriteMessage::FromClient req; @@ -822,7 +811,9 @@ Ydb::Topic::StreamWriteMessage::FromClient MakeStreamWriteRequest( if (deferredPublish) { auto* deferred = write->mutable_deferred_publish(); deferred->set_int_publication_id(deferredPublish->first); - deferred->set_ext_publication_id(deferredPublish->second); + if (deferredPublish->second) { + deferred->set_ext_publication_id(*deferredPublish->second); + } } if (tx) { write->mutable_tx()->set_session(tx->first); @@ -837,6 +828,14 @@ Ydb::Topic::StreamWriteMessage::FromClient MakeStreamWriteRequest( return req; } +std::pair<ui64, TMaybe<TString>> DeferredPublishOmitExt(ui64 intPublicationId) { + return {intPublicationId, Nothing()}; +} + +std::pair<ui64, TMaybe<TString>> DeferredPublishWithExt(ui64 intPublicationId, const TString& extPublicationId) { + return {intPublicationId, MakeMaybe(extPublicationId)}; +} + void WriteAndExpectWriteResponse( grpc::ClientReaderWriter<Ydb::Topic::StreamWriteMessage::FromClient, Ydb::Topic::StreamWriteMessage::FromServer>& stream, Ydb::Topic::StreamWriteMessage::FromClient req) @@ -1936,7 +1935,7 @@ Y_UNIT_TEST(StreamWriteDeferredPublishAcksWrite) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "deferred-payload", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); AssertDestinationRowCount(fixture.Server, "root@builtin", fixture.IntPublicationId, 1); @@ -1960,20 +1959,53 @@ Y_UNIT_TEST(StreamWriteDeferredPublishDisabledByDefault) { WriteAndExpectFailure( *session->Stream, - MakeStreamWriteRequest(1, "payload", std::make_pair(1u, TString("ext-disabled"))), + MakeStreamWriteRequest(1, "payload", DeferredPublishWithExt(1u, TString("ext-disabled"))), Ydb::StatusIds::UNSUPPORTED, TString(DisabledMessage)); } -Y_UNIT_TEST(StreamWriteRejectsEmptyExtPublicationId) { +Y_UNIT_TEST(StreamWriteAllowsOmitExtPublicationId) { + auto fixture = TDeferredStreamWriteFixture::Enabled(); + auto session = fixture.OpenWriteStream("producer-omit-ext"); + + WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( + 1, + "payload", + DeferredPublishOmitExt(fixture.IntPublicationId))); +} + +Y_UNIT_TEST(StreamWriteAllowsEmptyExtPublicationId) { auto fixture = TDeferredStreamWriteFixture::Enabled(); auto session = fixture.OpenWriteStream("producer-empty-ext"); + WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( + 1, + "payload", + DeferredPublishWithExt(fixture.IntPublicationId, TString()))); +} + +Y_UNIT_TEST(StreamWriteAllowsNonEmptyExtPublicationId) { + auto fixture = TDeferredStreamWriteFixture::Enabled(); + auto session = fixture.OpenWriteStream("producer-nonempty-ext"); + + WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( + 1, + "payload", + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); +} + +Y_UNIT_TEST(StreamWriteRejectsTooLongExtPublicationId) { + auto fixture = TDeferredStreamWriteFixture::Enabled(); + auto session = fixture.OpenWriteStream("producer-long-ext"); + WriteAndExpectFailure( *session->Stream, - MakeStreamWriteRequest(1, "payload", std::make_pair(fixture.IntPublicationId, TString(""))), + MakeStreamWriteRequest( + 1, + "payload", + DeferredPublishWithExt(fixture.IntPublicationId, TString(MaxDeferredPublishStringLength + 1, 'x'))), Ydb::StatusIds::BAD_REQUEST, - TString("WriteRequest.deferred_publish.ext_publication_id must not be empty")); + TString("WriteRequest.deferred_publish.ext_publication_id is too long")); } Y_UNIT_TEST(StreamWriteFailsOnUnknownIntPublicationId) { @@ -1984,7 +2016,7 @@ Y_UNIT_TEST(StreamWriteFailsOnUnknownIntPublicationId) { UNIT_ASSERT(session->Stream->Write(MakeStreamWriteRequest( 1, "payload", - std::make_pair(999999u, TString("missing-ext"))))); + DeferredPublishWithExt(999999u, TString("missing-ext"))))); UNIT_ASSERT(session->Stream->Read(&resp)); UNIT_ASSERT(resp.status() != Ydb::StatusIds::SUCCESS); } @@ -1996,7 +2028,7 @@ Y_UNIT_TEST(StreamWriteDeferredThenRegularInSameSession) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "deferred-part", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest(2, "regular-part")); } @@ -2008,14 +2040,14 @@ Y_UNIT_TEST(StreamWriteMergesPartitionsIntoDestinationBlob) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "partition-0", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } { auto session = fixture.OpenWriteStream("producer-part-1", 1); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "partition-1", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } AssertDestinationRowCount(fixture.Server, "root@builtin", fixture.IntPublicationId, 1); @@ -2035,7 +2067,6 @@ Y_UNIT_TEST_SUITE(TopicDeferredPublishFinalize) { Y_UNIT_TEST(PublishAfterStreamWriteClearsRegistryAndMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-publish-topic", "ext-publish"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "deferred-payload-visible"; { @@ -2043,7 +2074,7 @@ Y_UNIT_TEST(PublishAfterStreamWriteClearsRegistryAndMakesDataVisible) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto publishOutcome = CallPublish(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2059,7 +2090,6 @@ Y_UNIT_TEST(PublishAfterStreamWriteClearsRegistryAndMakesDataVisible) { Y_UNIT_TEST(PublishAfterStreamWriteToTwoPartitionsMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-two-partitions-topic", "ext-two-partitions"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload0 = "deferred-payload-part-0"; constexpr TStringBuf payload1 = "deferred-payload-part-1"; @@ -2068,14 +2098,14 @@ Y_UNIT_TEST(PublishAfterStreamWriteToTwoPartitionsMakesDataVisible) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload0), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } { auto session = fixture.OpenWriteStream("producer-part-1", 1); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload1), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto publishOutcome = CallPublish(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2094,21 +2124,20 @@ Y_UNIT_TEST(PublishAfterStreamWriteToTwoPartitionsMakesDataVisible) { Y_UNIT_TEST(CancelAfterStreamWriteToTwoPartitionsClearsRegistryWithoutData) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-cancel-two-partitions-topic", "ext-cancel-two-partitions"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-cancel-0", 0); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "deferred-payload-cancel-0", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } { auto session = fixture.OpenWriteStream("producer-cancel-1", 1); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "deferred-payload-cancel-1", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto cancelOutcome = CallCancelPublication(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2123,14 +2152,13 @@ Y_UNIT_TEST(CancelAfterStreamWriteToTwoPartitionsClearsRegistryWithoutData) { Y_UNIT_TEST(CancelAfterStreamWriteClearsRegistryWithoutData) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-cancel-topic", "ext-cancel"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-cancel"); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "deferred-payload-cancel", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto cancelOutcome = CallCancelPublication(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2144,14 +2172,13 @@ Y_UNIT_TEST(CancelAfterStreamWriteClearsRegistryWithoutData) { Y_UNIT_TEST(RepeatFinalizeReturnsNotFound) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-repeat-topic", "ext-repeat"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-repeat"); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "payload-repeat", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto firstPublish = CallPublish(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2209,21 +2236,20 @@ Y_UNIT_TEST(PublishMultipleDestinations) { auto topicStub = MakeTopicServiceStub(server); const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-multi")); - GrantPublicationRegistryDelete(server, "root@builtin"); { auto session = TStreamWriteSession::Open(*topicStub, "finalize-multi-topic-a", "producer-a", 0); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "topic-a-payload", - std::make_pair(intPublicationId, TString("ext-multi")))); + DeferredPublishWithExt(intPublicationId, TString("ext-multi")))); } { auto session = TStreamWriteSession::Open(*topicStub, "finalize-multi-topic-b", "producer-b", 0); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "topic-b-payload", - std::make_pair(intPublicationId, TString("ext-multi")))); + DeferredPublishWithExt(intPublicationId, TString("ext-multi")))); } const auto publishOutcome = CallPublish(*deferredStub, "/Root", intPublicationId); @@ -2233,7 +2259,6 @@ Y_UNIT_TEST(PublishMultipleDestinations) { Y_UNIT_TEST(PublishBeforeWriteAckKeepsRegistry) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-before-ack-topic", "ext-before-ack"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); TFinalizePublicationOutcome publishOutcome; std::thread publishThread([&]() { @@ -2244,7 +2269,7 @@ Y_UNIT_TEST(PublishBeforeWriteAckKeepsRegistry) { UNIT_ASSERT(session->Stream->Write(MakeStreamWriteRequest( 1, "payload-before-ack", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId)))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId)))); publishThread.join(); UNIT_ASSERT_VALUES_UNEQUAL(publishOutcome.Operation.status(), Ydb::StatusIds::SUCCESS); @@ -2253,14 +2278,13 @@ Y_UNIT_TEST(PublishBeforeWriteAckKeepsRegistry) { Y_UNIT_TEST(PublishFailureOnInvalidDestinationKeepsRegistry) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-bad-dest-topic", "ext-bad-dest"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-bad-dest"); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "payload-bad-dest", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const TString badBlob = NPQ::NDeferredPublish::SerializeDestinationBlob( @@ -2303,7 +2327,6 @@ Y_UNIT_TEST_SUITE(TopicDeferredPublishLifecycle) { Y_UNIT_TEST(PublishMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-publish-topic", "ext-lifecycle-publish"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-publish-payload"; { @@ -2311,7 +2334,7 @@ Y_UNIT_TEST(PublishMakesDataVisible) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto publishOutcome = CallPublish(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2327,14 +2350,13 @@ Y_UNIT_TEST(PublishMakesDataVisible) { Y_UNIT_TEST(CancelDiscardsData) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-cancel-topic", "ext-lifecycle-cancel"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-lifecycle-cancel"); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, "lifecycle-cancel-payload", - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto cancelOutcome = CallCancelPublication(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2346,7 +2368,6 @@ Y_UNIT_TEST(CancelDiscardsData) { Y_UNIT_TEST(StagingNotVisibleBeforePublish) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-staging-topic", "ext-lifecycle-staging"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-staging-payload"; { @@ -2354,7 +2375,7 @@ Y_UNIT_TEST(StagingNotVisibleBeforePublish) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } AssertTopicNotVisible(fixture.Server, fixture.TopicShortName); @@ -2380,7 +2401,6 @@ Y_UNIT_TEST(MultiDestinationSinglePublication) { auto topicStub = MakeTopicServiceStub(server); const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-lifecycle-multi-dest")); - GrantPublicationRegistryDelete(server, "root@builtin"); constexpr TStringBuf payloadA = "lifecycle-topic-a"; constexpr TStringBuf payloadB = "lifecycle-topic-b"; @@ -2389,14 +2409,14 @@ Y_UNIT_TEST(MultiDestinationSinglePublication) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payloadA), - std::make_pair(intPublicationId, TString("ext-lifecycle-multi-dest")))); + DeferredPublishWithExt(intPublicationId, TString("ext-lifecycle-multi-dest")))); } { auto session = TStreamWriteSession::Open(*topicStub, "lifecycle-multi-topic-b", "producer-b", 0); WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payloadB), - std::make_pair(intPublicationId, TString("ext-lifecycle-multi-dest")))); + DeferredPublishWithExt(intPublicationId, TString("ext-lifecycle-multi-dest")))); } const auto publishOutcome = CallPublish(*deferredStub, "/Root", intPublicationId); @@ -2409,7 +2429,6 @@ Y_UNIT_TEST(MultiDestinationSinglePublication) { Y_UNIT_TEST(RepeatFinalizeNotFound) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-repeat-topic", "ext-lifecycle-repeat"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-repeat-payload"; { @@ -2417,7 +2436,7 @@ Y_UNIT_TEST(RepeatFinalizeNotFound) { WriteAndExpectWriteResponse(*session->Stream, MakeStreamWriteRequest( 1, TString(payload), - std::make_pair(fixture.IntPublicationId, fixture.ExtPublicationId))); + DeferredPublishWithExt(fixture.IntPublicationId, fixture.ExtPublicationId))); } const auto firstPublish = CallPublish(*fixture.DeferredStub, "/Root", fixture.IntPublicationId); @@ -2449,7 +2468,6 @@ Y_UNIT_TEST(BeginOnlyPublishAborts) { const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-begin-only-publish")); - GrantPublicationRegistryDelete(server, "root@builtin"); const auto publishOutcome = CallPublish(*deferredStub, "/Root", intPublicationId); UNIT_ASSERT_VALUES_EQUAL(publishOutcome.Operation.status(), Ydb::StatusIds::ABORTED); @@ -2467,7 +2485,6 @@ Y_UNIT_TEST(BeginOnlyCancelDeletes) { const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-begin-only-cancel")); - GrantPublicationRegistryDelete(server, "root@builtin"); const auto cancelOutcome = CallCancelPublication(*deferredStub, "/Root", intPublicationId); UNIT_ASSERT_VALUES_EQUAL(cancelOutcome.Operation.status(), Ydb::StatusIds::SUCCESS); diff --git a/ydb/core/kqp/workload_service/actors/actors.h b/ydb/services/workload_manager/actors/actors.h index 1dfcaa85911..82b68c5667f 100644 --- a/ydb/core/kqp/workload_service/actors/actors.h +++ b/ydb/services/workload_manager/actors/actors.h @@ -1,10 +1,10 @@ #pragma once -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/protos/workload_manager_config.pb.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { // Pool state holder NActors::IActor* CreatePoolHandlerActor(const TString& databaseId, const TString& poolId, const NResourcePool::TPoolSettings& poolConfig, NMonitoring::TDynamicCounterPtr counters); @@ -22,4 +22,4 @@ NActors::IActor* CreateDatabaseFetcherActor(const NActors::TActorId& replyActorI // Cpu load fetcher actor NActors::IActor* CreateCpuLoadFetcherActor(const NActors::TActorId& replyActorId); -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp b/ydb/services/workload_manager/actors/cpu_load_actors.cpp index 761a4628f21..466e90da0b5 100644 --- a/ydb/core/kqp/workload_service/actors/cpu_load_actors.cpp +++ b/ydb/services/workload_manager/actors/cpu_load_actors.cpp @@ -1,11 +1,11 @@ #include "actors.h" -#include <ydb/core/kqp/workload_service/common/events.h> +#include <ydb/services/workload_manager/common/events.h> #include <ydb/library/query_actor/query_actor.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -74,4 +74,4 @@ IActor* CreateCpuLoadFetcherActor(const TActorId& replyActorId) { return new TQueryRetryActor<TCpuLoadFetcherActor, TEvPrivate::TEvCpuLoadResponse>(replyActorId); } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/actors/pool_handlers_actors.cpp b/ydb/services/workload_manager/actors/pool_handlers_actors.cpp index 136c137197c..b3eac3e270e 100644 --- a/ydb/core/kqp/workload_service/actors/pool_handlers_actors.cpp +++ b/ydb/services/workload_manager/actors/pool_handlers_actors.cpp @@ -4,13 +4,15 @@ #include <ydb/core/kqp/common/events/events.h> #include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/proxy_service/kqp_session_state.h> -#include <ydb/core/kqp/workload_service/common/events.h> -#include <ydb/core/kqp/workload_service/common/helpers.h> -#include <ydb/core/kqp/workload_service/tables/table_queries.h> +#include <ydb/services/workload_manager/session_updater.h> +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/services/workload_manager/common/helpers.h> +#include <ydb/services/workload_manager/tables/table_queries.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/core/kqp/common/events/events.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -151,7 +153,7 @@ public: // Worker actor events hFunc(TEvCleanupRequest, Handle); - IgnoreFunc(TEvKqp::TEvCancelQueryResponse); + IgnoreFunc(NKqp::TEvKqp::TEvCancelQueryResponse); // Schemeboard events hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); @@ -165,7 +167,7 @@ public: } SendPoolInfoUpdate(std::nullopt, std::nullopt, Subscribers); - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvStopPoolHandlerResponse(DatabaseId, PoolId)); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvStopPoolHandlerResponse(DatabaseId, PoolId)); Counters.OnCleanup(ResetCountersOnStrop); @@ -193,7 +195,7 @@ private: const TString& sessionId = event->Get()->SessionId; const TString& requestText = event->Get()->RequestText; auto wmSessionUpdater = event->Get()->WmSessionUpdater; - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvPlaceRequestIntoPoolResponse(DatabaseId, PoolId, sessionId)); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvPlaceRequestIntoPoolResponse(DatabaseId, PoolId, sessionId)); const TActorId& workerActorId = event->Sender; if (!InFlightLimit) { @@ -406,7 +408,7 @@ protected: auto event = std::make_unique<TEvPrivate::TEvFinishRequestInPool>( DatabaseId, PoolId, request->Duration, request->CpuConsumed, request->UsedCpuQuota ); - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), event.release()); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), event.release()); LocalSessions.erase(request->SessionId); if (StopHandler && LocalSessions.empty()) { @@ -456,9 +458,9 @@ private: } void ReplyCancel(const TRequest* request) { - auto ev = std::make_unique<TEvKqp::TEvCancelQueryRequest>(); + auto ev = std::make_unique<NKqp::TEvKqp::TEvCancelQueryRequest>(); ev->Record.MutableRequest()->SetSessionId(request->SessionId); - this->Send(MakeKqpProxyID(this->SelfId().NodeId()), ev.release()); + this->Send(NKqp::MakeKqpProxyID(this->SelfId().NodeId()), ev.release()); Counters.Cancelled->Inc(); Counters.CollectRequestLatency(request->ContinueTime); @@ -497,7 +499,7 @@ private: if (ShouldResign()) { const TActorId& newHandler = this->RegisterWithSameMailbox(CreatePoolHandlerActor(DatabaseId, PoolId, poolConfig, Counters.CountersRoot)); - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvResignPoolHandler(DatabaseId, PoolId, newHandler)); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvResignPoolHandler(DatabaseId, PoolId, newHandler)); } } @@ -664,7 +666,7 @@ protected: } if (!PreparingFinished) { - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvPrepareTablesRequest(DatabaseId, PoolId)); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvPrepareTablesRequest(DatabaseId, PoolId)); } RefreshState(); @@ -682,7 +684,7 @@ protected: void RefreshState(bool refreshRequired = false) override { if (!WaitingNodesInfo && TInstant::Now() - LastNodesInfoRefreshTime > LEASE_DURATION) { WaitingNodesInfo = true; - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvNodesInfoRequest()); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvNodesInfoRequest()); } RefreshRequired |= refreshRequired; @@ -880,7 +882,7 @@ private: auto event = std::make_unique<TEvPrivate::TEvRefreshPoolState>(); event->Record.SetPoolId(PoolId); event->Record.SetDatabase(DatabaseId); - this->Send(MakeKqpWorkloadServiceId(nodeId), std::move(event)); + this->Send(NWorkloadManager::MakeServiceId(nodeId), std::move(event)); RefreshState(); return; } @@ -1004,7 +1006,7 @@ private: } void RequestCpuQuota(double loadCpuThreshold, EStartRequestCase requestCase) const { - this->Send(MakeKqpWorkloadServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvCpuQuotaRequest(loadCpuThreshold / 100.0), 0, static_cast<ui64>(requestCase)); + this->Send(NWorkloadManager::MakeServiceId(this->SelfId().NodeId()), new TEvPrivate::TEvCpuQuotaRequest(loadCpuThreshold / 100.0), 0, static_cast<ui64>(requestCase)); } private: @@ -1094,4 +1096,4 @@ IActor* CreatePoolHandlerActor(const TString& databaseId, const TString& poolId, return new TFifoPoolHandlerActor(databaseId, poolId, poolConfig, counters); } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/actors/scheme_actors.cpp b/ydb/services/workload_manager/actors/scheme_actors.cpp index fd00b160ea2..ab94b9a9509 100644 --- a/ydb/core/kqp/workload_service/actors/scheme_actors.cpp +++ b/ydb/services/workload_manager/actors/scheme_actors.cpp @@ -6,9 +6,9 @@ #include <ydb/core/protos/schemeshard/operations.pb.h> #include <ydb/core/protos/workload_manager_config.pb.h> -#include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/common/events.h> -#include <ydb/core/kqp/workload_service/common/helpers.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/services/workload_manager/common/helpers.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/tx_proxy/proxy.h> @@ -16,7 +16,7 @@ #include <ydb/library/table_creator/table_creator.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -106,14 +106,14 @@ private: void Reply(NResourcePool::TPoolSettings poolConfig, TPathId pathId) { LOG_D("Pool info successfully resolved"); - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new TEvPrivate::TEvResolvePoolResponse(Ydb::StatusIds::SUCCESS, poolConfig, pathId, DefaultPoolCreated, std::move(Event))); + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new TEvPrivate::TEvResolvePoolResponse(Ydb::StatusIds::SUCCESS, poolConfig, pathId, DefaultPoolCreated, std::move(Event))); PassAway(); } void Reply(Ydb::StatusIds::StatusCode status, NYql::TIssues issues) { LOG_W("Failed to resolve pool, " << status << ", issues: " << issues.ToOneLineString()); - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new TEvPrivate::TEvResolvePoolResponse(status, {}, {}, DefaultPoolCreated, std::move(Event), std::move(issues))); + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new TEvPrivate::TEvResolvePoolResponse(status, {}, {}, DefaultPoolCreated, std::move(Event), std::move(issues))); PassAway(); } @@ -642,4 +642,4 @@ IActor* CreateDatabaseFetcherActor(const TActorId& replyActorId, const TString& return new TDatabaseFetcherActor(replyActorId, database, userToken, checkAccess); } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/actors/ya.make b/ydb/services/workload_manager/actors/ya.make index 1cb4c527b5a..b90c9e64e4f 100644 --- a/ydb/core/kqp/workload_service/actors/ya.make +++ b/ydb/services/workload_manager/actors/ya.make @@ -7,8 +7,8 @@ SRCS( ) PEERDIR( - ydb/core/kqp/workload_service/common - ydb/core/kqp/workload_service/tables + ydb/services/workload_manager/common + ydb/services/workload_manager/tables ydb/core/tx/tx_proxy ) diff --git a/ydb/core/kqp/workload_service/common/cpu_quota_manager.cpp b/ydb/services/workload_manager/common/cpu_quota_manager.cpp index dd3a6618342..b8b4b9c8968 100644 --- a/ydb/core/kqp/workload_service/common/cpu_quota_manager.cpp +++ b/ydb/services/workload_manager/common/cpu_quota_manager.cpp @@ -3,7 +3,7 @@ #include <util/string/builder.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { //// TCpuQuotaManager::TCounters @@ -153,4 +153,4 @@ void TCpuQuotaManager::AdjustCpuQuota(double quota, TDuration duration, double c } } -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/cpu_quota_manager.h b/ydb/services/workload_manager/common/cpu_quota_manager.h index ee9e8c2a246..3fe2dee520d 100644 --- a/ydb/core/kqp/workload_service/common/cpu_quota_manager.h +++ b/ydb/services/workload_manager/common/cpu_quota_manager.h @@ -7,7 +7,7 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { class TCpuQuotaManager { struct TCounters { @@ -73,4 +73,4 @@ private: bool Ready = false; }; -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/events.cpp b/ydb/services/workload_manager/common/events.cpp index ca8d9c554ac..33213f1ef04 100644 --- a/ydb/core/kqp/workload_service/common/events.cpp +++ b/ydb/services/workload_manager/common/events.cpp @@ -1,10 +1,10 @@ #include "events.h" -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { ui64 TPoolStateDescription::AmountRequests() const { return DelayedRequests + RunningRequests; } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/events.h b/ydb/services/workload_manager/common/events.h index 911dccd36f0..d91b2fe39cc 100644 --- a/ydb/core/kqp/workload_service/common/events.h +++ b/ydb/services/workload_manager/common/events.h @@ -1,13 +1,13 @@ #pragma once -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/scheme/scheme_pathid.h> #include <ydb/core/protos/kqp.pb.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { struct TPoolStateDescription { ui64 DelayedRequests = 0; @@ -319,4 +319,4 @@ struct TEvPrivate { }; }; -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/helpers.cpp b/ydb/services/workload_manager/common/helpers.cpp index ec551ef9ebb..147eda4402b 100644 --- a/ydb/core/kqp/workload_service/common/helpers.cpp +++ b/ydb/services/workload_manager/common/helpers.cpp @@ -4,7 +4,7 @@ #include <ydb/core/base/path.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { TString CreateDatabaseId(const TString& database, bool serverless, TPathId pathId) { TString databasePath = CanonizePath(database); @@ -93,4 +93,4 @@ NResourcePool::TPoolSettings PoolSettingsFromConfig(const NKikimrConfig::TWorklo return poolSettings; } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/helpers.h b/ydb/services/workload_manager/common/helpers.h index 2ef0e27c320..88dc669872d 100644 --- a/ydb/core/kqp/workload_service/common/helpers.h +++ b/ydb/services/workload_manager/common/helpers.h @@ -2,6 +2,7 @@ #include <library/cpp/retry/retry_policy.h> +#include <ydb/core/base/path.h> #include <ydb/core/protos/workload_manager_config.pb.h> #include <ydb/core/resource_pools/resource_pool_settings.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> @@ -13,8 +14,15 @@ #include <ydb/public/api/protos/ydb_status_codes.pb.h> +#include <util/string/builder.h> + + +namespace NKikimr::NWorkloadManager { + +inline TString GetPoolKey(const TString& databaseId, const TString& poolId) { + return CanonizePath(TStringBuilder() << databaseId << "/" << poolId); +} -namespace NKikimr::NKqp::NWorkload { #define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_WORKLOAD_SERVICE, "[WorkloadService] " << LogPrefix() << stream) #define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_WORKLOAD_SERVICE, "[WorkloadService] " << LogPrefix() << stream) @@ -111,4 +119,4 @@ ui64 SaturationSub(ui64 x, ui64 y); NResourcePool::TPoolSettings PoolSettingsFromConfig(const NKikimrConfig::TWorkloadManagerConfig& workloadManagerConfig); -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/common/ya.make b/ydb/services/workload_manager/common/ya.make index a8eb736baaa..1a6a565f35b 100644 --- a/ydb/core/kqp/workload_service/common/ya.make +++ b/ydb/services/workload_manager/common/ya.make @@ -7,11 +7,10 @@ SRCS( ) PEERDIR( - ydb/core/kqp/common/events + ydb/core/base ydb/core/scheme - ydb/core/tx/scheme_cache ydb/library/actors/core diff --git a/ydb/core/kqp/common/events/workload_service.h b/ydb/services/workload_manager/events.h index 5bbdeb5707c..f04a474feea 100644 --- a/ydb/core/kqp/common/events/workload_service.h +++ b/ydb/services/workload_manager/events.h @@ -1,8 +1,8 @@ #pragma once -#include <ydb/core/kqp/common/simple/kqp_event_ids.h> - +#include <ydb/core/base/events.h> #include <ydb/core/resource_pools/resource_pool_settings.h> +#include "session_updater.h" #include <ydb/core/scheme/scheme_pathid.h> #include <ydb/library/aclib/aclib.h> @@ -14,11 +14,22 @@ #include <memory> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { + +struct TWorkloadManagerEvents { + enum EEvents { + EvPlaceRequestIntoPool = EventSpaceBegin(TKikimrEvents::ES_WORKLOAD_MANAGER), + EvContinueRequest, + EvCleanupRequest, + EvCleanupResponse, + EvUpdatePoolInfo, + EvSubscribeOnPoolChanges, + EvFetchDatabaseResponse, + }; +}; -class ISessionUpdater; -struct TEvSubscribeOnPoolChanges : public NActors::TEventLocal<TEvSubscribeOnPoolChanges, TKqpWorkloadServiceEvents::EvSubscribeOnPoolChanges> { +struct TEvSubscribeOnPoolChanges : public NActors::TEventLocal<TEvSubscribeOnPoolChanges, TWorkloadManagerEvents::EvSubscribeOnPoolChanges> { TEvSubscribeOnPoolChanges(const TString& databaseId, const TString& poolId) : DatabaseId(databaseId) , PoolId(poolId) @@ -28,7 +39,7 @@ struct TEvSubscribeOnPoolChanges : public NActors::TEventLocal<TEvSubscribeOnPoo const TString PoolId; }; -struct TEvPlaceRequestIntoPool : public NActors::TEventLocal<TEvPlaceRequestIntoPool, TKqpWorkloadServiceEvents::EvPlaceRequestIntoPool> { +struct TEvPlaceRequestIntoPool : public NActors::TEventLocal<TEvPlaceRequestIntoPool, TWorkloadManagerEvents::EvPlaceRequestIntoPool> { TEvPlaceRequestIntoPool(const TString& databaseId, const TString& sessionId, const TString& poolId, TIntrusiveConstPtr<NACLib::TUserToken> userToken, const TString& requestText = "", std::shared_ptr<ISessionUpdater> wmSessionUpdater = nullptr) : DatabaseId(databaseId) , SessionId(sessionId) @@ -46,7 +57,7 @@ struct TEvPlaceRequestIntoPool : public NActors::TEventLocal<TEvPlaceRequestInto std::shared_ptr<ISessionUpdater> WmSessionUpdater; }; -struct TEvContinueRequest : public NActors::TEventLocal<TEvContinueRequest, TKqpWorkloadServiceEvents::EvContinueRequest> { +struct TEvContinueRequest : public NActors::TEventLocal<TEvContinueRequest, TWorkloadManagerEvents::EvContinueRequest> { TEvContinueRequest(Ydb::StatusIds::StatusCode status, const TString& poolId, const NResourcePool::TPoolSettings& poolConfig, NYql::TIssues issues = {}) : Status(status) , PoolId(poolId) @@ -71,7 +82,7 @@ struct TEvContinueRequest : public NActors::TEventLocal<TEvContinueRequest, TKqp const NYql::TIssues Issues; }; -struct TEvCleanupRequest : public NActors::TEventLocal<TEvCleanupRequest, TKqpWorkloadServiceEvents::EvCleanupRequest> { +struct TEvCleanupRequest : public NActors::TEventLocal<TEvCleanupRequest, TWorkloadManagerEvents::EvCleanupRequest> { TEvCleanupRequest(const TString& databaseId, const TString& sessionId, const TString& poolId, TDuration duration, TDuration cpuConsumed) : DatabaseId(databaseId) , SessionId(sessionId) @@ -87,7 +98,7 @@ struct TEvCleanupRequest : public NActors::TEventLocal<TEvCleanupRequest, TKqpWo const TDuration CpuConsumed; }; -struct TEvCleanupResponse : public NActors::TEventLocal<TEvCleanupResponse, TKqpWorkloadServiceEvents::EvCleanupResponse> { +struct TEvCleanupResponse : public NActors::TEventLocal<TEvCleanupResponse, TWorkloadManagerEvents::EvCleanupResponse> { explicit TEvCleanupResponse(Ydb::StatusIds::StatusCode status, NYql::TIssues issues = {}) : Status(status) , Issues(std::move(issues)) @@ -97,7 +108,7 @@ struct TEvCleanupResponse : public NActors::TEventLocal<TEvCleanupResponse, TKqp const NYql::TIssues Issues; }; -struct TEvUpdatePoolInfo : public NActors::TEventLocal<TEvUpdatePoolInfo, TKqpWorkloadServiceEvents::EvUpdatePoolInfo> { +struct TEvUpdatePoolInfo : public NActors::TEventLocal<TEvUpdatePoolInfo, TWorkloadManagerEvents::EvUpdatePoolInfo> { TEvUpdatePoolInfo(const TString& databaseId, const TString& poolId, const std::optional<NResourcePool::TPoolSettings>& config, const std::optional<NACLib::TSecurityObject>& securityObject) : DatabaseId(databaseId) , PoolId(poolId) @@ -111,7 +122,7 @@ struct TEvUpdatePoolInfo : public NActors::TEventLocal<TEvUpdatePoolInfo, TKqpWo const std::optional<NACLib::TSecurityObject> SecurityObject; }; -struct TEvFetchDatabaseResponse : public NActors::TEventLocal<TEvFetchDatabaseResponse, TKqpWorkloadServiceEvents::EvFetchDatabaseResponse> { +struct TEvFetchDatabaseResponse : public NActors::TEventLocal<TEvFetchDatabaseResponse, TWorkloadManagerEvents::EvFetchDatabaseResponse> { TEvFetchDatabaseResponse(Ydb::StatusIds::StatusCode status, const TString& database, const TString& databaseId, bool serverless, TPathId pathId, NYql::TIssues issues) : Status(status) , Database(database) @@ -129,4 +140,4 @@ struct TEvFetchDatabaseResponse : public NActors::TEventLocal<TEvFetchDatabaseRe const NYql::TIssues Issues; }; -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_has_full_scan_matcher.cpp b/ydb/services/workload_manager/has_full_scan_matcher.cpp index 85caab8d008..915844085b2 100644 --- a/ydb/core/kqp/workload_service/kqp_has_full_scan_matcher.cpp +++ b/ydb/services/workload_manager/has_full_scan_matcher.cpp @@ -1,9 +1,9 @@ -#include "kqp_has_full_scan_matcher.h" +#include "has_full_scan_matcher.h" #include <ydb/core/base/path.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -78,4 +78,4 @@ bool MatchesFullScan(const std::optional<NResourcePool::TRegexPredicate>& predic return false; } -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_has_full_scan_matcher.h b/ydb/services/workload_manager/has_full_scan_matcher.h index 3ae7374b40d..e4976c550e3 100644 --- a/ydb/core/kqp/workload_service/kqp_has_full_scan_matcher.h +++ b/ydb/services/workload_manager/has_full_scan_matcher.h @@ -6,7 +6,7 @@ #include <optional> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { /// /// Returns true if the classifier should accept the query based on the HAS_FULL_SCAN filter. @@ -18,4 +18,4 @@ namespace NKikimr::NKqp::NWorkload { bool MatchesFullScan(const std::optional<NResourcePool::TRegexPredicate>& predicate, const NKqpProto::TKqpPhyQuery& phyQuery); -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_has_path_matcher.cpp b/ydb/services/workload_manager/has_path_matcher.cpp index fd674228c7c..f6f35735113 100644 --- a/ydb/core/kqp/workload_service/kqp_has_path_matcher.cpp +++ b/ydb/services/workload_manager/has_path_matcher.cpp @@ -1,4 +1,4 @@ -#include "kqp_has_path_matcher.h" +#include "has_path_matcher.h" #include <ydb/core/base/path.h> @@ -10,7 +10,7 @@ #include <util/generic/vector.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -95,4 +95,4 @@ bool MatchesPath(const std::optional<NResourcePool::TRegexPredicate>& predicate, return false; } -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_has_path_matcher.h b/ydb/services/workload_manager/has_path_matcher.h index c1a835eba0c..07215e5764d 100644 --- a/ydb/core/kqp/workload_service/kqp_has_path_matcher.h +++ b/ydb/services/workload_manager/has_path_matcher.h @@ -9,7 +9,7 @@ #include <optional> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { /// /// Returns true if the classifier should accept the query based on the HAS_PATH filter. @@ -26,4 +26,4 @@ bool MatchesPath(const std::optional<NResourcePool::TRegexPredicate>& predicate, const TVector<TString>& queryTables, const NKqpProto::TKqpPhyQuery& phyQuery); -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/has_stream_matcher.cpp b/ydb/services/workload_manager/has_stream_matcher.cpp new file mode 100644 index 00000000000..dde424bf792 --- /dev/null +++ b/ydb/services/workload_manager/has_stream_matcher.cpp @@ -0,0 +1,14 @@ +#include "has_stream_matcher.h" + + +namespace NKikimr::NWorkloadManager { + +bool MatchesStream(const std::optional<bool>& hasStream, + const NKqp::TUserRequestContext& userRequestContext) { + if (!hasStream) { + return true; + } + return *hasStream ? userRequestContext.IsStreamingQuery : !userRequestContext.IsStreamingQuery; +} + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/has_stream_matcher.h b/ydb/services/workload_manager/has_stream_matcher.h new file mode 100644 index 00000000000..65e115c19f0 --- /dev/null +++ b/ydb/services/workload_manager/has_stream_matcher.h @@ -0,0 +1,14 @@ +#pragma once + +#include <ydb/core/kqp/common/kqp_user_request_context.h> + +#include <optional> + + +namespace NKikimr::NWorkloadManager { + +/// Returns true if the classifier should accept the query based on the HAS_STREAM filter. +bool MatchesStream(const std::optional<bool>& hasStream, + const NKqp::TUserRequestContext& userRequestContext); + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool/behaviour.cpp b/ydb/services/workload_manager/metadata_subscription/behaviour.cpp index a1b172b8741..0854488a194 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool/behaviour.cpp +++ b/ydb/services/workload_manager/metadata_subscription/behaviour.cpp @@ -4,7 +4,7 @@ #include <ydb/services/metadata/abstract/initialization.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { TResourcePoolBehaviour::TFactory::TRegistrator<TResourcePoolBehaviour> TResourcePoolBehaviour::Registrator(TResourcePoolConfig::GetTypeId()); @@ -25,4 +25,4 @@ TString TResourcePoolBehaviour::GetTypeId() const { return TResourcePoolConfig::GetTypeId(); } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool/behaviour.h b/ydb/services/workload_manager/metadata_subscription/behaviour.h index 01587224e41..24f98a2a0e1 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool/behaviour.h +++ b/ydb/services/workload_manager/metadata_subscription/behaviour.h @@ -3,7 +3,7 @@ #include <ydb/services/metadata/abstract/kqp_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolConfig { public: @@ -24,4 +24,4 @@ public: virtual TString GetTypeId() const override; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool/manager.cpp b/ydb/services/workload_manager/metadata_subscription/manager.cpp index bbaa2f8f585..fbd524f446c 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool/manager.cpp +++ b/ydb/services/workload_manager/metadata_subscription/manager.cpp @@ -9,9 +9,10 @@ #include <ydb/core/protos/feature_flags.pb.h> #include <ydb/core/protos/schemeshard/operations.pb.h> #include <ydb/core/resource_pools/resource_pool_settings.h> +#include <ydb/core/kqp/gateway/utils/metadata_helpers.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { @@ -20,7 +21,7 @@ using namespace NResourcePool; using TYqlConclusionStatus = TResourcePoolManager::TYqlConclusionStatus; using TAsyncStatus = TResourcePoolManager::TAsyncStatus; -struct TFeatureFlagExtractor : public IFeatureFlagExtractor { +struct TFeatureFlagExtractor : public NKqp::IFeatureFlagExtractor { bool IsEnabled(const NKikimrConfig::TFeatureFlags& flags) const override { return flags.GetEnableResourcePools(); } @@ -225,13 +226,13 @@ TAsyncStatus TResourcePoolManager::ExecutePrepared(const NKqpProto::TKqpSchemeOp TAsyncStatus TResourcePoolManager::ExecuteSchemeRequest(const NKikimrSchemeOp::TModifyScheme& schemeTx, const TExternalModificationContext& context, ui32 nodeId, NKqpProto::TKqpSchemeOperation::OperationCase operationCase) const { TAsyncStatus validationFuture = NThreading::MakeFuture<TYqlConclusionStatus>(TYqlConclusionStatus::Success()); if (operationCase != NKqpProto::TKqpSchemeOperation::kDropResourcePool) { - validationFuture = ChainFeatures(validationFuture, [context, nodeId] { + validationFuture = NKqp::ChainFeatures(validationFuture, [context, nodeId] { return CheckFeatureFlag(nodeId, MakeIntrusive<TFeatureFlagExtractor>(), context); }); } - return ChainFeatures(validationFuture, [schemeTx, context] { - return SendSchemeRequest(schemeTx, context); + return NKqp::ChainFeatures(validationFuture, [schemeTx, context] { + return NKqp::SendSchemeRequest(schemeTx, context); }); } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool/manager.h b/ydb/services/workload_manager/metadata_subscription/manager.h index 0c6ac1bf60e..6fbe30b5038 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool/manager.h +++ b/ydb/services/workload_manager/metadata_subscription/manager.h @@ -3,7 +3,7 @@ #include <ydb/services/metadata/manager/abstract.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolManager : public NMetadata::NModifications::IOperationsManager { public: @@ -33,4 +33,4 @@ private: TAsyncStatus ExecuteSchemeRequest(const NKikimrSchemeOp::TModifyScheme& schemeTx, const TExternalModificationContext& context, ui32 nodeId, NKqpProto::TKqpSchemeOperation::OperationCase operationCase) const; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/behaviour.cpp index aad9f783100..7b3122053d7 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/behaviour.cpp @@ -3,7 +3,7 @@ #include "manager.h" -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { TResourcePoolClassifierBehaviour::TFactory::TRegistrator<TResourcePoolClassifierBehaviour> TResourcePoolClassifierBehaviour::Registrator(TResourcePoolClassifierConfig::GetTypeId()); @@ -28,4 +28,4 @@ NMetadata::IClassBehaviour::TPtr TResourcePoolClassifierBehaviour::GetInstance() return result; } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/behaviour.h index 42c8440f80e..580ec31b8a0 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/behaviour.h @@ -6,7 +6,7 @@ #include <ydb/services/metadata/abstract/kqp_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierBehaviour : public NMetadata::TClassBehaviour<TResourcePoolClassifierConfig> { static TFactory::TRegistrator<TResourcePoolClassifierBehaviour> Registrator; @@ -22,4 +22,4 @@ public: static IClassBehaviour::TPtr GetInstance(); }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/checker.cpp index ec839ca3fd9..2e5a932ce38 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/checker.cpp @@ -3,8 +3,8 @@ #include <ydb/core/base/path.h> #include <ydb/core/cms/console/configs_dispatcher.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/common/events.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/common/events.h> #include <ydb/core/protos/console_config.pb.h> #include <ydb/core/protos/feature_flags.pb.h> #include <ydb/core/resource_pools/resource_pool_classifier_settings.h> @@ -14,13 +14,12 @@ #include <unordered_set> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { using namespace NActors; using namespace NResourcePool; -using namespace NWorkload; struct TEvCheckerPrivate { @@ -249,7 +248,7 @@ public: TryFinish(); } - void Handle(NWorkload::TEvPrivate::TEvFetchPoolResponse::TPtr& ev) { + void Handle(TEvPrivate::TEvFetchPoolResponse::TPtr& ev) { if (ev->Get()->Status == Ydb::StatusIds::NOT_FOUND) { FailAndPassAway(TStringBuilder() << "Resource pool '" << ev->Get()->PoolId << "' not found"); return; @@ -273,7 +272,7 @@ public: hFunc(TEvents::TEvUndelivered, Handle); hFunc(NConsole::TEvConfigsDispatcher::TEvGetConfigResponse, Handle); hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); - hFunc(NWorkload::TEvPrivate::TEvFetchPoolResponse, Handle); + hFunc(TEvPrivate::TEvFetchPoolResponse, Handle); ) private: @@ -323,6 +322,29 @@ private: return; } + if (!featureFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("has_full_scan") || configMap.count("has_path") || + configMap.count("has_stream") || configMap.count("has_app_name")) { + FailAndPassAway("Advanced resource pool classifier properties are disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + + if (!featureFlags.GetEnableRejectActionInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("action")) { + FailAndPassAway("Resource pool classifier action property is disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + FeatureFlagChecked = true; ValidateExistence(); ValidatePools(); @@ -363,7 +385,7 @@ private: const TString& databaseId = externalContext.GetDatabaseId(); const auto& workloadManagerConfig = AppData()->WorkloadManagerConfig; for (const auto& poolId : poolIds) { - Register(NWorkload::CreatePoolFetcherActor(SelfId(), databaseId, poolId, userToken, workloadManagerConfig)); + Register(CreatePoolFetcherActor(SelfId(), databaseId, poolId, userToken, workloadManagerConfig)); } } @@ -406,4 +428,4 @@ IActor* CreateResourcePoolClassifierPreparationActor(std::vector<TResourcePoolCl return new TResourcePoolClassifierPreparationActor(std::move(patchedObjects), std::move(controller), context, alterContext); } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/checker.h index 3018087f755..db7bd0ee391 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/checker.h @@ -5,8 +5,8 @@ #include <ydb/services/metadata/manager/generic_manager.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { NActors::IActor* CreateResourcePoolClassifierPreparationActor(std::vector<TResourcePoolClassifierConfig>&& patchedObjects, NMetadata::NModifications::IAlterPreparationController<TResourcePoolClassifierConfig>::TPtr controller, const NMetadata::NModifications::IOperationsManager::TInternalModificationContext& context, const NMetadata::NModifications::TAlterOperationContext& alterContext); -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.cpp index e9f7f3d59bb..23d164e7b85 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.cpp @@ -1,10 +1,10 @@ #include "fetcher.h" -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { std::vector<NMetadata::IClassBehaviour::TPtr> TResourcePoolClassifierSnapshotsFetcher::DoGetManagers() const { return {TResourcePoolClassifierConfig::GetBehaviour()}; } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h index 29611f0cbeb..9e99bea784b 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h @@ -3,11 +3,11 @@ #include "snapshot.h" -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierSnapshotsFetcher : public NMetadata::NFetcher::TSnapshotsFetcher<TResourcePoolClassifierSnapshot> { protected: virtual std::vector<NMetadata::IClassBehaviour::TPtr> DoGetManagers() const override; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/initializer.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/initializer.cpp index 39230296e4b..2ba7c4336eb 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/initializer.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/initializer.cpp @@ -2,7 +2,7 @@ #include "object.h" -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { @@ -38,4 +38,4 @@ void TResourcePoolClassifierInitializer::DoPrepare(NMetadata::NInitializer::IIni controller->OnPreparationFinished(result); } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/initializer.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/initializer.h index c1743cfcbcf..493d15f649f 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/initializer.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/initializer.h @@ -3,11 +3,11 @@ #include <ydb/services/metadata/abstract/initialization.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierInitializer : public NMetadata::NInitializer::IInitializationBehaviour { protected: virtual void DoPrepare(NMetadata::NInitializer::IInitializerInput::TPtr controller) const override; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/manager.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp index 7b0d7c7c882..ee78bd7b199 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/manager.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp @@ -5,7 +5,7 @@ #include <ydb/core/resource_pools/resource_pool_classifier_settings.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { @@ -118,4 +118,4 @@ void TResourcePoolClassifierManager::DoPrepareObjectsBeforeModification(std::vec actorSystem->Register(CreateResourcePoolClassifierPreparationActor(std::move(patchedObjects), std::move(controller), context, alterContext)); } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/manager.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.h index 947069f8a1a..fba335bb305 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/manager.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.h @@ -5,7 +5,7 @@ #include <ydb/services/metadata/manager/generic_manager.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierManager : public NMetadata::NModifications::TGenericOperationsManager<TResourcePoolClassifierConfig> { protected: @@ -18,4 +18,4 @@ private: NMetadata::NModifications::TOperationParsingResult FillDropInfo(const NYql::TObjectSettingsImpl& settings, const TInternalModificationContext& context) const; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/object.cpp index 93112f35b3b..6efd775c162 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/object.cpp @@ -4,7 +4,7 @@ #include <library/cpp/json/json_reader.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { @@ -133,4 +133,4 @@ TString TResourcePoolClassifierConfig::GetTypeId() { return "RESOURCE_POOL_CLASSIFIER"; } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/object.h index 3c11d0b9efd..31422f20893 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/object.h @@ -6,7 +6,7 @@ #include <ydb/services/metadata/manager/object.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierConfig : public NMetadata::NModifications::TObject<TResourcePoolClassifierConfig> { using TBase = NMetadata::NModifications::TObject<TResourcePoolClassifierConfig>; @@ -53,4 +53,4 @@ public: static TString GetTypeId(); }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.cpp index 501cf3ec781..40b13d8533e 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.cpp @@ -1,7 +1,7 @@ #include "snapshot.h" -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { bool TResourcePoolClassifierSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawData) { Y_ABORT_UNLESS(rawData.result_sets().size() == 1); @@ -41,4 +41,4 @@ std::optional<TResourcePoolClassifierConfig> TResourcePoolClassifierSnapshot::Ge return configIt->second; } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.h index 8d6eb6406f2..556cc76053d 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.h @@ -5,7 +5,7 @@ #include <ydb/services/metadata/abstract/fetcher.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { class TResourcePoolClassifierSnapshot : public NMetadata::NFetcher::ISnapshot { using TBase = NMetadata::NFetcher::ISnapshot; @@ -61,4 +61,4 @@ private: const TConfigsByRankMap* Configs = nullptr; }; -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/ya.make b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/ya.make index 53592996430..056947f4114 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/ya.make +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/ya.make @@ -12,7 +12,7 @@ SRCS( PEERDIR( ydb/core/cms/console - ydb/core/kqp/workload_service/actors + ydb/services/workload_manager/actors ydb/core/protos ydb/core/resource_pools ydb/library/query_actor diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool/ya.make b/ydb/services/workload_manager/metadata_subscription/ya.make index d3333254ed3..d3333254ed3 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool/ya.make +++ b/ydb/services/workload_manager/metadata_subscription/ya.make diff --git a/ydb/core/kqp/workload_service/kqp_query_classifier.cpp b/ydb/services/workload_manager/query_classifier.cpp index 244cdab017c..d5951780686 100644 --- a/ydb/core/kqp/workload_service/kqp_query_classifier.cpp +++ b/ydb/services/workload_manager/query_classifier.cpp @@ -1,17 +1,13 @@ -#include "kqp_has_full_scan_matcher.h" -#include "kqp_has_path_matcher.h" -#include "kqp_query_classifier.h" -#include "kqp_workload_service.h" +#include "has_full_scan_matcher.h" +#include "has_path_matcher.h" +#include "has_stream_matcher.h" +#include "query_classifier.h" -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/object.h> - -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { inline constexpr char RESOLVER_IS_USER[] = "User request"; inline constexpr char DEFAULT_RESOLVER[] = "Default"; -namespace NWorkload { - class TQueryClassifier : public IQueryClassifier { public: TQueryClassifier(TResourcePoolMapPtr resourcePoolMap, @@ -69,7 +65,7 @@ public: } [[nodiscard]] - TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery) override { + TPostCompileClassifyResult PostCompileClassify(const NKqp::TPreparedQueryHolder& preparedQuery, const NKqp::TUserRequestContext& userRequestContext) override { Y_VALIDATE(ClassifierView, "Post compile classify without configuration"); Y_VALIDATE(PreClassifyResult.has_value() && std::holds_alternative<TPendingCompilation>(*PreClassifyResult), "Post compile classify requires TPendingCompilation from pre-classification"); @@ -83,7 +79,7 @@ public: continue; } - if (!MatchesDynamic(settings, preparedQuery)) { + if (!MatchesDynamic(settings, preparedQuery, userRequestContext)) { continue; } @@ -179,7 +175,7 @@ private: } bool NeedsPreparedQuery(const NResourcePool::TClassifierSettings& settings) const { - return settings.HasFullScan.has_value() || settings.HasPath.has_value(); + return settings.HasFullScan.has_value() || settings.HasPath.has_value() || settings.HasStream.has_value(); } /// @@ -188,9 +184,10 @@ private: /// - Involves SQL analysis, plan building, or computations. /// - Depends on actual query structure and execution characteristics. /// - bool MatchesDynamic(const NResourcePool::TClassifierSettings& settings, const TPreparedQueryHolder& preparedQuery) const { + bool MatchesDynamic(const NResourcePool::TClassifierSettings& settings, const NKqp::TPreparedQueryHolder& preparedQuery, const NKqp::TUserRequestContext& userRequestContext) const { return MatchesFullScan(settings.HasFullScan, preparedQuery.GetPhysicalQuery()) - && MatchesPath(settings.HasPath, preparedQuery.GetQueryTables(), preparedQuery.GetPhysicalQuery()); + && MatchesPath(settings.HasPath, preparedQuery.GetQueryTables(), preparedQuery.GetPhysicalQuery()) + && MatchesStream(settings.HasStream, userRequestContext); } const TResourcePoolEntry* FindPool(const TString& poolId) const { @@ -298,6 +295,4 @@ std::shared_ptr<IQueryClassifier> CreateQueryClassifier(TResourcePoolMapPtr reso return std::make_shared<TQueryClassifier>(std::move(resourcePoolMap), std::move(classifierView), databaseId, std::move(context)); } -} // namespace NWorkload - -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_query_classifier.h b/ydb/services/workload_manager/query_classifier.h index bdbab934f22..79d35d5f87e 100644 --- a/ydb/core/kqp/workload_service/kqp_query_classifier.h +++ b/ydb/services/workload_manager/query_classifier.h @@ -1,15 +1,18 @@ #pragma once #include <ydb/core/kqp/common/simple/helpers.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h> #include <ydb/core/kqp/query_data/kqp_prepared_query.h> -#include <ydb/core/kqp/workload_service/kqp_workload_service.h> #include <ydb/core/resource_pools/resource_pool_classifier_settings.h> +#include <ydb/services/workload_manager/common/helpers.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.h> #include <ydb/library/aclib/aclib.h> -#include <util/string/builder.h> namespace NKikimr::NKqp { +struct TUserRequestContext; +} // namespace NKikimr::NKqp + +namespace NKikimr::NWorkloadManager { struct TClassifyContext { const TString PoolId; @@ -25,12 +28,6 @@ struct TResourcePoolEntry { using TResourcePoolMap = std::unordered_map<TString, TResourcePoolEntry>; using TResourcePoolMapPtr = std::shared_ptr<const TResourcePoolMap>; -inline TString GetPoolKey(const TString& databaseId, const TString& poolId) { - return TStringBuilder() << databaseId << "/" << poolId; -} - -namespace NWorkload { - /// /// Manages per-query workload manager policies /// @@ -77,7 +74,7 @@ public: /// Refines classification once the query plan is available [[nodiscard]] - virtual TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery) = 0; + virtual TPostCompileClassifyResult PostCompileClassify(const NKqp::TPreparedQueryHolder& preparedQuery, const NKqp::TUserRequestContext& userRequestContext) = 0; /// Get the current classification state [[nodiscard]] @@ -89,6 +86,4 @@ std::shared_ptr<IQueryClassifier> CreateQueryClassifier(TResourcePoolMapPtr reso const TString& databaseId, TClassifyContext context); -} // namespace NWorkload - -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_workload_service.cpp b/ydb/services/workload_manager/service/service.cpp index 0af61f06b83..c83334309a7 100644 --- a/ydb/core/kqp/workload_service/kqp_workload_service.cpp +++ b/ydb/services/workload_manager/service/service.cpp @@ -1,16 +1,17 @@ -#include "kqp_workload_service.h" -#include "kqp_workload_service_impl.h" +#include "service.h" +#include "workload_service_impl.h" #include <ydb/core/base/appdata_fwd.h> +#include <ydb/core/base/counters.h> #include <ydb/core/base/feature_flags.h> #include <ydb/core/base/path.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/common/helpers.h> -#include <ydb/core/kqp/workload_service/tables/table_queries.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/common/helpers.h> +#include <ydb/services/workload_manager/tables/table_queries.h> #include <ydb/core/mind/tenant_node_enumeration.h> @@ -21,16 +22,14 @@ #include <ydb/library/actors/interconnect/interconnect.h> -namespace NKikimr::NKqp { - -namespace NWorkload { +namespace NKikimr::NWorkloadManager { namespace { using namespace NActors; -class TKqpWorkloadService : public TActorBootstrapped<TKqpWorkloadService> { +class TWorkloadService : public TActorBootstrapped<TWorkloadService> { struct TCounters { const NMonitoring::TDynamicCounterPtr Counters; @@ -62,12 +61,12 @@ class TKqpWorkloadService : public TActorBootstrapped<TKqpWorkloadService> { }; public: - explicit TKqpWorkloadService(NMonitoring::TDynamicCounterPtr counters) + explicit TWorkloadService(NMonitoring::TDynamicCounterPtr counters) : Counters(counters) {} void Bootstrap() { - Become(&TKqpWorkloadService::MainState); + Become(&TWorkloadService::MainState); // Subscribe for FeatureFlags and WorkloadManagerConfig Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({ @@ -693,10 +692,6 @@ private: return nullptr; } - static TString GetPoolKey(const TString& databaseId, const TString& poolId) { - return CanonizePath(TStringBuilder() << databaseId << "/" << poolId); - } - TString LogPrefix() const { return "[Service] "; } @@ -722,10 +717,18 @@ private: } // anonymous namespace -} // namespace NWorkload +IActor* CreateService(NMonitoring::TDynamicCounterPtr counters) { + return new NWorkloadManager::TWorkloadService(counters); +} -IActor* CreateKqpWorkloadService(NMonitoring::TDynamicCounterPtr counters) { - return new NWorkload::TKqpWorkloadService(counters); +NMonitoring::TDynamicCounterPtr GetWorkloadManagerCounters(NMonitoring::TDynamicCounterPtr rootCounters) { + return GetServiceCounters(rootCounters, "kqp")->GetSubgroup("subsystem", "workload_manager"); } -} // namespace NKikimr::NKqp +NActors::TActorId MakeServiceId(ui32 nodeId) { + const char name[12] = "kqp_workld"; + return NActors::TActorId(nodeId, TStringBuf(name, 12)); +} + + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/service/service.h b/ydb/services/workload_manager/service/service.h new file mode 100644 index 00000000000..fc8f2a307d4 --- /dev/null +++ b/ydb/services/workload_manager/service/service.h @@ -0,0 +1,16 @@ +#pragma once + +#include <ydb/core/resource_pools/resource_pool_settings.h> + +#include <ydb/library/actors/core/actor.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> + +namespace NKikimr::NWorkloadManager { + +NActors::TActorId MakeServiceId(ui32 nodeId); + +NMonitoring::TDynamicCounterPtr GetWorkloadManagerCounters(NMonitoring::TDynamicCounterPtr rootCounters); + +NActors::IActor* CreateService(NMonitoring::TDynamicCounterPtr counters); + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/kqp_workload_service_impl.h b/ydb/services/workload_manager/service/workload_service_impl.h index 72856267bde..51b9d2a2835 100644 --- a/ydb/core/kqp/workload_service/kqp_workload_service_impl.h +++ b/ydb/services/workload_manager/service/workload_service_impl.h @@ -4,13 +4,13 @@ #include <ydb/core/kqp/common/events/events.h> #include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/common/cpu_quota_manager.h> -#include <ydb/core/kqp/workload_service/common/events.h> -#include <ydb/core/kqp/workload_service/common/helpers.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/common/cpu_quota_manager.h> +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/services/workload_manager/common/helpers.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { constexpr TDuration IDLE_DURATION = TDuration::Seconds(60); @@ -81,7 +81,7 @@ struct TDatabaseState { } if (Serverless != ev->Get()->Serverless) { - TActivationContext::Send(MakeKqpProxyID(SelfId.NodeId()), std::make_unique<TEvKqp::TEvUpdateDatabaseInfo>(ev->Get()->Database, ev->Get()->DatabaseId, ev->Get()->Serverless)); + TActivationContext::Send(NKqp::MakeKqpProxyID(SelfId.NodeId()), std::make_unique<NKqp::TEvKqp::TEvUpdateDatabaseInfo>(ev->Get()->Database, ev->Get()->DatabaseId, ev->Get()->Serverless)); } LastUpdateTime = TInstant::Now(); @@ -249,4 +249,4 @@ private: std::unordered_map<TActorId, double> HandlersLimits; }; -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/service/ya.make b/ydb/services/workload_manager/service/ya.make new file mode 100644 index 00000000000..959821993e6 --- /dev/null +++ b/ydb/services/workload_manager/service/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +SRCS( + service.cpp +) + +PEERDIR( + ydb/core/base + ydb/core/cms/console + ydb/core/mind + ydb/core/resource_pools + ydb/library/aclib + ydb/library/actors/interconnect + ydb/services/workload_manager/actors + ydb/services/workload_manager/common + ydb/services/workload_manager/tables +) + +YQL_LAST_ABI_VERSION() + +END() + diff --git a/ydb/core/kqp/proxy_service/kqp_session_state.h b/ydb/services/workload_manager/session_updater.h index bdb165a7b69..5c62b94b15c 100644 --- a/ydb/core/kqp/proxy_service/kqp_session_state.h +++ b/ydb/services/workload_manager/session_updater.h @@ -3,7 +3,7 @@ #include <util/datetime/base.h> #include <util/generic/string.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { /// /// Interface for updating the execution state of a KQP session within the Workload Manager (WM). @@ -24,4 +24,4 @@ public: virtual void SetPoolId(TString poolId) = 0; }; -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/tables/table_queries.cpp b/ydb/services/workload_manager/tables/table_queries.cpp index 0c6f9780e3d..4365bf8fab7 100644 --- a/ydb/core/kqp/workload_service/tables/table_queries.cpp +++ b/ydb/services/workload_manager/tables/table_queries.cpp @@ -3,15 +3,16 @@ #include <ydb/core/base/path.h> #include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/common/events.h> -#include <ydb/core/kqp/workload_service/common/helpers.h> +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/services/workload_manager/common/helpers.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/library/query_actor/query_actor.h> #include <ydb/library/table_creator/table_creator.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -289,7 +290,7 @@ private: LOG_E("Failed with issues: " << Issues.ToOneLineString()); } - Send(MakeKqpWorkloadServiceId(SelfId().NodeId()), new TEvPrivate::TEvCleanupTablesFinished(Success, TablesExists, std::move(Issues))); + Send(NWorkloadManager::MakeServiceId(SelfId().NodeId()), new TEvPrivate::TEvCleanupTablesFinished(Success, TablesExists, std::move(Issues))); PassAway(); } @@ -862,4 +863,4 @@ IActor* CreateCleanupRequestsActor(const TActorId& replyActorId, const TString& return TCleanupRequestsQuery::MakeRetry(replyActorId, databaseId, poolId, sessionIds, counters); } -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/tables/table_queries.h b/ydb/services/workload_manager/tables/table_queries.h index 599b6ba08a9..dc2cbfbc258 100644 --- a/ydb/core/kqp/workload_service/tables/table_queries.h +++ b/ydb/services/workload_manager/tables/table_queries.h @@ -3,7 +3,7 @@ #include <ydb/library/actors/core/actor.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { // Creates all needed tables for workload service NActors::IActor* CreateTablesCreator(); @@ -19,4 +19,4 @@ NActors::IActor* CreateDelayRequestActor(const NActors::TActorId& replyActorId, NActors::IActor* CreateStartRequestActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const std::optional<TString>& sessionId, TDuration leaseDuration, NMonitoring::TDynamicCounterPtr counters); NActors::IActor* CreateCleanupRequestsActor(const NActors::TActorId& replyActorId, const TString& databaseId, const TString& poolId, const std::vector<TString>& sessionIds, NMonitoring::TDynamicCounterPtr counters); -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/tables/ya.make b/ydb/services/workload_manager/tables/ya.make index 1f82c244588..9f7caa93fab 100644 --- a/ydb/core/kqp/workload_service/tables/ya.make +++ b/ydb/services/workload_manager/tables/ya.make @@ -5,7 +5,7 @@ SRCS( ) PEERDIR( - ydb/core/kqp/workload_service/common + ydb/services/workload_manager/common ydb/library/query_actor diff --git a/ydb/core/kqp/workload_service/ut/kqp_action_reject_ut.cpp b/ydb/services/workload_manager/ut/action_reject_ut.cpp index 1a0b8ab264b..818d75b9ff9 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_action_reject_ut.cpp +++ b/ydb/services/workload_manager/ut/action_reject_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; namespace { @@ -225,4 +225,4 @@ Y_UNIT_TEST_SUITE(ActionRejectDdl) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h b/ydb/services/workload_manager/ut/common/query_classifier_ut_common.h index ec531244a68..12058b2ea72 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h +++ b/ydb/services/workload_manager/ut/common/query_classifier_ut_common.h @@ -1,8 +1,9 @@ #pragma once #include <ydb/core/base/path.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h> -#include <ydb/core/kqp/workload_service/kqp_query_classifier.h> +#include <ydb/core/kqp/common/kqp_user_request_context.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/snapshot.h> +#include <ydb/services/workload_manager/query_classifier.h> #include <ydb/core/kqp/query_data/kqp_prepared_query.h> #include <ydb/library/aclib/aclib.h> @@ -13,7 +14,7 @@ #include <vector> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { inline constexpr char TEST_DB[] = "/Root/testdb"; @@ -24,6 +25,7 @@ inline TResourcePoolClassifierConfig MakeClassifierConfig( std::optional<TString> hasAppName = std::nullopt, std::optional<TString> hasFullScan = std::nullopt, std::optional<TString> hasPath = std::nullopt, + std::optional<bool> hasStream = std::nullopt, std::optional<TString> action = std::nullopt) { NJson::TJsonValue json(NJson::JSON_MAP); @@ -40,6 +42,9 @@ inline TResourcePoolClassifierConfig MakeClassifierConfig( if (hasPath) { json["has_path"] = *hasPath; } + if (hasStream) { + json["has_stream"] = *hasStream ? "true" : "false"; + } if (action) { json["action"] = *action; } @@ -93,6 +98,7 @@ struct TClassifyTestCase { std::optional<TString> ClassifierHasAppName; std::optional<TString> ClassifierHasFullScan; std::optional<TString> ClassifierHasPath; + std::optional<bool> ClassifierHasStream; std::optional<TString> ClassifierAction; TString ContextAppName; @@ -109,20 +115,21 @@ struct TClassifyTestCase { std::optional<TString> HasAppName; std::optional<TString> HasFullScan; std::optional<TString> HasPath; + std::optional<bool> HasStream; std::optional<TString> Action; }; std::vector<TExtraClassifier> ExtraClassifiers; - std::shared_ptr<NWorkload::IQueryClassifier> BuildClassifier() const { + std::shared_ptr<NWorkloadManager::IQueryClassifier> BuildClassifier() const { std::vector<TResourcePoolClassifierConfig> configs; configs.push_back(MakeClassifierConfig( TEST_DB, "c_main", Rank, ResourcePool, - ClassifierMemberName, ClassifierHasAppName, ClassifierHasFullScan, ClassifierHasPath, ClassifierAction)); + ClassifierMemberName, ClassifierHasAppName, ClassifierHasFullScan, ClassifierHasPath, ClassifierHasStream, ClassifierAction)); for (const auto& extra : ExtraClassifiers) { configs.push_back(MakeClassifierConfig( TEST_DB, extra.Name, extra.Rank, extra.ResourcePool, - extra.MemberName, extra.HasAppName, extra.HasFullScan, extra.HasPath, extra.Action)); + extra.MemberName, extra.HasAppName, extra.HasFullScan, extra.HasPath, extra.HasStream, extra.Action)); } auto classifierSnap = MakeClassifierSnapshot(std::move(configs)); @@ -150,10 +157,10 @@ struct TClassifyTestCase { .UserToken = token, }; - return NWorkload::CreateQueryClassifier(poolSnap, TClassifierConfigsView(classifierSnap, TEST_DB), TEST_DB, std::move(ctx)); + return NWorkloadManager::CreateQueryClassifier(poolSnap, TClassifierConfigsView(classifierSnap, TEST_DB), TEST_DB, std::move(ctx)); } - NWorkload::IQueryClassifier::TPreCompileClassifyResult RunPreClassify() const { + NWorkloadManager::IQueryClassifier::TPreCompileClassifyResult RunPreClassify() const { auto classifier = BuildClassifier(); return classifier->PreCompileClassify(); } @@ -163,7 +170,7 @@ struct TClassifyTestCase { /// TKqpPhyQuery holding one table op on `queryTablePath` that either performs /// a full scan (`isFullScan=true`) or a bounded read. /// - NWorkload::IQueryClassifier::TPostCompileClassifyResult RunPostClassify( + NWorkloadManager::IQueryClassifier::TPostCompileClassifyResult RunPostClassify( const TString& queryTablePath, bool isFullScan) const { auto classifier = BuildClassifier(); @@ -182,8 +189,9 @@ struct TClassifyTestCase { op->MutableReadRange()->MutableKeyRange()->MutableFrom()->AddValues(); } - TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); - return classifier->PostCompileClassify(holder); + NKqp::TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); + NKqp::TUserRequestContext userRequestContext; + return classifier->PostCompileClassify(holder, userRequestContext); } /// @@ -192,7 +200,7 @@ struct TClassifyTestCase { /// Exercises HAS_PATH's (B) walk over `tx.GetTables()`. One shape is enough /// for wiring verification; the matcher UT covers the full walk surface. /// - NWorkload::IQueryClassifier::TPostCompileClassifyResult RunPostClassifyForPath( + NWorkloadManager::IQueryClassifier::TPostCompileClassifyResult RunPostClassifyForPath( const TString& queryTablePath) const { auto classifier = BuildClassifier(); @@ -203,16 +211,30 @@ struct TClassifyTestCase { auto* tx = phyQuery->AddTransactions(); tx->AddTables()->MutableId()->SetPath(queryTablePath); - TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); - return classifier->PostCompileClassify(holder); + NKqp::TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); + NKqp::TUserRequestContext userRequestContext; + return classifier->PostCompileClassify(holder, userRequestContext); + } + + NWorkloadManager::IQueryClassifier::TPostCompileClassifyResult RunPostClassifyForStream( + bool isStreamingQuery) const + { + auto classifier = BuildClassifier(); + (void)classifier->PreCompileClassify(); + + auto proto = std::make_unique<NKikimrKqp::TPreparedQuery>(); + NKqp::TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = isStreamingQuery; + return classifier->PostCompileClassify(holder, userRequestContext); } }; -inline TString GetPoolId(const NWorkload::IQueryClassifier::TPreCompileClassifyResult& result) { - UNIT_ASSERT_C(std::holds_alternative<NWorkload::IQueryClassifier::TResolvedPoolId>(result), +inline TString GetPoolId(const NWorkloadManager::IQueryClassifier::TPreCompileClassifyResult& result) { + UNIT_ASSERT_C(std::holds_alternative<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result), TStringBuilder() << "Expected TResolvedPoolId, got variant with index: " << result.index() ); - return std::get<NWorkload::IQueryClassifier::TResolvedPoolId>(result).PoolId; + return std::get<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result).PoolId; } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp b/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp index b0cb6c030d7..c814295509e 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp +++ b/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp @@ -1,14 +1,14 @@ -#include "kqp_workload_service_ut_common.h" +#include "workload_service_ut_common.h" #include <ydb/core/base/backtrace.h> #include <ydb/core/base/counters.h> #include <ydb/core/kqp/common/events/events.h> -#include <ydb/core/kqp/common/simple/services.h> +#include <ydb/services/workload_manager/service/service.h> #include <ydb/core/kqp/executer_actor/kqp_executer.h> -#include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/fetcher.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/tables/table_queries.h> +#include <ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/fetcher.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/tables/table_queries.h> #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/node_whiteboard/node_whiteboard.h> @@ -16,7 +16,7 @@ #include <ydb/library/yql/utils/actor_log/log.h> #include <ydb/services/metadata/service.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { namespace { @@ -63,7 +63,7 @@ class TQueryRunnerActor : public TActorBootstrapped<TQueryRunnerActor> { using TBase = TActorBootstrapped<TQueryRunnerActor>; public: - TQueryRunnerActor(std::unique_ptr<TEvKqp::TEvQueryRequest> request, TPromise<TQueryRunnerResult> promise, const TQueryRunnerSettings& settings, ui32 targetNodeId) + TQueryRunnerActor(std::unique_ptr<NKqp::TEvKqp::TEvQueryRequest> request, TPromise<TQueryRunnerResult> promise, const TQueryRunnerSettings& settings, ui32 targetNodeId) : Request_(std::move(request)) , Promise_(promise) , Settings_(settings) @@ -77,19 +77,19 @@ public: void Bootstrap() { ActorIdToProto(SelfId(), Request_->Record.MutableRequestActorId()); - Send(MakeKqpProxyID(TargetNodeId_), std::move(Request_)); + Send(NKqp::MakeKqpProxyID(TargetNodeId_), std::move(Request_)); Become(&TQueryRunnerActor::StateFunc); } - void Handle(TEvKqpExecuter::TEvStreamData::TPtr& ev) { + void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev) { UNIT_ASSERT_C(Settings_.ExecutionExpected_ || ExecutionContinued, "Unexpected stream data, execution is not expected and was not continued"); if (!ExecutionStartReported) { ExecutionStartReported = true; SendNotification<TEvQueryRunner::TEvExecutionStarted>(); } - auto response = std::make_unique<TEvKqpExecuter::TEvStreamDataAck>(ev->Get()->Record.GetSeqNo(), ev->Get()->Record.GetChannelId()); + auto response = std::make_unique<NKqp::TEvKqpExecuter::TEvStreamDataAck>(ev->Get()->Record.GetSeqNo(), ev->Get()->Record.GetChannelId()); response->Record.SetFreeSpace(std::numeric_limits<i64>::max()); auto resultSetIndex = ev->Get()->Record.GetQueryResultIndex(); @@ -109,7 +109,7 @@ public: } } - void Handle(TEvKqp::TEvQueryResponse::TPtr& ev) { + void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev) { SendNotification<TEvQueryRunner::TEvExecutionFinished>(); Result_.Response = ev->Get()->Record; @@ -129,8 +129,8 @@ public: } STRICT_STFUNC(StateFunc, - hFunc(TEvKqpExecuter::TEvStreamData, Handle); - hFunc(TEvKqp::TEvQueryResponse, Handle); + hFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle); + hFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); hFunc(TEvQueryRunner::TEvContinueExecution, Handle); ) @@ -144,7 +144,7 @@ private: } private: - std::unique_ptr<TEvKqp::TEvQueryRequest> Request_; + std::unique_ptr<NKqp::TEvKqp::TEvQueryRequest> Request_; TPromise<TQueryRunnerResult> Promise_; const TQueryRunnerSettings Settings_; const ui32 TargetNodeId_; @@ -302,6 +302,8 @@ private: featureFlags.SetEnableResourcePoolsCounters(true); featureFlags.SetEnableStreamingQueries(Settings_.EnableStreamingQueries_); featureFlags.SetEnableResourcePoolsScheduler(Settings_.EnableResourcePoolsScheduler_); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(Settings_.EnableHasPredicatesInResourcePoolClassifiers_); + featureFlags.SetEnableRejectActionInResourcePoolClassifiers(Settings_.EnableRejectActionInResourcePoolClassifiers_); auto& queryServiceConfig = *appConfig.MutableQueryServiceConfig(); queryServiceConfig.SetAllExternalDataSourcesAreAvailable(true); @@ -660,14 +662,14 @@ public: UNIT_ASSERT_C(response, "Timed out waiting for resource pool classifier snapshot refresh"); runtime->Send( - MakeKqpProxyID(nodeId), + NKqp::MakeKqpProxyID(nodeId), edgeActor, new NMetadata::NProvider::TEvRefreshSubscriberData(response->Get()->GetSnapshot()), nodeIndex); } void StopWorkloadService(ui64 nodeIndex = 0) const override { - GetRuntime()->Send(MakeKqpWorkloadServiceId(GetRuntime()->GetNodeId(nodeIndex)), GetRuntime()->AllocateEdgeActor(), new TEvents::TEvPoison()); + GetRuntime()->Send(MakeServiceId(GetRuntime()->GetNodeId(nodeIndex)), GetRuntime()->AllocateEdgeActor(), new TEvents::TEvPoison()); Sleep(TDuration::Seconds(1)); } @@ -757,10 +759,10 @@ private: } } - std::unique_ptr<TEvKqp::TEvQueryRequest> GetQueryRequest(const TString& query, const TQueryRunnerSettings& settings) const { + std::unique_ptr<NKqp::TEvKqp::TEvQueryRequest> GetQueryRequest(const TString& query, const TQueryRunnerSettings& settings) const { UNIT_ASSERT_C(settings.PoolId_, "Query pool id is not specified"); - auto event = std::make_unique<TEvKqp::TEvQueryRequest>(); + auto event = std::make_unique<NKqp::TEvKqp::TEvQueryRequest>(); event->Record.SetUserToken(NACLib::TUserToken("", settings.UserSID_, settings.GroupSIDs_).SerializeAsString()); auto request = event->Record.MutableRequest(); @@ -777,8 +779,7 @@ private: } NMonitoring::TDynamicCounterPtr GetWorkloadManagerCounters(ui32 nodeIndex) const { - return GetServiceCounters(GetRuntime()->GetAppData(nodeIndex).Counters, "kqp") - ->GetSubgroup("subsystem", "workload_manager"); + return NWorkloadManager::GetWorkloadManagerCounters(GetRuntime()->GetAppData(nodeIndex).Counters); } static void CheckCommonCounters(NMonitoring::TDynamicCounterPtr subgroup, const TString& description) { @@ -935,10 +936,5 @@ void WaitForClassifierSuccess(TIntrusivePtr<IYdbSetup> ydb, const TQueryRunnerSe WaitForClassifierSuccess(ydb, TSampleQueries::TSelect42::Query, settings); } -//// TSampleQueriess -void TSampleQueries::CompareYson(const TString& expected, const TString& actual) { - NKqp::CompareYson(expected, actual); -} - -} // NKikimr::NKqp::NWorkload +} // NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h b/ydb/services/workload_manager/ut/common/workload_service_ut_common.h index 47b84ed1ef7..a5a1a5571b4 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h +++ b/ydb/services/workload_manager/ut/common/workload_service_ut_common.h @@ -1,7 +1,7 @@ #include <library/cpp/testing/unittest/registar.h> -#include <ydb/core/kqp/workload_service/common/events.h> - +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/testlib/actors/test_runtime.h> #include <ydb/core/testlib/test_client.h> @@ -16,7 +16,7 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h> -namespace NKikimr::NKqp::NWorkload { +namespace NKikimr::NWorkloadManager { inline constexpr TDuration FUTURE_WAIT_TIMEOUT = TDuration::Seconds(60); @@ -81,6 +81,8 @@ struct TYdbSetupSettings { FLUENT_SETTING_DEFAULT(bool, EnableMetadataObjectsOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableExternalDataSourcesOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableStreamingQueries, true); + FLUENT_SETTING_DEFAULT(bool, EnableHasPredicatesInResourcePoolClassifiers, true); + FLUENT_SETTING_DEFAULT(bool, EnableRejectActionInResourcePoolClassifiers, true); FLUENT_SETTING(NKikimrConfig::TWorkloadManagerConfig, WorkloadManagerConfig); FLUENT_SETTING_DEFAULT(i32, DedicatedDiskQuota, -1); @@ -193,12 +195,10 @@ struct TSampleQueries { static void CheckResult(const TResult& result) { CheckSuccess(result); UNIT_ASSERT_VALUES_EQUAL_C(result.GetResultSets().size(), 1, "Unexpected result set size"); - CompareYson("[[42]]", NYdb::FormatResultSetYson(result.GetResultSet(0))); + NKqp::CompareYson("[[42]]", NYdb::FormatResultSetYson(result.GetResultSet(0))); } }; -private: - static void CompareYson(const TString& expected, const TString& actual); }; -} // namespace NKikimr::NKqp::NWorkload +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/ut/common/ya.make b/ydb/services/workload_manager/ut/common/ya.make new file mode 100644 index 00000000000..d0c96fcd48c --- /dev/null +++ b/ydb/services/workload_manager/ut/common/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + query_classifier_ut_common.h + workload_service_ut_common.cpp +) + +PEERDIR( + ydb/core/kqp/ut/common + ydb/services/metadata + ydb/services/workload_manager/metadata_subscription/resource_pool_classifier +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_app_name_ut.cpp b/ydb/services/workload_manager/ut/has_app_name_ut.cpp index 58298f71107..6773b3ca83a 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_app_name_ut.cpp +++ b/ydb/services/workload_manager/ut/has_app_name_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; namespace { @@ -129,4 +129,4 @@ Y_UNIT_TEST_SUITE(HasAppNameDdl) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_full_scan_matcher_ut.cpp b/ydb/services/workload_manager/ut/has_full_scan_matcher_ut.cpp index 28c135ff3e3..77e9269f7c6 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_full_scan_matcher_ut.cpp +++ b/ydb/services/workload_manager/ut/has_full_scan_matcher_ut.cpp @@ -1,10 +1,10 @@ -#include <ydb/core/kqp/workload_service/kqp_has_full_scan_matcher.h> +#include <ydb/services/workload_manager/has_full_scan_matcher.h> #include <ydb/core/resource_pools/regex_predicate.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { using NResourcePool::TRegexPredicate; using NKqpProto::TKqpPhyQuery; @@ -36,7 +36,7 @@ TKqpReadRangesSource* AddSource(TKqpPhyQuery& phy, const TString& path) { const auto Matches = [](auto builder, auto configure) { TKqpPhyQuery phy; configure(builder(phy, TString("/Root/t"))); - return NWorkload::MatchesFullScan(TRegexPredicate::FromGlob("/Root/t"), phy); + return MatchesFullScan(TRegexPredicate::FromGlob("/Root/t"), phy); }; const auto AssertHasFullScan = [](auto c) { UNIT_ASSERT( Matches(AddOp, c)); }; @@ -51,21 +51,21 @@ Y_UNIT_TEST_SUITE(TFullScanMatcherPredicate) { Y_UNIT_TEST(EmptyPredicateAcceptsAny) { TKqpPhyQuery phy; - UNIT_ASSERT(NWorkload::MatchesFullScan(std::nullopt, phy)); + UNIT_ASSERT(MatchesFullScan(std::nullopt, phy)); } Y_UNIT_TEST(NonMatchingPathIsIgnored) { auto pred = TRegexPredicate::FromGlob("/Root/critical"); TKqpPhyQuery phy; AddOp(phy, "/Root/other")->MutableReadRange()->MutableKeyRange(); - UNIT_ASSERT(!NWorkload::MatchesFullScan(pred, phy)); + UNIT_ASSERT(!MatchesFullScan(pred, phy)); } Y_UNIT_TEST(GlobPatternMatches) { auto pred = TRegexPredicate::FromGlob("/Root/db/orders_archive*"); TKqpPhyQuery phy; AddOp(phy, "/Root/db/orders_archive_2024")->MutableReadRange()->MutableKeyRange(); - UNIT_ASSERT(NWorkload::MatchesFullScan(pred, phy)); + UNIT_ASSERT(MatchesFullScan(pred, phy)); } Y_UNIT_TEST(MixedOpsReturnTrueOnAnyMatch) { @@ -73,7 +73,7 @@ Y_UNIT_TEST_SUITE(TFullScanMatcherPredicate) { TKqpPhyQuery phy; AddOp(phy, "/Root/other")->MutableReadRange()->MutableKeyRange(); AddOp(phy, "/Root/t")->MutableReadRange()->MutableKeyRange(); - UNIT_ASSERT(NWorkload::MatchesFullScan(pred, phy)); + UNIT_ASSERT(MatchesFullScan(pred, phy)); } } @@ -161,4 +161,4 @@ Y_UNIT_TEST_SUITE(TFullScanMatcherLimit) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_full_scan_ut.cpp b/ydb/services/workload_manager/ut/has_full_scan_ut.cpp index ee69f7f5fa4..05c47bfda46 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_full_scan_ut.cpp +++ b/ydb/services/workload_manager/ut/has_full_scan_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; @@ -66,7 +66,7 @@ namespace { // Creates `orders(Id, Status)` with a global secondary index on Status, // grants the given user full access, and inserts a couple of rows so the planner // doesn't take an empty-table shortcut. -void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID) { +void SetupOrdersWithIndex(TIntrusivePtr<IYdbSetup> ydb, const TString& userSID) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; CREATE TABLE orders ( @@ -77,7 +77,7 @@ void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString ); )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO orders (Id, Status) VALUES (1u, "a"), @@ -91,7 +91,7 @@ void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString // classifier targeting it, and inserts a couple of rows. The classifier's target pool // has CONCURRENT_QUERY_LIMIT=0 so any match rejects with PRECONDITION_FAILED. void SetupOlapEventsAndClassifier( - TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID, const TString& poolId) + TIntrusivePtr<IYdbSetup> ydb, const TString& userSID, const TString& poolId) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; @@ -111,7 +111,7 @@ void SetupOlapEventsAndClassifier( ); )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO olap_events (Id, Payload) VALUES (1u, "a"), (2u, "b"); )", insertSettings); @@ -128,7 +128,7 @@ Y_UNIT_TEST_SUITE(HasFullScanSecondaryIndexOltpDdl) { // main table is not accessed. A classifier targeting only `/Root/orders` (not the impl path) // must NOT fire — the impl full scan happens on a path outside the classifier's regex. Y_UNIT_TEST(TestClassifierOnMainTableIgnoresIndexScan) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; @@ -145,8 +145,8 @@ Y_UNIT_TEST_SUITE(HasFullScanSecondaryIndexOltpDdl) { )"); const TString query = "SELECT * FROM orders VIEW by_status;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierSuccess(ydb, query, settings); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierSuccess(ydb, query, settings); } // Same query, but the classifier targets the index impl table path directly. @@ -157,7 +157,7 @@ Y_UNIT_TEST_SUITE(HasFullScanSecondaryIndexOltpDdl) { // `SELECT * FROM orders VIEW by_status` (no predicate) would trigger the planner // warning "Given predicate is not suitable for used index" and fall back to a main-table scan. Y_UNIT_TEST(TestClassifierOnIndexImplTableDetectsFullScan) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; @@ -174,8 +174,8 @@ Y_UNIT_TEST_SUITE(HasFullScanSecondaryIndexOltpDdl) { )"); const TString query = "SELECT * FROM orders VIEW by_status;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } } @@ -184,7 +184,7 @@ Y_UNIT_TEST_SUITE(HasFullScanIgnoresLimitDdl) { // LIMIT does not bypass full-scan detection. `SELECT * FROM orders LIMIT 1` still has an // unbounded ReadRange on /Root/orders — the classifier fires and rejects into the target pool. Y_UNIT_TEST(TestLimitDoesNotBypassClassifier) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; @@ -201,22 +201,22 @@ Y_UNIT_TEST_SUITE(HasFullScanIgnoresLimitDdl) { )"); const TString query = "SELECT * FROM orders LIMIT 1;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } // Same LIMIT semantic applied to OLAP: `ReadOlapRange` with an unbounded param name and // HasItemsLimit() set still counts as a full scan. Y_UNIT_TEST(TestLimitDoesNotBypassClassifierOnOlapTable) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOlapEventsAndClassifier(ydb, userSID, poolId); const TString query = "SELECT * FROM olap_events LIMIT 1;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } // Zero-selectivity filter behind a LIMIT: at runtime the executor scans the whole table @@ -225,15 +225,15 @@ Y_UNIT_TEST_SUITE(HasFullScanIgnoresLimitDdl) { // exactly the runtime full-scan the LIMIT rule used to miss. OLAP is used because there is // no secondary index to route the filter through, so the plan shape is unambiguous. Y_UNIT_TEST(TestFilterWithLimitDoesNotBypassClassifierOnOlapTable) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOlapEventsAndClassifier(ydb, userSID, poolId); const TString query = "SELECT * FROM olap_events WHERE Payload = 'nonexistent' LIMIT 1;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } } @@ -243,30 +243,30 @@ Y_UNIT_TEST_SUITE(HasFullScanOlapDdl) { // row-store's `ReadRange`/`ReadRangesSource`. This test proves the matcher's OLAP branch // is invoked end-to-end by the real planner, complementing the ReadOlapRange matcher UT. Y_UNIT_TEST(TestClassifierDetectsFullScanOnOlapTable) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOlapEventsAndClassifier(ydb, userSID, poolId); const TString query = "SELECT * FROM olap_events;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } // PK point lookup on OLAP compiles to a parametrized ReadOlapRange with a non-empty // KeyRanges.ParamName. Matcher must not flag it — proves the OLAP negative branch e2e. Y_UNIT_TEST(TestClassifierIgnoresPkPointLookupOnOlapTable) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOlapEventsAndClassifier(ydb, userSID, poolId); const TString query = "SELECT * FROM olap_events WHERE Id = 1u;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierSuccess(ydb, query, settings); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierSuccess(ydb, query, settings); } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_path_ddl_ut.cpp b/ydb/services/workload_manager/ut/has_path_ddl_ut.cpp index 114721f8651..260701a254d 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_path_ddl_ut.cpp +++ b/ydb/services/workload_manager/ut/has_path_ddl_ut.cpp @@ -1,20 +1,20 @@ #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; namespace { // Creates a simple row-store `t(Id, Payload)` and grants the given user full access. -void SetupRowStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID) { +void SetupRowStoreTable(TIntrusivePtr<IYdbSetup> ydb, const TString& userSID) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; CREATE TABLE t ( @@ -24,7 +24,7 @@ void SetupRowStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& ); )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO t (Id, Payload) VALUES (1u, "a"), (2u, "b"); )", insertSettings); @@ -33,7 +33,7 @@ void SetupRowStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& } // Creates a column-store `olap_t(Id, Payload)` and grants full access. -void SetupColumnStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID) { +void SetupColumnStoreTable(TIntrusivePtr<IYdbSetup> ydb, const TString& userSID) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; CREATE TABLE olap_t ( @@ -45,7 +45,7 @@ void SetupColumnStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TStrin ); )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO olap_t (Id, Payload) VALUES (1u, "a"), (2u, "b"); )", insertSettings); @@ -54,7 +54,7 @@ void SetupColumnStoreTable(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TStrin } // Creates `orders(Id, Status)` with a global secondary index on Status. -void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID) { +void SetupOrdersWithIndex(TIntrusivePtr<IYdbSetup> ydb, const TString& userSID) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; CREATE TABLE orders ( @@ -65,7 +65,7 @@ void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString ); )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO orders (Id, Status) VALUES (1u, "a"), (2u, "b"); )", insertSettings); @@ -76,7 +76,7 @@ void SetupOrdersWithIndex(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString // Creates `t(...)` and a view `v` over it, grants full access. // The view body must fully qualify the underlying table — the invoker's // session context doesn't resolve unqualified `t` when the view is expanded. -void SetupTableAndView(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& userSID) { +void SetupTableAndView(TIntrusivePtr<IYdbSetup> ydb, const TString& userSID) { ydb->ExecuteSchemeQuery(TStringBuilder() << R"( GRANT ALL ON `/)" << ydb->GetSettings().DomainName_ << R"(` TO `)" << userSID << R"(`; CREATE TABLE t ( @@ -87,7 +87,7 @@ void SetupTableAndView(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& u CREATE VIEW v WITH (security_invoker = true) AS SELECT * FROM `/Root/t`; )"); - auto insertSettings = NWorkload::TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); + auto insertSettings = TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID); auto insertResult = ydb->ExecuteQuery(R"( UPSERT INTO t (Id, Payload) VALUES (1u, "a"), (2u, "b"); )", insertSettings); @@ -96,7 +96,7 @@ void SetupTableAndView(TIntrusivePtr<NWorkload::IYdbSetup> ydb, const TString& u } // EDS/ET/Topic/CDC DDL ITs live in `ydb/core/kqp/ut/federated_query/datastreams/` -// (see `kqp_has_path_ut.cpp` there). Those kinds need TStreamingTestFixture's +// (see `has_path_ut.cpp` there). Those kinds need TStreamingTestFixture's // mock connector + mock PQ gateway + http gateway wiring, which is not present // in workload_service_ut's setup. This file covers the kinds whose fixture is // cheap: regular tables, sysview, secondary index, view (underlying-table gate). @@ -120,7 +120,7 @@ TString RejectClassifierDdl(const TString& poolId, const TString& classifierName Y_UNIT_TEST_SUITE(HasPathTableOltp) { Y_UNIT_TEST(TestClassifierMatchesTablePath) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupRowStoreTable(ydb, userSID); @@ -128,12 +128,12 @@ Y_UNIT_TEST_SUITE(HasPathTableOltp) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "row_table_classifier", "/Root/t")); const TString query = "SELECT * FROM t;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } Y_UNIT_TEST(TestClassifierIgnoresOtherTable) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupRowStoreTable(ydb, userSID); @@ -141,8 +141,8 @@ Y_UNIT_TEST_SUITE(HasPathTableOltp) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "row_table_classifier", "/Root/other_table")); const TString query = "SELECT * FROM t;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierSuccess(ydb, query, settings); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierSuccess(ydb, query, settings); } } @@ -152,7 +152,7 @@ Y_UNIT_TEST_SUITE(HasPathTableOltp) { Y_UNIT_TEST_SUITE(HasPathColumnTableOlap) { Y_UNIT_TEST(TestClassifierMatchesOlapTablePath) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupColumnStoreTable(ydb, userSID); @@ -160,14 +160,14 @@ Y_UNIT_TEST_SUITE(HasPathColumnTableOlap) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "olap_table_classifier", "/Root/olap_t")); const TString query = "SELECT * FROM olap_t;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } Y_UNIT_TEST(TestClassifierMatchesOlapPointLookup) { // Unlike HAS_FULL_SCAN, HAS_PATH doesn't care about read shape — a PK lookup // still touches the table and must fire. - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupColumnStoreTable(ydb, userSID); @@ -175,8 +175,8 @@ Y_UNIT_TEST_SUITE(HasPathColumnTableOlap) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "olap_table_classifier", "/Root/olap_t")); const TString query = "SELECT * FROM olap_t WHERE Id = 1u;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } } @@ -185,7 +185,7 @@ Y_UNIT_TEST_SUITE(HasPathColumnTableOlap) { Y_UNIT_TEST_SUITE(HasPathSecondaryIndex) { Y_UNIT_TEST(TestClassifierMatchesIndexImplPath) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOrdersWithIndex(ydb, userSID); @@ -195,14 +195,14 @@ Y_UNIT_TEST_SUITE(HasPathSecondaryIndex) { // Covering scan on the index — planner emits a full read of indexImplTable. const TString query = "SELECT Status FROM orders VIEW by_status;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } Y_UNIT_TEST(TestClassifierMatchesIndexOnWrite) { // UPSERT into an indexed table fans out to write both the main table and // the impl table. HAS_PATH targeting the impl path still fires. - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupOrdersWithIndex(ydb, userSID); @@ -211,8 +211,8 @@ Y_UNIT_TEST_SUITE(HasPathSecondaryIndex) { poolId, "index_write_classifier", "/Root/orders/by_status/indexImplTable")); const TString query = "UPSERT INTO orders (Id, Status) VALUES (3u, \"c\");"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } } @@ -221,7 +221,7 @@ Y_UNIT_TEST_SUITE(HasPathSecondaryIndex) { Y_UNIT_TEST_SUITE(HasPathSysView) { Y_UNIT_TEST(TestClassifierMatchesSysViewPath) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupRowStoreTable(ydb, userSID); @@ -230,12 +230,12 @@ Y_UNIT_TEST_SUITE(HasPathSysView) { poolId, "sysview_classifier", "/Root/.sys/partition_stats")); const TString query = "SELECT * FROM `.sys/partition_stats`;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } Y_UNIT_TEST(TestClassifierIgnoresRegularTableWhenSysViewTargeted) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupRowStoreTable(ydb, userSID); @@ -244,8 +244,8 @@ Y_UNIT_TEST_SUITE(HasPathSysView) { poolId, "sysview_classifier", "/Root/.sys/*")); const TString query = "SELECT * FROM t;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierSuccess(ydb, query, settings); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierSuccess(ydb, query, settings); } } @@ -260,7 +260,7 @@ Y_UNIT_TEST_SUITE(HasPathView) { // A regex on the view path `/Root/v` never fires. /* Y_UNIT_TEST(TestClassifierMatchesViewPath) { - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupTableAndView(ydb, userSID); @@ -268,15 +268,15 @@ Y_UNIT_TEST_SUITE(HasPathView) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "view_classifier", "/Root/v")); const TString query = "SELECT * FROM v;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } */ Y_UNIT_TEST(TestClassifierMatchesUnderlyingTableUnderView) { // Views are inlined — the underlying table's path also appears in QueryTables. // A regex targeting the underlying table matches queries going through the view. - auto ydb = NWorkload::TYdbSetupSettings().Create(); + auto ydb = TYdbSetupSettings().Create(); const TString& poolId = "reject_pool"; const TString& userSID = "test@user"; SetupTableAndView(ydb, userSID); @@ -284,9 +284,9 @@ Y_UNIT_TEST_SUITE(HasPathView) { ydb->ExecuteSchemeQuery(RejectClassifierDdl(poolId, "underlying_classifier", "/Root/t")); const TString query = "SELECT * FROM v;"; - auto settings = NWorkload::TQueryRunnerSettings().PoolId("").UserSID(userSID); - NWorkload::WaitForClassifierFail(ydb, query, settings, poolId); + auto settings = TQueryRunnerSettings().PoolId("").UserSID(userSID); + WaitForClassifierFail(ydb, query, settings, poolId); } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_path_matcher_ut.cpp b/ydb/services/workload_manager/ut/has_path_matcher_ut.cpp index 55cd3c86ef8..1cc1980b870 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_path_matcher_ut.cpp +++ b/ydb/services/workload_manager/ut/has_path_matcher_ut.cpp @@ -1,4 +1,4 @@ -#include <ydb/core/kqp/workload_service/kqp_has_path_matcher.h> +#include <ydb/services/workload_manager/has_path_matcher.h> #include <ydb/core/resource_pools/regex_predicate.h> #include <ydb/library/yql/providers/pq/proto/dq_io.pb.h> @@ -8,7 +8,7 @@ #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { using NResourcePool::TRegexPredicate; using NKqpProto::TKqpPhyQuery; @@ -41,7 +41,7 @@ void AddPqTopicSink(TKqpPhyQuery& phy, const TString& topicPath) { } // Fixed-pattern helpers: each suite pins the predicate so tests read as pure -// build-and-check one-liners (mirrors kqp_has_full_scan_matcher_ut.cpp style). +// build-and-check one-liners (mirrors has_full_scan_matcher_ut.cpp style). // Runs the matcher against a phyQuery holding one tx-level table entry. // Predicate + path both vary so predicate-behaviour tests can exercise glob @@ -49,27 +49,27 @@ void AddPqTopicSink(TKqpPhyQuery& phy, const TString& topicPath) { const auto MatchesTxTable = [](const TString& pattern, const TString& path) { TKqpPhyQuery phy; AddTxTable(phy, path); - return NWorkload::MatchesPath(TRegexPredicate::FromGlob(pattern), {}, phy); + return MatchesPath(TRegexPredicate::FromGlob(pattern), {}, phy); }; // Runs the matcher with pattern `/Root/t*` against the given queryTables list. const auto MatchesQueryTables = [](const TVector<TString>& queryTables) { TKqpPhyQuery phy; - return NWorkload::MatchesPath(TRegexPredicate::FromGlob("/Root/t*"), queryTables, phy); + return MatchesPath(TRegexPredicate::FromGlob("/Root/t*"), queryTables, phy); }; // Runs the matcher with pattern `/Root/db/target` against a phyQuery built by `build`. const auto MatchesTx = [](auto build) { TKqpPhyQuery phy; build(phy); - return NWorkload::MatchesPath(TRegexPredicate::FromGlob("/Root/db/target"), {}, phy); + return MatchesPath(TRegexPredicate::FromGlob("/Root/db/target"), {}, phy); }; // Runs the matcher with pattern `/Root/db/topic` against a phyQuery built by `build`. const auto MatchesTopics = [](auto build) { TKqpPhyQuery phy; build(phy); - return NWorkload::MatchesPath(TRegexPredicate::FromGlob("/Root/db/topic"), {}, phy); + return MatchesPath(TRegexPredicate::FromGlob("/Root/db/topic"), {}, phy); }; } // anonymous namespace @@ -80,7 +80,7 @@ Y_UNIT_TEST_SUITE(TPathMatcherPredicate) { Y_UNIT_TEST(EmptyPredicateAcceptsAny) { TKqpPhyQuery phy; AddTxTable(phy, "/Root/anything"); - UNIT_ASSERT(NWorkload::MatchesPath(std::nullopt, {}, phy)); + UNIT_ASSERT(MatchesPath(std::nullopt, {}, phy)); } Y_UNIT_TEST(NonMatchingPathIsNotAccepted) { @@ -166,4 +166,4 @@ Y_UNIT_TEST_SUITE(TPathMatcherTopics) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_path_ut.cpp b/ydb/services/workload_manager/ut/has_path_ut.cpp index 44f22c21661..760313b4c7f 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_has_path_ut.cpp +++ b/ydb/services/workload_manager/ut/has_path_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; namespace { @@ -61,4 +61,4 @@ Y_UNIT_TEST_SUITE(TQueryClassifierHasPath) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/ut/has_stream_matcher_ut.cpp b/ydb/services/workload_manager/ut/has_stream_matcher_ut.cpp new file mode 100644 index 00000000000..89ee1ab6dac --- /dev/null +++ b/ydb/services/workload_manager/ut/has_stream_matcher_ut.cpp @@ -0,0 +1,52 @@ +#include <ydb/services/workload_manager/has_stream_matcher.h> + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr::NWorkloadManager { + + +Y_UNIT_TEST_SUITE(TStreamMatcherPredicate) { + + + Y_UNIT_TEST(NulloptAcceptsAny) { + // No filter — every query passes regardless of streaming ops. + { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(MatchesStream(std::nullopt, userRequestContext)); + } + { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + UNIT_ASSERT(MatchesStream(std::nullopt, userRequestContext)); + } + } + + Y_UNIT_TEST(TrueMatchesStreamingQuery) { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + + UNIT_ASSERT(MatchesStream(true, userRequestContext)); + } + + Y_UNIT_TEST(TrueDoesNotMatchNonStreamingQuery) { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(!MatchesStream(true, userRequestContext)); + } + + Y_UNIT_TEST(FalseMatchesNonStreamingQuery) { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(MatchesStream(false, userRequestContext)); + } + + Y_UNIT_TEST(FalseDoesNotMatchStreamingQuery) { + NKqp::TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + UNIT_ASSERT(!MatchesStream(false, userRequestContext)); + } +} + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/ut/has_stream_ut.cpp b/ydb/services/workload_manager/ut/has_stream_ut.cpp new file mode 100644 index 00000000000..f12fdae8bd8 --- /dev/null +++ b/ydb/services/workload_manager/ut/has_stream_ut.cpp @@ -0,0 +1,61 @@ +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr::NWorkloadManager { + +using namespace NWorkloadManager; + + +namespace { + +TString GetPostPoolId(const IQueryClassifier::TPostCompileClassifyResult& result) { + UNIT_ASSERT_C(std::holds_alternative<IQueryClassifier::TResolvedPoolId>(result), + TStringBuilder() << "Expected TResolvedPoolId, got variant index " << result.index()); + return std::get<IQueryClassifier::TResolvedPoolId>(result).PoolId; +} + +} // anonymous namespace + + +Y_UNIT_TEST_SUITE(TQueryClassifierHasStream) { + + Y_UNIT_TEST(ShouldTriggerPendingCompilation) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPreClassify(); + UNIT_ASSERT_C(std::holds_alternative<IQueryClassifier::TPendingCompilation>(result), + TStringBuilder() << "Expected TPendingCompilation, got variant index " << result.index()); + } + + Y_UNIT_TEST(TrueMatchesStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPostClassifyForStream(true); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "pool_target"); + } + + Y_UNIT_TEST(TrueDoesNotMatchNonStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPostClassifyForStream(false); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "default"); + } + + Y_UNIT_TEST(FalseMatchesNonStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = false; + auto result = tc.RunPostClassifyForStream(false); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "pool_target"); + } + + Y_UNIT_TEST(FalseDoesNotMatchStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = false; + auto result = tc.RunPostClassifyForStream(true); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "default"); + } +} + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_member_name_ut.cpp b/ydb/services/workload_manager/ut/member_name_ut.cpp index 995713933cf..f51b41afa10 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_member_name_ut.cpp +++ b/ydb/services/workload_manager/ut/member_name_ut.cpp @@ -1,6 +1,6 @@ -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { Y_UNIT_TEST_SUITE(TQueryClassifierMemberName) { @@ -30,7 +30,7 @@ Y_UNIT_TEST_SUITE(TQueryClassifierMemberName) { }; auto view = TClassifierConfigsView(classifierSnap, TEST_DB); - auto classifier = NWorkload::CreateQueryClassifier(poolSnap, view, TEST_DB, ctx); + auto classifier = NWorkloadManager::CreateQueryClassifier(poolSnap, view, TEST_DB, ctx); auto result = classifier->PreCompileClassify(); UNIT_ASSERT_VALUES_EQUAL(GetPoolId(result), "pool_target"); } @@ -59,10 +59,10 @@ Y_UNIT_TEST_SUITE(TQueryClassifierMemberName) { }; auto view = TClassifierConfigsView(classifierSnap, TEST_DB); - auto classifier = NWorkload::CreateQueryClassifier(poolSnap, view, TEST_DB, ctx); + auto classifier = NWorkloadManager::CreateQueryClassifier(poolSnap, view, TEST_DB, ctx); auto result = classifier->PreCompileClassify(); UNIT_ASSERT_VALUES_EQUAL(GetPoolId(result), "pool_target"); } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_match_ut.cpp b/ydb/services/workload_manager/ut/query_classifier_match_ut.cpp index 7f4954171ba..378c589636b 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_match_ut.cpp +++ b/ydb/services/workload_manager/ut/query_classifier_match_ut.cpp @@ -1,6 +1,6 @@ -#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> +#include <ydb/services/workload_manager/ut/common/query_classifier_ut_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { @@ -23,27 +23,27 @@ void CheckDecision(std::function<void(NResourcePool::TPoolSettings&)> setup, EEx }); TClassifyContext ctx{.PoolId = "", .AppName = "", .UserToken = nullptr}; - auto classifier = NWorkload::CreateQueryClassifier( + auto classifier = NWorkloadManager::CreateQueryClassifier( poolSnap, TClassifierConfigsView(classifierSnap, TEST_DB), TEST_DB, std::move(ctx)); auto result = classifier->PreCompileClassify(); switch (expected) { case EExpected::Bypass: - UNIT_ASSERT_C(std::holds_alternative<NWorkload::IQueryClassifier::TBypass>(result), + UNIT_ASSERT_C(std::holds_alternative<NWorkloadManager::IQueryClassifier::TBypass>(result), TStringBuilder() << "Expected TBypass, got variant index " << result.index()); break; case EExpected::ResolvedAdmission: { - UNIT_ASSERT_C(std::holds_alternative<NWorkload::IQueryClassifier::TResolvedPoolId>(result), + UNIT_ASSERT_C(std::holds_alternative<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result), TStringBuilder() << "Expected TResolvedPoolId, got variant index " << result.index()); - UNIT_ASSERT_C(!std::get<NWorkload::IQueryClassifier::TResolvedPoolId>(result).SkipAdmission, + UNIT_ASSERT_C(!std::get<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result).SkipAdmission, "Expected SkipAdmission=false"); break; } case EExpected::ResolvedSkipAdmission: { - UNIT_ASSERT_C(std::holds_alternative<NWorkload::IQueryClassifier::TResolvedPoolId>(result), + UNIT_ASSERT_C(std::holds_alternative<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result), TStringBuilder() << "Expected TResolvedPoolId, got variant index " << result.index()); - UNIT_ASSERT_C(std::get<NWorkload::IQueryClassifier::TResolvedPoolId>(result).SkipAdmission, + UNIT_ASSERT_C(std::get<NWorkloadManager::IQueryClassifier::TResolvedPoolId>(result).SkipAdmission, "Expected SkipAdmission=true"); break; } @@ -134,4 +134,4 @@ Y_UNIT_TEST_SUITE(TQueryClassifierDecision) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp b/ydb/services/workload_manager/ut/query_classifier_ut.cpp index e7716bb7b73..4e1afd27e27 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp +++ b/ydb/services/workload_manager/ut/query_classifier_ut.cpp @@ -1,16 +1,16 @@ #include <fmt/format.h> #include <library/cpp/testing/unittest/registar.h> -#include <ydb/core/kqp/common/events/workload_service.h> +#include <ydb/services/workload_manager/events.h> #include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/kqp_query_classifier.h> +#include <ydb/services/workload_manager/query_classifier.h> #include <ydb/core/testlib/test_client.h> #include <ydb/core/kqp/common/events/events.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { -class TestQueryClassifier : public NWorkload::IQueryClassifier { +class TestQueryClassifier : public NWorkloadManager::IQueryClassifier { public: TestQueryClassifier(TPreCompileClassifyResult preClassifyResult, TPostCompileClassifyResult postClassifyResult) : PostCompileCalled(false) @@ -32,7 +32,7 @@ public: return EState::PreCompileDone; } - TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder&) override { + TPostCompileClassifyResult PostCompileClassify(const NKqp::TPreparedQueryHolder&, const NKqp::TUserRequestContext&) override { PostCompileCalled = true; return PostClassifyResult; } @@ -44,7 +44,7 @@ private: }; template<typename TPreResult, typename TPostResult> -TEvKqp::TEvQueryResponse::TPtr RunQueryWith(TPreResult preResult, TPostResult postResult) { +NKqp::TEvKqp::TEvQueryResponse::TPtr RunQueryWith(TPreResult preResult, TPostResult postResult) { TPortManager tp; auto mbusport = tp.GetPort(2134); auto settings = Tests::TServerSettings(mbusport); @@ -65,7 +65,7 @@ TEvKqp::TEvQueryResponse::TPtr RunQueryWith(TPreResult preResult, TPostResult po auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { if (ev->GetTypeRewrite() == NKqp::TEvKqp::TEvQueryRequest::EventType) { // Replace the classifier after proxy set it - auto* request = ev->Get<TEvKqp::TEvQueryRequest>(); + auto* request = ev->Get<NKqp::TEvKqp::TEvQueryRequest>(); request->SetWmQueryClassifier(std::make_shared<TestQueryClassifier>(preResult, postResult)); } @@ -80,20 +80,20 @@ TEvKqp::TEvQueryResponse::TPtr RunQueryWith(TPreResult preResult, TPostResult po ev->Record.MutableRequest()->SetQuery("SELECT 1;"); runtime.Send(new IEventHandle(proxyId, sender, ev.Release())); - return runtime.GrabEdgeEventRethrow<TEvKqp::TEvQueryResponse>(sender); + return runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(sender); } template<typename TPreResult> -TEvKqp::TEvQueryResponse::TPtr RunQueryWithPreClassify(TPreResult preResult) { - return RunQueryWith(preResult, NWorkload::IQueryClassifier::TBypass{}); +NKqp::TEvKqp::TEvQueryResponse::TPtr RunQueryWithPreClassify(TPreResult preResult) { + return RunQueryWith(preResult, NWorkloadManager::IQueryClassifier::TBypass{}); } template<typename TPostResult> -TEvKqp::TEvQueryResponse::TPtr RunQueryWithPostClassify(TPostResult postResult) { - return RunQueryWith(NWorkload::IQueryClassifier::TPendingCompilation{}, postResult); +NKqp::TEvKqp::TEvQueryResponse::TPtr RunQueryWithPostClassify(TPostResult postResult) { + return RunQueryWith(NWorkloadManager::IQueryClassifier::TPendingCompilation{}, postResult); } -TString GetErrorMessageFromResponse(TEvKqp::TEvQueryResponse::TPtr r) { +TString GetErrorMessageFromResponse(NKqp::TEvKqp::TEvQueryResponse::TPtr r) { NYql::TIssues issues; NYql::IssuesFromMessage( r->Get()->Record.GetResponse().GetQueryIssues(), issues); UNIT_ASSERT(issues.Size()); @@ -104,14 +104,14 @@ TString GetErrorMessageFromResponse(TEvKqp::TEvQueryResponse::TPtr r) { Y_UNIT_TEST_SUITE(KqpQueryPreClassifier) { Y_UNIT_TEST(ShouldBypassOnPreClassify) { - auto reply = RunQueryWithPreClassify(NWorkload::IQueryClassifier::TBypass()); + auto reply = RunQueryWithPreClassify(NWorkloadManager::IQueryClassifier::TBypass()); auto status = reply->Get()->Record.GetYdbStatus(); UNIT_ASSERT_EQUAL(status, Ydb::StatusIds::SUCCESS); } Y_UNIT_TEST(ShouldResolveDefaultOnPreClassify) { - auto resolve = NWorkload::IQueryClassifier::TResolvedPoolId{ + auto resolve = NWorkloadManager::IQueryClassifier::TResolvedPoolId{ .PoolId = NResourcePool::DEFAULT_POOL_ID }; auto reply = RunQueryWithPreClassify(resolve); @@ -119,7 +119,7 @@ Y_UNIT_TEST_SUITE(KqpQueryPreClassifier) { } Y_UNIT_TEST(ShouldRejectOnPreClassify) { - auto reject = NWorkload::IQueryClassifier::TReject{ + auto reject = NWorkloadManager::IQueryClassifier::TReject{ .Code = Ydb::StatusIds::ABORTED, .Message = "Reject by ShouldRejectOnPreClassify" }; @@ -132,14 +132,14 @@ Y_UNIT_TEST_SUITE(KqpQueryPreClassifier) { Y_UNIT_TEST_SUITE(KqpQueryPostClassifier) { Y_UNIT_TEST(ShouldBypassOnPostClassify) { - auto reply = RunQueryWithPostClassify(NWorkload::IQueryClassifier::TBypass()); + auto reply = RunQueryWithPostClassify(NWorkloadManager::IQueryClassifier::TBypass()); auto status = reply->Get()->Record.GetYdbStatus(); UNIT_ASSERT_EQUAL(status, Ydb::StatusIds::SUCCESS); } Y_UNIT_TEST(ShouldResolveDefaultOnPostClassify) { - auto resolve = NWorkload::IQueryClassifier::TResolvedPoolId{ + auto resolve = NWorkloadManager::IQueryClassifier::TResolvedPoolId{ .PoolId = NResourcePool::DEFAULT_POOL_ID }; auto reply = RunQueryWithPostClassify(resolve); @@ -147,7 +147,7 @@ Y_UNIT_TEST_SUITE(KqpQueryPostClassifier) { } Y_UNIT_TEST(ShouldRejectOnPostClassify) { - auto reject = NWorkload::IQueryClassifier::TReject{ + auto reject = NWorkloadManager::IQueryClassifier::TReject{ .Code = Ydb::StatusIds::ABORTED, .Message = "Rejected by ShouldRejectOnPostClassify" }; @@ -158,4 +158,4 @@ Y_UNIT_TEST_SUITE(KqpQueryPostClassifier) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp b/ydb/services/workload_manager/ut/stream_query_classification_ut.cpp index e13717696b9..f7d4dda2c07 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp +++ b/ydb/services/workload_manager/ut/stream_query_classification_ut.cpp @@ -1,23 +1,25 @@ -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <library/cpp/testing/unittest/registar.h> #include <chrono> #include <thread> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; namespace { -TIntrusivePtr<NWorkload::IYdbSetup> MakeStreamingYdb() { - return NWorkload::TYdbSetupSettings().Create([](auto) {}); +TIntrusivePtr<IYdbSetup> MakeStreamingYdb() { + return TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(true) + .Create([](auto) {}); } -void CreateTopic(TIntrusivePtr<NWorkload::IYdbSetup> ydb, TString name) { +void CreateTopic(TIntrusivePtr<IYdbSetup> ydb, TString name) { const auto& result = ydb->ExecuteQuery(TStringBuilder() << "CREATE TOPIC " << name); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToOneLineString()); } @@ -68,6 +70,46 @@ Y_UNIT_TEST_SUITE(StreamingQueryClassification) { } } + Y_UNIT_TEST(TestClassifierMatchesStreamingQuery) { + auto ydb = MakeStreamingYdb(); + + const TString& poolId = "streaming_pool"; + CreateTopic(ydb, "input_topic"); + CreateTopic(ydb, "output_topic"); + + { + const auto& result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL )" << poolId << R"( WITH ( + CONCURRENT_QUERY_LIMIT = 10, + QUEUE_SIZE = 100, + TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 10, + QUERY_CPU_LIMIT_PERCENT_PER_NODE = 1 + ); + CREATE RESOURCE POOL CLASSIFIER streaming_classifier WITH ( + RESOURCE_POOL=")" << poolId << R"(", + HAS_STREAM = "true" + ); + )"); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToOneLineString()); + } + ydb->WaitForClassifierPropagation(); + + { + const auto& result = ydb->ExecuteQuery(R"( + CREATE STREAMING QUERY MyStreamingQuery + AS DO BEGIN + INSERT INTO output_topic SELECT * FROM input_topic; + END DO + )"); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToOneLineString()); + ydb->WaitPoolState({.DelayedRequests = 0, .RunningRequests = 1}, poolId); + + } + { + std::this_thread::sleep_for(5s); + ydb->WaitPoolState({.DelayedRequests = 0, .RunningRequests = 1}, poolId); + } + } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_actors_ut.cpp b/ydb/services/workload_manager/ut/workload_service_actors_ut.cpp index acd8278a23c..71a2e363c5f 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_actors_ut.cpp +++ b/ydb/services/workload_manager/ut/workload_service_actors_ut.cpp @@ -1,15 +1,15 @@ #include <ydb/core/base/appdata_fwd.h> -#include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/services/workload_manager/actors/actors.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { -using namespace NWorkload; +using namespace NWorkloadManager; TEvPrivate::TEvFetchPoolResponse::TPtr FetchPool(TIntrusivePtr<IYdbSetup> ydb, const TString& poolId = "", const TString& userSID = "user@" BUILTIN_SYSTEM_DOMAIN) { @@ -171,7 +171,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceSubscriptions) { auto& runtime = *ydb->GetRuntime(); const auto& edgeActor = runtime.AllocateEdgeActor(); - runtime.Send(MakeKqpWorkloadServiceId(runtime.GetNodeId()), edgeActor, new TEvSubscribeOnPoolChanges(settings.DomainName_, settings.PoolId_)); + runtime.Send(MakeServiceId(runtime.GetNodeId()), edgeActor, new TEvSubscribeOnPoolChanges(settings.DomainName_, settings.PoolId_)); const auto& response = runtime.GrabEdgeEvent<TEvUpdatePoolInfo>(edgeActor, FUTURE_WAIT_TIMEOUT); UNIT_ASSERT_C(response, "Subscription update not found"); @@ -253,4 +253,4 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceSubscriptions) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_query_sessions_ut.cpp b/ydb/services/workload_manager/ut/workload_service_query_sessions_ut.cpp index e89e2d64e63..53adbee7fe0 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_query_sessions_ut.cpp +++ b/ydb/services/workload_manager/ut/workload_service_query_sessions_ut.cpp @@ -1,16 +1,16 @@ #include <fmt/format.h> -#include <ydb/core/kqp/common/events/workload_service.h> -#include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/proxy_service/kqp_session_state.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/events.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/services/workload_manager/session_updater.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; using namespace NActors; @@ -62,13 +62,13 @@ public: STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { - hFunc(NWorkload::TEvContinueRequest, HandleContinueRequest); + hFunc(NWorkloadManager::TEvContinueRequest, HandleContinueRequest); default: Send(ev->Forward(SessionActorId)); } } - void HandleContinueRequest(NWorkload::TEvContinueRequest::TPtr& ev) { + void HandleContinueRequest(NWorkloadManager::TEvContinueRequest::TPtr& ev) { senderId = ev->Sender; Send(new IEventHandle(EdgeActorId, SelfId(), ev->Release().Release())); Become(&TSessionProxyActor::StateWait); @@ -76,13 +76,13 @@ public: STFUNC(StateWait) { switch (ev->GetTypeRewrite()) { - hFunc(NWorkload::TEvContinueRequest, HandleRelease); + hFunc(NWorkloadManager::TEvContinueRequest, HandleRelease); default: Send(ev->Forward(SessionActorId)); } } - void HandleRelease(NWorkload::TEvContinueRequest::TPtr& ev) { + void HandleRelease(NWorkloadManager::TEvContinueRequest::TPtr& ev) { Send(new IEventHandle(SessionActorId, senderId, ev->Release().Release())); } @@ -136,17 +136,17 @@ public: STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { - hFunc(NWorkload::TEvPlaceRequestIntoPool, HandlePlaceRequest); + hFunc(NWorkloadManager::TEvPlaceRequestIntoPool, HandlePlaceRequest); default: Send(ev->Forward(WorkloadServiceId)); } } - void HandlePlaceRequest(NWorkload::TEvPlaceRequestIntoPool::TPtr& ev) { + void HandlePlaceRequest(NWorkloadManager::TEvPlaceRequestIntoPool::TPtr& ev) { auto* msg = ev->Get(); auto wrapper = std::make_shared<TWmSessionUpdaterWrapper>(FinalState, msg->WmSessionUpdater); - auto* proxyMsg = new NWorkload::TEvPlaceRequestIntoPool( + auto* proxyMsg = new NWorkloadManager::TEvPlaceRequestIntoPool( msg->DatabaseId, msg->SessionId, msg->PoolId, @@ -256,7 +256,7 @@ public: .Create(); auto& runtime = *Ydb->GetRuntime(); - auto workloadServiceId = MakeKqpWorkloadServiceId(runtime.GetNodeId(0)); + auto workloadServiceId = MakeServiceId(runtime.GetNodeId(0)); auto realWorkloadServiceId = runtime.GetLocalServiceId(workloadServiceId); auto proxyActor = new TKqpWorkloadProxyActor(State, realWorkloadServiceId, Rules); @@ -309,7 +309,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { // Stop a request before a real execution to prevent read races from a .sys/query_sessions // The request is now 'parked' in the interceptor actor - auto ev = runtime->GrabEdgeEvent<NWorkload::TEvContinueRequest>(edge); + auto ev = runtime->GrabEdgeEvent<NWorkloadManager::TEvContinueRequest>(edge); TQuerySessionReader reader(f.GetYdb()); reader.FetchAll(query); @@ -374,7 +374,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { // Stop a first request before a real execution to fill limits auto hangingRequest = f.GetYdb()->ExecuteQueryAsync(qHanging, myPool); - auto evHanging = runtime.GrabEdgeEvent<NWorkload::TEvContinueRequest>(edgeHanging); + auto evHanging = runtime.GrabEdgeEvent<NWorkloadManager::TEvContinueRequest>(edgeHanging); // Run a second request which has to be placed in a Delayed Queue auto delayedRequest = f.GetYdb()->ExecuteQueryAsync(qDelayed, myPool); @@ -392,7 +392,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { // Continue the first request and wait for the second request to be about to execute runtime.Send(new IEventHandle(evHanging->Sender, edgeHanging, evHanging->Release().Release())); - auto evDelayed = runtime.GrabEdgeEvent<NWorkload::TEvContinueRequest>(edgeDelayed); + auto evDelayed = runtime.GrabEdgeEvent<NWorkloadManager::TEvContinueRequest>(edgeDelayed); TQuerySessionReader reader2(f.GetYdb()); reader2.FetchAll(qDelayed); @@ -432,7 +432,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { const TString qFirst = "SELECT 11;"; TActorId edge1 = f.SetupInterceptor(qFirst); auto future1 = session.ExecuteDataQuery(qFirst, TTxControl::BeginTx().CommitTx()); - auto ev1 = runtime.GrabEdgeEvent<NWorkload::TEvContinueRequest>(edge1); + auto ev1 = runtime.GrabEdgeEvent<NWorkloadManager::TEvContinueRequest>(edge1); TQuerySessionReader reader(f.GetYdb()); reader.FetchAll(qFirst); @@ -450,7 +450,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { const TString qSecond = "SELECT 12;"; TActorId edge2 = f.SetupInterceptor(qSecond); auto future2 = session.ExecuteDataQuery(qSecond, TTxControl::BeginTx().CommitTx()); - auto ev2 = runtime.GrabEdgeEvent<NWorkload::TEvContinueRequest>(edge2); + auto ev2 = runtime.GrabEdgeEvent<NWorkloadManager::TEvContinueRequest>(edge2); TQuerySessionReader reader2(f.GetYdb()); reader2.FetchAll(qSecond); @@ -467,4 +467,4 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceQuerySessions) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp b/ydb/services/workload_manager/ut/workload_service_tables_ut.cpp index ec7fc061ad6..1f33ded14ce 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp +++ b/ydb/services/workload_manager/ut/workload_service_tables_ut.cpp @@ -2,17 +2,17 @@ #include <ydb/core/base/path.h> #include <ydb/core/kqp/common/simple/services.h> -#include <ydb/core/kqp/workload_service/kqp_workload_service.h> -#include <ydb/core/kqp/workload_service/common/events.h> -#include <ydb/core/kqp/workload_service/tables/table_queries.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/service/service.h> +#include <ydb/services/workload_manager/common/events.h> +#include <ydb/services/workload_manager/tables/table_queries.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { using namespace NActors; -using namespace NWorkload; +using namespace NWorkloadManager; void CheckPoolDescription(TIntrusivePtr<IYdbSetup> ydb, ui64 expectedQueueSize, ui64 expectedInFlight, TDuration leaseDuration = FUTURE_WAIT_TIMEOUT) { const auto& description = ydb->GetPoolDescription(leaseDuration); @@ -121,7 +121,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceTables) { // Restart workload service ydb->StopWorkloadService(); auto runtime = ydb->GetRuntime(); - runtime->Register(CreateKqpWorkloadService(runtime->GetAppData().Counters)); + runtime->Register(CreateService(NWorkloadManager::GetWorkloadManagerCounters(runtime->GetAppData().Counters))); // Check that tables will be cleanuped ydb->WaitPoolState({.DelayedRequests = 0, .RunningRequests = 0}); @@ -176,4 +176,4 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceTables) { } } -} // namespace NKikimr::NKqp +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp b/ydb/services/workload_manager/ut/workload_service_ut.cpp index c011a23cf45..d48bc6a6fad 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp +++ b/ydb/services/workload_manager/ut/workload_service_ut.cpp @@ -1,16 +1,28 @@ #include <ydb/core/base/appdata_fwd.h> #include <ydb/core/base/path.h> -#include <ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h> +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> -namespace NKikimr::NKqp { +namespace NKikimr::NWorkloadManager { namespace { -using namespace NWorkload; +using namespace NWorkloadManager; using namespace NYdb; +TIntrusivePtr<IYdbSetup> MakeYdbWithAdvancedDisabled() { + return TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(false) + .Create(); +} + +TIntrusivePtr<IYdbSetup> MakeYdbWithRejectActionDisabled() { + return TYdbSetupSettings() + .EnableRejectActionInResourcePoolClassifiers(false) + .Create(); +} + void StartTestConcurrentQueryLimit(const ui64 activeCountLimit, const ui64 queueSize, const ui64 nodeCount = 1) { auto ydb = TYdbSetupSettings() .NodeCount(nodeCount) @@ -876,11 +888,11 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifiersDdl) { } void WaitForFail(TIntrusivePtr<IYdbSetup> ydb, const TQueryRunnerSettings& settings, const TString& poolId) { - NWorkload::WaitForClassifierFail(ydb, settings, poolId); + NWorkloadManager::WaitForClassifierFail(ydb, settings, poolId); } void WaitForSuccess(TIntrusivePtr<IYdbSetup> ydb, const TQueryRunnerSettings& settings) { - NWorkload::WaitForClassifierSuccess(ydb, settings); + NWorkloadManager::WaitForClassifierSuccess(ydb, settings); } Y_UNIT_TEST(TestCreateResourcePoolClassifier) { @@ -2065,4 +2077,154 @@ Y_UNIT_TEST_SUITE(DefaultPoolSettings) { -} // namespace NKikimr::NKqp +Y_UNIT_TEST_SUITE(HasPredicatesClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithHasFullScanIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasPathIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_path WITH ( + RESOURCE_POOL="default", + HAS_PATH='/Root/mydb/.*', + RANK=200 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasStreamIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_stream WITH ( + RESOURCE_POOL="default", + HAS_STREAM='true', + RANK=300 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasAppNameIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_app_name WITH ( + RESOURCE_POOL="default", + HAS_APP_NAME='myapp', + RANK=350 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(AlterWithHasFullScanIsRejected) { + // Create a basic classifier first (no advanced props) — should succeed. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_basic WITH ( + RESOURCE_POOL="default", + RANK=500 + ); + )"); + + // ALTER to add HAS_FULL_SCAN — should fail. + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + ALTER RESOURCE POOL CLASSIFIER cl_basic SET ( + HAS_FULL_SCAN='/Root/.*' + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithoutAdvancedPropsSucceeds) { + // Plain classifier must still work even when the flag is off. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_plain WITH ( + RESOURCE_POOL="default", + RANK=600 + ); + )"); + // If we get here without an exception, the creation succeeded. + } +} + +Y_UNIT_TEST_SUITE(HasPredicatesFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithHasFullScanSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan_ok WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithActionRejectIsRejected) { + auto ydb = MakeYdbWithRejectActionDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_action_reject WITH ( + ACTION="reject", + RANK=400 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Resource pool classifier action property is disabled"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithActionRejectSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_reject_ok WITH ( + ACTION="reject", + RANK=200 + ); + )"); + } +} + +} // namespace NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/ut/ya.make b/ydb/services/workload_manager/ut/ya.make new file mode 100644 index 00000000000..525a582d59a --- /dev/null +++ b/ydb/services/workload_manager/ut/ya.make @@ -0,0 +1,40 @@ +UNITTEST_FOR(ydb/services/workload_manager) + +FORK_SUBTESTS() + +SIZE(MEDIUM) +IF (SANITIZER_TYPE) + REQUIREMENTS(cpu:4) +ELSE() + REQUIREMENTS(cpu:2) +ENDIF() + +SRCS( + has_app_name_ut.cpp + action_reject_ut.cpp + has_full_scan_matcher_ut.cpp + has_full_scan_ut.cpp + has_path_ddl_ut.cpp + has_path_matcher_ut.cpp + has_path_ut.cpp + stream_query_classification_ut.cpp + has_stream_matcher_ut.cpp + has_stream_ut.cpp + member_name_ut.cpp + query_classifier_match_ut.cpp + query_classifier_ut.cpp + workload_service_actors_ut.cpp + workload_service_query_sessions_ut.cpp + workload_service_tables_ut.cpp + workload_service_ut.cpp +) + +PEERDIR( + ydb/services/workload_manager/ut/common + + yql/essentials/sql/pg_dummy +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/services/workload_manager/ya.make b/ydb/services/workload_manager/ya.make new file mode 100644 index 00000000000..ca99fdb9abe --- /dev/null +++ b/ydb/services/workload_manager/ya.make @@ -0,0 +1,38 @@ +LIBRARY() + +SRCS( + has_full_scan_matcher.cpp + has_path_matcher.cpp + has_stream_matcher.cpp + query_classifier.cpp +) + +PEERDIR( + ydb/core/cms/console + + ydb/core/kqp/common + ydb/core/kqp/query_data + + ydb/core/mind + + ydb/core/resource_pools + + ydb/library/aclib + +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE( + actors + common + metadata_subscription + tables + service +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index b5f9759d2ed..c6c37c5f84c 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -154,7 +154,9 @@ public: sslData.Cert = TestSettings::GetServerCrt(); sslData.Key = TestSettings::GetServerKey(); sslData.Root = TestSettings::GetCaCrt(); - sslData.DoRequestClientCertificate = appConfig.GetClientCertificateAuthorization().GetRequestClientCertificate(); + const auto& clientCertificateAuthorization = appConfig.GetClientCertificateAuthorization(); + sslData.DoRequestClientCertificate = clientCertificateAuthorization.GetRequestClientCertificate(); + sslData.ClientCertificateRequired = clientCertificateAuthorization.GetClientCertificateRequired(); grpcOption.SetSslData(sslData); } diff --git a/ydb/services/ydb/ydb_register_node_ut.cpp b/ydb/services/ydb/ydb_register_node_ut.cpp index 8eb51610567..b905e4bb753 100644 --- a/ydb/services/ydb/ydb_register_node_ut.cpp +++ b/ydb/services/ydb/ydb_register_node_ut.cpp @@ -53,6 +53,7 @@ struct TKikimrServerForTestNodeRegistration : TBasicKikimrWithGrpcAndRootSchema< bool EnforceUserToken = false; bool EnforceCheckUserToken = false; // Takes effect when EnforceUserToken = false bool EnableDynamicNodeAuth = false; + bool EnableClientCertificateRequired = false; bool EnableWrongIdentity = false; bool SetNodeAuthValues = false; std::vector<TString> RegisterNodeAllowedSids = {"DefaultClientAuth@cert", BUILTIN_ACL_ROOT}; @@ -77,6 +78,11 @@ private: config.MutableClientCertificateAuthorization()->SetRequestClientCertificate(true); // config.MutableFeatureFlags()->SetEnableDynamicNodeAuthorization(true); } + if (serverInitialization.EnableClientCertificateRequired) { + auto* clientCertificateAuthorization = config.MutableClientCertificateAuthorization(); + clientCertificateAuthorization->SetRequestClientCertificate(true); + clientCertificateAuthorization->SetClientCertificateRequired(true); + } std::vector<TString> tmpRegisterNodeAllowedSids(serverInitialization.RegisterNodeAllowedSids); for (auto& sid : tmpRegisterNodeAllowedSids) { @@ -899,6 +905,73 @@ Y_UNIT_TEST(ServerWithoutCertVerification_ClientDoesNotProvideClientCerts) { } } +Y_UNIT_TEST(ServerWithClientCertificateRequired_ClientDoesNotProvideClientCerts) { + const TCertAndKey& caCert = TKikimrTestWithAuthAndSsl::GetCACertAndKey(); + const TString expectedTransportError = "connections to all backends failing"; + { + TKikimrServerForTestNodeRegistration server({ + .EnforceUserToken = true, + .EnableClientCertificateRequired = true, + }); + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + const TString database = "/" + server.ServerSettings->DomainName; + + SetLogPriority(server); + + TDriverConfig config; + config.UseSecureConnection(caCert.Certificate.c_str()) + .SetDatabase(database) + .SetEndpoint(location); + + CheckAccessDenied(RegisterNode(config), expectedTransportError); + CheckAccessDenied(RegisterNode(config.SetAuthToken(BUILTIN_ACL_ROOT)), expectedTransportError); + CheckAccessDenied(RegisterNode(config.SetAuthToken("wrong_token")), expectedTransportError); + } + { + TKikimrServerForTestNodeRegistration serverDoesNotRequireToken({ + .EnforceUserToken = false, + .EnableClientCertificateRequired = true, + }); + ui16 grpc = serverDoesNotRequireToken.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + const TString database = "/" + serverDoesNotRequireToken.ServerSettings->DomainName; + + SetLogPriority(serverDoesNotRequireToken); + + TDriverConfig config; + config.UseSecureConnection(caCert.Certificate.c_str()) + .SetDatabase(database) + .SetEndpoint(location); + + CheckAccessDenied(RegisterNode(config), expectedTransportError); + CheckAccessDenied(RegisterNode(config.SetAuthToken(BUILTIN_ACL_ROOT)), expectedTransportError); + CheckAccessDenied(RegisterNode(config.SetAuthToken("wrong_token")), expectedTransportError); + } +} + +Y_UNIT_TEST(ServerWithClientCertificateRequired_ClientProvidesClientCerts) { + const TCertAndKey& caCert = TKikimrTestWithAuthAndSsl::GetCACertAndKey(); + const TCertAndKey clientServerCert = GenerateSignedCert(caCert, TProps::AsClientServer()); + TKikimrServerForTestNodeRegistration server({ + .EnableClientCertificateRequired = true, + }); + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + const TString database = "/" + server.ServerSettings->DomainName; + + SetLogPriority(server); + + TDriverConfig config; + config.UseSecureConnection(caCert.Certificate.c_str()) + .UseClientCertificate(clientServerCert.Certificate.c_str(), clientServerCert.PrivateKey.c_str()) + .SetDatabase(database) + .SetEndpoint(location); + + CheckGood(RegisterNode(config)); + CheckGood(RegisterNode(config.SetAuthToken(BUILTIN_ACL_ROOT))); +} + Y_UNIT_TEST(ServerWithCertVerification_AuthNotRequired) { // Scenario when we want to turn on secure node registration, but to check it in safe way const TCertAndKey& caCert = TKikimrTestWithAuthAndSsl::GetCACertAndKey(); diff --git a/ydb/tests/compatibility/indexes/test_bloom_filter_index.py b/ydb/tests/compatibility/indexes/test_bloom_filter_index.py index b1bd6fee268..d4c9ec25b38 100644 --- a/ydb/tests/compatibility/indexes/test_bloom_filter_index.py +++ b/ydb/tests/compatibility/indexes/test_bloom_filter_index.py @@ -32,7 +32,7 @@ class TestBloomFilterIndex(RestartToAnotherVersionFixture): # Verify bloom filters exist as scheme objects via describe_table def callee(session): - return session.describe_table(f"/{self.database_root}/{self.table_name}") + return session.describe_table(f"/{self.database_path}/{self.table_name}") with ydb.SessionPool(self.driver, size=1) as pool: desc = pool.retry_operation_sync(callee) @@ -66,7 +66,7 @@ class TestBloomFilterIndex(RestartToAnotherVersionFixture): # Verify bloom filters still exist as scheme objects via describe_table after version change def callee_after(session): - return session.describe_table(f"/{self.database_root}/{self.table_name}") + return session.describe_table(f"/{self.database_path}/{self.table_name}") with ydb.SessionPool(self.driver, size=1) as pool: desc_after = pool.retry_operation_sync(callee_after) diff --git a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp index b77e4c69504..4d634c17b60 100644 --- a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp @@ -13,7 +13,6 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/tests/tools/fqrun/src/fq_setup.h> diff --git a/ydb/tests/fq/control_plane_storage/ya.make b/ydb/tests/fq/control_plane_storage/ya.make index 0d66ed059fc..9287a2e53f8 100644 --- a/ydb/tests/fq/control_plane_storage/ya.make +++ b/ydb/tests/fq/control_plane_storage/ya.make @@ -16,7 +16,6 @@ PEERDIR( library/cpp/testing/unittest ydb/core/base ydb/core/external_sources - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/init ydb/core/fq/libs/quota_manager/events ydb/core/fq/libs/rate_limiter/control_plane_service diff --git a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h index eab8e590c1a..d88b47553d8 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h +++ b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h @@ -6,7 +6,6 @@ #include <ydb/core/fq/libs/control_plane_storage/schema.h> #include <yql/essentials/providers/common/proto/gateways_config.pb.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/core/testlib/tablet_helpers.h> diff --git a/ydb/tests/fq/streaming/test_streaming.py b/ydb/tests/fq/streaming/test_streaming.py index 5e2f3f73765..0a9b50b0eba 100644 --- a/ydb/tests/fq/streaming/test_streaming.py +++ b/ydb/tests/fq/streaming/test_streaming.py @@ -9,6 +9,7 @@ from typing import Callable import ydb from ydb.tests.fq.streaming_common.common import Kikimr, StreamingTestBase +from ydb.tests.library.test_meta import link_test_case from ydb.tests.tools.datastreams_helpers.control_plane import create_read_rule logger = logging.getLogger(__name__) @@ -1543,3 +1544,248 @@ FROM `{table_name}`""" result_sets = kikimr.ydb_client.query(sql) assert result_sets[0].rows[0]['Data'] == message + + @link_test_case("#46136") + @link_test_case("#46137") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("additional_operator", ["hop", "mr", "join"]) + def test_precompute_and_other_ops(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], local_topics: bool, additional_operator: str) -> None: + inp, out, endpoint = self.get_io_names(kikimr, f"test_precompute_and_other_ops_{local_topics!s:.1}_{additional_operator}", local_topics, entity_name) + + precompute_row_table = f"test_precompute_and_other_ops_precompute_row_table_{local_topics!s:.1}_{additional_operator}" + precompute_column_table = f"test_precompute_and_other_ops_precompute_column_table_{local_topics!s:.1}_{additional_operator}" + output_table = f"test_precompute_and_other_ops_output_table_{local_topics!s:.1}_{additional_operator}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{precompute_row_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + CREATE TABLE `{precompute_column_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ) WITH ( + STORE = COLUMN + ); + CREATE TABLE `{output_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key, Value) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{precompute_row_table}` + (Key, Value) + VALUES + (1, "value-p-row"); + UPSERT INTO `{precompute_column_table}` + (Key, Value) + VALUES + (1, "value-p-column"); + """) + + if additional_operator == "hop": + process = """ + SELECT + Key, + SOME(Value) AS Value + FROM $in + GROUP BY + Key, + HOP(Ts, "PT10S", "PT10S", "PT10S") + """ + + expected_data1 = [] + expected_data2 = ["in1-value-p-row-value-p-column:1", "in1-value-p-row-value-p-column:2"] + elif additional_operator == "mr": + process = """ + SELECT * FROM $in MATCH_RECOGNIZE( + MEASURES + LAST(A.Key) as MatchKey + ALL ROWS PER MATCH + AFTER MATCH SKIP TO NEXT ROW + PATTERN ( A B ) + DEFINE + A as A.Key = 1, + B as B.Key = 2 + ); + """ + + expected_data1 = [] + expected_data2 = ["in1-value-p-row-value-p-column:1", "in1-value-p-row-value-p-column:2", "in2-value-p-row-value-p-column:1", "in2-value-p-row-value-p-column:2"] + elif additional_operator == "join": + join_table = f"test_precompute_and_other_ops_join_table_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{join_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{join_table}` + (Key, Value) + VALUES + (1, "value-j1"), + (2, "value-j2"); + """) + + process = f""" + SELECT + i.Key AS Key, + i.Value || "-" || j.Value AS Value + FROM $in AS i + LEFT JOIN `{join_table}` AS j ON i.Key = j.Key + """ + + expected_data1 = ["in1-value-p-row-value-j1-value-p-column:1", "in1-value-p-row-value-j2-value-p-column:2"] + expected_data2 = ["in2-value-p-row-value-j1-value-p-column:1", "in2-value-p-row-value-j2-value-p-column:2"] + + query_name = f"test_precompute_and_other_ops_query_{local_topics!s:.1}_{additional_operator}" + kikimr.ydb_client.query(f""" + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + PRAGMA FeatureR010 = "prototype"; + + $p_key = SELECT Key FROM `{precompute_row_table}`; + $p_row = SELECT Value FROM `{precompute_row_table}`; + $p_column = SELECT Value FROM `{precompute_column_table}`; + + $in = SELECT + Key, + Unwrap(Value || "-" || $p_row) AS Value, + Unwrap(CAST(Ts AS Timestamp)) AS Ts + FROM {inp} WITH ( + FORMAT = json_each_row, + SCHEMA = ( + Key Int32 NOT NULL, + Value String NOT NULL, + Ts String NOT NULL + ) + ) + WHERE Key % $p_key == 0; + + $processed = {process}; + + INSERT INTO {out} + SELECT + Unwrap(Value || "-" || $p_column || ":" || CAST(Key AS String)) AS Value + FROM $processed; + + UPSERT INTO `{output_table}` SELECT + Unwrap(Key) AS Key, + Unwrap(Value) AS Value + FROM $processed; + END DO; + """) + + path = f"/Root/{query_name}" + self.wait_completed_checkpoints(kikimr, path) + + def validate_table(expected): + result_sets = kikimr.ydb_client.query(f""" + SELECT * FROM `{output_table}` + ORDER BY Value || ":" || CAST(Key AS String); + """) + assert len(result_sets[0].rows) == len(expected) + + for row, expected_value in zip(result_sets[0].rows, sorted(expected)): + value, key = expected_value.split(":") + assert value.startswith(row["Value"].decode("utf-8")), row["Value"].decode("utf-8") + " vs " + value + assert row["Key"] == int(key) + + result_sets = kikimr.ydb_client.query(f""" + DELETE FROM `{output_table}`; + """) + + self.write_stream(['{"Key": 1, "Value": "in1", "Ts": "2026-07-17T07:20:53.428176Z"}'], endpoint=endpoint) + self.write_stream(['{"Key": 2, "Value": "in1", "Ts": "2026-07-17T08:20:53.428176Z"}'], endpoint=endpoint) + if expected_data1: + assert sorted(self.read_stream(len(expected_data1), topic_path=self.output_topic, endpoint=endpoint)) == sorted(expected_data1) + self.wait_completed_checkpoints(kikimr, path) + validate_table(expected_data1) + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (RUN = FALSE); + """) + + self.write_stream(['{"Key": 1, "Value": "in2", "Ts": "2026-07-17T09:20:53.428176Z"}'], endpoint=endpoint) + self.write_stream(['{"Key": 2, "Value": "in2", "Ts": "2026-07-17T10:20:53.428176Z"}'], endpoint=endpoint) + + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (RUN = TRUE); + """) + + assert sorted(self.read_stream(len(expected_data2), topic_path=self.output_topic, endpoint=endpoint)) == sorted(expected_data2) + self.wait_completed_checkpoints(kikimr, path) + validate_table(expected_data2) + + @link_test_case("#46139") + @pytest.mark.parametrize("local_topics", [True, False]) + def test_alter_query_with_precompute(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], local_topics: bool) -> None: + inp, out, endpoint = self.get_io_names(kikimr, f"test_alter_query_with_precompute_{local_topics!s:.1}", local_topics, entity_name) + + precompute_table = f"test_alter_query_with_precompute_table_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{precompute_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{precompute_table}` + (Key, Value) + VALUES + (1, "value-p-row"); + """) + + query_name = f"test_alter_query_with_precompute_query_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + INSERT INTO {out} + SELECT * FROM {inp} + END DO; + """) + + path = f"/Root/{query_name}" + self.wait_completed_checkpoints(kikimr, path) + + expected_data = ["test_data1"] + self.write_stream(expected_data, endpoint=endpoint) + assert self.read_stream(len(expected_data), topic_path=self.output_topic, endpoint=endpoint) == expected_data + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (FORCE = TRUE) AS + DO BEGIN + $p_value = SELECT Value FROM `{precompute_table}`; + INSERT INTO {out} + SELECT Unwrap(Data || $p_value) FROM {inp} + END DO; + """) + + self.wait_completed_checkpoints(kikimr, path) + self.write_stream(["test_data2"], endpoint=endpoint) + assert self.read_stream(1, topic_path=self.output_topic, endpoint=endpoint) == ["test_data2value-p-row"] + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (FORCE = TRUE) AS + DO BEGIN + INSERT INTO {out} + SELECT * FROM {inp} + END DO; + """) + + self.wait_completed_checkpoints(kikimr, path) + expected_data = ["test_data3"] + self.write_stream(expected_data, endpoint=endpoint) + assert self.read_stream(len(expected_data), topic_path=self.output_topic, endpoint=endpoint) == expected_data + + kikimr.ydb_client.query(f""" + DROP STREAMING QUERY `{query_name}`; + """) diff --git a/ydb/tests/fq/streaming/ya.make b/ydb/tests/fq/streaming/ya.make index e1b14d8871e..97c37186f39 100644 --- a/ydb/tests/fq/streaming/ya.make +++ b/ydb/tests/fq/streaming/ya.make @@ -29,6 +29,7 @@ IF (SANITIZER_TYPE) ELSE() SIZE(MEDIUM) FORK_SUBTESTS() + SPLIT_FACTOR(20) ENDIF() PEERDIR( diff --git a/ydb/tests/fq/streaming_common/common.py b/ydb/tests/fq/streaming_common/common.py index f24a1aa1b46..b02339eb7e7 100644 --- a/ydb/tests/fq/streaming_common/common.py +++ b/ydb/tests/fq/streaming_common/common.py @@ -71,6 +71,7 @@ def get_ydb_config(request): "enable_watermarks_advanced": enable_watermarks_advanced, "enable_streaming_partition_balancing": enable_streaming_partition_balancing, "enable_compile_cache_warmup": False, + "enable_channel_memory_tracking": False, # Remove after fix https://github.com/ydb-platform/ydb/issues/46891 }, replication_config={ "iam_service_control": { diff --git a/ydb/tests/functional/config/test_distconf_self_heal.py b/ydb/tests/functional/config/test_distconf_self_heal.py index 91c9846795e..375e1094ff3 100644 --- a/ydb/tests/functional/config/test_distconf_self_heal.py +++ b/ydb/tests/functional/config/test_distconf_self_heal.py @@ -26,6 +26,28 @@ def get_ring_group(request_config, config_name): return config["Ring"] +def extract_node_ids(ring): + """Recursively collect all node ids referenced by a (possibly nested) TRing structure.""" + node_ids = set() + if "Node" in ring: + node_ids.update(ring["Node"]) + for nested in ring.get("Ring", []): + node_ids.update(extract_node_ids(nested)) + return node_ids + + +def extract_all_node_ids(config, config_name): + """Collect all node ids present in either the RingGroups or single Ring representation.""" + config = config[f"{config_name}Config"] + node_ids = set() + if "RingGroups" in config: + for rg in config["RingGroups"]: + node_ids.update(extract_node_ids(rg)) + elif "Ring" in config: + node_ids.update(extract_node_ids(config["Ring"])) + return node_ids + + class KiKiMRDistConfSelfHealTest(object): nodes_count = 9 erasure = Erasure.MIRROR_3_DC @@ -33,6 +55,11 @@ class KiKiMRDistConfSelfHealTest(object): separate_node_configs = True state_storage_rings = None n_to_select = None + # extra fields merged into the static self_management_config section of the initial + # (bootstrap) YAML config; NB: Cfg->SelfManagementConfig used by distconf self-heal is read + # once at node startup and is NOT affected by dynamic ReplaceConfig calls, so options like + # automatic_*_management / *_self_heal_allowed_nodes must be set here, before cluster start. + self_management_extra_options = {} metadata_section = { "kind": "MainConfig", "version": 0, @@ -59,6 +86,9 @@ class KiKiMRDistConfSelfHealTest(object): n_to_select=cls.n_to_select, state_storage_rings=cls.state_storage_rings) + if cls.self_management_extra_options: + cls.configurator.yaml_config["self_management_config"].update(cls.self_management_extra_options) + cls.cluster = KiKiMR(configurator=cls.configurator) cls.cluster.start() @@ -234,3 +264,124 @@ class TestKiKiMRDistConfSelfHealParallelCall2(KiKiMRDistConfSelfHealTest): rg = self.do_request_config()[f"{configName}Config"]["Ring"] assert_eq(rg["NToSelect"], 9) assert_eq(len(rg["Ring"]), 9) + + +class TestKiKiMRDistConfSelfHealAllowedNodes(KiKiMRDistConfSelfHealTest): + # 3 extra spare nodes (10, 11, 12) relative to the base 9-node MIRROR_3_DC topology so that + # forbidding one node still leaves enough nodes for a valid, fully-healthy configuration. + erasure = Erasure.MIRROR_3_DC + nodes_count = 12 + rgOffset = 1 + + forbidden_node_id = nodes_count + allowed_node_ids = sorted(set(range(1, nodes_count + 1)) - {forbidden_node_id}) + + self_management_extra_options = { + "automatic_state_storage_management": True, + "state_storage_self_heal_allowed_nodes": allowed_node_ids, + "automatic_state_storage_board_management": True, + "state_storage_board_self_heal_allowed_nodes": allowed_node_ids, + "automatic_scheme_board_management": True, + "scheme_board_self_heal_allowed_nodes": allowed_node_ids, + } + + def do_test(self, configName): + self.do_bad_config(configName) + + logger.info("Start SelfHeal with allowed nodes restriction (forbidding node %s) for %s", + self.forbidden_node_id, configName) + logger.info(self.do_request({"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}})) + time.sleep(10) + + healed_config = self.do_request_config() + node_ids = extract_all_node_ids(healed_config, configName) + + assert_that(len(node_ids) > 0, f"Healed {configName} config has no nodes: {healed_config}") + assert_that(self.forbidden_node_id not in node_ids, + f"Forbidden node {self.forbidden_node_id} present in healed {configName} config: {node_ids}") + assert_that(node_ids <= set(self.allowed_node_ids), + f"Healed {configName} config uses nodes outside the allowed list: {node_ids - set(self.allowed_node_ids)}") + + +class TestKiKiMRDistConfSelfHealAutomaticManagementDisabled(KiKiMRDistConfSelfHealTest): + erasure = Erasure.MIRROR_3_DC + nodes_count = 9 + rgOffset = 1 + + self_management_extra_options = { + "automatic_state_storage_management": False, + "automatic_state_storage_board_management": False, + "automatic_scheme_board_management": False, + } + + def check_failed(self, req, message): + resp = self.do_request(req) + assert_that(resp.get("ErrorReason", "").startswith(message), {"Response": resp, "Expected": message}) + + def do_test(self, configName): + # Manually push the config into a "bad"/suboptimal state (ReconfigStateStorage is a + # direct manual API and is unaffected by AutomaticManagement flags). + self.do_bad_config(configName) + bad_config = get_ring_group(self.do_request_config(), configName) + + logger.info("Attempting SelfHeal with automatic management disabled for %s", configName) + self.check_failed( + {"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}}, + "Current configuration is recommended. Nothing to self-heal.") + time.sleep(5) + + # The config must remain completely untouched: self-heal must not have modified it since + # automatic management for this subsystem is disabled. + unchanged_config = get_ring_group(self.do_request_config(), configName) + assert_eq(unchanged_config, bad_config) + + +class TestKiKiMRDistConfSelfHealMixedAutomaticManagement(KiKiMRDistConfSelfHealTest): + # Verifies the mixed scenario: automatic management is disabled for one subsystem + # (StateStorage) while enabled for the other two (StateStorageBoard, SchemeBoard). Self-heal + # must heal the enabled subsystems while leaving the disabled one untouched, exercising the + # interaction with the shared usedNodes/nodesToReplace state in SelfHealStateStorage. + erasure = Erasure.MIRROR_3_DC + nodes_count = 9 + rgOffset = 1 + + disabled_config_name = "StateStorage" + + self_management_extra_options = { + "automatic_state_storage_management": False, + "automatic_state_storage_board_management": True, + "automatic_scheme_board_management": True, + } + + def check_failed(self, req, message): + resp = self.do_request(req) + assert_that(resp.get("ErrorReason", "").startswith(message), {"Response": resp, "Expected": message}) + + def do_test(self, configName): + self.do_bad_config(configName) + bad_config = get_ring_group(self.do_request_config(), configName) + + logger.info( + "Start SelfHeal with mixed automatic management (disabled for %s) for %s", + self.disabled_config_name, configName) + + if configName == self.disabled_config_name: + self.check_failed( + {"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}}, + "Current configuration is recommended. Nothing to self-heal.") + time.sleep(5) + + # The disabled subsystem's config must remain completely untouched, even though the + # other subsystems are actively being healed around it. + unchanged_config = get_ring_group(self.do_request_config(), configName) + assert_eq(unchanged_config, bad_config) + else: + disabled_before = get_ring_group(self.do_request_config(), self.disabled_config_name) + logger.info(self.do_request({"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}})) + time.sleep(10) + + disabled_after = get_ring_group(self.do_request_config(), self.disabled_config_name) + assert_eq(disabled_after, disabled_before) + rg = get_ring_group(self.do_request_config(), configName) + assert_eq(rg["NToSelect"], 9) + assert_eq(len(rg["Ring"]), 9) diff --git a/ydb/tests/functional/security/mon/functional/test_mon_mtls_auth.py b/ydb/tests/functional/security/mon/functional/test_mon_mtls_auth.py index 8d747ec1eeb..b8f5ce2df26 100644 --- a/ydb/tests/functional/security/mon/functional/test_mon_mtls_auth.py +++ b/ydb/tests/functional/security/mon/functional/test_mon_mtls_auth.py @@ -214,10 +214,7 @@ def client_certificates(certificates): } [email protected](scope='module') -def ydb_configurator(ydb_cluster_configuration, certificates): - config_generator = KikimrConfigGenerator(**ydb_cluster_configuration) - +def _setup_mtls_monitoring_config(config_generator, certificates, client_certificate_required): config_generator.yaml_config['client_certificate_authorization'] = { 'client_certificate_definitions': [ { @@ -249,11 +246,42 @@ def ydb_configurator(ydb_cluster_configuration, certificates): config_generator.monitoring_tls_cert_path = certificates['server_cert'] config_generator.monitoring_tls_key_path = certificates['server_key'] config_generator.monitoring_tls_ca_path = certificates['server_ca'] + config_generator.monitoring_tls_client_certificate_required = client_certificate_required + if client_certificate_required: + config_generator.monitoring_tls_admin_client_cert_path = certificates['admin_client_crt'] + config_generator.monitoring_tls_admin_client_key_path = certificates['admin_client_key'] + [email protected](scope='module') +def ydb_configurator(ydb_cluster_configuration, certificates): + config_generator = KikimrConfigGenerator(**ydb_cluster_configuration) + _setup_mtls_monitoring_config(config_generator, certificates, client_certificate_required=False) return config_generator -def _test_endpoint_with_certificate(trace_endpoint, client_certificates, cert_type, expected_status): [email protected](scope='module') +def ydb_configurator_required_mtls(ydb_cluster_configuration, certificates): + config_generator = KikimrConfigGenerator(**ydb_cluster_configuration) + _setup_mtls_monitoring_config(config_generator, certificates, client_certificate_required=True) + return config_generator + + [email protected](scope='module') +def ydb_cluster_required_mtls(ydb_configurator_required_mtls): + from ydb.tests.library.harness.kikimr_runner import KiKiMR + + cluster = KiKiMR(ydb_configurator_required_mtls) + cluster.start() + yield cluster + cluster.stop() + + +def _test_endpoint_without_client_certificate(trace_endpoint, ca_file): + with pytest.raises(requests.exceptions.SSLError): + requests.get(trace_endpoint, verify=ca_file, timeout=5) + + +def _test_endpoint_with_client_certificate(trace_endpoint, client_certificates, cert_type, expected_status): session = requests.Session() adapter = MTLSAdapter( cert_file=client_certificates[f'{cert_type}_cert'], @@ -290,10 +318,10 @@ def test_mtls_auth(ydb_cluster, client_certificates): _test_endpoint_without_any_auth(trace_endpoint) # With viewer certificate - _test_endpoint_with_certificate(trace_endpoint, client_certificates, 'viewer', 403) + _test_endpoint_with_client_certificate(trace_endpoint, client_certificates, 'viewer', 403) # With administration certificate - _test_endpoint_with_certificate(trace_endpoint, client_certificates, 'admin', 200) + _test_endpoint_with_client_certificate(trace_endpoint, client_certificates, 'admin', 200) def test_token_auth_when_mtls_is_enabled(ydb_cluster, client_certificates): @@ -303,7 +331,7 @@ def test_token_auth_when_mtls_is_enabled(ydb_cluster, client_certificates): trace_endpoint = f'{base_url}/trace' # Checks that mTLS is enabled - _test_endpoint_with_certificate(trace_endpoint, client_certificates, 'admin', 200) + _test_endpoint_with_client_certificate(trace_endpoint, client_certificates, 'admin', 200) # Without any authentication _test_endpoint_without_any_auth(trace_endpoint) @@ -313,3 +341,21 @@ def test_token_auth_when_mtls_is_enabled(ydb_cluster, client_certificates): # With an admin token _test_endpoint_with_token(trace_endpoint, 'root@builtin', 200) + + +def test_required_mtls_rejects_connection_without_client_cert(ydb_cluster_required_mtls, client_certificates): + host = ydb_cluster_required_mtls.nodes[1].host + mon_port = ydb_cluster_required_mtls.nodes[1].mon_port + base_url = f'https://{host}:{mon_port}' + trace_endpoint = f'{base_url}/trace' + + _test_endpoint_without_client_certificate(trace_endpoint, client_certificates['ca']) + + +def test_required_mtls_accepts_connection_with_client_cert(ydb_cluster_required_mtls, client_certificates): + host = ydb_cluster_required_mtls.nodes[1].host + mon_port = ydb_cluster_required_mtls.nodes[1].mon_port + base_url = f'https://{host}:{mon_port}' + trace_endpoint = f'{base_url}/trace' + + _test_endpoint_with_client_certificate(trace_endpoint, client_certificates, 'admin', 200) diff --git a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json index 4743e515a2b..821f0cec542 100644 --- a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json +++ b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json @@ -374,6 +374,10 @@ { "name": "MaintenanceStatus", "type": "Utf8?" + }, + { + "name": "ExpectedSlotSize", + "type": "Uint64?" } ], "primary_key": [ @@ -1027,6 +1031,10 @@ { "name": "HasPath", "type": "Utf8?" + }, + { + "name": "HasStream", + "type": "Bool?" } ], "primary_key": [ diff --git a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json index 32221b4c12c..9d8b2beb174 100644 --- a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json +++ b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json @@ -603,6 +603,10 @@ { "name": "HasPath", "type": "Utf8?" + }, + { + "name": "HasStream", + "type": "Bool?" } ], "primary_key": [ diff --git a/ydb/tests/functional/tenants/test_remove_storage_groups.py b/ydb/tests/functional/tenants/test_remove_storage_groups.py new file mode 100644 index 00000000000..f5a996ec2c5 --- /dev/null +++ b/ydb/tests/functional/tenants/test_remove_storage_groups.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +import copy +import logging +import os + +import pytest +from hamcrest import assert_that, equal_to + +import ydb.public.api.protos.ydb_cms_pb2 as cms_pb +from ydb.tests.library.common.wait_for import wait_for_and_assert +from ydb.tests.library.harness.util import LogLevels +from ydb.tests.oss.ydb_sdk_import import ydb + +logger = logging.getLogger(__name__) + +# local configuration for the ydb cluster (fetched by ydb_cluster_configuration fixture) +CLUSTER_CONFIG = dict( + additional_log_configs={ + # more logs + 'CMS_TENANTS': LogLevels.TRACE, + 'HIVE': LogLevels.TRACE, + # less logs + 'KQP_PROXY': LogLevels.CRIT, + 'KQP_WORKER': LogLevels.CRIT, + 'KQP_GATEWAY': LogLevels.CRIT, + 'GRPC_PROXY': LogLevels.ERROR, + 'TX_DATASHARD': LogLevels.ERROR, + 'TX_PROXY_SCHEME_CACHE': LogLevels.ERROR, + 'KQP_YQL': LogLevels.ERROR, + 'KQP_SESSION': LogLevels.CRIT, + 'KQP_COMPILE_ACTOR': LogLevels.CRIT, + 'PERSQUEUE_CLUSTER_TRACKER': LogLevels.CRIT, + 'SCHEME_BOARD_SUBSCRIBER': LogLevels.CRIT, + 'SCHEME_BOARD_POPULATOR': LogLevels.CRIT, + }, + extra_feature_flags=[ + 'enable_cut_history' + ], + # do not clutter logs with resource pools auto creation + enable_resource_pools=False, + hive_config={ + 'cut_history_allow_list': '' # allow all + }, +) + + +# fixtures.ydb_cluster_configuration local override [email protected](scope='module') +def ydb_cluster_configuration(): + conf = copy.deepcopy(CLUSTER_CONFIG) + return conf + + +def test_remove_storage_group(ydb_cluster, ydb_root, ydb_safe_test_name, ydb_client): + database = os.path.join(ydb_root, ydb_safe_test_name) + + ydb_cluster.create_database( + database, + storage_pool_units_count={'hdd': 1}, + ) + database_nodes = ydb_cluster.register_and_start_slots(database, count=1) + ydb_cluster.wait_tenant_up(database) + + ydb_cluster.alter_database( + database, + storage_units_to_add={'hdd': 2}, + ) + + driver = ydb_client(database) + with ydb.QuerySessionPool(driver, size=1) as pool: + table_name = f"`{database}/table`" + pool.execute_with_retries("CREATE TABLE " + table_name + " (key Int32, value String, primary key(key));") + + pool.execute_with_retries("""UPSERT INTO """ + table_name + """ (key, value) VALUES (1, "value1");""") + + ydb_cluster.alter_database( + database, + storage_units_to_remove={'hdd': 2}, + ) + + ydb_cluster.restart_slots() + + def get_storage_units(): + status = ydb_cluster.get_database_status(database) + units = sum([unit.count for unit in status.allocated_resources.storage_units]) + return units + + wait_for_and_assert( + get_storage_units, + equal_to(1), + timeout_seconds=120, + step_seconds=10, + ) + + status = ydb_cluster.get_database_status(database) + assert_that(status.state, equal_to(cms_pb.GetDatabaseStatusResult.RUNNING)) + + ydb_cluster.unregister_and_stop_slots(database_nodes) + ydb_cluster.remove_database(database) diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make index 724adb69b22..03963bb090a 100644 --- a/ydb/tests/functional/tenants/ya.make +++ b/ydb/tests/functional/tenants/ya.make @@ -13,6 +13,7 @@ TEST_SRCS( test_system_views.py test_auth_system_views.py test_publish_into_schemeboard_with_common_ssring.py + test_remove_storage_groups.py test_user_administration.py test_users_groups_with_acl.py ) diff --git a/ydb/tests/library/clients/kikimr_monitoring.py b/ydb/tests/library/clients/kikimr_monitoring.py index e755dadebeb..9dc7fa4c6f4 100644 --- a/ydb/tests/library/clients/kikimr_monitoring.py +++ b/ydb/tests/library/clients/kikimr_monitoring.py @@ -7,7 +7,17 @@ import urllib.request class KikimrMonitor(object): - def __init__(self, host, mon_port, update_interval_seconds=1.0, use_https=False, token=None): + def __init__( + self, + host, + mon_port, + update_interval_seconds=1.0, + use_https=False, + token=None, + client_cert_file=None, + client_key_file=None, + ca_file=None, + ): super(KikimrMonitor, self).__init__() self.__host = host self.__mon_port = mon_port @@ -16,6 +26,9 @@ class KikimrMonitor(object): self.__counters_url = "{monitoring_address}/counters/json".format(monitoring_address=self.__monitoring_address) self.__use_https = use_https self.__token = token + self.__client_cert_file = client_cert_file + self.__client_key_file = client_key_file + self.__ca_file = ca_file self.__pdisks = set() self.__data = {} self.__next_update_time = time.time() @@ -28,9 +41,15 @@ class KikimrMonitor(object): if self.__token: req.add_header('Authorization', self.__token) if self.__use_https: - ctx = ssl.create_default_context() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE + if self.__ca_file: + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cafile=self.__ca_file) + else: + ctx.verify_mode = ssl.CERT_NONE + if self.__client_cert_file and self.__client_key_file: + ctx.load_cert_chain(self.__client_cert_file, self.__client_key_file) return urllib.request.urlopen(req, context=ctx) else: return urllib.request.urlopen(req) diff --git a/ydb/tests/library/common/protobuf_console.py b/ydb/tests/library/common/protobuf_console.py index 925fe5dc0a0..6a6bb1da857 100644 --- a/ydb/tests/library/common/protobuf_console.py +++ b/ydb/tests/library/common/protobuf_console.py @@ -108,6 +108,18 @@ class AlterTenantRequest(AbstractProtobufBuilder): if soft is not None: quotas.data_size_soft_quota = soft + def add_storage_groups_to_add(self, pool_type, pool_size): + pool = self.protobuf.AlterTenantRequest.Request.storage_units_to_add.add() + pool.unit_kind = pool_type + pool.count = pool_size + return self + + def add_storage_groups_to_remove(self, pool_type, pool_size): + pool = self.protobuf.AlterTenantRequest.Request.storage_units_to_remove.add() + pool.unit_kind = pool_type + pool.count = pool_size + return self + class GetTenantStatusRequest(AbstractProtobufBuilder): """ diff --git a/ydb/tests/library/harness/kikimr_cluster_interface.py b/ydb/tests/library/harness/kikimr_cluster_interface.py index a8cb0106891..52b6015045e 100644 --- a/ydb/tests/library/harness/kikimr_cluster_interface.py +++ b/ydb/tests/library/harness/kikimr_cluster_interface.py @@ -400,6 +400,40 @@ class KiKiMRClusterInterface(object): ) return database_name + def alter_database( + self, + database_name, + storage_units_to_add=None, + storage_units_to_remove=None, + timeout_seconds=120, + token=None, + ): + req = AlterTenantRequest(database_name) + + if token is not None: + req.set_user_token(token) + + assert storage_units_to_add or storage_units_to_remove + if storage_units_to_add: + for pool_type, count in storage_units_to_add.items(): + req.add_storage_groups_to_add(pool_type, count) + + if storage_units_to_remove: + for pool_type, count in storage_units_to_remove.items(): + req.add_storage_groups_to_remove(pool_type, count) + + response = self.client.send_request(req.protobuf, method='ConsoleRequest') + operation = response.AlterTenantResponse.Response.operation + if not operation.ready and response.Status.Code != StatusIds.STATUS_CODE_UNSPECIFIED: + raise RuntimeError( + 'alter_database_storage_units failed: %s: %s' % (response.Status.Code, response.Status.Reason) + ) + if not operation.ready: + operation = self.__wait_console_op(operation.id, timeout_seconds=timeout_seconds, token=token) + if operation.status != StatusIds.SUCCESS: + raise RuntimeError('alter_database_storage_units failed: %s' % (operation.status,)) + return database_name + def remove_database( self, database_name, diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 4829367047d..58a8b5ffb47 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -174,6 +174,7 @@ class KikimrConfigGenerator(object): pg_compatible_expirement=False, generic_connector_config=None, # typing.Optional[TGenericConnectorConfig] kafka_api_port=None, + kafka_listen_address=None, metadata_section=None, column_shard_config=None, use_config_store=False, @@ -256,6 +257,9 @@ class KikimrConfigGenerator(object): self.monitoring_tls_cert_path = None self.monitoring_tls_key_path = None self.monitoring_tls_ca_path = None + self.monitoring_tls_admin_client_cert_path = None + self.monitoring_tls_admin_client_key_path = None + self._monitoring_tls_client_certificate_required = False self.enable_topic_cloud_events = enable_topic_cloud_events self.__binary_paths = binary_paths @@ -597,6 +601,8 @@ class KikimrConfigGenerator(object): kafka_proxy_config = dict() kafka_proxy_config['enable_kafka_proxy'] = True kafka_proxy_config["listening_port"] = self.get_kafka_api_port(node_id) + if kafka_listen_address is not None: + kafka_proxy_config["listening_address"] = kafka_listen_address self.yaml_config["kafka_proxy_config"] = kafka_proxy_config @@ -856,6 +862,19 @@ class KikimrConfigGenerator(object): def set_binary_paths(self, binary_paths): self.__binary_paths = binary_paths + @property + def monitoring_tls_client_certificate_required(self): + return self._monitoring_tls_client_certificate_required + + @monitoring_tls_client_certificate_required.setter + def monitoring_tls_client_certificate_required(self, value): + self._monitoring_tls_client_certificate_required = bool(value) + monitoring_config = self.yaml_config.setdefault('monitoring_config', {}) + if self._monitoring_tls_client_certificate_required: + monitoring_config['client_certificate_required'] = True + else: + monitoring_config.pop('client_certificate_required', None) + def write_tls_data(self): if self.__grpc_ssl_enable: for fpath, data in ( diff --git a/ydb/tests/library/harness/kikimr_node_interface.py b/ydb/tests/library/harness/kikimr_node_interface.py index d4fe3f981fa..99a4a5fdb64 100644 --- a/ydb/tests/library/harness/kikimr_node_interface.py +++ b/ydb/tests/library/harness/kikimr_node_interface.py @@ -41,7 +41,15 @@ class NodeInterface(object): @property def monitor(self): - return KikimrMonitor(self.host, self.mon_port, use_https=getattr(self, 'mon_uses_https', False), token=getattr(self, '_monitor_token', None)) + return KikimrMonitor( + self.host, + self.mon_port, + use_https=getattr(self, 'mon_uses_https', False), + token=getattr(self, '_monitor_token', None), + client_cert_file=getattr(self, '_monitor_client_cert_file', None), + client_key_file=getattr(self, '_monitor_client_key_file', None), + ca_file=getattr(self, '_monitor_ca_file', None), + ) @abc.abstractproperty def cwd(self): diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index 79d269c8959..1841ee636d3 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -75,6 +75,27 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface): self.grpc_port = port_allocator.grpc_port self.mon_port = port_allocator.mon_port self.mon_uses_https = self.__configurator.monitoring_tls_cert_path is not None + self._monitor_ca_file = self.__configurator.monitoring_tls_ca_path + if self.__configurator.monitoring_tls_client_certificate_required: + # Without proper client cert the cluster setup cannot be checked via monitoring: + # __wait_for_bs_controller_to_start polls /counters/json and hangs. + missing = [] + if not self.__configurator.monitoring_tls_admin_client_cert_path: + missing.append('monitoring_tls_admin_client_cert_path') + if not self.__configurator.monitoring_tls_admin_client_key_path: + missing.append('monitoring_tls_admin_client_key_path') + if not self.__configurator.monitoring_tls_ca_path: + missing.append('monitoring_tls_ca_path') + if missing: + raise ValueError( + 'monitoring TLS client certificate is required, but the following configurator ' + 'fields are not set: %s' % ', '.join(missing) + ) + self._monitor_client_cert_file = self.__configurator.monitoring_tls_admin_client_cert_path + self._monitor_client_key_file = self.__configurator.monitoring_tls_admin_client_key_path + else: + self._monitor_client_cert_file = None + self._monitor_client_key_file = None self.ic_port = port_allocator.ic_port self.grpc_ssl_port = port_allocator.grpc_ssl_port self.http_proxy_port = None @@ -805,6 +826,11 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): node.stop() node.start() + def restart_slots(self): + for slot in list(self.slots.values()): + slot.stop() + slot.start() + def prepare_node(self, configurator=None, seed_nodes_file=None): try: new_node_object = self.__register_node(configurator, seed_nodes_file) @@ -1009,7 +1035,10 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): node.host, node.mon_port, use_https=getattr(node, 'mon_uses_https', False), - token=token + token=token, + client_cert_file=getattr(node, '_monitor_client_cert_file', None), + client_key_file=getattr(node, '_monitor_client_key_file', None), + ca_file=getattr(node, '_monitor_ca_file', None), ) for node in self.nodes.values() ] diff --git a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py index 66817523528..95985bb74e8 100644 --- a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py +++ b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py @@ -773,7 +773,7 @@ class ScenarioTestHelper: def get_volumes_columns(self, table_name: str, name_column: str) -> tuple[int, int]: path = table_name if table_name.startswith('/') else self.get_full_path(table_name) - query = f'''SELECT * FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' + query = f'''SELECT SUM(RawBytes) AS RawBytes, SUM(BlobRangeSize) AS BlobRangeSize FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' if (len(name_column)): query += f' AND EntityName = \"{name_column}\"' result_sets = self.execute_query(query) diff --git a/ydb/tests/tools/kqprun/src/actors.cpp b/ydb/tests/tools/kqprun/src/actors.cpp index 4adafc08c76..9691ca2919d 100644 --- a/ydb/tests/tools/kqprun/src/actors.cpp +++ b/ydb/tests/tools/kqprun/src/actors.cpp @@ -4,7 +4,7 @@ #include <ydb/core/kqp/common/simple/services.h> #include <ydb/core/kqp/rm_service/kqp_rm_service.h> -#include <ydb/core/kqp/workload_service/actors/actors.h> +#include <ydb/services/workload_manager/actors/actors.h> using namespace NKikimrRun; @@ -180,7 +180,7 @@ public: } } - void Handle(NKikimr::NKqp::NWorkload::TEvFetchDatabaseResponse::TPtr& ev) { + void Handle(NKikimr::NWorkloadManager::TEvFetchDatabaseResponse::TPtr& ev) { const auto status = ev->Get()->Status; if (status == Ydb::StatusIds::SUCCESS) { HealthCheckStage_ = EHealthCheck::ScriptRequest; @@ -203,7 +203,7 @@ public: STRICT_STFUNC(StateFunc, sFunc(NActors::TEvents::TEvWakeup, DoHealthCheck); - hFunc(NKikimr::NKqp::NWorkload::TEvFetchDatabaseResponse, Handle); + hFunc(NKikimr::NWorkloadManager::TEvFetchDatabaseResponse, Handle); hFunc(NKikimr::NKqp::TEvKqp::TEvScriptResponse, Handle); ) @@ -229,7 +229,7 @@ private: } void FetchDatabase() { - Register(NKikimr::NKqp::NWorkload::CreateDatabaseFetcherActor(SelfId(), Settings_.Database)); + Register(NKikimr::NWorkloadManager::CreateDatabaseFetcherActor(SelfId(), Settings_.Database)); } void StartScriptQuery() { diff --git a/ydb/tests/tools/kqprun/src/ya.make b/ydb/tests/tools/kqprun/src/ya.make index b9a88f52573..3a9ff2e3bd4 100644 --- a/ydb/tests/tools/kqprun/src/ya.make +++ b/ydb/tests/tools/kqprun/src/ya.make @@ -10,7 +10,7 @@ PEERDIR( library/cpp/protobuf/json ydb/core/client/server ydb/core/grpc_services - ydb/core/kqp/workload_service/actors + ydb/services/workload_manager/actors ydb/core/testlib ydb/core/util ydb/library/aclib diff --git a/ydb/tools/cfg/base.py b/ydb/tools/cfg/base.py index d0bb80c5965..a7bfc343eff 100644 --- a/ydb/tools/cfg/base.py +++ b/ydb/tools/cfg/base.py @@ -63,12 +63,38 @@ def merge_with_default(dft, override): class KiKiMRDrive(object): - def __init__(self, type, path, shared_with_os=False, expected_slot_count=None, kind=None, pdisk_config=None, disk_scope=None): + def __init__(self, type, path, shared_with_os=False, expected_slot_count=None, + expected_slot_size=None, max_slots=None, kind=None, pdisk_config=None, disk_scope=None): + # the same settings may come as top-level drive fields or inside pdisk_config; + # validate the effective combination so that invalid mixes do not surface only + # as NodeWarden warnings after the config is deployed. Zero values are treated + # as 'not set' to match the C++ side + pdisk_config_dict = pdisk_config if isinstance(pdisk_config, dict) else {} + + def effective(top_level_value, key): + value = top_level_value if top_level_value is not None else pdisk_config_dict.get(key) + return value if value else None + + effective_slot_count = effective(expected_slot_count, "expected_slot_count") + effective_slot_size = effective(expected_slot_size, "expected_slot_size") + effective_max_slots = effective(max_slots, "max_slots") + slot_size_in_units = pdisk_config_dict.get("slot_size_in_units") + if effective_slot_size is not None: + if effective_slot_count is not None: + raise ValueError("expected_slot_size is mutually exclusive with expected_slot_count") + if slot_size_in_units: + raise ValueError("expected_slot_size is mutually exclusive with slot_size_in_units") + if effective_max_slots is None: + raise ValueError("expected_slot_size requires max_slots") + elif effective_max_slots is not None: + raise ValueError("max_slots requires expected_slot_size") self.type = type self.path = path self.shared_with_os = shared_with_os self.pdisk_config = pdisk_config self.expected_slot_count = expected_slot_count + self.expected_slot_size = expected_slot_size + self.max_slots = max_slots self.kind = kind self.disk_scope = disk_scope @@ -78,13 +104,25 @@ class KiKiMRDrive(object): and self.path == other.path and self.shared_with_os == other.shared_with_os and self.expected_slot_count == other.expected_slot_count + and self.expected_slot_size == other.expected_slot_size + and self.max_slots == other.max_slots and self.kind == other.kind and self.pdisk_config == other.pdisk_config and self.disk_scope == other.disk_scope ) def __hash__(self): - return hash("\0".join(map(str, (self.type, self.path, self.shared_with_os, self.expected_slot_count, self.kind, self.pdisk_config, self.disk_scope)))) + return hash("\0".join(map(str, ( + self.type, + self.path, + self.shared_with_os, + self.expected_slot_count, + self.expected_slot_size, + self.max_slots, + self.kind, + self.pdisk_config, + self.disk_scope, + )))) Domain = collections.namedtuple( diff --git a/ydb/tools/cfg/dynamic.py b/ydb/tools/cfg/dynamic.py index f4acce76b85..8857af288aa 100644 --- a/ydb/tools/cfg/dynamic.py +++ b/ydb/tools/cfg/dynamic.py @@ -198,6 +198,12 @@ class DynamicConfigGenerator(object): elif drive.expected_slot_count is not None: pc = pdisk_config.TPDiskConfig(ExpectedSlotCount=drive.expected_slot_count) kwargs.update(PDiskConfig=pc) + elif drive.expected_slot_size: # zero means 'not set', as on the C++ side + pc = pdisk_config.TPDiskConfig( + ExpectedSlotSize=drive.expected_slot_size, + MaxSlots=drive.max_slots, + ) + kwargs.update(PDiskConfig=pc) array.add(**kwargs) for host_config in self._cluster_details.host_configs: diff --git a/ydb/tools/cfg/static.py b/ydb/tools/cfg/static.py index 006c13834ba..2e216a659c1 100644 --- a/ydb/tools/cfg/static.py +++ b/ydb/tools/cfg/static.py @@ -692,11 +692,17 @@ class StaticConfigGenerator(object): # inside config.yaml we should use field drive in host_configs section host_config['drive'] = host_config.pop('drives') for drive in host_config['drive']: + pdisk_config = {} if 'expected_slot_count' in drive: # inside config.yaml we should use pdisk_config section for expected_slot_count - drive['pdisk_config'] = { - 'expected_slot_count': drive.pop('expected_slot_count') - } + pdisk_config['expected_slot_count'] = drive.pop('expected_slot_count') + if 'expected_slot_size' in drive: + # inside config.yaml we should use pdisk_config section for expected_slot_size/max_slots + pdisk_config['expected_slot_size'] = drive.pop('expected_slot_size') + if 'max_slots' in drive: + pdisk_config['max_slots'] = drive.pop('max_slots') + if pdisk_config: + drive.setdefault('pdisk_config', {}).update(pdisk_config) # support type-safe `pdisk_config` directly in `host_configs`, for example: # - path: /dev/disk/by-partlabel/ydb_disk_hdd_04 @@ -1004,7 +1010,14 @@ class StaticConfigGenerator(object): vdisk_location['pdisk_category'] = int(vdisk_location['pdisk_category']) if 'pdisk_config' in vdisk_location: if 'expected_slot_count' in vdisk_location['pdisk_config']: - vdisk_location['pdisk_config']['expected_slot_count'] = int(vdisk_location['pdisk_config']['expected_slot_count']) + vdisk_location['pdisk_config']['expected_slot_count'] = int( + vdisk_location['pdisk_config']['expected_slot_count']) + if 'expected_slot_size' in vdisk_location['pdisk_config']: + vdisk_location['pdisk_config']['expected_slot_size'] = int( + vdisk_location['pdisk_config']['expected_slot_size']) + if 'max_slots' in vdisk_location['pdisk_config']: + vdisk_location['pdisk_config']['max_slots'] = int( + vdisk_location['pdisk_config']['max_slots']) if self.__cluster_details.channel_profile_config is not None: normalized_config["channel_profile_config"] = self.__cluster_details.channel_profile_config @@ -1292,6 +1305,9 @@ class StaticConfigGenerator(object): if drive.expected_slot_count is not None: drive_pb.PDiskConfig.ExpectedSlotCount = drive.expected_slot_count + if drive.expected_slot_size: # zero means 'not set', as on the C++ side + drive_pb.PDiskConfig.ExpectedSlotSize = drive.expected_slot_size + drive_pb.PDiskConfig.MaxSlots = drive.max_slots if drive.disk_scope is not None: drive_pb.DiskScope = drive.disk_scope @@ -1375,6 +1391,9 @@ class StaticConfigGenerator(object): if drive.expected_slot_count is not None: drive_pb.PDiskConfig.ExpectedSlotCount = drive.expected_slot_count + if drive.expected_slot_size: # zero means 'not set', as on the C++ side + drive_pb.PDiskConfig.ExpectedSlotSize = drive.expected_slot_size + drive_pb.PDiskConfig.MaxSlots = drive.max_slots if drive.disk_scope is not None: drive_pb.DiskScope = drive.disk_scope diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py index 136fcaa3647..c103dfc715e 100644 --- a/ydb/tools/cfg/validation.py +++ b/ydb/tools/cfg/validation.py @@ -155,6 +155,8 @@ DRIVE_SCHEMA = { "path": dict(type="string", minLength=1), "shared_with_os": dict(type="boolean"), "expected_slot_count": dict(type="integer"), + "expected_slot_size": dict(type="integer"), + "max_slots": dict(type="integer"), "pdisk_config": { "type": "object", "additionalProperties": True, diff --git a/ydb/tools/stress_tool/README.md b/ydb/tools/stress_tool/README.md index e0608e7afab..d636240ee2b 100644 --- a/ydb/tools/stress_tool/README.md +++ b/ydb/tools/stress_tool/README.md @@ -84,10 +84,42 @@ Each `DDiskTestList` entry contains one or more `DDiskLoad` sources. These describe network load generated through the actors library interconnect subsystem (`ydb/library/actors/interconnect/load.h`), reusing the same `TInterconnectLoad` load actor used by the `InterconnectLoad` load type of the -regular YDB load actor service. Since the stress tool runs a single node, the -load actor and its counterpart (the load responder) both live in the same -process; traffic is routed through loopback using `NodeHops`, so no real -network connection is required to generate and measure interconnect traffic. +regular YDB load actor service. + +By default (no `--server`/`--client` options), the stress tool runs a single +node; the load actor and its counterpart (the load responder) both live in the +same process, and traffic is routed through loopback using `NodeHops`, so no +real network connection is required to generate and measure interconnect +traffic. + +To measure interconnect performance over a **real network** between two +hosts, run two copies of `ydb_stress_tool`: + +- On the **second** (remote) host, start a responder-only instance. Its + `--server NODE_ID` must match the NodeId that the client will assign this + server via `--endpoint` (see below) -- for a single server this is `1`: + ``` + ydb_stress_tool --cfg cfg.txt --server 1 --client 2 --ic-port 19001 + ``` + This registers the interconnect load responder for node 1 and listens on + port 19001 for an incoming connection from the client (node 2). No device + `--path` is required. + +- On the **first** host, run the actual `InterconnectTestList` from the config + as a client that connects out to the responder: + ``` + ydb_stress_tool --cfg cfg.txt --client 2 --endpoint remote-host:19001 + ``` + The client's own NodeId is `2` (from `--client`, must differ from all + server NodeIds); the server passed via `--endpoint` is assigned NodeId `1` + (the i-th `--endpoint` gets NodeId `i + 1`, matching the `--server NODE_ID` + used on the remote host). The config's `InterconnectLoad.NodeHops` must + reference this NodeId (e.g. `NodeHops: [1]`) so that load messages are + routed to the remote responder over the network instead of looping back + locally. + + Optional: `--use-uring` selects the io_uring transport instead of epoll for + the interconnect connection itself (Linux 5.19+). Each `InterconnectTestList` entry contains one or more `InterconnectLoad` sources, run sequentially. Detailed periodic throughput/RTT statistics are diff --git a/ydb/tools/stress_tool/device_test_tool.cpp b/ydb/tools/stress_tool/device_test_tool.cpp index 6362368ef29..53311967ed1 100644 --- a/ydb/tools/stress_tool/device_test_tool.cpp +++ b/ydb/tools/stress_tool/device_test_tool.cpp @@ -88,6 +88,15 @@ static void InstallServerSignalHandler() { signal(SIGTERM, ServerSignalHandler); } +static void InterconnectServerSignalHandler(int) { + NKikimr::InterconnectServerStopEvent.Signal(); +} + +static void InstallInterconnectServerSignalHandler() { + signal(SIGINT, InterconnectServerSignalHandler); + signal(SIGTERM, InterconnectServerSignalHandler); +} + static const char* ydb_logo = R"__( @@ -158,7 +167,9 @@ int main(int argc, char **argv) { // DDisk client/server options are hidden from the auto-generated --help list and // re-rendered below as a dedicated "DDisk client/server options" section. Keep the // section text in sync with the option definitions here. - opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has no DDiskTestList but has InterconnectTestList, runs as an interconnect load responder instead " + "(for testing interconnect over a real network between two ydb_stress_tool instances)") .RequiredArgument("NODE_ID").StoreResult(&serverNodeId).Hidden(); opts.AddLongOption("client", "client node ID; in server mode this is the expected client's ID (used in the nameserver), in client mode this is the client's own ID") .RequiredArgument("NODE_ID").StoreResult(&clientNodeId).Hidden(); @@ -166,7 +177,7 @@ int main(int argc, char **argv) { .RequiredArgument("HOST:PORT").AppendTo(&clientEndpoints).Hidden(); opts.AddLongOption("ic-port", "interconnect port for server to listen on (server only)") .RequiredArgument("PORT").StoreResult(&icPort).Hidden(); - opts.AddLongOption("num-server-devices", "number of devices per server (default 1)") + opts.AddLongOption("num-server-devices", "number of devices per server (default 1); ignored in interconnect server/client mode") .RequiredArgument("N").DefaultValue("1").Hidden(); bool useUring = false; opts.AddLongOption("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)") @@ -185,7 +196,9 @@ int main(int argc, char **argv) { TStringBuilder ddiskHelp; ddiskHelp << row("server NODE_ID", - "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has InterconnectTestList instead of DDiskTestList, runs as an interconnect load " + "responder for the network interconnect test (see InterconnectTestList in README)") << row("client NODE_ID", "client node ID; in server mode this is the expected client's ID (used in the nameserver), " "in client mode this is the client's own ID") @@ -194,11 +207,11 @@ int main(int argc, char **argv) { << row("ic-port PORT", "interconnect port for server to listen on (server only)") << row("num-server-devices N", - "number of devices per server (default 1)") + "number of devices per server (default 1); ignored for InterconnectTestList") << row("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)"); - opts.AddSection("DDisk client/server options", ddiskHelp); + opts.AddSection("DDisk / Interconnect client/server options", ddiskHelp); } TOptsParseResult res(&opts, argc, argv); @@ -206,6 +219,9 @@ int main(int argc, char **argv) { // Server mode is selected by --server. Client mode by --client without --server. // In server mode, --client is also required and provides the expected client's NodeID // so the interconnect handshake can resolve the peer in the static nameserver table. + // Both modes are shared between DDisk (real device paths, PDisk/DDisk actors) and + // Interconnect (no device paths, just a load responder / load actor) tests; which + // one is used is decided later based on the config file contents. const bool serverMode = res.Has("server"); const bool clientMode = !serverMode && res.Has("client"); if (serverMode) { @@ -221,10 +237,6 @@ int main(int argc, char **argv) { Cerr << "Error: --server and --client NODE_IDs must differ" << Endl; return 1; } - if (paths.empty()) { - Cerr << "Error: --server requires at least one --path" << Endl; - return 1; - } if (!icPort) { Cerr << "Error: --server requires --ic-port" << Endl; return 1; @@ -241,7 +253,7 @@ int main(int argc, char **argv) { } } - if (!clientMode && paths.empty()) { + if (!clientMode && !serverMode && paths.empty()) { Cerr << "Error: at least one --path must be specified" << Endl; return 1; } @@ -250,8 +262,11 @@ int main(int argc, char **argv) { Cout << ydb_logo << Flush; } - // For client mode, paths can be empty; use a dummy for TPerfTestConfig - if (clientMode && paths.empty()) { + const bool pathsProvided = !paths.empty(); + + // For client mode, and for server mode when running as an interconnect + // responder (no real devices), paths can be empty; use a dummy for TPerfTestConfig. + if ((clientMode || serverMode) && paths.empty()) { paths.push_back("unused"); } @@ -288,10 +303,77 @@ int main(int argc, char **argv) { } } - // Client-server mode dispatch + // Client-server mode dispatch. Two independent flavors share --server/--client/ + // --endpoint/--ic-port: DDisk (real device paths, PDisk/DDisk actors) is used when + // the config has DDiskTestList; Interconnect (no device paths, just a load + // responder / load actor talking over a real network connection) is used when the + // config has InterconnectTestList instead. This lets a second ydb_stress_tool + // instance on another host act as the interconnect load responder. if (serverMode || clientMode) { - if (protoTests.DDiskTestListSize() == 0) { - Cerr << "Error: --server/--client mode requires DDiskTestList in config" << Endl; + const bool hasDDiskTests = protoTests.DDiskTestListSize() > 0; + const bool hasInterconnectTests = protoTests.InterconnectTestListSize() > 0; + + if (!hasDDiskTests && !hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + if (hasDDiskTests && hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires exactly one of DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + + if (hasInterconnectTests) { + // Interconnect network test: no real device paths are used. + NDevicePerfTest::TInterconnectTest testProto = protoTests.GetInterconnectTestList(0); + + if (serverMode) { + InstallInterconnectServerSignalHandler(); + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + THolder<NKikimr::TPerfTest> test(new NKikimr::TInterconnectServer(config, serverNodeId, clientNodeId, icPort, useUring)); + test->SetPrinter(printer); + test->RunTest(); + return 0; + } + + if (clientMode) { + // Build server peer list. Server with index i (0-based) gets NodeId = i + 1. + // Config's InterconnectLoad.NodeHops entries should reference these NodeIds + // to route traffic to the corresponding remote server. + TVector<NKikimr::TInterconnectPeer> serverPeers; + serverPeers.reserve(clientEndpoints.size()); + for (size_t i = 0; i < clientEndpoints.size(); ++i) { + try { + auto [host, port] = ParseHostPort(clientEndpoints[i]); + serverPeers.push_back({static_cast<ui32>(i + 1), host, port}); + } catch (const yexception& ex) { + Cerr << "Error: --endpoint #" << (i + 1) << ": " << ex.what() << Endl; + return 1; + } + } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } + + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + for (ui32 run = 0; run < config.RunCount; ++run) { + THolder<NKikimr::TPerfTest> test( + new NKikimr::TInterconnectClient(config, testProto, clientNodeId, serverPeers, useUring)); + test->SetPrinter(printer); + test->RunTest(); + } + printer->EndTest(); + return 0; + } + } + + // DDisk test: real device paths (--path) are required. + if (serverMode && !pathsProvided) { + Cerr << "Error: --server requires at least one --path (DDisk mode)" << Endl; return 1; } NDevicePerfTest::TDDiskTest testProto = protoTests.GetDDiskTestList(0); @@ -320,6 +402,14 @@ int main(int argc, char **argv) { return 1; } } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } auto overrideDDiskInFlight = [](NDevicePerfTest::TDDiskTest& tp, ui32 inFlight) { for (size_t j = 0; j < tp.DDiskTestListSize(); ++j) { diff --git a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h index cdf4356b59e..57920621f43 100644 --- a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h +++ b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h @@ -30,11 +30,13 @@ #include <util/system/event.h> #include "device_test_tool.h" +#include "device_test_tool_ddisk_client_server.h" namespace NKikimr { static TSystemEvent InterconnectDoneEvent(TSystemEvent::rAuto); static TSystemEvent InterconnectResultsPrintedEvent(TSystemEvent::rAuto); +static TSystemEvent InterconnectServerStopEvent(TSystemEvent::rAuto); // Runs a sequence of interconnect load tests (NKikimr::TEvLoadTestRequest::TInterconnectLoad) // inside a single actor system, one after another, printing a short summary for each. @@ -258,7 +260,7 @@ struct TInterconnectTest : public TPerfTest { Printer->PrintResults(); InterconnectResultsPrintedEvent.Signal(); } - } catch (yexception ex) { + } catch (yexception& ex) { LastException = ex; IsLastExceptionSet = true; VERBOSE_COUT(ex.what()); @@ -282,4 +284,265 @@ struct TInterconnectTest : public TPerfTest { } }; +//////////////////////////////////////////////////////////////////////////////// +// Network (multi-host) interconnect test: a real TCP interconnect connection is +// established between two separate ydb_stress_tool processes running on +// different hosts. One side runs as a responder-only "server" (registers just +// the load responder actor and listens for incoming connections); the other +// side runs as a "client" that drives the InterconnectTestList load and +// connects out to the server via NodeHops pointing at the server's NodeId. +//////////////////////////////////////////////////////////////////////////////// + +// Server: registers the interconnect load responder actor for this node and +// listens for an incoming connection from the client. Does not run any load +// itself; it only replies to messages forwarded through TLoadResponderActor. +struct TInterconnectServer : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + volatile bool IsLastExceptionSet = false; + ui32 ServerNodeId; + ui32 ClientNodeId; + ui16 Port; + bool UseUring; + + TInterconnectServer(const TPerfTestConfig& cfg, ui32 serverNodeId, ui32 clientNodeId, ui16 port, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(serverNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , ServerNodeId(serverNodeId) + , ClientNodeId(clientNodeId) + , Port(port) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ServerNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 2, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 2, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Register the interconnect load responder actor for this node; it acts as + // the remote endpoint that the client's load actor routes messages to via + // NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ServerNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Server only accepts incoming connections, but the interconnect handshake + // still resolves the peer NodeId via the local nameserver, so register the + // client with a placeholder address (port 0; server never initiates a + // connection to the client). + auto common = MakeInterconnectCommon(Counters, ServerNodeId, UseUring); + TVector<TInterconnectPeer> peers = {{ClientNodeId, "::", 0}}; + SetupInterconnectServices(Setup.Get(), common, ServerNodeId, "::", Port, peers, /*listen=*/true); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ServerNodeId, "logger"), std::move(loggerActorCmd)); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect server init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + Cerr << "Interconnect load responder (node " << ServerNodeId << ", expecting client node " << ClientNodeId << ")" + << " is listening on port " << Port << ". Press Ctrl+C to stop." << Endl; + + InterconnectServerStopEvent.Wait(); + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + } + + void Finish() override { + } + + ~TInterconnectServer() override { + } +}; + +// Client: connects out to a set of remote interconnect servers and drives the +// InterconnectTestList load. The load actor's NodeHops must reference the +// server's NodeId (see ServerPeers) so that traffic is routed over the real +// network connection rather than looped back locally. +struct TInterconnectClient : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + yexception LastException; + volatile bool IsLastExceptionSet = false; + TActorId TestId; + NDevicePerfTest::TInterconnectTest TestProto; + ui32 ClientNodeId; + TVector<TInterconnectPeer> ServerPeers; + bool UseUring; + + TInterconnectClient(const TPerfTestConfig& cfg, const NDevicePerfTest::TInterconnectTest& testProto, + ui32 clientNodeId, const TVector<TInterconnectPeer>& serverPeers, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(clientNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , TestProto(testProto) + , ClientNodeId(clientNodeId) + , ServerPeers(serverPeers) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ClientNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 4, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 4, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Set up interconnect with all server peers + auto common = MakeInterconnectCommon(Counters, ClientNodeId, UseUring); + SetupInterconnectServices(Setup.Get(), common, ClientNodeId, "::", 0, ServerPeers, /*listen=*/false); + + // The load actor queries a local load responder for a shared traffic + // counter on Bootstrap (see NInterconnect::TLoadActor::Bootstrap), so a + // load responder must be registered on this node too, even though the + // actual load messages are routed to the remote server via NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ClientNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + // INTERCONNECT is too chatty at DEBUG/TRACE; floor it at INFO regardless of --log-level. + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ClientNodeId, "logger"), std::move(loggerActorCmd)); + + TestId = MakeBlobStorageProxyID(1); + TActorSetupCmd testSetup(new TInterconnectPerfTestActor(Cfg, TestProto, Printer, Counters), + TMailboxType::ReadAsFilled, 2); + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(TestId, std::move(testSetup))); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + + // Give interconnect some time to establish connections to the server(s) + Sleep(TDuration::Seconds(3)); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect client init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + try { + ActorSystem->Send(TestId, new TEvTablet::TEvBoot(MakeTabletID(0, 0, 1), 0, nullptr, TActorId(), nullptr)); + + for (ui32 i = 0; i < TestProto.InterconnectTestListSize(); ++i) { + InterconnectDoneEvent.Wait(); + Printer->PrintResults(); + InterconnectResultsPrintedEvent.Signal(); + } + } catch (yexception& ex) { + LastException = ex; + IsLastExceptionSet = true; + VERBOSE_COUT(ex.what()); + } + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + InterconnectDoneEvent.Reset(); + if (IsLastExceptionSet) { + IsLastExceptionSet = false; + ythrow LastException; + } + } + + void Finish() override { + } + + ~TInterconnectClient() override { + } +}; + } // namespace NKikimr diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp index 85ff6111c49..948cf86ef63 100644 --- a/yql/essentials/tools/sql2yql/sql2yql.cpp +++ b/yql/essentials/tools/sql2yql/sql2yql.cpp @@ -275,7 +275,12 @@ int BuildAST(int argc, char** argv) { opts.AddLongOption("mem-limit", "Set memory limit in megabytes").Handler1T<ui32>(0, NYql::SetAddressSpaceLimit); opts.AddLongOption("gateways-cfg", "Gateways configuration file").Optional().RequiredArgument("FILE").Handler1T<TString>([&gatewaysConfig, &clusterMapping](const TString& file) { gatewaysConfig = ParseProtoConfig<NYql::TGatewaysConfig>(file); - GetClusterMappingFromGateways(*gatewaysConfig, clusterMapping); + + THashMap<TString, TString> local; + GetClusterMappingFromGateways(*gatewaysConfig, local); + for (const auto& [cluster, service] : local) { + clusterMapping.emplace(cluster, service); // priority to argv + } }); opts.AddLongOption("pg-ext", "Pg extensions config file").Optional().RequiredArgument("FILE").Handler1T<TString>([](const TString& file) { auto pgExtConfig = ParseProtoConfig<NYql::NProto::TPgExtensions>(file); diff --git a/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp b/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp index 1e798159ed3..9b3845fdede 100644 --- a/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp @@ -211,6 +211,7 @@ IGraphTransformer::TStatus TWalkFoldersImpl::AfterListFolderOp(TExprContext& ctx ProcessingState_ = PreHandling; } else { folderListVal.ReportIssues(ctx.IssueManager); + return IGraphTransformer::TStatus::Error; } BatchFolderListFuture_ = Nothing(); diff --git a/yt/yt/client/formats/parser.h b/yt/yt/client/formats/parser.h index 6993e7e0b9c..61acf3aa517 100644 --- a/yt/yt/client/formats/parser.h +++ b/yt/yt/client/formats/parser.h @@ -13,8 +13,7 @@ namespace NYT::NFormats { struct IParser : public TNonCopyable { - virtual ~IParser() - { } + virtual ~IParser() = default; virtual void Read(TStringBuf data) = 0; virtual void Finish() = 0; diff --git a/yt/yt/core/misc/persistent_queue-inl.h b/yt/yt/core/misc/persistent_queue-inl.h index 829d13d1728..ee9e6ecbb16 100644 --- a/yt/yt/core/misc/persistent_queue-inl.h +++ b/yt/yt/core/misc/persistent_queue-inl.h @@ -118,6 +118,7 @@ void TPersistentQueue<T, ChunkSize>::Enqueue(T value) auto& size = this->Size_; if (!head.CurrentChunk_) { + ++this->ChunkCount_; auto chunk = New<TChunk>(); head.CurrentChunk_ = tail.CurrentChunk_ = chunk; head.CurrentIndex_ = tail.CurrentIndex_ = 0; @@ -127,6 +128,7 @@ void TPersistentQueue<T, ChunkSize>::Enqueue(T value) ++size; if (head.CurrentIndex_ == ChunkSize) { + ++this->ChunkCount_; auto chunk = New<TChunk>(); head.CurrentChunk_->Next = chunk; head.CurrentChunk_ = chunk; @@ -148,6 +150,9 @@ T TPersistentQueue<T, ChunkSize>::Dequeue() if (tail.CurrentIndex_ == ChunkSize) { tail.CurrentChunk_ = tail.CurrentChunk_->Next; tail.CurrentIndex_ = 0; + + YT_VERIFY(this->ChunkCount_ > 0); + --this->ChunkCount_; } return result; @@ -159,6 +164,13 @@ void TPersistentQueue<T, ChunkSize>::Clear() this->Head_ = TPersistentQueueIterator<T, ChunkSize>(); this->Tail_ = TPersistentQueueIterator<T, ChunkSize>(); this->Size_ = 0; + this->ChunkCount_ = 0; +} + +template <class T, size_t ChunkSize> +i64 TPersistentQueue<T, ChunkSize>::GetByteSize() const +{ + return this->ChunkCount_ * sizeof(TChunk); } template <class T, size_t ChunkSize> @@ -211,6 +223,12 @@ void TIndexedPersistentQueue<T, ChunkSize>::Clear() } template <class T, size_t ChunkSize> +i64 TIndexedPersistentQueue<T, ChunkSize>::GetByteSize() const +{ + return TBase::GetByteSize() + Chunks_.size() * sizeof(TChunk*); +} + +template <class T, size_t ChunkSize> void TIndexedPersistentQueue<T, ChunkSize>::Freeze() { auto& tail = TBase::Tail_; diff --git a/yt/yt/core/misc/persistent_queue.h b/yt/yt/core/misc/persistent_queue.h index d5d4458ae4c..f3925461bf8 100644 --- a/yt/yt/core/misc/persistent_queue.h +++ b/yt/yt/core/misc/persistent_queue.h @@ -116,6 +116,9 @@ public: virtual T Dequeue(); virtual void Clear(); + // NB: Chunks kept only by snapshots of the persistent queue do not get accounted here. + virtual i64 GetByteSize() const; + using TSnapshot = TPersistentQueueSnapshot<T, ChunkSize>; TSnapshot MakeSnapshot() const; @@ -125,6 +128,8 @@ public: private: using TChunk = TPersistentQueueChunk<T, ChunkSize>; using TChunkPtr = TIntrusivePtr<TChunk>; + + int ChunkCount_ = 0; }; // Almost zero-cost until used extension over TPersistentQueue. @@ -144,6 +149,8 @@ public: T Dequeue() override; void Clear() override; + i64 GetByteSize() const override; + void Freeze(); const T& operator[](int index) const; |